diff --git a/.github/workflows/llvm-dist.yml b/.github/workflows/llvm-dist.yml new file mode 100644 index 000000000..233e97e98 --- /dev/null +++ b/.github/workflows/llvm-dist.yml @@ -0,0 +1,93 @@ +name: LLVM distribution + +# Builds the LLVM that the Linux release helpers link +# (native/llvm-codegen/llvm-dist.mjs): LLVM from its release sources with +# ThinLTO and native/llvm-codegen/llvm.profdata.gz, for each Linux +# architecture. Each job builds a helper against it and one against LLVM's +# release archive, requires the two to give identical objects on this +# checkout's helper inputs (scripts/check-llvm-dist.mjs), prints both +# helpers' times, and attaches the archive to the LLVM_DIST_TAG release. +# The arm64 build uses the profile collected on x64; its job log reports how +# many functions the profile no longer matches. Run it after a profile refresh +# or an LLVM version change, with a new LLVM_DIST_TAG. +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: ${{ github.workflow }} + +jobs: + build: + name: Build LLVM (${{ matrix.arch }}) + if: github.repository == 'vercel-labs/scriptc' + strategy: + fail-fast: false + matrix: + include: + - arch: x64 + runner: ubuntu-24.04 + llvm_asset: LLVM-22.1.8-Linux-X64 + - arch: arm64 + runner: ubuntu-24.04-arm + llvm_asset: LLVM-22.1.8-Linux-ARM64 + runs-on: ${{ matrix.runner }} + # About an hour for LLVM on 4 cores, and 5 minutes for each helper link. + timeout-minutes: 240 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version: "24" + cache: pnpm + - name: Install LLVM's release archive and sources + run: | + sudo apt-get update + sudo apt-get install --yes ninja-build zlib1g-dev libzstd-dev + base=https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8 + curl -fL --retry 3 -o "$RUNNER_TEMP/llvm.tar.xz" "$base/${{ matrix.llvm_asset }}.tar.xz" + tar -xJf "$RUNNER_TEMP/llvm.tar.xz" -C "$RUNNER_TEMP" + mv "$RUNNER_TEMP/${{ matrix.llvm_asset }}" "$RUNNER_TEMP/llvm-22.1.8" + curl -fL --retry 3 -o "$RUNNER_TEMP/llvm-src.tar.xz" "$base/llvm-project-22.1.8.src.tar.xz" + tar -xJf "$RUNNER_TEMP/llvm-src.tar.xz" -C "$RUNNER_TEMP" + rm "$RUNNER_TEMP/llvm.tar.xz" "$RUNNER_TEMP/llvm-src.tar.xz" + echo "$RUNNER_TEMP/llvm-22.1.8/bin" >> "$GITHUB_PATH" + - run: pnpm install --frozen-lockfile + - run: pnpm --filter @scriptc/compiler --filter scriptc build + - name: Build the distribution + run: >- + node scripts/build-llvm-dist.mjs --source "$RUNNER_TEMP/llvm-project-22.1.8.src" + --prefix "$RUNNER_TEMP/dist/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}" + --archive "$RUNNER_TEMP/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}.tar.xz" + --work "$RUNNER_TEMP/llvm-dist-work" + - name: Build a helper against each LLVM + env: + HELPER: llvm-linux-${{ matrix.arch }}-gnu + run: | + LLVM_DIR="$RUNNER_TEMP/llvm-22.1.8/lib/cmake/llvm" pnpm --filter "@scriptc/$HELPER" build:native + cp "packages/$HELPER/bin/scriptc-llvm-codegen" "$RUNNER_TEMP/helper-release-archive" + LLVM_DIR="$RUNNER_TEMP/dist/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}/lib/cmake/llvm" \ + SCRIPTC_LLVM_THINLTO=1 pnpm --filter "@scriptc/$HELPER" build:native + cp "packages/$HELPER/bin/scriptc-llvm-codegen" "$RUNNER_TEMP/helper-profiled" + - name: Compare the helpers + run: >- + node scripts/check-llvm-dist.mjs --baseline "$RUNNER_TEMP/helper-release-archive" + --candidate "$RUNNER_TEMP/helper-profiled" --rounds 3 --work "$RUNNER_TEMP/llvm-check" + - name: Attach the archive to the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ARCHIVE: ${{ runner.temp }}/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}.tar.xz + run: | + tag=$(node --input-type=module -e "import { LLVM_DIST_TAG } from './native/llvm-codegen/llvm-dist.mjs'; console.log(LLVM_DIST_TAG)") + # Both architectures' jobs create the release if it is missing. + gh release view "$tag" > /dev/null 2>&1 || + gh release create "$tag" --target "$GITHUB_SHA" --title "$tag" --latest=false \ + --notes "LLVM 22.1.8 built with ThinLTO and the helper profile, for the Linux LLVM helpers (scripts/build-llvm-dist.mjs)." || + gh release view "$tag" > /dev/null + gh release upload "$tag" "$ARCHIVE" "$ARCHIVE.sha256" --clobber diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 799337e0a..0b6748edf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,6 +83,7 @@ jobs: spawnSync('git', ['diff', '--quiet', run.head_sha, 'HEAD', '--', '.', ':(exclude).github/workflows/release.yml', ':(exclude)scripts/verify-native-cli.mjs', ':(exclude)scripts/package-native-cli.mjs', ':(exclude)tests/harness/native-cli-packaging.test.ts', + ':(exclude)scripts/wait-npm-available.mjs', ':(exclude)tests/harness/npm-available.test.ts', ]).status !== 0) continue; const artifacts = api(`repos/${repo}/actions/runs/${run.id}/artifacts?per_page=100`).artifacts .filter((artifact) => !artifact.expired && artifact.size_in_bytes > 0); @@ -100,11 +101,11 @@ jobs: if (receiptRun) appendFileSync(process.env.GITHUB_OUTPUT, `receipt_run=${receiptRun}\n`); NODE - # Collects the native compiler's profile on this commit - # (scripts/refresh-cli-profile.mjs), so every platform's CLI build optimizes - # with a profile of the code it compiles. The committed profile falls behind - # main within a day or two. If this job fails, the CLI builds use the - # committed profile. + # Collects the native compiler's profile and function layout on this commit + # (scripts/refresh-cli-profile.mjs, scripts/refresh-cli-layout.mjs), so every + # platform's CLI build optimizes and lays out the code it compiles. The + # committed ones fall behind main within a day or two. If this job fails, + # the CLI builds use the committed profile and layout. cli-profile: name: Collect the native compiler profile needs: check-release @@ -138,11 +139,17 @@ jobs: - name: Collect the profile run: >- node scripts/refresh-cli-profile.mjs --tsc-ts "$RUNNER_TEMP/tsc-ts" --jobs 4 - --output "$RUNNER_TEMP/cli-profile/cli.profdata.gz" + --output "$RUNNER_TEMP/cli-profile/cli.profdata.gz" --work "$RUNNER_TEMP/cli-profile-work" + - name: Collect the function layout + run: >- + node scripts/refresh-cli-layout.mjs --jobs 4 --instrumented "$RUNNER_TEMP/cli-profile-work/cli" + --output "$RUNNER_TEMP/cli-profile/cli.layout.gz" - uses: actions/upload-artifact@v4 with: name: cli-profile - path: ${{ runner.temp }}/cli-profile/cli.profdata.gz + path: | + ${{ runner.temp }}/cli-profile/cli.profdata.gz + ${{ runner.temp }}/cli-profile/cli.layout.gz retention-days: 7 if-no-files-found: error @@ -173,6 +180,7 @@ jobs: runtime: runtime-linux-x64-gnu cli: cli-linux-x64-gnu llvm_asset: LLVM-22.1.8-Linux-X64 + llvm_dist: x64 use_zig: true - platform: linux-arm64 runner: ubuntu-24.04-arm @@ -180,6 +188,7 @@ jobs: runtime: runtime-linux-arm64-gnu cli: cli-linux-arm64-gnu llvm_asset: LLVM-22.1.8-Linux-ARM64 + llvm_dist: arm64 use_zig: true - platform: windows-x64 runner: windows-2022 @@ -195,6 +204,7 @@ jobs: seed_helper: llvm-linux-x64-gnu cli_target: x86_64-linux-musl llvm_asset: LLVM-22.1.8-Linux-X64 + llvm_dist: x64 use_zig: true - platform: linux-arm64-musl runner: ubuntu-24.04-arm @@ -204,12 +214,14 @@ jobs: seed_helper: llvm-linux-arm64-gnu cli_target: aarch64-linux-musl llvm_asset: LLVM-22.1.8-Linux-ARM64 + llvm_dist: arm64 use_zig: true - platform: wasm32-wasi runner: ubuntu-24.04 helper: llvm-linux-x64-gnu runtime: runtime-wasm32-wasi llvm_asset: LLVM-22.1.8-Linux-X64 + llvm_dist: x64 use_zig: true runs-on: ${{ matrix.runner }} # The Intel macOS build took 24 minutes for 0.3.2 and passed 30 for 0.3.3. @@ -240,6 +252,28 @@ jobs: mv "$RUNNER_TEMP/${{ matrix.llvm_asset }}" "$RUNNER_TEMP/llvm-22.1.8" echo "LLVM_DIR=$RUNNER_TEMP/llvm-22.1.8/lib/cmake/llvm" >> "$GITHUB_ENV" echo "$RUNNER_TEMP/llvm-22.1.8/bin" >> "$GITHUB_PATH" + # The Linux helpers link LLVM built with ThinLTO and a profile of the + # helper's code generation (native/llvm-codegen/llvm-dist.mjs), from a + # release of this repository. Without that archive they link LLVM's + # release archive, which gives the same objects more slowly. The + # wasm32-wasi job also uploads the x64 GNU helper, so it selects the same. + - name: Select the profiled LLVM distribution (Linux) + if: matrix.llvm_dist != '' + env: + ARCH: ${{ matrix.llvm_dist }} + run: | + tag=$(node --input-type=module -e "import { LLVM_DIST_TAG } from './native/llvm-codegen/llvm-dist.mjs'; console.log(LLVM_DIST_TAG)") + asset=$(node --input-type=module -e "import { llvmDistAsset } from './native/llvm-codegen/llvm-dist.mjs'; console.log(llvmDistAsset(process.env.ARCH))") + base="https://github.com/$GITHUB_REPOSITORY/releases/download/$tag" + if curl -fsSL --retry 3 -o "$RUNNER_TEMP/$asset" "$base/$asset" && + curl -fsSL --retry 3 -o "$RUNNER_TEMP/$asset.sha256" "$base/$asset.sha256" && + (cd "$RUNNER_TEMP" && sha256sum --check "$asset.sha256"); then + tar -xJf "$RUNNER_TEMP/$asset" -C "$RUNNER_TEMP" + echo "SCRIPTC_HELPER_LLVM_DIR=$RUNNER_TEMP/${asset%.tar.xz}/lib/cmake/llvm" >> "$GITHUB_ENV" + echo "The helper links $asset from release $tag" + else + echo "::warning::$asset is not available from release $tag; the helper links LLVM's release archive" + fi - uses: vercel-labs/setup-zig@v1 if: matrix.use_zig == true with: @@ -278,7 +312,15 @@ jobs: - name: Build native packages (POSIX) if: matrix.platform != 'windows-x64' run: | - pnpm --filter @scriptc/${{ matrix.helper }} build:native + if [ -n "${SCRIPTC_HELPER_LLVM_DIR:-}" ]; then + LLVM_DIR="$SCRIPTC_HELPER_LLVM_DIR" SCRIPTC_LLVM_THINLTO=1 pnpm --filter @scriptc/${{ matrix.helper }} build:native || { + echo "::warning::the helper did not build against the profiled LLVM; it links LLVM's release archive" + rm -rf node_modules/.cache/scriptc-${{ matrix.helper }} + pnpm --filter @scriptc/${{ matrix.helper }} build:native + } + else + pnpm --filter @scriptc/${{ matrix.helper }} build:native + fi if [ "${{ matrix.use_zig }}" = true ]; then CC=zig AR=zig pnpm --filter @scriptc/${{ matrix.runtime }} build:native else @@ -300,8 +342,8 @@ jobs: with: name: cli-profile path: ${{ runner.temp }}/cli-profile - # Without the artifact, SCRIPTC_NATIVE_PROFILE_USE stays unset and - # build-native-cli.mts uses the committed profile. + # Without the artifact, SCRIPTC_NATIVE_PROFILE_USE and SCRIPTC_NATIVE_LAYOUT + # stay unset and build-native-cli.mts uses the committed profile and layout. - name: Select the native compiler profile if: matrix.cli != '' shell: bash @@ -311,6 +353,11 @@ jobs: if [ -n "$COLLECTED" ]; then echo "SCRIPTC_NATIVE_PROFILE_USE=$COLLECTED" >> "$GITHUB_ENV" echo "The native CLI builds with the profile collected on $GITHUB_SHA" + layout="$(dirname "$COLLECTED")/cli.layout.gz" + if [ -f "$layout" ]; then + echo "SCRIPTC_NATIVE_LAYOUT=$layout" >> "$GITHUB_ENV" + echo "The native CLI builds with the function layout collected on $GITHUB_SHA" + fi else echo "::warning::No profile was collected on this commit; the native CLI builds with the committed profile" fi @@ -394,6 +441,8 @@ jobs: contents: read actions: read id-token: write + outputs: + uploads: ${{ steps.publish.outputs.uploads }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -413,7 +462,7 @@ jobs: VERSION: ${{ needs.check-release.outputs.version }} RECEIPT_RUN: ${{ needs.check-release.outputs.receipt_run }} run: | - touch "$RUNNER_TEMP/npm-accepted.txt" + touch "$RUNNER_TEMP/npm-accepted.txt" "$RUNNER_TEMP/npm-accepted-times.txt" # Include this run so re-running only failed jobs restores its receipts. for run in "$GITHUB_RUN_ID" "$RECEIPT_RUN"; do if [ -z "$run" ]; then continue; fi @@ -422,6 +471,7 @@ jobs: if [ "$count" -eq 0 ]; then continue; fi gh run download "$run" --repo "$GITHUB_REPOSITORY" --pattern "npm-accepted-$VERSION-*" --dir "$RUNNER_TEMP/npm-receipts/$run" find "$RUNNER_TEMP/npm-receipts/$run" -name npm-accepted.txt -exec cat {} + >> "$RUNNER_TEMP/npm-accepted.txt" + find "$RUNNER_TEMP/npm-receipts/$run" -name npm-accepted-times.txt -exec cat {} + >> "$RUNNER_TEMP/npm-accepted-times.txt" done - name: Setup pnpm @@ -471,6 +521,7 @@ jobs: done - name: Publish to npm + id: publish shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -490,10 +541,14 @@ jobs: # Upload acceptance and public availability are separate states. # Save acceptance immediately so retries never depend on registry lag. + # npm-release.txt lists every package of the release, and + # npm-accepted-times.txt the time npm accepted each upload, for the + # npm-available job. publish_dir() { dir="$1" packed="${2:-}" name=$(node -p "require('./$dir/package.json').name") + echo "$name@$VERSION" >> "$RUNNER_TEMP/npm-release.txt" if grep -Fxq "$name@$VERSION" "$RUNNER_TEMP/npm-accepted.txt"; then echo "$name@$VERSION upload already accepted, skipping" return 0 @@ -510,6 +565,7 @@ jobs: publish_log="$RUNNER_TEMP/npm-publish.log" if npm publish "$packed" $PROVENANCE --access public 2>&1 | tee "$publish_log"; then echo "$name@$VERSION" >> "$RUNNER_TEMP/npm-accepted.txt" + echo "$name@$VERSION $(date +%s)" >> "$RUNNER_TEMP/npm-accepted-times.txt" return 0 else publish_status=${PIPESTATUS[0]} @@ -533,7 +589,8 @@ jobs: # pnpm pack resolves workspace dependencies; npm handles OIDC. # Publish dependencies first. Registry propagation after an accepted - # upload must not block the remaining packages or the GitHub release. + # upload must not block the remaining packages; the npm-available + # job waits for it. publish_dir packages/runtime publish_dir packages/runtime-darwin-arm64 publish_dir packages/llvm-darwin-arm64 @@ -561,23 +618,66 @@ jobs: publish_dir packages/cli echo "All package uploads accepted; npm registry propagation may continue after this job." >> "$GITHUB_STEP_SUMMARY" + # Each package of the release with the time its upload was accepted, + # when a run recorded it. + { + echo "uploads<> "$GITHUB_OUTPUT" + - name: Save accepted upload receipts if: always() uses: actions/upload-artifact@v4 with: name: npm-accepted-${{ needs.check-release.outputs.version }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/npm-accepted.txt + path: | + ${{ runner.temp }}/npm-accepted.txt + ${{ runner.temp }}/npm-accepted-times.txt if-no-files-found: ignore retention-days: 90 - # Publish standalone native distributions and the surface manifest after - # npm publishing succeeds. The release body comes from the marked changelog. - github-release: - name: Create GitHub Release + # Waits until every package of the release resolves on the npm registry, + # and reports each package's time from upload to available. Fails with the + # packages that are still missing after 2 hours. It only reads from the + # registry; the next release run retries it, and publishing skips uploads + # npm already accepted. + npm-available: + name: Wait for npm availability needs: [check-release, publish] if: >- !cancelled() && needs.check-release.outputs.should_release == 'true' && needs.publish.result == 'success' runs-on: ubuntu-latest + timeout-minutes: 135 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + registry-url: "https://registry.npmjs.org" + + - name: Wait for every package to resolve + env: + UPLOADS: ${{ needs.publish.outputs.uploads }} + run: | + printf '%s\n' "$UPLOADS" > "$RUNNER_TEMP/npm-uploads.txt" + node scripts/wait-npm-available.mjs "$RUNNER_TEMP/npm-uploads.txt" + + # Publish standalone native distributions and the surface manifest after + # every npm package resolves. The release body comes from the marked + # changelog. + github-release: + name: Create GitHub Release + needs: [check-release, publish, npm-available] + if: >- + !cancelled() && needs.check-release.outputs.should_release == 'true' && needs.npm-available.result == 'success' + runs-on: ubuntu-latest timeout-minutes: 10 permissions: contents: write diff --git a/RELEASING.md b/RELEASING.md index 42bfe75ad..8eac23a8e 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -18,6 +18,8 @@ The release job builds each pinned LLVM helper, precompiled runtime pack, and na Release builds of the native CLI optimize the compiler with a profile collected on the release commit. The `cli-profile` job runs `node scripts/refresh-cli-profile.mjs` on Linux x64, which builds an instrumented compiler, runs it on tsc-ts, a slice of the corpus, and the runtime benchmark workloads, and writes the merged sparse profile. Every platform's CLI build uses that profile (`SCRIPTC_NATIVE_PROFILE_USE`), and the GitHub release attaches it as `scriptc--cli.profdata.gz`. Code the profile does not describe, or describes from older compiler sources, is optimized as without a profile. When the profile job fails, the CLI builds use the committed profile `packages/compiler/native/cli.profdata.gz`, which local and CI builds of the native compiler also use. Refresh the committed profile when compiler code has changed substantially since the last refresh: run `node scripts/refresh-cli-profile.mjs --tsc-ts ` on a quiet Linux x64 host and commit the new profile on its own. +Builds of the native compiler also order its functions by the committed layout `packages/compiler/native/cli.layout.gz`: the functions that `--version` and rebuilds of unchanged programs run come first, then the ones other builds run, ordered by the share of the layout's training builds that run each, so a run maps fewer pages of the executable. Functions the layout does not list keep their usual order. Function names of closures and generated helpers change with nearby edits, so the `cli-profile` job also collects a layout on the release commit (`scripts/refresh-cli-layout.mjs`, reusing the job's instrumented compiler), and the CLI builds use it (`SCRIPTC_NATIVE_LAYOUT`). Refresh the committed layout the same way as the profile, with `node scripts/refresh-cli-layout.mjs`, and commit it on its own. + Publishing uses npm trusted publishing (OIDC) — there is no npm token secret. Each published package must have a GitHub Actions trusted publisher for `release.yml` and the `Release` environment. A missing configuration fails diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 0157503fd..9af4ace00 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -16,6 +16,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `dynamic-methods` | cpu | untyped JavaScript calling builtin methods on values it does not type: string `slice`/`indexOf`/`toUpperCase`/`padStart`, array `push`/`pop`/`includes`, `Map` and `Set`, and an object's own methods | | `dynamic-properties` | cpu | untyped JavaScript reading and writing properties of values it does not type: records built by object literals in two key orders, nested records, and instances of a class and its subclass | | `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | +| `handler-tables` | cpu | dispatch tables of functions: `Record` read as `t[k]!(…)`, `t[k]?.(…)` and through a checked local, a `Map`, an array of handlers, and an object literal of methods; some handlers declare fewer parameters than the table type | | `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | | `json-unions` | cpu | `JSON.parse` into discriminated unions: an event log with eight kinds and nested expression trees with nine, evaluated after the cast | diff --git a/benchmarks/runtime/workloads/handler-tables.json b/benchmarks/runtime/workloads/handler-tables.json new file mode 100644 index 000000000..1408ae91e --- /dev/null +++ b/benchmarks/runtime/workloads/handler-tables.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/handler-tables.ts", + "args": [ + "4" + ] +} diff --git a/benchmarks/runtime/workloads/handler-tables.ts b/benchmarks/runtime/workloads/handler-tables.ts new file mode 100644 index 000000000..b0a61fcfe --- /dev/null +++ b/benchmarks/runtime/workloads/handler-tables.ts @@ -0,0 +1,108 @@ +// Dispatch tables of functions: a `Record` of opcode handlers +// called as `ops[name]!(…)`, `ops[name]?.(…)` and through a checked local, +// a `Map` of the same handlers, an array of handlers indexed by +// opcode number, and an object literal of methods. Some handlers declare +// fewer parameters than the table's function type. +type Op = (acc: number, arg: number) => number; + +const ops: Record = { + add: (acc, arg) => acc + arg, + sub: (acc, arg) => acc - arg, + mul: (acc, arg) => (acc * arg) % 1000003, + neg: (acc) => -acc, + half: (acc) => Math.floor(acc / 2), + mix: (acc, arg) => (acc ^ arg) & 0xffff, +}; + +const byName = new Map(Object.entries(ops)); +const byCode: Op[] = Object.values(ops); +const names = Object.keys(ops); + +const methods = { + add(acc: number, arg: number): number { + return acc + arg; + }, + sub(acc: number, arg: number): number { + return acc - arg; + }, + mul(acc: number, arg: number): number { + return (acc * arg) % 1000003; + }, +}; + +interface Insn { + name: string; + code: number; + arg: number; +} + +function program(length: number): Insn[] { + const out: Insn[] = []; + let seed = 12345; + for (let i = 0; i < length; i++) { + seed = (seed * 1103515245 + 12345) % 2147483648; + const code = seed % (names.length + 1); + // One in seven instructions names an opcode no table holds. + out.push({ name: code < names.length ? names[code]! : "nop", code, arg: (seed >> 8) % 97 }); + } + return out; +} + +function runAsserted(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) if (insn.name !== "nop") acc = ops[insn.name]!(acc, insn.arg); + return acc; +} + +function runOptional(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) acc = ops[insn.name]?.(acc, insn.arg) ?? acc + 1; + return acc; +} + +function runChecked(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) { + const handler = ops[insn.name]; + if (handler) acc = handler(acc, insn.arg); + } + return acc; +} + +function runMap(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) { + const handler = byName.get(insn.name); + if (handler !== undefined) acc = handler(acc, insn.arg); + } + return acc; +} + +function runArray(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) if (insn.code < byCode.length) acc = byCode[insn.code]!(acc, insn.arg); + return acc; +} + +function runMethods(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) { + if (insn.code === 0) acc = methods.add(acc, insn.arg); + else if (insn.code === 1) acc = methods.sub(acc, insn.arg); + else if (insn.code === 2) acc = methods["mul"](acc, insn.arg); + } + return acc; +} + +const rounds = Number(process.argv[2] ?? "4"); +const insns = program(200_000); +const totals = [0, 0, 0, 0, 0, 0]; +for (let round = 0; round < rounds * 10; round++) { + totals[0] = (totals[0]! + runAsserted(insns)) % 1e9; + totals[1] = (totals[1]! + runOptional(insns)) % 1e9; + totals[2] = (totals[2]! + runChecked(insns)) % 1e9; + totals[3] = (totals[3]! + runMap(insns)) % 1e9; + totals[4] = (totals[4]! + runArray(insns)) % 1e9; + totals[5] = (totals[5]! + runMethods(insns)) % 1e9; +} +console.log(totals.join(" ")); diff --git a/docs/content/docs/cli.mdx b/docs/content/docs/cli.mdx index 4c1b7e7c6..0a82775ca 100644 --- a/docs/content/docs/cli.mdx +++ b/docs/content/docs/cli.mdx @@ -2,7 +2,7 @@ title: "CLI Reference" --- -The `scriptc` command has five subcommands: `build`, `run`, `coverage`, `cache warm`, and `merge-profiles`. `scriptc --help` prints this summary and the options below, and `scriptc --version` prints the installed version. +The `scriptc` command has six subcommands: `build`, `run`, `coverage`, `types`, `cache warm`, and `merge-profiles`. `scriptc --help` prints this summary and the options below, and `scriptc --version` prints the installed version. ```console $ scriptc --help @@ -21,6 +21,13 @@ Usage: exporting the profile symbols; a profile with a sidecar section also gets the contract sidecar JSON beside the archive + scriptc types patch [--range ] [--dry-run] + fix a wrong declaration of an installed + package: shows what the selector + matches and adds the type patch to + package.json ("| undefined" widens) + scriptc types list the project's type patches and what + they match scriptc cache warm [runtime|tls…] verify and load precompiled runtime families for the current compiler/SDK/target scriptc merge-profiles -o [--sparse] @@ -88,6 +95,24 @@ $ scriptc run fib.ts Analyzes the program without building it and reports what compiles and what blocks the rest. See [Coverage Reports](/docs/coverage). +## scriptc types + +Writes and lists [type patches](/docs/dependencies#fix-a-packages-types), the project's fixes for wrong declarations of npm packages. + +
+
scriptc types patch <package> <selector> <type>
+
Finds the package installed for the working directory, prints every declaration the selector matches in its declaration files, its JSDoc and its @types/ package with the type before and after, and adds the patch to the nearest package.json. The patch joins an entry whose range covers the installed version, or gets the range ^<installed version>. Nothing is written when the selector matches nothing. --range <range> chooses the range, and --dry-run only prints.
+
scriptc types list
+
Prints each type patch in the nearest package.json, whether it applies to the installed version, and what it matches.
+
+ +```console +$ scriptc types list +commander@^15.0.0 + HelpContext.helpWidth: | undefined + node_modules/commander/lib/command.js:2481 number -> number | undefined +``` + ## scriptc cache warm Verifies and loads the installed precompiled runtime objects and vendored libraries, and prints the cache directory. Pass `runtime`, `tls`, or both to select which to warm. Ordinary builds don't need this step. With `--sanitize`, it builds the instrumented runtime objects using a C toolchain. @@ -138,11 +163,14 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
--target-cpu <baseline|x86-64-v2|x86-64-v3|x86-64-v4|native>
On x86-64 targets, compile the program's code for an x86-64 microarchitecture level. The default, baseline, runs on any x86-64 CPU. A higher level lets LLVM use more instructions, and the executable then runs only on CPUs that support that level. native uses the highest level the build machine supports. Valid for executable, assembly, and object output on x86-64 targets. See Target CPU.
+
--cpu-dispatch
+
On x86-64 targets, compile the program's code twice, for the baseline and for x86-64-v3, and choose the copy when the program starts. The executable runs on any x86-64 CPU and uses the x86-64-v3 code on CPUs that support it. Not used with a --target-cpu level above baseline. Valid only for executable builds. See Runtime CPU dispatch.
+
--profile-generate <pattern>
On Linux, build an instrumented executable for profile-guided optimization. Each run writes a raw profile to the path pattern, where %p expands to the process ID; the LLVM_PROFILE_FILE environment variable overrides the pattern at run time. A relative pattern resolves from the current working directory. Instrumented programs run slower. Valid only for executable builds with --optimization=release or speed.
--profile-use <file.profdata>
-
Optimize the executable with a profile from scriptc merge-profiles: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain. A new or changed profile compiles every partition of the program again. Valid only for executable builds with --optimization=release or speed.
+
Optimize the executable with a profile from scriptc merge-profiles: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain. A new or changed profile compiles every partition of the program again. scriptc build prints how the profile matched the program's functions on stderr; see Profile-guided optimization. Valid only for executable builds with --optimization=release or speed.
--npm-static <pkg[,pkg…]|auto>
Name npm packages to compile even if they don't ship their own type declarations (repeatable). A trailing ..., as in fs-extra..., also names every package that package depends on. auto uses the default automatic selection. Packages that can't compile produce diagnostics. See npm Dependencies.
@@ -234,6 +262,25 @@ $ scriptc build app.ts --optimization=speed --profile-use app.profdata Train on workloads like the ones the program will run: code the training never reached is optimized as without a profile. The profile can be committed and reused as the program changes. Functions whose control flow changed since the profile was collected lose their profile, and the rest keep theirs, so regenerate it when much of the program has changed. [Build parallelism](/docs/how-it-works#build-parallelism) describes what a profile adds to build time. +`scriptc build --profile-use` prints one line on stderr with how the profile matched the program's functions: + +``` +scriptc: profile app.profdata: 9170 functions annotated, 0 mismatched, 28636 without a record +``` + +- **annotated:** functions optimized with the profile's counts. +- **mismatched:** functions whose control flow changed since the profile was collected. They are optimized without counts. +- **without a record:** functions the profile does not describe: new functions, and functions the profiled runs never reached. +- When the build reuses cached partitions whose counts were not recorded, the line ends with the number of those partitions. + +When more than 10% of the functions the profile describes (annotated and mismatched) are mismatched, the build also prints a warning: + +``` +scriptc: warning: 3739 of the 8947 functions app.profdata describes (41.8%) changed since it was collected and are optimized without their counts; collect the profile again +``` + +The build still succeeds. Collecting the profile again with the current program and scriptc version restores the counts. + ## Target CPU By default, x86-64 executables run on any x86-64 CPU, so the compiler uses only the baseline instruction set (SSE2). `--target-cpu` compiles the program's code for a higher x86-64 microarchitecture level. Each level includes the ones below it: @@ -271,6 +318,22 @@ An executable built for a level runs only on CPUs that support that level. On an The level applies to the program's own code. The runtime pack keeps its baseline code. Builds for different levels never share cached artifacts, and `baseline` builds are the same as builds without the option. +### Runtime CPU dispatch + +`--cpu-dispatch` keeps the executable portable and still uses the higher level where the CPU has it. The program's functions are compiled for the baseline and for `x86-64-v3`. At startup, the executable checks the CPU and the operating system's register support, and then runs the `x86-64-v3` code on CPUs that support every feature of that level. Calls between the program's functions stay within the chosen copy. + +```console +$ scriptc build app.ts --optimization=speed --cpu-dispatch -o app +``` + +The second copy makes the executable larger and the first build of the program slower, since LLVM compiles the program's code twice. Unchanged rebuilds are not affected. Both copies compute the same results: neither contracts floating-point operations into fused multiply-adds. + +The `SCRIPTC_CPU_LEVEL` environment variable caps the level the executable uses, for example to test the baseline code on a newer machine. It takes a level from `0` to `4`, `baseline`, or a level name such as `x86-64-v3`; any other value selects the baseline code. + +```console +$ SCRIPTC_CPU_LEVEL=baseline ./app +``` + ## LLVM backend scriptc generates LLVM IR and compiles it to native code. Executable builds link the result with the matching precompiled runtime pack. Use --emit=llvm to inspect the generated code. diff --git a/docs/content/docs/dependencies.mdx b/docs/content/docs/dependencies.mdx index 9b660c76f..2288aeb83 100644 --- a/docs/content/docs/dependencies.mdx +++ b/docs/content/docs/dependencies.mdx @@ -145,7 +145,24 @@ A JavaScript package can contain operations scriptc can't compile, often in func scriptc trusts declared types: a package's `.d.ts` files, its `@types/` package, and the JSDoc types in its JavaScript. A value that contradicts its declared type throws a `TypeError` instead of flowing on, so a package whose declarations are wrong can stop a program that runs under Node.js. A common case is a member declared `string` or `number` that is sometimes `undefined`. -Fix such a declaration in your project with a type patch in `package.json`: +When the type comes from a package's declarations, the error names the declaration and the patch that would admit the value: + +```text +Uncaught TypeError: expected string at $, got undefined + profilekit@3.1.0 declares Profile.email as string (index.d.ts:7) + if that declaration is wrong, patch it: scriptc types patch profilekit "Profile.email" "| undefined" +``` + +Fix such a declaration in your project with a type patch. `scriptc types patch` shows what a declaration path matches in the installed package and adds the patch to your `package.json`: + +```console +$ scriptc types patch commander "HelpContext.helpWidth" "| undefined" +commander 15.0.0 (node_modules/commander) + lib/command.js:2481 number -> number | undefined +wrote package.json scriptc.typePatches["commander@^15.0.0"].declarations["HelpContext.helpWidth"] +``` + +The patch it writes looks like this: ```json:package.json { diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index 561840268..25427a307 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -4,7 +4,13 @@ When an untyped value reaches `NodeJS.ReadableStream` or `NodeJS.WritableStream`, those two interfaces hold any value, so JavaScript stream objects (constructor-function streams, plain objects with `on` and `pipe`) pass through TypeScript functions typed with them and come back to JavaScript as the same objects. Member calls on such a value use the native stream when it is one and call the object's own members otherwise; `readable.pipe()` into a JavaScript destination follows Node's pipe protocol. The first `pipeline` stage may arrive as an untyped value that is a native stream, a `Cipheriv` or `Decipheriv` value, or an iterable: a string, an array, a generator object, or any value with a `Symbol.asyncIterator` or `Symbol.iterator` method, which the pipeline pumps into the next stage as Node does. - Later `pipeline` stages and `for await` sources that arrive as untyped values must be native streams, or `Cipheriv` and `Decipheriv` values, at run time; other values throw a not-supported error. + It may also be a source function, which the pipeline calls with `{ signal }` and whose iterable result it pumps, or a `{ readable, writable }` pair, whose readable side it pumps. + Later `pipeline` stages that arrive as untyped values may be native streams, `Cipheriv` and `Decipheriv` values, functions, and iterables. + A function is called with the previous stage and `{ signal }` as a function stage: before the last stage it must return an async iterable, and as the last stage it may return a promise, whose value resolves the pipeline, or an async iterable, which feeds the PassThrough that `pipeline()` returns. + An untyped source function, function stage, or destination that returns a stream is unsupported. + An iterable becomes `Duplex.from` of it, which the next stage reads from while the stages before it are not read, as in Node. + A pipeline branches at run time on whether each of its first three untyped stages after the first is a function; a function in a later untyped stage, an untyped web stream, and a `{ readable, writable }` pair after the first stage throw a not-supported error. + `for await` sources that arrive as untyped values must be native streams, or `Cipheriv` and `Decipheriv` values, at run time; other values throw a not-supported error. Native streams expose no `emit` to JavaScript, so a JavaScript stream whose own `pipe()` emits on its destination (the legacy `Stream` pipe) cannot pipe into one. `Readable.from` accepts strings, Buffers, arrays, Sets, Maps, Map and Set iterators, generators, async generators, and class iterables and async iterables, and in JavaScript any iterable or async iterable. In JavaScript, a sync iterator value with a `then` method that is not a promise, and an object whose `[Symbol.asyncIterator]` or `[Symbol.iterator]` method is a generator method, are unsupported. diff --git a/docs/content/limitations/native-addons.mdx b/docs/content/limitations/native-addons.mdx index 8fa7f94bf..988df3660 100644 --- a/docs/content/limitations/native-addons.mdx +++ b/docs/content/limitations/native-addons.mdx @@ -2,12 +2,18 @@ Node-API (N-API) and V8 `.node` addons depend on Node.js itself, so compiled pro Direct `createRequire` calls to local addons are rejected at compile time. A package that requires an addon needs a JavaScript implementation that compiles, or a replacement through native FFI. -A package that looks for an optional addon at run time compiles with `--npm-static` when the lookup is inside a `try` block. -There, a `createRequire` require call with a computed specifier, and a `require.resolve` or `require.resolve.paths` call with a computed request, evaluate their arguments and then throw an error. -Its `require.resolve` read as a value is a function that throws when called. -Coverage lists these as deferred to runtime. -The package's `catch` clause then runs as it does in Node.js when the addon is missing. -Outside a `try` block, the same uses keep the package refused. +A package that looks for an optional addon at run time compiles with `--npm-static`. +In JavaScript dependencies, a `createRequire` require call with a computed specifier, and `require.resolve` called with a computed request or read as a value, resolve the specifier with the Node.js algorithm when the program runs. +`require.resolve` returns the path Node.js returns. +When nothing resolves, both throw the `MODULE_NOT_FOUND` error that Node.js throws, so a package that probes for a missing addon takes the same branch as in Node.js. +When the specifier of a require call resolves to a native addon, a builtin, or a file that is not part of the compiled program, the program prints an SC2020 error that names it and exits with status 1, because Node.js would load that module. +A `catch` clause around the call does not run in that case. +Resolution steps that the runtime does not reproduce, such as package `imports` specifiers, `require.resolve` options, and subpaths that a package does not export, end the program the same way. +Coverage lists these calls as deferred to runtime. +Other computed uses compile when they are inside a `try` block: a require call with spread arguments, a `require.resolve` call with another argument count, and `require.resolve.paths` called with a computed request evaluate their arguments and then throw an error, and `require.resolve.paths` read as a value is a function that throws when called. +Coverage lists these as deferred to runtime as well, and the package's `catch` clause then runs as it does in Node.js when the addon is missing. +Outside a `try` block, these uses keep the package refused. +This applies until compiled programs can load addons through Node.js. Use [Native FFI](/docs/ffi#replacing-a-node-api-addon) to call a plain C ABI function linked from an object or static archive. The guide includes a replacement for a small Node-API helper. diff --git a/native/llvm-codegen/CMakeLists.txt b/native/llvm-codegen/CMakeLists.txt index 07abb35ec..7ce20086f 100644 --- a/native/llvm-codegen/CMakeLists.txt +++ b/native/llvm-codegen/CMakeLists.txt @@ -17,6 +17,14 @@ set(SCRIPTC_HELPER_ARCH "arm64" CACHE STRING # Linux packages ship this process-isolated helper as a fully static host # executable so it runs independently of the destination libc and libstdc++. option(SCRIPTC_STATIC_LINUX_BUILD "Build a static Linux helper executable" OFF) +# The release Linux helpers link an LLVM built with ThinLTO and a profile of +# the helper's code generation (llvm-dist.mjs). Its archives hold bitcode, so +# the helper compiles and links with ThinLTO too, which needs clang and lld. +option(SCRIPTC_LLVM_THINLTO "Compile and link the helper with ThinLTO" OFF) +set(SCRIPTC_LLVM_PROFILE_USE "" CACHE FILEPATH + "Indexed profile the helper's own sources are optimized with") +option(SCRIPTC_LLVM_PROFILE_GENERATE + "Instrument the helper's sources to collect that profile" OFF) # These must be cache variables before project() enables the Apple toolchain. # A target property named OSX_DEPLOYMENT_TARGET is not recognized by CMake and @@ -219,6 +227,31 @@ else() endif() target_link_libraries(scriptc-llvm-codegen PRIVATE ${SCRIPTC_LLVM_LIBS} ${SCRIPTC_LLVM_SYSTEM_LIBS}) +if(SCRIPTC_LLVM_THINLTO) + target_compile_options(scriptc-llvm-codegen PRIVATE -flto=thin) + # The thin link optimizes and compiles all of LLVM's code the helper + # reaches; the cache makes a relink after a helper edit incremental. + target_link_options(scriptc-llvm-codegen PRIVATE + -flto=thin + -fuse-ld=lld + "LINKER:--thinlto-jobs=all" + "LINKER:--thinlto-cache-dir=${CMAKE_BINARY_DIR}/thinlto-cache" + ) +endif() +if(SCRIPTC_LLVM_PROFILE_USE) + # The profile covers LLVM's sources and the helper's. Functions it does + # not cover, or whose control flow changed since, are optimized without it. + target_compile_options(scriptc-llvm-codegen PRIVATE + "-fprofile-use=${SCRIPTC_LLVM_PROFILE_USE}" + -Wno-profile-instr-unprofiled + -Wno-profile-instr-out-of-date + ) +endif() +if(SCRIPTC_LLVM_PROFILE_GENERATE) + target_compile_options(scriptc-llvm-codegen PRIVATE -fprofile-generate) + target_link_options(scriptc-llvm-codegen PRIVATE -fprofile-generate) +endif() + if(APPLE) # Homebrew's static LLVM archives contain broad component object files. # Export only the process entry point so archive-global LLVM symbols do not diff --git a/native/llvm-codegen/README.md b/native/llvm-codegen/README.md index 0e8abbc47..02ac6e3ef 100644 --- a/native/llvm-codegen/README.md +++ b/native/llvm-codegen/README.md @@ -37,7 +37,7 @@ Repeating `--output` divides the program into that many partitions, which are co WebAssembly code generation differs from LLVM's default pipeline in two ways that do not change its output. The pass that gives multiple-entry loops a single entry runs only on functions whose control flow is irreducible; its reachability analysis grows with the square of a function's block count even when it has nothing to change. Object emission also leaves out the module pass that collects external symbols for assembly declarations, so each function's machine code is freed once it is printed instead of when the whole module is compiled. -`--cache-dir ` names a private directory the caller owns for one helper build. Simplified partitions are stored under the hash of their exact source and objects under the hash of the exact module after importing, each with a trailing digest and published by rename, so a rebuild recompiles only the partitions an edit can affect. +`--cache-dir ` names a private directory the caller owns for one helper build. Simplified partitions are stored under the hash of their exact source and objects under the hash of the exact module after importing, each with a trailing digest and published by rename, so a rebuild recompiles only the partitions an edit can affect. Partitions drop the names of their struct types before the thin link, which would otherwise number same-named types from different partitions in an order that any edit can shift, changing the module after importing in partitions the edit did not touch. `--import-bitcode `, given once per runtime-pack unit that ships bitcode, imports runtime functions for inlining; scriptc's program builds do not pass it. Small runtime functions reachable from the program's runtime calls are imported as `available_externally` so LLVM can inline them; the runtime objects remain the only definitions. The helper never runs a sanitizer pass, so with import it also drops the emitter's inert `sanitize_address` attributes, which would otherwise block that inlining. Without `--import-bitcode` the module is optimized exactly as emitted. @@ -58,3 +58,20 @@ scriptc-llvm-codegen prune-import --input unit.opt.bc --output unit.bc ``` Both commands keep only what a program build can import: the importable bodies within the size limit at their call distance from any public function, and the constants they copy. Every other definition becomes a declaration, so a program imports exactly what it would from the whole unit while reading a fraction of it. Unpromoted unit-local symbols stay local, so release bitcode offers only bodies that need none of them. + +## Linux release builds + +The Linux helper packages are released linked against LLVM 22.1.8 built from its release sources with ThinLTO and with `llvm.profdata.gz`, a profile of the helper's own code generation (`llvm-dist.mjs`). The helper's code generation is 5-9% faster with it than when the helper links LLVM's release archive, and its objects are identical. The release workflow downloads that build from the `LLVM_DIST_TAG` release of this repository; when the archive is missing, the helper links LLVM's release archive instead. + +To build and use it locally, with clang, lld and llvm-profdata 22.1.8 on `PATH` (LLVM's release archive for the host has them): + +```console +$ node scripts/build-llvm-dist.mjs --source llvm-project-22.1.8.src --prefix +$ LLVM_DIR=/lib/cmake/llvm SCRIPTC_LLVM_THINLTO=1 pnpm --filter @scriptc/llvm-linux-x64-gnu build:native +``` + +`SCRIPTC_LLVM_THINLTO=1` compiles and links the helper with clang, lld and ThinLTO, since that LLVM's archives hold bitcode, and optimizes the helper's own sources with the same profile. The link takes about 4 minutes on 4 cores. + +The `llvm-dist` workflow (run by hand) builds the archive for x64 and arm64, checks that a helper linked against it gives the same objects as one linked against LLVM's release archive (`scripts/check-llvm-dist.mjs`), and attaches it to the release. The arm64 build uses the profile collected on x64, and its log reports how many functions the profile does not match there. + +The profile depends on LLVM's sources and the kind of code scriptc emits, not on scriptc's commits. `scripts/refresh-llvm-profile.mjs` collects a new one after an LLVM version change: it builds LLVM with IR instrumentation and an instrumented helper, runs that helper on tsc-ts, five Effect fixtures, the runtime benchmark workloads and every 50th corpus program (`scripts/llvm-helper-inputs.mjs`), and merges the result. A new profile or LLVM version gets a new `LLVM_DIST_TAG`. diff --git a/native/llvm-codegen/linux-helper.mjs b/native/llvm-codegen/linux-helper.mjs new file mode 100644 index 000000000..68d738af6 --- /dev/null +++ b/native/llvm-codegen/linux-helper.mjs @@ -0,0 +1,79 @@ +/* The static Linux helper build shared by the four Linux helper packages, + * scripts/refresh-llvm-profile.mjs and scripts/check-llvm-dist.mjs. */ +import { execFileSync } from "node:child_process"; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { helperLlvmDistArgs } from "./llvm-dist.mjs"; +import { nativeTargetCmakeArgs } from "./targets.mjs"; + +const DATA_LAYOUTS = { + x64: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", + arm64: + "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32", +}; + +/** Configures and builds the helper for `arch` (`x64` or `arm64`) and `libc` + * (`gnu` or `musl`) in `build` against LLVM_DIR, and writes the stripped + * executable to `output`. musl builds take the compilers from CC and CXX. */ +export function buildLinuxHelper({ arch, libc, version, build, output }) { + const triple = `${arch === "x64" ? "x86_64" : "aarch64"}-unknown-linux-${libc}`; + // CMake drops the other definitions when a reconfigure changes the + // compiler, so a build directory configured for another LLVM or build + // kind starts over. + const kind = JSON.stringify({ + llvm: process.env.LLVM_DIR ?? null, + thinLTO: process.env.SCRIPTC_LLVM_THINLTO === "1", + profileGenerate: process.env.SCRIPTC_LLVM_PROFILE_GENERATE === "1", + cc: process.env.CC ?? null, + cxx: process.env.CXX ?? null, + }); + const stamp = join(build, "scriptc-helper-build.json"); + if (existsSync(build) && (!existsSync(stamp) || readFileSync(stamp, "utf8") !== kind)) + rmSync(build, { recursive: true, force: true }); + mkdirSync(build, { recursive: true }); + writeFileSync(stamp, kind); + execFileSync( + "cmake", + [ + "-S", + fileURLToPath(new URL(".", import.meta.url)), + "-B", + build, + "-G", + "Ninja", + ...(libc === "musl" + ? [ + `-DCMAKE_C_COMPILER=${process.env.CC ?? "cc"}`, + `-DCMAKE_CXX_COMPILER=${process.env.CXX ?? "c++"}`, + ] + : []), + `-DLLVM_DIR=${process.env.LLVM_DIR ?? "/usr/lib/llvm-22/lib/cmake/llvm"}`, + `-DSCRIPTC_PACKAGE_VERSION=${version}`, + `-DSCRIPTC_DEFAULT_TARGET=${triple}`, + `-DSCRIPTC_DEFAULT_DATA_LAYOUT=${DATA_LAYOUTS[arch]}`, + ...nativeTargetCmakeArgs, + "-DSCRIPTC_STATIC_LINUX_BUILD=ON", + "-DCMAKE_EXE_LINKER_FLAGS=-static", + "-DCMAKE_BUILD_TYPE=Release", + // Later definitions win, so these can select clang for musl builds. + ...helperLlvmDistArgs(build), + ], + { stdio: "inherit" }, + ); + execFileSync("cmake", ["--build", build, "--target", "scriptc-llvm-codegen"], { + stdio: "inherit", + }); + mkdirSync(dirname(output), { recursive: true }); + copyFileSync(join(build, "scriptc-llvm-codegen"), output); + execFileSync("strip", [output], { stdio: "inherit" }); + chmodSync(output, 0o755); +} diff --git a/native/llvm-codegen/llvm-dist.mjs b/native/llvm-codegen/llvm-dist.mjs new file mode 100644 index 000000000..aea90dd54 --- /dev/null +++ b/native/llvm-codegen/llvm-dist.mjs @@ -0,0 +1,96 @@ +/* The LLVM distribution the Linux helpers link in releases: LLVM built from + * the release sources with ThinLTO and with a profile of the helper's own + * code generation (llvm.profdata.gz). scripts/build-llvm-dist.mjs builds it, + * the llvm-dist workflow attaches it to the LLVM_DIST_TAG release of this + * repository, and scripts/refresh-llvm-profile.mjs collects the profile. */ +import { execFileSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { gunzipSync } from "node:zlib"; + +export const LLVM_VERSION = "22.1.8"; +/** The release of this repository that holds the distribution archives. A + * new profile or LLVM version gets a new tag. */ +export const LLVM_DIST_TAG = "llvm-22.1.8-pgo.1"; +export const llvmProfile = fileURLToPath(new URL("./llvm.profdata.gz", import.meta.url)); + +/** The archive name for a Linux architecture (`x64` or `arm64`). */ +export function llvmDistAsset(arch) { + return `scriptc-llvm-${LLVM_VERSION}-linux-${arch}.tar.xz`; +} + +/** Writes the uncompressed profile into `dir` and returns its path. */ +export function writeLlvmProfile(dir, profile = llvmProfile) { + mkdirSync(dir, { recursive: true }); + const path = join(dir, "llvm.profdata"); + writeFileSync(path, gunzipSync(readFileSync(profile))); + return path; +} + +/** CMake arguments for an LLVM build with the helper's targets, static, + * without tools or tests. `profile` builds with that indexed profile and + * ThinLTO; `instrumented` builds the IR-instrumented libraries the profile + * is collected with. clang and lld come from PATH and must be LLVM_VERSION, + * since the profile's format and hashes follow the compiler. */ +export function llvmCmakeArgs({ source, build, prefix, profile, instrumented }) { + const version = execFileSync("clang", ["--version"], { encoding: "utf8" }); + if (!version.includes(`clang version ${LLVM_VERSION}`)) + throw new Error(`the LLVM distribution is built with clang ${LLVM_VERSION}; found ${version}`); + return [ + "-S", + join(source, "llvm"), + "-B", + build, + "-G", + "Ninja", + "-DCMAKE_BUILD_TYPE=Release", + `-DCMAKE_INSTALL_PREFIX=${prefix}`, + "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_CXX_COMPILER=clang++", + "-DLLVM_USE_LINKER=lld", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64;WebAssembly", + "-DLLVM_ENABLE_PROJECTS=", + "-DLLVM_ENABLE_ASSERTIONS=OFF", + "-DLLVM_ENABLE_ZLIB=FORCE_ON", + "-DLLVM_ENABLE_ZSTD=FORCE_ON", + "-DLLVM_USE_STATIC_ZSTD=ON", + "-DLLVM_ENABLE_TERMINFO=OFF", + "-DLLVM_ENABLE_LIBXML2=OFF", + "-DLLVM_ENABLE_LIBEDIT=OFF", + "-DLLVM_ENABLE_LIBPFM=OFF", + "-DLLVM_INCLUDE_TESTS=OFF", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_INCLUDE_BENCHMARKS=OFF", + "-DLLVM_INCLUDE_DOCS=OFF", + "-DLLVM_BUILD_TOOLS=OFF", + ...(instrumented ? ["-DLLVM_BUILD_INSTRUMENTED=IR", "-DLLVM_BUILD_RUNTIME=OFF"] : []), + ...(profile ? ["-DLLVM_ENABLE_LTO=Thin", `-DLLVM_PROFDATA_FILE=${profile}`] : []), + ]; +} + +/** Extra CMake arguments for a Linux helper package build. With + * SCRIPTC_LLVM_THINLTO=1, LLVM_DIR names a distribution from + * scripts/build-llvm-dist.mjs: its archives hold ThinLTO bitcode, so the + * helper compiles and links with clang, lld and ThinLTO, and its own sources + * use the same profile. SCRIPTC_LLVM_PROFILE_GENERATE=1 instruments the + * helper's sources against an instrumented LLVM. */ +export function helperLlvmDistArgs(build) { + const args = []; + if (process.env.SCRIPTC_LLVM_THINLTO === "1") + args.push( + "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_CXX_COMPILER=clang++", + "-DSCRIPTC_LLVM_THINLTO=ON", + `-DSCRIPTC_LLVM_PROFILE_USE=${writeLlvmProfile(build)}`, + ); + else args.push("-DSCRIPTC_LLVM_THINLTO=OFF", "-DSCRIPTC_LLVM_PROFILE_USE="); + if (process.env.SCRIPTC_LLVM_PROFILE_GENERATE === "1") + args.push( + "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_CXX_COMPILER=clang++", + "-DSCRIPTC_LLVM_PROFILE_GENERATE=ON", + ); + else args.push("-DSCRIPTC_LLVM_PROFILE_GENERATE=OFF"); + return args; +} diff --git a/native/llvm-codegen/llvm.profdata.gz b/native/llvm-codegen/llvm.profdata.gz new file mode 100644 index 000000000..5ebb5076f Binary files /dev/null and b/native/llvm-codegen/llvm.profdata.gz differ diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index 14bcd1522..4a7e2333b 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -173,6 +173,23 @@ void limitProfileImports() { (void)Limited; } +// Drops the names of a partition's struct types before its summary is +// written. Every partition declares the program's types under the same +// names, and a thin link loads the importing partition and the partitions +// it imports from into one context, which tells same-named types apart by +// appending a counter that runs across the whole link. An edit that adds a +// type anywhere shifts the counter, so the module after importing, the +// object cache key, changed in partitions whose code had not. Unnamed types +// carry no such suffix; importing still matches them by their bodies. +struct UnnameStructTypes : PassInfoMixin { + PreservedAnalyses run(Module &M, ModuleAnalysisManager &) { + for (StructType *Type : M.getIdentifiedStructTypes()) + Type->setName(""); + return PreservedAnalyses::all(); + } + static bool isRequired() { return true; } +}; + // How the --profile-use profile matched the functions it annotated, summed // over partitions: functions that received its counts, functions whose // control flow changed since it was collected (hash mismatch), and functions @@ -733,6 +750,7 @@ compileOptimizedPartitions(const EmitOptions &Options, if (Annotating) P.countAnnotated(PartitionMatch.Annotated); ModulePassManager MPM = P.PB.buildThinLTOPreLinkDefaultPipeline(Level); + MPM.addPass(UnnameStructTypes()); MPM.addPass(BitcodeWriterPass(Stream, /*ShouldPreserveUseListOrder=*/false, /*EmitSummaryIndex=*/true, /*EmitModuleHash=*/true)); diff --git a/packages/cli/scripts/install-native.mjs b/packages/cli/scripts/install-native.mjs index 84eb846bc..4db76cc24 100644 --- a/packages/cli/scripts/install-native.mjs +++ b/packages/cli/scripts/install-native.mjs @@ -49,6 +49,7 @@ export function relocateToolchain(manifest, sourceDirectory, destinationDirector "declarations", "comptime", "wasi_node_runner", + "lib_diagnostics", ]) { if (typeof relocated[name] === "string") relocated[name] = pathFrom(relocated[name]); } diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index 127a11064..506e4cf6b 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -141,6 +141,7 @@ async function tryFastPath(): Promise { ...(optimization === "dev" || optimization === "speed" ? { optimization } : {}), ...(values.strip ? { strip: true as const } : {}), ...(values["detached-exit"] ? { detachedExit: true as const } : {}), + ...(values["cpu-dispatch"] === true ? { cpuDispatch: true as const } : {}), ...(checker === undefined ? {} : { checker }), npmStatic, ffiProfile: ffiPath === null ? null : { path: ffiPath, bytes: ffiBytes! }, @@ -220,6 +221,7 @@ async function coverageFastPath( values.strip || values["detached-exit"] || values["target-cpu"] !== undefined || + values["cpu-dispatch"] !== undefined || values.lib || values.profile !== undefined || values.ffi !== undefined || diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 8203b959e..5deb4ca29 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -10,6 +10,7 @@ import { mergeProfiles, resolveProvenanceSources, sourceTargetPlatform, + typePatches, warmNativeCaches, } from "@scriptc/compiler"; import { runCli } from "./command.js"; @@ -39,6 +40,7 @@ process.exitCode = await runCli(process.argv.slice(2), { sourceTargetPlatform, warmNativeCaches, mergeProfiles, + typePatches, run: (binary) => new Promise((resolveExit) => { let child; diff --git a/packages/cli/test/cpu-dispatch.test.ts b/packages/cli/test/cpu-dispatch.test.ts new file mode 100644 index 000000000..c20202aa0 --- /dev/null +++ b/packages/cli/test/cpu-dispatch.test.ts @@ -0,0 +1,82 @@ +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; + +const execFileAsync = promisify(execFile); +const require = createRequire(import.meta.url); +const repoRoot = join(import.meta.dirname, "../../.."); +const cliEntry = join(repoRoot, "packages/cli/src/main.ts"); +const bootstrapEntry = join(repoRoot, "packages/cli/dist/bootstrap.js"); +const tsxLoader = join(dirname(require.resolve("tsx/package.json")), "dist/loader.mjs"); + +const program = `function mix(a: Float64Array, b: Float64Array, k: number): number { + let s = 0; + for (let i = 0; i < a.length; i++) s += a[i]! * k + b[i]! / (i + 1); + return s; +} +const a = new Float64Array(4096); +const b = new Float64Array(4096); +for (let i = 0; i < a.length; i++) { + a[i] = ((i * 7919) % 1000) / 7 - 50; + b[i] = 1 / (i + 3); +} +let total = 0; +for (let r = 0; r < 32; r++) total += mix(a, b, r * 0.1); +console.log(total, [3, 1, 2].map((x) => mix(a.subarray(x), b, x)).join(" ")); +`; + +test.runIf(process.platform === "linux" && process.arch === "x64")( + "--cpu-dispatch builds one executable that matches Node at every level", + async () => { + const dir = await mkdtemp(join("/tmp", "scriptc-cpu-dispatch-")); + const entry = join(dir, "main.ts"); + const outPath = join(dir, "program"); + const llvmPath = join(dir, "main.ll"); + const env = { ...process.env, SCRIPTC_CACHE_DIR: join(dir, "cache") }; + const build = async (args: string[], bootstrap = false): Promise => { + await execFileAsync( + process.execPath, + [ + ...(bootstrap ? [bootstrapEntry] : ["--import", tsxLoader, cliEntry]), + "build", + entry, + "-o", + outPath, + ...args, + ], + { env, maxBuffer: 4 * 1024 * 1024 }, + ); + return readFile(llvmPath, "utf8"); + }; + const run = async (level?: string): Promise => + ( + await execFileAsync(outPath, [], { + env: { ...process.env, ...(level === undefined ? {} : { SCRIPTC_CPU_LEVEL: level }) }, + }) + ).stdout; + try { + await writeFile(entry, program); + const expected = `${(await execFileAsync(process.execPath, [entry])).stdout}`; + const plain = await build(["--optimization=speed"]); + expect(plain).not.toContain("@sc_cpu_level"); + const dispatching = await build(["--optimization=speed", "--cpu-dispatch"]); + expect(dispatching).toMatch(/^define internal double @sc_bf_mix\.v3\(/m); + for (const level of [undefined, "0", "baseline", "3", "x86-64-v3"]) + expect(await run(level), `SCRIPTC_CPU_LEVEL=${level}`).toBe(expected); + // The bootstrap's cached route keeps the two builds apart. + expect(await build(["--optimization=speed"], true)).toBe(plain); + expect(await build(["--optimization=speed", "--cpu-dispatch"], true)).toBe(dispatching); + expect(await build(["--cpu-dispatch"])).toContain("@sc_cpu_level_init()"); + expect(await run("0")).toBe(expected); + expect(await build(["--target-cpu=x86-64-v3", "--cpu-dispatch"])).not.toContain( + "@sc_cpu_level", + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, + 240_000, +); diff --git a/packages/cli/test/native-install.test.ts b/packages/cli/test/native-install.test.ts index c75b54267..5e51a2447 100644 --- a/packages/cli/test/native-install.test.ts +++ b/packages/cli/test/native-install.test.ts @@ -53,6 +53,7 @@ function fixture() { dsymutil: "dsymutil", comptime: "../lib/comptime", wasi_node_runner: "../lib/wasi/cli/wasi-runner.js", + lib_diagnostics: "../lib/lib-diagnostics.json", }; writeFileSync(binary + ".json", JSON.stringify(toolchain)); const helpers = [ @@ -93,6 +94,9 @@ test("installs a direct executable and relocatable references to platform assets expect(resolve(dirname(installed), manifest.wasi_node_runner)).toBe( join(platform, "wasi/cli/wasi-runner.js"), ); + expect(resolve(dirname(installed), manifest.lib_diagnostics)).toBe( + join(platform, "lib-diagnostics.json"), + ); expect(resolve(dirname(installed), manifest.runtime_packs[0].path)).toBe( join(platform, "runtime"), ); diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index f63cc790c..24db2c4c2 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 45, marker: "scr_runtime_abi_v45" }, + runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v45"); + expect(error?.stderr).toContain("scr_runtime_abi_v46"); }); }); diff --git a/packages/cli/test/output-options.test.ts b/packages/cli/test/output-options.test.ts index 21c5726c7..b9eb94843 100644 --- a/packages/cli/test/output-options.test.ts +++ b/packages/cli/test/output-options.test.ts @@ -48,6 +48,7 @@ describe("output option compatibility", () => { [{ emit: "llvm", strip: true }, /strip/], [{ emit: "obj", detachedExit: true }, /detached-exit/], [{ emit: "llvm", detachedExit: true }, /detached-exit/], + [{ emit: "asm", cpuDispatch: true }, /cpu-dispatch/], [{ emit: "obj", profileGenerate: "/tmp/%p.profraw" }, /profile-generate/], [{ emit: "llvm", profileUse: "/tmp/a.profdata" }, /profile-use/], [{ profileGenerate: "/tmp/%p.profraw", profileUse: "/tmp/a.profdata" }, /cannot be combined/], diff --git a/packages/cli/test/profile.test.ts b/packages/cli/test/profile.test.ts index 24c759e92..da40a15b6 100644 --- a/packages/cli/test/profile.test.ts +++ b/packages/cli/test/profile.test.ts @@ -63,8 +63,16 @@ test.skipIf(process.platform !== "linux")( expect(merge.stdout).toBe(`${merged}\n`); const optimized = join(dir, "optimized"); - await scriptc(["build", entry, "--profile-use", merged, "-o", optimized], env); + const fresh = await scriptc(["build", entry, "--profile-use", merged, "-o", optimized], env); expect((await execFileAsync(optimized, ["1000"])).stdout).toBe("total 167499\n"); + // The build reports how the profile matched the program's functions. + expect(fresh.stderr).toMatch( + /^scriptc: profile .*: [1-9]\d* functions annotated, 0 mismatched, \d+ without a record\n$/, + ); + expect(fresh.stderr).toContain(`scriptc: profile ${merged}: `); + // A build from the cache reports the same match. + const again = await scriptc(["build", entry, "--profile-use", merged, "-o", optimized], env); + expect(again.stderr).toBe(fresh.stderr); // A sparse profile leaves out the runtime functions the runs never // reached. @@ -77,10 +85,15 @@ test.skipIf(process.platform !== "linux")( await scriptc(["build", entry, "--profile-use", sparse, "-o", optimized], env); expect((await execFileAsync(optimized, ["1000"])).stdout).toBe("total 167499\n"); - // A profile from an older version of the program still builds it. + // A profile from an older version of the program still builds it. The + // edited function no longer matches its counts, and the build warns. await writeFile(entry, PROGRAM + 'console.log("edited");\n'); - await scriptc(["build", entry, "--profile-use", merged, "-o", optimized], env); + const stale = await scriptc(["build", entry, "--profile-use", merged, "-o", optimized], env); expect((await execFileAsync(optimized, ["1000"])).stdout).toBe("total 167499\nedited\n"); + expect(stale.stderr).toMatch(/ [1-9]\d* mismatched, /); + expect(stale.stderr).toMatch( + /\nscriptc: warning: [1-9]\d* of the \d+ functions .* describes \([\d.]+%\) changed since it was collected and are optimized without their counts; collect the profile again\n$/, + ); } finally { await rm(dir, { recursive: true, force: true }); } diff --git a/packages/compiler/ambient/node-fallback/fs.d.ts b/packages/compiler/ambient/node-fallback/fs.d.ts index 8d22ec66c..6399558bd 100644 --- a/packages/compiler/ambient/node-fallback/fs.d.ts +++ b/packages/compiler/ambient/node-fallback/fs.d.ts @@ -171,6 +171,16 @@ declare module "node:fs" { ): void; export function rmdirSync(path: string | URL): void; export function readdirSync(path: string | URL): string[]; + /* The recursive form: every entry below the path, relative to it. */ + export function readdirSync( + path: string | URL, + options: { + recursive?: boolean; + withFileTypes?: false; + encoding?: "utf8" | "utf-8"; + [option: string]: unknown; + }, + ): string[]; /* The withFileTypes form: Dirent rows (name + parentPath + the type * probes — the honest subset of Node's Dirent surface). The type is * interned explicitly in type-mapper.ts (the record's hidden %dtype field diff --git a/packages/compiler/native/cli.layout.gz b/packages/compiler/native/cli.layout.gz new file mode 100644 index 000000000..4ede69568 Binary files /dev/null and b/packages/compiler/native/cli.layout.gz differ diff --git a/packages/compiler/native/host.c b/packages/compiler/native/host.c index 24d330cc0..bffdf933f 100644 --- a/packages/compiler/native/host.c +++ b/packages/compiler/native/host.c @@ -56,7 +56,9 @@ void scr_collect_cycles_phase(void); * frontend's syntax trees and program state refer to each other, so * reference counting frees almost none of them when the frontend is * disposed, and the collector's schedule alone may not run a full pass - * before the build ends. */ + * before the build ends. The sweep then returns the freed pages to the + * system (scr_heap_trim), so code generation does not grow on top of the + * frontend's heap. */ void scriptc_native_collect_cycles(void) { scr_collect_cycles_phase(); } @@ -80,9 +82,30 @@ void scriptc_native_unlock(void) {} #include #include -int32_t scriptc_native_fork(void) { return (int32_t)fork(); } +#if defined(__linux__) +/* LLVM's profile runtime, linked into instrumented builds of the compiler + * (SCRIPTC_NATIVE_PROFILE_GENERATE); other builds leave these null. A worker + * starts with zeroed counters and writes its profile before _exit, which + * skips the runtime's atexit writer, so the profile counts the worker's work + * once. */ +extern void __llvm_profile_reset_counters(void) __attribute__((weak)); +extern int __llvm_profile_write_file(void) __attribute__((weak)); +#endif -void scriptc_native_exit(int32_t code) { _exit(code); } +int32_t scriptc_native_fork(void) { + pid_t pid = fork(); +#if defined(__linux__) + if (pid == 0 && __llvm_profile_reset_counters != NULL) __llvm_profile_reset_counters(); +#endif + return (int32_t)pid; +} + +void scriptc_native_exit(int32_t code) { +#if defined(__linux__) + if (__llvm_profile_write_file != NULL) (void)__llvm_profile_write_file(); +#endif + _exit(code); +} /* The worker's exit status, 128 plus the signal for a worker a signal * ended, or -1 when waitpid fails. */ diff --git a/packages/compiler/native/ts7-process.test.c b/packages/compiler/native/ts7-process.test.c index b6d760abd..85de63db1 100644 --- a/packages/compiler/native/ts7-process.test.c +++ b/packages/compiler/native/ts7-process.test.c @@ -1,6 +1,10 @@ #ifndef _WIN32 #define _POSIX_C_SOURCE 200809L #endif +/* macOS declares mkdtemp only with the Darwin extensions visible. */ +#if defined(__APPLE__) && !defined(_DARWIN_C_SOURCE) +#define _DARWIN_C_SOURCE 1 +#endif #include "ts7-process.h" #include #include diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 03af418a5..d1c39d8ac 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -40,7 +40,7 @@ }, "dependencies": { "@scriptc/runtime": "workspace:*", - "tsc-ts": "0.0.9", + "tsc-ts": "0.0.10", "typescript": "7.0.2", "typescript5": "npm:typescript@5.9.3" }, diff --git a/packages/compiler/src/backend/executable-features.ts b/packages/compiler/src/backend/executable-features.ts index b31317cb5..a3350eb89 100644 --- a/packages/compiler/src/backend/executable-features.ts +++ b/packages/compiler/src/backend/executable-features.ts @@ -31,12 +31,14 @@ export function executableLinkFeatures(mod: IrModule): NativeLinkFeatures { ...(features.streamDyn ? { streamDyn: true } : {}), ...(features.handleProps ? { handleProps: true } : {}), ...(features.declAdopt ? { declAdopt: true } : {}), + ...(features.typeOrigins ? { typeOrigins: true } : {}), ...(features.dynForAwait ? { dynForAwait: true } : {}), ...(features.eventsOnceSignal ? { eventsOnceSignal: true } : {}), ...(features.mime ? { mime: true } : {}), ...(features.utilTiming ? { utilTiming: true } : {}), ...(features.dynBuffer ? { dynBuffer: true } : {}), ...(features.builtinValues ? { builtinValues: true } : {}), + ...(features.computedRequire ? { computedRequire: true } : {}), events: features.processEvents, emitter: features.emitter, symbol: features.symbol, diff --git a/packages/compiler/src/backend/library-pack.ts b/packages/compiler/src/backend/library-pack.ts index 521f1d53e..14ffdff28 100644 --- a/packages/compiler/src/backend/library-pack.ts +++ b/packages/compiler/src/backend/library-pack.ts @@ -166,6 +166,7 @@ export async function compilePackedLibrary( options.programPublicSymbols, stem, target.platform, + { inProcessElf: true }, ); } else { program = await emit( diff --git a/packages/compiler/src/backend/link-plan-core.test.ts b/packages/compiler/src/backend/link-plan-core.test.ts new file mode 100644 index 000000000..9c33e6fd0 --- /dev/null +++ b/packages/compiler/src/backend/link-plan-core.test.ts @@ -0,0 +1,20 @@ +import { expect, test } from "vitest"; +import { systemLibraryLinkerArgs } from "./link-plan-core.js"; +import { MACOS_ARM64_TARGET, LINUX_ARM64_GNU_TARGET, LINUX_X64_GNU_TARGET } from "./targets.js"; + +test("Linux links libm as needed and other system libraries as given", () => { + for (const target of [LINUX_X64_GNU_TARGET, LINUX_ARM64_GNU_TARGET]) { + expect(systemLibraryLinkerArgs(target, ["m"])).toEqual([ + "-Wl,--as-needed", + "-lm", + "-Wl,--no-as-needed", + ]); + expect(systemLibraryLinkerArgs(target, ["z", "m"])).toEqual([ + "-lz", + "-Wl,--as-needed", + "-lm", + "-Wl,--no-as-needed", + ]); + } + expect(systemLibraryLinkerArgs(MACOS_ARM64_TARGET, ["System"])).toEqual(["-lSystem"]); +}); diff --git a/packages/compiler/src/backend/link-plan-core.ts b/packages/compiler/src/backend/link-plan-core.ts index de5da1657..fc05d2961 100644 --- a/packages/compiler/src/backend/link-plan-core.ts +++ b/packages/compiler/src/backend/link-plan-core.ts @@ -70,3 +70,19 @@ export function executableLinkInputs(options: { ], }; } + +/** The linker arguments for an executable's system libraries. On Linux, libm + * links as needed: the runtime defines the libm functions most programs + * reach (scr_glibc_math.c) and Math's transcendentals (scr_ieee754.c), so a + * program that calls no other libm function, such as pow, starts without + * loading libm.so. */ +export function systemLibraryLinkerArgs( + target: Pick, + names: readonly string[], +): string[] { + return names.flatMap((name) => + name === "m" && target.platform === "linux" + ? ["-Wl,--as-needed", "-lm", "-Wl,--no-as-needed"] + : [`-l${name}`], + ); +} diff --git a/packages/compiler/src/backend/linker.ts b/packages/compiler/src/backend/linker.ts index a54956ed9..c4f04133e 100644 --- a/packages/compiler/src/backend/linker.ts +++ b/packages/compiler/src/backend/linker.ts @@ -22,6 +22,7 @@ import { type NativeArtifactDependency } from "./native/contracts.js"; import { RuntimePackError, stageRuntimePackArtifacts } from "./runtime-pack.js"; import type { NativeLinkPlan } from "./link-plan.js"; import { lldLinkerArgs } from "./lld.js"; +import { systemLibraryLinkerArgs } from "./link-plan-core.js"; import { createDarwinDebugSymbols, installDarwinDebugSymbols, @@ -248,7 +249,7 @@ export async function linkNativeExecutable( ...lld, ...plan.driverFlags, ...plan.inputs.map((input) => staged.replacements.get(input) ?? input), - ...plan.systemLibraries.map((name) => `-l${name}`), + ...systemLibraryLinkerArgs(plan.target, plan.systemLibraries), "-o", privateOut, ]; diff --git a/packages/compiler/src/backend/llvm/bytes-headers.ts b/packages/compiler/src/backend/llvm/bytes-headers.ts new file mode 100644 index 000000000..f5b95cf75 --- /dev/null +++ b/packages/compiler/src/backend/llvm/bytes-headers.ts @@ -0,0 +1,77 @@ +/* Typed-array header fields read once at a speculative loop's entry. + * + * A view's length, data pointer and shared storage are set at construction + * and never change, and a view held by a const local or a parameter that + * the loop does not rebind is the same view in every iteration. The entry + * reads those fields once, and every element access in the loop's copies + * (the ordinary copy, the fast copy and the copy a failed check resumes in) + * uses the values. No header read is then left inside the copies for LLVM's + * load PRE to move ahead of the fast copy's exits: such a move keeps the + * fast loop's first iteration in the loop, with the checks that one + * iteration of it makes redundant. */ +import type { IrExpr, IrStmt } from "../../ir/ir.js"; +import { everyStmt } from "../../ir/traverse.js"; +import { emitBytesData, emitBytesHeaderLength, emitBytesSharedStorage } from "./expr-bytes.js"; +import type { LlEmitter } from "./emitter.js"; + +export interface BytesHeader { + length: string; + data: string; + /** The view's shared storage (worker programs only). */ + shared: string | null; +} + +export interface BytesHeaderScope { + /** The function the scope's values belong to. */ + fn: string; + locals: ReadonlyMap; + /** The header of each receiver value read from one of `locals`. */ + values: Map; +} + +/** Reads the header fields of the views `loop` accesses through a const + * local or a parameter it does not rebind, at the current block, which must + * dominate every copy of the loop. Null when there are none. */ +export function emitLoopBytesHeaders( + emitter: LlEmitter, + loop: IrStmt, + parameters: ReadonlySet, +): BytesHeaderScope | null { + const receivers = new Set(); + const rebound = new Set(); + const receiver = (e: IrExpr): void => { + if (e.kind === "varRef") receivers.add(e.localId); + }; + everyStmt(loop, { + expr: (e) => { + if (e.kind === "bytesIntrinsic") receiver(e.receiver); + else if (e.kind === "assignExpr" || e.kind === "incDec") rebound.add(e.localId); + return true; + }, + stmt: (s) => { + if (s.kind === "bytesSet") receiver(s.arr); + else if (s.kind === "assign" || s.kind === "forOf" || s.kind === "varDecl") + rebound.add(s.localId); + else if (s.kind === "tryCatch" && s.catchLocalId !== null) rebound.add(s.catchLocalId); + return true; + }, + }); + const locals = new Map(); + for (const id of receivers) { + if (rebound.has(id)) continue; + const local = emitter.currentLocals.get(id); + if (local === undefined || local.boxed === true || local.type.kind !== "bytes") continue; + if (local.mutable && !parameters.has(id)) continue; + if (emitter.integerLoopBindings.has(id) || emitter.int32Locals.has(id)) continue; + const B = emitter.B; + const view = B.tmp(); + B.line(`${view} = load ptr, ptr ${emitter.binding(id).slot}${emitter.slotAlias(id)}`); + locals.set(id, { + length: emitBytesHeaderLength(emitter, view), + data: emitBytesData(emitter, view), + shared: emitter.mod.workers ? emitBytesSharedStorage(emitter, view) : null, + }); + } + if (locals.size === 0) return null; + return { fn: emitter.currentFunctionName, locals, values: new Map() }; +} diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts index 0fa8fea34..742d256b7 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts @@ -320,16 +320,17 @@ test("worker programs test each receiver's shared storage once in the guard", () const fast = /^bytes\.version\.fast\d+:$[^]*?^bytes\.version\.fast\.shared\d+:$/m.exec(body)![0]; expect(fast).not.toContain("bytes.shared"); expect(fast).not.toMatch(/(load|store) atomic double/); - // The checked copy keeps the shared-memory path for each access, and so - // does the copy for shared views whose bounds hold. + // The checked copy keeps the shared-memory path for each access. The copy + // for shared views whose bounds hold is a counted loop that polls between + // blocks, so its accesses at the counter are plain. const slow = body.slice(body.search(/^bytes\.version\.slow\d+:$/m), body.indexOf(fast)); expect(slow).toContain("load atomic double"); expect(slow).toContain("store atomic double"); const shared = /^bytes\.version\.fast\.shared\d+:$[^]*?^bytes\.version\.done\d+:$/m.exec( body, )![0]; - expect(shared).toContain("load atomic double"); - expect(shared).toContain("store atomic double"); + expect(shared).toMatch(/^loop\.poll\d+:$/m); + expect(shared).not.toMatch(/(load|store) atomic double/); }); test("index temps of a version's counter are integers in the fast copy", () => { diff --git a/packages/compiler/src/backend/llvm/checked-narrows.ts b/packages/compiler/src/backend/llvm/checked-narrows.ts index c1514df1e..5fbaf6d08 100644 --- a/packages/compiler/src/backend/llvm/checked-narrows.ts +++ b/packages/compiler/src/backend/llvm/checked-narrows.ts @@ -101,6 +101,13 @@ export class CheckedNarrows { } } +/** A call to a checked extraction whose helper borrows its argument, so + * emitCheckedNarrowReceiver can lend the payload instead of calling it. */ +export function borrowsCheckedNarrow(host: LlvmEmitterContext, e: IrExpr): boolean { + if (e.kind !== "call" || host.checkedNarrows.get(e) === null) return false; + return host.callLifetimes.borrowed.get(e.callee)?.has(0) === true && host.mayThrow.has(e.callee); +} + /** A checked extraction consumed at once by a field or tag load. The * successful arm borrows the payload from the union, which itself borrows * from its source: a dense array read keeps the element in the array slot, @@ -113,7 +120,7 @@ export function emitCheckedNarrowReceiver( narrow: CheckedNarrowCall, ): LlValue | null { const callee = narrow.call.callee; - if (!host.callLifetimes.borrowed.get(callee)?.has(0) || !host.mayThrow.has(callee)) return null; + if (!borrowsCheckedNarrow(host, narrow.call)) return null; const B = host.B; const read = host.optionalArrayReads.get(narrow.value); const union = read diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index 699755157..74880453f 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -573,13 +573,21 @@ export function classIdLoadLines( * 64-bit word. The count is the i32 at offset 0. */ export const CLASS_ID_OFFSET = 4; +/** The bound on layout ids (scr_rc.h SCR_RC_LAYOUT_ID_LIMIT): the high + * half of a 64-bit rc word keeps the layout id in its low 30 bits, beside + * the program-lifetime and frozen bits. A class id is the layout id of its + * class's canonical layout. */ +export const LAYOUT_ID_LIMIT = 0x3fffffff; + /** The rc word an LlClassMeta.idInRc instance is created with: the class * id, its preorder number plus 1, in the high 32 bits, and in the low 32 bits * a count of 1, or for a program-lifetime instance the midpoint of the 32-bit - * count (as programLifetimeRc is of the 32-bit word). The id is never 0, - * 0x80000000 or 0xffffffff, the high halves of other objects' rc words, so - * the runtime can tell an object with a class id (scr_rc.h scr_rc_class_id). */ + * count (as programLifetimeRc is of the 32-bit word). The id is a layout id + * (LAYOUT_ID_LIMIT), so it never sets the program-lifetime or frozen bit + * and the runtime reads it back (scr_rc.h scr_rc_layout_id). */ export function classIdRcWord(meta: LlClassMeta, lifetime: boolean): string { + if (meta.pre + 1 >= LAYOUT_ID_LIMIT) + throw new InternalCompilerError(`class id ${meta.pre + 1} is not a layout id`); return (BigInt(meta.pre + 1) * 2n ** 32n + (lifetime ? 2n ** 31n : 1n)).toString(); } diff --git a/packages/compiler/src/backend/llvm/common.ts b/packages/compiler/src/backend/llvm/common.ts index cca1b603b..f1bdeb557 100644 --- a/packages/compiler/src/backend/llvm/common.ts +++ b/packages/compiler/src/backend/llvm/common.ts @@ -277,6 +277,10 @@ export function emitFieldAbsentTest(B: BlockBuilder, value: string, undefinedTag return absent; } +/** ScrClosure.function_kind's SCR_FN_RECEIVER_FREE bit (scr_closure.h): + * the body never observes the ambient receiver. */ +export const CLOSURE_RECEIVER_FREE = 16; + /** The right-hand side of a JS function-identity comparison: two closures * are one function when their identity roots match (a signature adapter * stands for the function it adapts). Both operands must be closures. */ diff --git a/packages/compiler/src/backend/llvm/constant-callbacks.test.ts b/packages/compiler/src/backend/llvm/constant-callbacks.test.ts index e8ca5571f..8bdab0408 100644 --- a/packages/compiler/src/backend/llvm/constant-callbacks.test.ts +++ b/packages/compiler/src/backend/llvm/constant-callbacks.test.ts @@ -299,6 +299,47 @@ test("receiver readers propagate backwards through direct calls and recursion", ).toBe(false); }); +test("signature adapters that only forward the receiver ignore it with their original", () => { + const forward = (receiver: IrExpr): IrStmt => ({ + kind: "return", + value: { + kind: "callValue", + callee: ref("f.0"), + receiver, + args: [{ kind: "varRef", localId: "a.0", type: F64, loc }], + type: BOOL, + loc, + }, + loc, + }); + const ambient: IrExpr = { kind: "libCall", fn: "dyn.this", args: [], type: { kind: "dyn" }, loc }; + const adapter: IrFunction = { + name: "%fn.adapt.x.impl", + params: [{ localId: "a.0", name: "a", type: F64 }], + captures: [{ localId: "f.0", name: "f", type: callback }], + locals: [ + { id: "f.0", name: "f", type: callback, mutable: false, boxed: true }, + { id: "a.0", name: "a", type: F64, mutable: false }, + ], + returnType: BOOL, + body: [forward(ambient)], + loc, + }; + const readers = (): AmbientReceiverReaders => + new AmbientReceiverReaders(new Map([[adapter.name, adapter]])); + // The adapter itself reads the receiver, only to forward it. + expect(readers().ignores(adapter)).toBe(false); + expect(readers().forwardsOnly(adapter)).toBe(true); + // Any other read of the receiver, or forwarding something else, does not. + adapter.body = [thisRead, forward(ambient)]; + expect(readers().forwardsOnly(adapter)).toBe(false); + adapter.body = [forward(ref("a.0", F64))]; + expect(readers().forwardsOnly(adapter)).toBe(false); + adapter.body = [forward(ambient)]; + adapter.captures = []; + expect(readers().forwardsOnly(adapter)).toBe(false); +}); + test("virtual calls admit every same-named method", () => { const call: IrStmt = { kind: "exprStmt", diff --git a/packages/compiler/src/backend/llvm/constant-choices.test.ts b/packages/compiler/src/backend/llvm/constant-choices.test.ts new file mode 100644 index 000000000..f19b5b19d --- /dev/null +++ b/packages/compiler/src/backend/llvm/constant-choices.test.ts @@ -0,0 +1,109 @@ +import { expect, test } from "vitest"; +import { BOOL, F64, type IrExpr, type IrStmt } from "../../ir/ir.js"; +import { matchConstantChoice } from "./constant-choices.js"; + +const loc = { file: "choices.ts", start: 0, end: 0 }; +const bytes = { kind: "bytes", elem: "u8" } as const; +const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc }); +const ref = (localId: string, type: IrExpr["type"] = F64): IrExpr => ({ + kind: "varRef", + localId, + type, + loc, +}); +const is = (left: IrExpr, right: IrExpr): IrExpr => ({ + kind: "bin", + op: "===", + left, + right, + type: BOOL, + loc, +}); +const store = (value: IrExpr, index: IrExpr = ref("i"), arr = "a"): IrStmt => ({ + kind: "bytesSet", + arr: ref(arr, bytes), + index, + value, + loc, +}); +const branch = (cond: IrExpr, then: IrStmt[], else_: IrStmt[] | null): IrStmt => ({ + kind: "if", + cond, + then, + else_, + loc, +}); +/** `if (x === c0) a[i] = r0; else if ... else a[i] = fallback`. */ +function chain(cases: [number, number][], fallback: IrStmt[] | null, subject = "x"): IrStmt { + let rest = fallback; + for (let k = cases.length - 1; k >= 0; k--) { + const [value, result] = cases[k]!; + rest = [branch(is(ref(subject), num(value)), [store(num(result))], rest)]; + } + return rest![0]!; +} + +test("a chain of literal stores to one element is a constant choice", () => { + const choice = matchConstantChoice( + chain( + [ + [97, 0], + [116, 1], + [99, 2], + ], + [store(num(3))], + ), + ); + expect(choice?.cases).toEqual([ + { value: 97, result: 0 }, + { value: 116, result: 1 }, + { value: 99, result: 2 }, + ]); + expect(choice?.fallback).toBe(3); + expect(choice?.stores).toHaveLength(4); +}); + +test("a literal on the left, -0 and a repeated value keep strict equality's first arm", () => { + const first: IrStmt = branch( + is(num(-0), ref("x")), + [store(num(5))], + [ + chain( + [ + [1, 6], + [0, 7], + [2, 8], + ], + [store(num(9))], + ), + ], + ); + expect(matchConstantChoice(first)?.cases).toEqual([ + { value: 0, result: 5 }, + { value: 1, result: 6 }, + { value: 2, result: 8 }, + ]); +}); + +test("chains that are not one choice of literals keep their arms", () => { + const three: [number, number][] = [ + [1, 10], + [2, 20], + [3, 30], + ]; + // Two values, no final else, a fraction, or cases spanning more than 256 values. + expect(matchConstantChoice(chain(three.slice(0, 2), [store(num(0))]))).toBeNull(); + expect(matchConstantChoice(chain(three, null))).toBeNull(); + expect(matchConstantChoice(chain([...three, [1.5, 40]], [store(num(0))]))).toBeNull(); + expect(matchConstantChoice(chain([...three, [256, 40]], [store(num(0))]))).not.toBeNull(); + expect(matchConstantChoice(chain([...three, [257, 40]], [store(num(0))]))).toBeNull(); + // A final else that is not a literal, or that stores elsewhere. + expect(matchConstantChoice(chain(three, [store(ref("y"))]))).toBeNull(); + expect(matchConstantChoice(chain(three, [store(num(0), ref("j"))]))).toBeNull(); + expect(matchConstantChoice(chain(three, [store(num(0), ref("i"), "b")]))).toBeNull(); + // An arm with a second statement. + expect(matchConstantChoice(chain(three, [store(num(0)), store(num(1))]))).toBeNull(); + // A second subject. + const mixed = branch(is(ref("y"), num(0)), [store(num(1))], [chain(three, [store(num(0))])]); + expect(matchConstantChoice(mixed)).toBeNull(); +}); diff --git a/packages/compiler/src/backend/llvm/constant-choices.ts b/packages/compiler/src/backend/llvm/constant-choices.ts new file mode 100644 index 000000000..6e7bb4c84 --- /dev/null +++ b/packages/compiler/src/backend/llvm/constant-choices.ts @@ -0,0 +1,241 @@ +import { createHash } from "node:crypto"; +import { InternalCompilerError } from "../../errors.js"; +import type { IrExpr, IrStmt } from "../../ir/ir.js"; +import { exactInteger, widenInteger } from "./integer-values.js"; +import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; +import { f64Lit } from "./common.js"; + +type BytesSet = Extract; + +/** An if/else-if chain that stores one of several numeric literals to the + * same typed-array element, choosing it by comparing one number local with + * integer literals: + * + * if (x === 97) a[i] = 0; else if (x === 116) a[i] = 1; ... else a[i] = 3; + * + * The chain is emitted as one store of a value read from a constant table, + * with no branch on the comparisons. Branches on unpredictable input (the + * bases of a DNA sequence, the bytes of a text) mispredict about half the + * time; LLVM keeps them, because sparse cases (97, 99, 116) are too far + * apart for its own switch tables. */ +export interface ConstantChoice { + subject: IrExpr; + /** The first arm for each distinct value, in chain order. */ + cases: readonly { value: number; result: number }[]; + fallback: number; + /** Every arm's store, the final else's last. */ + stores: readonly BytesSet[]; +} + +/** At least this many distinct values: shorter chains predict as well as a + * table load. */ +const MIN_CASES = 3; +/** The table covers the subject's values from the smallest case (or the + * subject's proven minimum) up to this many entries. */ +const MAX_TABLE = 256; + +function integerLiteral(expr: IrExpr): number | null { + if (expr.kind !== "numLit" || !Number.isSafeInteger(expr.value)) return null; + // `x === -0` and `x === 0` select the same values. + return expr.value === 0 ? 0 : expr.value; +} + +function sameVarRef(a: IrExpr, b: IrExpr): boolean { + return a.kind === "varRef" && b.kind === "varRef" && a.localId === b.localId; +} + +/** A local read or a number literal: reading it has no effect. */ +function sameOperand(a: IrExpr, b: IrExpr): boolean { + if (a.kind === "numLit" && b.kind === "numLit") return Object.is(a.value, b.value); + return sameVarRef(a, b); +} + +/** The store of a single-statement arm, when it stores a number literal to + * an element named by a local and a local or a literal index (so evaluating + * them once, after the comparisons, reads what every arm would). */ +function literalStore(body: readonly IrStmt[] | null): BytesSet | null { + if (body === null || body.length !== 1) return null; + const s = body[0]!; + if (s.kind !== "bytesSet" || s.value.kind !== "numLit") return null; + if (s.arr.kind !== "varRef" || (s.index.kind !== "varRef" && s.index.kind !== "numLit")) + return null; + if (s.arr.type.kind !== "bytes" || s.index.type.kind !== "f64") return null; + return s; +} + +export function matchConstantChoice(s: IrStmt): ConstantChoice | null { + if (s.kind !== "if") return null; + let subject: IrExpr | null = null; + const cases: { value: number; result: number }[] = []; + const seen = new Set(); + const stores: BytesSet[] = []; + let current: IrStmt = s; + for (;;) { + if (current.kind !== "if") return null; + const cond = current.cond; + if (cond.kind !== "bin" || cond.op !== "===") return null; + const [local, literal] = + cond.left.kind === "varRef" ? [cond.left, cond.right] : [cond.right, cond.left]; + if (local.kind !== "varRef" || local.type.kind !== "f64") return null; + if (subject === null) subject = local; + else if (!sameVarRef(subject, local)) return null; + const value = integerLiteral(literal); + if (value === null) return null; + const store = literalStore(current.then); + if (store === null) return null; + stores.push(store); + // A repeated value never reaches its later arm. + if (!seen.has(value)) { + seen.add(value); + cases.push({ value, result: (store.value as IrExpr & { kind: "numLit" }).value }); + } + const rest: readonly IrStmt[] | null = current.else_; + if (rest === null || rest.length !== 1) return null; + if (rest[0]!.kind === "if") { + current = rest[0]!; + continue; + } + const last = literalStore(rest); + if (last === null) return null; + stores.push(last); + break; + } + const first = stores[0]!; + for (const store of stores) { + if (!sameVarRef(store.arr, first.arr) || !sameOperand(store.index, first.index)) return null; + if (store.arr.type.kind !== "bytes" || first.arr.type.kind !== "bytes") return null; + if (store.arr.type.elem !== first.arr.type.elem) return null; + } + if (cases.length < MIN_CASES) return null; + const values = cases.map((c) => c.value); + if (Math.max(...values) - Math.min(...values) >= MAX_TABLE) return null; + const fallback = (stores[stores.length - 1]!.value as IrExpr & { kind: "numLit" }).value; + return { subject: subject!, cases, fallback, stores }; +} + +function int32(value: number): boolean { + return ( + Number.isInteger(value) && value >= -2147483648 && value <= 2147483647 && !Object.is(value, -0) + ); +} + +/** The chosen number: the table entry for the subject's value, or the final + * else's literal for a value no case names (including NaN, a fraction and + * every value outside the table). */ +function emitChoice(host: LlvmEmitterContext, choice: ConstantChoice): LlValue { + const B = host.B; + const subject = host.emitExpr(choice.subject); + const results = [...choice.cases.map((c) => c.result), choice.fallback]; + const integral = results.every(int32); + const element = integral ? "i32" : "double"; + const literal = (value: number): string => (integral ? String(value) : f64Lit(value)); + + const proven = exactInteger(host, subject, choice.subject); + const values = choice.cases.map((c) => c.value); + let low = Math.min(...values); + let high = Math.max(...values); + // A proven range that fits a table needs no test: every value it allows + // has an entry. + const covered = + proven !== null && proven.range.max - proven.range.min < MAX_TABLE ? proven : null; + if (covered !== null) { + low = covered.range.min; + high = covered.range.max; + } + const size = high - low + 1; + const entries: string[] = []; + for (let value = low; value <= high; value++) { + const hit = choice.cases.find((c) => c.value === value); + entries.push(`${element} ${literal(hit === undefined ? choice.fallback : hit.result)}`); + } + const table = `@sc_choice_${createHash("sha256") + .update(`${element}:${entries.join(",")}`) + .digest("hex") + .slice(0, 16)}`; + host.declare( + `${table} = private unnamed_addr constant [${size} x ${element}] [${entries.join(", ")}]`, + ); + + let index: string; + let ok: string | null = null; + if (covered !== null) { + const wide = widenInteger(host, covered); + index = B.tmp(); + B.line(`${index} = sub i64 ${wide}, ${low}`); + } else { + // freeze keeps NaN and out-of-range conversions from producing poison; + // the round trip then rejects every value that is not an exact integer + // (-0 converts to 0, which `=== 0` selects too). + const raw = B.tmp(); + const selector = B.tmp(); + const restored = B.tmp(); + const exact = B.tmp(); + const offset = B.tmp(); + const inside = B.tmp(); + ok = B.tmp(); + index = B.tmp(); + B.line(`${raw} = fptosi double ${subject.name} to i64`); + B.line(`${selector} = freeze i64 ${raw}`); + B.line(`${restored} = sitofp i64 ${selector} to double`); + B.line(`${exact} = fcmp oeq double ${subject.name}, ${restored}`); + B.line(`${offset} = sub i64 ${selector}, ${low}`); + B.line(`${inside} = icmp ult i64 ${offset}, ${size}`); + B.line(`${ok} = and i1 ${exact}, ${inside}`); + B.line(`${index} = select i1 ${ok}, i64 ${offset}, i64 0`); + } + const slot = B.tmp(); + const entry = B.tmp(); + B.line( + `${slot} = getelementptr inbounds [${size} x ${element}], ptr ${table}, i64 0, i64 ${index}`, + ); + B.line(`${entry} = load ${element}, ptr ${slot}`); + let chosen = entry; + if (ok !== null) { + chosen = B.tmp(); + B.line( + `${chosen} = select i1 ${ok}, ${element} ${entry}, ${element} ${literal(choice.fallback)}`, + ); + } + if (!integral) return { name: chosen, type: choice.stores[0]!.value.type }; + const number = B.tmp(); + B.line(`${number} = sitofp i32 ${chosen} to double`); + return { + name: number, + type: choice.stores[0]!.value.type, + uint32: chosen, + integer: { + name: chosen, + type: "i32", + signed: Math.min(...results) < 0, + range: { min: Math.min(...results), max: Math.max(...results) }, + }, + }; +} + +/** Emits a matched chain as one store of the chosen literal. The element + * write is the arms' own: every arm names the same receiver and index. */ +export function emitConstantChoice(host: LlvmEmitterContext, choice: ConstantChoice): void { + const first = choice.stores[0]!; + if (first.arr.type.kind !== "bytes") + throw new InternalCompilerError("llvm emitter bug: constant choice on non-bytes"); + // An arm's index carries the ranges proven inside that arm: an index that + // is the compared local holds the arm's value there. The one store runs + // for every value, so such an index reads the local through the first + // comparison's operand, whose range holds before the chain branches. + const indexExpr = sameVarRef(first.index, choice.subject) ? choice.subject : first.index; + const arr = host.emitStableReceiver(first.arr, [indexExpr]); + const index = host.emitExpr(indexExpr); + const value = emitChoice(host, choice); + const scopes = choice.stores.map((store) => host.bytesAliasScopes.get(store)); + host.emitBytesSet( + first.arr.type.elem, + arr.name, + index, + value, + indexExpr, + undefined, + choice.stores.every((store) => host.bytesBounds.has(store)), + choice.stores.every((store) => host.bytesOwned.has(store)), + scopes.every((scope) => scope === scopes[0]) ? (scopes[0] ?? "") : "", + ); +} diff --git a/packages/compiler/src/backend/llvm/cpu-dispatch.ts b/packages/compiler/src/backend/llvm/cpu-dispatch.ts new file mode 100644 index 000000000..f24fc8674 --- /dev/null +++ b/packages/compiler/src/backend/llvm/cpu-dispatch.ts @@ -0,0 +1,330 @@ +/* Runtime CPU dispatch for x86-64 executables (cloneForCpuDispatch): the + * program's functions get a copy compiled for x86-64-v3, chosen once per + * call by the CPU level the executable detects at startup. */ +import type { TargetCpu } from "../target-cpu.js"; + +/** The levels a dispatching executable carries a copy of its program's + * functions for, above the baseline. */ +export const CPU_DISPATCH_TIERS: readonly TargetCpu[] = ["x86-64-v3"]; + +/** The CPU level the executable runs at: 1 (the x86-64 baseline) or 3 + * (x86-64-v3), set by @sc_cpu_level_init before the program runs. */ +const LEVEL = "@sc_cpu_level"; + +/** The attribute groups of the x86-64-v3 copies: #0's and #2's (see + * emit-module.ts functionAttributes) with the level's `target-cpu`. */ +const COPY_GROUPS: ReadonlyMap = new Map([ + ["0", "3"], + ["2", "4"], +]); + +export function cpuDispatchAttributes(asan: string): string[] { + return [ + `attributes #3 = {${asan} "target-cpu"="x86-64-v3" }`, + `attributes #4 = { noinline${asan} "target-cpu"="x86-64-v3" }`, + ]; +} + +/** Bits of CPUID leaf 1's ECX that x86-64-v3 needs: SSE3 (0), SSSE3 (9), + * FMA (12), CMPXCHG16B (13), SSE4.1 (19), SSE4.2 (20), MOVBE (22), POPCNT + * (23), XSAVE (26), OSXSAVE (27), AVX (28) and F16C (29). */ +const LEAF1_ECX = 0x3cd83201; +/** Leaf 7's EBX: BMI1 (3), AVX2 (5) and BMI2 (8). */ +const LEAF7_EBX = 0x128; +/** Leaf 0x80000001's ECX: LAHF/SAHF (0) and LZCNT (5). */ +const EXT1_ECX = 0x21; + +/** `SCRIPTC_CPU_LEVEL` values and the level each caps the executable at. */ +const LEVEL_NAMES: readonly [string, number][] = [ + ["0", 0], + ["1", 1], + ["2", 2], + ["3", 3], + ["4", 4], + ["baseline", 1], + ["v1", 1], + ["v2", 2], + ["v3", 3], + ["v4", 4], + ["x86-64", 1], + ["x86-64-v2", 2], + ["x86-64-v3", 3], + ["x86-64-v4", 4], +]; + +/** @sc_cpu_level and @sc_cpu_level_init, which main calls first: + * - CPUID reports every x86-64-v3 feature, and XGETBV that the operating + * system saves the SSE and AVX registers, for level 3; otherwise 1; + * - `SCRIPTC_CPU_LEVEL` (a level from 0 to 4, `baseline` or an `x86-64-vN` + * name) caps the level; another value caps it at 0. */ +export function cpuLevelDefs(declare: (line: string) => void): string[] { + declare(`declare ptr @getenv(ptr)`); + declare(`declare i32 @strcmp(ptr, ptr)`); + const bytes = (text: string): string => `[${text.length + 1} x i8] c"${text}\\00"`; + const lines = [ + `${LEVEL} = internal global i32 1`, + `@sc_cpu_level_env = private unnamed_addr constant ${bytes("SCRIPTC_CPU_LEVEL")}`, + ...LEVEL_NAMES.map( + ([name], i) => `@sc_cpu_level_name_${i} = private unnamed_addr constant ${bytes(name)}`, + ), + `define internal void @sc_cpu_level_init() #0 {`, + `entry:`, + ` %max = call { i32, i32, i32, i32 } asm "cpuid", "={ax},={bx},={cx},={dx},{ax},{cx}"(i32 0, i32 0)`, + ` %max.leaf = extractvalue { i32, i32, i32, i32 } %max, 0`, + ` %has.leaf7 = icmp uge i32 %max.leaf, 7`, + ` br i1 %has.leaf7, label %leaf1, label %detected`, + `leaf1:`, + ` %l1 = call { i32, i32, i32, i32 } asm "cpuid", "={ax},={bx},={cx},={dx},{ax},{cx}"(i32 1, i32 0)`, + ` %l1.ecx = extractvalue { i32, i32, i32, i32 } %l1, 2`, + ` %l1.bits = and i32 %l1.ecx, ${LEAF1_ECX}`, + ` %l1.ok = icmp eq i32 %l1.bits, ${LEAF1_ECX}`, + ` br i1 %l1.ok, label %leaf7, label %detected`, + `leaf7:`, + ` %l7 = call { i32, i32, i32, i32 } asm "cpuid", "={ax},={bx},={cx},={dx},{ax},{cx}"(i32 7, i32 0)`, + ` %l7.ebx = extractvalue { i32, i32, i32, i32 } %l7, 1`, + ` %l7.bits = and i32 %l7.ebx, ${LEAF7_EBX}`, + ` %l7.ok = icmp eq i32 %l7.bits, ${LEAF7_EBX}`, + ` %ext = call { i32, i32, i32, i32 } asm "cpuid", "={ax},={bx},={cx},={dx},{ax},{cx}"(i32 -2147483648, i32 0)`, + ` %ext.max = extractvalue { i32, i32, i32, i32 } %ext, 0`, + ` %ext.has = icmp uge i32 %ext.max, -2147483647`, + ` %l7.ext = and i1 %l7.ok, %ext.has`, + ` br i1 %l7.ext, label %ext1, label %detected`, + `ext1:`, + ` %e1 = call { i32, i32, i32, i32 } asm "cpuid", "={ax},={bx},={cx},={dx},{ax},{cx}"(i32 -2147483647, i32 0)`, + ` %e1.ecx = extractvalue { i32, i32, i32, i32 } %e1, 2`, + ` %e1.bits = and i32 %e1.ecx, ${EXT1_ECX}`, + ` %e1.ok = icmp eq i32 %e1.bits, ${EXT1_ECX}`, + ` br i1 %e1.ok, label %xcr0, label %detected`, + // Leaf 1 reported OSXSAVE, so XGETBV is available. + `xcr0:`, + ` %xcr = call { i32, i32 } asm "xgetbv", "={ax},={dx},{cx}"(i32 0)`, + ` %xcr.lo = extractvalue { i32, i32 } %xcr, 0`, + ` %xcr.bits = and i32 %xcr.lo, 6`, + ` %xcr.ok = icmp eq i32 %xcr.bits, 6`, + ` %v3 = select i1 %xcr.ok, i32 3, i32 1`, + ` br label %detected`, + `detected:`, + ` %level = phi i32 [ 1, %entry ], [ 1, %leaf1 ], [ 1, %leaf7 ], [ 1, %ext1 ], [ %v3, %xcr0 ]`, + ` %env = call ptr @getenv(ptr @sc_cpu_level_env)`, + ` %has.env = icmp ne ptr %env, null`, + ` br i1 %has.env, label %cap, label %store`, + `cap:`, + ]; + // The matching name's level, or 0 when none matches. + let cap = "0"; + LEVEL_NAMES.forEach(([, level], i) => { + lines.push( + ` %name.${i} = call i32 @strcmp(ptr %env, ptr @sc_cpu_level_name_${i})`, + ` %is.${i} = icmp eq i32 %name.${i}, 0`, + ` %cap.${i} = select i1 %is.${i}, i32 ${level}, i32 ${cap}`, + ); + cap = `%cap.${i}`; + }); + lines.push( + ` %capped.below = icmp ult i32 ${cap}, %level`, + ` %capped = select i1 %capped.below, i32 ${cap}, i32 %level`, + ` br label %store`, + `store:`, + ` %final = phi i32 [ %level, %detected ], [ %capped, %cap ]`, + ` store i32 %final, ptr ${LEVEL}`, + ` ret void`, + `}`, + ``, + ); + return lines; +} + +/** The call main makes before anything else. */ +export const CPU_LEVEL_INIT_CALL = ` call void @sc_cpu_level_init()`; + +/** A function definition's header, split around its name and parameters. */ +interface Header { + /** `define` and the linkage, up to the return type. */ + linkage: string; + /** The return type with its attributes. */ + ret: string; + name: string; + /** The parameter list between the parentheses. */ + params: string; + /** Function attributes before the group (`noinline`, `inlinehint`). */ + attrs: string; + group: string; + /** What follows the group: a personality, then `{` and a comment. */ + rest: string; +} + +const HEADER_START = /^define ((?:internal |private )?)(.*?)@([\w.$-]+)\(/; +const HEADER_END = /^((?: [a-z_]+)*) #(\d+)( .*)$/; +const RETURN_ATTRIBUTES = + /\b(?:zeroext|signext|noalias|nonnull|noundef|inreg|dereferenceable(?:_or_null)?\(\d+\)|align \d+)\s+/g; + +function parseHeader(line: string): Header | null { + const start = HEADER_START.exec(line); + if (start === null) return null; + const open = start[0].length - 1; + let depth = 0; + let close = open; + for (; close < line.length; close++) { + const c = line.charAt(close); + if (c === "(") depth++; + else if (c === ")" && --depth === 0) break; + } + const end = HEADER_END.exec(line.slice(close + 1)); + if (end === null) return null; + return { + linkage: `define ${start[1]!}`, + ret: start[2]!.trim(), + name: start[3]!, + params: line.slice(open + 1, close), + attrs: end[1]!, + group: end[2]!, + rest: end[3]!, + }; +} + +/** The parameters split at their top-level commas. */ +function splitParams(params: string): string[] { + const out: string[] = []; + let depth = 0; + let from = 0; + for (let i = 0; i < params.length; i++) { + const c = params.charAt(i); + if (c === "(" || c === "{" || c === "[" || c === "<") depth++; + else if (c === ")" || c === "}" || c === "]" || c === ">") depth--; + else if (c === "," && depth === 0) { + out.push(params.slice(from, i).trim()); + from = i + 1; + } + } + if (params.trim() !== "") out.push(params.slice(from).trim()); + return out; +} + +/** A definition the pass copies: one in an attribute group it has a copy + * group for (coroutine bodies, #1, keep their one body), with named + * parameters, no variable arguments and no debug locations. */ +function cloneable(header: Header, text: string): boolean { + if (!COPY_GROUPS.has(header.group)) return false; + if (text.includes("!dbg")) return false; + return splitParams(header.params).every((param) => /%[\w.$-]+$/.test(param)); +} + +/** The dispatching definition under the function's own name: it tail-calls + * the x86-64-v3 copy at level 3 or higher and the baseline copy otherwise. */ +function dispatcher(header: Header): string { + const args = splitParams(header.params).join(", "); + const type = header.ret.replace(RETURN_ATTRIBUTES, "").trim(); + const call = (tier: string, result: string): string[] => + type === "void" + ? [` musttail call void @${header.name}.${tier}(${args})`, ` ret void`] + : [ + ` ${result} = musttail call ${header.ret} @${header.name}.${tier}(${args})`, + ` ret ${type} ${result}`, + ]; + return [ + `${header.linkage}${header.ret} @${header.name}(${header.params}) #0 {`, + `entry:`, + ` %sc.cpu.level = load i32, ptr ${LEVEL}`, + ` %sc.cpu.v3 = icmp uge i32 %sc.cpu.level, 3`, + ` br i1 %sc.cpu.v3, label %sc.cpu.v3.call, label %sc.cpu.base.call`, + `sc.cpu.v3.call:`, + ...call("v3", "%sc.cpu.v3.result"), + `sc.cpu.base.call:`, + ...call("base", "%sc.cpu.base.result"), + `}`, + ].join("\n"); +} + +/** A definition split at its header line and its closing brace. */ +interface Definition { + header: Header; + /** The lines after the header, through the closing `}`. */ + body: string; + /** Text after the closing brace, kept as it is. */ + after: string; +} + +/** Rewrites the program's function definitions in `parts` for runtime + * dispatch. A definition is a part, or a `define` chunk of a part, that + * follows a partition group line (emit-module.ts partitionGroupLines); + * runtime helpers and the module's entry points stay as they are. Each + * cloneable function `@f` becomes: + * - `@f.base`, its body as before; + * - `@f.v3`, the same body in an x86-64-v3 attribute group; + * - `@f`, a dispatcher with the same signature, which every reference to + * the function as a value and every call from outside the program's + * functions still names; a function with no such reference gets none. + * Direct calls between cloned functions go to the caller's own tier, so the + * level is tested once on entry into the program's code. */ +export function cloneForCpuDispatch(parts: readonly string[]): string[] { + const program: { index: number; chunks: string[] }[] = []; + for (let i = 1; i < parts.length; i++) + if (parts[i - 1]!.startsWith("; scriptc-group") && parts[i]!.startsWith("define ")) + program.push({ index: i, chunks: parts[i]!.split(/\n(?=define )/) }); + const defs = new Map(); + for (const { chunks } of program) + for (const chunk of chunks) { + const newline = chunk.indexOf("\n"); + const header = newline < 0 ? null : parseHeader(chunk.slice(0, newline)); + if (header === null) continue; + const close = chunk.indexOf("\n}", newline); + if (close < 0) continue; + const body = chunk.slice(newline + 1, close + 2); + if (!cloneable(header, body)) continue; + defs.set(header.name, { header, body, after: chunk.slice(close + 2) }); + } + if (defs.size === 0) return [...parts]; + const callee = /@([\w.$-]+)\(/g; + const retarget = (body: string, tier: string): string => + body.replace(callee, (all, name: string) => (defs.has(name) ? `@${name}.${tier}(` : all)); + const named = namedFunctions(parts, program, defs, (body) => retarget(body, "base")); + const out = [...parts]; + for (const { index, chunks } of program) + out[index] = chunks + .map((chunk) => { + const header = HEADER_START.exec(chunk); + const def = header === null ? undefined : defs.get(header[3]!); + if (def === undefined) return chunk; + const h = def.header; + const copy = (tier: string, group: string): string => + `define internal ${h.ret} @${h.name}.${tier}(${h.params})${h.attrs} #${group}${h.rest}\n${retarget(def.body, tier)}`; + return ( + [ + copy("base", h.group), + copy("v3", COPY_GROUPS.get(h.group)!), + ...(named.has(h.name) ? [dispatcher(h)] : []), + ].join("\n") + def.after + ); + }) + .join("\n"); + return out; +} + +/** The cloned functions whose own name the module still uses once the + * copies call each other: as a value, or in a call from code that is not + * copied (main, runtime helpers, coroutine bodies). Only these get a + * dispatcher. A function without one keeps the callers it had, so LLVM + * inlines and removes its copies as it would the original. */ +function namedFunctions( + parts: readonly string[], + program: readonly { index: number; chunks: string[] }[], + defs: ReadonlyMap, + retargeted: (body: string) => string, +): Set { + const named = new Set(); + const scan = (text: string): void => { + for (const m of text.matchAll(/@([\w.$-]+)/g)) if (defs.has(m[1]!)) named.add(m[1]!); + }; + const inProgram = new Set(program.map(({ index }) => index)); + parts.forEach((part, i) => { + if (!inProgram.has(i)) scan(part); + }); + for (const { chunks } of program) + for (const chunk of chunks) { + const header = HEADER_START.exec(chunk); + const def = header === null ? undefined : defs.get(header[3]!); + if (def === undefined) scan(chunk); + else scan(retargeted(def.body) + def.after); + } + return named; +} diff --git a/packages/compiler/src/backend/llvm/emit-function.ts b/packages/compiler/src/backend/llvm/emit-function.ts index 8c9ace9f3..a901445d1 100644 --- a/packages/compiler/src/backend/llvm/emit-function.ts +++ b/packages/compiler/src/backend/llvm/emit-function.ts @@ -83,6 +83,7 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { emitter.unwindCleanups.clear(); emitter.jumpTargets = []; emitter.currentLocals = new Map(fn.locals.map((l) => [l.id, l])); + emitter.currentParameters = new Set(fn.params.map((p) => p.localId)); emitter.lazyCaptureBoxes.clear(); emitter.lazyStackBoxes.clear(); // An unchanged captured parameter keeps its value in a plain slot; the diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index b1118f8bd..d25427e4f 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -65,7 +65,9 @@ import { STRING_READ_HELPERS, stringScanDecl, stringScanInline } from "./string- import { emitLibraryIdentityLines } from "../library-identity-markers.js"; import { InternalCompilerError } from "../../errors.js"; import { targetCpuAttribute } from "../target-cpu.js"; +import { CPU_LEVEL_INIT_CALL, cpuDispatchAttributes, cpuLevelDefs } from "./cpu-dispatch.js"; import type { IrFunction } from "../../ir/ir.js"; +import { type FunctionLayout, layoutTierOf } from "./function-layout.js"; /** The function bodies emitted in `chunks` contiguous ranges, each by its * own emitter (LlEmitter.chunkEmitter), then merged in function order: the @@ -198,7 +200,8 @@ function readRange(dir: string, range: number): { result: ChunkResult; texts: st * #1 for coroutine bodies and #2 for out-of-line record clones. Each carries * the build's `--target-cpu` level, and `sanitize_address` in sanitized * builds. A group needs one attribute: without either, #0 holds a string - * attribute that LLVM ignores. */ + * attribute that LLVM ignores. Dispatching builds add the x86-64-v3 copies' + * #3 and #4 (cpu-dispatch.ts). */ function functionAttributes(emitter: LlEmitter, hasNoInlineRecordClone: boolean): string[] { const cpu = targetCpuAttribute(emitter.targetCpu); const asan = emitter.sanitizeAttributes ? " sanitize_address" : ""; @@ -207,6 +210,7 @@ function functionAttributes(emitter: LlEmitter, hasNoInlineRecordClone: boolean) `attributes #0 = {${base} }`, ...(emitter.usesCoroutines ? [`attributes #1 = {${asan} presplitcoroutine${cpu} }`] : []), ...(hasNoInlineRecordClone ? [`attributes #2 = { noinline${asan}${cpu} }`] : []), + ...(emitter.cpuDispatch ? cpuDispatchAttributes(asan) : []), ]; } @@ -230,6 +234,61 @@ function partitionGroupLines(functions: readonly IrFunction[]): string[] { /** Ends the group of the definitions that follow. */ const PARTITION_GROUP_END = "; scriptc-group"; +/** Writes the generated helpers, then the function definitions with their + * group lines. With a layout, the definitions it lists, helpers included, + * move after the unlisted helpers and before the other functions, by tier, + * each in its tier's group (`layout-`). The helper compiles each + * partition's definitions in text order, so listed ones come first in their + * partitions, and it places a group's definitions in the same partitions. + * Everything else keeps its order and its source-file group. */ +function writeDefinitions( + out: string[], + layout: FunctionLayout | undefined, + helperTexts: readonly (readonly string[])[], + fnDefs: readonly string[], + groups: readonly string[], +): void { + const tierLine = (tier: number) => `${PARTITION_GROUP_END} layout-${tier}`; + const listed = new Map(); + const listIn = (tier: number, lines: readonly string[]) => { + const list = listed.get(tier) ?? []; + if (!listed.has(tier)) listed.set(tier, list); + for (const line of lines) list.push(line); + }; + if (layout === undefined || layout.size === 0) { + for (const texts of helperTexts) for (const text of texts) out.push(text); + } else { + // A helper's text can span several entries (one per line), so the + // definitions are found in the joined text. A listed definition moves + // through its closing brace; what follows it stays in place. + const joined: string[] = []; + for (const texts of helperTexts) for (const text of texts) joined.push(text); + for (const part of joined.join("\n").split(/\n(?=define )/)) { + const tier = part.startsWith("define ") ? layoutTierOf(layout, part) : undefined; + const close = tier === undefined ? -1 : part.search(/\n}(\n|$)/); + if (tier === undefined || close < 0) { + out.push(part); + continue; + } + listIn(tier, [tierLine(tier), part.slice(0, close + 2)]); + const rest = part.slice(close + 3); + if (rest !== "") out.push(rest); + } + } + const unlisted: number[] = []; + fnDefs.forEach((text, i) => { + const tier = layout === undefined || layout.size === 0 ? undefined : layoutTierOf(layout, text); + if (tier === undefined) unlisted.push(i); + else listIn(tier, ["", tierLine(tier), text]); + }); + for (const tier of [...listed.keys()].sort((a, b) => a - b)) + for (const line of listed.get(tier)!) out.push(line); + for (const [k, i] of unlisted.entries()) { + if (k > 0 || listed.size > 0) out.push(""); + out.push(groups[i] ?? PARTITION_GROUP_END, fnDefs[i]!); + } +} + export function emitModuleParts(emitter: LlEmitter): string[] { // Function bodies first (the literal/unit/fn-value tables fill as they // emit), then the file assembles around them — the runtime ABI’s order. @@ -585,6 +644,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { ...(emitter.publishWalkersCache?.defs() ?? []), ...emitter.publishedClassNamesDefs(), ...(emitter.regions?.moduleDefs() ?? []), + ...(emitter.cpuDispatch ? cpuLevelDefs((line) => emitter.declare(line)) : []), ]; if (emitter.constantNumericTables.size > 0) emitter.declare(`declare double @scr_arr_get_number(ptr, double)`); @@ -944,21 +1004,24 @@ export function emitModuleParts(emitter: LlEmitter): string[] { ); } if (globals.length > 0) out.push(``); - for (const line of helpers) out.push(line); - for (const line of ffiCallbacks.defs) out.push(line); - for (const line of shapes.defs) out.push(line); - for (const line of classShapes.defs) out.push(line); - for (const line of classObjDefs) out.push(line); - for (const line of emitter.walkers.defs) out.push(line); - for (const line of emitter.dyn.defs) out.push(line); - for (const line of wrappers) out.push(line); - for (const line of asyncDefs) out.push(line); - for (const line of emitter.resolveThunkDefs) out.push(line); - const groups = partitionGroupLines(emitter.mod.functions); - for (let i = 0; i < fnDefs.length; i++) { - if (i > 0) out.push(""); - out.push(groups[i] ?? PARTITION_GROUP_END, fnDefs[i]!); - } + writeDefinitions( + out, + emitter.targetOptions.functionLayout, + [ + helpers, + ffiCallbacks.defs, + shapes.defs, + classShapes.defs, + classObjDefs, + emitter.walkers.defs, + emitter.dyn.defs, + wrappers, + asyncDefs, + emitter.resolveThunkDefs, + ], + fnDefs, + partitionGroupLines(emitter.mod.functions), + ); if (fnDefs.length === 0) out.push(""); out.push(PARTITION_GROUP_END, ""); // WebAssembly modules translated into this program, and the registry. @@ -1043,6 +1106,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { `define i32 @${emitter.mod.workers ? "sc_context_entry" : emitter.wasi ? "__main_argc_argv" : "main"}(i32 %argc, ptr %argv) ${FN_ATTRS} {`, `entry:`, ...(emitter.runtimeAbiMarker ? [` call void @${RUNTIME_ABI_MARKER}()`] : []), + ...(emitter.cpuDispatch && !emitter.mod.workers ? [CPU_LEVEL_INIT_CALL] : []), ...(emitter.mod.workers ? [` call void @scr_runtime_workers_v8()`] : []), ` call void @scr_init()`, // Compressed fields hold addresses in the program-lifetime region, which @@ -1201,6 +1265,8 @@ export function emitModuleParts(emitter: LlEmitter): string[] { `}`, `define i32 @main(i32 %argc, ptr %argv) ${FN_ATTRS} {`, `entry:`, + // Worker threads enter at sc_worker_entry and read the level main set. + ...(emitter.cpuDispatch ? [CPU_LEVEL_INIT_CALL] : []), ` %result = call i32 @sc_context_entry(i32 %argc, ptr %argv)`, ` call void @scr_context_cleanup()`, ` ret i32 %result`, diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index cc0af8ebb..c1f3cf18f 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -1,4 +1,6 @@ /* LlEmitter's statement emission: the statement switch, try/catch, switch, loop copies, speculative writes and string self-concatenation. */ +import { emitLoopBytesHeaders } from "./bytes-headers.js"; +import { noteBytesReceiver } from "./expr-bytes.js"; import { type LlEmitter, type LlScopeEntry, @@ -81,6 +83,7 @@ import { planProvenIntegerLoop, planSpeculativeLoop, type ProvenLoopPlan, + type SpeculativeLoopPlan, speculativeReads, speculativeWriteTarget, } from "./speculative-integers.js"; @@ -93,6 +96,7 @@ import { import { emitStringInputs } from "./string-lifetimes.js"; import { LOOPS } from "./loop-state.js"; import { loopAccessesTypedArrays } from "../loop-polls.js"; +import { emitConstantChoice, matchConstantChoice } from "./constant-choices.js"; /** A callback loop version copies the loop; keep the copies small. */ const MAX_CALLBACK_VERSION_COST = 400; @@ -576,6 +580,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { // IrBytesElem is static, so never rediscover it through the // generic runtime switch in a hot loop. const arr = emitter.emitStableReceiver(s.arr, [s.index, s.value]); + noteBytesReceiver(emitter, s.arr, arr.name); const idx = emitter.emitExpr(s.index); const v = emitter.emitExpr(s.value); if (s.arr.type.kind !== "bytes") @@ -837,6 +842,13 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { break; } case "if": { + // A chain storing one of several literals to one element stores a + // table entry instead (debug builds keep the arms to step through). + const choice = emitter.debug === null ? matchConstantChoice(s) : null; + if (choice !== null) { + emitConstantChoice(emitter, choice); + break; + } const cond = emitter.emitCondition(s.cond); const trueLabel = B.newLabel("if.t"); const joinLabel = B.newLabel("if.j"); @@ -1265,6 +1277,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { if (stripMined) { const counter: StripMinedCounter = countedLoop ? { + localId: countedLoop.localId, slot: integerSlot!, type: "i64", signed: true, @@ -1276,6 +1289,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { limit: () => countedLimit!, } : { + localId: integerLoop!.localId, slot: integerSlot!, type: emitter.sizeType, signed: false, @@ -2371,6 +2385,22 @@ export function emitLoopCopies( else ordinary(exit); return; } + const outer = emitter.bytesHeaders; + emitter.bytesHeaders = emitLoopBytesHeaders(emitter, loop, emitter.currentParameters) ?? outer; + try { + emitSpeculativeLoop(emitter, plan, exit, emitCopy); + } finally { + emitter.bytesHeaders = outer; + } +} + +/** The copies of a speculative loop (emitLoopCopies). */ +function emitSpeculativeLoop( + emitter: LlEmitter, + plan: SpeculativeLoopPlan, + exit: string, + emitCopy: (exit: string) => void, +): void { const B = emitter.B; const entries = new Map(); let ok = "true"; @@ -2705,10 +2735,11 @@ function boundedLoopRuns(emitter: LlEmitter, loop: IrStmt): number { return trips * (1 + nested); } -/** The counter of a strip-mined loop: its slot, the LLVM integer type and - * signedness of its values, its step, the counters stepped beside it, and - * the limit, emitted in the outer header before each block. */ +/** The counter of a strip-mined loop: its binding, its slot, the LLVM + * integer type and signedness of its values, its step, the counters stepped + * beside it, and the limit, emitted in the outer header before each block. */ interface StripMinedCounter { + localId: string; slot: string; type: string; signed: boolean; @@ -2800,7 +2831,7 @@ function emitStripMinedCountedLoop( const after = runSlot === null ? poll : B.newLabel("loop.end"); B.condBr(inBlock, lb, after); B.startBlock(lb); - emitter.jumpTargets.push({ + const target: LlEmitter["jumpTargets"][number] = { kind: "loop", brkLabel: exit, contLabel: lu, @@ -2808,8 +2839,19 @@ function emitStripMinedCountedLoop( frameDepth: emitter.frames.length, scopeDepth: emitter.scopes.length, finallyDepth: emitter.finallyStack.length, - }); - emitter.emitBlock(s.body); + }; + emitter.jumpTargets.push(target); + // The counter steps by a nonzero constant and the body does not assign + // it, so an element at the counter is a different element on every + // iteration: its plain load or store is not merged with another + // iteration's or moved out of the loop. + const outer = emitter.plainSharedLoop; + emitter.plainSharedLoop = { counter: counter.localId, target, depth: emitter.loopDepth }; + try { + emitter.emitBlock(s.body); + } finally { + emitter.plainSharedLoop = outer; + } emitter.jumpTargets.pop(); B.br(lu); B.startBlock(lu); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index f1f1fa7d6..459dd2d9e 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1,4 +1,5 @@ import type { TargetCpu } from "../target-cpu.js"; +import type { FunctionLayout } from "./function-layout.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; import { wasmBoundsChecks } from "./wasm-modules.js"; import { jsTruthy } from "./jsval.js"; @@ -63,6 +64,7 @@ import { captureFreeFunctions, observesStacks } from "../stack-observations.js"; import { findInitializedByteLoopBindings } from "./byte-windows.js"; import { matchBytesLoopVersion } from "./bytes-loop-versions.js"; import { globalSlotAlias } from "./bytes-alias.js"; +import type { BytesHeaderScope } from "./bytes-headers.js"; import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import { materializeSplitPiece, @@ -147,6 +149,7 @@ import { bridgePlainUnwindCalls, unwindAbiFunctions, } from "./unwind-abi.js"; +import { cloneForCpuDispatch } from "./cpu-dispatch.js"; import { mangleAsyncInlineAwaitCall, mangleAsyncSpawn, @@ -183,14 +186,14 @@ import { unionBoxBits, } from "./inline-unions.js"; import { OptionalArrayReads, type LocalArrayRead } from "./local-array-reads.js"; -import { CheckedNarrows } from "./checked-narrows.js"; +import { borrowsCheckedNarrow, CheckedNarrows } from "./checked-narrows.js"; import { emitDenseArraySet, emitDenseSlotUnlink, SCR_ELEM_I32, type ArrayAccess, } from "./dense-array-access.js"; -import { matchMapRead, type MapReadLifetimes } from "./map-read-lifetimes.js"; +import { matchMapRead, pointerMapGet, type MapReadLifetimes } from "./map-read-lifetimes.js"; import { type MapEntryFusion } from "./map-entry-fusion.js"; import { emitBorrowedFieldSequence, guardedValue } from "./borrowed-receivers.js"; import { emitBorrowedInputs } from "./borrowed-inputs.js"; @@ -213,7 +216,13 @@ import { } from "./string-slices.js"; import { LazyCaptures } from "./lazy-captures.js"; import { StackCaptures } from "./stack-captures.js"; -import { emitFieldAbsentTest, f64Lit, SCR_ARR_FIELD, SCR_ARR_FROZEN } from "./common.js"; +import { + CLOSURE_RECEIVER_FREE, + emitFieldAbsentTest, + f64Lit, + SCR_ARR_FIELD, + SCR_ARR_FROZEN, +} from "./common.js"; import { ffiExtendsNarrowIntegers } from "../targets.js"; import { emitLiteralExpr, @@ -285,7 +294,12 @@ import { emitBytesSet, emitBytesIntrinsic, } from "./expr-bytes.js"; -import { emitRegexIntrinsic, emitRecordKeyGet, keyedRecordReadInto } from "./expr-records.js"; +import { + emitRecordKeyGet, + emitRegexIntrinsic, + keyedRecordReadInto, + pointerRecordKeyGet, +} from "./expr-records.js"; import { dynPromiseAdapter, streamTypedRefCommitAdapter, @@ -487,6 +501,8 @@ export interface LlvmTargetOptions { /** The host's process primitives, where it can fork (the native * compiler's). */ forkPrimitives?: ForkPrimitives | undefined; + /** Functions to write first, by tier (function-layout.ts). */ + functionLayout?: FunctionLayout; /** Exact frontend sources for dev-build locations and variable metadata. */ debugSources?: ReadonlyMap; /** Pointer width of the target C ABI. Native targets are 64-bit today. */ @@ -533,6 +549,11 @@ export interface LlvmTargetOptions { * every emitted function carries it as its `target-cpu`. Unset, code * runs on any x86-64 CPU. */ targetCpu?: TargetCpu; + /** Give an x86-64 executable without a `targetCpu` an x86-64-v3 copy of + * each program function, which it runs on CPUs at that level + * (cpu-dispatch.ts). Ignored for other targets, libraries and builds + * with debug information. */ + cpuDispatch?: boolean; } /** LlvmTargetOptions.lowRegion for an executable: Linux targets without @@ -1277,6 +1298,8 @@ export class LlEmitter { readonly targetCpu: TargetCpu | undefined; /** Emitted functions carry `sanitize_address` (LlvmTargetOptions). */ readonly sanitizeAttributes: boolean; + /** LlvmTargetOptions.cpuDispatch, where it applies. */ + readonly cpuDispatch: boolean; readonly wasmBoundsChecks: boolean; /** Interned string literals: UTF-8 text → { symbol, byte length } — * first-use order, the runtime ABI’s determinism discipline. */ @@ -1804,11 +1827,26 @@ export class LlEmitter { /** Worker programs: typed-array accesses whose views an enclosing version * guard proved thread-owned. */ bytesOwned: ReadonlySet = new Set(); + /** Worker programs: the strip-mined counted loop whose body is being + * emitted (emitStripMinedCountedLoop), with its counter, its jump target + * and its loopDepth. An element access of a view that may be shared, at + * the counter and directly in that loop's body, is a plain load or store + * (plainSharedElement in expr-bytes.ts). */ + plainSharedLoop: { + counter: string; + target: LlEmitter["jumpTargets"][number]; + depth: number; + } | null = null; /** The alias-scope attachment of each typed-array element read and write * in the current loop version's fast copy (emitVersionedLoop), and the * one the element access being emitted carries (bytesElementAlias). */ bytesAliasScopes: ReadonlyMap = new Map(); bytesAccessScope = ""; + /** Header fields a speculative loop's entry read for its views + * (bytes-headers.ts), while its copies are emitted. */ + bytesHeaders: BytesHeaderScope | null = null; + /** The parameters of the function being emitted. */ + currentParameters: ReadonlySet = new Set(); /** Loop versions with alias scopes in the current function; it names * their scopes. */ aliasScopedVersions = 0; @@ -1980,6 +2018,13 @@ export class LlEmitter { this.detachedExit = options.detachedExit === true; this.targetCpu = options.targetCpu; this.sanitizeAttributes = options.sanitizeAttributes !== false; + this.cpuDispatch = + options.cpuDispatch === true && + options.targetCpu === undefined && + !this.wasi && + mod.lib === undefined && + this.debug === null && + /^(x86_64|amd64|x64)\b/.test(options.targetTriple || process.arch); this.wasmBoundsChecks = options.wasmBoundsChecks === true || wasmBoundsChecks({ @@ -2486,7 +2531,8 @@ export class LlEmitter { } emitParts(): string[] { const parts = emitParts(this); - return this.unwindAbi.size > 0 ? bridgePlainUnwindCalls(parts) : parts; + const bridged = this.unwindAbi.size > 0 ? bridgePlainUnwindCalls(parts) : parts; + return this.cpuDispatch ? cloneForCpuDispatch(bridged) : bridged; } /** The dispose-observer gates the emitted frees and record writers test @@ -2560,7 +2606,7 @@ export class LlEmitter { ret === "void" ? ` ${call}` : ` %r = ${call}`, ret === "void" ? ` ret void` : ` ret ${ret} %r`, `}`, - `@${mangleFnClosure(name)} = internal ${this.mod.workers === true || this.mod.lib?.threadInstances === true ? "thread_local " : ""}global %ScrClosure { ${this.sizeType} -1, ptr @${mangleWrapper(name)}, ${this.sizeType} 0, ptr null, i32 ${(fn.generator ? 1 : 0) + (fn.async ? 2 : 0) + (fn.ownsPrototype ? 4 : 0)}, i32 0, ptr null, ptr ${lending ? `@${mangleBorrowingWrapper(name)}` : "null"} }`, + `@${mangleFnClosure(name)} = internal ${this.mod.workers === true || this.mod.lib?.threadInstances === true ? "thread_local " : ""}global %ScrClosure { ${this.sizeType} -1, ptr @${mangleWrapper(name)}, ${this.sizeType} 0, ptr null, i32 ${this.closureFunctionKind(fn)}, i32 0, ptr null, ptr ${lending ? `@${mangleBorrowingWrapper(name)}` : "null"} }`, ``, ); if (lending) @@ -3112,12 +3158,13 @@ export class LlEmitter { // ── plumbing (frame and scope ownership) ────── - /** A symbol derived from the constant's text rather than its first-use - * index, so an edit that interns one more string leaves every other - * constant's symbol, and the code referencing it, unchanged. This lets - * native builds reuse the compiled partitions an edit does not touch. A - * collision retries with a counter, still in first-use order. */ - private contentSymbol(prefix: string, text: string): string { + /** A symbol derived from a constant's text, or a helper's key, rather + * than its first-use index, so an edit that interns one more string or + * helper leaves every other symbol, and the code referencing it, + * unchanged. This lets native builds reuse the compiled partitions an + * edit does not touch. A collision retries with a counter, still in + * first-use order. */ + contentSymbol(prefix: string, text: string): string { for (let attempt = 0; ; attempt++) { const digest = createHash("sha256") .update(attempt === 0 ? text : `${attempt}\0${text}`) @@ -3243,7 +3290,7 @@ export class LlEmitter { const key = `${unionId}:${tag}`; let sym = this.unitInstances.get(key); if (!sym) { - sym = `sc_unit_${this.unitInstances.size}`; + sym = this.contentSymbol("sc_unit_", key); this.unitInstances.set(key, sym); } this.journal?.entry("unit", key, sym); @@ -3700,6 +3747,9 @@ export class LlEmitter { passThrough?: { name: string; type: IrType }, poll = false, guard?: string, + /** Runs at the start of the cold unwind block, before the unwind (a + * package-declared check's origin note). */ + onUnwind?: () => void, ): void { const through = passThrough !== undefined && @@ -3724,6 +3774,7 @@ export class LlEmitter { const lk = B.newLabel("exc.k"); B.condBr(p, lu, lk); B.startBlock(lu); + onUnwind?.(); this.emitUnwind(through); B.startBlock(lk); if (skip !== undefined) { @@ -3800,6 +3851,7 @@ export class LlEmitter { B.condBr(cold, lu, lk); } B.startBlock(lu); + onUnwind?.(); this.emitUnwind(through); B.startBlock(lk); if (skip !== undefined) { @@ -5449,8 +5501,15 @@ export class LlEmitter { case "retainWrapper": addEntry(this.nullableUnions.retainWrappers, key, value as string); return; - case "unit": - return addNumbered(this.unitInstances, key, value as string, "sc_unit_", names); + case "unit": { + let own = this.unitInstances.get(key); + if (own === undefined) { + own = this.contentSymbol("sc_unit_", key); + this.unitInstances.set(key, own); + } + if (own !== value) names.set(value as string, own); + return; + } case "absent": return addNumbered(this.absentInstances, key, value as string, "sc_absent_", names); case "retype": @@ -5458,9 +5517,9 @@ export class LlEmitter { case "slots": return addNumbered(this.threadSlotsBlocks, key, value as string, "@sc_slots_", names); case "regex": - return addNumberedSymbol(this.regexInstances, key, value as never, "sc_re_", names); + return this.addContentEntry(this.regexInstances, key, value as never, "sc_re_", names); case "regexReplacer": - return addNumberedSymbol(this.regexReplacers, key, value as never, "sc_rr_", names); + return this.addContentEntry(this.regexReplacers, key, value as never, "sc_rr_", names); case "templateStrings": return addNumberedSymbol( this.templateStringsInstances, @@ -5737,7 +5796,10 @@ export class LlEmitter { unwrittenRead(e: IrExpr, writes: Writes): boolean { const rw = this.referenceWrites; if (rw === null) return false; - const keys = rw.readKeys(e, (call) => this.borrowedReturnKeys(call.callee)); + // A checked extraction reads nothing besides the union it narrows. + const keys = rw.readKeys(e, (call) => + borrowsCheckedNarrow(this, call) ? [] : this.borrowedReturnKeys(call.callee), + ); return keys !== null && !rw.conflicts(keys, writes); } @@ -6014,7 +6076,21 @@ export class LlEmitter { } case "fieldGet": case "recordGet": + case "recordKeyGet": return this.borrowsWithoutOwning(e) && this.unownedWalk(e.obj); + case "mapIntrinsic": { + // A Map binding's own pointer (maps are not plainReference types). + if (!this.borrowsWithoutOwning(e) || e.receiver.kind !== "varRef") return false; + const binding = this.binding(e.receiver.localId); + return ( + binding.kind === "global" || + (binding.kind === "local" && + binding.local !== undefined && + !binding.local.boxed && + !binding.local.tdz && + !this.localUnionStorage.has(e.receiver.localId)) + ); + } case "unionNarrow": case "downcast": case "upcast": @@ -7272,6 +7348,11 @@ export class LlEmitter { return this.nullableFieldGet(e) === null; case "recordGet": return true; + // A keyed read whose hits are all the result's own pointer. + case "recordKeyGet": + return pointerRecordKeyGet(this, e); + case "mapIntrinsic": + return pointerMapGet(this, e); // emitReadReceiver lends a required element of a borrowed array when // computing the index removes no edge. case "arrayGet": @@ -7417,8 +7498,20 @@ export class LlEmitter { return wrapEmitterListener(this, target, adapterFn); } - unwrapNullableClosure(u: string, funcTag: number): string { - return unwrapNullableClosure(this, u, funcTag); + /** ScrClosure.function_kind for a closure over `fn`: generator 1, async + * 2, own prototype 4, and SCR_FN_RECEIVER_FREE (16) when its body never + * observes the ambient receiver, so indirect callers skip the bracket. */ + closureFunctionKind(fn: IrFunction): number { + return ( + (fn.generator ? 1 : 0) + + (fn.async ? 2 : 0) + + (fn.ownsPrototype ? 4 : 0) + + (this.receiverReaders.ignores(fn) ? CLOSURE_RECEIVER_FREE : 0) + ); + } + + unwrapNullableClosure(u: string, unionId: string, funcTag: number): string { + return unwrapNullableClosure(this, u, unionId, funcTag); } closeBindThunkFor(cbUnion: IrType, retServer: boolean): string { diff --git a/packages/compiler/src/backend/llvm/expr-async.ts b/packages/compiler/src/backend/llvm/expr-async.ts index ee9ff2983..037179d8e 100644 --- a/packages/compiler/src/backend/llvm/expr-async.ts +++ b/packages/compiler/src/backend/llvm/expr-async.ts @@ -372,6 +372,21 @@ export function emitSerializationExpr( // the dummy harmlessly. // `JSON.parse(text) as T` fuses into one schema-directed parse that // falls back to exactly this route whenever the fast path declines. + // A package-declared target: the cold path names the declaration in + // the pending TypeError (scr_type_origin.c). + // Library archives and reactors link a fixed runtime set without the + // note unit: their checks keep the plain message. + const origin = + e.origin === undefined || host.mod.lib !== undefined + ? undefined + : host.mod.declOrigins?.[e.origin]; + const note = + origin === undefined + ? undefined + : (): void => { + host.declare("declare void @scr_type_origin_note(ptr)"); + B.line(`call void @scr_type_origin_note(ptr ${host.cstr(origin)})`); + }; const parsed = e.value; if (!e.preserveRefs && parsed.kind === "libCall" && parsed.fn === "json.parse") { const fused = host.dyn.jsonParseHelper(e.type); @@ -380,7 +395,7 @@ export function emitSerializationExpr( const r = B.tmp(); B.line(`${r} = call ptr @${fused}(ptr ${text.name})`); const fusedOut = host.own({ name: r, type: e.type }); - host.emitPendingCheck(); + host.emitPendingCheck(undefined, false, undefined, note); return fusedOut; } } @@ -394,14 +409,14 @@ export function emitSerializationExpr( } const dynV = emitBorrowedInput(host, e.value); const helper = host.dyn.dynCheckHelper(e.type, e.preserveRefs); - if (isDynTypedRefType(e.type)) return emitClassCheck(host, e.type, dynV.name, helper); + if (isDynTypedRefType(e.type)) return emitClassCheck(host, e.type, dynV.name, helper, note); const ty = host.llType(e.type); const t = B.tmp(); B.line( `${t} = call ${ty === "i1" ? "zeroext i1" : ty} @${helper}(ptr ${dynV.name}, ptr null)`, ); const out = host.own({ name: t, type: e.type }); - host.emitPendingCheck(); + host.emitPendingCheck(undefined, false, undefined, note); return out; } default: { @@ -420,6 +435,7 @@ function emitClassCheck( type: IrType & { kind: "object" }, value: string, helper: string, + note?: () => void, ): LlValue { const B = host.B; const slot = B.slot(); @@ -460,7 +476,7 @@ function emitClassCheck( B.line(`${result} = call ptr @${helper}(ptr ${value}, ptr null)`); B.line(`store ptr ${result}, ptr ${slot}`); // A throwing check returns null, so the unwind has nothing to release. - host.emitPendingCheck(); + host.emitPendingCheck(undefined, false, undefined, note); B.br(done); B.startBlock(done); const out = B.tmp(); diff --git a/packages/compiler/src/backend/llvm/expr-bytes.ts b/packages/compiler/src/backend/llvm/expr-bytes.ts index 6f0b0380d..72ab4ec13 100644 --- a/packages/compiler/src/backend/llvm/expr-bytes.ts +++ b/packages/compiler/src/backend/llvm/expr-bytes.ts @@ -1,6 +1,7 @@ /* Focused LLVM expression emission extracted from emitter.ts. */ import { InternalCompilerError } from "../../errors.js"; import { emitBorrowedInput } from "./borrowed-inputs.js"; +import type { BytesHeader } from "./bytes-headers.js"; import { BYTES_ELEMENT_SIZE, F64, type IrBytesElem, type IrExpr } from "../../ir/ir.js"; import type { LlvmEmitterContext, LlValue } from "./expr-context.js"; import { F64_INF, exactIndexLines, f64Lit, sizeToDouble } from "./common.js"; @@ -160,6 +161,22 @@ export function emitBytesIndex( return idx; } +/** Records that `value` is the view `receiver` reads, so the header reads of + * an access through it use the values a loop entry read (bytes-headers.ts). */ +export function noteBytesReceiver(host: LlvmEmitterContext, receiver: IrExpr, value: string): void { + const scope = host.bytesHeaders; + if (scope === null || receiver.kind !== "varRef" || scope.fn !== host.currentFunctionName) return; + const header = scope.locals.get(receiver.localId); + if (header !== undefined) scope.values.set(value, header); +} + +/** The header fields a loop entry read for the view `receiver`, if any. */ +function cachedHeader(host: LlvmEmitterContext, receiver: string): BytesHeader | undefined { + const scope = host.bytesHeaders; + if (scope === null || scope.fn !== host.currentFunctionName) return undefined; + return scope.values.get(receiver); +} + /** The element count of a typed array, in the target's size type. `value`: * the program reads the count itself (bytesHeaderValueAlias), rather than * only bounding an access. */ @@ -168,6 +185,8 @@ export function emitBytesHeaderLength( receiver: string, value = false, ): string { + const cached = cachedHeader(host, receiver); + if (cached !== undefined) return cached.length; const p = host.B.tmp(); const len = host.B.tmp(); const alias = value ? bytesHeaderValueAlias(host) : bytesHeaderAlias(host); @@ -177,6 +196,8 @@ export function emitBytesHeaderLength( } export function emitBytesData(host: LlvmEmitterContext, receiver: string): string { + const cached = cachedHeader(host, receiver); + if (cached !== undefined) return cached.data; const p = host.B.tmp(); const data = host.B.tmp(); host.B.line(`${p} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 3`); @@ -203,6 +224,8 @@ export function emitBytesLength( /** A view's shared backing (null for a buffer this thread owns). It is * set at construction, before script code can see the view. */ export function emitBytesSharedStorage(host: LlvmEmitterContext, receiver: string): string { + const cached = cachedHeader(host, receiver)?.shared; + if (cached !== undefined && cached !== null) return cached; const B = host.B; const slot = B.tmp(), storage = B.tmp(); @@ -303,7 +326,7 @@ function emitScopedBytesGet( const signed = elem === "i8" || elem === "i16" || elem === "i32"; B.line(`${number} = ${signed ? "sitofp" : "uitofp"} i32 ${u32} to double`); B.line(`${out} = select i1 ${valid}, double ${number}, double ${f64Lit(NaN)}`); - return { name: out, type: F64, uint32: u32 }; + return { name: out, type: F64, uint32: u32, ordered: valid }; } function emitBytesLoad( @@ -317,7 +340,7 @@ function emitBytesLoad( owned: boolean, ): LlValue { // `owned`: an enclosing version guard proved the view thread-owned. - if (host.mod.workers && !owned) { + if (host.mod.workers && !owned && !plainSharedElement(host, elem, expr)) { const idx = emitBytesIndex(host, receiver, index, expr, invalid, inBounds); return emitSharedOrLocalLoad(host, elem, receiver, idx); } @@ -328,6 +351,31 @@ function emitBytesLoad( return emitBytesLoadAt(host, elem, data, idx); } +/** Worker programs: whether an element access at `index` of a view that + * may be shared is a plain load or store, like a thread-owned view's. It is + * when the index is the counter of the strip-mined counted loop whose body + * holds the access, outside any loop nested in that body (its condition and + * update included: a nested loop that waits on the element reads it on + * every iteration, so the read stays atomic): every iteration then reaches + * a different element, so LLVM cannot merge an element's reads across + * iterations or move them out of the loop, and the loop can be vectorized. A racing write from another thread can still be seen or + * missed within one iteration, the documented gap of non-atomic shared + * accesses. Float16 elements keep their runtime path. */ +function plainSharedElement( + host: LlvmEmitterContext, + elem: IrBytesElem, + index: IrExpr | undefined, +): boolean { + const loop = host.plainSharedLoop; + if (loop === null || elem === "f16" || index?.kind !== "varRef") return false; + // A nested loop raises loopDepth before its condition is emitted. + if (index.localId !== loop.counter || host.loopDepth !== loop.depth) return false; + let innermost: (typeof host.jumpTargets)[number] | undefined; + for (let i = host.jumpTargets.length - 1; i >= 0 && innermost === undefined; i--) + if (host.jumpTargets[i]!.kind === "loop") innermost = host.jumpTargets[i]; + return innermost === loop.target; +} + /** Worker programs: the element at a checked index of a view that may be * shared. An integer element joins as its 32-bit value and converts to a * number after the join, as the local load does, so compares and @@ -666,7 +714,7 @@ function emitScopedBytesSet( ty = "i32"; operand = stored!; } - if (host.mod.workers && !owned) { + if (host.mod.workers && !owned && !plainSharedElement(host, elem, indexExpr)) { // A shared view's element takes a relaxed atomic store of its width. const local = emitSharedBytesSplit(host, receiver); const p = B.tmp(); @@ -788,6 +836,7 @@ export function emitBytesIntrinsic( const r = directElementAccess ? host.emitStableReceiver(e.receiver, e.args) : emitBorrowedInput(host, e.receiver); + if (directElementAccess) noteBytesReceiver(host, e.receiver, r.name); // Byte operations borrow every input; views and identity results return // independent owners, while writes retain no operand beyond the call. const args = e.args.map((a) => emitBorrowedInput(host, a)); diff --git a/packages/compiler/src/backend/llvm/expr-callbacks.ts b/packages/compiler/src/backend/llvm/expr-callbacks.ts index 398aebb21..7a17f4fab 100644 --- a/packages/compiler/src/backend/llvm/expr-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-callbacks.ts @@ -8,7 +8,7 @@ import { typeEquals, typeKey, } from "../../ir/ir.js"; -import { mangleGenResThunk, mangleRecordNew, mangleRecordStruct } from "../mangle.js"; +import { mangleRecordNew, mangleRecordStruct } from "../mangle.js"; import { FN_ATTRS, llFieldType, releaseSym, retainSym, traceArg, vAdapters } from "./shapes.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlvmEmitterContext } from "./expr-context.js"; @@ -54,7 +54,7 @@ export function raceAdapterFor(host: LlvmEmitterContext, from: IrType, to: IrTyp const key = `race:${typeKey(from)}=>${typeKey(to)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_race_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_race_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "raceAdapterFor", [from, to]); const toArms = to.kind === "union" ? (host.unionsById.get(to.unionId)?.arms ?? []) : []; @@ -217,7 +217,7 @@ export function genResultThunkFor( const key = `gr:${typeKey(genT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = mangleGenResThunk(host.resolveThunks.size); + sym = host.contentSymbol("sc_gres_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "genResultThunkFor", [genT, recT]); const shape = host.recordsById.get(recT.shapeId); @@ -474,7 +474,7 @@ export function childExitThunkFor(host: LlvmEmitterContext, param: IrType): stri const key = `cx:${param.unionId}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_cx_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_cx_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "childExitThunkFor", [param]); const def = host.unionsById.get(param.unionId); @@ -519,7 +519,7 @@ export function childExitSignalThunkFor( const key = `cx2:${codeParam.unionId}+${sigParam.unionId}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_cx_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_cx_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "childExitSignalThunkFor", [codeParam, sigParam]); const codeDef = host.unionsById.get(codeParam.unionId); @@ -580,7 +580,7 @@ export function execFileThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: const key = `exec:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_exec_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_exec_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "execFileThunkFor", [cbT]); const errorParam = cbT.params[0]; @@ -652,7 +652,7 @@ export function ipcMessageThunkFor( const key = `ipcmsg:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_ipc_msg_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ipc_msg_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "ipcMessageThunkFor", [cbT]); const param = cbT.params[0]; @@ -703,7 +703,7 @@ export function ipcSendThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: const key = `ipcsend:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_ipc_send_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ipc_send_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "ipcSendThunkFor", [cbT]); const param = cbT.params[0]; @@ -764,7 +764,7 @@ export function childDataThunkFor(host: LlvmEmitterContext, param: IrType): stri const key = `cd:${param.unionId}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_cd_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_cd_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "childDataThunkFor", [param]); const def = host.unionsById.get(param.unionId); @@ -799,7 +799,7 @@ export function childDataDynThunkFor(host: LlvmEmitterContext, chunk: "bytes" | const key = `cdd:${chunk}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_cdd_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_cdd_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "childDataDynThunkFor", [chunk]); const box = chunk === "bytes" ? "scr_dyn_new_buffer" : "scr_dyn_new_str"; @@ -834,7 +834,7 @@ export function emitterFixedAdapter( const key = `ee:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return { fn: sym, shim }; - sym = `sc_ee_ad_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ee_ad_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "emitterFixedAdapter", [cbT]); host.declare(`declare ptr @scr_box_get_ref(ptr)`); @@ -905,7 +905,7 @@ export function emitterOnceAdapter( : `eeOnce:${fire}:${typeKey(boxerT)}`; let sym = host.resolveThunks.get(key); if (sym) return { fn: sym, shim }; - sym = `sc_ee_once_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ee_once_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "emitterOnceAdapter", [boxerT, fire]); host.declare(`declare ptr @scr_box_get_ref(ptr)`); @@ -974,10 +974,20 @@ export function wrapEmitterListener( export function unwrapNullableClosure( host: LlvmEmitterContext, u: string, + unionId: string, funcTag: number, ): string { const B = host.B; host.declare(`declare ptr @scr_closure_retain_v(ptr)`); + const nullable = host.nullableUnions.get(unionId); + if (nullable !== null) { + // The union is the closure pointer itself (NULL for the unit arm). + if (nullable.repr !== "single" || nullable.nullTag >= 0 || nullable.refTag !== funcTag) + throw new InternalCompilerError("llvm emitter bug: nullable callback union shape"); + const r = B.tmp(); + B.line(`${r} = call ptr ${retainSym(host.shapeHost, { kind: "union", unionId })}(ptr ${u})`); + return r; + } const slot = B.slot(); B.entryAllocas.push(`${slot} = alloca ptr`); B.line(`store ptr null, ptr ${slot}`); @@ -1015,7 +1025,7 @@ export function closeBindThunkFor( const key = `ncb:${cbUnion.unionId}:${retServer ? "srv" : "void"}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_ncb_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ncb_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "closeBindThunkFor", [cbUnion, retServer]); const def = host.unionsById.get(cbUnion.unionId); @@ -1026,9 +1036,9 @@ export function closeBindThunkFor( "llvm emitter bug: bound-close callback union lacks its func arm", ); const oneParam = funcArm.params.length === 1; + const cbNullable = host.nullableUnions.get(cbUnion.unionId); host.declare(`declare ptr @scr_box_get_ref(ptr)`); host.declare(`declare void @scr_closure_release(ptr)`); - host.declare(`declare void @scr_union_release(ptr)`); host.declare(`declare ptr @scr_closure_retain_v(ptr)`); host.declare(`declare void @scr_net_server_close_direct(ptr, ptr)`); let trampoline: string | null = null; @@ -1072,13 +1082,11 @@ export function closeBindThunkFor( ` %capsp = getelementptr inbounds %ScrClosure, ptr %self, i64 1`, ` %bx = load ptr, ptr %capsp`, ` %srv = call ptr @scr_box_get_ref(ptr %bx) ; +1`, - ` %tagp = getelementptr inbounds %ScrUnion, ptr %cbu, i64 0, i32 1`, - ` %tag = load i32, ptr %tagp`, + ...unionTagLines(cbNullable, "%cbu", "%tag"), ` %isfn = icmp eq i32 %tag, ${funcTag}`, ` br i1 %isfn, label %fn, label %go`, `fn:`, - ` %pp = getelementptr inbounds %ScrUnion, ptr %cbu, i64 0, i32 5`, - ` %pv = load ptr, ptr %pp`, + ...unionPeekLines(cbNullable, "%cbu", "%pv"), ]; if (oneParam) { d.push( @@ -1097,7 +1105,7 @@ export function closeBindThunkFor( ` br label %go`, `go:`, ` %regv = load ptr, ptr %regslot`, - ` call void @scr_union_release(ptr %cbu) ; the callee owns its +1 param`, + ` call void ${releaseSym(host.shapeHost, cbUnion)}(ptr %cbu) ; the callee owns its +1 param`, ` call void @scr_net_server_close_direct(ptr %srv, ptr %regv) ; reg moves`, ); if (retServer) { @@ -1121,7 +1129,7 @@ export function closeOverrideWrapFor( const key = `ncw:${cbUnion.unionId}:${retServer ? "srv" : "void"}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_ncw_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ncw_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "closeOverrideWrapFor", [cbUnion, retServer]); const def = host.unionsById.get(cbUnion.unionId); @@ -1169,7 +1177,7 @@ export function settledStoreFor( const key = `settled:${typeKey(from)}=>${typeKey(to)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_settled_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_settled_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "settledStoreFor", [from, to]); const def = host.unionsById.get(to.unionId); diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index d411d9172..3ecf33a79 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -42,7 +42,13 @@ import { } from "./classes.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlvmEmitterContext, ExprOf, LlValue } from "./expr-context.js"; -import { f64Lit, ffiNativeTypeLl, ffiNativeParamLl, ffiNativeReturnLl } from "./common.js"; +import { + CLOSURE_RECEIVER_FREE, + f64Lit, + ffiNativeTypeLl, + ffiNativeParamLl, + ffiNativeReturnLl, +} from "./common.js"; import { canStackUnion, emitStackUnion } from "./stack-unions.js"; import { emitStackRecordKeyGet, scalarRecordKeyGet } from "./expr-records.js"; import { emitCallArrayRead } from "./local-array-reads.js"; @@ -761,8 +767,7 @@ export function emitCallExpr( `${c} = call ptr @scr_closure_new(ptr @${host.callTarget(e.fnName)}, ${host.sizeType} ${e.captures.length})`, ); const out = host.own({ name: c, type: e.type }); - const functionKind = - (target.generator ? 1 : 0) + (target.async ? 2 : 0) + (target.ownsPrototype ? 4 : 0); + const functionKind = host.closureFunctionKind(target); if (functionKind) { const kindPtr = B.tmp(); B.line(`${kindPtr} = getelementptr inbounds %ScrClosure, ptr ${c}, i64 0, i32 4`); @@ -800,6 +805,25 @@ export function emitCallExpr( : 0; host.declare(`declare void @scr_closure_adopt_identity(ptr, ptr, i32)`); B.line(`call void @scr_closure_adopt_identity(ptr ${c}, ptr ${original}, i32 ${length})`); + if (host.receiverReaders.forwardsOnly(target)) { + // The adapter forwards the ambient receiver to its original, + // so it ignores the receiver exactly when the original does. + const fromPtr = B.tmp(), + from = B.tmp(), + bit = B.tmp(), + toPtr = B.tmp(), + to = B.tmp(), + kind = B.tmp(); + B.line( + `${fromPtr} = getelementptr inbounds %ScrClosure, ptr ${original}, i64 0, i32 4`, + ); + B.line(`${from} = load i32, ptr ${fromPtr}`); + B.line(`${bit} = and i32 ${from}, ${CLOSURE_RECEIVER_FREE}`); + B.line(`${toPtr} = getelementptr inbounds %ScrClosure, ptr ${c}, i64 0, i32 4`); + B.line(`${to} = load i32, ptr ${toPtr}`); + B.line(`${kind} = or i32 ${to}, ${bit}`); + B.line(`store i32 ${kind}, ptr ${toPtr}`); + } } }); return out; @@ -813,7 +837,18 @@ export function emitCallExpr( const ft = e.callee.type; if (e.args.length !== ft.params.length) throw new LlvmUnsupportedError("callValue:arity", e.loc); - const callee = host.stackCallbacks.emit(host, e.callee) ?? emitBorrowedInput(host, e.callee); + // A receiver that reads the ambient `this` binds what the callee + // would see without a binding: the call forwards it unchanged and + // needs no receiver bracket (signature adapters forward this way). + const forwardsReceiver = + e.receiver?.kind === "libCall" && + e.receiver.fn === "dyn.this" && + e.receiver.args.length === 0; + const callee = + host.stackCallbacks.emit(host, e.callee) ?? + (lendsCallee(host, e, forwardsReceiver) + ? host.emitReadReceiver(e.callee) + : emitBorrowedInput(host, e.callee)); // A class instance bound as the callee's `this` (`h.fn(x)`) is boxed // only if the callee reads it: the receiver window records the // instance (owned by this frame for the whole call) with its typed-ref @@ -825,7 +860,10 @@ export function emitCallExpr( !host.shapeHost.isErrorClass(e.receiver.value.type.className) ? host.emitExpr(e.receiver.value) : null; - const receiver = e.receiver === undefined || lazyReceiver ? null : host.emitExpr(e.receiver); + const receiver = + e.receiver === undefined || lazyReceiver || forwardsReceiver + ? null + : host.emitExpr(e.receiver); const pushReceiver = (): void => { if (lazyReceiver) { const t = lazyReceiver.type; @@ -865,7 +903,7 @@ export function emitCallExpr( ? borrowableInputs(host, e.args, false, host.referenceWrites?.summary(known!.name)) : []; let dynamic = false; - if (!direct && e.receiver === undefined) { + if (!direct && (e.receiver === undefined || forwardsReceiver)) { const references = new Set(); ft.params.forEach((param, index) => { if (isRefCounted(param)) references.add(index); @@ -898,16 +936,51 @@ export function emitCallExpr( lent.has(i) ? { name: host.retainFrame(a.name, a.type), type: a.type } : a, ) : args; - const invoke = (target: string, needsReceiver: boolean, values = args): string => { + // An entry through the closure's own pointer brackets the call only + // when the closure may observe the receiver (SCR_FN_RECEIVER_FREE). + const invoke = ( + target: string, + bracket: boolean, + values = args, + indirect = false, + ): string => { + const needsReceiver = bracket && !forwardsReceiver; + let observes: string | null = null; if (needsReceiver) { host.declare("declare void @scr_dyn_this_pop()"); - pushReceiver(); + if (indirect) { + const kindPtr = B.tmp(), + kind = B.tmp(), + free = B.tmp(); + observes = B.tmp(); + B.line( + `${kindPtr} = getelementptr inbounds %ScrClosure, ptr ${callee.name}, i64 0, i32 4`, + ); + B.line(`${kind} = load i32, ptr ${kindPtr}`); + B.line(`${free} = and i32 ${kind}, ${CLOSURE_RECEIVER_FREE}`); + B.line(`${observes} = icmp eq i32 ${free}, 0`); + const push = B.newLabel("receiver.push"), + pushed = B.newLabel("receiver.call"); + B.condBr(observes, push, pushed); + B.startBlock(push); + pushReceiver(); + B.br(pushed); + B.startBlock(pushed); + } else pushReceiver(); } const result = e.type.kind === "void" ? "" : B.tmp(); B.line( `${result ? `${result} = ` : ""}call ${host.llType(e.type)} ${target}(${argListOf(values)})`, ); - if (needsReceiver) B.line("call void @scr_dyn_this_pop()"); + if (observes !== null) { + const pop = B.newLabel("receiver.pop"), + popped = B.newLabel("receiver.done"); + B.condBr(observes, pop, popped); + B.startBlock(pop); + B.line("call void @scr_dyn_this_pop()"); + B.br(popped); + B.startBlock(popped); + } else if (needsReceiver) B.line("call void @scr_dyn_this_pop()"); return result; }; const lending = @@ -953,11 +1026,11 @@ export function emitCallExpr( B.line(`${has} = icmp ne ptr ${entry}, null`); B.condBr(has, borrowing, owning); B.startBlock(borrowing); - const lentResult = invoke(entry, true); + const lentResult = invoke(entry, true, args, true); const borrowingEnd = B.currentLabel(); B.br(done); B.startBlock(owning); - const ownedResult = invoke(fn, true, ownedArgs()); + const ownedResult = invoke(fn, true, ownedArgs(), true); const owningEnd = B.currentLabel(); B.br(done); B.startBlock(done); @@ -966,7 +1039,7 @@ export function emitCallExpr( B.line( `${t} = phi ${host.llType(e.type)} [ ${lentResult}, %${borrowingEnd} ], [ ${ownedResult}, %${owningEnd} ]`, ); - } else if (!direct) t = invoke(fn, true, ownedArgs()); + } else if (!direct) t = invoke(fn, true, ownedArgs(), true); else { // A global initializer may not have run, or a callback may enter // through a boundary the census cannot prove. Validate the actual @@ -984,7 +1057,7 @@ export function emitCallExpr( const fastEnd = B.currentLabel(); B.br(done); B.startBlock(fallback); - const indirect = invoke(fn, true, ownedArgs()); + const indirect = invoke(fn, true, ownedArgs(), true); const fallbackEnd = B.currentLabel(); B.br(done); B.startBlock(done); @@ -1358,6 +1431,22 @@ function ctorArguments( * walk consumes without owning it (LlEmitter.canBorrowReceiver proved every * reference argument borrowable and every argument edge-preserving). The * result stays reachable from the borrowed arguments until the consumer. */ +/** A closure value read from a location the call cannot write (a keyed + * table entry, a field, a checked extraction of either) is entered without + * a reference of its own: the location keeps the closure alive until the + * call returns. Plain stable bindings already lend (emitBorrowedInput). */ +function lendsCallee( + host: LlvmEmitterContext, + e: ExprOf<"callValue">, + forwardsReceiver: boolean, +): boolean { + const writes = host.referenceWrites; + if (writes === null || host.canBorrowCallArgument(e.callee) || !host.canBorrowReceiver(e.callee)) + return false; + const later = e.receiver === undefined || forwardsReceiver ? e.args : [e.receiver, ...e.args]; + return borrowableInputs(host, [e.callee, ...later], false, writes.ownWrites(e))[0] === true; +} + export function emitBorrowedResultCall(host: LlvmEmitterContext, e: ExprOf<"call">): LlValue { const callee = host.fnByName.get(e.callee); if (!callee || !host.borrowedReturns.has(e.callee)) diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 696dad3ab..7eec14e4e 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -4,7 +4,6 @@ import { arraySearchHelperName, arraySearchKind } from "./array-search.js"; import { InternalCompilerError } from "../../errors.js"; import { isStableReceiverOperand, undefinedArmTag } from "../../ir/analysis.js"; import { F64, type IrExpr, type IrType, isRefCounted, typeEquals, typeKey } from "../../ir/ir.js"; -import { mangleResolveThunk } from "../mangle.js"; import { arrOps, elemAccess, @@ -76,7 +75,7 @@ export function resolveThunkFor(host: LlvmEmitterContext, inner: IrType): string const key = typeKey(inner); let sym = host.resolveThunks.get(key); if (!sym) { - sym = mangleResolveThunk(host.resolveThunks.size); + sym = host.contentSymbol("sc_rslv_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "resolveThunkFor", [inner]); const v = vAdapters(host.shapeHost, inner); diff --git a/packages/compiler/src/backend/llvm/expr-context.ts b/packages/compiler/src/backend/llvm/expr-context.ts index ac5262571..aea9e4f50 100644 --- a/packages/compiler/src/backend/llvm/expr-context.ts +++ b/packages/compiler/src/backend/llvm/expr-context.ts @@ -21,6 +21,10 @@ export interface LlValue { * its absolute value is below `below` (a zero may be -0). Index consumers * use it in place of a round trip (integer-values.ts `indexInteger`). */ wideInteger?: WideInteger; + /** An i1 SSA value (or `true`) that is true exactly when this number is + * not NaN: set by NaN-coded integer element reads and the wrapping sums + * of them, so ToInt32 consumers need no floating compare. */ + ordered?: string; slot?: boolean; /** A sequence local held in a capture box; release the box itself. */ boxed?: boolean; diff --git a/packages/compiler/src/backend/llvm/expr-control.ts b/packages/compiler/src/backend/llvm/expr-control.ts index 5ffa54eb5..6e0fd4acb 100644 --- a/packages/compiler/src/backend/llvm/expr-control.ts +++ b/packages/compiler/src/backend/llvm/expr-control.ts @@ -499,10 +499,7 @@ export function emitOptionalChainReceiver( // needs no reference of its own while the body cannot remove an edge // (so nothing can overwrite the projection before the body finishes). const borrowReceiver = - !multiple && - isRefCounted(narrowed) && - host.canBorrowReceiver(e.receiver) && - host.referenceEffects.preserves(e.body); + !multiple && isRefCounted(narrowed) && borrowsChainReceiver(host, e.receiver, e.body); // An optional array read of a nullable-pointer union is the element // pointer itself, read inline and owned by the statement frame until // the body finishes; the bind slot borrows it like an emitExpr value. @@ -562,6 +559,16 @@ export function emitOptionalChainReceiver( }; } +/** An optional chain's receiver can lend its payload to the bind slot when + * the body leaves the receiver's locations alone: it removes no edge at + * all, or it writes none of the locations the receiver reads. */ +function borrowsChainReceiver(host: LlvmEmitterContext, receiver: IrExpr, body: IrExpr): boolean { + if (!host.canBorrowReceiver(receiver)) return false; + if (host.referenceEffects.preserves(body)) return true; + const writes = host.referenceWrites; + return writes !== null && host.unwrittenRead(receiver, writes.expr(body)); +} + /** `a?.b ?? d` narrowed to the chain body's present arm. The body wraps a * value of the result type in a non-unit arm, so the default runs exactly * when the receiver is nullish and the intermediate result box is never @@ -588,16 +595,21 @@ function emitNullishOptionalChain(host: LlvmEmitterContext, e: ExprOf<"nullish"> if (unitTags.length === 0 || present.length !== 1) return null; const B = host.B; const narrowed = receiverDef.arms[present[0]!]!; - const r = host.canStackReceiver(chain.receiver) - ? host.emitReadReceiver(chain.receiver) - : host.emitExpr(chain.receiver); + // A receiver read from a location the body cannot write lends its + // payload to the bind slot for the whole body (`table[k]?.(…) ?? d`). + const borrowed = + isRefCounted(narrowed) && borrowsChainReceiver(host, chain.receiver, chain.body.value); + const r = + borrowed || host.canStackReceiver(chain.receiver) + ? host.emitReadReceiver(chain.receiver) + : host.emitExpr(chain.receiver); const bindTy = host.llType(narrowed); const bind = B.slot(); B.entryAllocas.push(`${bind} = alloca ${bindTy}`); B.line( `store ${bindTy} ${bindTy === "ptr" ? "null" : bindTy === "double" ? f64Lit(0) : "false"}, ptr ${bind}`, ); - host.ownSlot(bind, narrowed); + if (!borrowed) host.ownSlot(bind, narrowed); const isUnit = host.tagInSet(r.name, receiverDef.id, unitTags); const ty = host.llType(e.type); const slot = B.slot(); @@ -610,7 +622,10 @@ function emitNullishOptionalChain(host: LlvmEmitterContext, e: ExprOf<"nullish"> host.emitBranchInto(slot, e.right); B.br(lj); B.startBlock(lb); - B.line(`store ${bindTy} ${host.unionExtract(r.name, receiverDef.id, narrowed)}, ptr ${bind}`); + const payload = borrowed + ? host.unionPeek(r.name, receiverDef.id) + : host.unionExtract(r.name, receiverDef.id, narrowed); + B.line(`store ${bindTy} ${payload}, ptr ${bind}`); host.chainSlots.set(chain.id, { name: bind, type: narrowed, slot: true }); host.emitBranchInto(slot, chain.body.value); host.chainSlots.delete(chain.id); diff --git a/packages/compiler/src/backend/llvm/expr-dynamic.ts b/packages/compiler/src/backend/llvm/expr-dynamic.ts index 55bbb828b..7f4b7e48a 100644 --- a/packages/compiler/src/backend/llvm/expr-dynamic.ts +++ b/packages/compiler/src/backend/llvm/expr-dynamic.ts @@ -11,7 +11,7 @@ import { bindLiveArrayOps } from "./expr-stream-bridges.js"; /* Focused LLVM expression emission extracted from emitter.ts. */ import { InternalCompilerError } from "../../errors.js"; import { emitNullableIsTag, emitNullablePresent } from "./union-repr.js"; -import { isObjectArm } from "./nullable-unions.js"; +import { isPointerIdentityArm } from "./nullable-unions.js"; import { streamTypedRefEligible } from "../../ir/analysis.js"; import { DYN, @@ -908,13 +908,13 @@ export function emitDynamicExpr( if (!def) throw new InternalCompilerError(`llvm emitter bug: equality of unknown union ${e.unionId}`); const nullable = host.nullableUnions.get(def.id); - if (nullable && isObjectArm(nullable)) { + if (nullable && isPointerIdentityArm(nullable)) { // Object identity, and NULL === NULL for the unit arm: one compare. const eq = B.tmp(); B.line(`${eq} = icmp ${e.negated ? "ne" : "eq"} ptr ${l.name}, ${r.name}`); return { name: eq, type: e.type }; } - if (nullable) { + if (nullable && nullable.repr === "single" && nullable.arm.kind === "string") { // String arm: identical pointers are equal (the same string or the // same unit encoding); otherwise equal only when both hold strings // with the same bytes. diff --git a/packages/compiler/src/backend/llvm/expr-primitives.test.ts b/packages/compiler/src/backend/llvm/expr-primitives.test.ts index f343c956e..93555c101 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.test.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.test.ts @@ -278,6 +278,18 @@ test("sums that only feed a bitwise conversion of int32 operands add in 32 bits" returning("float_read_add", bin("|", bin("+", length, read("floats")), zero)), returning("product", bin("|", bin("*", length, length), zero)), returning("number_add", bin("+", length, read("bytes"))), + returning( + "read_products", + bin( + ">>>", + bin( + "+", + bin("*", read("bytes"), { kind: "numLit", value: 77, type: F64, loc }), + bin("*", read("bytes"), { kind: "numLit", value: 150, type: F64, loc }), + ), + zero, + ), + ), ]; const llvm = emitLlvmModule(mod); const body = (name: string): string => @@ -292,8 +304,10 @@ test("sums that only feed a bitwise conversion of int32 operands add in 32 bits" expect(body(name), name).not.toContain("uint32.coerce"); } // A NaN-coded read converts to 0 through the sum: the result selects 0 - // unless every NaN-coded operand is ordered. - expect(body("read_add")).toMatch(/ = fcmp ord double .*, 0\.0$/m); + // unless every NaN-coded operand is ordered. A read's own validity bit + // tells, so the number is not compared. + expect(body("read_add")).toMatch(/ = phi i1 \[ true, /); + expect(body("read_add")).not.toMatch(/fcmp ord/); expect(body("read_add")).toMatch(/ = add i32 /); expect(body("read_add")).toMatch(/ = select i1 .*, i32 .*, i32 0$/m); expect(body("read_chain")).toMatch(/ = and i1 /); @@ -309,6 +323,12 @@ test("sums that only feed a bitwise conversion of int32 operands add in 32 bits" expect(body(name), name).not.toMatch(/ = (add|mul) i32 /); } expect(body("number_add")).not.toMatch(/ = add i32 |fcmp ord/); + // Products of NaN-coded reads and constants are exact integers or NaN, so + // their sum adds in 32 bits too, selected by the reads' validity bits. + expect(body("read_products")).toMatch(/ = mul i32 /); + expect(body("read_products")).toMatch(/ = add i32 /); + expect(body("read_products")).not.toContain("uint32.coerce"); + expect(body("read_products")).not.toMatch(/fcmp ord/); }); test.each([32, 64] as const)( diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index eed138fe2..6e1c1ac4e 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -47,7 +47,7 @@ import { type ArrayAccess, } from "./dense-array-access.js"; import { emitWrappingSum, integerOrNaNRange, markWrappingOperands } from "./wrapping-sums.js"; -import { emitPowerOfTwoQuotient, isPowerOfTwoCall } from "./lib-filesystem.js"; +import { emitPowF64, emitPowerOfTwoQuotient, isPowerOfTwoCall } from "./lib-filesystem.js"; /** The signed integer predicates of the number compares. */ const INTEGER_COMPARES: Record = { @@ -305,7 +305,6 @@ export function emitOperatorExpr( "===": "oeq", "!==": "une", }; - const libm: Record = { "**": "pow" }; const bit: Record = { "&": "and", "|": "or", @@ -367,8 +366,8 @@ export function emitOperatorExpr( `${result} = ${e.op === "+" ? "add" : e.op === "-" ? "sub" : "mul"} nsw i64 ${left}, ${right}`, ); const number = integerNumber(host, result, range, e.type); - const uint32 = emitWrappingSum(host, e, l, r, li, ri); - return uint32 === undefined ? number : { ...number, uint32 }; + const wrapped = emitWrappingSum(host, e, l, r, li, ri); + return wrapped === undefined ? number : { ...number, ...wrapped }; } if (e.op === "%" && li.range.min >= 0 && ri.range.min > 0) { const range = { min: 0, max: Math.min(li.range.max, ri.range.max - 1) }; @@ -411,12 +410,12 @@ export function emitOperatorExpr( return { name: emitPowerOfTwoQuotient(host, l.name, r.name), type: e.type }; if (arith[e.op] !== undefined) { B.line(`${t} = ${arith[e.op]} double ${l.name}, ${r.name}`); - const uint32 = emitWrappingSum(host, e, l, r, li, ri); + const wrapped = emitWrappingSum(host, e, l, r, li, ri); // The exact integer beside the double, for index consumers. const wideInteger = wideArithmetic(host, e.op, l, li, r, ri); - if (uint32 !== undefined || wideInteger) { + if (wrapped !== undefined || wideInteger) { const value: LlValue = { name: t, type: e.type }; - if (uint32 !== undefined) value.uint32 = uint32; + if (wrapped !== undefined) Object.assign(value, wrapped); if (wideInteger) value.wideInteger = wideInteger; return value; } @@ -471,32 +470,8 @@ export function emitOperatorExpr( B.line(`${t} = ${e.op === ">>>" ? "uitofp" : "sitofp"} i32 ${result} to double`); return { name: t, type: e.type, uint32: result }; } else { - const fn = libm[e.op]; - if (fn === undefined) throw new LlvmUnsupportedError(`bin:${e.op}`, e.loc); - host.declare(`declare double @${fn}(double, double)`); - // libm's pow answers 1 for pow(±1, ±Infinity) and pow(1, NaN), where - // `**` answers NaN (and LLVM folds pow(1.0, x) to 1.0): an inline - // guard selects NaN for a base of ±1 with a non-finite exponent and - // keeps the direct pow call. A finite literal exponent, or a literal - // base other than ±1, cannot meet those cases and needs no guard. - if (fn === "pow" && !libmPowAgrees(e.left, e.right)) { - host.declare(`declare double @llvm.fabs.f64(double)`); - const power = B.tmp(); - B.line(`${power} = call double @pow(double ${l.name}, double ${r.name})`); - const base = B.tmp(); - const unit = B.tmp(); - const magnitude = B.tmp(); - const finite = B.tmp(); - const reject = B.tmp(); - B.line(`${base} = call double @llvm.fabs.f64(double ${l.name})`); - B.line(`${unit} = fcmp oeq double ${base}, 1.0`); - B.line(`${magnitude} = call double @llvm.fabs.f64(double ${r.name})`); - B.line(`${finite} = fcmp olt double ${magnitude}, 0x7FF0000000000000`); - B.line(`${reject} = select i1 ${finite}, i1 false, i1 ${unit}`); - B.line(`${t} = select i1 ${reject}, double 0x7FF8000000000000, double ${power}`); - } else { - B.line(`${t} = call double @${fn}(double ${l.name}, double ${r.name})`); - } + if (e.op !== "**") throw new LlvmUnsupportedError(`bin:${e.op}`, e.loc); + return { name: emitPowF64(host, l.name, r.name, e.right), type: e.type }; } return { name: t, type: e.type }; } @@ -649,7 +624,7 @@ export function regexLiteralTemplate( let re = host.regexInstances.get(key); if (!re) { re = { - sym: `sc_re_${host.regexInstances.size}`, + sym: host.contentSymbol("sc_re_", key), src: host.internLiteral(e.pattern), fl: host.internLiteral(e.flags), pattern: e.pattern, @@ -673,7 +648,7 @@ export function regexLiteralReplacer( let rr = host.regexReplacers.get(key); if (!rr) { rr = { - sym: `sc_rr_${host.regexReplacers.size}`, + sym: host.contentSymbol("sc_rr_", key), pattern: e.pattern, flags: e.flags, replacement, @@ -1473,19 +1448,3 @@ export function emitRecordLitFields( host.initField(ptr, type, inline, v); } } - -/** Whether libm pow matches `**` for these operands: every case except a - * base of ±1 with a NaN or infinite exponent agrees. */ -function libmPowAgrees(base: IrExpr, exponent: IrExpr): boolean { - const power = numericLiteral(exponent); - if (power !== null && Number.isFinite(power)) return true; - const value = numericLiteral(base); - return value !== null && !Number.isNaN(value) && Math.abs(value) !== 1; -} - -function numericLiteral(expr: IrExpr): number | null { - if (expr.kind === "numLit") return expr.value; - if (expr.kind === "unary" && expr.op === "-" && expr.operand.kind === "numLit") - return -expr.operand.value; - return null; -} diff --git a/packages/compiler/src/backend/llvm/expr-records.ts b/packages/compiler/src/backend/llvm/expr-records.ts index 161d1eccc..af4bb092f 100644 --- a/packages/compiler/src/backend/llvm/expr-records.ts +++ b/packages/compiler/src/backend/llvm/expr-records.ts @@ -10,6 +10,7 @@ import { regexLiteralReplacer, regexLiteralTemplate } from "./expr-primitives.js import { canReplaceWith, regexLiteralCompiles } from "./regex-matchers.js"; import { isScalarUnionDef, loadInlineUnion, presentBits } from "./inline-unions.js"; import { preservesRegexInputs } from "./reference-effects.js"; +import { emitNumberMapSlot, emitStringMapEntry } from "./map-number-lookup.js"; /** Widen one tagged read. The source is borrowed unless the overflow map * supplied an owned value. Each selected reference payload is retained @@ -417,8 +418,8 @@ export function emitRecordKeyGet( e: IrExpr & { kind: "recordKeyGet" }, ): LlValue { const B = host.B; - const obj = host.emitExpr(e.obj); - const k = host.emitExpr(e.key); + // The read runs no user code: both inputs lend for its duration. + const [obj, k] = emitBorrowedInputs(host, [e.obj, e.key]) as [LlValue, LlValue]; const ty = host.llType(e.type); const slot = B.slot(); B.entryAllocas.push(`${slot} = alloca ${ty}`); @@ -440,6 +441,80 @@ export function emitRecordKeyGet( return host.own({ name: t, type: e.type }); } +/** A keyed read whose every hit is the result's own pointer: the result + * is a nullable-pointer union (nullable-unions.ts) with an undefined arm, + * and each declared field and the overflow value hold exactly the result + * type or its reference arm in a plain (non-inline) slot. A miss is the + * immortal undefined encoding. Such a read can lend the stored pointer: + * borrowedRecordKeyGet reads it without a reference. */ +export function pointerRecordKeyGet(host: LlvmEmitterContext, e: IrExpr): boolean { + if (e.kind !== "recordKeyGet" || e.missNaN === true || e.type.kind !== "union") return false; + const resultType = e.type; + const nullable = host.nullableUnions.single(resultType.unionId); + if (nullable === null || undefinedArmTag(resultType, host.unionsById) < 0) return false; + const exact = (t: IrType): boolean => typeEquals(t, resultType) || typeEquals(t, nullable.arm); + const shape = host.recordShape(e.shapeId); + if (e.overflowOnly !== true) { + for (const f of shape.fields) + if (host.recordFieldInline(e.shapeId, f.name) || !exact(f.type)) return false; + } + return shape.indexValue === undefined || typeEquals(shape.indexValue, nullable.arm); +} + +/** A pointerRecordKeyGet read without a reference: the declared field or + * overflow entry the key names, or the undefined encoding for a miss. The + * record (and through it the stored value) must stay unchanged and alive + * until the consumer is done with the pointer (canBorrowReceiver). */ +export function emitBorrowedRecordKeyGet( + host: LlvmEmitterContext, + e: IrExpr & { kind: "recordKeyGet" }, +): LlValue { + const B = host.B; + if (e.type.kind !== "union") + throw new InternalCompilerError("llvm emitter bug: borrowed keyed read of a non-union"); + const resultType = e.type; + const obj = host.emitReadReceiver(e.obj); + const key = host.canBorrowReceiver(e.key) ? host.emitReadReceiver(e.key) : host.emitExpr(e.key); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ptr`); + const join = B.newLabel("rkb.j"); + const shape = host.recordShape(e.shapeId); + for (const f of e.overflowOnly === true ? [] : shape.fields) { + const hit = B.tmp(); + B.line( + `${hit} = call zeroext i1 ${host.literalEquals(f.name)}(ptr ${key.name}) ; ${llvmCommentText(f.name)}`, + ); + const lh = B.newLabel("rkb.h"); + const ln = B.newLabel("rkb.n"); + B.condBr(hit, lh, ln); + B.startBlock(lh); + const { ptr, type } = host.recordFieldPtr(obj.name, e.shapeId, f.name); + B.line(`store ptr ${host.loadRecordField(ptr, type, false)}, ptr ${slot}`); + B.br(join); + B.startBlock(ln); + } + if (shape.indexValue) { + const ovf = host.recordOvfPtr(obj.name, e.shapeId); + const entry = emitStringMapEntry(host, ovf, key.name); + const found = B.tmp(); + B.line(`${found} = icmp ne ptr ${entry}, null`); + const lh = B.newLabel("rkb.h"); + const ln = B.newLabel("rkb.n"); + B.condBr(found, lh, ln); + B.startBlock(lh); + B.line(`store ptr ${emitNumberMapSlot(host, entry, "ptr")}, ptr ${slot}`); + B.br(join); + B.startBlock(ln); + } + const undefTag = undefinedArmTag(resultType, host.unionsById); + B.line(`store ptr ${host.unitInstanceRef(resultType.unionId, undefTag)}, ptr ${slot}`); + B.br(join); + B.startBlock(join); + const t = B.tmp(); + B.line(`${t} = load ptr, ptr ${slot}`); + return { name: t, type: e.type }; +} + /** A keyed read whose result is a scalar union (`r[k]` on a * `Record`, at `number | undefined`) and whose every hit * is a number or boolean arm of it, or an inline field of the same union. @@ -735,27 +810,17 @@ export function keyedRecordReadInto( B.br(join); B.startBlock(ln); } else { - host.declare(`declare ptr @scr_map_get_str_ref(ptr, ptr)`); + // The entry lookup filters absent keys inline; a hit retains the + // stored value with its own type's retain (map-number-lookup.ts). A + // nullable value's NULL is a stored unit arm: presence decides. const lh = B.newLabel("rkg.h"); const ln = B.newLabel("rkg.n"); - let raw: string; - if (iv.kind === "union" && host.nullableUnions.has(iv.unionId)) { - // NULL is a stored unit arm of a nullable value: presence decides. - host.declare(`declare zeroext i1 @scr_map_has_str(ptr, ptr)`); - const has = B.tmp(); - B.line(`${has} = call zeroext i1 @scr_map_has_str(ptr ${ovf}, ptr ${keyName})`); - B.condBr(has, lh, ln); - B.startBlock(lh); - raw = B.tmp(); - B.line(`${raw} = call ptr @scr_map_get_str_ref(ptr ${ovf}, ptr ${keyName})`); - } else { - raw = B.tmp(); - const isnull = B.tmp(); - B.line(`${raw} = call ptr @scr_map_get_str_ref(ptr ${ovf}, ptr ${keyName})`); - B.line(`${isnull} = icmp eq ptr ${raw}, null`); - B.condBr(isnull, ln, lh); - B.startBlock(lh); - } + const entry = emitStringMapEntry(host, ovf, keyName); + const found = B.tmp(); + B.line(`${found} = icmp ne ptr ${entry}, null`); + B.condBr(found, lh, ln); + B.startBlock(lh); + const raw = host.retainHeap(emitNumberMapSlot(host, entry, "ptr"), iv); B.line(`store ${ty} ${surface(iv, raw, true)}, ptr ${slot}`); B.br(join); B.startBlock(ln); diff --git a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts index b368c91f4..285f9ca9a 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts @@ -13,7 +13,7 @@ export function streamDataAdapter( const key = `eed:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_ee_dad_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ee_dad_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "streamDataAdapter", [cbT]); const p = cbT.params[0]; @@ -221,7 +221,7 @@ export function streamDoneFnFor( const key = `sd:${kind}:${typeKey(doneT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_sd_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_sd_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "streamDoneFnFor", [kind, doneT]); const errT = doneT.params[0]; @@ -365,7 +365,7 @@ export function fsRenameThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: const key = `fsren:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_fsren_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_fsren_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "fsRenameThunkFor", [cbT]); const param = cbT.params[0]!; @@ -454,7 +454,7 @@ export function cryptoIntThunkFor( const key = `cryptoint:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_cryptoint_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_cryptoint_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "cryptoIntThunkFor", [cbT]); const d: string[] = [ @@ -516,7 +516,7 @@ export function zlibBytesThunkFor( const key = `${family}bytes:${arrayBuffer ? "arraybuffer:" : ""}${mode}${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_${family}bytes_${host.resolveThunks.size}`; + sym = host.contentSymbol(`sc_${family}bytes_`, key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "zlibBytesThunkFor", [cbT, arrayBuffer, family, valueMode]); const d: string[] = [ @@ -644,7 +644,7 @@ export function streamCbThunkFor( const key = `scb:${kind}:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_scb_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_scb_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "streamCbThunkFor", [kind, cbT]); const runtimeParams = diff --git a/packages/compiler/src/backend/llvm/function-layout.ts b/packages/compiler/src/backend/llvm/function-layout.ts new file mode 100644 index 000000000..00cf233db --- /dev/null +++ b/packages/compiler/src/backend/llvm/function-layout.ts @@ -0,0 +1,41 @@ +/** Function layouts: the functions that runs of a program execute, in tiers + * (scripts/refresh-cli-layout.mjs writes the native compiler's). The emitter + * writes a listed function before the others, in a partition group of its + * tier, so the code that runs together sits together in the executable. */ + +/** Layout key → tier (1 first). */ +export type FunctionLayout = ReadonlyMap; + +/** The layout key of an LLVM symbol: module numbers (`%m12.`, `%init.12`), + * which change when modules are added, are left out. */ +export function functionLayoutKey(symbol: string): string { + return symbol.replace(/_x25_m\d+_/g, "_x25_m_").replace(/_x25_init_\d+$/, "_x25_init_"); +} + +/** Reads a layout's lines, each ` `. A function listed in two + * tiers keeps the first. */ +export function parseFunctionLayout(text: string): FunctionLayout { + const layout = new Map(); + for (const line of text.split("\n")) { + const space = line.indexOf(" "); + if (space < 1) continue; + const tier = Number(line.slice(0, space)); + const key = functionLayoutKey(line.slice(space + 1).trim()); + if (Number.isInteger(tier) && tier > 0 && !layout.has(key)) layout.set(key, tier); + } + return layout; +} + +/** The lowest tier of the functions `text` defines, or undefined when the + * layout lists none of them. */ +export function layoutTierOf(layout: FunctionLayout, text: string): number | undefined { + let tier: number | undefined; + for (let at = text.indexOf("define "); at >= 0; at = text.indexOf("\ndefine ", at + 1)) { + const name = text.indexOf("@", at); + const open = text.indexOf("(", name); + if (name < 0 || open < 0) break; + const listed = layout.get(functionLayoutKey(text.slice(name + 1, open))); + if (listed !== undefined && (tier === undefined || listed < tier)) tier = listed; + } + return tier; +} diff --git a/packages/compiler/src/backend/llvm/lib-connect.ts b/packages/compiler/src/backend/llvm/lib-connect.ts index 4d151fc7e..2ab72c188 100644 --- a/packages/compiler/src/backend/llvm/lib-connect.ts +++ b/packages/compiler/src/backend/llvm/lib-connect.ts @@ -17,7 +17,7 @@ function connectAdapter(host: LlvmEmitterContext, cb: LlValue, h2: boolean): str const key = `connect:${h2}:${typeKey(type)}`; const previous = host.resolveThunks.get(key); if (previous !== undefined) return previous; - const symbol = `sc_connect_${host.resolveThunks.size}`; + const symbol = host.contentSymbol("sc_connect_", key); host.resolveThunks.set(key, symbol); host.journal?.allocated(symbol, "connectAdapter", [cb, h2]); const retain = h2 ? "scr_http_res_retain_v" : "scr_net_sock_retain_v"; diff --git a/packages/compiler/src/backend/llvm/lib-datagram.ts b/packages/compiler/src/backend/llvm/lib-datagram.ts index 07de4e54e..7a83f1d30 100644 --- a/packages/compiler/src/backend/llvm/lib-datagram.ts +++ b/packages/compiler/src/backend/llvm/lib-datagram.ts @@ -19,7 +19,7 @@ function messageAdapter(host: LlvmEmitterContext, type: IrType): string { const key = "dgram:" + type.shapeId; const previous = host.resolveThunks.get(key); if (previous !== undefined) return previous; - const symbol = `sc_dgram_message_${host.resolveThunks.size}`; + const symbol = host.contentSymbol("sc_dgram_message_", key); host.resolveThunks.set(key, symbol); host.journal?.allocated(symbol, "messageAdapter", [type]); const shape = host.recordsById.get(type.shapeId); @@ -68,7 +68,7 @@ function lookupAdapter(host: LlvmEmitterContext, cb: LlValue): string { const key = `dns:${error.unionId}:${arity}`; const previous = host.resolveThunks.get(key); if (previous !== undefined) return previous; - const symbol = `sc_dns_lookup_${host.resolveThunks.size}`; + const symbol = host.contentSymbol("sc_dns_lookup_", key); host.resolveThunks.set(key, symbol); host.journal?.allocated(symbol, "lookupAdapter", [cb]); host.declare("declare ptr @scr_error_new(i32, ptr)"); diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 8cefda8f0..bd2e42c1e 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -17,6 +17,7 @@ import type { LlvmEmitterContext, LibCallExpr, LibCallPrefix, LlValue } from "./ import { ASSERT_VERDICT_LIB_FNS, LIB_FN_SYMS, + PURE_MATH_LIB_FNS, mockTimerSym, USES_TIMERS_LIB_FNS, } from "./lib-shared.js"; @@ -671,11 +672,13 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll // which also takes those streams. e.fn === "stream.unboxDyn" && host.usesDynBuiltinStream ? "scr_dyn_builtin_stream_unbox" - : // for await over a checked-dynamic value iterates a native - // stream behind it (scr_dyn_for_await_stream.c) - e.fn === "dyn.asyncIteratorOpen" && host.dynForAwaitStreams - ? "scr_stream_dyn_async_iterator_open" - : LIB_FN_SYMS[e.fn]); + : e.fn === "stream.unboxDynStage" && host.usesDynBuiltinStream + ? "scr_dyn_builtin_stream_unbox_stage" + : // for await over a checked-dynamic value iterates a native + // stream behind it (scr_dyn_for_await_stream.c) + e.fn === "dyn.asyncIteratorOpen" && host.dynForAwaitStreams + ? "scr_stream_dyn_async_iterator_open" + : LIB_FN_SYMS[e.fn]); if (sym === undefined) throw new LlvmUnsupportedError(`libCall:${e.fn}`, e.loc); // The runtime's Date functions take and answer time values: a Date // operand passes its time value, read at once from the borrowed object, @@ -777,7 +780,8 @@ function emitSymbolLibCall( return { name: "", type: e.type }; } const t = B.tmp(); - B.line(`${t} = call ${retDecl} @${sym}(${argList})`); + const attrs = PURE_MATH_LIB_FNS.has(e.fn) ? " nounwind willreturn memory(none)" : ""; + B.line(`${t} = call ${retDecl} @${sym}(${argList})${attrs}`); // The result joins its frame BEFORE the pending check so an unwind // releases the dummy (NULL for refcounted returns) harmlessly. const out = host.own({ name: t, type: e.type }); @@ -1196,7 +1200,7 @@ function allTupleAdapterFor(host: LlvmEmitterContext, fnName: string, tuple: IrT const key = `allTuple:${fnName}`; let adapter = host.resolveThunks.get(key); if (adapter !== undefined) return adapter; - adapter = `sc_all_tuple_${host.resolveThunks.size}`; + adapter = host.contentSymbol("sc_all_tuple_", key); host.resolveThunks.set(key, adapter); host.journal?.allocated(adapter, "allTupleAdapterFor", [fnName, tuple]); const rc = vAdapters(host.shapeHost, tuple); diff --git a/packages/compiler/src/backend/llvm/lib-filesystem.ts b/packages/compiler/src/backend/llvm/lib-filesystem.ts index f0b766936..9174f4775 100644 --- a/packages/compiler/src/backend/llvm/lib-filesystem.ts +++ b/packages/compiler/src/backend/llvm/lib-filesystem.ts @@ -1027,6 +1027,54 @@ function emitPowerOfTwo(host: LlvmEmitterContext, n: string): string { return out; } +/** `base ** exponent` and `Math.pow(base, exponent)` as V8's math::pow + * answers them: a literal exponent 2 is `base * base`, a literal 0.5 is + * `sqrt(base + 0)` (+Infinity for an infinite base), and every other + * exponent calls scr_math_pow, which applies V8's other special cases + * before the platform pow. The platform pow is never called by its libm + * name, which LLVM would rewrite (pow(2, x) to exp2(x), pow(x, -1) to 1/x) + * into answers that can differ from V8's. */ +export function emitPowF64( + host: LlvmEmitterContext, + base: string, + exponent: string, + exponentExpr: IrExpr, +): string { + const B = host.B; + const literal = + exponentExpr.kind === "numLit" + ? exponentExpr.value + : exponentExpr.kind === "unary" && + exponentExpr.op === "-" && + exponentExpr.operand.kind === "numLit" + ? -exponentExpr.operand.value + : null; + const out = B.tmp(); + if (literal === 2) { + B.line(`${out} = fmul double ${base}, ${base}`); + return out; + } + if (literal === 0.5) { + host.declare(`declare double @llvm.sqrt.f64(double)`); + host.declare(`declare double @llvm.fabs.f64(double)`); + const plus = B.tmp(), + root = B.tmp(), + magnitude = B.tmp(), + infinite = B.tmp(); + B.line(`${plus} = fadd double ${base}, ${f64Lit(0)}`); + B.line(`${root} = call double @llvm.sqrt.f64(double ${plus})`); + B.line(`${magnitude} = call double @llvm.fabs.f64(double ${base})`); + B.line(`${infinite} = fcmp oeq double ${magnitude}, 0x7FF0000000000000`); + B.line(`${out} = select i1 ${infinite}, double 0x7FF0000000000000, double ${root}`); + return out; + } + host.declare(`declare double @scr_math_pow(double, double)`); + B.line( + `${out} = call double @scr_math_pow(double ${base}, double ${exponent}) nounwind willreturn`, + ); + return out; +} + /** `Math.pow(2, n)`, the divisor emitPowerOfTwoQuotient takes. */ export function isPowerOfTwoCall(e: IrExpr): boolean { return ( @@ -1099,6 +1147,11 @@ export function emitPrimitiveLibCall(host: LlvmEmitterContext, e: LibCallExpr): const v = host.emitExpr(e.args[1]!); return { name: emitPowerOfTwo(host, v.name), type: e.type }; } + if (e.fn === "math.pow") { + const base = host.emitExpr(e.args[0]!); + const exponent = host.emitExpr(e.args[1]!); + return { name: emitPowF64(host, base.name, exponent.name, e.args[1]!), type: e.type }; + } if (e.fn === "math.floor" || e.fn === "math.ceil") { const intr = e.fn === "math.floor" ? "floor" : "ceil"; const v = host.emitExpr(e.args[0]!); diff --git a/packages/compiler/src/backend/llvm/lib-http2.ts b/packages/compiler/src/backend/llvm/lib-http2.ts index d750fb7c5..038e98dc4 100644 --- a/packages/compiler/src/backend/llvm/lib-http2.ts +++ b/packages/compiler/src/backend/llvm/lib-http2.ts @@ -22,7 +22,7 @@ function sniAnswer(host: LlvmEmitterContext, type: IrType): string { const key = "sni:" + typeKey(type); const previous = host.resolveThunks.get(key); if (previous !== undefined) return previous; - const symbol = `sc_sni_answer_${host.resolveThunks.size}`; + const symbol = host.contentSymbol("sc_sni_answer_", key); host.resolveThunks.set(key, symbol); host.journal?.allocated(symbol, "sniAnswer", [type]); const params = ["ptr %self"], @@ -95,11 +95,12 @@ export function sniCallbackArgs(host: LlvmEmitterContext, cb: LlValue): AbiArgum let type = cb.type, value = cb.name; if (type.kind === "union") { - const arms = host.unionsById.get(type.unionId)?.arms ?? []; + const unionId = type.unionId; + const arms = host.unionsById.get(unionId)?.arms ?? []; const tag = arms.findIndex((arm) => arm.kind === "func"); if (tag < 0) throw new InternalCompilerError("SNI callback union lacks its function arm"); type = arms[tag]!; - value = host.unwrapNullableClosure(value, tag); + value = host.unwrapNullableClosure(value, unionId, tag); } else host.toHeap(cb); if (type.kind !== "func" || type.params[1] === undefined) throw new InternalCompilerError("SNI callback lacks its answer function"); diff --git a/packages/compiler/src/backend/llvm/lib-lookup.ts b/packages/compiler/src/backend/llvm/lib-lookup.ts index 56e94461f..620833819 100644 --- a/packages/compiler/src/backend/llvm/lib-lookup.ts +++ b/packages/compiler/src/backend/llvm/lib-lookup.ts @@ -37,7 +37,7 @@ function lookupAnswer(host: LlvmEmitterContext, answer: IrType & { kind: "func" const key = "lookup:" + typeKey(answer); let symbol = host.resolveThunks.get(key); if (symbol === undefined) { - symbol = `sc_lookup_answer_${host.resolveThunks.size}`; + symbol = host.contentSymbol("sc_lookup_answer_", key); host.resolveThunks.set(key, symbol); host.journal?.allocated(symbol, "lookupAnswer", [answer]); for (const declaration of [ diff --git a/packages/compiler/src/backend/llvm/lib-network.ts b/packages/compiler/src/backend/llvm/lib-network.ts index 35eb885cd..63661ab4c 100644 --- a/packages/compiler/src/backend/llvm/lib-network.ts +++ b/packages/compiler/src/backend/llvm/lib-network.ts @@ -165,7 +165,7 @@ export function emitNetworkHttpLibCall(host: LlvmEmitterContext, e: LibCallExpr) const funcTag = def ? def.arms.findIndex((a) => a.kind === "func") : -1; if (funcTag < 0) throw new InternalCompilerError(`llvm emitter bug: ${e.fn} union lacks its func arm`); - cb = host.unwrapNullableClosure(args[cbIndex]!.name, funcTag); + cb = host.unwrapNullableClosure(args[cbIndex]!.name, cbT.unionId, funcTag); } } const valueCount = reusePort ? 5 : 4; diff --git a/packages/compiler/src/backend/llvm/lib-process.ts b/packages/compiler/src/backend/llvm/lib-process.ts index 75d8038fc..bdb1fba27 100644 --- a/packages/compiler/src/backend/llvm/lib-process.ts +++ b/packages/compiler/src/backend/llvm/lib-process.ts @@ -497,7 +497,7 @@ function boundCallbackThunk(host: LlvmEmitterContext, cbT: IrType & { kind: "fun const key = `alsbound:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_alsbound_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_alsbound_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "boundCallbackThunk", [cbT]); host.declare("declare ptr @scr_als_bound_target(ptr)"); diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 4b3f34b80..182ac49ab 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -35,6 +35,31 @@ export const ASSERT_VERDICT_LIB_FNS: ReadonlyMap = new Ma ["assert.partialDeepResult", null], ]); +/** The Math functions that call scr_ieee754.c. They read and write no + * memory, so calls with the same argument fold and loops may hoist them. */ +export const PURE_MATH_LIB_FNS: ReadonlySet = new Set([ + "math.sin", + "math.sinh", + "math.cos", + "math.cosh", + "math.tan", + "math.tanh", + "math.asin", + "math.asinh", + "math.acos", + "math.acosh", + "math.atan", + "math.atanh", + "math.cbrt", + "math.exp", + "math.expm1", + "math.log", + "math.log1p", + "math.log2", + "math.log10", + "math.atan2", +]); + export const LIB_FN_SYMS: Record = { "assert.expectsErrDyn": "scr_assert_expects_err_dyn", "assert.throwsValidate": "scr_assert_throws_validate", @@ -118,27 +143,29 @@ export const LIB_FN_SYMS: Record = { "str.codePoints": "scr_str_code_points", "num.fromString": "scr_string_to_number", "math.round": "scr_math_round", - "math.sin": "sin", - "math.sinh": "sinh", - "math.cos": "cos", - "math.cosh": "cosh", - "math.tan": "tan", - "math.tanh": "tanh", - "math.asin": "asin", - "math.asinh": "asinh", - "math.acos": "acos", - "math.acosh": "acosh", - "math.atan": "atan", - "math.atanh": "atanh", - "math.cbrt": "cbrt", - "math.exp": "exp", - "math.expm1": "expm1", + // Math's transcendental functions are V8's (scr_ieee754.c), so their + // results are the ones Node prints. + "math.sin": "scr_ieee754_sin", + "math.sinh": "scr_ieee754_sinh", + "math.cos": "scr_ieee754_cos", + "math.cosh": "scr_ieee754_cosh", + "math.tan": "scr_ieee754_tan", + "math.tanh": "scr_ieee754_tanh", + "math.asin": "scr_ieee754_asin", + "math.asinh": "scr_ieee754_asinh", + "math.acos": "scr_ieee754_acos", + "math.acosh": "scr_ieee754_acosh", + "math.atan": "scr_ieee754_atan", + "math.atanh": "scr_ieee754_atanh", + "math.cbrt": "scr_ieee754_cbrt", + "math.exp": "scr_ieee754_exp", + "math.expm1": "scr_ieee754_expm1", "math.sqrt": "sqrt", - "math.log": "log", - "math.log1p": "log1p", - "math.log2": "log2", - "math.log10": "log10", - "math.atan2": "atan2", + "math.log": "scr_ieee754_log", + "math.log1p": "scr_ieee754_log1p", + "math.log2": "scr_ieee754_log2", + "math.log10": "scr_ieee754_log10", + "math.atan2": "scr_ieee754_atan2", "math.pow": "scr_math_pow", // encodeURIComponent throws URIError on a lone surrogate // (MAY_THROW_LIB_FNS); decodeUriComponent is with the throwing tier below. @@ -286,6 +313,8 @@ export const LIB_FN_SYMS: Record = { "module.id": "scr_module_id", "module.path": "scr_module_path", "module.paths": "scr_module_paths", + "module.requireComputed": "scr_require_computed", + "module.resolveComputed": "scr_require_resolve", "module.children": "scr_module_children", "module.parent": "scr_module_parent", "module.loaded": "scr_module_loaded", @@ -394,6 +423,7 @@ export const LIB_FN_SYMS: Record = { "fs.rmRetrySync": "scr_fs_rm_opts_retry", "fs.rmdirSync": "scr_fs_rmdir", "fs.readdirSync": "scr_fs_readdir", + "fs.readdirRecursiveSync": "scr_fs_readdir_recursive", "fs.globSync": "scr_fs_glob_strings", "fs.callbackGlob": "scr_fs_glob_callback", "fsp.globStrings": "scr_fsp_glob_strings", @@ -818,6 +848,9 @@ export const LIB_FN_SYMS: Record = { "stream.stdioWritable": "scr_stream_stdio_writable", "stream.unboxDyn": "scr_stream_unbox_dyn", "stream.dynIterableSource": "scr_stream_dyn_iterable_source", + "stream.dynSourceValue": "scr_stream_dyn_source_value", + "stream.unboxDynStage": "scr_stream_unbox_dyn_stage", + "stream.dynTransformResult": "scr_stream_dyn_transform_result", "stream.handleStream": "scr_stream_handle", "stream.pipeDyn": "scr_stream_pipe_dyn", "stream.unpipeDyn": "scr_stream_unpipe_dyn", @@ -1683,6 +1716,7 @@ export const LIB_FN_SYMS: Record = { "dyn.objEntries": "scr_dyn_obj_entries", "dyn.structuredClone": "scr_structured_clone", "dyn.declAdopt": "scr_dyn_decl_adopt", + "dyn.adoptShared": "scr_dyn_adopt_shared", "dyn.cloneMissing": "scr_structured_clone_missing", "dyn.cloneTransferFail": "scr_structured_clone_transfer_fail", "dyn.cloneTransferFailWith": "scr_structured_clone_transfer_fail_with", diff --git a/packages/compiler/src/backend/llvm/lib-stream.ts b/packages/compiler/src/backend/llvm/lib-stream.ts index fdbf13d4f..53f455964 100644 --- a/packages/compiler/src/backend/llvm/lib-stream.ts +++ b/packages/compiler/src/backend/llvm/lib-stream.ts @@ -743,7 +743,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV const helpers = packed(5 + n + k, helperCount); let at = 5 + n + k + helperCount; const take = (present: boolean): string => (present ? args[at++]!.name : "null"); - const box = take(((flags >> 1) & 3) === 2); + const box = take(((flags >> 1) & 3) >= 2); const signal = take((flags & 32) !== 0); const abort = take((flags & 32) !== 0); const head = diff --git a/packages/compiler/src/backend/llvm/map-read-lifetimes.ts b/packages/compiler/src/backend/llvm/map-read-lifetimes.ts index ae741463d..fbb21dcd2 100644 --- a/packages/compiler/src/backend/llvm/map-read-lifetimes.ts +++ b/packages/compiler/src/backend/llvm/map-read-lifetimes.ts @@ -12,13 +12,18 @@ import type { CallLifetimes } from "./call-lifetimes.js"; import type { ReferenceEffects } from "./reference-effects.js"; import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; import { + emitCompactMapSlot, emitNumberMapGetRef, emitNumberMapGetScalar, + emitNumberMapSlot, + emitStringMapEntry, emitStringMapGetScalar, emitStringMapGetRef, compactMapValues, noteNumberMapKey, } from "./map-number-lookup.js"; +import { InternalCompilerError } from "../../errors.js"; +import { undefinedArmTag } from "../../ir/analysis.js"; import { mapKeyAccess, mapKeyParamType } from "./shapes.js"; import { rememberMapEntry } from "./map-entry-fusion.js"; @@ -490,3 +495,62 @@ export function emitStackMapRead(host: LlvmEmitterContext, read: LocalMapRead): B.line(`store i32 ${selected}, ptr ${tag}`); return { value: { name: box, type: read.type }, owner }; } + +/** A string-keyed Map.get whose result is a nullable-pointer union over the + * map's own value type (`Map` read as `F | undefined`): a hit is + * the stored pointer itself and a miss the immortal undefined encoding, so + * the read can lend the stored value (emitBorrowedMapGet). */ +export function pointerMapGet(host: LlvmEmitterContext, e: IrExpr): boolean { + if ( + e.kind !== "mapIntrinsic" || + e.method !== "get" || + e.args.length !== 1 || + e.type.kind !== "union" || + e.receiver.type.kind !== "map" || + e.receiver.type.key.kind !== "string" || + !borrowsMapReadInputs(e) + ) + return false; + const nullable = host.nullableUnions.single(e.type.unionId); + return ( + nullable !== null && + typeEquals(nullable.arm, e.receiver.type.value) && + undefinedArmTag(e.type, host.unionsById) >= 0 + ); +} + +/** A pointerMapGet read without a reference: the map (and through it the + * stored value) must stay unchanged and alive until the consumer is done + * with the pointer (canBorrowReceiver). */ +export function emitBorrowedMapGet( + host: LlvmEmitterContext, + e: IrExpr & { kind: "mapIntrinsic" }, +): LlValue { + const B = host.B; + const mapType = e.receiver.type; + if (e.type.kind !== "union" || mapType.kind !== "map") + throw new InternalCompilerError("llvm emitter bug: borrowed Map.get shape"); + const receiver = host.emitReadReceiver(e.receiver); + const keyExpr = e.args[0]!; + const key = host.canBorrowReceiver(keyExpr) + ? host.emitReadReceiver(keyExpr) + : host.emitExpr(keyExpr); + const compact = compactMapValues(host, mapType.key, mapType.value); + const entry = emitStringMapEntry(host, receiver.name, key.name, compact); + const found = B.tmp(); + B.line(`${found} = icmp ne ptr ${entry}, null`); + const hit = B.newLabel("mapb.hit"), + miss = B.newLabel("mapb.miss"), + join = B.newLabel("mapb.join"); + B.condBr(found, hit, miss); + B.startBlock(hit); + const value = compact ? emitCompactMapSlot(host, entry) : emitNumberMapSlot(host, entry, "ptr"); + B.br(join); + B.startBlock(miss); + B.br(join); + B.startBlock(join); + const absent = host.unitInstanceRef(e.type.unionId, undefinedArmTag(e.type, host.unionsById)); + const result = B.tmp(); + B.line(`${result} = phi ptr [ ${value}, %${hit} ], [ ${absent}, %${miss} ]`); + return { name: result, type: e.type }; +} diff --git a/packages/compiler/src/backend/llvm/nullable-unions.test.ts b/packages/compiler/src/backend/llvm/nullable-unions.test.ts index bf0941652..f4e6a6866 100644 --- a/packages/compiler/src/backend/llvm/nullable-unions.test.ts +++ b/packages/compiler/src/backend/llvm/nullable-unions.test.ts @@ -7,7 +7,7 @@ import { type IrType, type IrUnionDef, } from "../../ir/ir.js"; -import { NullableUnions } from "./nullable-unions.js"; +import { isObjectArm, isPointerIdentityArm, NullableUnions } from "./nullable-unions.js"; const loc = { file: "nullable-unions.ts", start: 0, end: 0 }; const a: IrType = { kind: "record", shapeId: "a" }; @@ -111,3 +111,22 @@ test("library, lifetime-region and debug builds keep record unions boxed", () => expect(new NullableUnions(module(unions, { lifetimeRegions: true })).get("ab")).toBeNull(); expect(new NullableUnions(module(unions)).shapes("ab")).not.toBeNull(); }); + +test("function arms are nullable pointers compared by identity root", () => { + const fn: IrType = { kind: "func", params: [F64], ret: F64 }; + const unions = new NullableUnions( + module([ + { id: "fu", arms: [fn, UNDEFINED_T] }, + { id: "fnu", arms: [NULL_T, fn, UNDEFINED_T] }, + { id: "ff", arms: [fn, F64] }, + ]), + ); + expect(unions.single("fu")).toMatchObject({ refTag: 0, unitTag: 1, nullTag: -1, arm: fn }); + expect(unions.single("fnu")).toMatchObject({ refTag: 1, unitTag: 2, nullTag: 0 }); + expect(unions.get("ff")).toBeNull(); + const single = unions.single("fu")!; + // Always truthy, but `===` compares adapters by the function they adapt. + expect(isObjectArm(single)).toBe(true); + expect(isPointerIdentityArm(single)).toBe(false); + expect(unions.armTypes().get("fu")).toEqual(fn); +}); diff --git a/packages/compiler/src/backend/llvm/nullable-unions.ts b/packages/compiler/src/backend/llvm/nullable-unions.ts index c51f3b5f6..eedb922b3 100644 --- a/packages/compiler/src/backend/llvm/nullable-unions.ts +++ b/packages/compiler/src/backend/llvm/nullable-unions.ts @@ -89,21 +89,30 @@ export const NULLABLE_ABSENT = "@sc_nullable_absent"; /** The reference arms whose instances are never NULL. Class and record * entry points all tolerate NULL; strings, Stats snapshots, arrays, maps and - * sets retain through a NULL-skipping wrapper (shapes.ts). Object arms are - * always truthy and compare by identity; string arms are truthy when - * non-empty and compare bytes. */ + * sets retain through a NULL-skipping wrapper (shapes.ts), as do closures + * (function arms). Object arms are always truthy and compare by identity + * (function arms by identity root); string arms are truthy when non-empty + * and compare bytes. */ export type NullableRefArm = | { kind: "object"; className: string } | { kind: "record"; shapeId: string } | { kind: "string" } | { kind: "stats" } - | (IrType & { kind: "array" | "map" | "set" }); + | (IrType & { kind: "array" | "map" | "set" | "func" }); -/** The arm is a JS object: always truthy, compared by identity. */ +/** The arm is a JS object: always truthy. */ export function isObjectArm(nullable: PointerUnion): boolean { return nullable.repr === "shapes" || nullable.arm.kind !== "string"; } +/** The arm compares by pointer identity. Function arms compare identity + * roots instead: a signature adapter is the function it adapts. */ +export function isPointerIdentityArm(nullable: PointerUnion): boolean { + return ( + nullable.repr === "shapes" || (nullable.arm.kind !== "string" && nullable.arm.kind !== "func") + ); +} + /** The representation decision for every union of one module. It is a * pure function of the module, so serialized-IR and in-memory emission * agree. */ @@ -264,7 +273,8 @@ function nullableShape( return { ...base, arm: { kind: "record", shapeId: arm.shapeId } }; if (arm.kind === "string") return { ...base, arm: { kind: "string" } }; if (arm.kind === "stats") return { ...base, arm: { kind: "stats" } }; - if (arm.kind === "array" || arm.kind === "map" || arm.kind === "set") return { ...base, arm }; + if (arm.kind === "array" || arm.kind === "map" || arm.kind === "set" || arm.kind === "func") + return { ...base, arm }; return null; } diff --git a/packages/compiler/src/backend/llvm/receivers.ts b/packages/compiler/src/backend/llvm/receivers.ts index 2fe9dd66b..8524a51fb 100644 --- a/packages/compiler/src/backend/llvm/receivers.ts +++ b/packages/compiler/src/backend/llvm/receivers.ts @@ -11,14 +11,24 @@ import type { LlValue } from "./expr-context.js"; import { guardedValue } from "./borrowed-receivers.js"; import { InternalCompilerError } from "../../errors.js"; import { emitBorrowedArrayRead, emitProjectedArrayRead } from "./local-array-reads.js"; -import { emitStackMapRead, matchMapRead } from "./map-read-lifetimes.js"; +import { + emitBorrowedMapGet, + emitStackMapRead, + matchMapRead, + pointerMapGet, +} from "./map-read-lifetimes.js"; import { emitBorrowedResultCall, emitCallExpr } from "./expr-calls.js"; -import { emitCheckedNarrowReceiver } from "./checked-narrows.js"; +import { borrowsCheckedNarrow, emitCheckedNarrowReceiver } from "./checked-narrows.js"; import { canStackUnion, emitStackUnion } from "./stack-unions.js"; import { emitDowncastGuard, emitUnionNarrowGuard } from "./narrow-guards.js"; import { emitScalarOptional, isScalarOptionalSource } from "./local-union-storage.js"; import { borrowableInputs } from "./borrowed-inputs.js"; -import { emitStackRecordKeyGet, scalarRecordKeyGet } from "./expr-records.js"; +import { + emitBorrowedRecordKeyGet, + emitStackRecordKeyGet, + pointerRecordKeyGet, + scalarRecordKeyGet, +} from "./expr-records.js"; import { type WalkBorrowHost, analyzeWalks } from "./walk-borrows.js"; /** Read a receiver whose value is consumed immediately by a field or tag @@ -92,6 +102,10 @@ export function emitReadReceiver(emitter: LlEmitter, e: IrExpr): LlValue { return emitBorrowedResultCall(emitter, e); const arrayRead = emitter.optionalArrayReads.get(e); if (arrayRead) return emitProjectedArrayRead(emitter, arrayRead); + if (e.kind === "recordKeyGet" && emitter.canBorrowReceiver(e)) + return emitBorrowedRecordKeyGet(emitter, e); + if (e.kind === "mapIntrinsic" && emitter.canBorrowReceiver(e)) + return emitBorrowedMapGet(emitter, e); const narrow = emitter.checkedNarrows.get(e); const narrowed = narrow ? emitCheckedNarrowReceiver(emitter, narrow) : null; if (narrowed) return narrowed; @@ -294,7 +308,27 @@ export function canBorrowReceiver(emitter: LlEmitter, e: IrExpr): boolean { // snapshot, so neither the callee nor a later argument may replace // what it was read from: a callee that preserves every edge cannot, // and any other one must leave the argument's locations unwritten. + // A keyed read lends the stored pointer while the record keeps it. + case "recordKeyGet": + return ( + pointerRecordKeyGet(emitter, e) && + emitter.canBorrowReceiver(e.obj) && + emitter.referenceEffects.preserves(e.key) + ); + // So does a Map.get while the map keeps it. + case "mapIntrinsic": + return ( + pointerMapGet(emitter, e) && + emitter.canBorrowReceiver(e.receiver) && + emitter.referenceEffects.preserves(e.args[0]!) + ); case "call": { + // A checked extraction lends the payload of the union it narrows + // (emitCheckedNarrowReceiver). + if (borrowsCheckedNarrow(emitter, e)) { + const narrow = emitter.checkedNarrows.get(e)!; + return emitter.canBorrowReceiver(narrow.value); + } if (!emitter.borrowedReturns.has(e.callee)) return false; if (!e.args.every((arg) => emitter.referenceEffects.preserves(arg))) return false; const preserving = emitter.referenceEffects.functions.has(e.callee); diff --git a/packages/compiler/src/backend/llvm/reference-writes.test.ts b/packages/compiler/src/backend/llvm/reference-writes.test.ts index cbbe49d3b..db575babe 100644 --- a/packages/compiler/src/backend/llvm/reference-writes.test.ts +++ b/packages/compiler/src/backend/llvm/reference-writes.test.ts @@ -97,3 +97,46 @@ test("an array that is read back or escapes keeps its element writes", () => { const input = fill("input", [store("input", element("input"))]); expect(writes(input).conflicts(["a"], writes(input).summary("input"))).toBe(true); }); + +test("a keyed record read depends on every field and keyed entry of records", () => { + const handler: IrType = { kind: "func", params: [F64], ret: F64 }; + const table: IrType = { kind: "record", shapeId: "t" }; + const key: IrExpr = { kind: "strLit", value: "k", type: { kind: "string" }, loc }; + const read: IrExpr = { + kind: "recordKeyGet", + obj: ref("table", table), + shapeId: "t", + key, + type: { kind: "union", unionId: "hu" }, + loc, + }; + const keyedWrite: IrStmt = { + kind: "recordKeySet", + obj: ref("table", table), + shapeId: "t", + key, + value: ref("h", handler), + loc, + }; + const fn = (name: string, body: IrStmt[]): IrFunction => ({ + name, + loc, + returnType: F64, + params: [{ localId: "table", name: "table", type: table }], + locals: [ + { id: "table", name: "table", type: table, mutable: false }, + { id: "h", name: "h", type: handler, mutable: false }, + ], + body, + }); + const records = [{ id: "t", fields: [{ name: "add", type: handler }], indexValue: handler }]; + const quiet = fn("quiet", []); + const writer = fn("writer", [keyedWrite]); + const functions = new Map([quiet, writer].map((f) => [f.name, f])); + const rw = new ReferenceWrites(functions, [], () => false, new Set(), { records }); + rw.enter(quiet); + const keys = rw.readKeys(read); + expect(keys).toEqual(expect.arrayContaining(["r:*", "r:add"])); + expect(rw.conflicts(keys!, rw.summary("quiet"))).toBe(false); + expect(rw.conflicts(keys!, rw.summary("writer"))).toBe(true); +}); diff --git a/packages/compiler/src/backend/llvm/reference-writes.ts b/packages/compiler/src/backend/llvm/reference-writes.ts index c37af81d4..79ac98a7d 100644 --- a/packages/compiler/src/backend/llvm/reference-writes.ts +++ b/packages/compiler/src/backend/llvm/reference-writes.ts @@ -92,6 +92,8 @@ const INERT_LIB_PREFIXES = [ ]; const INERT_LIB_CALLS = new Set([ "dyn.compare", + // Reads the ambient receiver (a fresh reference to the bound value). + "dyn.this", "error.new", "error.nodeThrow", "dyn.typedRefIs", @@ -923,6 +925,17 @@ export class ReferenceWrites { case "arrayGet": out.push(ARRAY_KEY); return this.expr(node.index) === null && walk(node.arr); + case "recordKeyGet": + // Any declared field or overflow entry the key may name. + out.push(ANY_RECORD_KEY); + for (const f of this.records.get(node.shapeId)?.fields ?? []) out.push(`r:${f.name}`); + return this.expr(node.key) === null && walk(node.obj); + case "mapIntrinsic": + // A typed Map.get: any entry of any map. + if (node.method !== "get" || node.args.length !== 1 || !borrowsMapReadInputs(node)) + return false; + out.push(MAP_KEY); + return this.expr(node.args[0]!) === null && walk(node.receiver); case "unionNarrow": case "upcast": case "downcast": diff --git a/packages/compiler/src/backend/llvm/shape-union-tags.test.ts b/packages/compiler/src/backend/llvm/shape-union-tags.test.ts new file mode 100644 index 000000000..e07626e77 --- /dev/null +++ b/packages/compiler/src/backend/llvm/shape-union-tags.test.ts @@ -0,0 +1,80 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../../index.js"; +import { validateModule } from "../../ir/validate.js"; +import { emitLlvmModule } from "./emitter.js"; + +async function llvmFor(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-shape-union-tags-")); + try { + const entry = join(dir, "main.ts"), + output = join(dir, "main.ir.json"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath: output, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((item) => `${item.code}: ${item.message}`).join("\n")); + const module = deserializeModule(await readFile(output, "utf8")); + expect(validateModule(module)).toEqual([]); + return emitLlvmModule(module); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +function tagHelpers(ir: string): string[] { + return [...ir.matchAll(/^define internal i32 @sc_stag\.[^\n]*\{[^]*?^}/gm)].map( + (match) => match[0], + ); +} + +test("a union whose arms span few shape numbers switches on the shape word", async () => { + const ir = await llvmFor(` +type Num = { kind: "num"; value: number }; +type Neg = { kind: "neg"; operand: Node }; +type Add = { kind: "add"; left: Node; right: Node }; +type Node = Num | Neg | Add; +function evaluate(node: Node): number { + switch (node.kind) { + case "num": return node.value; + case "neg": return -evaluate(node.operand); + case "add": return evaluate(node.left) + evaluate(node.right); + } +} +const one: Node = { kind: "num", value: 1 }; +console.log(evaluate({ kind: "add", left: one, right: { kind: "neg", operand: one } })); +`); + expect(ir).not.toMatch(/@sc_stagtab\./); + const helpers = tagHelpers(ir); + expect(helpers.length).toBeGreaterThan(0); + for (const helper of helpers) expect(helper).toMatch(/switch i32 %w/); +}); + +test("a union whose arms span many shape numbers reads the arm from its table", async () => { + const arms = Array.from({ length: 40 }, (_, i) => i); + const ir = await llvmFor(` +${arms.map((i) => `type K${i} = { kind: "k${i}"; v${i}: number };`).join("\n")} +type Any = ${arms.map((i) => `K${i}`).join(" | ")}; +function value(item: Any): number { + switch (item.kind) { +${arms.map((i) => ` case "k${i}": return item.v${i};`).join("\n")} + } +} +const items: Any[] = [${arms.map((i) => `{ kind: "k${i}", v${i}: ${i} }`).join(", ")}]; +let total = 0; +for (const item of items) total += value(item); +console.log(total); +`); + // One byte per shape number from the arms' lowest to highest; 255 names no arm. + const table = + /^(@sc_stagtab\.\S+) = private unnamed_addr constant \[(\d+) x i8\] \[([^\]]*)\]$/m.exec(ir); + expect(table, "arm table").not.toBeNull(); + const entries = table![3]!.split(", ").map((entry) => Number(entry.replace("i8 ", ""))); + expect(entries).toHaveLength(Number(table![2])); + expect(entries.filter((entry) => entry !== 255).sort((a, b) => a - b)).toEqual(arms); + const helper = tagHelpers(ir).find((text) => text.includes(table![1]!)); + expect(helper, "tag helper").toBeDefined(); + expect(helper).toContain(`getelementptr inbounds [${table![2]} x i8], ptr ${table![1]}`); + expect(helper).not.toMatch(/switch i32 %w/); +}); diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 6d03db222..79a79f36c 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -313,19 +313,46 @@ function shapeUnionRc(host: ShapeHost, union: ShapeUnion): { retain: string; rel }; } +/** The widest span of shape numbers a union's arms may cover for its tag + * helper to switch on the shape word (emitShapeUnionHelpers). */ +const SHAPE_TAG_SWITCH_SPAN = 32; + /** The tag, release and trace helpers of every shape union * (nullable-unions.ts). The tag helper maps a value to its arm: NULL and * the null sentinel are the unit arms, anything else a tagged record whose - * shape word names its arm (the switch folds to a lookup table). */ + * shape word names its arm. + * - Arms within SHAPE_TAG_SWITCH_SPAN shape numbers: the helper switches on + * the shape word. Inlined, the switch folds into the caller's branch on + * the arm, one indirect jump with a jump table of at most that many + * entries per site. + * - Wider unions: the shape word indexes the union's arm table, one entry + * per shape number from the arms' lowest to highest, and callers branch + * on the small arm tag. Folded switches would give every site of such a + * union a jump table spanning all of its arms' shape numbers. */ export function emitShapeUnionHelpers(host: ShapeHost): string[] { const out: string[] = []; const S = host.sizeType; for (const union of host.nullableUnions.allShapeUnions()) { const header = shapeWordHeader(S); const tagFn = shapeTagFn(union.unionId); - const cases = union.arms - .map((arm) => `i32 ${host.nullableUnions.shapeNumber(arm.shapeId)}, label %a${arm.tag}`) - .join(" "); + const numbers = union.arms.map((arm) => host.nullableUnions.shapeNumber(arm.shapeId)); + const lo = Math.min(...numbers); + const span = Math.max(...numbers) - lo + 1; + const byTable = span > SHAPE_TAG_SWITCH_SPAN; + // Entries are bytes, or 16-bit words for a union of 255 arms or more. + // A shape number that names no arm maps to the all-ones entry, which is + // above every arm tag, so callers take their path for no arm as they do + // for the -1 of a shape number outside the table. + const wide = union.arms.some((arm) => arm.tag >= 255); + const entry = wide ? "i16" : "i8"; + const none = wide ? 65535 : 255; + const table = new Array(span).fill(none); + union.arms.forEach((arm, i) => (table[numbers[i]! - lo] = arm.tag)); + const tableName = tagFn.replace("@sc_stag.", "@sc_stagtab."); + if (byTable) + out.push( + `${tableName} = private unnamed_addr constant [${span} x ${entry}] [${table.map((t) => `${entry} ${t}`).join(", ")}]`, + ); out.push( `define internal i32 ${tagFn}(ptr %u) alwaysinline ${FN_ATTRS} { ; shape union tag`, `entry:`, @@ -343,16 +370,35 @@ export function emitShapeUnionHelpers(host: ShapeHost): string[] { ` ret i32 ${union.nullTag}`, `rec:`, ); - out.push( - ` %wp = getelementptr inbounds ${header}, ptr %u, i32 0, i32 ${SHAPE_WORD_INDEX}`, - ` %w = load i32, ptr %wp`, - ` switch i32 %w, label %bad [ ${cases} ]`, - ...union.arms.flatMap((arm) => [`a${arm.tag}:`, ` ret i32 ${arm.tag}`]), - `bad:`, - ` ret i32 -1`, - `}`, - ``, - ); + const cases = union.arms.map((arm, i) => `i32 ${numbers[i]}, label %a${arm.tag}`).join(" "); + if (!byTable) + out.push( + ` %wp = getelementptr inbounds ${header}, ptr %u, i32 0, i32 ${SHAPE_WORD_INDEX}`, + ` %w = load i32, ptr %wp`, + ` switch i32 %w, label %bad [ ${cases} ]`, + ...union.arms.flatMap((arm) => [`a${arm.tag}:`, ` ret i32 ${arm.tag}`]), + `bad:`, + ` ret i32 -1`, + `}`, + ``, + ); + else + out.push( + ` %wp = getelementptr inbounds ${header}, ptr %u, i32 0, i32 ${SHAPE_WORD_INDEX}`, + ` %w = load i32, ptr %wp`, + ` %i = sub i32 %w, ${lo}`, + ` %inrange = icmp ult i32 %i, ${span}`, + ` br i1 %inrange, label %look, label %bad`, + `look:`, + ` %tp = getelementptr inbounds [${span} x ${entry}], ptr ${tableName}, i32 0, i32 %i`, + ` %t8 = load ${entry}, ptr %tp`, + ` %tag = zext ${entry} %t8 to i32`, + ` ret i32 %tag`, + `bad:`, + ` ret i32 -1`, + `}`, + ``, + ); // Release and trace: one definition for the unions over the same // record arms (helperOwner). Unit encodings are NULL or the immortal // null sentinel; the shape word picks the arm. diff --git a/packages/compiler/src/backend/llvm/stack-callbacks.ts b/packages/compiler/src/backend/llvm/stack-callbacks.ts index 5c404896a..20de50372 100644 --- a/packages/compiler/src/backend/llvm/stack-callbacks.ts +++ b/packages/compiler/src/backend/llvm/stack-callbacks.ts @@ -32,10 +32,12 @@ export class StackCallbacks { const count = B.tmp(); B.line(`${count} = getelementptr inbounds %ScrClosure, ptr ${closure}, i32 0, i32 2`); B.line(`store ${host.sizeType} ${value.captures.length}, ptr ${count}`); - if (this.functions.get(value.fnName)?.ownsPrototype) { + const target = this.functions.get(value.fnName); + const functionKind = target ? host.closureFunctionKind(target) : 0; + if (functionKind !== 0) { const kind = B.tmp(); B.line(`${kind} = getelementptr inbounds %ScrClosure, ptr ${closure}, i32 0, i32 4`); - B.line(`store i32 4, ptr ${kind}`); + B.line(`store i32 ${functionKind}, ptr ${kind}`); } value.captures.forEach((localId, index) => { const box = host.retainBox(host.captureBox(localId)); diff --git a/packages/compiler/src/backend/llvm/unwind-abi.test.ts b/packages/compiler/src/backend/llvm/unwind-abi.test.ts index 5f7f5895b..bcef1c379 100644 --- a/packages/compiler/src/backend/llvm/unwind-abi.test.ts +++ b/packages/compiler/src/backend/llvm/unwind-abi.test.ts @@ -283,3 +283,43 @@ test("calls that cannot unwind are marked nounwind", () => { expect(out).toContain("%r = call ptr %b(ptr %b) nounwind"); expect(out).toContain(" call void @scr_unwind_raise()\n"); }); + +test("functions without a plain call of a raising function change only their call lines", () => { + const raising = [ + "define internal void @sc_bf_raises() #0 {", + "entry:", + " call void @scr_unwind_raise()", + " unreachable", + "}", + ].join("\n"); + const other = [ + "define internal void @sc_bf_raises.copy() #0 {", + "entry:", + " ret void", + "}", + ].join("\n"); + const plain = [ + "define internal double @sc_f_plain(ptr %p) #0 {", + "entry: ; call sites below", + " %a = call double @sc_f_other(ptr %p)", + " %b = tail call double @sc_f_other(ptr %p) ; mentions @ and @sc_bf_raises.copy", + " store ptr @sc_bf_raises.copy, ptr %p", + " ret double %a", + "}", + ].join("\n"); + const parts = [`${raising}\n${other}`, plain, "@table = internal constant ptr @sc_f_plain"]; + const out = bridgePlainUnwindCalls(parts); + expect(out[0]).toBe(parts[0]); + expect(out[1]).toBe( + plain + .replace("@sc_f_other(ptr %p)\n", "@sc_f_other(ptr %p) nounwind\n") + .replace("@sc_f_other(ptr %p) ;", "@sc_f_other(ptr %p) nounwind ;"), + ); + expect(out[2]).toBe(parts[2]); + expect(() => + bridgePlainUnwindCalls([ + raising, + plain.replace("store ptr @sc_bf_raises.copy", "store ptr @sc_bf_raises"), + ]), + ).toThrow(UnwindAbiViolation); +}); diff --git a/packages/compiler/src/backend/llvm/unwind-abi.ts b/packages/compiler/src/backend/llvm/unwind-abi.ts index 4dc611913..91722fb1d 100644 --- a/packages/compiler/src/backend/llvm/unwind-abi.ts +++ b/packages/compiler/src/backend/llvm/unwind-abi.ts @@ -301,6 +301,46 @@ function markNounwind(line: string): string { : `${line.slice(0, close + 1)} nounwind${line.slice(close + 1)}`; } +/** The functions of `part`: its text split before each `define` line + * (`part.split(/\n(?=define )/)`), found with indexOf. */ +function splitFunctions(part: string): string[] { + const chunks: string[] = []; + let start = 0; + for (let at = part.indexOf("\ndefine "); at >= 0; at = part.indexOf("\ndefine ", at + 1)) { + chunks.push(part.slice(start, at)); + start = at + 1; + } + chunks.push(part.slice(start)); + return chunks; +} + +/** Whether `code` can be part of a symbol name (`[\w.$%-]`). */ +function isNameCode(code: number): boolean { + return ( + (code >= 97 && code <= 122) || + (code >= 65 && code <= 90) || + (code >= 48 && code <= 57) || + code === 95 /* _ */ || + code === 46 /* . */ || + code === 36 /* $ */ || + code === 37 /* % */ || + code === 45 /* - */ + ); +} + +/** Calls `each` with every line of `chunk` that holds `call `, by its start + * and end offsets, once per line and in order. Lines without `call ` match + * neither CALL_LINE nor PLAIN_CALL. */ +function eachCallLine(chunk: string, each: (start: number, end: number) => void): void { + for (let at = chunk.indexOf("call "); at >= 0;) { + const start = chunk.lastIndexOf("\n", at) + 1; + let end = chunk.indexOf("\n", at); + if (end < 0) end = chunk.length; + each(start, end); + at = chunk.indexOf("call ", end); + } +} + /** The value a bridged call yields on its landing pad: the dummy a * pending-flag callee returns with an exception pending. */ function dummyValue(type: string): string { @@ -318,7 +358,7 @@ function dummyValue(type: string): string { * these functions that emitDirectCall lowered are invokes already. */ export function bridgePlainUnwindCalls(parts: string[]): string[] { const raisers = new Set(); - const split = parts.map((part) => part.split(/\n(?=define )/)); + const split = parts.map(splitFunctions); for (const chunks of split) for (const chunk of chunks) { const m = DEFINE_NAME.exec(chunk); @@ -359,20 +399,29 @@ export function bridgePlainUnwindCalls(parts: string[]): string[] { } } for (const name of invoked) if (!raisers.has(name) && testsFlag.has(name)) violations.add(name); + // Every mention of a raising function outside its `define` line, other + // than as the callee of a call or invoke, found with indexOf. for (const chunks of split) for (const chunk of chunks) - for (const line of chunk.split("\n")) { - if (line.startsWith("define ")) continue; - for (const m of line.matchAll(/@([\w.$%-]+)/g)) { - if (!raisers.has(m[1]!)) continue; - const at = m.index!; - if ( - /(?:call|invoke) [^@]*$/.test(line.slice(0, at)) && - line.charAt(at + m[0].length) === "(" - ) - continue; - violations.add(m[1]!); + for (let at = chunk.indexOf("@"); at >= 0;) { + let end = at + 1; + while (end < chunk.length && isNameCode(chunk.charCodeAt(end))) end++; + const next = chunk.indexOf("@", end); + if (end === at + 1) { + at = next; + continue; } + const name = chunk.slice(at + 1, end); + if (raisers.has(name)) { + const start = chunk.lastIndexOf("\n", at) + 1; + const line = chunk.slice(start, at); + if ( + !line.startsWith("define ") && + !(/(?:call|invoke) [^@]*$/.test(line) && chunk.charAt(end) === "(") + ) + violations.add(name); + } + at = next; } if (violations.size > 0) throw new UnwindAbiViolation(violations); let personality = false; @@ -381,6 +430,34 @@ export function bridgePlainUnwindCalls(parts: string[]): string[] { chunks .map((chunk) => { if (!DEFINE_NAME.test(chunk) || !chunk.includes("call ")) return chunk; + // A function without a plain call of a raising function only marks + // its calls nounwind: rewrite just its call lines. + let bridged = false; + const marks: { start: number; end: number; line: string }[] = []; + eachCallLine(chunk, (start, end) => { + if (bridged) return; + const line = chunk.slice(start, end); + const m = PLAIN_CALL.exec(line); + const callee = m === null ? undefined : (m[4] ?? m[7]); + if (m !== null && callee !== undefined && raisers.has(callee)) { + bridged = true; + return; + } + if (!CALL_LINE.test(line)) return; + const marked = markNounwind(line); + if (marked !== line) marks.push({ start, end, line: marked }); + }); + if (!bridged) { + if (marks.length === 0) return chunk; + const pieces: string[] = []; + let from = 0; + for (const mark of marks) { + pieces.push(chunk.slice(from, mark.start), mark.line); + from = mark.end; + } + pieces.push(chunk.slice(from)); + return pieces.join(""); + } const lines = chunk.split("\n"); const resume = resumers.has(DEFINE_NAME.exec(chunk)![1]!); const result: string[] = []; diff --git a/packages/compiler/src/backend/llvm/wrapping-sums.ts b/packages/compiler/src/backend/llvm/wrapping-sums.ts index 5b487d809..49b3dde8b 100644 --- a/packages/compiler/src/backend/llvm/wrapping-sums.ts +++ b/packages/compiler/src/backend/llvm/wrapping-sums.ts @@ -22,8 +22,9 @@ export function markWrappingOperands(host: LlvmEmitterContext, ...operands: IrEx /** The values an f64 expression can have when each one is an exact integer * (never -0) or NaN: proven ranges, safe integer literals, NaN-coded reads * of integer typed-array elements and of Int32[] elements, `charCodeAt`, - * and sums and differences of those whose every integer result is still - * exact in a double. */ + * and sums, differences and products of those whose every integer result + * is a safe integer, which the double holds exactly. A product that can be + * -0 (a zero times a negative) has no range. */ export function integerOrNaNRange(host: LlvmEmitterContext, e: IrExpr): IntegerRange | null { if (e.type.kind !== "f64") return null; const range = host.integerRanges.get(e); @@ -51,7 +52,7 @@ export function integerOrNaNRange(host: LlvmEmitterContext, e: IrExpr): IntegerR // A UTF-16 code unit, or NaN past either end. if (e.kind === "strIntrinsic" && e.method === "charCodeAt") return { min: 0, max: 65535 }; if (e.kind === "varRef") return host.integerOrNaNLocals.get(e.localId) ?? null; - if (e.kind === "bin" && (e.op === "+" || e.op === "-")) { + if (e.kind === "bin" && (e.op === "+" || e.op === "-" || e.op === "*")) { const left = integerOrNaNRange(host, e.left); const right = left && integerOrNaNRange(host, e.right); return right ? integerArithmeticRange(e.op, left!, right) : null; @@ -113,9 +114,14 @@ function wrappingOperand( return { low, valid: null }; } // The number is an exact integer or NaN, and its ToUint32 view is the - // integer's low 32 bits or 0. An ordered compare tells the two apart. - const valid = B.tmp(); - B.line(`${valid} = fcmp ord double ${value.name}, 0.0`); + // integer's low 32 bits or 0. Its ordered bit, or an ordered compare, + // tells the two apart. + let valid: string | null = value.ordered ?? null; + if (valid === "true") valid = null; + else if (valid === null) { + valid = B.tmp(); + B.line(`${valid} = fcmp ord double ${value.name}, 0.0`); + } if (value.uint32 !== undefined) return { low: value.uint32, valid }; // Without a view, convert once. The conversion of NaN is poison, but the // result only selects the sum when every operand is ordered. @@ -137,8 +143,9 @@ function wrappingOperand( * low halves is the converted exact result; a NaN operand makes the result * NaN, which converts to 0. A product is exact when the operands' magnitude * bounds multiply to at most 2^53; it may be -0 (`0 * -1`), which converts - * to 0 like the integer 0. Returns undefined, emitting nothing, for any - * other expression. */ + * to 0 like the integer 0. Returns the converted result with its ordered + * bit (`true` when no operand can be NaN), or undefined, emitting nothing, + * for any other expression. */ export function emitWrappingSum( host: LlvmEmitterContext, e: IrExpr & { kind: "bin" }, @@ -146,7 +153,7 @@ export function emitWrappingSum( r: LlValue, li: LlInteger | null, ri: LlInteger | null, -): string | undefined { +): { uint32: string; ordered: string } | undefined { if ((e.op !== "+" && e.op !== "-" && e.op !== "*") || !host.wrappingSums.has(e)) return undefined; const leftRange = li ? li.range : integerOrNaNRange(host, e.left); const rightRange = leftRange && (ri ? ri.range : integerOrNaNRange(host, e.right)); @@ -165,14 +172,14 @@ export function emitWrappingSum( `${sum} = ${e.op === "+" ? "add" : e.op === "-" ? "sub" : "mul"} i32 ${left.low}, ${right.low}`, ); let valid = left.valid ?? right.valid; - if (valid === null) return sum; + if (valid === null) return { uint32: sum, ordered: "true" }; if (left.valid && right.valid) { valid = B.tmp(); B.line(`${valid} = and i1 ${left.valid}, ${right.valid}`); } const out = B.tmp(); B.line(`${out} = select i1 ${valid}, i32 ${sum}, i32 0`); - return out; + return { uint32: out, ordered: valid }; } /** Whether every product of a value in `left` and one in `right` is an diff --git a/packages/compiler/src/backend/mangle.ts b/packages/compiler/src/backend/mangle.ts index 14736ead7..8542a1c64 100644 --- a/packages/compiler/src/backend/mangle.ts +++ b/packages/compiler/src/backend/mangle.ts @@ -221,15 +221,6 @@ export function mangleGenDrop(fnName: string): string { export function mangleGenSourceClose(fnName: string): string { return `sc_gc_${sanitize(fnName)}`; } -/** Emitted ref-kind resolve thunk, interned per inner-type key. */ -export function mangleResolveThunk(n: number): string { - return `sc_rslv_${n}`; -} -/** Emitted generator-resume result builder (the IteratorResult record), - * interned per generator-type key. */ -export function mangleGenResThunk(n: number): string { - return `sc_gres_${n}`; -} /** Emitted Promise.race fulfillment adapter (entry inner type → result * inner type), interned per type pair. */ export function mangleRaceThunk(n: number): string { diff --git a/packages/compiler/src/backend/native-codegen.test.ts b/packages/compiler/src/backend/native-codegen.test.ts index 5463b868e..64961bcaa 100644 --- a/packages/compiler/src/backend/native-codegen.test.ts +++ b/packages/compiler/src/backend/native-codegen.test.ts @@ -1,4 +1,4 @@ -import { chmod, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { afterEach, expect, test } from "vitest"; @@ -26,6 +26,7 @@ async function fakePackage( emptyOutput?: boolean; missingOutput?: boolean; changePackageDuringEmit?: boolean; + profileReport?: string; } = {}, ) { const root = await mkdtemp(join(tmpdir(), "scriptc-native-helper-test-")); @@ -72,6 +73,7 @@ ${ ? ":" : 'for each in $outputs; do cp "$input" "$each"; done' } +${options.profileReport === undefined ? "" : `printf '%s\\n' '${options.profileReport}'`} `, ); await chmod(bin, 0o755); @@ -196,6 +198,41 @@ test("profiles reach the helper and key the native cache by their bytes", async expect(calls[2]).toContain("--profile-use /profiles/a.profdata"); }); +test("a cached artifact keeps the profile match of the build that produced it", async () => { + const pkg = await fakePackage({ + profileReport: + '{"ok":true,"profile":{"annotated":120,"mismatched":7,"missing":40,"unknown_partitions":0}}', + }); + const used = { + ...request(pkg.root, pkg.packageJson, join(pkg.root, "used.o")), + profile: { use: { path: "/profiles/a.profdata", digest: "a".repeat(64) } }, + }; + const match = { annotated: 120, mismatched: 7, missing: 40, unknownPartitions: 0 }; + expect((await emitNativeArtifact(used)).profileMatch).toEqual(match); + const again = await emitNativeArtifact({ ...used, outputPath: join(pkg.root, "again.o") }); + expect(again.profileMatch).toEqual(match); + expect((await readFile(pkg.log, "utf8")).trim().split("\n")).toHaveLength(1); + // Without the cached match, the cached partition's counts are unknown. + const cache = join(pkg.root, "cache", "native-codegen-v1"); + const [shard] = await readdir(cache); + for (const name of await readdir(join(cache, shard!))) + if (name.includes(".profile-match")) await rm(join(cache, shard!, name)); + const unknown = await emitNativeArtifact({ ...used, outputPath: join(pkg.root, "unknown.o") }); + expect(unknown.profileMatch).toEqual({ + annotated: 0, + mismatched: 0, + missing: 0, + unknownPartitions: 1, + }); + // Builds without a profile carry no match, from the helper or the cache. + const plain = request(pkg.root, pkg.packageJson, join(pkg.root, "plain.o")); + expect((await emitNativeArtifact(plain)).profileMatch).toBeUndefined(); + expect( + (await emitNativeArtifact({ ...plain, outputPath: join(pkg.root, "plain-again.o") })) + .profileMatch, + ).toBeUndefined(); +}); + test("chunked LLVM input preserves emitted bytes and the native cache identity", async () => { const pkg = await fakePackage(); const first = request(pkg.root, pkg.packageJson, join(pkg.root, "chunks.o")); diff --git a/packages/compiler/src/backend/native-codegen.ts b/packages/compiler/src/backend/native-codegen.ts index 413fb7e02..84c76ce91 100644 --- a/packages/compiler/src/backend/native-codegen.ts +++ b/packages/compiler/src/backend/native-codegen.ts @@ -2,7 +2,17 @@ import { execFile } from "node:child_process"; import { createHash, type Hash } from "node:crypto"; import { closeSync, constants, openSync, readSync } from "node:fs"; import { createRequire } from "node:module"; -import { access, chmod, copyFile, mkdir, readFile, rename, rm, stat } from "node:fs/promises"; +import { + access, + chmod, + copyFile, + mkdir, + readFile, + rename, + rm, + stat, + writeFile, +} from "node:fs/promises"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; import { @@ -436,7 +446,10 @@ export async function emitNativeArtifact( dependencies: helper.dependencies, outputPaths: outputs, } satisfies NativeCodegenArtifact; - if (await installCachedPartitions(cached, outputs)) return artifact; + if (await installCachedPartitions(cached, outputs)) { + if (options.profile?.use === undefined) return artifact; + return { ...artifact, profileMatch: await cachedProfileMatch(cached[0]!, outputs.length) }; + } // Partitioned programs also reuse individual partition artifacts, so an // edit recompiles only the partitions it affects. The directory names the // exact helper build and target; the helper keys entries by content. @@ -459,6 +472,7 @@ export async function emitNativeArtifact( // The helper reads a private copy: the caller's .ll may be shared with a // concurrent build of another entry that has the same output stem. const input = privateSiblingPath(options.outputPath, "native-input"); + const matchStage = privateSiblingPath(options.outputPath, "native-profile-match"); try { if (isLlvmFile(options.llvm)) await copyFile(options.llvm.file, input); else await writeLlvmSource(input, options.llvm, 0o600); @@ -493,6 +507,8 @@ export async function emitNativeArtifact( } await chmod(stage, artifactMode()); } + const profileMatch = + options.profile?.use === undefined ? null : parseNativeProfileMatch(stdout); // Cache publication is an optimization boundary. The helper has already // produced a valid caller artifact, so a read-only/full cache must not // discard it or turn an otherwise successful build into an exception. @@ -500,23 +516,68 @@ export async function emitNativeArtifact( root !== null && (await nativeArtifactDependenciesStillMatch(helper.dependencies).catch(() => false)) ) { + // The match goes in first: a reader that finds every partition finds it too. + if (profileMatch !== null) { + await writeFile( + matchStage, + `${JSON.stringify({ profile: helperProfileMatch(profileMatch) })}\n`, + { + mode: 0o600, + }, + ) + .then(() => publishCachedFile(matchStage, cachedProfileMatchPath(cached[0]!))) + .catch(() => undefined); + } for (let index = 0; index < stages.length; index++) await publishCachedFile(stages[index]!, cached[index]!).catch(() => undefined); } for (let index = 0; index < stages.length; index++) await rename(stages[index]!, outputs[index]!); await pruneBuildCache(root); - const profileMatch = - options.profile?.use === undefined ? null : parseNativeProfileMatch(stdout); return profileMatch === null ? artifact : { ...artifact, profileMatch }; } finally { await Promise.all([ ...stages.map((stage) => rm(stage, { force: true }).catch(() => undefined)), rm(input, { force: true }).catch(() => undefined), + rm(matchStage, { force: true }).catch(() => undefined), ]); } } +/** The cache entry beside a cached artifact's first partition that holds the + * profile match of the build that produced it. */ +function cachedProfileMatchPath(firstPartition: string): string { + return `${firstPartition}.profile-match`; +} + +/** The match in the helper's own JSON form, as parseNativeProfileMatch reads it. */ +function helperProfileMatch(match: NativeProfileMatch): Record { + return { + annotated: match.annotated, + mismatched: match.mismatched, + missing: match.missing, + unknown_partitions: match.unknownPartitions, + }; +} + +/** The profile match of a cached artifact. Without a valid entry, every + * partition's counts are unknown. */ +async function cachedProfileMatch( + firstPartition: string, + partitions: number, +): Promise { + const path = cachedProfileMatchPath(firstPartition); + const text = (await validCachedFile(path)) ? await readFile(path, "utf8").catch(() => "") : ""; + return ( + parseNativeProfileMatch(text) ?? { + annotated: 0, + mismatched: 0, + missing: 0, + unknownPartitions: partitions, + } + ); +} + /** Install a cached artifact only when every partition is present and valid. */ async function installCachedPartitions( cached: readonly (string | null)[], diff --git a/packages/compiler/src/backend/native-link-info-core.ts b/packages/compiler/src/backend/native-link-info-core.ts index b29da246f..574eceb9d 100644 --- a/packages/compiler/src/backend/native-link-info-core.ts +++ b/packages/compiler/src/backend/native-link-info-core.ts @@ -55,6 +55,7 @@ export interface NativeLinkFeatures { * Absent (false) for programs without them. */ handleProps?: boolean; declAdopt?: boolean; + typeOrigins?: boolean; /** for await over checked-dynamic values: scr_dyn_for_await.c. Absent * (false) for programs without it. */ dynForAwait?: boolean; @@ -71,6 +72,9 @@ export interface NativeLinkFeatures { /** Builtin functions read as values (Buffer.byteLength, Buffer.alloc, * fs.realpathSync) link scr_builtin_values.c; absent (false) otherwise. */ builtinValues?: boolean; + /** createRequire's computed require and require.resolve link + * scr_require.c; absent (false) otherwise. */ + computedRequire?: boolean; events: boolean; emitter: boolean; symbol: boolean; diff --git a/packages/compiler/src/backend/native-profile.test.ts b/packages/compiler/src/backend/native-profile.test.ts index b2cb0c75d..18f0c6f73 100644 --- a/packages/compiler/src/backend/native-profile.test.ts +++ b/packages/compiler/src/backend/native-profile.test.ts @@ -3,6 +3,7 @@ import { executableLinkInputs } from "./link-plan-core.js"; import { parseNativeProfileMatch, nativeProfileHelperArgs, + renderNativeProfileMatch, nativeProfileKey, nativeProfileRefusal, profileGenerateLinkerArgs, @@ -84,3 +85,49 @@ test("the helper's profile match is read from its output", () => { expect(parseNativeProfileMatch('{"ok":true}\n')).toBeNull(); expect(parseNativeProfileMatch('{"ok":true,"profile":{"annotated":-1}}\n')).toBeNull(); }); + +test("the build reports the match, and warns when the profile is stale", () => { + expect( + renderNativeProfileMatch("app.profdata", { + annotated: 9170, + mismatched: 0, + missing: 28636, + unknownPartitions: 0, + }), + ).toBe( + "scriptc: profile app.profdata: 9170 functions annotated, 0 mismatched, 28636 without a record\n", + ); + // 10% of the described functions changed: at the threshold, no warning. + expect( + renderNativeProfileMatch("app.profdata", { + annotated: 90, + mismatched: 10, + missing: 5, + unknownPartitions: 1, + }), + ).toBe( + "scriptc: profile app.profdata: 90 functions annotated, 10 mismatched, 5 without a record, 1 partition without counts\n", + ); + expect( + renderNativeProfileMatch("app.profdata", { + annotated: 5208, + mismatched: 3739, + missing: 28859, + unknownPartitions: 2, + }), + ).toBe( + "scriptc: profile app.profdata: 5208 functions annotated, 3739 mismatched, 28859 without a record, 2 partitions without counts\n" + + "scriptc: warning: 3739 of the 8947 functions app.profdata describes (41.8%) changed since it was collected and are optimized without their counts; collect the profile again\n", + ); + // Nothing described: no share to warn about. + expect( + renderNativeProfileMatch("app.profdata", { + annotated: 0, + mismatched: 0, + missing: 12, + unknownPartitions: 0, + }), + ).toBe( + "scriptc: profile app.profdata: 0 functions annotated, 0 mismatched, 12 without a record\n", + ); +}); diff --git a/packages/compiler/src/backend/native-profile.ts b/packages/compiler/src/backend/native-profile.ts index 904994a7d..85b1fdca4 100644 --- a/packages/compiler/src/backend/native-profile.ts +++ b/packages/compiler/src/backend/native-profile.ts @@ -83,6 +83,32 @@ export function parseNativeProfileMatch(stdout: string): NativeProfileMatch | nu return null; } +/** The share of the functions a profile describes whose control flow changed + * since it was collected, above which `scriptc build` warns that the profile + * is stale. A freshly collected profile has none. */ +export const STALE_PROFILE_SHARE = 0.1; + +/** `scriptc build --profile-use`'s report on stderr: one line with the match, + * and a warning line when more than STALE_PROFILE_SHARE of the functions the + * profile describes changed since it was collected. */ +export function renderNativeProfileMatch(profile: string, match: NativeProfileMatch): string { + const unknown = + match.unknownPartitions === 0 + ? "" + : `, ${match.unknownPartitions} partition${match.unknownPartitions === 1 ? "" : "s"} without counts`; + const lines = [ + `scriptc: profile ${profile}: ${match.annotated} functions annotated, ${match.mismatched} mismatched, ${match.missing} without a record${unknown}`, + ]; + const described = match.annotated + match.mismatched; + if (described > 0 && match.mismatched / described > STALE_PROFILE_SHARE) { + const percent = ((match.mismatched / described) * 100).toFixed(1); + lines.push( + `scriptc: warning: ${match.mismatched} of the ${described} functions ${profile} describes (${percent}%) changed since it was collected and are optimized without their counts; collect the profile again`, + ); + } + return `${lines.join("\n")}\n`; +} + /** Cache-key part; empty without a profile so other keys are unchanged. */ export function nativeProfileKey(profile: NativeProfileOptions | undefined): string { if (profile?.generate !== undefined) return `profile-generate:${profile.generate}`; diff --git a/packages/compiler/src/backend/native-toolchain.test.ts b/packages/compiler/src/backend/native-toolchain.test.ts index f1bcde404..0ac21e853 100644 --- a/packages/compiler/src/backend/native-toolchain.test.ts +++ b/packages/compiler/src/backend/native-toolchain.test.ts @@ -24,6 +24,7 @@ import { compileLibArchive, executableSectionEliminationFlags, executableNativeEnvironmentFingerprint, + implicitDependencyIncludeDirective, implicitDependencyProbeIncludes, parseLinkTraceFiles, resolveBuildCacheRoot, @@ -624,6 +625,46 @@ test("implicit dependency seeds include separately compiled vendor system header expect(includes).toContain(""); }); +test.skipIf(clangExecutable === undefined)( + "the implicit toolchain probe guards target-specific compiler headers", + async () => { + // clang ships its x86 intrinsic headers (, ...) for every + // target, so __has_include finds them on arm64, where including them is + // an #error, and the same holds for the ARM headers on x86. The probe + // must guard them by target. Only the compiler's own freestanding + // headers are visible here, so the result does not depend on the host's + // C library. + const includes = (await implicitDependencyProbeIncludes(runtimeSrcDir())).filter( + (include) => + include !== "" && /intrin\.h>$|^$|^$/.test(include), + ); + expect(includes).toContain(""); + const resourceDir = execFileSync(clangExecutable!, ["-print-resource-dir"], { + encoding: "utf8", + }).trim(); + const source = includes.map(implicitDependencyIncludeDirective).join(""); + for (const target of ["aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]) { + const result = spawnSync( + clangExecutable!, + [ + `--target=${target}`, + "-ffreestanding", + "-nostdinc", + "-isystem", + join(resourceDir, "include"), + "-fsyntax-only", + "-x", + "c", + "-", + ], + { input: source, encoding: "utf8" }, + ); + expect(result.stderr, target).toBe(""); + expect(result.status, target).toBe(0); + } + }, +); + test.skipIf( process.platform === "win32" || clangExecutable === undefined || diff --git a/packages/compiler/src/backend/native-toolchain.ts b/packages/compiler/src/backend/native-toolchain.ts index 05f1f7122..6c024c6a5 100644 --- a/packages/compiler/src/backend/native-toolchain.ts +++ b/packages/compiler/src/backend/native-toolchain.ts @@ -26,7 +26,10 @@ export { export { CcCompileError, subprocessFailureDetail } from "./native/process.js"; export { compileLibArchive } from "./native/library.js"; export { localizeLibraryObjects } from "./native/object-merge.js"; -export { implicitDependencyProbeIncludes } from "./native/compiler-fingerprint.js"; +export { + implicitDependencyIncludeDirective, + implicitDependencyProbeIncludes, +} from "./native/compiler-fingerprint.js"; export { parseLinkTraceFiles, nativeLinkerDependencyPaths } from "./native/linker-fingerprint.js"; export { clearCcCaches, stageRuntimeObjects } from "./native/runtime-objects.js"; export { diff --git a/packages/compiler/src/backend/native-tools.ts b/packages/compiler/src/backend/native-tools.ts index 2f1b88ed0..9d6f2cf94 100644 --- a/packages/compiler/src/backend/native-tools.ts +++ b/packages/compiler/src/backend/native-tools.ts @@ -8,7 +8,12 @@ import { validateNativeCodegenVersion, type NativeCodegenOutputKind, } from "./native-codegen-core.js"; -import { nativeProfileHelperArgs, type NativeProfileOptions } from "./native-profile.js"; +import { + nativeProfileHelperArgs, + parseNativeProfileMatch, + type NativeProfileMatch, + type NativeProfileOptions, +} from "./native-profile.js"; export function runNativeTool( executable: string, @@ -48,10 +53,10 @@ export function emitNativeObject(options: { partitionCache?: string; /** Profile-guided optimization: instrumentation or an indexed profile. */ profile?: NativeProfileOptions; -}): string[] { +}): { outputs: string[]; profileMatch: NativeProfileMatch | null } { verifyNativeHelper(options); const outputs = nativePartitionPaths(options.outputPath, options.partitions ?? 1); - runNativeTool(options.executable, [ + const stdout = runNativeTool(options.executable, [ "emit", "--input", options.inputPath, @@ -74,7 +79,10 @@ export function emitNativeObject(options: { ...nativeProfileHelperArgs(options.profile), ]); for (const output of outputs) requireNativeArtifact(output); - return outputs; + return { + outputs, + profileMatch: options.profile?.use === undefined ? null : parseNativeProfileMatch(stdout), + }; } /** Merges raw or indexed IR profiles into one indexed profile. */ diff --git a/packages/compiler/src/backend/native/compiler-fingerprint.ts b/packages/compiler/src/backend/native/compiler-fingerprint.ts index ec427ce0a..6f92707f6 100644 --- a/packages/compiler/src/backend/native/compiler-fingerprint.ts +++ b/packages/compiler/src/backend/native/compiler-fingerprint.ts @@ -195,15 +195,25 @@ export async function implicitDependencyProbeIncludes(rtDir: string): Promise" || include === "" - ? " && (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64))" - : include === "" - ? " && defined(_WIN32)" - : include === "" || include === "" || include === "" - ? " && (defined(__arm__) || defined(__aarch64__) || defined(_M_ARM) || defined(_M_ARM64))" - : ""; +/** Compiler-provided x86 headers: clang ships them for every target, so + * __has_include finds them on arm64 too, where including them is an + * #error. Covers the SSE/AVX intrinsic families (, + * , ...), , and . */ +function x86OnlyHeader(include: string): boolean { + return include === "" || /^<(x86|x86gpr|[a-z]*mm)intrin\.h>$/.test(include); +} + +/** One guarded line of the implicit-toolchain probe source: the header is + * included only when the compiler can find it and it belongs to the target + * being probed. */ +export function implicitDependencyIncludeDirective(include: string): string { + const targetCondition = x86OnlyHeader(include) + ? " && (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64))" + : include === "" + ? " && defined(_WIN32)" + : include === "" || include === "" || include === "" + ? " && (defined(__arm__) || defined(__aarch64__) || defined(_M_ARM) || defined(_M_ARM64))" + : ""; return `#if __has_include(${include})${targetCondition}\n#include ${include}\n#endif\n`; } diff --git a/packages/compiler/src/backend/native/contracts.ts b/packages/compiler/src/backend/native/contracts.ts index e144b7f24..49fd792ba 100644 --- a/packages/compiler/src/backend/native/contracts.ts +++ b/packages/compiler/src/backend/native/contracts.ts @@ -126,6 +126,7 @@ export interface CcOptions { /** Own properties JavaScript adds to children: compiles scr_dyn_handle_props.c. */ handleProps?: boolean; declAdopt?: boolean; + typeOrigins?: boolean; /** for await over checked-dynamic values: compiles scr_dyn_for_await.c. */ dynForAwait?: boolean; /** events.once with options (events.onceSignal): compiles @@ -138,6 +139,9 @@ export interface CcOptions { /** Buffer.prototype methods called on checked-dynamic values: * compiles scr_bytes_dyn.c. */ dynBuffer?: boolean; + /** createRequire's computed require and require.resolve: compiles + * scr_require.c. */ + computedRequire?: boolean; /** Builtin functions read as values: compiles scr_builtin_values.c. */ builtinValues?: boolean; /** The program constructs EventTarget/Event/CustomEvent values or diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 2164368ad..abacd5c8c 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -191,6 +191,8 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ unit((u) => u.opts.dynBuffer === true && u.opts.dynInvoke, "scr_bytes_dyn.c"), // Builtin functions read as values in JavaScript dependencies. unit((u) => u.opts.builtinValues === true, "scr_builtin_values.c"), + // createRequire's computed require and require.resolve. + unit((u) => u.opts.computedRequire === true, "scr_require.c"), unit((u) => u.opts.dc, "scr_dc.c"), unit((u) => u.opts.dynTier, "scr_js_value.c", "scr_js_object.c"), unit((u) => u.opts.eventTarget || u.nativeFetch, "scr_event_target.c"), @@ -302,6 +304,7 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ unit((u) => u.opts.handleProps === true, "scr_dyn_handle_props.c"), // The declaration-typed boundary's adoption. unit((u) => u.opts.declAdopt === true, "scr_decl_adopt.c"), + unit((u) => u.opts.typeOrigins === true, "scr_type_origin.c"), // for await over checked-dynamic values. unit((u) => u.opts.dynForAwait === true, "scr_dyn_for_await.c"), // for await over checked-dynamic values that may hold native streams. @@ -1516,6 +1519,8 @@ export async function compileCInternal( [], programPublicSymbols, `${stem}.program`, + undefined, + { inProcessElf: true }, ); const publishable = entries.filter((entry) => entry.missed); if (publishable.length > 0) { diff --git a/packages/compiler/src/backend/native/library.ts b/packages/compiler/src/backend/native/library.ts index f0577f3ca..06b4ce1df 100644 --- a/packages/compiler/src/backend/native/library.ts +++ b/packages/compiler/src/backend/native/library.ts @@ -517,6 +517,8 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise [], programPublicSymbols!, `${stem}.program`, + undefined, + { inProcessElf: true }, ); // Keep the canonical archive member spelling. The fact that native // compilation used shards is an implementation detail; consumers and diff --git a/packages/compiler/src/backend/native/object-merge.ts b/packages/compiler/src/backend/native/object-merge.ts index 39a14785e..43f15fef8 100644 --- a/packages/compiler/src/backend/native/object-merge.ts +++ b/packages/compiler/src/backend/native/object-merge.ts @@ -5,6 +5,17 @@ import { localizeElfObject, mergeAndLocalizeCoffObjects } from "../object-locali import { type CcDriver, targetPlatform } from "./driver.js"; import { execFileAsync, subprocessFailureDetail } from "./process.js"; +/* Demotes in process (localizeElfObject), rewriting the object in place. */ +async function localizeInProcess(object: string, keepSymbols: readonly string[]): Promise { + try { + await writeFile(object, localizeElfObject(await readFile(object), new Set(keepSymbols))); + } catch (err) { + throw new Error( + `ELF symbol localization failed while localizing the library archive's runtime symbols (abi.localize_runtime).\n\n${(err as Error).message}`, + ); + } +} + /* Multi-instance library mode's localization step: combine the program * object with exactly the runtime/vendor members it reaches into ONE * relocatable object, then demote every scriptc external definition except @@ -46,7 +57,11 @@ import { execFileAsync, subprocessFailureDetail } from "./process.js"; * multi-archive link from discarding one archive's copies), * then binutils objcopy --keep-global-symbols localizes every * other DEFINED global (objcopy leaves undefined symbols - * global by its own rule). The exact historical recipe. + * global by its own rule). The exact historical recipe. With + * `inProcessElf` (program-shard merges), localizeElfObject + * demotes instead of objcopy: a sanitized dev program merges + * into an object with hundreds of thousands of sections, which + * objcopy takes minutes over and localizeElfObject seconds. * ELF, cross triples from any host — `zig cc -target -r` * merges (zig is already the cross driver's hard requirement), * then localizeElfObject demotes in process and resolves @@ -68,6 +83,7 @@ export async function localizeLibraryObjects( keepSymbols: readonly string[], stem: string, platform = targetPlatform(driver), + options: { inProcessElf?: boolean } = {}, ): Promise { const combined = join(buildDir, `${stem}.localized.o`); const staging = join(buildDir, `${stem}.localize-staging.a`); @@ -79,7 +95,7 @@ export async function localizeLibraryObjects( const stderr = subprocessFailureDetail(err); throw new Error( `${argv[0]} failed while localizing the library archive's runtime symbols (abi.localize_runtime).\n` + - `Runtime symbol localization needs the ${platform === "darwin" ? "host toolchain's ld" : driver.target === null ? "host toolchain's ld and objcopy" : "cross driver's relocatable link"} beside the C compiler.\n\n${stderr}`, + `Runtime symbol localization needs the ${platform === "darwin" || (driver.target === null && options.inProcessElf === true) ? "host toolchain's ld" : driver.target === null ? "host toolchain's ld and objcopy" : "cross driver's relocatable link"} beside the C compiler.\n\n${stderr}`, ); } }; @@ -122,7 +138,6 @@ export async function localizeLibraryObjects( keepFile, ]); } else if (platform === "linux" && driver.target === null) { - await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join("")); await run([ "ld", "-r", @@ -132,7 +147,12 @@ export async function localizeLibraryObjects( "-o", combined, ]); - await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]); + if (options.inProcessElf === true) { + await localizeInProcess(combined, keepSymbols); + } else { + await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join("")); + await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]); + } } else if (platform === "linux") { // Cross ELF: the cross driver's own lld performs the relocatable merge // (with the staging archive's member semantics); demotion and section- @@ -152,13 +172,7 @@ export async function localizeLibraryObjects( "-o", combined, ]); - try { - await writeFile(combined, localizeElfObject(await readFile(combined), new Set(keepSymbols))); - } catch (err) { - throw new Error( - `ELF symbol localization failed while localizing the library archive's runtime symbols (abi.localize_runtime).\n\n${(err as Error).message}`, - ); - } + await localizeInProcess(combined, keepSymbols); } else { throw new InternalCompilerError( `runtime symbol localization (abi.localize_runtime) has no ${platform} arm; compileLibrary admits darwin, linux, and win32 builds only`, diff --git a/packages/compiler/src/backend/native/program-shards.ts b/packages/compiler/src/backend/native/program-shards.ts index 1afd87f7b..2948299b2 100644 --- a/packages/compiler/src/backend/native/program-shards.ts +++ b/packages/compiler/src/backend/native/program-shards.ts @@ -52,13 +52,13 @@ export async function resolveProgramShardMergeIdentity(driver: CcDriver): Promis } if (platform === "linux") { if (driver.target === null) { - const [ld, objcopy] = await Promise.all([resolvedTool("ld"), resolvedTool("objcopy")]); - return ld === null || objcopy === null + // ld -r merges; localizeElfObject demotes in process. + const ld = await resolvedTool("ld"); + return ld === null ? null - : rememberFingerprintDependencies( - `program-shard-merge-linux-v1\0${ld.cacheIdentity}\0${objcopy.cacheIdentity}`, - [ld.canonicalPath, objcopy.canonicalPath], - ); + : rememberFingerprintDependencies(`program-shard-merge-linux-v2\0${ld.cacheIdentity}`, [ + ld.canonicalPath, + ]); } const arch = driver.target.split("-", 1)[0]; if (arch !== "x86_64" && arch !== "aarch64") return null; diff --git a/packages/compiler/src/backend/native/runtime-inputs.ts b/packages/compiler/src/backend/native/runtime-inputs.ts index d32867331..891fa9312 100644 --- a/packages/compiler/src/backend/native/runtime-inputs.ts +++ b/packages/compiler/src/backend/native/runtime-inputs.ts @@ -74,6 +74,8 @@ export const EXECUTABLE_RUNTIME_SOURCES = [ "scr_slots.c", "scr_alloc.c", "scr_cipher_info.c", + "scr_ieee754.c", + "scr_glibc_math.c", ] as const; export function runtimeSrcDir(): string { diff --git a/packages/compiler/src/backend/object-localize.test.ts b/packages/compiler/src/backend/object-localize.test.ts index 9e4e30285..1a1e9d4bd 100644 --- a/packages/compiler/src/backend/object-localize.test.ts +++ b/packages/compiler/src/backend/object-localize.test.ts @@ -28,7 +28,10 @@ const SHF_EXECINSTR = 0x4n; const SHF_GROUP = 0x200n; const STB_GLOBAL = 1; const STB_WEAK = 2; +const SHT_SYMTAB_SHNDX = 18; +const SHN_LORESERVE = 0xff00; const SHN_COMMON = 0xfff2; +const SHN_XINDEX = 0xffff; interface ElfSymbolSpec { name: string; @@ -53,7 +56,11 @@ function buildStrtab(names: string[]): { data: Uint8Array; offsets: Map s.name).filter((n) => n !== "")); const symData = new Uint8Array((symbols.length + 1) * 24); const symView = new DataView(symData.buffer); + const shndxData = new Uint8Array((symbols.length + 1) * 4); + let usesShndx = false; symbols.forEach((sym, i) => { const at = (i + 1) * 24; symView.setUint32(at, sym.name === "" ? 0 : strtab.offsets.get(sym.name)!, true); symView.setUint8(at + 4, (sym.binding << 4) | 0); - symView.setUint16(at + 6, sym.shndx, true); + if (sym.shndx >= SHN_LORESERVE && sym.shndx !== SHN_COMMON) { + symView.setUint16(at + 6, SHN_XINDEX, true); + new DataView(shndxData.buffer).setUint32((i + 1) * 4, sym.shndx, true); + usesShndx = true; + } else { + symView.setUint16(at + 6, sym.shndx, true); + } symView.setBigUint64(at + 8, sym.value ?? 0n, true); }); const all: ElfSectionSpec[] = [ @@ -82,9 +97,22 @@ function buildElf( }, { name: ".strtab", type: SHT_STRTAB, data: strtab.data }, { name: ".shstrtab", type: SHT_STRTAB }, + ...(usesShndx + ? [ + { + name: ".symtab_shndx", + type: SHT_SYMTAB_SHNDX, + data: shndxData, + link: sections.length + 1, + entsize: 4n, + }, + ] + : []), ]; + const shstrndx = sections.length + 3; + const extended = all.length >= SHN_LORESERVE; const shstr = buildStrtab(all.map((s) => s.name).filter((n) => n !== "")); - all[all.length - 1]!.data = shstr.data; + all[shstrndx]!.data = shstr.data; let offset = 64; const offsets = all.map((s) => { const data = s.data ?? new Uint8Array(0); @@ -106,8 +134,8 @@ function buildElf( view.setBigUint64(40, BigInt(shoff), true); view.setUint16(52, 64, true); view.setUint16(58, 64, true); - view.setUint16(60, all.length, true); - view.setUint16(62, all.length - 1, true); + view.setUint16(60, extended ? 0 : all.length, true); + view.setUint16(62, shstrndx >= SHN_LORESERVE ? SHN_XINDEX : shstrndx, true); all.forEach((s, i) => { const at = shoff + i * 64; const data = s.data ?? new Uint8Array(0); @@ -116,8 +144,8 @@ function buildElf( view.setUint32(at + 4, s.type, true); view.setBigUint64(at + 8, s.flags ?? 0n, true); view.setBigUint64(at + 24, BigInt(s.type === 0 ? 0 : offsets[i]!), true); - view.setBigUint64(at + 32, BigInt(data.length), true); - view.setUint32(at + 40, s.link ?? 0, true); + view.setBigUint64(at + 32, BigInt(i === 0 && extended ? all.length : data.length), true); + view.setUint32(at + 40, i === 0 && shstrndx >= SHN_LORESERVE ? shstrndx : (s.link ?? 0), true); view.setUint32(at + 44, s.info ?? 0, true); view.setBigUint64(at + 48, s.addralign ?? 1n, true); view.setBigUint64(at + 56, s.entsize ?? 0n, true); @@ -142,8 +170,10 @@ interface ReadElf { function readElf(bytes: Uint8Array): ReadElf { const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const shoff = Number(view.getBigUint64(40, true)); - const shnum = view.getUint16(60, true); - const shstrndx = view.getUint16(62, true); + let shnum = view.getUint16(60, true); + let shstrndx = view.getUint16(62, true); + if (shnum === 0) shnum = Number(view.getBigUint64(shoff + 32, true)); + if (shstrndx === SHN_XINDEX) shstrndx = view.getUint32(shoff + 40, true); const raw = [] as { nameOff: number; type: number; @@ -182,12 +212,16 @@ function readElf(bytes: Uint8Array): ReadElf { const symtabIdx = raw.findIndex((s) => s.type === SHT_SYMTAB); const symtab = raw[symtabIdx]!; const strtab = raw[symtab.link]!; + const shndxTable = raw.find((s) => s.type === SHT_SYMTAB_SHNDX && s.link === symtabIdx); const symbols = [] as ReadElf["symbols"]; for (let at = symtab.offset; at < symtab.offset + symtab.size; at += 24) { + let shndx = view.getUint16(at + 6, true); + if (shndx === SHN_XINDEX) + shndx = view.getUint32(shndxTable!.offset + ((at - symtab.offset) / 24) * 4, true); symbols.push({ name: name(strtab, view.getUint32(at, true)), binding: view.getUint8(at + 4) >> 4, - shndx: view.getUint16(at + 6, true), + shndx, }); } return { sections, symbols, symtabInfo: symtab.info }; @@ -241,6 +275,65 @@ describe("localizeElfObject", () => { expect(info & 0xffffffffn).toBe(2n); }); + test("keeps extended section numbering when the object has SHN_LORESERVE or more sections", () => { + // A sanitized dev program merges into an object with more sections than + // e_shnum and st_shndx can index: section 0 carries the count and the + // section-name table's index, and .symtab_shndx the symbols' indices. + const count = SHN_LORESERVE + 400; + const texts: ElfSectionSpec[] = Array.from({ length: count }, (_, i) => ({ + name: `.text.f${i}`, + type: SHT_PROGBITS, + flags: SHF_ALLOC | SHF_EXECINSTR, + data: new Uint8Array(4), + })); + const rela = new Uint8Array(24); + new DataView(rela.buffer).setBigUint64(8, (2n << 32n) | 2n, true); // sym 2 (high) + const object = buildElf( + [ + ...texts, + { + name: ".rela.text.last", + type: SHT_RELA, + data: rela, + link: count + 2, + info: count, + entsize: 24n, + }, + ], + [ + { name: "low", binding: STB_GLOBAL, shndx: 1 }, + { name: "high", binding: STB_GLOBAL, shndx: count }, + { name: "main", binding: STB_GLOBAL, shndx: count - 1 }, + { name: "libc_ref", binding: STB_GLOBAL, shndx: 0 }, + { name: "shared_common", binding: STB_GLOBAL, shndx: SHN_COMMON, value: 8n }, + ], + 0, + ); + const output = localizeElfObject(object, new Set(["main"])); + const header = new DataView(output.buffer, output.byteOffset, output.byteLength); + expect(header.getUint16(60, true)).toBe(0); + expect(header.getUint16(62, true)).toBe(SHN_XINDEX); + const localized = readElf(output); + expect(localized.sections.length).toBe(count + 6); + expect(localized.sections.filter((s) => s.type === SHT_SYMTAB_SHNDX)).toHaveLength(1); + const byName = new Map(localized.symbols.map((s) => [s.name, s])); + expect(byName.get("low")!.binding).toBe(0); + expect(byName.get("high")!.binding).toBe(0); + expect(localized.sections[byName.get("high")!.shndx]!.name).toBe(`.text.f${count - 1}`); + expect(byName.get("main")!.binding).toBe(STB_GLOBAL); + expect(localized.sections[byName.get("main")!.shndx]!.name).toBe(`.text.f${count - 2}`); + expect(byName.get("libc_ref")!.shndx).toBe(0); + expect(byName.get("shared_common")!.shndx).toBe(SHN_COMMON); + const relaOut = localized.sections.find((s) => s.name === ".rela.text.last")!; + expect(localized.sections[relaOut.info]!.name).toBe(`.text.f${count - 1}`); + const info = new DataView(relaOut.data.buffer, relaOut.data.byteOffset).getBigUint64(8, true); + expect(localized.symbols[Number(info >> 32n)]!.name).toBe("high"); + // The output reads back as the same object. + expect(readElf(localizeElfObject(output, new Set(["main"]))).symbols).toEqual( + localized.symbols, + ); + }); + test("resolves section groups: SHT_GROUP dropped, SHF_GROUP cleared, members and indices intact", () => { const group = new Uint8Array(8); const groupView = new DataView(group.buffer); diff --git a/packages/compiler/src/backend/object-localize.ts b/packages/compiler/src/backend/object-localize.ts index 302027484..fb91ea85b 100644 --- a/packages/compiler/src/backend/object-localize.ts +++ b/packages/compiler/src/backend/object-localize.ts @@ -84,9 +84,13 @@ interface ElfSymbol { name: string; info: number; other: number; - /** Resolved section index (SHT_SYMTAB_SHNDX applied), or a reserved - * SHN_* value below/at SHN_LORESERVE semantics. */ + /** Section index (SHT_SYMTAB_SHNDX applied), or the reserved SHN_* value + * when `reserved`. */ shndx: number; + /** st_shndx holds a reserved value (SHN_ABS, SHN_COMMON, ...), not a + * section index. An index from SHT_SYMTAB_SHNDX can itself be at or above + * SHN_LORESERVE. */ + reserved: boolean; } /** Demote every defined global/weak symbol not named in `keep` to a local @@ -143,8 +147,8 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) if (strtab === undefined) fail("ELF symbol table names an invalid string table"); // SHT_SYMTAB_SHNDX (only present when section indices overflow st_shndx): - // resolve the real indices up front and drop the section — the rebuilt - // object's section count only shrinks, so inline indices always fit. + // resolve the real indices up front and drop the section. The rebuilt + // object writes a new one when its own section count still overflows. const shndxSection = sections.find( (s) => s.type === SHT_SYMTAB_SHNDX && sections[s.link] === symtab, ); @@ -170,9 +174,11 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) for (let i = 0; i < symCount; i++) { const at = i * 24; let shndx: number = symView.getUint16(at + 6, true); + let reserved = shndx >= SHN_LORESERVE; if (shndx === SHN_XINDEX) { if (shndxView === null) fail("ELF symbol uses SHN_XINDEX without a SHT_SYMTAB_SHNDX table"); shndx = shndxView.getUint32(i * 4, true); + reserved = false; } symbols.push({ nameOffset: symView.getUint32(at, true), @@ -180,6 +186,7 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) info: symView.getUint8(at + 4), other: symView.getUint8(at + 5), shndx, + reserved, }); } @@ -212,11 +219,11 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) const symbolKept: boolean[] = []; for (let i = 0; i < symCount; i++) { const sym = symbols[i]!; - const regular = sym.shndx > SHN_UNDEF && sym.shndx < SHN_LORESERVE; + const regular = !sym.reserved && sym.shndx !== SHN_UNDEF; symbolKept.push(!(regular && sectionKept[sym.shndx] !== true)); if (i === 0) continue; const binding = sym.info >> 4; - const defined = sym.shndx !== SHN_UNDEF && sym.shndx !== SHN_COMMON; + const defined = sym.shndx !== SHN_UNDEF && !(sym.reserved && sym.shndx === SHN_COMMON); if (binding !== STB_LOCAL && defined && !keep.has(sym.name)) { sym.info = (STB_LOCAL << 4) | (sym.info & 0xf); } @@ -236,6 +243,16 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) symbolMap[oldIndex] = newIndex; }); + // Extended section numbering: with SHN_LORESERVE or more sections, an + // index from SHN_LORESERVE up does not fit st_shndx, e_shnum or + // e_shstrndx. Such a symbol's st_shndx is SHN_XINDEX and its index goes in + // a SHT_SYMTAB_SHNDX section (one entry per symbol, appended after the + // kept sections); the counts go in section 0 (sh_size, sh_link). + let keptCount = 0; + for (const keptSection of sectionKept) if (keptSection) keptCount++; + const extended = keptCount >= SHN_LORESERVE; + const shndxOut = extended ? new Uint8Array(symbolOrder.length * 4) : null; + const shndxOutView = shndxOut === null ? null : new DataView(shndxOut.buffer); const newSymtabData = new Uint8Array(symbolOrder.length * 24); const newSymtabView = new DataView(newSymtabData.buffer); symbolOrder.forEach((oldIndex, newIndex) => { @@ -244,9 +261,14 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) newSymtabView.setUint32(at, sym.nameOffset, true); newSymtabView.setUint8(at + 4, sym.info); newSymtabView.setUint8(at + 5, sym.other); - const shndx = - sym.shndx > SHN_UNDEF && sym.shndx < SHN_LORESERVE ? sectionMap[sym.shndx]! : sym.shndx; - newSymtabView.setUint16(at + 6, shndx, true); + const regular = !sym.reserved && sym.shndx !== SHN_UNDEF; + const shndx = regular ? sectionMap[sym.shndx]! : sym.shndx; + if (regular && shndx >= SHN_LORESERVE) { + newSymtabView.setUint16(at + 6, SHN_XINDEX, true); + shndxOutView!.setUint32(newIndex * 4, shndx, true); + } else { + newSymtabView.setUint16(at + 6, shndx, true); + } newSymtabView.setBigUint64(at + 8, symView.getBigUint64(oldIndex * 24 + 8, true), true); newSymtabView.setBigUint64(at + 16, symView.getBigUint64(oldIndex * 24 + 16, true), true); }); @@ -278,6 +300,33 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) // Rebuild. sh_link is a section index in every use this object can carry; // sh_info is one only on relocation sections. const kept = sections.filter((_, i) => sectionKept[i] === true); + const newSymtabIndex = sectionMap[symtabIndex]!; + if (shndxOut !== null) { + // The extended index table goes last, named in the section-name table. + const shstrtab = sections[shstrndx]!; + let nameOffset = shndxSection?.nameOffset; + if (nameOffset === undefined) { + const name = textEncoder.encode(".symtab_shndx\0"); + const grown = new Uint8Array(shstrtab.data.length + name.length); + grown.set(shstrtab.data); + grown.set(name, shstrtab.data.length); + nameOffset = shstrtab.data.length; + shstrtab.data = grown; + } + kept.push({ + nameOffset, + type: SHT_SYMTAB_SHNDX, + flags: 0n, + addr: 0n, + size: BigInt(shndxOut.length), + link: symtabIndex, + info: 0, + addralign: 4n, + entsize: 4n, + data: shndxOut, + }); + } + const newShstrndx = sectionMap[shstrndx]!; let offset = 64; const offsets: number[] = []; for (const section of kept) { @@ -301,8 +350,8 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) outView.setBigUint64(40, BigInt(shoffOut), true); outView.setUint16(54, 0, true); // e_phentsize outView.setUint16(56, 0, true); // e_phnum - outView.setUint16(60, kept.length, true); - outView.setUint16(62, sectionMap[shstrndx]!, true); + outView.setUint16(60, extended ? 0 : kept.length, true); + outView.setUint16(62, newShstrndx >= SHN_LORESERVE ? SHN_XINDEX : newShstrndx, true); kept.forEach((section, i) => { if (section.type !== SHT_NULL && section.type !== SHT_NOBITS) { out.set(section.data, offsets[i]!); @@ -313,12 +362,27 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) outView.setBigUint64(at + 8, section.flags, true); outView.setBigUint64(at + 16, section.addr, true); outView.setBigUint64(at + 24, BigInt(offsets[i]!), true); + if (i === 0) { + // Section 0 carries the counts that overflow the ELF header. + outView.setBigUint64(at + 32, extended ? BigInt(kept.length) : 0n, true); + outView.setUint32(at + 40, newShstrndx >= SHN_LORESERVE ? newShstrndx : 0, true); + outView.setUint32(at + 44, 0, true); + return; + } outView.setBigUint64( at + 32, section.type === SHT_NOBITS ? section.size : BigInt(section.data.length), true, ); - outView.setUint32(at + 40, section.link === 0 ? 0 : (sectionMap[section.link] ?? 0), true); + outView.setUint32( + at + 40, + section.type === SHT_SYMTAB_SHNDX + ? newSymtabIndex + : section.link === 0 + ? 0 + : (sectionMap[section.link] ?? 0), + true, + ); outView.setUint32( at + 44, section.type === SHT_RELA || section.type === SHT_REL diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index fe13879a2..892e9281e 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 45 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v45" as const; +export const RUNTIME_ABI_VERSION = 46 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v46" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-core.ts b/packages/compiler/src/backend/runtime-pack-core.ts index 730adec96..c2e9dd07f 100644 --- a/packages/compiler/src/backend/runtime-pack-core.ts +++ b/packages/compiler/src/backend/runtime-pack-core.ts @@ -87,12 +87,14 @@ export interface RuntimeFeatureSet extends NativeLinkFeatures { streamDyn: boolean; handleProps: boolean; declAdopt: boolean; + typeOrigins: boolean; dynForAwait: boolean; eventsOnceSignal: boolean; mime: boolean; utilTiming: boolean; dynBuffer: boolean; builtinValues: boolean; + computedRequire: boolean; warningReplace: boolean; stdioExtended: boolean; brotli: boolean; @@ -289,12 +291,14 @@ export function effectiveRuntimeFeatures( streamDyn: features.streamDyn === true, handleProps: features.handleProps === true, declAdopt: features.declAdopt === true, + typeOrigins: features.typeOrigins === true, dynForAwait: features.dynForAwait === true, eventsOnceSignal: features.eventsOnceSignal === true, mime: features.mime === true, utilTiming: features.utilTiming === true, dynBuffer: features.dynBuffer === true, builtinValues: features.builtinValues === true, + computedRequire: features.computedRequire === true, warningReplace: features.warningReplace === true, stdioExtended: features.stdioExtended === true, brotli: features.brotli === true, diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 492b71217..9c5b61f58 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 45, marker: "scr_runtime_abi_v45" }, + runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index fdb0ba29e..812fd9531 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -129,7 +129,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 45, marker: "scr_runtime_abi_v45" }, + runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/backend/target-cpu.ts b/packages/compiler/src/backend/target-cpu.ts index 51db87bbb..d2967ca68 100644 --- a/packages/compiler/src/backend/target-cpu.ts +++ b/packages/compiler/src/backend/target-cpu.ts @@ -105,6 +105,11 @@ export function targetCpuField(cpu: TargetCpu | undefined): { targetCpu?: Target return cpu === undefined || cpu === "baseline" ? {} : { targetCpu: cpu }; } +/** Cache-key parts for a dispatching build: the tiers it carries. */ +export function cpuDispatchKeyParts(cpuDispatch: boolean | undefined): string[] { + return cpuDispatch === true ? ["cpu-dispatch-x86-64-v3"] : []; +} + /** The error for a level on a target that is not x86-64, or null. */ export function targetCpuTargetRefusal( cpu: TargetCpu | undefined, diff --git a/packages/compiler/src/cli/command.ts b/packages/compiler/src/cli/command.ts index 8b2e4c480..4c3ce111f 100644 --- a/packages/compiler/src/cli/command.ts +++ b/packages/compiler/src/cli/command.ts @@ -1,3 +1,4 @@ +import type { TypePatchCommandRequest } from "../frontend/type-patches/command.js"; import { rmSync, statSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { parseArgs } from "node:util"; @@ -27,6 +28,7 @@ import { import { isTypePatchCode } from "../diagnostics/diagnostic.js"; import { isPerformanceCode, renderPerformanceWarnings } from "../diagnostics/dynamic-tier.js"; import { isNativeOptimization } from "../backend/optimization.js"; +import { renderNativeProfileMatch } from "../backend/native-profile.js"; import { hostTargetCpu, isTargetCpuOption, @@ -116,6 +118,7 @@ async function main(args: string[], host: CliHost): Promise { values["windows-subsystem"] !== undefined || values["detached-exit"] || values["target-cpu"] !== undefined || + values["cpu-dispatch"] !== undefined || values["profile-generate"] !== undefined || values["profile-use"] !== undefined || values.sparse || @@ -176,6 +179,7 @@ async function main(args: string[], host: CliHost): Promise { values["windows-subsystem"] !== undefined || values["detached-exit"] || values["target-cpu"] !== undefined || + values["cpu-dispatch"] !== undefined || values["profile-generate"] !== undefined || values["profile-use"] !== undefined || (values["npm-static"] ?? []).length > 0 || @@ -205,6 +209,39 @@ async function main(args: string[], host: CliHost): Promise { process.stdout.write(`${outputPath}\n`); return 0; } + if (command === "types") { + const sub = positionals[1]; + const rest = positionals.slice(2); + let request: TypePatchCommandRequest; + if (sub === "patch" && rest.length === 3) { + request = { + action: "patch", + cwd: process.cwd(), + pkg: rest[0]!, + selector: rest[1]!, + type: rest[2]!, + range: values.range ?? null, + dryRun: values["dry-run"], + }; + } else if ( + sub === "list" && + rest.length === 0 && + values.range === undefined && + !values["dry-run"] + ) { + request = { action: "list", cwd: process.cwd() }; + } else { + fail( + `scriptc types takes "patch " (with --range or --dry-run) or "list"\n\n${USAGE}`, + ); + } + const result = await host.typePatches(request); + process.stdout.write(result.stdout); + process.stderr.write(result.stderr); + return result.exitCode; + } + if (values.range !== undefined || values["dry-run"]) + fail(`--range and --dry-run are scriptc types patch options\n\n${USAGE}`); if (values.sparse) fail(`--sparse is a merge-profiles option\n\n${USAGE}`); if (command !== "build" && command !== "run" && command !== "coverage") { fail(`unknown command "${command}"\n\n${USAGE}`); @@ -234,6 +271,7 @@ async function main(args: string[], host: CliHost): Promise { values["windows-subsystem"] !== undefined || values["detached-exit"] || values["target-cpu"] !== undefined || + values["cpu-dispatch"] !== undefined || values["profile-generate"] !== undefined || values["profile-use"] !== undefined || (values["npm-static"] ?? []).length > 0 || @@ -243,7 +281,7 @@ async function main(args: string[], host: CliHost): Promise { hasNodeIslandFlags(values) ) { fail( - "scriptc build --lib takes no --backend/--emit/--print/--optimization/--strip/--windows-subsystem/--detached-exit/--target-cpu/--profile-generate/--profile-use/--npm-static/--ffi/--external-types/--tier/--explain-dynamic: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, external type mappings belong to coverage, and library artifacts are static-tier only", + "scriptc build --lib takes no --backend/--emit/--print/--optimization/--strip/--windows-subsystem/--detached-exit/--target-cpu/--cpu-dispatch/--profile-generate/--profile-use/--npm-static/--ffi/--external-types/--tier/--explain-dynamic: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, external type mappings belong to coverage, and library artifacts are static-tier only", ); } const profilePath = resolve(profileArg); @@ -289,6 +327,9 @@ async function main(args: string[], host: CliHost): Promise { if (command === "coverage" && values["target-cpu"] !== undefined) { fail(`--target-cpu is a build/run option\n\n${USAGE}`); } + if (command === "coverage" && values["cpu-dispatch"] !== undefined) { + fail(`--cpu-dispatch is a build/run option\n\n${USAGE}`); + } if (command === "coverage" && values["detached-exit"]) { fail(`--detached-exit is a build/run option\n\n${USAGE}`); } @@ -469,6 +510,7 @@ async function main(args: string[], host: CliHost): Promise { ...(values.optimization === undefined ? {} : { optimization: values.optimization }), strip: values.strip, detachedExit: values["detached-exit"], + ...(values["cpu-dispatch"] === undefined ? {} : { cpuDispatch: values["cpu-dispatch"] }), ...(values["profile-generate"] === undefined ? {} : { profileGenerate: values["profile-generate"] }), @@ -589,6 +631,7 @@ async function main(args: string[], host: CliHost): Promise { ...(windowsSubsystem !== undefined ? { windowsSubsystem } : {}), ...(values["detached-exit"] ? { detachedExit: true } : {}), ...targetCpuField(targetCpu), + ...(values["cpu-dispatch"] === undefined ? {} : { cpuDispatch: values["cpu-dispatch"] }), // The instrumented program resolves a relative pattern against its own // working directory; resolve it here, where the user wrote it. ...(values["profile-generate"] === undefined @@ -670,6 +713,15 @@ async function main(args: string[], host: CliHost): Promise { } } if (result.artifact.kind === "exe") { + // How the profile matched the program's functions; `run` keeps stderr + // for the program's own output. + const profileUse = values["profile-use"]; + if ( + command === "build" && + profileUse !== undefined && + result.artifact.profileMatch !== undefined + ) + process.stderr.write(renderNativeProfileMatch(profileUse, result.artifact.profileMatch)); if (!values["keep-llvm"]) rmSync(result.artifact.translationUnitPath, { force: true }); } else if (result.artifact.kind === "obj") { nativeLinkInfo = result.artifact.nativeLinkInfo; diff --git a/packages/compiler/src/cli/host.ts b/packages/compiler/src/cli/host.ts index 7a927ee39..eafe1882b 100644 --- a/packages/compiler/src/cli/host.ts +++ b/packages/compiler/src/cli/host.ts @@ -1,3 +1,7 @@ +import type { + TypePatchCommandRequest, + TypePatchCommandResult, +} from "../frontend/type-patches/command.js"; import type { AnalyzeOptions, AnalyzeResult, @@ -30,5 +34,7 @@ export interface CliHost { /** Merges raw or indexed profiles into one indexed profile; a sparse one * leaves out functions that never ran. */ mergeProfiles: (outputPath: string, inputPaths: string[], sparse: boolean) => Promise; + /** `scriptc types patch|list` (frontend/type-patches/command.ts). */ + typePatches: (request: TypePatchCommandRequest) => Promise; run: (binary: string) => Promise; } diff --git a/packages/compiler/src/cli/output-options.ts b/packages/compiler/src/cli/output-options.ts index eb8f9a8e6..fb68eb050 100644 --- a/packages/compiler/src/cli/output-options.ts +++ b/packages/compiler/src/cli/output-options.ts @@ -11,6 +11,7 @@ export interface OutputOptionValues { strip?: boolean; windowsSubsystem?: string; detachedExit?: boolean; + cpuDispatch?: boolean; profileGenerate?: string; profileUse?: string; ffi?: string; @@ -75,6 +76,9 @@ export function resolveOutputOptions( if (values.detachedExit && emit !== "exe") { return { ok: false, message: `--detached-exit is only supported with --emit=exe` }; } + if (values.cpuDispatch && emit !== "exe") { + return { ok: false, message: `--cpu-dispatch is only supported with --emit=exe` }; + } if (values.profileGenerate !== undefined && values.profileUse !== undefined) { return { ok: false, message: `--profile-generate and --profile-use cannot be combined` }; } diff --git a/packages/compiler/src/cli/usage.ts b/packages/compiler/src/cli/usage.ts index f8b6a4869..94a4641c5 100644 --- a/packages/compiler/src/cli/usage.ts +++ b/packages/compiler/src/cli/usage.ts @@ -13,6 +13,13 @@ Usage: exporting the profile symbols; a profile with a sidecar section also gets the contract sidecar JSON beside the archive + scriptc types patch [--range ] [--dry-run] + fix a wrong declaration of an installed + package: shows what the selector + matches and adds the type patch to + package.json ("| undefined" widens) + scriptc types list the project's type patches and what + they match scriptc cache warm [runtime|tls…] verify and load precompiled runtime families for the current compiler/SDK/target scriptc merge-profiles -o [--sparse] @@ -31,7 +38,7 @@ Options: (native-link-info implies --emit=obj and never links; diagnostics prints the versioned JSON diagnostics envelope for build and coverage) - --fail-on + --fail-on coverage only: exit 1 when the entry path has blockers (a build would fail), or also when it has Node divergences, likely-hot functions that use untyped @@ -62,6 +69,12 @@ Options: executable then requires that level: v3 adds AVX2, BMI2 and FMA, v4 adds AVX-512. native uses the build host's level. The runtime keeps its baseline code + --cpu-dispatch x86-64: also compile the program's code for x86-64-v3 + (AVX2), and run that copy on CPUs that support it. The + executable still runs on any x86-64 CPU. Not used with + --target-cpu above baseline. The SCRIPTC_CPU_LEVEL + environment variable (0 to 4, or a level name) caps the + level at run time (opt in; --emit=exe only) --profile-generate Linux: instrument the executable for profile-guided optimization. Each run writes a raw profile to the @@ -73,8 +86,10 @@ Options: code is laid out, inlined and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is - optimized as without one (release or speed; --emit=exe - only) + optimized as without one. build prints how the profile + matched on stderr, and warns when more than 10% of the + functions it describes changed (release or speed; + --emit=exe only) --keep-llvm keep generated LLVM IR beside the executable (default) --no-keep-llvm delete generated LLVM IR after compiling --emit-ir also write IR beside an executable or library archive; @@ -154,9 +169,12 @@ export const CLI_OPTIONS = { "windows-subsystem": { type: "string" }, "detached-exit": { type: "boolean", default: false }, "target-cpu": { type: "string" }, + "cpu-dispatch": { type: "boolean" }, "profile-generate": { type: "string" }, "profile-use": { type: "string" }, sparse: { type: "boolean", default: false }, + range: { type: "string" }, + "dry-run": { type: "boolean", default: false }, "keep-llvm": { type: "boolean", default: true }, "emit-ir": { type: "boolean", default: false }, sanitize: { type: "boolean", default: false }, diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts index 8ec7b5ffd..08de807f9 100644 --- a/packages/compiler/src/compile-types.ts +++ b/packages/compiler/src/compile-types.ts @@ -45,6 +45,11 @@ export interface CompileBaseOptions { * Unset or `baseline`, the code runs on any x86-64 CPU. x86-64 targets * only; the runtime pack keeps its baseline code. */ targetCpu?: TargetCpu; + /** Give the executable an x86-64-v3 copy of each program function, which + * it runs instead of the baseline code on CPUs that support that level + * (`SCRIPTC_CPU_LEVEL` caps the level at run time). Executables for x86-64 + * targets without a `targetCpu` only; off by default. */ + cpuDispatch?: boolean; /** Instrument the executable for profile-guided optimization. Each run * writes a raw profile to this path pattern (`%p` expands to the process * id; `LLVM_PROFILE_FILE` overrides the pattern at run time). Linux @@ -54,6 +59,11 @@ export interface CompileBaseOptions { * merge-profiles`). Code the profile does not describe, or describes * from an older version of the program, is optimized as without one. */ profileUse?: string; + /** A function layout (scripts/refresh-cli-layout.mjs; gzip-compressed when + * the name ends in .gz): the listed functions are written first, by tier, + * so the code that runs together sits together in the executable. The + * native compiler's build uses it. Executables and LLVM output. */ + functionLayout?: string; /** Eligible packages compile automatically. Explicit names override automatic selection heuristics. */ npmStatic?: readonly string[] | "auto"; diff --git a/packages/compiler/src/coverage/report.ts b/packages/compiler/src/coverage/report.ts index 58a2cafa6..d78120f89 100644 --- a/packages/compiler/src/coverage/report.ts +++ b/packages/compiler/src/coverage/report.ts @@ -22,9 +22,18 @@ import { type TiersEnvelope, } from "../diagnostics/envelope.js"; import { renderDiagnostics } from "../diagnostics/render.js"; +import { + dynamicSlotLabel, + isDynamicSlotCode, + type DynamicSlotEntry, +} from "../diagnostics/dynamic-slots.js"; import type { LowerStats } from "../frontend/lowering/lowerer.js"; import type { ProvenanceSources } from "../frontend/provenance-registry.js"; -import type { DemotionUnit, TierReport } from "../frontend/tier-analysis.js"; +import type { + DemotionUnit, + LocalizedFailureReport, + TierReport, +} from "../frontend/tier-analysis.js"; import type { WasmModuleReport } from "../wasm/discover.js"; import { describe as describeReason, @@ -55,6 +64,9 @@ export interface CoverageInput { * leaves untyped. Never blockers; `--fail-on=dynamic` opts into failing * on it. */ performance?: ScrDiagnostic[]; + /** Localized failure: the checked dynamic slots (SC6104, also in + * `performance`). `--fail-on=dynamic-slots` fails on them. */ + dynamicSlots?: DynamicSlotEntry[]; /** Each dependency's native compilation outcome, with the first refusal * reason for a refused package. */ npmStatic?: NpmStaticStatus[]; @@ -234,6 +246,10 @@ export function renderCoverage(input: CoverageInput, opts: CoverageRenderOptions ` ${c(DIM, `(${skipped} function${skipped === 1 ? "" : "s"} not analyzed — signature blocked; their statements count as not static)`)}`, ); } + renderDynamicSlots(input.dynamicSlots ?? [], out, c, (loc) => { + const p = locator.position(loc.file, loc.start); + return `${displayPath(loc.file, opts.root)}:${p.line}:${p.column}`; + }); out.push(""); // Report native package compilation and the first reason for each refusal. @@ -391,7 +407,7 @@ export function renderCoverage(input: CoverageInput, opts: CoverageRenderOptions renderRows(groupBlockers(divergences), YELLOW); }; // The performance warning: its message names the functions. - const performance = input.performance ?? []; + const performance = (input.performance ?? []).filter((d) => !isDynamicSlotCode(d.code)); const renderPerformance = () => { for (const d of performance) { out.push(""); @@ -525,6 +541,7 @@ export type CoverageFailOn = | "blockers" | "divergences" | "dynamic" + | "dynamic-slots" | "node-islands" | "eager-node-islands" | "island-gaps"; @@ -533,6 +550,7 @@ export const COVERAGE_FAIL_ON: readonly CoverageFailOn[] = [ "blockers", "divergences", "dynamic", + "dynamic-slots", "node-islands", "eager-node-islands", "island-gaps", @@ -548,7 +566,9 @@ export function coveragePasses(input: CoverageInput, failOn?: CoverageFailOn): b // runtime fences build (the statement throws only if it executes). if (input.diagnostics.length > 0) return false; if (failOn === "divergences" && (input.divergences?.length ?? 0) > 0) return false; - if (failOn === "dynamic" && (input.performance?.length ?? 0) > 0) return false; + if (failOn === "dynamic" && (input.performance ?? []).some((d) => !isDynamicSlotCode(d.code))) + return false; + if (failOn === "dynamic-slots" && (input.dynamicSlots?.length ?? 0) > 0) return false; // The Node island gates (report-only islands count: they are what a // build would choose). Every island evaluates eagerly until deferred // evaluation lands, so eager-node-islands equals node-islands for now. @@ -620,6 +640,19 @@ export function coverageEnvelope( percentStatic: total === 0 ? 100 : Math.floor((ok / total) * 100), }, }), + ...(input.dynamicSlots !== undefined + ? { + dynamicSlots: input.dynamicSlots.map((slot) => ({ + file: slot.loc.file, + ...locator.position(slot.loc.file, slot.loc.start), + name: slot.name, + kind: slot.slot, + declaredType: slot.declaredType, + code: slot.code, + reason: slot.reason, + })), + } + : {}), ...(input.tiers !== undefined ? { tiers: tiersEnvelope(input.tiers, locator) } : {}), ...(input.wasmModules !== undefined ? { wasmModules: input.wasmModules } : {}), ...(input.nodeIslands !== undefined @@ -668,6 +701,26 @@ function tiersEnvelope(tiers: TierReport, locator: SourceLocator): TiersEnvelope ...at(r.loc), })), units: tiers.units.map((u) => ({ ...unit(u), requests: u.requests, sites: u.sites })), + localizedFailure: { + errors: tiers.localizedFailure.errors, + typeFailures: tiers.localizedFailure.typeFailures, + cascades: tiers.localizedFailure.cascades, + clearedThroughL2: tiers.localizedFailure.clearedThroughL2, + clearedThroughL4: tiers.localizedFailure.clearedThroughL4, + remainingByCode: tiers.localizedFailure.remainingByCode, + slots: tiers.localizedFailure.slots.map((slot) => ({ + ...at(slot.loc), + slot: slot.slot, + name: slot.name, + declaredType: slot.declaredType, + family: slot.family, + code: slot.code, + message: slot.message, + stage: slot.stage, + ...(slot.rejects !== null ? { rejects: slot.rejects } : {}), + cascade: slot.cascade, + })), + }, }; } @@ -695,6 +748,59 @@ function unitMatches(u: DemotionUnit, query: string): boolean { ); } +const DYNAMIC_SLOTS_SHOWN = 20; + +/** The dynamic slots under the statement counts: each slot's declared + * type, and why it has no static lowering. */ +function renderDynamicSlots( + slots: readonly DynamicSlotEntry[], + out: string[], + c: (code: string, s: string) => string, + where: (loc: SrcLoc) => string, +): void { + if (slots.length === 0) return; + out.push( + ` dynamic slots ${slots.length} ${c(DIM, "(declared types with no static lowering, held as checked-dynamic values; SC6104)")}`, + ); + for (const slot of slots.slice(0, DYNAMIC_SLOTS_SHOWN)) + out.push(` ${dynamicSlotLabel(slot)} ${c(DIM, where(slot.loc))}`); + if (slots.length > DYNAMIC_SLOTS_SHOWN) + out.push(` ${c(DIM, `and ${slots.length - DYNAMIC_SLOTS_SHOWN} more`)}`); +} + +const SLOTS_SHOWN = 40; + +/** The localized-failure section of --explain-dynamic (report-only): the + * committed type failures that would become checked dynamic slots, and + * the cascade each one causes today. */ +function renderLocalizedFailure( + report: LocalizedFailureReport, + out: string[], + c: (code: string, s: string) => string, + where: (loc: { file: string; start: number }) => string, +): void { + if (report.typeFailures.degradable === 0 && report.typeFailures.refused === 0) return; + out.push(""); + out.push( + ` ${report.slots.length} would-be dynamic slot${report.slots.length === 1 ? "" : "s"} ${c(DIM, "(localized failure, report-only: type failures a checked dynamic slot would hold)")}`, + ); + out.push( + ` ${c(DIM, `${report.errors} errors: ${report.clearedThroughL2} clear with declaration slots (L1-L2), ${report.clearedThroughL4} with member and expression slots too (L3-L4); ${report.cascades} cascade${report.cascades === 1 ? "" : "s"} attributed; ${report.typeFailures.refused} type failure${report.typeFailures.refused === 1 ? " stays" : "s stay"} refused`)}`, + ); + for (const slot of report.slots.slice(0, SLOTS_SHOWN)) { + const cascade = slot.cascade > 0 ? `, clears ${slot.cascade}` : ""; + out.push( + ` ${slot.stage} ${slot.slot} ${slot.name}: ${capMessage(slot.declaredType)} ${c(DIM, `${where(slot.loc)} (${slot.code} ${slot.family}${cascade})`)}`, + ); + if (slot.rejects !== null) + out.push(` ${c(DIM, `rejects ${slot.rejects.split(" @")[0]}`)}`); + } + if (report.slots.length > SLOTS_SHOWN) + out.push( + ` ${c(DIM, `${report.slots.length - SLOTS_SHOWN} more (--print=diagnostics lists all)`)}`, + ); +} + /** `scriptc coverage --explain-dynamic[=]`: the dynamic tier today * and the demotion chains the report-only analysis computed. With a query, * only units whose name matches it (`x`, `Point.x`, `parse:x`). */ @@ -735,6 +841,7 @@ export function renderExplainDynamic( (excluded > 0 ? c(DIM, `, ${excluded} excluded by policy`) : ""), ); } + if (query === undefined) renderLocalizedFailure(tiers.localizedFailure, out, c, where); const units = tiers.units.filter((u) => query === undefined || unitMatches(u, query)); if (query !== undefined && units.length === 0) { out.push(` no demoted unit matches '${query}'`); diff --git a/packages/compiler/src/coverage/verdict-cache.ts b/packages/compiler/src/coverage/verdict-cache.ts index c92cdc45e..dfaaa5c98 100644 --- a/packages/compiler/src/coverage/verdict-cache.ts +++ b/packages/compiler/src/coverage/verdict-cache.ts @@ -7,6 +7,7 @@ * still hold. With a semantic matcher (the Node frontend's), edits that * only touch TypeScript comments also reuse it, with every reported * location moved to the edited text. */ +import type { DynamicSlotEntry } from "../diagnostics/dynamic-slots.js"; import { createHash } from "node:crypto"; import { mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -23,7 +24,7 @@ import { type FrontendInputSnapshot, } from "../frontend/input-tracker.js"; -const SCHEMA = "scriptc.coverage-verdict.v3"; +const SCHEMA = "scriptc.coverage-verdict.v4"; /** Key-value storage for verdict entries (best effort: failures miss). */ export interface VerdictStore { @@ -41,6 +42,7 @@ export interface CachedCoverage { unreached?: { stats: LowerStats; diagnostics: ScrDiagnostic[]; runtimeFences?: ScrDiagnostic[] }; divergences?: ScrDiagnostic[]; performance?: ScrDiagnostic[]; + dynamicSlots?: DynamicSlotEntry[]; npmStatic?: NpmStaticStatus[]; wasmModules?: WasmModuleReport[]; nodeIslands?: NodeIslandReport; @@ -158,6 +160,7 @@ function cacheable(coverage: AnalyzeResult["coverage"]): CachedCoverage | null { if (coverage.unreached !== undefined) out.unreached = coverage.unreached; if (coverage.divergences !== undefined) out.divergences = coverage.divergences; if (coverage.performance !== undefined) out.performance = coverage.performance; + if (coverage.dynamicSlots !== undefined) out.dynamicSlots = coverage.dynamicSlots; if (coverage.npmStatic !== undefined) out.npmStatic = coverage.npmStatic; if (coverage.wasmModules !== undefined) out.wasmModules = coverage.wasmModules; if (coverage.nodeIslands !== undefined) out.nodeIslands = coverage.nodeIslands; @@ -177,6 +180,7 @@ function restore(cached: CachedCoverage, sources: CachedSource[]): AnalyzeResult if (cached.unreached !== undefined) coverage.unreached = cached.unreached; if (cached.divergences !== undefined) coverage.divergences = cached.divergences; if (cached.performance !== undefined) coverage.performance = cached.performance; + if (cached.dynamicSlots !== undefined) coverage.dynamicSlots = cached.dynamicSlots; if (cached.npmStatic !== undefined) coverage.npmStatic = cached.npmStatic; if (cached.wasmModules !== undefined) coverage.wasmModules = cached.wasmModules; if (cached.nodeIslands !== undefined) coverage.nodeIslands = cached.nodeIslands; diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 8010cbb17..8d6d397da 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -110,6 +110,64 @@ export interface ScrDiagnostic { * and class member body: it throws when the module initializes, which is * program startup for a module the entry imports. */ startup?: true; + /** Localized failure, report-only (stage L0): set where a type with no + * static lowering poisoned a slot (Lowerer.badType and the dyn class + * field policy). Never part of the rendered message. */ + typeFailure?: TypeFailure; + /** Localized failure, report-only: the declaration this diagnostic + * rejected, as a failure-group key. Set when a reference flushes a + * deferred declaration's diagnostics, and on the diagnostics of a + * declaration statement whose bindings were blocked. */ + rejects?: string; + /** Localized failure, report-only: the failure groups (rejected + * declarations, blocked statements) that the statement committing this + * diagnostic used. A cascade clears when every one of them would + * degrade instead of failing. */ + cause?: readonly string[]; +} + +/** Where a type failure sits (localized-failure design §3.6): a + * declaration slot, a member of a declared type, or an expression's own + * type. */ +export type TypeFailureSlot = + | "field" + | "param" + | "return" + | "local" + | "global" + | "loop" + | "member" + | "expression"; + +/** Which badType branch (or policy fence) refused the type. */ +export type TypeFailureFamily = + | "node-types" + | "fetch-ambient" + | "enum-object" + | "stdlib-index" + | "index-signature" + | "package" + | "generic-signature" + | "overloads" + | "intersection" + | "component" + | "stdlib-nominal" + | "record-member" + | "unsupported" + | "dyn-field-policy"; + +/** A type failure's structured payload: what the commit points of the + * localized-failure design inspect (PoisonError.failure carries the same + * object). `degradable` is the §3.7 classification: a checked-dynamic slot + * would keep the program's meaning. */ +export interface TypeFailure { + slot: TypeFailureSlot; + /** The slot's name (`_customTools`, `listAll`, `x`), or the expression + * text for expression slots. */ + name: string; + declaredType: string; + family: TypeFailureFamily; + degradable: boolean; } /** SC3002–SC3004 — native-helper target, installation, and execution @@ -879,6 +937,21 @@ export function nativeAddonDiag(specifier: string, loc: SrcLoc): ScrDiagnostic { }; } +/** A createRequire require or require.resolve call whose specifier is + * computed at run time, where the runtime resolves it like Node + * (scr_require.c). Coverage lists the call as deferred: a require whose + * specifier resolves to a module ends the program with this code, because + * the module graph is fixed at build time. */ +export function computedRequireDiag(surface: string, loc: SrcLoc): ScrDiagnostic { + return { + code: "SC2020", + message: `${surface} with a computed specifier resolves at run time; a require of a specifier that resolves to a module, such as a native addon, ends the program`, + loc, + milestone: "later", + hint: 'require("") compiles the module into the program; native addons need a replacement through native FFI (https://scriptc.dev/ffi#replacing-a-node-api-addon)', + }; +} + /** Records are monomorphic structs, so a value's shape must match the * expected shape exactly OR narrow through the width-copy family * (recordWidthPlan / the overflow capture: target fields copied off the diff --git a/packages/compiler/src/diagnostics/dynamic-slots.ts b/packages/compiler/src/diagnostics/dynamic-slots.ts new file mode 100644 index 000000000..09b4cd670 --- /dev/null +++ b/packages/compiler/src/diagnostics/dynamic-slots.ts @@ -0,0 +1,49 @@ +/* SC6104, the dynamic-slot performance warning (localized-failure design + * §3.9): a slot whose declared TypeScript type has no static lowering holds + * a checked-dynamic value instead of rejecting the declaration around it. + * Its wording lives here; the slots come from the lowering's commit points + * (Lowerer.degradeAtCommit). Imports nothing heavy, like dynamic-tier.ts. */ +import type { SrcLoc } from "../ir/ir.js"; +import type { ScrDiagnostic, TypeFailureSlot } from "./diagnostic.js"; + +export const DYNAMIC_SLOT_CODE = "SC6104"; + +/** One dynamic slot, as coverage reports it. */ +export interface DynamicSlotEntry { + loc: SrcLoc; + name: string; + slot: TypeFailureSlot; + declaredType: string; + /** The diagnostic the slot's declaration would have failed with. */ + code: string; + reason: string; +} + +const SLOT_WORD: Record = { + field: "field", + param: "parameter", + return: "return value of", + local: "local", + global: "module global", + loop: "loop variable", + member: "member", + expression: "expression", +}; + +/** `'_customTools' (declared 'ToolDefinition[]')`, with the slot kind. */ +export function dynamicSlotLabel(entry: DynamicSlotEntry): string { + return `${SLOT_WORD[entry.slot]} '${entry.name}' (declared '${entry.declaredType}')`; +} + +export function dynamicSlotDiag(entry: DynamicSlotEntry): ScrDiagnostic { + return { + code: DYNAMIC_SLOT_CODE, + message: `${dynamicSlotLabel(entry)} is held as a checked-dynamic value: its type has no static lowering (${entry.reason}); reads into typed code are checked`, + loc: entry.loc, + hint: "restate the declared type inside the compilable set to keep this slot static; scriptc coverage lists dynamic slots, and --fail-on=dynamic-slots fails on them", + }; +} + +export function isDynamicSlotCode(code: string): boolean { + return code === DYNAMIC_SLOT_CODE; +} diff --git a/packages/compiler/src/diagnostics/envelope.ts b/packages/compiler/src/diagnostics/envelope.ts index bdb545ff3..f6f8b9d3d 100644 --- a/packages/compiler/src/diagnostics/envelope.ts +++ b/packages/compiler/src/diagnostics/envelope.ts @@ -114,6 +114,27 @@ export interface TiersEnvelope { message: string; })[]; units: (TierUnitPosition & { requests: number; sites: number })[]; + /** Localized failure (report-only): committed type failures that would + * become checked dynamic slots, and what they would clear. */ + localizedFailure: { + errors: number; + typeFailures: { degradable: number; refused: number }; + cascades: number; + clearedThroughL2: number; + clearedThroughL4: number; + remainingByCode: Record; + slots: (SourcePosition & { + slot: string; + name: string; + declaredType: string; + family: string; + code: string; + message: string; + stage: string; + rejects?: string; + cascade: number; + })[]; + }; } export interface DiagnosticsEnvelope { @@ -143,6 +164,15 @@ export interface DiagnosticsEnvelope { }; /** build only: the produced artifact path. */ artifact?: string; + /** coverage only, when there are any: the checked dynamic slots + * (localized failure, SC6104). */ + dynamicSlots?: (SourcePosition & { + name: string; + kind: string; + declaredType: string; + code: string; + reason: string; + })[]; /** coverage --explain-dynamic only: the report-only tier assignment. */ tiers?: TiersEnvelope; /** coverage only: the WebAssembly modules the build can see, with their diff --git a/packages/compiler/src/executable/executable-cache.test.ts b/packages/compiler/src/executable/executable-cache.test.ts index afae8b2c1..3ba6a7c54 100644 --- a/packages/compiler/src/executable/executable-cache.test.ts +++ b/packages/compiler/src/executable/executable-cache.test.ts @@ -220,7 +220,7 @@ test("early executable cache misses on source and resolution changes", async () expect(await readEarlyExecutableCache(f.root, f.options)).toBeNull(); }); -test("early executable keys isolate compile modes, paths, implementation, checker, and FFI manifests", async () => { +test("early executable keys isolate compile modes, paths, implementation, checker, function layouts, and FFI manifests", async () => { const f = await fixture(); const tracker = new FrontendInputTracker(); tracker.run(() => trackedReadFile(f.source)); @@ -239,8 +239,10 @@ test("early executable keys isolate compile modes, paths, implementation, checke { ...f.options, detachedExit: true }, { ...f.options, targetCpu: "x86-64-v3" }, { ...f.options, targetCpu: "x86-64-v4" }, + { ...f.options, cpuDispatch: true }, { ...f.options, profile: "profile-generate:/tmp/%p.profraw" }, { ...f.options, profile: "profile-use:digest" }, + { ...f.options, functionLayout: "layout-digest" }, { ...f.options, checker: "tsc-ts-executable-identity" }, { ...f.options, npmStatic: "auto" }, { ...f.options, npmStatic: ["commander"] }, diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index ece1dc971..97b737f66 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -12,13 +12,14 @@ import { } from "node:fs/promises"; import { basename, dirname, join, resolve } from "node:path"; import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import type { NativeProfileMatch } from "../backend/native-profile.js"; import type { FrontendInputSnapshot } from "../frontend/input-tracker.js"; import { frontendInputsStillMatch, validFrontendInputSnapshot } from "../frontend/input-tracker.js"; import { compilerReleaseVersion } from "../library/sidecar.js"; import { nativeArtifactDependenciesStillMatch } from "../backend/native/artifact-stamps.js"; import { type NativeArtifactDependency } from "../backend/native/contracts.js"; import { optimizationKeyParts } from "../backend/optimization.js"; -import { type TargetCpu, targetCpuKeyParts } from "../backend/target-cpu.js"; +import { type TargetCpu, cpuDispatchKeyParts, targetCpuKeyParts } from "../backend/target-cpu.js"; import { privateSiblingPath } from "../backend/build-cache.js"; import type { CompilerImplementationDependency } from "../library/compiler-self-identity.js"; import { @@ -74,12 +75,14 @@ export interface EarlyExecutableNativeFeatures { streamDyn?: boolean; handleProps?: boolean; declAdopt?: boolean; + typeOrigins?: boolean; dynForAwait?: boolean; eventsOnceSignal?: boolean; mime?: boolean; utilTiming?: boolean; dynBuffer?: boolean; builtinValues?: boolean; + computedRequire?: boolean; events: boolean; emitter: boolean; symbol: boolean; @@ -127,6 +130,8 @@ interface EarlyExecutableCacheStamp { native: EarlyExecutableNativeFeatures; /** The build's performance warnings (SC61xx), which a cache hit repeats. */ performance?: ScrDiagnostic[]; + /** How a --profile-use build's profile matched, which a cache hit reports. */ + profileMatch?: NativeProfileMatch; integrity: string; } @@ -147,9 +152,13 @@ export interface EarlyExecutableCacheOptions { detachedExit?: true; /** Omitted for the x86-64 baseline (`--target-cpu`). */ targetCpu?: TargetCpu; + /** Omitted for builds without runtime CPU dispatch. */ + cpuDispatch?: true; /** Omitted without --profile-generate/--profile-use; otherwise the * pattern, or the digest of the profile's bytes. */ profile?: string; + /** Omitted without a function layout; otherwise the digest of its bytes. */ + functionLayout?: string; /** Omitted for the default tsgo checker; with SCRIPTC_CHECKER=tsc-ts, the * tsc-ts executable's file identity. */ checker?: string; @@ -209,12 +218,23 @@ export interface EarlyExecutableCacheHit { frontend: FrontendInputSnapshot; /** The performance warnings the build reported (SC61xx), repeated on a hit. */ performance?: ScrDiagnostic[]; + /** How a --profile-use build's profile matched, reported again on a hit. */ + profileMatch?: NativeProfileMatch; } export interface EarlyExecutableCachePublish extends EarlyExecutableCacheHit { nativeDependencies?: NativeArtifactDependency[]; } +function isProfileMatch(value: unknown): value is NativeProfileMatch { + if (typeof value !== "object" || value === null) return false; + const match = value as Record; + return ["annotated", "mismatched", "missing", "unknownPartitions"].every((key) => { + const count = match[key]; + return typeof count === "number" && Number.isInteger(count) && count >= 0; + }); +} + const BOOLEAN_NATIVE_KEYS = [ "regex", "copying", @@ -291,7 +311,9 @@ function cacheKey(options: EarlyExecutableCacheOptions): string { ...(options.windowsSubsystem === "gui" ? ["windows-subsystem-gui"] : []), ...(options.detachedExit ? ["detached-exit"] : []), ...targetCpuKeyParts(options.targetCpu), + ...cpuDispatchKeyParts(options.cpuDispatch), ...(options.profile !== undefined ? [options.profile] : []), + ...(options.functionLayout !== undefined ? [`layout:${options.functionLayout}`] : []), ...(options.checker !== undefined ? [`checker:${options.checker}`] : []), ...(options.youngNursery ? ["young-nursery"] : []), options.npmStatic === null @@ -334,7 +356,9 @@ function routeKey(options: Omit 0 ? { performance: stamp.performance } : {}), + ...(isProfileMatch(stamp.profileMatch) ? { profileMatch: stamp.profileMatch } : {}), ...(ir === null ? {} : { irPath: paths.irPath }), ...(llvmSourcePath === undefined ? {} : { llvmSourcePath }), ...(irSourcePath === undefined ? {} : { irSourcePath }), @@ -855,6 +880,7 @@ export async function publishEarlyExecutableCache( ...(result.performance === undefined || result.performance.length === 0 ? {} : { performance: result.performance }), + ...(result.profileMatch === undefined ? {} : { profileMatch: result.profileMatch }), }; const stamp: EarlyExecutableCacheStamp = { ...unsigned, integrity: stampIntegrity(unsigned) }; await writeFile(join(stage, "stamp.json"), `${JSON.stringify(stamp)}\n`, { mode: 0o600 }); diff --git a/packages/compiler/src/frontend/analysis.ts b/packages/compiler/src/frontend/analysis.ts index 848306b66..d2e881473 100644 --- a/packages/compiler/src/frontend/analysis.ts +++ b/packages/compiler/src/frontend/analysis.ts @@ -132,6 +132,7 @@ export function analyzeWithFrontend( ...(lowered.runtimeFences.length > 0 ? { runtimeFences: lowered.runtimeFences } : {}), ...(lowered.divergences !== undefined ? { divergences: lowered.divergences } : {}), ...(lowered.performance !== undefined ? { performance: lowered.performance } : {}), + ...(lowered.dynamicSlots !== undefined ? { dynamicSlots: lowered.dynamicSlots } : {}), ...(lowered.unreached ? { unreached: lowered.unreached } : {}), ...(fe.npmStatic.length > 0 ? { npmStatic: fe.npmStatic } : {}), // --provenance-sources: the per-package attribution inputs (the diff --git a/packages/compiler/src/frontend/canon.ts b/packages/compiler/src/frontend/canon.ts new file mode 100644 index 000000000..5d75ae806 --- /dev/null +++ b/packages/compiler/src/frontend/canon.ts @@ -0,0 +1,479 @@ +/* Canonical type normalization (localized-failure design part 2, §4), + * stage N0: report-only. Nothing in the compiler consumes it yet. + * + * `canon(type, env)` resolves a checker type, under the current generic + * bindings, to a small set of canonical shapes: primitive, literal, unit, + * top, object (nominal or structural, with lazily resolved members), array, + * tuple, flat union, unbound parameter, and an opaque leaf for what it + * cannot reduce. The per-form rules of type-mapper.ts (generic aliases, + * equal-member intersections, narrowed parameters, recursive placeholders) + * are what this replaces family by family in later stages; N1 runs it in + * shadow mode beside mapType, N2 as mapType's fallback. + * + * Nodes intern by checker type identity and the bindings of the type + * parameters the walk consulted, so a recursive type closes on itself once + * (a back-reference returns the node under construction). */ +import * as ts from "./ts7/adapter.js"; + +export type CanonPrim = "number" | "string" | "boolean" | "bigint" | "symbol"; + +export interface CanonProp { + name: string; + optional: boolean; + readonly: boolean; + type: () => Canon; +} + +export interface CanonSig { + params: { name: string; optional: boolean; rest: boolean; type: () => Canon }[]; + ret: () => Canon; + /** The signature keeps type parameters of its own. */ + generic: boolean; +} + +export interface CanonIndex { + key: Canon; + value: () => Canon; +} + +export interface AliasRef { + name: string; + args: readonly Canon[]; + /** The checker type the alias reference came from (identity preserved + * for tables keyed by ts.Type, such as boundAliasInstances). */ + tsType: ts.Type; +} + +export type Canon = + | { k: "prim"; p: CanonPrim } + | { k: "lit"; value: string | number | boolean } + | { k: "unit"; u: "undefined" | "null" | "void" | "never" } + | { k: "top"; t: "unknown" | "any" | "object" } + | { + k: "obj"; + /** Class or standard-library declaration name, when nominal. */ + nominal?: string; + /** The type arguments of a nominal generic instantiation. */ + typeArgs: () => readonly Canon[]; + alias?: AliasRef; + props: () => readonly CanonProp[]; + index: () => readonly CanonIndex[]; + call: () => readonly CanonSig[]; + construct: () => readonly CanonSig[]; + } + | { k: "array"; elem: Canon; readonly: boolean } + | { k: "tuple"; elems: readonly { type: Canon; optional: boolean; rest: boolean }[] } + | { k: "union"; arms: readonly Canon[] } + | { k: "param"; name: string } + | { k: "opaque"; ts: ts.Type; reason: string }; + +/** Type-parameter bindings: the checker type a parameter is bound to. */ +export type CanonEnv = ReadonlyMap; + +const PRIM_FLAGS: readonly [number, CanonPrim][] = [ + [ts.TypeFlags.Number, "number"], + [ts.TypeFlags.String, "string"], + [ts.TypeFlags.Boolean, "boolean"], + [ts.TypeFlags.BigInt, "bigint"], + [ts.TypeFlags.ESSymbol | ts.TypeFlags.UniqueESSymbol, "symbol"], +]; + +/** A stable key for a canonical node, for deduplication and tests. */ +export function canonKey(c: Canon, depth = 0): string { + if (depth > 6) return "…"; + switch (c.k) { + case "prim": + return c.p; + case "lit": + return JSON.stringify(c.value); + case "unit": + return c.u; + case "top": + return c.t; + case "param": + return `param ${c.name}`; + case "opaque": + return `opaque(${c.reason})`; + case "array": + return `${c.readonly ? "readonly " : ""}${canonKey(c.elem, depth + 1)}[]`; + case "tuple": + return `[${c.elems.map((e) => `${e.rest ? "..." : ""}${canonKey(e.type, depth + 1)}${e.optional ? "?" : ""}`).join(", ")}]`; + case "union": + return c.arms.map((a) => canonKey(a, depth + 1)).join(" | "); + case "obj": { + if (c.nominal !== undefined) { + const args = c.typeArgs(); + return args.length === 0 + ? c.nominal + : `${c.nominal}<${args.map((a) => canonKey(a, depth + 1)).join(", ")}>`; + } + const props = c + .props() + .map((p) => `${p.name}${p.optional ? "?" : ""}: ${canonKey(p.type(), depth + 1)}`); + const sig = (s: CanonSig): string => + `(${s.params + .map( + (p) => `${p.rest ? "..." : ""}${p.optional ? "?" : ""}${canonKey(p.type(), depth + 1)}`, + ) + .join(", ")}) => ${canonKey(s.ret(), depth + 1)}`; + const calls = c.call().map(sig); + const constructs = c.construct().map((s) => `new ${sig(s)}`); + const index = c + .index() + .map((i) => `[${canonKey(i.key, depth + 1)}]: ${canonKey(i.value(), depth + 1)}`); + return `{ ${[...calls, ...constructs, ...index, ...props].join("; ")} }`; + } + } +} + +/** The pairs a comparison takes as equal while it runs, and the order they + * were assumed in, so that a failed comparison withdraws what it assumed. */ +interface Assumptions { + pairs: Map>; + trail: [Canon, Canon][]; +} + +/** Whether two canonical nodes are the same type, structurally: unlike + * canonKey, every part is compared (signature parameters, index + * signatures, type arguments, any depth). A pair met again while it is + * being compared is taken as equal, which closes the comparison of + * recursive types; when the comparison fails, the pairs assumed during it + * are withdrawn. */ +export function canonEqual( + a: Canon, + b: Canon, + assumed: Assumptions = { pairs: new Map(), trail: [] }, +): boolean { + if (a === b) return true; + if (a.k !== b.k) return false; + let pairs = assumed.pairs.get(a); + if (pairs?.has(b)) return true; + if (pairs === undefined) assumed.pairs.set(a, (pairs = new Set())); + const mark = assumed.trail.length; + pairs.add(b); + assumed.trail.push([a, b]); + const equal = sameCanon(a, b, assumed); + if (!equal) + while (assumed.trail.length > mark) { + const [x, y] = assumed.trail.pop()!; + assumed.pairs.get(x)!.delete(y); + } + return equal; +} + +function sameCanon(a: Canon, b: Canon, assumed: Assumptions): boolean { + const eq = (x: Canon, y: Canon): boolean => canonEqual(x, y, assumed); + const sigsEqual = (xs: readonly CanonSig[], ys: readonly CanonSig[]): boolean => + xs.length === ys.length && + xs.every((x, i) => { + const y = ys[i]!; + return ( + x.generic === y.generic && + x.params.length === y.params.length && + x.params.every( + (p, j) => + p.optional === y.params[j]!.optional && + p.rest === y.params[j]!.rest && + eq(p.type(), y.params[j]!.type()), + ) && + eq(x.ret(), y.ret()) + ); + }); + switch (a.k) { + case "prim": + return a.p === (b as typeof a).p; + case "lit": + return a.value === (b as typeof a).value; + case "unit": + return a.u === (b as typeof a).u; + case "top": + return a.t === (b as typeof a).t; + case "param": + return a.name === (b as typeof a).name; + case "opaque": + return a.ts === (b as typeof a).ts; + case "array": { + const o = b as typeof a; + return a.readonly === o.readonly && eq(a.elem, o.elem); + } + case "tuple": { + const o = b as typeof a; + return ( + a.elems.length === o.elems.length && + a.elems.every( + (e, i) => + e.optional === o.elems[i]!.optional && + e.rest === o.elems[i]!.rest && + eq(e.type, o.elems[i]!.type), + ) + ); + } + case "union": { + const o = b as typeof a; + return ( + a.arms.length === o.arms.length && + a.arms.every((x) => o.arms.some((y) => eq(x, y))) && + o.arms.every((y) => a.arms.some((x) => eq(x, y))) + ); + } + case "obj": { + const o = b as typeof a; + if (a.nominal !== o.nominal) return false; + if (a.nominal !== undefined) { + const ax = a.typeArgs(); + const ox = o.typeArgs(); + return ax.length === ox.length && ax.every((x, i) => eq(x, ox[i]!)); + } + const ap = a.props(); + const op = o.props(); + if (ap.length !== op.length) return false; + for (const p of ap) { + const q = op.find((x) => x.name === p.name); + if (q === undefined || q.optional !== p.optional || !eq(p.type(), q.type())) return false; + } + const ai = a.index(); + const oi = o.index(); + return ( + ai.length === oi.length && + ai.every((x, i) => eq(x.key, oi[i]!.key) && eq(x.value(), oi[i]!.value())) && + sigsEqual(a.call(), o.call()) && + sigsEqual(a.construct(), o.construct()) + ); + } + } +} + +export class Normalizer { + private readonly interned = new Map(); + /** A number for each type parameter an environment binds: parameters + * with the same name are distinct keys. */ + private readonly paramIds = new Map(); + + constructor(private readonly checker: ts.TypeChecker) {} + + canon(type: ts.Type, env: CanonEnv = new Map()): Canon { + const id = (type as { id?: number }).id; + const key = id === undefined ? null : `${id}|${this.envKey(env)}`; + if (key !== null) { + const known = this.interned.get(key); + if (known !== undefined) return known; + } + const node = this.build(type, env, key); + if (key !== null && !this.interned.has(key)) this.interned.set(key, node); + return node; + } + + private envKey(env: CanonEnv): string { + if (env.size === 0) return ""; + const parts: string[] = []; + for (const [param, bound] of env) { + let id = this.paramIds.get(param); + if (id === undefined) this.paramIds.set(param, (id = this.paramIds.size)); + parts.push(`${id}=${(bound as { id?: number }).id ?? "?"}`); + } + return parts.sort().join(","); + } + + private build(type: ts.Type, env: CanonEnv, key: string | null): Canon { + const flags = type.flags; + if (flags & ts.TypeFlags.Any) return { k: "top", t: "any" }; + if (flags & ts.TypeFlags.Unknown) return { k: "top", t: "unknown" }; + if (flags & ts.TypeFlags.NonPrimitive) return { k: "top", t: "object" }; + if (flags & ts.TypeFlags.Never) return { k: "unit", u: "never" }; + if (flags & ts.TypeFlags.Void) return { k: "unit", u: "void" }; + if (flags & ts.TypeFlags.Undefined) return { k: "unit", u: "undefined" }; + if (flags & ts.TypeFlags.Null) return { k: "unit", u: "null" }; + if (flags & ts.TypeFlags.TemplateLiteral) return { k: "prim", p: "string" }; + if (flags & (ts.TypeFlags.StringLiteral | ts.TypeFlags.NumberLiteral)) { + const value = (type as { value?: string | number }).value; + if (value !== undefined) return { k: "lit", value }; + } + if (flags & ts.TypeFlags.BooleanLiteral) + return { k: "lit", value: this.checker.typeToString(type) === "true" }; + if (!type.isUnionType()) + for (const [flag, p] of PRIM_FLAGS) if (flags & flag) return { k: "prim", p }; + if (flags & ts.TypeFlags.TypeParameter) { + const symbol = type.getSymbol(); + const bound = symbol === undefined ? undefined : env.get(symbol); + if (bound !== undefined && bound !== type) return this.canon(bound, env); + return { k: "param", name: symbol === undefined ? "?" : ts.symbolName(symbol) }; + } + if (type.isUnionType()) return this.union(ts.constituentTypes(type), env); + if (type.isIntersectionType()) return this.intersection(type, env, key); + if (flags & (ts.TypeFlags.Conditional | ts.TypeFlags.IndexedAccess | ts.TypeFlags.Index)) + return { k: "opaque", ts: type, reason: "symbolic type operator" }; + if (flags & ts.TypeFlags.Object) return this.object(type, env, key); + return { k: "opaque", ts: type, reason: `flags ${flags}` }; + } + + /** Flat, deduplicated, never-free; `void` reads as `undefined`. */ + private union(parts: readonly ts.Type[], env: CanonEnv): Canon { + const arms: Canon[] = []; + const add = (c: Canon): void => { + if (c.k === "union") { + for (const a of c.arms) add(a); + return; + } + if (c.k === "unit" && c.u === "never") return; + const arm: Canon = c.k === "unit" && c.u === "void" ? { k: "unit", u: "undefined" } : c; + // Objects dedupe by identity (they intern by checker type): comparing + // their members would walk into a recursive type still being built. + if (arms.some((a) => (arm.k === "obj" ? a === arm : canonEqual(a, arm)))) return; + arms.push(arm); + }; + for (const part of parts) add(this.canon(part, env)); + // Units last, in a fixed order; the rest keep the checker's order. + const unitRank = (c: Canon): number => + c.k !== "unit" ? 0 : c.u === "null" ? 1 : c.u === "undefined" ? 2 : 3; + arms.sort((a, b) => unitRank(a) - unitRank(b)); + if (arms.length === 0) return { k: "unit", u: "never" }; + if (arms.length === 1) return arms[0]!; + // `true | false` is `boolean`. + const bools = arms.filter((a) => a.k === "lit" && typeof a.value === "boolean"); + if (bools.length === 2) { + const rest = arms.filter((a) => !(a.k === "lit" && typeof a.value === "boolean")); + const merged: Canon[] = [{ k: "prim", p: "boolean" }, ...rest]; + return merged.length === 1 ? merged[0]! : { k: "union", arms: merged }; + } + return { k: "union", arms }; + } + + /** Equal constituents collapse; object-like constituents without + * signatures merge through the checker's resolved properties; a + * primitive branded with an object is the primitive. */ + private intersection(type: ts.Type, env: CanonEnv, key: string | null): Canon { + const parts = ts.constituentTypes(type).map((t) => this.canon(t, env)); + if (parts.every((p) => canonEqual(p, parts[0]!))) return parts[0]!; + // Narrowed parameters: `T & {}` (and `T & ({} | null)`'s object part) + // keep T without its undefined and null arms. + const empty = (p: Canon): boolean => + p.k === "obj" && + p.nominal === undefined && + p.props().length === 0 && + p.index().length === 0 && + p.call().length === 0 && + p.construct().length === 0; + const rest = parts.filter((p) => !empty(p)); + if (rest.length === 1 && rest.length < parts.length) { + const only = rest[0]!; + if (only.k !== "union") return only.k === "unit" ? { k: "unit", u: "never" } : only; + const arms = only.arms.filter((a) => a.k !== "unit"); + return arms.length === 1 ? arms[0]! : { k: "union", arms }; + } + const prims = parts.filter((p) => p.k === "prim" || p.k === "lit"); + if (prims.length === 1 && parts.every((p) => p === prims[0] || p.k === "obj")) return prims[0]!; + if (parts.every((p) => p.k === "obj" || (p.k === "top" && p.t === "object"))) + return this.object(type, env, key); + return { k: "opaque", ts: type, reason: "intersection" }; + } + + private object(type: ts.Type, env: CanonEnv, key: string | null): Canon { + const checker = this.checker; + // Arrays and tuples enter the intern table before their elements are + // normalized, so a recursive alias (`type Tree = Tree[] | number`) + // closes on the node under construction. + if (checker.isArrayType(type)) { + const [elem] = checker.getTypeArguments(type as ts.TypeReference); + const node: Canon & { k: "array" } = { + k: "array", + elem: { k: "top", t: "unknown" }, + readonly: + type.getSymbol() !== undefined && ts.symbolName(type.getSymbol()!) === "ReadonlyArray", + }; + if (key !== null) this.interned.set(key, node); + if (elem !== undefined) node.elem = this.canon(elem, env); + return node; + } + if (checker.isTupleType(type)) { + const ref = type as ts.TupleTypeReference; + // The element flags live on the tuple shape: the type itself, or the + // target of a reference to it. + const shape = + (ref.elementFlags as ts.ElementFlags[] | undefined) !== undefined + ? ref + : (ref.getTarget() as ts.TupleType | undefined); + const flags = shape?.elementFlags ?? []; + const elems: { type: Canon; optional: boolean; rest: boolean }[] = []; + const node: Canon = { k: "tuple", elems }; + if (key !== null) this.interned.set(key, node); + checker.getTypeArguments(ref).forEach((a, i) => { + const f = flags[i] ?? ts.ElementFlags.Required; + elems.push({ + type: this.canon(a, env), + optional: (f & ts.ElementFlags.Optional) !== 0, + rest: (f & (ts.ElementFlags.Rest | ts.ElementFlags.Variadic)) !== 0, + }); + }); + return node; + } + const symbol = type.getSymbol(); + const decl = symbol?.declarations[0]; + const nominal = + decl !== undefined && (ts.isClassDeclaration(decl) || ts.isClassExpression(decl)) + ? ts.symbolName(symbol!) + : decl !== undefined && + ts.isInterfaceDeclaration(decl) && + ts.getSourceFileOfNode(decl).isDeclarationFile && + /\/lib\.[^/]*\.d\.ts$/.test(ts.getSourceFileOfNode(decl).fileName) + ? ts.symbolName(symbol!) + : undefined; + const aliasSymbol = type.getAliasSymbol(); + const alias: AliasRef | undefined = + aliasSymbol === undefined + ? undefined + : { + name: ts.symbolName(aliasSymbol), + args: type.getAliasTypeArguments().map((a) => this.canon(a, env)), + tsType: type, + }; + let typeArgs: readonly Canon[] | undefined; + let props: readonly CanonProp[] | undefined; + let index: readonly CanonIndex[] | undefined; + let call: readonly CanonSig[] | undefined; + let construct: readonly CanonSig[] | undefined; + const sigs = (list: readonly ts.Signature[]): CanonSig[] => + list.map((s) => ({ + params: s.getParameters().map((p) => { + const pd = p.declarations[0]; + const param = pd !== undefined && ts.isParameter(pd) ? pd : undefined; + return { + name: ts.symbolName(p), + optional: param?.questionToken !== undefined || param?.initializer !== undefined, + rest: param?.dotDotDotToken !== undefined, + type: () => this.canon(checker.getTypeOfSymbol(p), env), + }; + }), + ret: () => this.canon(checker.getReturnTypeOfSignature(s), env), + generic: s.getTypeParameters().length > 0, + })); + const node: Canon = { + k: "obj", + ...(nominal !== undefined ? { nominal } : {}), + ...(alias !== undefined ? { alias } : {}), + typeArgs: () => + (typeArgs ??= + nominal !== undefined && + ((type as ts.ObjectType).objectFlags & ts.ObjectFlags.Reference) !== 0 + ? checker.getTypeArguments(type as ts.TypeReference).map((a) => this.canon(a, env)) + : []), + props: () => + (props ??= checker.getPropertiesOfType(type).map((p) => ({ + name: ts.symbolName(p), + optional: (p.flags & ts.SymbolFlags.Optional) !== 0, + readonly: false, + type: () => this.canon(checker.getTypeOfSymbol(p), env), + }))), + index: () => + (index ??= checker.getIndexInfosOfType(type).map((info) => ({ + key: this.canon(info.keyType, env), + value: () => this.canon(info.type, env), + }))), + call: () => (call ??= sigs(checker.getCallSignatures(type))), + construct: () => (construct ??= sigs(checker.getConstructSignatures(type))), + }; + // Recursion closes here: members resolve lazily, so a back-reference + // reaches this node through the intern table. + if (key !== null) this.interned.set(key, node); + return node; + } +} diff --git a/packages/compiler/src/frontend/decl-projection.ts b/packages/compiler/src/frontend/decl-projection.ts index a999559c2..eebe1f773 100644 --- a/packages/compiler/src/frontend/decl-projection.ts +++ b/packages/compiler/src/frontend/decl-projection.ts @@ -27,6 +27,59 @@ * scriptc-perf/results/design-decl-boundary/DESIGN.md. */ import * as ts from "./ts7/adapter.js"; +import { declarationSelector, originAt } from "./lowering/decl-origin.js"; + +/** Per user file, the origin (lowering/decl-origin.ts) of each boundary + * cast in source order: the package declaration the cast's type comes + * from, so a failing cast names it. Set by planDeclProjection, reset per + * program load. */ +const boundaryOrigins = new Map(); + +export function resetDeclBoundaryOrigins(): void { + boundaryOrigins.clear(); +} + +/** The origin of the `index`-th boundary cast of a file, or null. */ +export function declBoundaryOrigin(fileName: string, index: number): string | null { + return boundaryOrigins.get(fileName)?.[index] ?? null; +} + +/** The package declaration a projected expression's type comes from, in + * the declaration world. */ +function castOrigin(checker: ts.TypeChecker, node: ts.Node, type: ts.Type): string | null { + let inner = node as ts.Expression; + while ( + ts.isParenthesizedExpression(inner) || + ts.isAwaitExpression(inner) || + ts.isNonNullExpression(inner) + ) + inner = inner.expression; + let decl: ts.Node | undefined; + let suffix = ""; + try { + if (ts.isCallExpression(inner) || ts.isNewExpression(inner)) { + decl = checker.getResolvedSignature(inner)?.declaration; + suffix = "()"; + } else { + const name = ts.isPropertyAccessExpression(inner) + ? inner.name + : ts.isIdentifier(inner) + ? inner + : null; + if (name === null) return null; + let symbol = checker.getSymbolAtLocation(name); + if (symbol !== undefined && symbol.flags & ts.SymbolFlags.Alias) + symbol = checker.getAliasedSymbol(symbol); + decl = symbol?.declarations[0]; + } + if (decl === undefined) return null; + const selector = declarationSelector(decl); + if (selector === null) return null; + return originAt(decl, `${selector}${suffix}`, checker.typeToString(type)); + } catch { + return null; + } +} /** Alias names of projected types start with this prefix; the lowering * recognizes boundary casts and parameters by it. */ @@ -573,6 +626,7 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | const declSf = input.declProgram.getSourceFile(file); if (jsSf === undefined || declSf === undefined || jsSf.text !== declSf.text) continue; const fileEdits: Edit[] = []; + const fileOrigins: (string | null)[] = []; /** Candidate expressions (by js node) with their declared type, named * only for the casts that survive suppression. */ const exprCandidates = new Map(); @@ -946,6 +1000,12 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | const candidate = exprCandidates.get(e)!; const alias = candidate === "any" ? "any" : printer.project(candidate); if (alias === null) continue; + // Only casts to a projected alias are boundary casts the lowering + // counts (an `as any` cast is not one), so only they take a slot. + if (candidate !== "any") { + const twin = twins.get(e); + fileOrigins.push(twin === undefined ? null : castOrigin(declChecker, twin, candidate)); + } const start = ts.tokenStartOfNode(e); const stmt = e.parent; let prefix = "("; @@ -980,6 +1040,7 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | if (fileEdits.length === 0) continue; edits.set(file, fileEdits); + if (fileOrigins.some((o) => o !== null)) boundaryOrigins.set(file, fileOrigins); } if (edits.size === 0) return null; diff --git a/packages/compiler/src/frontend/lowering/array-values.ts b/packages/compiler/src/frontend/lowering/array-values.ts index ed52b8da5..a8c2d699b 100644 --- a/packages/compiler/src/frontend/lowering/array-values.ts +++ b/packages/compiler/src/frontend/lowering/array-values.ts @@ -3,8 +3,10 @@ import { InternalCompilerError } from "../../errors.js"; import { dynUndefinedExpr, helperLoc, varRef } from "../../ir/build.js"; import { BOOL, + DYN, F64, type IrExpr, + type IrLocal, type IrStmt, type IrType, isIntegerBytesElem, @@ -446,6 +448,167 @@ export function appendElement( ]; } +/** The argument a helper passes in place of an element its callback never + * reads (lowerArrayCallback's `unread`): a constant of the parameter's + * type, so the slot stays unread and an element the type cannot hold + * (stored through a dynamic view) passes. Null for a type without one. */ +export function unreadArgument(lowerer: Lowerer, type: IrType, loc: SrcLoc): IrExpr | null { + if (type.kind === "f64") return { kind: "numLit", value: NaN, type: F64, loc }; + if (type.kind === "bool") return { kind: "boolLit", value: false, type: BOOL, loc }; + if (type.kind === "string") return { kind: "strLit", value: "", type: STRING, loc }; + if (type.kind === "union" && lowerer.armTag(type.unionId, UNDEFINED_T) >= 0) + return lowerer.wrappedUndefined(type, loc) ?? null; + return null; +} + +/** The locals of an element snapshot (arraySlotSnapshot): whether the slot + * held an element outside the element type, the value read otherwise, and + * that element's dynamic value otherwise. */ +export interface SlotSnapshot { + isForeign: string; + value: string; + foreign: string; +} + +/** The locals a snapshot of an `elem` element uses. */ +export function slotSnapshotLocals(lowerer: Lowerer, elem: IrType, ids: SlotSnapshot): IrLocal[] { + return [ + { id: ids.isForeign, name: "foreign", type: BOOL, mutable: false }, + { id: ids.value, name: "value", type: arrayValueType(lowerer, elem), mutable: false }, + { id: ids.foreign, name: "element", type: DYN, mutable: false }, + ]; +} + +/** Read the element at `arr[index]` as it is stored, before a callback runs, + * as JavaScript reads it before calling a callback that ignores it: a value + * or a present undefined into `ids.value`, or an element outside the element + * type (stored through a dynamic view) as its dynamic value into + * `ids.foreign`, which `ids.isForeign` tells apart. */ +export function arraySlotSnapshot( + lowerer: Lowerer, + arr: IrExpr, + index: IrExpr, + elem: IrType, + ids: SlotSnapshot, + loc: SrcLoc, +): IrStmt[] { + const valueT = arrayValueType(lowerer, elem); + const isForeign: IrExpr = { kind: "varRef", localId: ids.isForeign, type: BOOL, loc }; + const missing = + valueT.kind === "dyn" ? dynUndefinedExpr(loc) : lowerer.wrappedUndefined(valueT, loc)!; + return [ + { + kind: "varDecl", + localId: ids.isForeign, + init: { + kind: "bin", + op: "===", + left: { kind: "arrayState", arr, index, type: F64, loc }, + right: { kind: "numLit", value: 3, type: F64, loc }, + type: BOOL, + loc, + }, + loc, + }, + { + kind: "varDecl", + localId: ids.value, + init: { + kind: "ternary", + cond: isForeign, + then: missing, + else_: arrayValueRead(lowerer, arr, index, elem, loc), + type: valueT, + loc, + }, + loc, + }, + { + kind: "varDecl", + localId: ids.foreign, + init: { + kind: "ternary", + cond: isForeign, + then: { kind: "libCall", fn: "array.foreign", args: [arr, index], type: DYN, loc }, + else_: dynUndefinedExpr(loc), + type: DYN, + loc, + }, + loc, + }, + ]; +} + +/** Append a snapshot (arraySlotSnapshot) of an element to `out`, an array of + * the same element type, as it was stored: an element outside the element + * type stays one, appended as the view's push appends it. `out` is a + * binding. */ +export function arraySnapshotAppend( + lowerer: Lowerer, + out: IrExpr, + elem: IrType, + ids: SlotSnapshot, + loc: SrcLoc, +): IrStmt { + const end: IrExpr = { + kind: "arrIntrinsic", + method: "length", + receiver: out, + args: [], + type: F64, + loc, + }; + const value: IrExpr = { + kind: "varRef", + localId: ids.value, + type: arrayValueType(lowerer, elem), + loc, + }; + return { + kind: "if", + cond: { kind: "varRef", localId: ids.isForeign, type: BOOL, loc }, + then: [ + { + kind: "exprStmt", + expr: { + kind: "dynInvoke", + recv: { kind: "dynFrom", value: out, liveRef: true, type: DYN, loc }, + method: "push", + calleeName: "push", + args: [{ kind: "varRef", localId: ids.foreign, type: DYN, loc }], + type: DYN, + loc, + }, + loc, + }, + ], + else_: [arrayValueStore(lowerer, out, end, value, elem, loc)], + loc, + }; +} + +/** A snapshot (arraySlotSnapshot) as a value of `type`: the value read, or + * the element outside the element type through the checked conversion, + * which such an element fails as a typed read does. `typed` converts the + * value read. */ +export function arraySnapshotValue( + lowerer: Lowerer, + elem: IrType, + ids: SlotSnapshot, + type: IrType, + typed: (value: IrExpr) => IrExpr, + loc: SrcLoc, +): IrExpr { + return { + kind: "ternary", + cond: { kind: "varRef", localId: ids.isForeign, type: BOOL, loc }, + then: lowerer.coerceToExpected({ kind: "varRef", localId: ids.foreign, type: DYN, loc }, type), + else_: typed({ kind: "varRef", localId: ids.value, type: arrayValueType(lowerer, elem), loc }), + type, + loc, + }; +} + /** Store a stabilized value without changing the array's element ABI. */ export function arrayValueStore( lowerer: Lowerer, @@ -529,6 +692,9 @@ export function lowerSafeIndexRead( const elem = arr.type.elem; if (elem.kind === "void") return null; const resultT = arrayValueType(lowerer, elem); + // An element union that cannot take an undefined arm has no value for a + // missing element. + if (resultT.kind !== "dyn" && lowerer.wrappedUndefined(resultT, loc) === null) return null; const key = `idxOr:${typeKey(elem)}`; let name = lowerer.arrHofHelpers.get(key); if (!name) { diff --git a/packages/compiler/src/frontend/lowering/builtins/disposable-stack.ts b/packages/compiler/src/frontend/lowering/builtins/disposable-stack.ts index 375c3f1e7..866358b06 100644 --- a/packages/compiler/src/frontend/lowering/builtins/disposable-stack.ts +++ b/packages/compiler/src/frontend/lowering/builtins/disposable-stack.ts @@ -73,8 +73,7 @@ export function lowerDisposableStackUseCall( if (valueT?.kind !== "object") return null; const loc = locOf(call); const className = valueT.className; - const info = lowerer.classes.get(className); - if (!info) lowerer.flushDeferredClass(className); + const info = lowerer.classes.get(className) ?? lowerer.flushDeferredClass(className); const found = info ? lowerer.findMethodOn(info, "sym:dispose") : null; if (!found) return null; if (found.sig.params.length !== 0) { diff --git a/packages/compiler/src/frontend/lowering/builtins/filesystem.ts b/packages/compiler/src/frontend/lowering/builtins/filesystem.ts index 6b988c491..d8272c795 100644 --- a/packages/compiler/src/frontend/lowering/builtins/filesystem.ts +++ b/packages/compiler/src/frontend/lowering/builtins/filesystem.ts @@ -1306,6 +1306,7 @@ export function lowerFsReaddirTypesCall( ); } let sawWithFileTypes = false; + let recursive: ts.Expression | null = null; for (const p of optsNode.properties.nodes) { const m = optionMember(p); if (!m) { @@ -1334,9 +1335,11 @@ export function lowerFsReaddirTypesCall( "names decode as utf8 (the default); encoding: 'buffer' has no lowering", ); } + } else if (m.name === "recursive" && !promiseForm) { + recursive = m.value; } else { // The options-record stance: recursive (a documented knob with - // no lowering) fences by name; undocumented keys drop like Node. + // no lowering here) fences by name; undocumented keys drop like Node. fenceOrDropOptionKey( lowerer, p, @@ -1347,6 +1350,26 @@ export function lowerFsReaddirTypesCall( ); } } + if (recursive !== null) { + // Only the `true`/`false` keywords fold; any other expression, even one + // typed as a literal, evaluates at run time for its effects. + let flag = recursive; + while (ts.isParenthesizedExpression(flag)) flag = flag.expression; + const literal = + flag.kind === ts.SyntaxKind.TrueKeyword + ? "true" + : flag.kind === ts.SyntaxKind.FalseKeyword + ? "false" + : null; + if (sawWithFileTypes && literal !== "false") { + lowerer.noLowering( + `${operation} with withFileTypes and recursive`, + recursive, + "recursive: true lists relative names; withFileTypes: true lists one directory's Dirents", + ); + } + if (!sawWithFileTypes) return lowerFsReaddirRecursive(lowerer, call, recursive, literal, loc); + } if (!sawWithFileTypes) { lowerer.noLowering( `${operation} with 2 arguments`, @@ -1388,6 +1411,75 @@ export function lowerFsReaddirTypesCall( }; } +/** `fs.readdirSync(path, { recursive })` without withFileTypes: the + * relative names of every entry below the path when `recursive` is true + * (scr_fs_readdir_recursive, Node's queue order), the directory's own names + * when it is false. A recursive flag that is not a boolean literal selects + * between the two at run time, after the path is evaluated. */ +function lowerFsReaddirRecursive( + lowerer: Lowerer, + call: ts.CallExpression, + recursive: ts.Expression, + literal: string | null, + loc: SrcLoc, +): IrExpr { + const names = arrayOf(STRING); + // @types/node types the call `string[] | Buffer[]` when the options name + // no encoding; the names are strings, so the call is the string[] arm. + const callType = lowerer.mapTypeOf(lowerer.typeOf(call)); + if ( + !callType || + !( + typeEquals(callType, names) || + (callType.kind === "union" && lowerer.armTag(callType.unionId, names) >= 0) + ) + ) { + lowerer.noLowering( + "readdirSync(path, { recursive }) where the result is not a string array", + call, + "names decode as utf8 strings (the default encoding)", + ); + } + return lowerFsReaddirNames(lowerer, call, recursive, literal, names, loc); +} + +function lowerFsReaddirNames( + lowerer: Lowerer, + call: ts.CallExpression, + recursive: ts.Expression, + literal: string | null, + names: IrType, + loc: SrcLoc, +): IrExpr { + const path = lowerFsPath(lowerer, call.arguments.nodes[0]!); + const listing = (fn: IrLibFn, arg: IrExpr): IrExpr => ({ + kind: "libCall", + fn, + args: [arg], + type: names, + loc, + }); + if (literal === "true") return listing("fs.readdirRecursiveSync", path); + if (literal === "false") return listing("fs.readdirSync", path); + const pathLocal = lowerer.declareHiddenLocal("%readdirPath", STRING); + const pathRef = varRef(pathLocal.id, STRING, loc); + const flag = lowerer.lowerExprExpecting(recursive, BOOL); + return { + kind: "seqExpr", + stmts: [{ kind: "varDecl", localId: pathLocal.id, init: path, loc }], + result: { + kind: "ternary", + cond: flag, + then: listing("fs.readdirRecursiveSync", pathRef), + else_: listing("fs.readdirSync", pathRef), + type: names, + loc, + }, + type: names, + loc, + }; +} + /** fs.statSync / fs.lstatSync's documented option keys. */ const FS_STAT_DOCUMENTED_OPTIONS: ReadonlySet = new Set(["bigint", "throwIfNoEntry"]); diff --git a/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts b/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts index 48e7783a3..60dd8007f 100644 --- a/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts +++ b/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts @@ -1,4 +1,4 @@ -import { nodeThrowExpr, boolLit, strLit, varRef } from "../../../ir/build.js"; +import { dynUndefinedExpr, nodeThrowExpr, boolLit, strLit, varRef } from "../../../ir/build.js"; import { dirname, isAbsolute, resolve } from "node:path"; import * as ts from "../../ts7/adapter.js"; import { type Lowerer, PoisonError } from "../lowerer.js"; @@ -15,12 +15,14 @@ import { type RuntimeResolveResult, } from "../../runtime-resolve.js"; import { + computedRequireDiag, invalidJsonModuleDiag, nativeAddonDiag, noLoweringDiag, unsupportedPackageImportDiag, } from "../../../diagnostics/diagnostic.js"; -import { NODE_BUILTIN_MODULES_V24 } from "../surfaces.js"; +import { NODE_BUILTIN_MODULES_V24, builtinModulesArrayLit } from "../surfaces.js"; +import { moduleFileName } from "../import-meta.js"; import { BOOL, DYN, @@ -138,6 +140,51 @@ export function lowerNodeModuleCall( }; } +/** The run-time arguments of scr_require.c's computed require and + * require.resolve after the specifier: the requiring file (Node's require + * stack names it), the call's "file:line" for the SC2020 refusal, and the + * builtin ids. The call joins the runtime-fence ledger, so coverage lists + * it as deferred to runtime. */ +function computedRequireArgs( + lowerer: Lowerer, + surface: string, + node: ts.Node, + baseFile: ts.SourceFile, +): IrExpr[] { + const loc = locOf(node); + lowerer.runtimeFences.push(computedRequireDiag(surface, loc)); + const line = ts.getLineAndCharacterOfPosition(ts.getSourceFileOfNode(node), loc.start).line + 1; + return [ + strLit(moduleFileName(lowerer, baseFile), loc), + strLit(`${loc.file}:${line}`, loc), + builtinModulesArrayLit(loc), + ]; +} + +/** A createRequire loader's require.resolve with a computed request, in + * JavaScript source: Node's resolution at run time (scr_require.c), with + * the options argument checked there. */ +function computedResolveCall( + lowerer: Lowerer, + node: ts.Node, + baseFile: ts.SourceFile, + request: IrExpr, + options: IrExpr, + loc: SrcLoc, +): IrExpr { + return { + kind: "libCall", + fn: "module.resolveComputed", + args: [ + request, + options, + ...computedRequireArgs(lowerer, "createRequire's require.resolve", node, baseFile), + ], + type: STRING, + loc, + }; +} + /** The fence a createRequire use with no build-time lowering throws in * JavaScript source, where preflight admits it inside a try block * (createRequireUse7 in program.ts). The diagnostic joins the runtime-fence @@ -160,18 +207,67 @@ function createRequireFence( }; } -/** A createRequire loader's `resolve` (or `resolve.paths`) read as a value - * in JavaScript source: a function that throws the fence when called, so - * `(options.resolve ?? require.resolve)(id)` still calls the option when - * one is given. */ +/** A createRequire loader's `resolve` read as a value in JavaScript source + * (passed on, or picked by `??`): a function that resolves its request at + * run time, so `(options.resolve ?? require.resolve)(id)` behaves as in + * Node. */ +export function createRequireResolveValue( + lowerer: Lowerer, + access: ts.PropertyAccessExpression, + loc: SrcLoc, +): IrExpr { + const baseFile = createRequireCalleeFileOf(lowerer, access.expression); + if (baseFile === null || !isJsSourceFile(ts.getSourceFileOfNode(access))) { + lowerer.noLowering( + "createRequire's require.resolve as a value", + access, + 'call it with a string literal (require.resolve("")), which resolves at build time', + ); + } + const name = keyedHelperName(lowerer, "%createRequire.resolve", `${loc.file}:${loc.start}`); + if (!lowerer.liftedFns.some((fn) => fn.name === name)) { + const call = computedResolveCall( + lowerer, + access, + baseFile, + varRef("request", DYN, loc), + varRef("options", DYN, loc), + loc, + ); + lowerer.liftedFns.push({ + name, + params: [ + { localId: "request", name: "request", type: DYN }, + { localId: "options", name: "options", type: DYN }, + ], + locals: [ + { id: "request", name: "request", type: DYN, mutable: false }, + { id: "options", name: "options", type: DYN, mutable: false }, + ], + returnType: STRING, + body: [{ kind: "return", value: call, loc }], + loc, + }); + } + return { + kind: "dynFrom", + value: { kind: "closure", fnName: name, captures: [], type: funcOf([DYN, DYN], STRING), loc }, + fnName: access.name.text, + type: DYN, + loc, + }; +} + +/** A createRequire loader's `resolve.paths` read as a value in JavaScript + * source: a function that throws the fence when called. */ export function createRequireResolveFenceValue( lowerer: Lowerer, access: ts.PropertyAccessExpression, loc: SrcLoc, ): IrExpr { - const surface = "createRequire's require.resolve as a value"; + const surface = "createRequire's require.resolve.paths as a value"; const hint = - 'call it with a string literal (require.resolve("")), which resolves at build time'; + 'call it with a string literal (require.resolve.paths("")), which resolves at build time'; if (!isJsSourceFile(ts.getSourceFileOfNode(access))) lowerer.noLowering(surface, access, hint); const name = keyedHelperName(lowerer, "%createRequire.fence", `${loc.file}:${loc.start}`); if (!lowerer.liftedFns.some((fn) => fn.name === name)) { @@ -234,12 +330,38 @@ export function lowerCreateRequireCall( const cr = createRequireSpecOf(lowerer, call); if (cr === null) return null; if (cr.spec === null) { + const args = call.arguments.nodes; const surface = "createRequire's require with a computed specifier or this argument shape"; const hint = 'the compiled module graph is fixed at build time — the one lowered form is require(""); native addons are not loaded'; if (!isJsSourceFile(ts.getSourceFileOfNode(call))) lowerer.noLowering(surface, call, hint, undefined, "dynamic-require"); - return deferredCreateRequireThrow(lowerer, call, surface, hint, loc); + // Spread arguments have no run-time resolution: they run, then the + // call throws the fence. + if (args.some((a) => ts.isSpreadElement(a))) + return deferredCreateRequireThrow(lowerer, call, surface, hint, loc); + // Node evaluates every argument, then requires the first. + const stmts: IrStmt[] = []; + let id = args.length > 0 ? lowerer.lowerExprExpecting(args[0]!, DYN) : dynUndefinedExpr(loc); + if (args.length > 1) { + const slot = lowerer.declareHiddenLocal("%require.id", DYN); + stmts.push({ kind: "varDecl", localId: slot.id, init: id, loc }); + for (const arg of args.slice(1)) + stmts.push({ kind: "exprStmt", expr: lowerer.lowerExpr(arg), loc: locOf(arg) }); + id = varRef(slot.id, DYN, loc); + } + const require: IrExpr = { + kind: "libCall", + fn: "module.requireComputed", + args: [id, ...computedRequireArgs(lowerer, "createRequire's require", call, cr.baseFile)], + type: DYN, + loc, + }; + return lowerer.coerceInto( + call, + stmts.length === 0 ? require : { kind: "seqExpr", stmts, result: require, type: DYN, loc }, + lowerer.irTypeOf(call), + ); } const spec = cr.spec; if (spec === "node:ffi") { @@ -492,6 +614,26 @@ export function lowerRequireResolveCall(lowerer: Lowerer, call: ts.CallExpressio ); } const specifier = staticString(call.arguments.nodes[0]); + // A computed request (or options) through a createRequire loader in + // JavaScript source resolves at run time. + if ( + (specifier === null || staticResolveOptionPaths(call.arguments.nodes[1]) === null) && + !pathsCall && + createRequireCalleeFileOf(lowerer, receiver) !== null && + isJsSourceFile(ts.getSourceFileOfNode(call)) + ) { + const [request, options] = call.arguments.nodes; + return computedResolveCall( + lowerer, + call, + baseFile, + lowerer.lowerExprExpecting(request!, DYN), + options === undefined + ? dynUndefinedExpr(locOf(call)) + : lowerer.lowerExprExpecting(options, DYN), + locOf(call), + ); + } if (specifier === null) { const fenced = deferred("with a computed request", resolveHint); if (fenced) return fenced; diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index 2af84f8fb..1c6df3d5c 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -81,6 +81,7 @@ import { isClassCallback } from "./class-callbacks.js"; import { CaughtValueDispatch } from "./caught-value-dispatch.js"; import { refreshDescriptorGuards } from "./class-descriptors.js"; import { ClassConstructionDispatch } from "./class-construction.js"; +import { keyedHelperName } from "./stable-names.js"; import { DispatchSites } from "./dispatch-sites.js"; import { emitterRooted, @@ -3915,7 +3916,7 @@ export class ClassDynamicDispatch { if (!dispatch.fn.params.some((param) => param.localId === dispatch.keyLocal)) params.push({ localId: dispatch.keyLocal, name: "key", type: STRING }); const helper: IrFunction = { - name: `%dyn.class.keys.${lowerer.liftedFns.length}`, + name: keyedHelperName(lowerer, "%dyn.class.keys", helperKey), params, returnType: dispatch.fn.returnType, locals: params.map((p) => ({ @@ -6793,24 +6794,41 @@ export class ClassDynamicDispatch { type: result, loc, }; + // A void method dispatches its overrides with statements: a ternary + // has no void form. + let voidDispatch: IrStmt[] = [{ kind: "exprStmt", expr: call, loc }]; for (const override of overrides.toReversed()) { const subType: IrType = { kind: "object", className: override.cls.def.name }; + const isOverride: IrExpr = { + kind: "instanceOf", + value: receiver, + className: override.cls.def.name, + type: BOOL, + loc, + }; + const overrideCall: IrExpr = { + kind: "call", + callee: override.callee, + args: [{ kind: "downcast", value: receiver, type: subType, loc }, ...args], + type: result, + loc, + }; + if (result.kind === "void") { + voidDispatch = [ + { + kind: "if", + cond: isOverride, + then: [{ kind: "exprStmt", expr: overrideCall, loc }], + else_: voidDispatch, + loc, + }, + ]; + continue; + } call = { kind: "ternary", - cond: { - kind: "instanceOf", - value: receiver, - className: override.cls.def.name, - type: BOOL, - loc, - }, - then: { - kind: "call", - callee: override.callee, - args: [{ kind: "downcast", value: receiver, type: subType, loc }, ...args], - type: result, - loc, - }, + cond: isOverride, + then: overrideCall, else_: call, type: result, loc, @@ -6818,10 +6836,7 @@ export class ClassDynamicDispatch { } let body: IrStmt[]; if (result.kind === "void") - body = [ - { kind: "exprStmt", expr: call, loc }, - { kind: "return", value: dynUndefinedExpr(loc), loc }, - ]; + body = [...voidDispatch, { kind: "return", value: dynUndefinedExpr(loc), loc }]; else { const boxed = lowerer.coerceToExpected(call, DYN); if (boxed.type.kind !== "dyn") return fence(); diff --git a/packages/compiler/src/frontend/lowering/coercions/asserted.ts b/packages/compiler/src/frontend/lowering/coercions/asserted.ts new file mode 100644 index 000000000..43c447cb0 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/coercions/asserted.ts @@ -0,0 +1,87 @@ +/* Slots reached only through what a type assertion claims. */ +import type { IrExpr, IrType, SrcLoc } from "../../../ir/ir.js"; +import { canDynCheckTo, DYN, isUnitType, typeEquals } from "../../../ir/ir.js"; +import { lowerRecordOvfCaptureHelper } from "../containers/indexed-objects.js"; +import type { Lowerer } from "../lowerer.js"; +import { unionRetagHelper } from "./unions.js"; + +/** A value an erased assertion produced (`x as T`, lowerer.assertedValues) + * flowing into a slot that no ordinary conversion reaches. Runs only where + * the slot would otherwise refuse the value (SC2002/SC2003), so every + * conversion that already lowered keeps its code. + * + * - A union whose arms only partly convert (`api as Partial>` on `S | Partial> | undefined`): the arms that convert + * into T convert into the slot, and an arm the assertion excludes (no + * conversion into T) throws the catchable TypeError of a lying assertion. + * - A record or array whose member types the assertion narrows (`models as + * Model<"one">[]` on a union-element array): the checked deep conversion + * a cast of an unknown value takes, which validates the value member by + * member and names the deepest failing path. */ +export function assertedConversion( + lowerer: Lowerer, + expr: IrExpr, + expected: IrType, +): IrExpr | null { + const asserted = lowerer.assertedValues.get(expr); + if (!asserted) return null; + const loc = expr.loc; + if (expr.type.kind === "union") { + const from = lowerer.unions.get(expr.type.unionId); + if (!from) return null; + const values = from.arms.filter((arm) => !isUnitType(arm)); + const kept = values.filter((arm) => convertsInto(lowerer, arm, asserted, loc)); + if (kept.length === 0 || kept.length === values.length) return null; + if (asserted.kind === "union" && expected.kind === "union") { + // The arms T excludes trap; the others (units included) re-tag. + const trappable = new Set(); + from.arms.forEach((arm, tag) => { + if (!isUnitType(arm) && !kept.includes(arm)) trappable.add(tag); + }); + const helper = unionRetagHelper(lowerer, from.id, expected.unionId, loc, trappable); + return helper ? { kind: "call", callee: helper, args: [expr], type: expected, loc } : null; + } + // A record or array T holds one arm's values: extract that arm (every + // other arm, a unit too, contradicts the assertion) and convert it. + if (asserted.kind === "union" || kept.length !== 1) return null; + const arm = kept[0]!; + const helper = lowerer.narrowedArmHelper(from.id, arm, loc); + if (!helper) return null; + const narrowed = lowerer.checkedNarrowCall(helper, expr, arm, loc); + if (typeEquals(arm, expected)) return narrowed; + const converted = lowerer.coerceToExpected(narrowed, expected); + return typeEquals(converted.type, expected) ? converted : null; + } + if (expected.kind !== "record" && expected.kind !== "array") return null; + if (expected.kind === "record" && lowerer.shapes.get(expected.shapeId)?.indexValue !== undefined) + return null; + if ( + !lowerer.dynConvertible(expr.type) || + !canDynCheckTo( + expected, + (id) => lowerer.shapes.get(id), + (id) => lowerer.unions.get(id), + ) + ) + return null; + return { kind: "dynCheck", value: lowerer.coerceToExpected(expr, DYN), type: expected, loc }; +} + +/** True when a value of `arm` has a structural conversion into `target`: + * the same type, one of its arms, a width lift, or (into an + * index-signature record) the overflow capture. */ +function convertsInto(lowerer: Lowerer, arm: IrType, target: IrType, loc: SrcLoc): boolean { + if (typeEquals(arm, target)) return true; + if ( + target.kind === "union" && + lowerer.unions.get(target.unionId)?.arms.some((candidate) => typeEquals(candidate, arm)) + ) + return true; + if (lowerer.widthLiftPlan(arm, target) !== null) return true; + return ( + arm.kind === "record" && + target.kind === "record" && + lowerer.shapes.get(target.shapeId)?.indexValue !== undefined && + lowerRecordOvfCaptureHelper(lowerer, arm.shapeId, target.shapeId, loc) !== null + ); +} diff --git a/packages/compiler/src/frontend/lowering/coercions/state.ts b/packages/compiler/src/frontend/lowering/coercions/state.ts index 76e5ac434..9fa815081 100644 --- a/packages/compiler/src/frontend/lowering/coercions/state.ts +++ b/packages/compiler/src/frontend/lowering/coercions/state.ts @@ -16,7 +16,14 @@ export class CoercionState { // The extracted arm of each checked (non-defaulting) extraction helper. readonly checkedNarrowArms = new Map< string, - { unionId: string; tag: number; target: IrType; widens?: true; message?: string } + { + unionId: string; + tag: number; + target: IrType; + widens?: true; + message?: string; + origin?: string; + } >(); readonly retags = new Map(); readonly narrows = new Map(); diff --git a/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts b/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts index bff419d48..2a394c4d1 100644 --- a/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts +++ b/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts @@ -1,6 +1,6 @@ import { planRecordUnionLift, planRecordUnionWrap } from "../union-retag.js"; import type { WidthLift } from "../width-lift.js"; -import type { IrType } from "../../../ir/ir.js"; +import type { IrRecordShape, IrType } from "../../../ir/ir.js"; import { canDynCheckTo, isUnitType, typeEquals, UNDEFINED_T } from "../../../ir/ir.js"; import { type ClassInfo, findMethodOn, findGenericMethodOn } from "../lower-classes.js"; import type { Lowerer } from "../lowerer.js"; @@ -104,7 +104,15 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width }) ); }); - if (absentOnly.length === 1) selected = absentOnly; + // That layout does not win when it holds the record only by dropping + // fields and another arm keeps every field (`{ kind; type }` beside + // `{ kind; name; adapts?: true; type }`): the value is not its variant. + const dropsFields = (arm: IrType): boolean => !keepsEveryField(lowerer, source, arm); + if ( + absentOnly.length === 1 && + !(dropsFields(absentOnly[0]!.arm) && selected.some(({ arm }) => !dropsFields(arm))) + ) + selected = absentOnly; } // A record that fits several arms of a discriminated union selects one // by its discriminant at run time. Without this, an arm that a nested @@ -131,21 +139,19 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width // A record whose fields some arm keeps EVERY one of, each lifting // (`{ id; limit; opts?: A | B }` into `{ id; limit; opts?: A | B | C } | // { id; opts?: C }`): that arm is the value's own layout, widened; the - // others hold it only by dropping fields. Prefer the unique lossless arm. + // others hold it only by dropping fields. Prefer the unique lossless arm + // with the record's own fields, then the unique one that also declares + // optional fields the record lacks, which complete as absent. if (selected.length > 1 && src.kind === "record") { const source = lowerer.shapes.get(src.shapeId); - const lossless = selected.filter(({ arm }) => { - const target = arm.kind === "record" ? lowerer.shapes.get(arm.shapeId) : undefined; - return ( - source !== undefined && - target !== undefined && - !source.tuple && - !target.tuple && - target.fields.length === source.fields.length && - source.fields.every((field) => target.fields.some((kept) => kept.name === field.name)) - ); - }); - if (lossless.length === 1) selected = lossless; + const lossless = selected.filter(({ arm }) => keepsEveryField(lowerer, source, arm)); + const exact = lossless.filter( + ({ arm }) => + arm.kind === "record" && + lowerer.shapes.get(arm.shapeId)?.fields.length === source?.fields.length, + ); + if (exact.length === 1) selected = exact; + else if (exact.length === 0 && lossless.length === 1) selected = lossless; } if (selected.length !== 1) return null; return { how: "liftWrap", tag: selected[0]!.tag, arm: selected[0]!.arm }; @@ -241,6 +247,23 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width return null; } +/** True when the record arm `arm` has a field of every name the record + * `source` has. Tuples keep their own positional layouts. */ +function keepsEveryField( + lowerer: Lowerer, + source: IrRecordShape | undefined, + arm: IrType, +): boolean { + const target = arm.kind === "record" ? lowerer.shapes.get(arm.shapeId) : undefined; + return ( + source !== undefined && + target !== undefined && + !source.tuple && + !target.tuple && + source.fields.every((field) => target.fields.some((kept) => kept.name === field.name)) + ); +} + /** Two record arms a source record fits, where one arm declares a field * the other lacks and the source types that field `T | undefined` * (`{ count, errors: errs.length ? errs : undefined }` into `{ count; diff --git a/packages/compiler/src/frontend/lowering/coercions/structural.ts b/packages/compiler/src/frontend/lowering/coercions/structural.ts index 93ae1c231..863472910 100644 --- a/packages/compiler/src/frontend/lowering/coercions/structural.ts +++ b/packages/compiler/src/frontend/lowering/coercions/structural.ts @@ -1,3 +1,4 @@ +import { fieldOrigin } from "../decl-origin.js"; import { keyedHelperName } from "../stable-names.js"; import { constructedValue } from "../constructor-returns.js"; import { countedFor, dynUndefinedExpr, varRef } from "../../../ir/build.js"; @@ -394,7 +395,15 @@ export function recordWidthHelper( // orphan one). const plan = lowerer.recordWidthPlan(fromId, toId); if (!plan) return null; - const key = `rec:${fromId}:${toId}`; + // Package code converting into a package-declared record type (an + // object literal returned under `@returns {HelpContext}`): its field + // extractions name the declared fields (decl-origin.ts), so the helper + // is keyed by that type's declaration too (two packages may declare + // records of the same name and shape). Every other projection is shared + // as before. + const declared = lowerer.coercionRecordOrigin(); + const key = + declared === null ? `rec:${fromId}:${toId}` : `rec:${fromId}:${toId}:origin:${declared.key}`; const existing = lowerer.valueHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, "%rec.width", key); @@ -402,7 +411,21 @@ export function recordWidthHelper( // (self-referential shapes) resolves to this helper itself. lowerer.valueHelpers.set(key, name); const fromT: IrType = { kind: "record", shapeId: fromId }; - lowerer.liftedFns.push(buildRecordProjection(lowerer, name, fromT, to, plan, loc)); + const outerNode = lowerer.coercionOriginNode; + const outerField = lowerer.coercionFieldOrigin; + // Nested conversions see only their own field's origin. + lowerer.coercionOriginNode = null; + lowerer.coercionFieldOrigin = null; + try { + lowerer.liftedFns.push( + buildRecordProjection(lowerer, name, fromT, to, plan, loc, (field) => + declared === null ? null : fieldOrigin(lowerer.checker, declared, field), + ), + ); + } finally { + lowerer.coercionOriginNode = outerNode; + lowerer.coercionFieldOrigin = outerField; + } return name; } @@ -799,6 +822,7 @@ function buildRecordProjection( target: IrRecordShape, plan: Map, loc: SrcLoc, + originOf: (field: string) => string | null = () => null, ): IrFunction { const parameter = source.kind === "record" ? "r" : "o"; const localId = parameter + ".0"; @@ -855,7 +879,12 @@ function buildRecordProjection( type: conversion.src, loc, }; + // Only the field's own extraction (a plain narrow) carries the + // declared field's origin; nested conversions are shared. + lowerer.coercionFieldOrigin = + conversion.lift.how === "narrow" ? originOf(field.name) : null; const lifted = lowerer.applyWidthLift(conversion.lift, read, field.type, loc); + lowerer.coercionFieldOrigin = null; return { name: field.name, value: diff --git a/packages/compiler/src/frontend/lowering/coercions/unions.ts b/packages/compiler/src/frontend/lowering/coercions/unions.ts index 6bd72e989..bc4ca4b0e 100644 --- a/packages/compiler/src/frontend/lowering/coercions/unions.ts +++ b/packages/compiler/src/frontend/lowering/coercions/unions.ts @@ -1,3 +1,4 @@ +import { renderOriginNote } from "../decl-origin.js"; import { keyedHelperName } from "../stable-names.js"; import { buildUnionNarrow, unionNarrowFailureMessage } from "../union-narrow.js"; import { isSafeToRepeat } from "../expressions/evaluation-safety.js"; @@ -408,7 +409,24 @@ export function narrowedArmHelper( if (!from || isUnitType(target)) return null; const tag = lowerer.armTag(fromId, target); if (tag < 0) return null; - const key = message === undefined ? `${fromId}:${tag}` : `${fromId}:${tag}:${message}`; + // A slot in package code typed by a package declaration: the TypeError + // names the declaration (decl-origin.ts). Other extractions keep their + // shared helper. + const originNode = + message === undefined && lowerer.coercionOriginUnion === fromId + ? lowerer.coercionOriginNode + : null; + const origin = + message !== undefined + ? null + : (lowerer.coercionFieldOrigin ?? + (originNode === null ? null : lowerer.slotOriginOf(originNode))); + const key = + message !== undefined + ? `${fromId}:${tag}:${message}` + : origin !== null + ? `${fromId}:${tag}:origin:${origin}` + : `${fromId}:${tag}`; const existing = lowerer.coercions.narrows.get(key); if (existing) { lowerer.coercions.checkedNarrows.add(existing); @@ -425,6 +443,9 @@ export function narrowedArmHelper( (type) => lowerer.fmt(type), null, message ?? null, + null, + origin, + lowerer.targetPlatform === "win32", ); if (!fn) throw new InternalCompilerError("lowerer bug: invalid checked union extraction"); lowerer.coercions.checkedNarrowArms.set(name, { @@ -432,6 +453,7 @@ export function narrowedArmHelper( tag, target: from.arms[tag]!, ...(message === undefined ? {} : { message }), + ...(origin === null ? {} : { origin }), }); lowerer.liftedFns.push(fn); return name; @@ -516,8 +538,20 @@ export function checkedNarrowCall( let failure: IrExpr | null = null; for (let tag = from.arms.length - 1; tag >= 0; tag--) { if (tag === arm.tag) continue; + const failing = from.arms[tag]!; const message = - arm.message ?? unionNarrowFailureMessage(from.arms[tag]!, (t) => lowerer.fmt(t)); + (arm.message ?? unionNarrowFailureMessage(failing, (t) => lowerer.fmt(t))) + + (arm.origin === undefined + ? "" + : renderOriginNote( + arm.origin, + isUnitType(failing) + ? failing.kind === "undefinedT" + ? "undefined" + : "null" + : lowerer.fmt(failing), + lowerer.targetPlatform === "win32", + )); const thrown = nodeThrowExpr(1, "", message, type, loc); failure = failure === null diff --git a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts index 2c4d5d9b4..497b50a27 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts @@ -38,6 +38,9 @@ import { } from "../lower-array-sort.js"; import { arrayIndexPresent, + arraySlotSnapshot, + arraySnapshotAppend, + arraySnapshotValue, arrayValueRead, arrayValueStore, arrayValueType, @@ -45,6 +48,9 @@ import { elementMayBeForeign, foreignElementThrow, primitiveElement, + slotSnapshotLocals, + unreadArgument, + type SlotSnapshot, } from "../array-values.js"; import { typeKey } from "../../type-mapper.js"; import { @@ -101,9 +107,12 @@ export function lowerArrayHofCall( } } const callback = booleanFilter - ? { fnArg: truthyFilterCallback(lowerer, valueT, loc), arity: 1 } + ? { fnArg: truthyFilterCallback(lowerer, valueT, loc), arity: 1, unread: false } : lowerArrayCallback(lowerer, argNode, [valueT], arrayOf(elem)); const { fnArg, arity } = callback; + // Only an array that can hold elements outside its element type needs to + // leave an element its callback ignores unread. + const unread = callback.unread && !present && elementMayBeForeign(lowerer, elem); const fnRet = fnArg.type.ret; if (method === "map" && (fnRet.kind === "void" || fnRet.kind === "func")) { // The result array U[] is unrepresentable here (no void elements; @@ -133,8 +142,9 @@ export function lowerArrayHofCall( : null; const outElem = filtered !== null ? filtered.elem : callbackArrayElement(lowerer, call, fnRet); // A filter that re-tags its elements keeps the single union-typed visit. + // A callback that never reads its element has no use for a typed one. const fast = - present || booleanFilter || (method === "filter" && !typeEquals(outElem, elem)) + present || booleanFilter || unread || (method === "filter" && !typeEquals(outElem, elem)) ? null : lowerFastCallback(lowerer, argNode, [valueT], 0, elem, arrayOf(elem), callback); const helper = arrayHofHelper( @@ -148,6 +158,7 @@ export function lowerArrayHofCall( callbackSite(fast ?? fnArg), fast?.type ?? null, present, + unread, ); const resultType: IrType = method === "map" || method === "filter" ? arrayOf(outElem) : VOID; const result: IrExpr = { @@ -541,12 +552,21 @@ function callbackValueRead( elem: IrType, present: boolean, loc: SrcLoc, + unread = false, ): IrExpr { + if (unread) { + const placeholder = unreadArgument(lowerer, callbackValueType(lowerer, elem, present), loc); + if (placeholder) return placeholder; + } return present ? { kind: "arrayGet", arr, index, type: elem, loc } : arrayValueRead(lowerer, arr, index, elem, loc); } +/** The snapshot locals of the filter and find helpers; `value` is their + * element local `v.0`. */ +const FILTER_SNAPSHOT: SlotSnapshot = { isForeign: "vf.0", value: "v.0", foreign: "vd.0" }; + /** Interned synthetic loop function for one (method, elem, fnRet, arity) * combo. Named `%arr..` ('%' keeps it out of the user * namespace); rides `liftedFns` into the module like a lifted lambda (it @@ -562,13 +582,23 @@ function arrayHofHelper( site = "", fastT: CallbackType | null = null, present = false, + unread = false, ): string { - const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}:${typeKey(outElem)}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}`; + const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}:${typeKey(outElem)}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, `%arr.${method}`, key); lowerer.arrHofHelpers.set(key, name); - lowerer.arrHofInfo.set(name, { method, elem, fnRet, arity, outElem, present, split: !!fastT }); + lowerer.arrHofInfo.set(name, { + method, + elem, + fnRet, + arity, + outElem, + present, + split: !!fastT, + unread, + }); const fn = buildArrayHofFn( lowerer, name, @@ -580,6 +610,7 @@ function arrayHofHelper( outElem, fastT, present, + unread, ); lowerer.liftedFns.push(fastT ? withFastCallback(fn, fastT, elem) : fn); return name; @@ -794,6 +825,7 @@ function buildArrayHofFn( outElem: IrType, fastT: CallbackType | null = null, present = false, + unread = false, ): IrFunction { const arrT = arrayOf(elem); const valueT = callbackValueType(lowerer, elem, present); @@ -810,6 +842,19 @@ function buildArrayHofFn( present, loc, ); + // What the callback takes: the element, or a placeholder for one it + // ignores (callbackValueRead's `unread`). + const argElem = unread + ? callbackValueRead( + lowerer, + varRef("a.0", arrT, loc), + varRef("i.0", F64, loc), + elem, + present, + loc, + true, + ) + : getElem; const hasElem: IrExpr = { kind: "arrayHas", arr: varRef("a.0", arrT, loc), @@ -858,7 +903,7 @@ function buildArrayHofFn( kind: "if", cond: hasElem, then: [ - { kind: "varDecl", localId: "mapped.0", init: callF(getElem), loc }, + { kind: "varDecl", localId: "mapped.0", init: callF(argElem), loc }, arrayValueStore( lowerer, varRef("out.0", outT, loc), @@ -994,6 +1039,19 @@ function buildArrayHofFn( ); returnType = outT; const retained = filterRetainedValue(lowerer, varRef("v.0", valueT, loc), elem, outElem, loc); + // A predicate that ignores the element of an array that can hold + // elements outside its element type keeps a snapshot of the slot, read + // before the predicate runs, as stored (arraySlotSnapshot). + const copied = + (arity === 0 || unread) && + !present && + typeEquals(outElem, elem) && + elementMayBeForeign(lowerer, elem); + // `v.0` is declared above; the snapshot adds its other two locals. + if (copied) + locals.push( + ...slotSnapshotLocals(lowerer, elem, FILTER_SNAPSHOT).filter((l) => l.id !== "v.0"), + ); body = [ { kind: "varDecl", @@ -1006,34 +1064,62 @@ function buildArrayHofFn( { kind: "if", cond: hasElem, - then: [ - { kind: "varDecl", localId: "v.0", init: getElem, loc }, - { - kind: "if", - // ToBoolean over the predicate's answer — inert when it already - // returned bool, the per-union helper when it returned a union. - cond: filterCond(callF(varRef("v.0", valueT, loc)), fnRet, loc), - then: [ - arrayValueStore( + // A kept element is appended as it was stored, so one the element + // type cannot hold stays in the result. + then: copied + ? [ + ...arraySlotSnapshot( lowerer, - varRef("out.0", outT, loc), - { - kind: "arrIntrinsic", - method: "length", - receiver: varRef("out.0", outT, loc), - args: [], - type: F64, - loc, - }, - retained, - outElem, + varRef("a.0", arrT, loc), + varRef("i.0", F64, loc), + elem, + FILTER_SNAPSHOT, loc, ), + { + kind: "if", + cond: filterCond(callF(argElem), fnRet, loc), + then: [ + arraySnapshotAppend( + lowerer, + varRef("out.0", outT, loc), + elem, + FILTER_SNAPSHOT, + loc, + ), + ], + else_: null, + loc, + }, + ] + : [ + { kind: "varDecl", localId: "v.0", init: getElem, loc }, + { + kind: "if", + // ToBoolean over the predicate's answer — inert when it already + // returned bool, the per-union helper when it returned a union. + cond: filterCond(callF(varRef("v.0", valueT, loc)), fnRet, loc), + then: [ + arrayValueStore( + lowerer, + varRef("out.0", outT, loc), + { + kind: "arrIntrinsic", + method: "length", + receiver: varRef("out.0", outT, loc), + args: [], + type: F64, + loc, + }, + retained, + outElem, + loc, + ), + ], + else_: null, + loc, + }, ], - else_: null, - loc, - }, - ], else_: null, loc, }, @@ -1048,7 +1134,7 @@ function buildArrayHofFn( { kind: "if", cond: hasElem, - then: [{ kind: "exprStmt", expr: callF(getElem), loc }], + then: [{ kind: "exprStmt", expr: callF(argElem), loc }], else_: null, loc, }, @@ -1088,13 +1174,17 @@ export function lowerArrayFindLikeCall( presentElements(lowerer, call, elem) && ((method !== "find" && method !== "findLast") || (!bindUntyped && lowerer.coercibleValue(elem, lowerer.irTypeOf(call)))); - const { fnArg, arity } = lowerArrayCallback( + const lowered = lowerArrayCallback( lowerer, argNode, [callbackValueType(lowerer, elem, present)], arrT, bindUntyped, ); + const { fnArg, arity } = lowered; + // Only an array that can hold elements outside its element type needs to + // leave an element its callback ignores unread. + const unread = lowered.unread && !present && elementMayBeForeign(lowerer, elem); // JS takes the ToBoolean of the predicate's result — allowed wherever // that ToBoolean has a static answer (bool passes through; f64/string // by value; a truthy-answerable union by its arm — the @@ -1109,18 +1199,19 @@ export function lowerArrayFindLikeCall( method === "findIndex" || method === "findLastIndex" ) { - const fast = present - ? null - : lowerFastCallback( - lowerer, - argNode, - [arrayValueType(lowerer, elem)], - 0, - elem, - arrT, - { fnArg, arity }, - bindUntyped, - ); + const fast = + present || unread + ? null + : lowerFastCallback( + lowerer, + argNode, + [arrayValueType(lowerer, elem)], + 0, + elem, + arrT, + { fnArg, arity }, + bindUntyped, + ); const fastT = fast?.type ?? null; const helper = method === "findIndex" || method === "findLastIndex" @@ -1134,6 +1225,7 @@ export function lowerArrayFindLikeCall( callbackSite(fast ?? fnArg), fastT, present, + unread, ) : someEveryHelper( lowerer, @@ -1145,6 +1237,7 @@ export function lowerArrayFindLikeCall( callbackSite(fast ?? fnArg), fastT, present, + unread, ); return { kind: "call", @@ -1182,6 +1275,7 @@ export function lowerArrayFindLikeCall( retag, callbackSite(fnArg), present, + unread, ); return { kind: "call", callee: helper, args: [receiver, fnArg], type: resultT, loc }; } @@ -1275,9 +1369,10 @@ function findHelper( retag: string | null, site = "", present = false, + unread = false, ): string { const method = last ? "findLast" : "find"; - const key = `${method}:${typeKey(elem)}:${typeKey(resultT)}:${typeKey(fnRet)}:${arity}${site}${present ? ":present" : ""}`; + const key = `${method}:${typeKey(elem)}:${typeKey(resultT)}:${typeKey(fnRet)}:${arity}${site}${present ? ":present" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, `%arr.${method}`, key); @@ -1296,28 +1391,62 @@ function findHelper( type: resultT, loc, }; - const visitBody: IrStmt[] = [ - { - kind: "varDecl", - localId: "v.0", - init: callbackValueRead( - lowerer, - varRef("a.0", arrT, loc), - varRef("i.0", F64, loc), - elem, - present, - loc, - ), + const read: IrStmt = { + kind: "varDecl", + localId: "v.0", + init: callbackValueRead( + lowerer, + varRef("a.0", arrT, loc), + varRef("i.0", F64, loc), + elem, + present, loc, - }, - { - kind: "if", - cond: predicateToBoolean(callArrayCallback(arrT, fnT, fnRet, arity, v, loc)), - then: [{ kind: "return", value: found, loc }], - else_: null, - loc, - }, - ]; + ), + loc, + }; + // A predicate that ignores the element of an array that can hold + // elements outside its element type keeps a snapshot of the slot, read + // before the predicate runs as JavaScript reads it (arraySlotSnapshot): + // the predicate is called without reading it as typed, and a found element + // outside the element type takes the checked conversion into the result. + const ignored = (arity === 0 || unread) && !present && elementMayBeForeign(lowerer, elem); + const placeholder = unread ? unreadArgument(lowerer, valueT, loc) : null; + const visitBody: IrStmt[] = ignored + ? [ + ...arraySlotSnapshot( + lowerer, + varRef("a.0", arrT, loc), + varRef("i.0", F64, loc), + elem, + FILTER_SNAPSHOT, + loc, + ), + { + kind: "if", + cond: predicateToBoolean( + callArrayCallback(arrT, fnT, fnRet, arity, placeholder ?? v, loc), + ), + then: [ + { + kind: "return", + value: arraySnapshotValue(lowerer, elem, FILTER_SNAPSHOT, resultT, () => found, loc), + loc, + }, + ], + else_: null, + loc, + }, + ] + : [ + read, + { + kind: "if", + cond: predicateToBoolean(callArrayCallback(arrT, fnT, fnRet, arity, v, loc)), + then: [{ kind: "return", value: found, loc }], + else_: null, + loc, + }, + ]; const visit: IrStmt[] = visitBody; const loop = last ? reverseArrayLoop(loc, visit) @@ -1334,6 +1463,9 @@ function findHelper( locals: [ ...arrayCallbackLocals(arrT, fnT), { id: "v.0", name: "v", type: valueT, mutable: false }, + ...(ignored + ? slotSnapshotLocals(lowerer, elem, FILTER_SNAPSHOT).filter((l) => l.id !== "v.0") + : []), ], body, loc, @@ -1375,9 +1507,10 @@ function findIndexHelper( site = "", fastT: CallbackType | null = null, present = false, + unread = false, ): string { const method = last ? "findLastIndex" : "findIndex"; - const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}`; + const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, `%arr.${method}`, key); @@ -1391,6 +1524,7 @@ function findIndexHelper( outElem: elem, present, split: !!fastT, + unread, }); const arrT = arrayOf(elem); const fnT = funcOf([callbackValueType(lowerer, elem, present), F64, arrT].slice(0, arity), fnRet); @@ -1411,6 +1545,7 @@ function findIndexHelper( elem, present, loc, + unread, ), loc, ), @@ -1460,8 +1595,9 @@ function someEveryHelper( site = "", fastT: CallbackType | null = null, present = false, + unread = false, ): string { - const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}`; + const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, `%arr.${method}`, key); @@ -1474,6 +1610,7 @@ function someEveryHelper( outElem: elem, present, split: !!fastT, + unread, }); const arrT = arrayOf(elem); const fnT = funcOf([callbackValueType(lowerer, elem, present), F64, arrT].slice(0, arity), fnRet); @@ -1490,6 +1627,7 @@ function someEveryHelper( elem, present, loc, + unread, ), loc, ), @@ -1553,12 +1691,16 @@ export function lowerArrayFlatMapCall( const argNode = call.arguments.nodes[0]; if (!argNode) lowerer.unsupported("SC1090", call, "this call form"); // tsc-guarded const present = presentElements(lowerer, call, elem); - const { fnArg, arity } = lowerArrayCallback( + const lowered = lowerArrayCallback( lowerer, argNode, [callbackValueType(lowerer, elem, present)], arrT, ); + const { fnArg, arity } = lowered; + // Only an array that can hold elements outside its element type needs to + // leave an element its callback ignores unread. + const unread = lowered.unread && !present && elementMayBeForeign(lowerer, elem); const fnRet = fnArg.type.ret; if ( fnRet.kind === "dyn" || @@ -1603,12 +1745,13 @@ export function lowerArrayFlatMapCall( if (fnRet.kind === "void" || fnRet.kind === "func") lowerer.badType(call, lowerer.typeOf(call)); requireProducedArrayElement(lowerer, call, "'.flatMap()'", fnRet); const outElem = callbackArrayElement(lowerer, call, fnRet); - const fast = present - ? null - : lowerFastCallback(lowerer, argNode, [arrayValueType(lowerer, elem)], 0, elem, arrT, { - fnArg, - arity, - }); + const fast = + present || unread + ? null + : lowerFastCallback(lowerer, argNode, [arrayValueType(lowerer, elem)], 0, elem, arrT, { + fnArg, + arity, + }); const helper = arrayHofHelper( lowerer, "flatMap", @@ -1620,6 +1763,7 @@ export function lowerArrayFlatMapCall( callbackSite(fast ?? fnArg), fast?.type ?? null, present, + unread, ); return { kind: "call", @@ -1629,12 +1773,13 @@ export function lowerArrayFlatMapCall( loc, }; } - const fast = present - ? null - : lowerFastCallback(lowerer, argNode, [arrayValueType(lowerer, elem)], 0, elem, arrT, { - fnArg, - arity, - }); + const fast = + present || unread + ? null + : lowerFastCallback(lowerer, argNode, [arrayValueType(lowerer, elem)], 0, elem, arrT, { + fnArg, + arity, + }); const helper = flatMapHelper( lowerer, elem, @@ -1644,6 +1789,7 @@ export function lowerArrayFlatMapCall( callbackSite(fast ?? fnArg), fast?.type ?? null, present, + unread, ); return { kind: "call", @@ -1943,8 +2089,9 @@ function flatMapHelper( site = "", fastT: CallbackType | null = null, present = false, + unread = false, ): string { - const key = `flatMap:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}`; + const key = `flatMap:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, "%arr.flatMap", key); @@ -1957,6 +2104,7 @@ function flatMapHelper( outElem: fnRet.elem, present, split: !!fastT, + unread, }); const arrT = arrayOf(elem); const inner = fnRet.elem; @@ -2053,6 +2201,7 @@ function flatMapHelper( elem, present, loc, + unread, ), loc, ), @@ -2125,7 +2274,7 @@ export function lowerArrayReduceCall( lowerer.runtimeOptionalReduceTypes.get(call) ?? (hasInit ? lowerer.irTypeOf(call) : arrayValueType(lowerer, elem)); if (accT.kind === "void" || accT.kind === "func") lowerer.badType(call, lowerer.typeOf(call)); - const { fnArg, arity } = lowerArrayCallback( + const lowered = lowerArrayCallback( lowerer, argNode, [accT, arrayValueType(lowerer, elem)], @@ -2133,17 +2282,23 @@ export function lowerArrayReduceCall( false, accT, ); - const fast = lowerFastCallback( - lowerer, - argNode, - [accT, arrayValueType(lowerer, elem)], - 1, - elem, - arrT, - { fnArg, arity }, - false, - accT, - ); + const { fnArg, arity } = lowered; + // Only an array that can hold elements outside its element type needs to + // leave an element its callback ignores unread. + const unread = lowered.unread && elementMayBeForeign(lowerer, elem); + const fast = unread + ? null + : lowerFastCallback( + lowerer, + argNode, + [accT, arrayValueType(lowerer, elem)], + 1, + elem, + arrT, + { fnArg, arity }, + false, + accT, + ); const helper = reduceHelper( lowerer, method, @@ -2155,6 +2310,7 @@ export function lowerArrayReduceCall( loc, callbackSite(fast ?? fnArg), fast?.type ?? null, + unread, ); const args: IrExpr[] = [receiver, fnArg]; if (hasInit) args.push(lowerer.lowerExprExpecting(call.arguments.nodes[1]!, accT)); @@ -2185,8 +2341,9 @@ function reduceHelper( loc: SrcLoc, site = "", fastT: CallbackType | null = null, + unread = false, ): string { - const key = `${method}:${typeKey(elem)}:${typeKey(accT)}:${typeKey(fnRet)}:${arity}:${hasInit ? "init" : "seed"}${site}${fastT ? ":split" : ""}`; + const key = `${method}:${typeKey(elem)}:${typeKey(accT)}:${typeKey(fnRet)}:${arity}:${hasInit ? "init" : "seed"}${site}${fastT ? ":split" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, `%arr.${method}`, key); @@ -2201,6 +2358,7 @@ function reduceHelper( present: false, split: !!fastT, hasInit, + unread, }); const arrT = arrayOf(elem); const fnT = funcOf([accT, arrayValueType(lowerer, elem), F64, arrT].slice(0, arity), fnRet); @@ -2210,6 +2368,9 @@ function reduceHelper( const right = method === "reduceRight"; const at = (index: IrExpr): IrExpr => arrayValueRead(lowerer, varRef("a.0", arrT, loc), index, elem, loc); + // The visited element, or a placeholder for a callback that never reads it. + const placeholder = unread ? unreadArgument(lowerer, arrayValueType(lowerer, elem), loc) : null; + const visited = (): IrExpr => placeholder ?? at(i); const locals: IrLocal[] = [ { id: "a.0", name: "a", type: arrT, mutable: true }, { id: "f.0", name: "f", type: fnT, mutable: true }, @@ -2320,7 +2481,7 @@ function reduceHelper( { kind: "callValue", callee: varRef("f.0", fnT, loc), - args: [varRef("acc.0", accT, loc), at(i), i, varRef("a.0", arrT, loc)].slice( + args: [varRef("acc.0", accT, loc), visited(), i, varRef("a.0", arrT, loc)].slice( 0, arity, ), diff --git a/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts b/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts index 6307d871f..5cdbe4d31 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts @@ -6,6 +6,7 @@ import { AmbientReceiverReaders } from "../../../ir/receiver-readers.js"; import { boolLit, countedFor, numLit, varRef } from "../../../ir/build.js"; import { BOOL, + DYN, F64, VOID, type IrBindingSource, @@ -32,9 +33,14 @@ import { import { arrayIndexPresent, arrayValueRead, + arraySlotSnapshot, + arraySnapshotAppend, + arraySnapshotValue, arrayValueStore, arrayValueType, elementMayBeForeign, + unreadArgument, + type SlotSnapshot, } from "../array-values.js"; import type { Lowerer } from "../lowerer.js"; @@ -62,6 +68,9 @@ export interface ArrayHofInfo { present: boolean; split: boolean; hasInit?: boolean; + /** The callback never reads the element it declares + * (lowerArrayCallback's `unread`). */ + unread?: boolean; } type FuncType = IrType & { kind: "func" }; @@ -610,7 +619,13 @@ function chainCallbacks(chain: Chain): Stage[] | null { /** The values a fused loop hands from one stage to the next: a local of a * known type, or (before the first read) the source slot itself. */ -type Flow = { kind: "value"; id: string; type: IrType } | { kind: "slot" }; +/** What reaches a stage: a value, the source slot not yet read, or a + * snapshot of the slot read before an earlier callback that ignored it + * (arraySlotSnapshot). */ +type Flow = + | { kind: "value"; id: string; type: IrType } + | { kind: "slot" } + | { kind: "snapshot"; ids: SlotSnapshot }; interface Plan { chain: Chain; @@ -1595,13 +1610,21 @@ class FusedLoop { stage: Stage, k: number, lead: IrExpr[], - flow: Flow & { kind: "value" }, + flow: Flow, ): { pre: IrStmt[]; value: IrExpr } | null => { const arity = stage.info.arity; - const value = varRef(flow.id, flow.type, loc); const at = lead.length; const pick = (): { arg: IrExpr & { type: FuncType }; value: IrExpr | null } | null => { if (arity <= at) return { arg: stage.slow, value: null }; + // A callback that reads the value reaches it read (`read`); one + // that never reads it takes a placeholder for the unread slot. + if (flow.kind !== "value") { + const placeholder = stage.info.unread + ? unreadArgument(this.lowerer, stage.slow.type.params[at]!, loc) + : null; + return placeholder ? { arg: stage.slow, value: placeholder } : null; + } + const value = varRef(flow.id, flow.type, loc); const fast = stage.fast; if (fast && typeEquals(fast.type.params[at]!, flow.type)) return { arg: fast, value }; const paramT = stage.slow.type.params[at]!; @@ -1630,11 +1653,62 @@ class FusedLoop { return { pre: [...before, ...called.pre], value: called.value }; }; + /** The stage's callback never reads the element at parameter `at`, and + * a placeholder of its type stands for it (unreadArgument). */ + const unreadSlot = (stage: Stage, at: number): boolean => + stage.info.unread === true && + unreadArgument(this.lowerer, stage.slow.type.params[at]!, loc) !== null; + const fail = (): IrStmt[] => { this.failed = true; return []; }; + /** Bring `flow` to the callback of `stage`, whose element parameter is + * at `at`. A slot is read for a callback that reads it. For one that + * ignores it, the slot of an array that can hold elements outside its + * element type is read as stored before the callback runs (a snapshot, + * as JavaScript reads the element first), so such an element passes the + * callback as it is; other arrays read the slot as usual. A snapshot + * reaching a callback that reads it becomes its value, and an element + * outside the element type takes the ordinary read's checked + * conversion there. */ + const reach = ( + stage: Stage, + at: number, + flow: Flow, + rest: (flow: Flow) => IrStmt[], + ): IrStmt[] => { + const reads = stage.info.arity > at && !unreadSlot(stage, at); + if (flow.kind === "slot") { + if (reads || !elementMayBeForeign(this.lowerer, arrT.elem)) + return read(stage, (value) => rest(value)); + const ids: SlotSnapshot = { + isForeign: this.local("sf", BOOL), + value: this.local("sv", arrayValueType(this.lowerer, arrT.elem)), + foreign: this.local("sd", DYN), + }; + return [ + ...arraySlotSnapshot(this.lowerer, a, i, arrT.elem, ids, loc), + ...rest({ kind: "snapshot", ids }), + ]; + } + if (flow.kind === "snapshot" && reads) { + const valueT = arrayValueType(this.lowerer, arrT.elem); + const u = this.local("u", valueT); + return [ + { + kind: "varDecl", + localId: u, + init: arraySnapshotValue(this.lowerer, arrT.elem, flow.ids, valueT, (v) => v, loc), + loc, + }, + ...rest({ kind: "value", id: u, type: valueT }), + ]; + } + return rest(flow); + }; + /** Turn the source slot into a value for the callback of `stage`. */ const read = (stage: Stage, rest: (flow: Flow & { kind: "value" }) => IrStmt[]): IrStmt[] => { if (stage.info.present) { @@ -1690,11 +1764,12 @@ class FusedLoop { }; /** The statements that run one value through stages k.. and the terminal. */ - const visit = (k: number, flow: Flow): IrStmt[] => { + const visit = (k: number, flow: Flow, reached = false): IrStmt[] => { if (k === chain.stages.length) return finish(flow); const stage = chain.stages[k]!; const info = stage.info; - if (flow.kind === "slot") return read(stage, (value) => visit(k, value)); + if (flow.kind !== "value" && !reached) + return reach(stage, 0, flow, (next) => visit(k, next, true)); const called = deliver(stage, k, [], flow); if (!called) return fail(); const { pre, value: result } = called; @@ -1791,11 +1866,23 @@ class FusedLoop { } }; + /** Append a snapshot of the source slot to the collected result: a + * filter whose callback ignored the element keeps it as it was stored + * (arraySnapshotAppend). */ + const copySlot = (flow: Flow & { kind: "snapshot" }): IrStmt[] => { + const outT = resultT as IrType & { kind: "array" }; + if (holes || !typeEquals(outT.elem, arrT.elem)) return fail(); + return [ + arraySnapshotAppend(this.lowerer, varRef(out!, resultT, loc), arrT.elem, flow.ids, loc), + ]; + }; /** The terminal's action for one value. */ - const finish = (flow: Flow): IrStmt[] => { + const finish = (flow: Flow, reached = false): IrStmt[] => { if (terminal.kind === "length") return [increment(count!, loc)]; if (terminal.kind === "collect") { - if (flow.kind !== "value") return fail(); + if (flow.kind === "snapshot") return copySlot(flow); + if (flow.kind === "slot") + return read(chain.stages[chain.stages.length - 1]!, (value) => finish(value)); const last = chain.stages[chain.stages.length - 1]!; const outT = resultT as IrType & { kind: "array" }; if (!typeEquals(outT.elem, last.info.outElem)) return fail(); @@ -1816,7 +1903,9 @@ class FusedLoop { } const stage = terminal.stage; const method = stage.info.method; - if (flow.kind === "slot") return read(stage, (value) => finish(value)); + const lead = method === "reduce" ? 1 : 0; + if (flow.kind !== "value" && !reached) + return reach(stage, lead, flow, (next) => finish(next, true)); if (method === "reduce") { const accT = stage.info.outElem; const called = deliver(stage, chain.stages.length, [varRef(acc!, accT, loc)], flow); diff --git a/packages/compiler/src/frontend/lowering/containers/callback-arguments.ts b/packages/compiler/src/frontend/lowering/containers/callback-arguments.ts index de027e08c..3b504861a 100644 --- a/packages/compiler/src/frontend/lowering/containers/callback-arguments.ts +++ b/packages/compiler/src/frontend/lowering/containers/callback-arguments.ts @@ -53,7 +53,7 @@ export function lowerArrayCallback( arrT: IrType, bindUntyped = false, expectedReturn?: IrType, -): { fnArg: IrExpr & { type: IrType & { kind: "func" } }; arity: number } { +): { fnArg: IrExpr & { type: IrType & { kind: "func" } }; arity: number; unread: boolean } { // `xs.map(((x) => x))`: a parenthesized callback is the function itself. while (ts.isParenthesizedExpression(argNode)) argNode = argNode.expression; // The ambient `Boolean` as the callback (`flags.every(Boolean)`, @@ -72,7 +72,11 @@ export function lowerArrayCallback( if (valueT === undefined || valueT.kind === "void" || isUnitType(valueT)) lowerer.badType(argNode, lowerer.typeOf(argNode)); if (valueT.kind === "union") lowerer.requireTruthyUnion(valueT.unionId, argNode); - return { fnArg: truthyFilterCallback(lowerer, valueT, locOf(argNode)), arity: 1 }; + return { + fnArg: truthyFilterCallback(lowerer, valueT, locOf(argNode)), + arity: 1, + unread: false, + }; } // The ambient `String` (`values.map(String)`): String(value) converts its // first argument as a direct call does, whatever the element type; the @@ -86,7 +90,7 @@ export function lowerArrayCallback( const valueT = lead[0]; if (valueT === undefined || valueT.kind === "void") lowerer.badType(argNode, lowerer.typeOf(argNode)); - return { fnArg: stringConversionCallback(lowerer, valueT, argNode), arity: 1 }; + return { fnArg: stringConversionCallback(lowerer, valueT, argNode), arity: 1, unread: false }; } const full = [...lead, F64, arrT]; // A DYN-receiver HOF's callback (`parsed.flatMap((value) => ...)`): @@ -276,9 +280,32 @@ export function lowerArrayCallback( return { fnArg: fnArg as IrExpr & { type: IrType & { kind: "func" } }, arity: fnArg.type.params.length, + unread: !readsParameter(argNode, lead.length - 1), }; } +/** Whether a callback can observe its parameter at `index`: false for a + * function or arrow expression that declares no such parameter, or binds + * it to a plain name that appears nowhere else in it (any identifier of + * that name counts, including shorthand properties and shadowing + * bindings). Any other callback may. */ +function readsParameter(argNode: ts.Expression, index: number): boolean { + if (!ts.isArrowFunction(argNode) && !ts.isFunctionExpression(argNode)) return true; + const param = argNode.parameters.nodes[index]; + if (!param) return false; + if (!ts.isIdentifier(param.name) || param.initializer || param.dotDotDotToken) return true; + const name = param.name.text; + const mentions = (node: ts.Node): boolean => + (node !== param.name && + ts.isIdentifier(node) && + (node.text === name || node.text === "arguments")) || + (ts.forEachChild(node, (child) => (mentions(child) ? true : undefined)) ?? false); + return ( + argNode.parameters.nodes.some((other) => other !== param && mentions(other)) || + mentions(argNode.body) + ); +} + /** Preserve the checker-selected payload ABI when a callback's runtime * return gained only undefined. The array state byte carries that value. */ export function callbackArrayElement( diff --git a/packages/compiler/src/frontend/lowering/cycle-initialization.ts b/packages/compiler/src/frontend/lowering/cycle-initialization.ts index 8c8ce06e6..30c52e382 100644 --- a/packages/compiler/src/frontend/lowering/cycle-initialization.ts +++ b/packages/compiler/src/frontend/lowering/cycle-initialization.ts @@ -233,6 +233,38 @@ function earlyHeritageReference(lowerer: Lowerer, binding: CycleEarlyBinding): t return null; } +/** The identifiers of `sf` by text, each with its position in the file's + * preorder walk: one walk per file serves every class's reference scan. */ +function fileIdentifiers( + lowerer: Lowerer, + sf: ts.SourceFile, +): ReadonlyMap { + const memo = lowerer.program.analysis.fileIdentifiers; + const known = memo.get(sf); + if (known !== undefined) return known; + const byText = new Map(); + let order = 0; + ts.walkPreorder(sf, (node) => { + order++; + if (!ts.isIdentifier(node)) return undefined; + const list = byText.get(node.text); + if (list === undefined) byText.set(node.text, [{ order, node }]); + else list.push({ order, node }); + return undefined; + }); + memo.set(sf, byText); + return byText; +} + +/** Whether `node` lies inside `ancestor` (or is it): the reference scans + * skip the class's own declaration. */ +function within(node: ts.Node, ancestor: ts.Node | undefined): boolean { + if (ancestor === undefined) return false; + for (let current: ts.Node | undefined = node; current !== undefined; current = current.parent) + if (current === ancestor) return true; + return false; +} + /** Whether `node` names `symbol`, through import aliases when `aliases`. */ function resolvesTo( lowerer: Lowerer, @@ -294,12 +326,14 @@ function classReferences( // the class's name or a local one. const names = classLocalNames(lowerer, sf, symbol, name.text, cycle); if (names.size === 0) continue; - ts.walkPreorder(sf, (node) => { - if (node === declaration) return "skip"; - if (ts.isIdentifier(node) && names.has(node.text) && resolvesTo(lowerer, node, symbol, cycle)) + // The file's identifiers under those names, in walk order. + const identifiers = fileIdentifiers(lowerer, sf); + const named: { order: number; node: ts.Identifier }[] = []; + for (const text of names) for (const entry of identifiers.get(text) ?? []) named.push(entry); + if (names.size > 1) named.sort((a, b) => a.order - b.order); + for (const { node } of named) + if (!within(node, declaration) && resolvesTo(lowerer, node, symbol, cycle)) out.push({ loc: locOf(node), name: node.text }); - return undefined; - }); } return out; } diff --git a/packages/compiler/src/frontend/lowering/decl-origin.ts b/packages/compiler/src/frontend/lowering/decl-origin.ts new file mode 100644 index 000000000..6eae57966 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/decl-origin.ts @@ -0,0 +1,436 @@ +/* Origins of package-declared types at checked conversions. + * + * scriptc enforces declared types: a value that contradicts the type an + * npm package declares (its .d.ts, @types package or JSDoc) throws a + * TypeError where Node runs on. When the declaration is what is wrong, the + * fix is a type patch (package.json scriptc.typePatches), so the error + * names the declaration and the patch that would admit the value: + * + * TypeError: expected number at $, got undefined + * termctx@1.2.0 declares Terminal.columns() as number (index.js:14) + * if that declaration is wrong, patch it in package.json: "scriptc": + * { "typePatches": { "termctx@^1.2.0": { "declarations": + * { "Terminal.columns()": "| undefined" } } } } + * + * The origin is found for three kinds of slots: a value a package function + * returns under a written return type, a property of an object literal + * typed by a package-declared type, and a declaration-boundary cast (a + * value user code receives from a package; decl-projection.ts records + * those with originAt). Generic declarations get none: their types come + * from how the program instantiates them. Checks in + * code that names no package declaration carry no origin, so programs + * without npm packages lower exactly as before. + * + * The origin string's fields (separated by \x1f) are read by the runtime + * (scr_type_origin.c) for checked conversions, whose path and actual kind + * are only known when they fail, and rendered here for union extractions, + * whose failing arm is known statically. */ + +import type { IrExpr, IrModule } from "../../ir/ir.js"; +import { everyModuleNode } from "../../ir/traverse.js"; +import { trackedReadFile } from "../input-tracker.js"; +import * as ts from "../ts7/adapter.js"; + +export const ORIGIN_SEPARATOR = "\x1f"; + +interface PackageFile { + pkg: string; + version: string; + /** The file relative to the package root. */ + rel: string; +} + +/** Installed package versions by package root, read once per program load + * (resetDeclOriginVersions): a dependency reinstalled at another version + * between two builds of one process is read again. */ +const versions = new Map(); + +export function resetDeclOriginVersions(): void { + versions.clear(); +} + +function packageFileOf(fileName: string): PackageFile | null { + const path = fileName.split("\\").join("/"); + const marker = "/node_modules/"; + const at = path.lastIndexOf(marker); + if (at < 0) return null; + const parts = path.slice(at + marker.length).split("/"); + const nameParts = parts[0]!.startsWith("@") ? 2 : 1; + if (parts.length <= nameParts) return null; + let pkg = parts.slice(0, nameParts).join("/"); + const root = path.slice(0, at + marker.length) + pkg; + let versionRoot = root; + if (pkg.startsWith("@types/")) { + // A twin describes the runtime package installed beside it. + const bare = pkg.slice("@types/".length); + const i = bare.indexOf("__"); + pkg = i > 0 ? `@${bare.slice(0, i)}/${bare.slice(i + 2)}` : bare; + versionRoot = path.slice(0, at + marker.length) + pkg; + } + let version = versions.get(versionRoot); + if (version === undefined) { + version = ""; + try { + const parsed = JSON.parse(trackedReadFile(`${versionRoot}/package.json`) ?? "{}") as { + version?: unknown; + }; + if (typeof parsed.version === "string") version = parsed.version; + } catch { + version = ""; + } + versions.set(versionRoot, version); + } + return { pkg, version, rel: parts.slice(nameParts).join("/") }; +} + +/** True for a node in an installed package's file. */ +export function inPackageFile(node: ts.Node): boolean { + return packageFileOf(ts.getSourceFileOfNode(node).fileName) !== null; +} + +function lineOf(node: ts.Node): number { + const sf = ts.getSourceFileOfNode(node); + return ts.getLineAndCharacterOfPosition(sf, ts.tokenStartOfNode(node)).line + 1; +} + +function encode(file: PackageFile, selector: string, where: string, declared: string): string { + const range = file.version === "" ? "*" : `^${file.version}`; + return [file.pkg, file.version, range, selector, where, declared.replace(/\s+/g, " ")].join( + ORIGIN_SEPARATOR, + ); +} + +function nameOf(name: ts.Node | undefined): string | null { + if (name === undefined) return null; + if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isPrivateIdentifier(name)) + return name.text; + return null; +} + +/** `module.exports` → "export="; `exports.x` / `module.exports.x` → "x". */ +function exportTarget(left: ts.Node): string | null { + if (!ts.isPropertyAccessExpression(left)) return null; + const obj = left.expression; + const prop = nameOf(left.name); + if (prop === null) return null; + if (ts.isIdentifier(obj) && obj.text === "module" && prop === "exports") return "export="; + if (ts.isIdentifier(obj) && obj.text === "exports") return prop; + if ( + ts.isPropertyAccessExpression(obj) && + ts.isIdentifier(obj.expression) && + obj.expression.text === "module" && + nameOf(obj.name) === "exports" + ) + return prop; + return null; +} + +/** The selector naming a function-like declaration (without `()`), or + * null when it has no name a selector can reach. */ +function functionSelector(fn: ts.Node): string | null { + if (ts.isFunctionDeclaration(fn)) return nameOf(fn.name); + if (ts.isMethodDeclaration(fn) || ts.isGetAccessor(fn)) { + const method = nameOf(fn.name); + const holder = fn.parent; + if (method === null || holder === undefined) return null; + if (ts.isClassDeclaration(holder) || ts.isClassExpression(holder)) { + const cls = nameOf(holder.name); + return cls === null ? null : `${cls}.${method}`; + } + if (ts.isObjectLiteralExpression(holder)) { + const owner = valueSelector(holder); + return owner === null ? null : `${owner}.${method}`; + } + return null; + } + if (ts.isFunctionExpression(fn) || ts.isArrowFunction(fn)) return valueSelector(fn); + return null; +} + +/** The selector of the declaration an expression is the value of. */ +function valueSelector(value: ts.Node): string | null { + let e = value; + while (e.parent !== undefined && ts.isParenthesizedExpression(e.parent)) e = e.parent; + const p = e.parent; + if (p === undefined) return null; + if (ts.isVariableDeclaration(p) && p.initializer === e) return nameOf(p.name); + if ( + ts.isBinaryExpression(p) && + p.right === e && + p.operatorToken.kind === ts.SyntaxKind.EqualsToken + ) + return exportTarget(p.left); + if (ts.isPropertyAssignment(p) && p.initializer === e && p.parent !== undefined) { + const name = nameOf(p.name); + const owner = ts.isObjectLiteralExpression(p.parent) ? valueSelector(p.parent) : null; + return name === null || owner === null ? null : `${owner}.${name}`; + } + return null; +} + +/** True when a function-like declaration writes its return type: a type + * annotation, or a JSDoc @returns tag. */ +function writesReturnType(fn: ts.Node): boolean { + if ((fn as ts.FunctionDeclaration).type !== undefined) return true; + const sf = ts.getSourceFileOfNode(fn); + let owner: ts.Node = fn; + // The JSDoc of `exports.f = function`, `const f = () =>` sits on the + // statement. + while ( + owner.parent !== undefined && + !ts.isSourceFile(owner.parent) && + !ts.isBlock(owner.parent) && + !ts.isClassDeclaration(owner.parent) && + !ts.isClassExpression(owner.parent) && + !ts.isObjectLiteralExpression(owner.parent) + ) + owner = owner.parent; + const leading = sf.text.slice(owner.pos, ts.tokenStartOfNode(owner)); + const doc = leading.lastIndexOf("/**"); + return doc >= 0 && /@returns?\s*\{/.test(leading.slice(doc)); +} + +function enclosingFunction(node: ts.Node): ts.Node | null { + for (let p = node.parent; p !== undefined; p = p.parent) { + if ( + ts.isFunctionDeclaration(p) || + ts.isFunctionExpression(p) || + ts.isArrowFunction(p) || + ts.isMethodDeclaration(p) || + ts.isGetAccessor(p) + ) + return p; + if (ts.isSourceFile(p) || ts.isClassDeclaration(p)) return null; + } + return null; +} + +function typeText(checker: ts.TypeChecker, type: ts.Type | undefined): string { + if (type === undefined) return ""; + try { + return checker.typeToString(type); + } catch { + return ""; + } +} + +/** The origin of the slot a value expression in a package file flows + * into, or null when no package declaration types that slot. */ +export function slotOrigin(checker: ts.TypeChecker, node: ts.Node): string | null { + let value = node; + while (value.parent !== undefined && ts.isParenthesizedExpression(value.parent)) + value = value.parent; + const parent = value.parent; + if (parent === undefined) return null; + // A property of an object literal typed by a declared type. + if ( + (ts.isPropertyAssignment(parent) && parent.initializer === value) || + ts.isShorthandPropertyAssignment(value) || + (ts.isShorthandPropertyAssignment(parent) && parent.name === value) + ) { + const property = ts.isShorthandPropertyAssignment(value) ? value : parent; + const literal = property.parent; + const name = nameOf((property as ts.PropertyAssignment).name); + if (literal === undefined || !ts.isObjectLiteralExpression(literal) || name === null) + return null; + const contextual = checker.getContextualType(literal); + if (contextual === undefined) return null; + const typeSymbol = contextual.getAliasSymbol() ?? contextual.getSymbol(); + const typeName = typeSymbol === undefined ? "" : ts.symbolName(typeSymbol); + if (!/^[A-Za-z_$][\w$]*$/.test(typeName)) return null; + const symbol = checker.getPropertyOfType(contextual, name); + const decl = symbol?.declarations[0]; + if (symbol === undefined || decl === undefined || genericDeclaration(decl)) return null; + const file = packageFileOf(ts.getSourceFileOfNode(decl).fileName); + if (file === null) return null; + return encode( + file, + `${typeName}.${name}`, + `${file.rel}:${lineOf(decl)}`, + typeText(checker, checker.getTypeOfSymbol(symbol)), + ); + } + // A value a package function returns under its written return type. + let fn: ts.Node | null = null; + if (ts.isReturnStatement(parent) && parent.expression === value) fn = enclosingFunction(parent); + else if (ts.isArrowFunction(parent) && parent.body === value) fn = parent; + if (fn === null || !writesReturnType(fn)) return null; + const file = packageFileOf(ts.getSourceFileOfNode(fn).fileName); + const selector = functionSelector(fn); + if (file === null || selector === null) return null; + let declared = ""; + try { + const signature = checker.getSignatureFromDeclaration(fn as ts.SignatureDeclaration); + declared = + signature === undefined ? "" : typeText(checker, checker.getReturnTypeOfSignature(signature)); + } catch { + declared = ""; + } + return encode(file, `${selector}()`, `${file.rel}:${lineOf(fn)}`, declared); +} + +/** A package-declared record type a value is converted to (the + * contextual type of a value in package code, such as an object literal + * returned under `@returns {HelpContext}`), or null. */ +export interface RecordOrigin { + type: ts.Type; + name: string; + /** The declaration's identity (package, version, file, line and name): + * records of the same shape and name declared by two packages differ. */ + key: string; +} + +export function recordOrigin(checker: ts.TypeChecker, node: ts.Node): RecordOrigin | null { + const contextual = checker.getContextualType(node); + if (contextual === undefined) return null; + const symbol = contextual.getAliasSymbol() ?? contextual.getSymbol(); + const name = symbol === undefined ? "" : ts.symbolName(symbol); + if (!/^[A-Za-z_$][\w$]*$/.test(name)) return null; + const decl = symbol?.declarations[0]; + if (decl === undefined || genericDeclaration(decl)) return null; + const file = packageFileOf(ts.getSourceFileOfNode(decl).fileName); + if (file === null) return null; + const key = [file.pkg, file.version, file.rel, lineOf(decl), name].join(ORIGIN_SEPARATOR); + return { type: contextual, name, key }; +} + +/** The origin of one field of a package-declared record type. */ +export function fieldOrigin( + checker: ts.TypeChecker, + record: RecordOrigin, + field: string, +): string | null { + const symbol = checker.getPropertyOfType(record.type, field); + const decl = symbol?.declarations[0]; + if (symbol === undefined || decl === undefined || genericDeclaration(decl)) return null; + const file = packageFileOf(ts.getSourceFileOfNode(decl).fileName); + if (file === null) return null; + return encode( + file, + `${record.name}.${field}`, + `${file.rel}:${lineOf(decl)}`, + typeText(checker, checker.getTypeOfSymbol(symbol)), + ); +} + +/** The origin of a declaration in a package file named by `selector`. */ +export function originAt(decl: ts.Node, selector: string, declared: string): string | null { + const file = packageFileOf(ts.getSourceFileOfNode(decl).fileName); + if (file === null) return null; + return encode(file, selector, `${file.rel}:${lineOf(decl)}`, declared); +} + +/** True when a declaration or the class, interface or type alias holding + * it has type parameters: its declared type then comes from how the + * program instantiates it (a lying `z.custom()`), not from the package. */ +function genericDeclaration(decl: ts.Node): boolean { + for (let n: ts.Node | undefined = decl; n !== undefined && !ts.isSourceFile(n); n = n.parent) { + const params = (n as ts.FunctionDeclaration).typeParameters; + if (params !== undefined && params.nodes.length > 0) return true; + } + return false; +} + +/** The selector of a declaration in a declaration file: a function, a + * member of a class, interface or named object type, or null (also for + * generic declarations, whose types the program supplies). */ +export function declarationSelector(decl: ts.Node): string | null { + if (genericDeclaration(decl)) return null; + const own = nameOf((decl as ts.FunctionDeclaration).name); + if (own === null) return null; + if (ts.isFunctionDeclaration(decl) || ts.isVariableDeclaration(decl)) return own; + let holder = decl.parent; + if (holder !== undefined && ts.isTypeLiteralNode(holder)) holder = holder.parent; + if ( + holder !== undefined && + (ts.isClassDeclaration(holder) || + ts.isInterfaceDeclaration(holder) || + ts.isTypeAliasDeclaration(holder)) + ) { + const owner = nameOf(holder.name); + return owner === null ? null : `${owner}.${own}`; + } + return null; +} + +/** One argument of the suggested `scriptc types patch` command, quoted so + * the shell passes it as written (scr_type_origin.c quotes the same way): + * bare when it has only characters no shell treats specially; otherwise, + * for a POSIX shell, in single quotes with each `'` as `'\''`, and for + * Windows (cmd and PowerShell), in double quotes when it has none of the + * characters either shell expands, escapes or ends a string with there. Null when no quoting + * is inert there: the note then names the command without arguments. + * Package names, selectors and types come from package declarations. */ +export function shellArgument(value: string, windows: boolean): string | null { + if (/^[A-Za-z0-9_@+=:,./-]+$/.test(value)) return value; + if (!windows) + return /^[A-Za-z0-9_@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`; + for (const ch of value) { + const code = ch.codePointAt(0)!; + // PowerShell also takes the typographic quotes U+2018-U+201F as quotes. + if (code < 0x20 || code === 0x7f || (code >= 0x2018 && code <= 0x201f)) return null; + if ('"%!^$`'.includes(ch)) return null; + } + return `"${value}"`; +} + +/** The suggested patch for a declaration: `scriptc types patch` with the + * package, the selector and the type, each quoted for the target's shell. */ +export function patchSuggestion( + pkg: string, + selector: string, + type: string, + windows: boolean, +): string { + const args = [pkg, selector, type].map((a) => shellArgument(a, windows)); + return args.every((a) => a !== null) + ? `if that declaration is wrong, patch it: scriptc types patch ${args.join(" ")}` + : "if that declaration is wrong, patch it with scriptc types patch"; +} + +/** The note a union extraction's TypeError carries for a failing `arm` + * (the same text the runtime renders for checked conversions). `windows`: + * the program targets Windows, whose shells quote differently. */ +export function renderOriginNote(origin: string, arm: string, windows = false): string { + const [pkg = "", version = "", , selector = "", where = "", declared = ""] = + origin.split(ORIGIN_SEPARATOR); + return ( + `\n ${pkg}${version === "" ? "" : `@${version}`} declares ${selector}` + + `${declared === "" ? "" : ` as ${declared}`} (${where})` + + `\n ${patchSuggestion(pkg, selector, `| ${arm}`, windows)}` + ); +} + +/** Keeps only the origins the module's checks still reference (a coercion + * whose check was later replaced leaves its origin behind), renumbered in + * first-use order; drops the table when none is left, so the runtime unit + * is not linked for nothing. */ +export function compactDeclOrigins(mod: IrModule): void { + const table = mod.declOrigins ?? []; + const remap = new Map(); + const used: string[] = []; + // A node the IR shares at several places is renumbered once. + const checks = new Set(); + everyModuleNode(mod, { + expr: (e) => { + if (e.kind === "dynCheck" && e.origin !== undefined) { + // An exact capsule check never throws: nothing to name. + if (e.exact === true) { + delete e.origin; + return true; + } + checks.add(e); + if (!remap.has(e.origin)) { + remap.set(e.origin, used.length); + used.push(table[e.origin]!); + } + } + return true; + }, + stmt: () => true, + type: () => true, + }); + for (const check of checks) check.origin = remap.get(check.origin!)!; + if (used.length === 0) delete mod.declOrigins; + else mod.declOrigins = used; +} diff --git a/packages/compiler/src/frontend/lowering/expressions/element-access.ts b/packages/compiler/src/frontend/lowering/expressions/element-access.ts index 29f249282..bbeeb7891 100644 --- a/packages/compiler/src/frontend/lowering/expressions/element-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/element-access.ts @@ -1,4 +1,6 @@ /* Element reads and writes (`a[i]`, `a[i] = v`), compound element updates, and keyed record reads. */ +import { OBJECT_PROTOTYPE_MEMBERS } from "../../shape-inference/domain.js"; +import { lowerUnionTupleRead } from "./union-fields.js"; import * as ts from "../../ts7/adapter.js"; import { lowerOptionalIndex, @@ -465,6 +467,12 @@ export function lowerElementAccess(lowerer: Lowerer, expr: ts.ElementAccessExpre "tuple indexing with a non-literal index (tuples are fixed-shape — use t[0], t[1], ...)", ); } + // A union of tuples asserted to one tuple view (`(r as readonly + // [boolean, T | E])[0]`) still holds the union. + if (obj.type.kind === "union") { + const element = lowerUnionTupleRead(lowerer, obj, idx, locOf(expr)); + if (element) return lowerer.maybeNarrow(element, expr); + } const shapeId = obj.type.kind === "record" ? obj.type.shapeId : receiverIr.shapeId; const actualShape = lowerer.shapes.get(shapeId) ?? shape; const fieldType = actualShape.fields.find((f) => f.name === String(idx))?.type; @@ -597,6 +605,15 @@ export function lowerElementAccess(lowerer: Lowerer, expr: ts.ElementAccessExpre } } } + if (key.type.kind === "f64" && ts.isNumericLiteral(expr.argumentExpression)) { + const element = lowerUnionTupleRead( + lowerer, + value, + Number(expr.argumentExpression.text), + locOf(expr), + ); + if (element) return lowerer.maybeNarrow(element, expr); + } if (key.type.kind === "f64") { const optional = optionalArmIndexRead(lowerer, value, key, locOf(expr)); if (optional) return lowerer.maybeNarrow(optional, expr); @@ -1319,6 +1336,21 @@ export function lowerRecordKeyRead( }; return lowerer.maybeNarrow(get, expr); } + // A literal key the shape does not declare, on a shape without an index + // signature, names no own property of any of its values: the read + // answers undefined, as Node's does. The checker's key check cannot vouch + // for it here, since the receiver can be typed from a package's + // JavaScript while the checker read its declarations. Object.prototype + // members keep the keyed read below. + if (shape.indexValue === undefined && !OBJECT_PROTOTYPE_MEMBERS.has(litKey)) { + return { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: obj, loc }], + result: dynUndefinedExpr(loc), + type: DYN, + loc, + }; + } } let key = litKey !== null @@ -1489,7 +1521,7 @@ function lowerRecordPropertyKey(lowerer: Lowerer, key: IrExpr, node: ts.Expressi * exactly the key JS derives; negative/computed keys stay runtime * expressions and canonicalize through ensureString instead). Null for * everything else. */ -function recordKeyLiteralText(node: ts.Expression): string | null { +export function recordKeyLiteralText(node: ts.Expression): string | null { if (ts.isStringLiteral(node)) return node.text; if (ts.isNumericLiteral(node)) return String(Number(node.text)); return null; diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index 62b2c7960..a6957401a 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -90,7 +90,7 @@ import { lowerGlobalValue } from "../lower-global-value.js"; import { fenceEnumObjectValue } from "../lower-enums.js"; import { mixinFnOfCallee } from "../lower-mixins.js"; import { isNavigatorGlobal, refuseNavigator } from "../builtins/navigator.js"; -import { wasiGuestPath } from "../../../wasi-paths.js"; +import { moduleFileName } from "../import-meta.js"; export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: SrcLoc): IrExpr { if (lowerer.isStdlibGlobal(expr, "console")) { @@ -318,10 +318,12 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: // that does not exist. The blocked-binding cascade names the use; // without this the validator's registration check ICEs on the live // reference (`@this {T}` inference — signature 07). - const unregistered = lowerer.unregisteredClassOf(g.type, new Set()); + let unregistered = lowerer.unregisteredClassOf(g.type, new Set()); + // The class's own rejection is the root cause the cascade names + // (unless the flush registers it with dynamic slots). + if (unregistered !== null && lowerer.flushDeferredClass(unregistered) !== undefined) + unregistered = lowerer.unregisteredClassOf(g.type, new Set()); if (unregistered !== null) { - // The class's own rejection is the root cause the cascade names. - lowerer.flushDeferredClass(unregistered); lowerer.pushDiag(blockedBindingUseDiag(expr.text, loc)); throw new PoisonError(); } @@ -1067,12 +1069,6 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: ); } -function moduleFileName(lowerer: Lowerer, sf: ts.SourceFile): string { - return lowerer.targetPlatform === "wasi" - ? (wasiGuestPath(sf.fileName) ?? sf.fileName.replace(/\\/g, "/")) - : sf.fileName; -} - /** A computed-key access (`value[key]` after `typeof value === "object"`) * of a runtime-optional binding the checker narrowed to one record arm of a * wider union. The binding re-tags to that record or undefined, so the keyed diff --git a/packages/compiler/src/frontend/lowering/expressions/message-listeners.ts b/packages/compiler/src/frontend/lowering/expressions/message-listeners.ts new file mode 100644 index 000000000..698bf4dc3 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/expressions/message-listeners.ts @@ -0,0 +1,22 @@ +import * as ts from "../../ts7/adapter.js"; + +const LISTENER_METHODS = new Set([ + "on", + "once", + "addListener", + "prependListener", + "prependOnceListener", +]); + +/** `.on("message", …)` and the other listener registrations. */ +export function isMessageListenerCall(call: ts.CallExpression): boolean { + const callee = call.expression; + const event = call.arguments.nodes[0]; + return ( + ts.isPropertyAccessExpression(callee) && + LISTENER_METHODS.has(callee.name.text) && + event !== undefined && + ts.isStringLiteral(event) && + event.text === "message" + ); +} diff --git a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts index 9f0559672..91b63cabc 100644 --- a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts +++ b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts @@ -1,4 +1,5 @@ /* Type narrowing of lowered values, conditional and as-expressions, and runtime-optional receiver reads. */ +import { declBoundaryOrigin } from "../../decl-projection.js"; import * as ts from "../../ts7/adapter.js"; import { type Lowerer, dynFallbackType } from "../lowerer.js"; import { @@ -50,6 +51,7 @@ import { isConstAssertionTypeNode, unitOnlyUnion } from "../../type-mapper.js"; import { assertsGlobalObject, extractionCopiesReference } from "../assertion-identity.js"; import { iterableSlotValue, iterableUnionOf } from "../iterable-values.js"; import { isDeclBoundaryTypeNode } from "../../decl-projection.js"; +import { isMessageListenerCall } from "./message-listeners.js"; /** `c ? a : b`. An explicit destination constructs each branch at that * layout before joining it. Without one, retain the expression's inferred @@ -1481,6 +1483,7 @@ export function lowerAsExpression( target?.kind === "record" && inner.type.shapeId !== target.shapeId ) { + lowerer.assertedValues.set(inner, target); return lowerer.coerceInto(expr, inner, target); } // `u as Arm` on a UNION value is `u!`'s spelling with a named arm @@ -1515,6 +1518,15 @@ export function lowerAsExpression( const asserted = checkedClassUnionAssertion(lowerer, inner, target, locOf(expr)); if (asserted) return asserted; } + // The erased value still answers to the assertion: a slot it cannot + // reach by an ordinary conversion converts it as asserted + // (assertedConversion). + if ( + target && + (target.kind === "union" || target.kind === "record" || target.kind === "array") && + (inner.type.kind === "union" || inner.type.kind === "array") + ) + lowerer.assertedValues.set(inner, target); return inner; // erasure, unchanged } @@ -1531,7 +1543,7 @@ export function lowerAsExpression( // The declaration-typed boundary (decl-projection.ts): an identity- // preserving checked conversion of a value package code produced. if (isDeclBoundaryTypeNode(expr.type)) { - const adopted = declBoundaryCast(lowerer, inner, target, locOf(expr)); + const adopted = declBoundaryCast(lowerer, inner, target, locOf(expr), expr); if (adopted) return adopted; } // An all-`unknown`-fields record target (`err as { code?: unknown }` @@ -1575,7 +1587,7 @@ export function lowerAsExpression( // contains undefined, so casts over parse results keep failing on // non-string values with the usual path-annotated TypeError. if (target.kind === "union" && lowerer.dynConvertible(target)) { - return { kind: "dynCheck", value: inner, type: target, loc: locOf(expr) }; + return checkedDynCast(lowerer, expr, inner, target); } // Uint8Array targets: the checked-dynamic tree carries a bytes kind now (converted // stdin chunks) — the extraction validates the kind and copies out. @@ -1619,9 +1631,145 @@ export function lowerAsExpression( `number, string, boolean, records, arrays, and unions of those)`, ); } + return checkedDynCast(lowerer, expr, inner, target); +} + +/** The checked conversion of a dynamic value. A record or array built from + * a value another holder may still reference keeps one identity with it + * (adoptingCast); only a source nothing else can hold (freshDynSource) + * converts outright, which keeps the fused `JSON.parse(text) as T`. */ +function checkedDynCast( + lowerer: Lowerer, + expr: ts.AsExpression | ts.TypeAssertion, + inner: IrExpr, + target: IrType, +): IrExpr { + if ( + holdsReference(lowerer, target) && + !freshDynSource(lowerer, expr, inner) && + !resultOnlyRead(lowerer, expr) + ) { + return adoptingCast(lowerer, inner, target, locOf(expr), "dyn.adoptShared"); + } return { kind: "dynCheck", value: inner, type: target, loc: locOf(expr) }; } +/** A cast whose result only serves one read of a primitive member + * (`(value as T).name`, `(value as T[]).length`): the converted value is + * never stored, compared, written or called through, and the read yields no + * reference into it, so whether it is the source itself or a copy cannot be + * observed. */ +function resultOnlyRead(lowerer: Lowerer, expr: ts.AsExpression | ts.TypeAssertion): boolean { + let node: ts.Node = expr; + while (node.parent !== undefined && ts.isParenthesizedExpression(node.parent)) node = node.parent; + const access = node.parent; + if ( + access === undefined || + !( + (ts.isPropertyAccessExpression(access) || ts.isElementAccessExpression(access)) && + access.expression === node + ) + ) + return false; + const primitive = + ts.TypeFlags.StringLike | + ts.TypeFlags.NumberLike | + ts.TypeFlags.BooleanLike | + ts.TypeFlags.BigIntLike | + ts.TypeFlags.Undefined | + ts.TypeFlags.Null; + const read = lowerer.typeOf(access); + const parts = read.isUnionType() ? ts.constituentTypes(read) : [read]; + if (!parts.every((part) => (part.flags & primitive) !== 0)) return false; + let use: ts.Node = access; + while (use.parent !== undefined && ts.isParenthesizedExpression(use.parent)) use = use.parent; + const parent = use.parent; + if (parent === undefined) return true; + if (ts.isCallExpression(parent) && parent.expression === use) return false; + if ( + ts.isBinaryExpression(parent) && + parent.left === use && + ts.isAssignmentOperator(parent.operatorToken.kind) + ) + return false; + if ( + (ts.isPrefixUnaryExpression(parent) || ts.isPostfixUnaryExpression(parent)) && + (parent.operator === ts.SyntaxKind.PlusPlusToken || + parent.operator === ts.SyntaxKind.MinusMinusToken) + ) + return false; + if (ts.isDeleteExpression(parent)) return false; + return true; +} + +/** A cast operand nothing else can hold: a value this expression creates + * (`JSON.parse(text)`, `await response.json()`), or a structured clone the + * runtime delivers once and the program reads at this site only (`workerData` + * read nowhere else; the parameter of the program's one `"message"` + * listener, read nowhere else in it). Anything else is adopted. */ +function freshDynSource( + lowerer: Lowerer, + expr: ts.AsExpression | ts.TypeAssertion, + inner: IrExpr, +): boolean { + if (inner.kind === "libCall" && inner.fn === "json.parse") return true; + if ( + inner.kind === "awaitExpr" && + inner.value.kind === "libCall" && + inner.value.fn === "fetch.responseJson" + ) + return true; + const facts = lowerer.structuredCloneReads(); + if (inner.kind === "libCall" && inner.fn === "worker.data") return facts.workerData === 1; + let operand: ts.Expression = expr.expression; + while (ts.isParenthesizedExpression(operand)) operand = operand.expression; + if (!ts.isIdentifier(operand) || inner.kind !== "varRef" || facts.messageListeners !== 1) + return false; + const symbol = lowerer.checker.getSymbolAtLocation(operand); + const param = symbol?.valueDeclaration; + if (!param || !ts.isParameter(param) || !ts.isIdentifier(param.name)) return false; + const fn = param.parent; + if ( + !(ts.isArrowFunction(fn) || ts.isFunctionExpression(fn)) || + fn.parameters.nodes[0] !== param || + !isMessageListener(fn) + ) + return false; + let reads = 0; + const name = param.name.text; + const visit = (node: ts.Node): void => { + if (ts.isIdentifier(node) && node.text === name && node !== param.name) reads++; + ts.forEachChild(node, visit); + }; + ts.forEachChild(fn, visit); + return reads === 1; +} + +/** The listener argument of `.on("message", fn)` (also once, + * addListener, prependListener, prependOnceListener) registered once: at + * module level, outside any function, class or loop, so one emitter can + * never hand the same message to two registrations of it. */ +function isMessageListener(fn: ts.Node): boolean { + const call = fn.parent; + if (!call || !ts.isCallExpression(call) || call.arguments.nodes[1] !== fn) return false; + if (!isMessageListenerCall(call)) return false; + for (let p = call.parent; p !== undefined && !ts.isSourceFile(p); p = p.parent) { + if ( + ts.isFunctionLike(p) || + ts.isClassDeclaration(p) || + ts.isClassExpression(p) || + ts.isClassStaticBlockDeclaration(p) || + ts.isForStatement(p) || + ts.isForInStatement(p) || + ts.isForOfStatement(p) || + ts.isWhileStatement(p) || + ts.isDoStatement(p) + ) + return false; + } + return true; +} + /** True when a type holds a record or an array somewhere (values whose * identity can be observed). */ function holdsReference(lowerer: Lowerer, t: IrType, seen = new Set()): boolean { @@ -1639,11 +1787,42 @@ function holdsReference(lowerer: Lowerer, t: IrType, seen = new Set()): * becomes a live capsule of the value built from it * (scr_dyn_decl_adopt), so identity and mutation visibility hold on both * sides. Scalar targets are ordinary checked casts. */ +/** The origin the projection recorded for a boundary cast (the package + * declaration its type comes from, decl-projection.ts), found by the + * cast's position among the file's boundary casts in source order. */ +function recordedBoundaryOrigin(lowerer: Lowerer, cast: ts.Node): string | null { + const sf = ts.getSourceFileOfNode(cast); + let order = lowerer.boundaryCastOrder.get(sf); + if (order === undefined) { + const found = new Map(); + const visit = (node: ts.Node): void => { + if (ts.isAsExpression(node) && isDeclBoundaryTypeNode(node.type)) found.set(node, found.size); + ts.forEachChild( + node, + (child) => { + visit(child); + return undefined; + }, + (list) => { + for (const child of list.nodes) visit(child); + return undefined; + }, + ); + }; + visit(sf); + order = found; + lowerer.boundaryCastOrder.set(sf, order); + } + const index = order.get(cast); + return index === undefined ? null : declBoundaryOrigin(sf.fileName, index); +} + function declBoundaryCast( lowerer: Lowerer, inner: IrExpr, target: IrType, loc: SrcLoc, + cast: ts.Node, ): IrExpr | null { if ( !canDynCheckTo( @@ -1654,9 +1833,42 @@ function declBoundaryCast( !(target.kind === "union" && lowerer.dynConvertible(target)) ) return null; - if (!holdsReference(lowerer, target)) { - return { kind: "dynCheck", value: inner, type: target, loc }; + // The package call or read the value comes from (decl-origin.ts): a + // failing check names it and the patch admitting the value. + let originText: string | null = null; + try { + originText = recordedBoundaryOrigin(lowerer, cast); + } catch { + originText = null; } + const originId = originText === null ? undefined : lowerer.declOriginIndex(originText); + if (!holdsReference(lowerer, target)) { + return { + kind: "dynCheck", + value: inner, + type: target, + loc, + ...(originId === undefined ? {} : { origin: originId }), + }; + } + return adoptingCast(lowerer, inner, target, loc, "dyn.declAdopt", originId); +} + +/** The checked conversion of `inner` to `target` followed by adoption + * (`dyn.declAdopt` at the boundary, the lenient `dyn.adoptShared` + * elsewhere): every source object or array that something else still + * references becomes a live capsule of the native value built from it. + * `originId` names the package declaration a failing check reports + * (decl-origin.ts). */ +function adoptingCast( + lowerer: Lowerer, + inner: IrExpr, + target: IrType, + loc: SrcLoc, + fn: "dyn.declAdopt" | "dyn.adoptShared", + originId?: number, +): IrExpr { + const boundary = fn === "dyn.declAdopt"; const source = lowerer.declareHiddenLocal("%boundarySource", DYN); const value = lowerer.declareHiddenLocal("%boundaryValue", target); return { @@ -1669,9 +1881,10 @@ function declBoundaryCast( init: { kind: "dynCheck", value: varRef(source.id, DYN, loc), - adopt: true, + ...(boundary ? { adopt: true as const } : {}), type: target, loc, + ...(originId === undefined ? {} : { origin: originId }), }, loc, }, @@ -1679,7 +1892,7 @@ function declBoundaryCast( kind: "exprStmt", expr: { kind: "libCall", - fn: "dyn.declAdopt", + fn, args: [ varRef(source.id, DYN, loc), { diff --git a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts index f57e52159..31479e4b0 100644 --- a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts +++ b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts @@ -419,7 +419,12 @@ export function lowerDynObjectLiteral( : !boxValue && (ts.isObjectLiteralExpression(valueExpr) || (ts.isArrayLiteralExpression(valueExpr) && - holdsEvolvingArray(lowerer, valueExpr))) + // An empty literal takes whatever the object's users + // store into it (`{ [kChildren]: [] }`), so it is a + // checked-dynamic array rather than its `never[]` layout. + (valueExpr.elements.nodes.length === 0 || + holdsEvolvingArray(lowerer, valueExpr) || + lowerer.untypedLiteralContext(valueExpr)))) ? lowerer.lowerExprExpecting(valueExpr, DYN) : lowerer.lowerExpr(valueExpr as ts.Expression); raw = fenceClosureProbe(lowerer, valueExpr, undefined, lowerValue) ?? lowerValue(); @@ -2529,6 +2534,20 @@ export function lowerObjectLiteral( : { kind: "seqExpr", stmts: spreadStmts, result, type, loc }; } +/** The field type is an array, or a union with exactly one array arm. */ +function singleArrayArm(lowerer: Lowerer, type: IrType): boolean { + if (type.kind === "array") return true; + if (type.kind !== "union") return false; + const arms = lowerer.unions.get(type.unionId)?.arms ?? []; + return arms.filter((arm) => arm.kind === "array").length === 1; +} + +function isEmptyArrayLiteral(node: ts.Node): boolean { + let inner = node; + while (ts.isParenthesizedExpression(inner)) inner = inner.expression; + return ts.isArrayLiteralExpression(inner) && inner.elements.nodes.length === 0; +} + /** A union of plain records can preserve its existing variant while * replacing shared fields. Dispatch before constructing the copy: only * the selected arm evaluates overrides, and recordClone snapshots fields @@ -2577,8 +2596,11 @@ function lowerUnionRecordClone(lowerer: Lowerer, expr: ts.ObjectLiteralExpressio for (let index = 0; index < props.length; index++) { const prop = props[index]!; const node = ts.isPropertyAssignment(prop) ? prop.initializer : prop.name; - const valueType = lowerer.mapTypeOf(lowerer.typeOf(node)); const fieldType = shape.fields.find((field) => field.name === names[index])!.type; + // An empty array literal (`content: []`, typed `never[]`) builds a + // fresh array of each arm's own element type. + if (isEmptyArrayLiteral(node) && singleArrayArm(lowerer, fieldType)) continue; + const valueType = lowerer.mapTypeOf(lowerer.typeOf(node)); const lift = valueType ? lowerer.widthLiftPlan(valueType, fieldType) : null; // The inferred result must retain an override's own fields and // aliases. A structural copy into the source field's narrower shape diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 6466857a5..97ec9a320 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -156,7 +156,10 @@ import { blockedBindingUseDiag } from "../../../diagnostics/diagnostic.js"; import { npmStaticPackageOfPath } from "../../npm-static.js"; import { importMetaField, isImportMeta } from "../import-meta.js"; import { createRequireCalleeFileOf } from "../builtins/module-bindings.js"; -import { createRequireResolveFenceValue } from "../builtins/module-resolution.js"; +import { + createRequireResolveFenceValue, + createRequireResolveValue, +} from "../builtins/module-resolution.js"; import { builtinPrototypeMethod, lowerArrayIsArrayValue, @@ -307,6 +310,7 @@ function isCreateRequireResolveValue(lowerer: Lowerer, expr: ts.PropertyAccessEx ) resolve = expr.expression; return ( + !expr.questionDotToken && ts.isPropertyAccessExpression(resolve) && resolve.name.text === "resolve" && createRequireCalleeFileOf(lowerer, resolve.expression) !== null @@ -1352,10 +1356,13 @@ export function lowerPropertyAccessExpr( } } // A createRequire loader's resolve read as a value (passed on, or - // picked by `??`): resolution happens at build time, so only calls - // with a literal specifier lower. + // picked by `??`): a function that resolves its request at run time. + // resolve.paths read as a value has no run-time lowering: a function + // that throws the fence when called. if (isCreateRequireResolveValue(lowerer, expr)) - return createRequireResolveFenceValue(lowerer, expr, loc); + return expr.name.text === "paths" + ? createRequireResolveFenceValue(lowerer, expr, loc) + : createRequireResolveValue(lowerer, expr, loc); // The lib fence's PROPERTY chokepoint: a stdlib-declared member that // no lowering above claimed ([1,2].entries, Math.SQRT2, Promise.all, // re.exec as a value, ...) reports SC2020 here. @@ -3369,6 +3376,18 @@ export function lowerUnionProperty( expr, ); } + // An array the checker sees through a union of arrays (`readdirSync`'s + // `string[] | Buffer[]`, whose lowering answers the string[] arm). + if (value.type.kind === "array" && expr.name.text === "length") { + return { + kind: "arrIntrinsic", + method: "length", + receiver: value, + args: [], + type: F64, + loc: locOf(expr), + }; + } if (value.type.kind !== "union") { lowerer.unsupported( "SC1090", @@ -3644,13 +3663,13 @@ export function classFieldTarget( ) { return { container: "dynamic", obj: represented, field: fieldName, fieldType: DYN }; } - const info = lowerer.classes.get(receiverType.className); - if (!info) { - // A receiver typed as a class whose collection deferred: the - // deferred diagnostics are what explains the miss. + // A receiver typed as a class whose collection deferred: the deferred + // diagnostics are what explains the miss (unless the flush registers + // the class with dynamic slots, retryDeferred). + const info = + lowerer.classes.get(receiverType.className) ?? lowerer.flushDeferredClass(receiverType.className); - return null; - } + if (!info) return null; const fieldType = info.fields.get(fieldName); const lowerObjectReceiver = (): IrExpr => { let obj = represented; diff --git a/packages/compiler/src/frontend/lowering/expressions/relational.ts b/packages/compiler/src/frontend/lowering/expressions/relational.ts index b323162c0..89817e057 100644 --- a/packages/compiler/src/frontend/lowering/expressions/relational.ts +++ b/packages/compiler/src/frontend/lowering/expressions/relational.ts @@ -535,12 +535,13 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc }; } if (rhsClassval?.kind === "classval" && !caught) { - const targetInfo = lowerer.classes.get(rhsClassval.className); + const targetInfo = + lowerer.classes.get(rhsClassval.className) ?? + lowerer.flushDeferredClass(rhsClassval.className); if (!targetInfo) { // The type world names a class the lowering never registered - // (a fenced class expression, a deferred declaration): flush - // its diagnostics and poison the test site, never an ICE. - lowerer.flushDeferredClass(rhsClassval.className); + // (a fenced class expression, a deferred declaration): its + // diagnostics flushed above; poison the test site, never an ICE. lowerer.unsupported( "SC1090", expr, diff --git a/packages/compiler/src/frontend/lowering/expressions/union-fields.ts b/packages/compiler/src/frontend/lowering/expressions/union-fields.ts index 4f4532a6a..01c841fdd 100644 --- a/packages/compiler/src/frontend/lowering/expressions/union-fields.ts +++ b/packages/compiler/src/frontend/lowering/expressions/union-fields.ts @@ -281,3 +281,58 @@ export function lowerUnionFieldWrite( } return { kind: "seqExpr", stmts: body, result: dispatch, type: result.type, loc }; } + +/** Read the element at a literal index from a union of tuples whose element + * types differ (`readonly [true, number] | readonly [false, string]`, index + * 1): each tag reads its own tuple's slot and widens it into the union of + * the arms' element types; the caller narrows the result to the checker's + * flow type. Null when an arm is not a tuple with that position. */ +export function lowerUnionTupleRead( + lowerer: Lowerer, + receiver: IrExpr, + index: number, + loc: IrExpr["loc"], +): IrExpr | null { + if (receiver.type.kind !== "union" || !Number.isInteger(index) || index < 0) return null; + const unionId = receiver.type.unionId; + const arms = lowerer.unions.get(unionId)?.arms; + if (!arms || arms.length < 2) return null; + const name = String(index); + const fields: UnionFieldArm[] = []; + const parts: IrType[] = []; + for (const arm of arms) { + if (arm.kind !== "record" || lowerer.shapes.get(arm.shapeId)?.tuple !== true) return null; + const field = unionFieldArm(lowerer, arm, name); + if (!field || field.type.kind === "void") return null; + fields.push(field); + const own = + field.type.kind === "union" ? lowerer.unions.get(field.type.unionId)?.arms : [field.type]; + if (!own) return null; + for (const part of own) if (!parts.some((p) => typeEquals(p, part))) parts.push(part); + } + const type: IrType = + parts.length === 1 ? parts[0]! : { kind: "union", unionId: lowerer.unions.intern(parts) }; + const object = lowerer.declareHiddenLocal("%unionReadObject", receiver.type); + const objectRef = varRef(object.id, object.type, loc); + const read = (tag: number): IrExpr | null => { + const probe = unionFieldRead(unionId, tag, objectRef, fields[tag]!, name, loc); + const converted = lowerer.coerceToExpected(probe, type); + return typeEquals(converted.type, type) && isWidening(converted, probe) ? converted : null; + }; + let dispatch = read(fields.length - 1); + if (!dispatch) return null; + for (let tag = fields.length - 2; tag >= 0; tag--) { + const then = read(tag); + if (!then) return null; + dispatch = { + kind: "ternary", + cond: { kind: "unionIsTag", unionId, tag, value: objectRef, negated: false, type: BOOL, loc }, + then, + else_: dispatch, + type, + loc, + }; + } + const init: IrStmt = { kind: "varDecl", localId: object.id, init: receiver, loc }; + return { kind: "seqExpr", stmts: [init], result: dispatch, type, loc }; +} diff --git a/packages/compiler/src/frontend/lowering/import-meta.ts b/packages/compiler/src/frontend/lowering/import-meta.ts index 796e64b69..ccc86bfba 100644 --- a/packages/compiler/src/frontend/lowering/import-meta.ts +++ b/packages/compiler/src/frontend/lowering/import-meta.ts @@ -17,14 +17,19 @@ export function isImportMeta(expr: ts.Expression): boolean { ); } +/** The module's file name as the compiled program reports it + * (import.meta.filename): the WASI guest path on wasi targets. */ +export function moduleFileName(lowerer: Lowerer, sf: ts.SourceFile): string { + return lowerer.targetPlatform === "wasi" + ? (wasiGuestPath(sf.fileName) ?? sf.fileName.replace(/\\/g, "/")) + : sf.fileName; +} + /** File-backed metadata is fixed by the compiled module graph. Reads share * one implementation across dot access, folded keys and selected bindings. */ export function importMetaField(lowerer: Lowerer, node: ts.Node, name: string): IrExpr { const sf = ts.getSourceFileOfNode(node); - const fileName = - lowerer.targetPlatform === "wasi" - ? (wasiGuestPath(sf.fileName) ?? sf.fileName.replace(/\\/g, "/")) - : sf.fileName; + const fileName = moduleFileName(lowerer, sf); const loc = locOf(node); switch (name) { case "env": diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 0a845b3bd..0c4afa9f8 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -10107,7 +10107,13 @@ function lowerObjLitGenericMethodCall( type: DYN, loc, }; - return lowerer.coerceInto(call, invoked, lowerer.irTypeOf(call)); + const resultType = lowerer.irTypeOf(call); + // A void method's result stays the runtime's untyped value, as a + // call through a non-generic void member's dispatch keeps it: a class + // may implement the member with a method that returns a value, which + // JavaScript passes on (`console.log(c.publish(m))`). + if (resultType.kind === "void") return invoked; + return lowerer.coerceInto(call, invoked, resultType); } } if (onInterface) { @@ -10229,8 +10235,8 @@ export function lowerObjectMethodCall( ? lowerer.coerceInto(access.expression, receiver, receiverIr) : receiver; }; - const info = lowerer.classes.get(receiverIr.className); - if (!info) lowerer.flushDeferredClass(receiverIr.className); + const info = + lowerer.classes.get(receiverIr.className) ?? lowerer.flushDeferredClass(receiverIr.className); const found = info ? lowerer.findMethodOn(info, access.name.text) : null; // The zlib stream members (flush/params/close/reset) are declared on // the zlib classes; their receivers are runtime Transform streams. @@ -10583,11 +10589,8 @@ function lowerUnionObjectMethodCall( found: NonNullable>; }[] = []; for (const arm of def.arms as (IrType & { kind: "object" })[]) { - const info = lowerer.classes.get(arm.className); - if (!info) { - lowerer.flushDeferredClass(arm.className); - return null; - } + const info = lowerer.classes.get(arm.className) ?? lowerer.flushDeferredClass(arm.className); + if (!info) return null; const found = lowerer.findMethodOn(info, method); if (!found) return null; if (found.sig.abstract === true && !lowerer.overrideBelow(info, method)) return null; diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index d941d7b60..006917319 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -116,6 +116,7 @@ import { requireSpecOf, } from "../program.js"; import { PoisonError, dynFallbackType, jsFuncNameOf, newFnCtx, own } from "./lowerer.js"; +import { declarationGroupKey } from "./type-failure.js"; import { isAssertClassNew } from "./assert-instances.js"; import { isCjsRequireGlobal } from "./lower-node-module.js"; import { lowerArrayConstructor } from "./containers/array-construction.js"; @@ -2297,11 +2298,21 @@ export function collectClassShapeInner( base = named; if (!base) { if (adoptCallableBase(t.expression)) continue; - lowerer.unsupported( + // A base whose own collection deferred can still register later + // (a flush retry with dynamic slots): the derived class retries + // with it (Lowerer.retryDeferred). + const diag = unsupportedDiag( "SC1090", - t, + locOf(t), `extending classes not declared in the program ('${t.expression.text}')`, ); + lowerer.retriableDiags.add(diag); + // A base the program declares but rejected: this rejection is its + // cascade (localized failure, report-only). + const rejectedBase = rejectedProgramClassGroup(lowerer, symbol); + if (rejectedBase !== null) diag.cause = [rejectedBase]; + lowerer.pushDiag(diag); + throw new PoisonError(); } } @@ -2537,13 +2548,13 @@ export function collectClassShapeInner( if (type.kind === "void") lowerer.badType(member.name, lowerer.typeOf(member.name)); if ( type.kind === "dyn" && + !lowerer.liftDynFieldPolicy && !isJsSourceFile(ts.getSourceFileOfNode(member)) && !lowerer.classFieldStoresDyn(lowerer.typeOf(member.name)) && (member.type !== undefined || (lowerer.typeOf(member.name).flags & ts.TypeFlags.Any) === 0) ) { - lowerer.unsupported( - "SC1090", + lowerer.dynFieldPolicyFence( member.name, `${dynFieldTypeSpelling(lowerer, member.name)}-typed static fields`, ); @@ -3053,14 +3064,16 @@ export function collectClassShapeInner( if (type.kind === "void") lowerer.badType(member.name, lowerer.typeOf(member.name)); // Bundled JS declares fields without annotations before assigning // them in the constructor. Use the same native checked-dynamic - // storage as implicit constructor-assigned JS fields. + // storage as implicit constructor-assigned JS fields. In + // TypeScript the refusal is a policy fence that a flush retry lifts + // (Lowerer.liftDynFieldPolicy). if ( type.kind === "dyn" && + !lowerer.liftDynFieldPolicy && !isJsSourceFile(ts.getSourceFileOfNode(member)) && !lowerer.classFieldStoresDyn(lowerer.typeOf(member.name)) ) { - lowerer.unsupported( - "SC1090", + lowerer.dynFieldPolicyFence( member.name, `${dynFieldTypeSpelling(lowerer, member.name)}-typed class fields`, ); @@ -3196,9 +3209,12 @@ export function collectClassShapeInner( const type = shape.bodyType ?? shape.type; if (type.kind === "void") lowerer.badType(p.name, lowerer.typeOf(p.name)); // The class-field dyn rule verbatim (KEEP NARROW). - if (type.kind === "dyn" && !lowerer.classFieldStoresDyn(lowerer.typeOf(p))) { - lowerer.unsupported( - "SC1090", + if ( + type.kind === "dyn" && + !lowerer.liftDynFieldPolicy && + !lowerer.classFieldStoresDyn(lowerer.typeOf(p)) + ) { + lowerer.dynFieldPolicyFence( p.name, `${dynFieldTypeSpelling(lowerer, p)}-typed class fields`, ); @@ -6205,6 +6221,7 @@ function lowerClassValue( const context = newFnCtx(true, null, null, VOID); info.localClass.context = context; lowerer.fnStack.push(context); + const diagsBefore = lowerer.diags.length; try { const bodies = lowerClassMembers(lowerer, info); for (const body of bodies) info.localClass.bodies.set(body.name, body); @@ -6219,6 +6236,8 @@ function lowerClassValue( }); } info.localClass.ready = true; + // A retry of the enclosing statement reuses these bodies. + for (const d of lowerer.diags.slice(diagsBefore)) lowerer.loweredOnceDiags.add(d); lowerer.onExprClassCollected?.(info); } finally { lowerer.fnStack.pop(); @@ -11315,12 +11334,13 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { ); info = lowerer.classes.get(calleeT.className); } + // The TYPE world names a class the lowering never registered — a + // fenced class expression, an abstract/deferred declaration: flush + // its own diagnostics (they tell the real story, unless the flush + // registers it with dynamic slots) and poison this construction + // site, never an ICE. + if (!info) info = lowerer.flushDeferredClass(calleeT.className); if (!info) { - // The TYPE world names a class the lowering never registered — - // a fenced class expression, an abstract/deferred declaration: - // flush its own diagnostics (they tell the real story) and - // poison this construction site, never an ICE. - lowerer.flushDeferredClass(calleeT.className); lowerer.unsupported( "SC1090", expr, @@ -11850,3 +11870,14 @@ function definedThroughReflection(lowerer: Lowerer, decl: ts.ClassLikeDeclaratio }); return found; } + +/** The failure group of a class the program declares but did not collect + * (its own declaration was rejected), or null for any other symbol. */ +function rejectedProgramClassGroup(lowerer: Lowerer, symbol: ts.Symbol | null): string | null { + if (symbol === null) return null; + const recorded = lowerer.rejectedGroups.get(symbol); + if (recorded !== undefined) return recorded; + const decl = symbol.declarations.find((d) => ts.isClassDeclaration(d) || ts.isClassExpression(d)); + if (decl === undefined || ts.getSourceFileOfNode(decl).isDeclarationFile) return null; + return declarationGroupKey(symbol); +} diff --git a/packages/compiler/src/frontend/lowering/lower-exprs.ts b/packages/compiler/src/frontend/lowering/lower-exprs.ts index 006975085..3bc9ed7b0 100644 --- a/packages/compiler/src/frontend/lowering/lower-exprs.ts +++ b/packages/compiler/src/frontend/lowering/lower-exprs.ts @@ -96,7 +96,8 @@ import { lowerUnionKeyedRead, } from "./expressions/property-access.js"; import { lowerNullishPair } from "./expressions/binary.js"; -import { lowerElementCompoundParts } from "./expressions/element-access.js"; +import { lowerElementCompoundParts, recordKeyLiteralText } from "./expressions/element-access.js"; +import { OBJECT_PROTOTYPE_MEMBERS } from "../shape-inference/domain.js"; export { DEFERRED_READ_MESSAGE, abstractPropertyDeclOf, @@ -1964,6 +1965,23 @@ export function lowerAbsenceProbe(lowerer: Lowerer, node: ts.Expression): IrExpr if (shape && !shape.tuple && shape.indexValue && shape.indexValue.kind !== "dyn") { return lowerer.lowerRecordKeyRead(expr, receiverT.shapeId, shape, true); } + // A literal key the shape does not declare is absent from every value + // (lowerRecordKeyRead answers undefined), whatever type the checker gave + // the read. An optional chain keeps its own lowering, which reads a + // nullish receiver as undefined instead of converting it. + const literal = recordKeyLiteralText(expr.argumentExpression); + if ( + !expr.questionDotToken && + !hasOptionalChainGuard(expr.expression) && + shape && + !shape.tuple && + !shape.indexValue && + literal !== null && + !OBJECT_PROTOTYPE_MEMBERS.has(literal) && + !shape.fields.some((field) => field.name === literal) + ) { + return lowerer.lowerRecordKeyRead(expr, receiverT.shapeId, shape, true); + } return null; } // `m[2]` on a JavaScript `RegExpExecArray | null` that a previous @@ -3136,11 +3154,9 @@ export function symbolFieldInfo( if (lowerer.chainBlocked(expr)) return null; const receiverIr = lowerer.mapTypeOf(lowerer.typeOf(expr.expression)); if (receiverIr?.kind !== "object") return null; - const info = lowerer.classes.get(receiverIr.className); - if (!info) { - lowerer.flushDeferredClass(receiverIr.className); - return null; - } + const info = + lowerer.classes.get(receiverIr.className) ?? lowerer.flushDeferredClass(receiverIr.className); + if (!info) return null; const key = classSymbolKeyOf(lowerer, expr.argumentExpression); if (!key) return null; const field = info.symbolFields?.get(key.identity); diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index 3f4a70c74..57c58a760 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -1,3 +1,4 @@ +import type { TypeFailureSlot } from "../../diagnostics/diagnostic.js"; import { THREADS_MODULE } from "../threads-import.js"; import { positionTag } from "./structural-positions.js"; import { EXPERIMENTAL_MODULE } from "../experimental-import.js"; @@ -178,6 +179,17 @@ export interface AppendGraph { * the require() edges preflight resolved — a CommonJS file's only edges * (the compiled CommonJS package a workspace module loaded on demand * imports, and the sibling packages and files IT requires). */ +/** Whether a class declaration has a static field with an initializer, + * which Node runs where the class statement evaluates. */ +function hasStaticFieldInitializer(decl: ts.ClassDeclaration): boolean { + return decl.members.nodes.some( + (member) => + ts.isPropertyDeclaration(member) && + member.initializer !== undefined && + member.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.StaticKeyword) === true, + ); +} + function appendedEdgesOf( program: ts.Program, graph: AppendGraph | undefined, @@ -1035,13 +1047,11 @@ export function moduleArtifacts( } while (pendingClasses.length > 0) { const name = pendingClasses.pop()!; - const info = lowerer.classes.get(name); - if (!info) { - // Referenced by an emitted type but never registered: collection - // deferred its diagnostics — a reached reference makes them count. - lowerer.flushDeferredClass(name); - continue; - } + // Referenced by an emitted type but never registered: collection + // deferred its diagnostics — a reached reference makes them count + // (or registers the class with dynamic slots, retryDeferred). + const info = lowerer.classes.get(name) ?? lowerer.flushDeferredClass(name); + if (!info) continue; for (const field of info.def.fields) visitType(field.type); if (info.base) visitClass(info.base.def.name); if (info.def.genericOf) visitClass(info.def.genericOf); @@ -2048,7 +2058,15 @@ export function collectGlobals( // Destructuring declarations register EVERY bound identifier (the // desugar in the init function assigns the pre-registered globals, // exactly like plain declarations). - for (const nameNode of boundIdentifiersOf(decl.name)) { + const names = boundIdentifiersOf(decl.name); + // Localized failure: a degradable type failure at a module global + // degrades that type and the same name registers again + // (globalSlotRetry), at most MAX_GLOBAL_RETRIES times. + let retries = 0; + let retriedIndex = -1; + for (let nameIndex = 0; nameIndex < names.length; nameIndex++) { + const nameNode = names[nameIndex]!; + if (nameIndex !== retriedIndex) retries = 0; const diagsBefore = lowerer.diags.length; try { // Checked JavaScript factories can expose inferred callables by @@ -2700,6 +2718,12 @@ export function collectGlobals( } } catch (e) { if (!(e instanceof PoisonError)) throw e; + if (retries < MAX_GLOBAL_RETRIES && globalSlotRetry(lowerer, sf, e)) { + retries++; + retriedIndex = nameIndex; + nameIndex--; + continue; + } // diagnostic already recorded; lowering the statement poisons // too — and in a JS file BOTH defer to the statement's // runtime fence (deferJsCollectionDiags). @@ -2710,6 +2734,20 @@ export function collectGlobals( } } +/** Localized failure's commit point for module globals (design §3.3): + * collectGlobals commits a global's type failure directly, before any body + * lowers, so it degrades here and the registration runs again. */ +function globalSlotRetry(lowerer: Lowerer, sf: ts.SourceFile, e: PoisonError): boolean { + if (isJsSourceFile(sf)) return false; + const slot = lowerer.degradeAtCommit(e, GLOBAL_SLOTS); + if (slot === null) return false; + lowerer.noteDynamicSlot(slot); + return true; +} + +const MAX_GLOBAL_RETRIES = 16; +const GLOBAL_SLOTS: ReadonlySet = new Set(["global"]); + /** Registers the entry-init note for a `var` module global: JS hoists * module vars to `undefined` at module entry, so an undefined-armed slot * must hold the interned undefined arm BEFORE the body runs (a function @@ -3202,7 +3240,19 @@ export function lowerFileInit( // by position) — exactly when JS evaluates decorators, static // initializers, and blocks, so code reading an earlier module // binding sees its assigned value. - const statics = [...lowerer.classes.values()] + const runsStatics = (c: ClassInfo): boolean => + lowerer.classInitFlags.has(c.def.name) || + c.callableBase !== undefined || + c.def.baseValueGlobal !== undefined || + c.runtimeStatics !== undefined || + c.staticFields.length > 0 || + (c.staticBlocks?.length ?? 0) > 0 || + c.classDecorators !== undefined; + const statics: { + pos: number; + info: ClassInfo | null; + deferred?: { symbol: ts.Symbol; decl: ts.ClassDeclaration }; + }[] = [...lowerer.classes.values()] .filter( (c) => // Class DECLARATIONS only: expression classes run their static @@ -3214,15 +3264,30 @@ export function lowerFileInit( ts.isClassDeclaration(c.decl) && !c.mixinInstance && ts.getSourceFileOfNode(c.decl) === sf && - (lowerer.classInitFlags.has(c.def.name) || - c.callableBase !== undefined || - c.def.baseValueGlobal !== undefined || - c.runtimeStatics !== undefined || - c.staticFields.length > 0 || - (c.staticBlocks?.length ?? 0) > 0 || - c.classDecorators !== undefined), + runsStatics(c), ) .map((c) => ({ pos: ts.tokenStartOfNode(c.decl!), info: c })); + // Localized failure: a deferred class with static field initializers + // is reached at its statement, where Node runs them. Its place retries + // it (retryDeferredAtDeclaration), and once it has registered there or + // through an earlier flush, its initializers lower in place. + for (const symbol of lowerer.deferredCollectors.keys()) + for (const decl of symbol.declarations) + if ( + ts.isClassDeclaration(decl) && + ts.getSourceFileOfNode(decl) === sf && + hasStaticFieldInitializer(decl) + ) + statics.push({ pos: ts.tokenStartOfNode(decl), info: null, deferred: { symbol, decl } }); + const lowerStaticsAt = (entry: (typeof statics)[number]): IrStmt[] => { + let info = entry.info; + if (entry.deferred) { + if (!lowerer.retryDeferredAtDeclaration(entry.deferred.symbol)) return []; + info = lowerer.classes.get(lowerer.classNamer(entry.deferred.decl)) ?? null; + if (info === null || !runsStatics(info)) return []; + } + return withClassInitFlag(lowerer, info!, lowerer.lowerStaticFieldInits(info!)); + }; // Statics-bearing MIXIN instantiations whose call evaluates in THIS // file: their declaration-time code runs when the call does — the // top-level statement lower-mixins recorded. Insertion order is @@ -3243,13 +3308,7 @@ export function lowerFileInit( // declarations never share a kept statement's start, so `<=` // changes nothing for them. while (at < statics.length && statics[at]!.pos <= ts.tokenStartOfNode(stmt)) { - body.push( - ...withClassInitFlag( - lowerer, - statics[at]!.info, - lowerer.lowerStaticFieldInits(statics[at]!.info), - ), - ); + body.push(...lowerStaticsAt(statics[at]!)); at++; } // A cycle binding's declaration stays at its position: its @@ -3268,13 +3327,7 @@ export function lowerFileInit( body.push(...lowerer.pendingClassExprInits.splice(0), ...stmtIr); } while (at < statics.length) { - body.push( - ...withClassInitFlag( - lowerer, - statics[at]!.info, - lowerer.lowerStaticFieldInits(statics[at]!.info), - ), - ); + body.push(...lowerStaticsAt(statics[at]!)); at++; } for (const { at, g } of unionEntryInits) { diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 84196476a..09f5baab0 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -40,7 +40,9 @@ import { isJsonRequireDecl } from "./json-require.js"; import { isTimeValueDateDeclaration } from "./date-values.js"; import { bindingSource } from "../binding-source.js"; import { lowerConstructorReturn } from "./constructor-returns.js"; -import type { Lowerer } from "./lowerer.js"; +import type { DynamicSlot, Lowerer } from "./lowerer.js"; +import type { ScrDiagnostic, TypeFailureSlot } from "../../diagnostics/diagnostic.js"; +import { statementGroupKey } from "./type-failure.js"; import { cjsExportValueByName, cjsNamespaceDestructureOf, @@ -415,8 +417,8 @@ function cleanupCall( ): { call: IrExpr; async: boolean } | null { const loc = locOf(node); if (value.type.kind === "object") { - const info = lowerer.classes.get(value.type.className); - if (!info) lowerer.flushDeferredClass(value.type.className); + const info = + lowerer.classes.get(value.type.className) ?? lowerer.flushDeferredClass(value.type.className); const asyncFound = awaited && info ? lowerer.findMethodOn(info, "sym:asyncDispose") : null; const syncFound = info ? lowerer.findMethodOn(info, "sym:dispose") : null; const member = asyncFound ? "sym:asyncDispose" : syncFound ? "sym:dispose" : null; @@ -750,57 +752,38 @@ export function lowerStmts(lowerer: Lowerer, stmts: readonly ts.Statement[]): Ir if (!lowerer.suppressStats) lowerer.countStatement(stmt, sf?.fileName); const diagsBefore = lowerer.diags.length; const sinkMark = lowerer.diagSinkMark(); + const touched = new Set(); + lowerer.failureTouches.push(touched); + const pendingSlots: DynamicSlot[] = []; + // A declarator that initializes a forward-captured TDZ box takes it + // out of tdzPredeclared; a retry after a later declarator degrades + // must initialize the same box again. + const tdzBefore = new Map(lowerer.ctx.tdzPredeclared); try { - let lowered: IrStmt | IrStmt[] | null; - let appended = false; - if ( - ts.isVariableStatement(stmt) && - (stmt.declarationList.flags & ts.NodeFlags.Using) !== 0 - ) { - const registrations = lowerUsingStatement(lowerer, stmt); - const accounting: IrStmt[] = []; - for (const registration of registrations) { - out.push(...registration.acquire); - accounting.push(...registration.acquire, ...registration.cleanup); - if (registration.cleanup.length > 0) { - usingMarkers.push({ - start: out.length, - cleanup: registration.cleanup, - loc: registration.loc, - }); - } + for (let attempt = 0; ; attempt++) { + try { + lowerOneStatement(lowerer, stmt, out, usingMarkers, diagsBefore, deferFences); + break; + } catch (e) { + // Localized failure's statement commit point (design §3.3): a + // degradable type failure at a local, loop variable or module + // global degrades that type and the statement lowers again. + // A declaration this statement flushed inside a probe (where + // no retry runs) can retry here too. + if (attempt >= MAX_DEGRADE_RETRIES || deferFences) throw e; + const slot = lowerer.degradeAtCommit(e, STATEMENT_SLOTS); + if (slot === null && !lowerer.reopenFlushedAtCommit(diagsBefore)) throw e; + // A failed attempt adds nothing to `out` but the TDZ boxes + // forward captures declared (they stay declared), so the + // retry appends after them. + lowerer.dropAttemptDiags(diagsBefore); + if (slot !== null) pendingSlots.push(slot); + for (const [symbol, local] of tdzBefore) + if (!lowerer.ctx.tdzPredeclared.has(symbol)) + lowerer.ctx.tdzPredeclared.set(symbol, local); } - lowered = accounting; - appended = true; - } else { - lowered = lowerer.lowerStmt(stmt); - } - // The lib-boundary chokepoint (lib-boundary.ts): checked-dynamic - // arguments that reached builtin-call slots without the checked - // coercion get their dynCheck here, and the uncoercible ones - // fence — inside this statement's poison window, so a JS fence - // becomes the statement's runtimeFence exactly like every other - // deferred rejection. - if (lowered) { - const body: IrStmt[] = []; - if (Array.isArray(lowered)) body.push(...lowered); - else body.push(lowered); - const rewritten = lowerSurplusCalls(lowerer, body); - enforceLibBoundary(lowerer, rewritten); - if (!appended) out.push(...rewritten); - } - // A JS statement that lowered CLEAN can still have flushed - // DEFERRED collection diagnostics (a symbol lookup on an alias - // resolves through flushDeferred — informational here, not a - // blocker of THIS statement): those belong to the runtime-fence - // ledger, not the build — real uses of the broken declaration - // meet their own per-site fences. - if (deferFences && lowerer.diagSink === null && lowerer.diags.length > diagsBefore) { - const flushed = lowerer.diags.splice(diagsBefore); - const ice = flushed.filter((d) => d.code === "SC9001"); - lowerer.diags.push(...ice); - lowerer.runtimeFences.push(...flushed.filter((d) => d.code !== "SC9001")); } + for (const slot of pendingSlots) lowerer.noteDynamicSlot(slot); } catch (e) { // A Go panic inside tsgo, surfaced by the sync channel as a // thrown Error (server intact — the prefetch fence's finding): @@ -813,7 +796,17 @@ export function lowerStmts(lowerer: Lowerer, stmts: readonly ts.Statement[]): Ir throw e; } if (!lowerer.suppressStats) lowerer.countStatementFailed(stmt, sf?.fileName); - lowerer.noteBlockedBindings(stmt); + // Localized failure (report-only): the statement's committed + // diagnostics that are not themselves type failures or rejected + // declarations are cascades of the rejected declarations it used. + const committed = + lowerer.diagSink === null && !deferFences ? lowerer.diags.slice(diagsBefore) : []; + if (touched.size > 0) { + const cause = [...touched]; + for (const d of committed) + if (d.typeFailure === undefined && d.rejects === undefined) d.cause ??= cause; + } + lowerer.noteBlockedBindings(stmt, committed); // The runtime-fence conversion (JS sources, direct diagnostics // only — a capture wrapper in force means collection owns the // report). The statement's diagnostics move OFF the build and @@ -837,6 +830,8 @@ export function lowerStmts(lowerer: Lowerer, stmts: readonly ts.Statement[]): Ir const fence = lowerer.capturedRuntimeFence(sinkMark, stmt); if (fence) out.push(fence); } + } finally { + lowerer.failureTouches.pop(); } } // Each acquisition protects the remainder of its lexical statement @@ -862,6 +857,81 @@ export function lowerStmts(lowerer: Lowerer, stmts: readonly ts.Statement[]): Ir return out; } +/** Localized failure: retries a degradable statement-level type failure + * at most this many times (each retry degrades one more type). */ +const MAX_DEGRADE_RETRIES = 16; + +/** The slot kinds lowerStmts' commit point degrades: locals, loop + * variables and module globals (stage L1), and the parameters and returns + * of the function expressions, arrows and methods a statement holds + * (stage L2). */ +const STATEMENT_SLOTS: ReadonlySet = new Set([ + "local", + "global", + "loop", + "param", + "return", + "field", +]); + +/** One attempt at lowering a statement of lowerStmts into `out`. */ +function lowerOneStatement( + lowerer: Lowerer, + stmt: ts.Statement, + out: IrStmt[], + usingMarkers: UsingMarker[], + diagsBefore: number, + deferFences: boolean, +): void { + let lowered: IrStmt | IrStmt[] | null; + let appended = false; + if (ts.isVariableStatement(stmt) && (stmt.declarationList.flags & ts.NodeFlags.Using) !== 0) { + const registrations = lowerUsingStatement(lowerer, stmt); + const accounting: IrStmt[] = []; + for (const registration of registrations) { + out.push(...registration.acquire); + accounting.push(...registration.acquire, ...registration.cleanup); + if (registration.cleanup.length > 0) { + usingMarkers.push({ + start: out.length, + cleanup: registration.cleanup, + loc: registration.loc, + }); + } + } + lowered = accounting; + appended = true; + } else { + lowered = lowerer.lowerStmt(stmt); + } + // The lib-boundary chokepoint (lib-boundary.ts): checked-dynamic + // arguments that reached builtin-call slots without the checked + // coercion get their dynCheck here, and the uncoercible ones + // fence — inside this statement's poison window, so a JS fence + // becomes the statement's runtimeFence exactly like every other + // deferred rejection. + if (lowered) { + const body: IrStmt[] = []; + if (Array.isArray(lowered)) body.push(...lowered); + else body.push(lowered); + const rewritten = lowerSurplusCalls(lowerer, body); + enforceLibBoundary(lowerer, rewritten); + if (!appended) out.push(...rewritten); + } + // A JS statement that lowered CLEAN can still have flushed + // DEFERRED collection diagnostics (a symbol lookup on an alias + // resolves through flushDeferred — informational here, not a + // blocker of THIS statement): those belong to the runtime-fence + // ledger, not the build — real uses of the broken declaration + // meet their own per-site fences. + if (deferFences && lowerer.diagSink === null && lowerer.diags.length > diagsBefore) { + const flushed = lowerer.diags.splice(diagsBefore); + const ice = flushed.filter((d) => d.code === "SC9001"); + lowerer.diags.push(...ice); + lowerer.runtimeFences.push(...flushed.filter((d) => d.code !== "SC9001")); + } +} + /** Whether a statement list belongs to a function body (at any block * depth) rather than to a module's top level. */ function enclosedByFunction(stmt: ts.Statement): boolean { @@ -1302,6 +1372,7 @@ function jsDeclarationExpandsObject( inferred !== null && jsOpenObjectType(decl, inferred, lowerer.shapes, lowerer.unions).kind === "dyn") || arms.some((arm) => arm.kind === "record" && jsObjectBindingExpands(lowerer, decl, arm)) || + arms.some((arm) => arm.kind === "record" && jsObjectLiteralFilledLater(lowerer, decl, arm)) || (isJsSourceFile(ts.getSourceFileOfNode(decl)) && !decl.type && !hasJsTypeAnnotation(decl) && @@ -1522,7 +1593,11 @@ function noteEarlyReadVar(lowerer: Lowerer, symbol: ts.Symbol, local: IrLocal): * lowerVarDecl's salvage path declares the local when it can, and * resolution checks locals/globals before the fallthroughs consult * blockedBindings. */ -export function noteBlockedBindings(lowerer: Lowerer, stmt: ts.Statement): void { +export function noteBlockedBindings( + lowerer: Lowerer, + stmt: ts.Statement, + diags: readonly ScrDiagnostic[] = [], +): void { let list: ts.VariableDeclarationList | null = null; if (ts.isVariableStatement(stmt)) list = stmt.declarationList; else if ( @@ -1533,12 +1608,20 @@ export function noteBlockedBindings(lowerer: Lowerer, stmt: ts.Statement): void list = stmt.initializer; } if (!list) return; + // Localized failure (report-only): the bindings form one failure group, + // the statement, whose diagnostics are the group's. + const group = statementGroupKey(stmt); + let bound = false; for (const decl of list.declarations.nodes) { for (const nameNode of boundIdentifiersOf(decl.name)) { const symbol = lowerer.checker.getSymbolAtLocation(nameNode); - if (symbol) lowerer.blockedBindings.add(symbol); + if (!symbol) continue; + lowerer.blockedBindings.add(symbol); + if (!lowerer.rejectedGroups.has(symbol)) lowerer.rejectedGroups.set(symbol, group); + bound = true; } } + if (bound) for (const d of diags) d.rejects ??= group; } /** True when a reference/assignment fell through resolution because the @@ -1548,12 +1631,13 @@ export function noteBlockedBindings(lowerer: Lowerer, stmt: ts.Statement): void * flushed by the emit pass). */ export function isBlockedBinding(lowerer: Lowerer, symbol: ts.Symbol | null): boolean { if (!symbol) return false; - return ( + const blocked = lowerer.blockedBindings.has(symbol) || lowerer.deferredDiags.has(symbol) || lowerer.flushedSymbols.has(symbol) || - lowerer.alreadyFlushed.has(symbol) - ); + lowerer.alreadyFlushed.has(symbol); + if (blocked) lowerer.touchRejected(symbol); + return blocked; } /** Lowers a statement in a fresh lexical scope (if/while/for bodies). */ @@ -1878,7 +1962,16 @@ export function lowerStmt(lowerer: Lowerer, stmt: ts.Statement): IrStmt | IrStmt } // Exception-representable values move into the runtime's exception // cell. `throw f()` of a void call has no value. - let value = lowerer.lowerExpr(stmt.expression); + // A thrown object or array literal has no declared type: it is built as + // a dynamic object, so the catch binding's checked casts adopt it + // (lowerer.untypedLiteralContext). + let literal: ts.Expression = stmt.expression; + while (ts.isParenthesizedExpression(literal)) literal = literal.expression; + let value = + (ts.isObjectLiteralExpression(literal) || ts.isArrayLiteralExpression(literal)) && + lowerer.untypedLiteralContext(literal) + ? lowerer.lowerExprExpecting(stmt.expression, DYN) + : lowerer.lowerExpr(stmt.expression); if (value.type.kind === "void") lowerer.badType(stmt.expression, lowerer.typeOf(stmt.expression)); // Preserve primitive reasons and readable record identity across the @@ -3222,8 +3315,8 @@ export function lowerBindingPattern( // chain declares keep named fences (a detached method would lose its // receiver without an error; JS binds the prototype function). if (srcType.kind === "object") { - const info = lowerer.classes.get(srcType.className); - if (!info) lowerer.flushDeferredClass(srcType.className); + const info = + lowerer.classes.get(srcType.className) ?? lowerer.flushDeferredClass(srcType.className); if (info) { for (const el of pattern.elements.nodes) { if (el.name === undefined) continue; @@ -6515,6 +6608,41 @@ function lowerVariableInitializer( } } +/** A JS object literal that leaves out a required field of the binding's + * inferred record: tsc types `var p = {}; p.id = id;` by the members later + * assignments add, so the object only gains them after its initializer, and + * until then it is not a value of that record. Keep it a checked-dynamic + * object, as an object that gains keys outside the record is. */ +function jsObjectLiteralFilledLater( + lowerer: Lowerer, + decl: ts.VariableDeclaration, + type: IrType & { kind: "record" }, +): boolean { + if (decl.type || !decl.initializer || !isJsSourceFile(ts.getSourceFileOfNode(decl))) return false; + let initializer = decl.initializer; + while (ts.isParenthesizedExpression(initializer)) initializer = initializer.expression; + if (!ts.isObjectLiteralExpression(initializer) || hasJsTypeAnnotation(decl)) return false; + const shape = lowerer.shapes.get(type.shapeId); + if (!shape || shape.tuple) return false; + const given = new Set(); + for (const property of initializer.properties.nodes) { + if (ts.isSpreadAssignment(property)) return false; + const name = property.name; + if (name === undefined) return false; + if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNumericLiteral(name)) + given.add(name.text); + else return false; + } + // A field whose type holds undefined reads as it would before the write. + const holdsUndefined = (t: IrType): boolean => + t.kind === "dyn" || + t.kind === "undefinedT" || + (t.kind === "union" && lowerer.armTag(t.unionId, UNDEFINED_T) >= 0); + return shape.fields.some( + (field) => !given.has(field.name) && !field.name.startsWith("%") && !holdsUndefined(field.type), + ); +} + /** A JS object literal's inferred fields do not close its later storage. * Keep the original native object when writes add keys or delete fields; * converting it to a fixed record would discard that behavior. */ diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index c8699d856..2ae952aaa 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -2837,20 +2837,24 @@ export function streamClassAliasDecl( const PIPELINE_STAGE_EXPECTED = "of type function or an instance of Blob, ReadableStream, WritableStream, Stream, Iterable, AsyncIterable, or Promise or { readable, writable } pair"; -/** A checked-dynamic pipeline stage (an untyped source or destination): - * a boxed native stream unwraps at run time; other values (functions, - * pairs) throw the not-supported Error there. */ +/** A checked-dynamic pipeline stage taken as a stream: a boxed native + * stream unwraps at run time, and a value that is not stream-like or a + * function (an iterable after the first stage) becomes Duplex.from of it, + * as pipelineImpl's last branch does. Stream-like objects, web streams and + * { readable, writable } pairs throw the not-supported Error there, and so + * do functions, which reach this lowering only past + * MAX_DYN_FUNCTION_STAGES. */ function unboxDynStage(lowerer: Lowerer, v: IrExpr, node: ts.Expression): IrExpr { noteUntypedStreamFlow(lowerer, null); return { kind: "libCall", - fn: "stream.unboxDyn", + fn: "stream.unboxDynStage", args: [ v, strLit( lowerer.dynNodeStreams - ? "pipeline stages other than native streams (JavaScript stream objects, iterables, generators, plain functions, and { readable, writable } pairs) are not supported yet" - : "pipeline stages other than native streams (iterables, generators, plain functions, and { readable, writable } pairs) are not supported yet", + ? "pipeline stages other than native streams, iterables and functions (JavaScript stream objects, web streams, { readable, writable } pairs after the first stage, and functions after the first three untyped stages) are not supported yet" + : "pipeline stages other than native streams, iterables and functions (web streams, { readable, writable } pairs after the first stage, and functions after the first three untyped stages) are not supported yet", locOf(node), ), ], @@ -2930,6 +2934,102 @@ const BOXED_SOURCE_KINDS: ReadonlySet = new Set([ "string", ]); +/** The most checked-dynamic stages after the first that a pipeline + * branches on as functions; each one doubles the lowered pipeline calls. */ +const MAX_DYN_FUNCTION_STAGES = 3; + +/** A pipeline's arguments for one of the run-time branches + * lowerDynStagePipeline lowers. */ +interface DynStagePlan { + /** the first stage when it is checked-dynamic */ + source: DynPipelineSource | null; + /** argument values evaluated before the branches, in argument order */ + bound: ReadonlyMap; + /** a typed iterable first stage's source, lowered before the branches */ + preSource?: { closure: IrExpr; out: SyncGenType } | undefined; + /** the checked-dynamic stages this branch takes as functions */ + fns: ReadonlySet; +} + +/** pipeline() whose stages after the first include untyped values: + * pipelineImpl calls a stage that is a function with the previous stage + * and { signal }, and pipes into or takes any other value, so the call is + * lowered once per combination of those stages being functions or not, + * up to MAX_DYN_FUNCTION_STAGES of them, and a typeof test on each + * picks the branch at run time. The arguments other than function stages + * are evaluated first, in argument order. */ +function lowerDynStagePipeline( + lowerer: Lowerer, + stageArgs: readonly ts.Expression[], + src: DynPipelineSource | null, + loc: SrcLoc, + build: (plan: DynStagePlan) => IrExpr, +): IrExpr { + const candidates: number[] = []; + for (let j = 1; j < stageArgs.length; j++) { + const arg = stageArgs[j]!; + if (pipelineStageFunction(lowerer, arg) !== null || isValueStream(lowerer, arg)) continue; + if ((lowerer.typeOf(arg).flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) { + candidates.push(j); + } + } + if (candidates.length === 0) return build({ source: src, bound: new Map(), fns: new Set() }); + const stmts: IrStmt[] = []; + const bind = (init: IrExpr): IrExpr => { + const local = lowerer.declareHiddenLocal("%pipelineArg", init.type); + stmts.push({ kind: "varDecl", localId: local.id, init, loc: init.loc }); + return varRef(local.id, init.type, init.loc); + }; + const source = src !== null && !src.iterable ? { expr: bind(src.expr), iterable: false } : src; + const bound = new Map(); + let preSource: DynStagePlan["preSource"]; + for (let j = src !== null ? 1 : 0; j < stageArgs.length; j++) { + const arg = stageArgs[j]!; + if (pipelineStageFunction(lowerer, arg) !== null) continue; + if (j === 0 && iterableSourceType(lowerer, arg) !== null) { + const lowered = lowerIterableSource(lowerer, arg, null, loc); + if (lowered === null) lowerer.noLowering("pipeline over this source", arg); + preSource = { closure: bind(lowered!.closure), out: lowered!.out }; + continue; + } + let value = lowerer.lowerExpr(arg); + // an untyped array, string, Set, Map or generator object that keeps + // its static type is taken through its checked-dynamic value + if ( + candidates.includes(j) && + BOXED_SOURCE_KINDS.has(value.type.kind) && + lowerer.dynConvertible(value.type) + ) { + value = { kind: "dynFrom", value, type: DYN, loc: value.loc }; + } + bound.set(j, bind(value)); + } + const dynStages = candidates + .filter((j) => bound.get(j)!.type.kind === "dyn") + .slice(0, MAX_DYN_FUNCTION_STAGES); + const branch = (i: number, fns: ReadonlySet): IrExpr => { + if (i === dynStages.length) return build({ source, bound, preSource, fns }); + const j = dynStages[i]!; + const asStream = branch(i + 1, fns); + const asFunction = branch(i + 1, new Set([...fns, j])); + // a function destination's promise resolves to its value + const type = + asFunction.type.kind === "promise" && asFunction.type.inner.kind === "dyn" + ? asFunction.type + : asStream.type; + return { + kind: "ternary", + cond: { kind: "dynTest", test: "function", value: bound.get(j)!, type: BOOL, loc }, + then: lowerer.coerceToExpected(asFunction, type), + else_: lowerer.coerceToExpected(asStream, type), + type, + loc, + }; + }; + const result = branch(0, new Set()); + return { kind: "seqExpr", stmts, result, type: result.type, loc }; +} + /** lowerPipelineStages' source arguments for a checked-dynamic first stage. */ function stagesSource( src: DynPipelineSource | null, @@ -2938,6 +3038,22 @@ function stagesSource( return src.iterable ? [undefined, src.expr] : [new Map([[0, src.expr]]), undefined]; } +/** A pipeline stage taken as a stream under a DynStagePlan: the + * checked-dynamic first stage's stream, a value bound before the + * branches, or the argument lowered here. */ +function planStreamArg( + lowerer: Lowerer, + plan: DynStagePlan, + node: ts.Expression, + i: number, +): IrExpr { + if (i === 0 && plan.source !== null) return plan.source.expr; + const bound = plan.bound.get(i); + return bound !== undefined + ? streamArgOf(lowerer, node, bound, "pipeline") + : lowerStreamArg(lowerer, node, "pipeline"); +} + /** A stream-rooted argument, lowered — or the pointed fence (pipeline's * iterable/generator/function stages have no lowering yet). `what` picks * the validation ladder: finished() rejects EVERY non-stream through @@ -3304,7 +3420,9 @@ export function lowerStreamModuleCall( : null; const stageArgs = opts ? args.slice(0, -1) : args; const build = (src: DynPipelineSource | null): IrExpr => - lowerPromisePipeline(lowerer, stageArgs, opts, src, loc); + lowerDynStagePipeline(lowerer, stageArgs, src, loc, (plan) => + lowerPromisePipeline(lowerer, stageArgs, opts, plan, loc), + ); return lowerDynSourcePipeline(lowerer, stageArgs[0]!, loc, build) ?? build(null); } return null; @@ -3312,17 +3430,23 @@ export function lowerStreamModuleCall( return lowerStreamModuleCallbackCall(lowerer, call, bi, loc); } -/** stream/promises pipeline(...stages[, options]), with the first stage - * pre-lowered when it is checked-dynamic (lowerDynSourcePipeline). */ +/** stream/promises pipeline(...stages[, options]), for one branch of + * lowerDynStagePipeline's plan. */ function lowerPromisePipeline( lowerer: Lowerer, stageArgs: readonly ts.Expression[], opts: ReturnType | null, - src: DynPipelineSource | null, + plan: DynStagePlan, loc: SrcLoc, ): IrExpr { - if (src?.iterable === true || hasPipelineFunctionStage(lowerer, stageArgs)) { - const staged = lowerPipelineStages(lowerer, stageArgs, loc, ...stagesSource(src)); + const src = plan.source; + if ( + src?.iterable === true || + plan.fns.size > 0 || + plan.preSource !== undefined || + hasPipelineFunctionStage(lowerer, stageArgs) + ) { + const staged = lowerPipelineStages(lowerer, stageArgs, loc, ...stagesSource(src), plan); const type = staged.destPromise ?? { kind: "promise", inner: VOID }; // A promise destination's value settles the result (same type). if (!opts) { @@ -3346,9 +3470,7 @@ function lowerPromisePipeline( } let streams: IrExpr[]; try { - streams = stageArgs.map((a, i) => - i === 0 && src !== null ? src.expr : lowerStreamArg(lowerer, a, "pipeline"), - ); + streams = stageArgs.map((a, i) => planStreamArg(lowerer, plan, a, i)); } catch (e) { if (e instanceof StreamArgTypeThrow) return e.expr; throw e; @@ -3530,7 +3652,9 @@ function lowerStreamModuleCallbackCall( ); } const build = (src: DynPipelineSource | null): IrExpr => - lowerCallbackPipeline(lowerer, call, args, src, loc); + lowerDynStagePipeline(lowerer, args.slice(0, -1), src, loc, (plan) => + lowerCallbackPipeline(lowerer, call, args, plan, loc), + ); return lowerDynSourcePipeline(lowerer, args[0]!, loc, build) ?? build(null); } @@ -3544,23 +3668,32 @@ function lowerStreamModuleCallbackCall( return null; } -/** pipeline(...stages, callback), with the first stage pre-lowered when it - * is checked-dynamic (lowerDynSourcePipeline). */ +/** pipeline(...stages, callback), for one branch of lowerDynStagePipeline's + * plan. */ function lowerCallbackPipeline( lowerer: Lowerer, call: ts.CallExpression, args: readonly ts.Expression[], - src: DynPipelineSource | null, + plan: DynStagePlan, loc: SrcLoc, ): IrExpr { - if (src?.iterable === true || hasPipelineFunctionStage(lowerer, args.slice(0, -1))) { - const staged = lowerPipelineStages(lowerer, args.slice(0, -1), loc, ...stagesSource(src)); - const cbNode = args[args.length - 1]!; - const cbParams = - ts.isArrowFunction(cbNode) || ts.isFunctionExpression(cbNode) - ? cbNode.parameters.nodes.length - : 0; - if (staged.dest === 2 || (staged.dest === 1 && cbParams > 1)) { + const src = plan.source; + const cbNode = args[args.length - 1]!; + const cbParams = + ts.isArrowFunction(cbNode) || ts.isFunctionExpression(cbNode) + ? cbNode.parameters.nodes.length + : 0; + // A branch over untyped stages whose destination is a stream: the + // callback still receives (err, value), value undefined, as in Node. + const dynCallback = plan.bound.size > 0 && cbParams > 1; + if ( + src?.iterable === true || + plan.fns.size > 0 || + plan.preSource !== undefined || + hasPipelineFunctionStage(lowerer, args.slice(0, -1)) + ) { + const staged = lowerPipelineStages(lowerer, args.slice(0, -1), loc, ...stagesSource(src), plan); + if (staged.dest >= 2 || (staged.dest === 1 && cbParams > 1) || dynCallback) { // callback(err, value): the destination's value (undefined for // an async generator) reaches the callback through the // checked-dynamic path. @@ -3595,15 +3728,15 @@ function lowerCallbackPipeline( } let streams: IrExpr[]; try { - streams = args - .slice(0, -1) - .map((a, i) => (i === 0 && src !== null ? src.expr : lowerStreamArg(lowerer, a, "pipeline"))); + streams = args.slice(0, -1).map((a, i) => planStreamArg(lowerer, plan, a, i)); } catch (e) { if (e instanceof StreamArgTypeThrow) return e.expr; throw e; } const last = streams[streams.length - 1]!; - const { cb, dyn } = lowerEosCallback(lowerer, "pipeline", args[args.length - 1]!, last.type); + const { cb, dyn } = dynCallback + ? { cb: lowerer.lowerExprExpecting(cbNode, DYN), dyn: true } + : lowerEosCallback(lowerer, "pipeline", cbNode, last.type); return { kind: "libCall", fn: dyn ? "stream.pipelineDyn" : "stream.pipeline", @@ -4213,6 +4346,397 @@ function lowerDynIterableSource( }; } +/** The async generator a checked-dynamic stage's iterable is pumped + * through. */ +function dynStageGenType(): SyncGenType { + return { kind: "generator", async: true, yieldT: DYN, retT: VOID, nextT: UNDEFINED_T }; +} + +/** A checked-dynamic pipeline stage after the first that is a function at + * run time, as the stage closure the runtime calls with the previous + * stage and { signal } (pipelineImpl's `typeof stream === 'function'` + * branch): it calls the function with the previous stream or generator + * as an untyped value. Before the last stage the result must be async + * iterable (ERR_INVALID_RETURN_VALUE otherwise) and is pumped on through + * lowerDynIterableSource's generator; the last stage answers the result, + * which the runtime classifies (SCR_PIPE_DEST_DYN). A previous stage + * whose type has no untyped form throws the not-supported Error. */ +function lowerDynFunctionStage( + lowerer: Lowerer, + fnValue: IrExpr, + srcT: IrType, + j: number, + isLast: boolean, + loc: SrcLoc, + handoff?: DynStageHandoff, +): { closure: IrExpr; out: IrType } { + const out: IrType = isLast ? DYN : dynStageGenType(); + const stageT = funcOf([srcT, DYN], out); + const take = handoff?.take === true; + const give = handoff?.give === true && !isLast; + const key = `dynFunctionStage:${isLast ? "dest" : "transform"}:${typeKey(srcT)}:${take ? 1 : 0}${give ? 1 : 0}`; + let maker = lowerer.valueHelpers.get(key); + if (!maker) { + maker = `%stream.dynFunctionStage.${lowerer.valueHelpers.size}`; + lowerer.valueHelpers.set(key, maker); + const stageName = `${maker}.stage`; + const frame = new LiftedFrame(loc); + const fnId = frame.local("fn", DYN, false); + const labelId = frame.local("label", STRING, false); + const cellId = frame.local("cell", DYN, false); + const srcId = frame.param("src", srcT); + const optsId = frame.param("opts", DYN); + const inputId = frame.local("input", DYN, false); + const resultId = frame.local("result", DYN, false); + const nextId = frame.local("next", DYN, false); + const cell = (): IrExpr => varRef(cellId, DYN, loc); + const body: IrStmt[] = []; + if (take) { + // the previous untyped stage's value, handed over as it is + // (makeAsyncIterable answers an iterable itself) + body.push( + { kind: "varDecl", localId: inputId, init: handoffTake(cell(), loc), loc }, + handoffPut(cell(), dynUndefinedExpr(loc), loc), + ); + } + if (!take && !lowerer.dynConvertible(srcT)) { + body.push({ + kind: "return", + value: nodeThrowExpr( + 0, + "SC2020", + `pipeline function stages after a stage of type '${lowerer.fmt(srcT)}' are not supported yet`, + out, + loc, + ), + loc, + }); + } else { + body.push({ + kind: "varDecl", + localId: resultId, + init: { + kind: "dynCall", + callee: varRef(fnId, DYN, loc), + receiver: dynUndefinedExpr(loc), + calleeName: "transform", + args: [ + take + ? varRef(inputId, DYN, loc) + : { kind: "dynFrom", value: varRef(srcId, srcT, loc), type: DYN, loc }, + varRef(optsId, DYN, loc), + ], + type: DYN, + loc, + }, + loc, + }); + if (isLast) { + body.push({ kind: "return", value: varRef(resultId, DYN, loc), loc }); + } else { + body.push({ + kind: "varDecl", + localId: nextId, + init: { + kind: "libCall", + fn: "stream.dynTransformResult", + args: [varRef(resultId, DYN, loc), varRef(labelId, STRING, loc)], + type: DYN, + loc, + }, + loc, + }); + // the next untyped function stage takes the result itself; the + // generator returned for the runtime is then never read + if (give) body.push(handoffPut(cell(), varRef(nextId, DYN, loc), loc)); + const iterable = lowerDynIterableSource(lowerer, varRef(nextId, DYN, loc), loc); + if (!iterable) + throw new InternalCompilerError("dyn iterable generator without a result record"); + body.push({ + kind: "return", + value: { kind: "callValue", callee: iterable.closure, args: [], type: out, loc }, + loc, + }); + } + } + const captured = take || give ? [fnId, labelId, cellId] : [fnId, labelId]; + for (const id of captured) frame.locals.find((l) => l.id === id)!.boxed = true; + lowerer.liftedFns.push({ + name: stageName, + params: frame.params, + returnType: out, + captures: [ + { localId: fnId, name: "fn", type: DYN }, + { localId: labelId, name: "label", type: STRING }, + ...(take || give ? [{ localId: cellId, name: "cell", type: DYN }] : []), + ], + locals: frame.locals, + body, + loc, + }); + const makerFn: IrLocal = { id: "fn.0", name: "fn", type: DYN, mutable: false, boxed: true }; + const makerLabel: IrLocal = { + id: "label.1", + name: "label", + type: STRING, + mutable: false, + boxed: true, + }; + const makerCell: IrLocal = { + id: "cell.2", + name: "cell", + type: DYN, + mutable: false, + boxed: true, + }; + const makerLocals = take || give ? [makerFn, makerLabel, makerCell] : [makerFn, makerLabel]; + lowerer.liftedFns.push({ + name: maker, + params: makerLocals.map((l) => ({ localId: l.id, name: l.name, type: l.type })), + returnType: stageT, + locals: makerLocals, + body: [ + { + kind: "return", + value: { + kind: "closure", + fnName: stageName, + captures: makerLocals.map((l) => l.id), + type: stageT, + loc, + }, + loc, + }, + ], + loc, + }); + } + return { + closure: { + kind: "call", + callee: maker, + args: [fnValue, strLit(`transform[${j - 1}]`, loc), ...(take || give ? [handoff!.cell] : [])], + type: stageT, + loc, + }, + out, + }; +} + +/** The cell that carries an untyped stage's value to the untyped function + * stage after it, created per pipeline call. A stage that gives stores its + * value; the next stage takes it instead of the generator the runtime + * passes, as Node's makeAsyncIterable answers an iterable itself. */ +interface DynStageHandoff { + cell: IrExpr; + take: boolean; + give: boolean; +} + +function handoffPut(cell: IrExpr, value: IrExpr, loc: SrcLoc): IrStmt { + return { + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "dyn.keySet", + args: [cell, strLit("v", loc), value], + type: VOID, + loc, + }, + loc, + }; +} + +function handoffTake(cell: IrExpr, loc: SrcLoc): IrExpr { + return { kind: "dynKeyGet", value: cell, key: strLit("v", loc), type: DYN, loc }; +} + +/** `(value) => generator over value`: the async generator an untyped + * destination function's async iterable result is pumped through + * (SCR_PIPE_DEST_DYN's box), interned. */ +function dynStageIterHelper(lowerer: Lowerer, loc: SrcLoc): IrExpr { + const genT = dynStageGenType(); + const type = funcOf([DYN], genT); + const key = "dynStageIter"; + let name = lowerer.valueHelpers.get(key); + if (!name) { + name = `%stream.dynStageIter.${lowerer.valueHelpers.size}`; + lowerer.valueHelpers.set(key, name); + const v: IrLocal = { id: "value.0", name: "value", type: DYN, mutable: false }; + const iterable = lowerDynIterableSource(lowerer, varRef(v.id, DYN, loc), loc); + if (!iterable) + throw new InternalCompilerError("dyn iterable generator without a result record"); + lowerer.liftedFns.push({ + name, + params: [{ localId: v.id, name: v.name, type: DYN }], + returnType: genT, + locals: [v], + body: [ + { + kind: "return", + value: { kind: "callValue", callee: iterable.closure, args: [], type: genT, loc }, + loc, + }, + ], + loc, + }); + } + return { kind: "closure", fnName: name, captures: [], type, loc }; +} + +/** A checked-dynamic first pipeline stage pumped as an iterable + * (pipelineImpl's i === 0 branch), as the stage closure the runtime calls + * with { signal } while it sets the pipeline up: a function is called + * with { signal } and its result must be iterable, a { readable, writable } + * pair pumps its readable side, and any other value is pumped itself, + * through lowerDynIterableSource's generator. */ +function lowerDynSourceStage( + lowerer: Lowerer, + value: IrExpr, + loc: SrcLoc, + cell?: IrExpr, +): { closure: IrExpr; out: SyncGenType } | null { + const out: SyncGenType = { + kind: "generator", + async: true, + yieldT: DYN, + retT: VOID, + nextT: UNDEFINED_T, + }; + const stageT = funcOf([DYN], out); + const give = cell !== undefined; + const key = give ? "dynSourceStage:give" : "dynSourceStage"; + let maker = lowerer.valueHelpers.get(key); + if (!maker) { + maker = `%stream.dynSourceStage.${lowerer.valueHelpers.size}`; + const stageName = `${maker}.stage`; + const frame = new LiftedFrame(loc); + const srcId = frame.local("src", DYN, false); + const cellId = frame.local("cell", DYN, false); + const optsId = frame.param("opts", DYN); + const valueId = frame.local("value", DYN, false); + const genId = frame.local("gen", out, false); + const src = (): IrExpr => varRef(srcId, DYN, loc); + const sourceValue = (v: IrExpr, returned: boolean): IrExpr => ({ + kind: "libCall", + fn: "stream.dynSourceValue", + args: [v, boolLit(returned, loc)], + type: DYN, + loc, + }); + const iterable = lowerDynIterableSource(lowerer, varRef(valueId, DYN, loc), loc); + if (!iterable) return null; + lowerer.valueHelpers.set(key, maker); + const body: IrStmt[] = [ + { + kind: "varDecl", + localId: valueId, + init: { + kind: "ternary", + cond: { kind: "dynTest", test: "function", value: src(), type: BOOL, loc }, + then: sourceValue( + { + kind: "dynCall", + callee: src(), + receiver: dynUndefinedExpr(loc), + calleeName: "source", + args: [varRef(optsId, DYN, loc)], + type: DYN, + loc, + }, + true, + ), + else_: sourceValue(src(), false), + type: DYN, + loc, + }, + loc, + }, + { + kind: "varDecl", + localId: genId, + init: { kind: "callValue", callee: iterable.closure, args: [], type: out, loc }, + loc, + }, + ]; + if (give) { + // the untyped function stage after the source takes the iterable + // itself, as Node's makeAsyncIterable answers it; a stream (a pair's + // readable side) goes as the generator over it, as Node iterates a + // stream there + const v = varRef(valueId, DYN, loc); + body.push( + handoffPut( + varRef(cellId, DYN, loc), + { + kind: "ternary", + cond: { kind: "libCall", fn: "stream.dynIterableSource", args: [v], type: BOOL, loc }, + then: v, + else_: { kind: "dynFrom", value: varRef(genId, out, loc), type: DYN, loc }, + type: DYN, + loc, + }, + loc, + ), + ); + } + body.push({ kind: "return", value: varRef(genId, out, loc), loc }); + const captured = give ? [srcId, cellId] : [srcId]; + for (const id of captured) frame.locals.find((l) => l.id === id)!.boxed = true; + lowerer.liftedFns.push({ + name: stageName, + params: frame.params, + returnType: out, + captures: [ + { localId: srcId, name: "src", type: DYN }, + ...(give ? [{ localId: cellId, name: "cell", type: DYN }] : []), + ], + locals: frame.locals, + body, + loc, + }); + const makerSrc: IrLocal = { id: "src.0", name: "src", type: DYN, mutable: false, boxed: true }; + const makerCell: IrLocal = { + id: "cell.1", + name: "cell", + type: DYN, + mutable: false, + boxed: true, + }; + const makerLocals = give ? [makerSrc, makerCell] : [makerSrc]; + lowerer.liftedFns.push({ + name: maker, + params: makerLocals.map((l) => ({ localId: l.id, name: l.name, type: l.type })), + returnType: stageT, + locals: makerLocals, + body: [ + { + kind: "return", + value: { + kind: "closure", + fnName: stageName, + captures: makerLocals.map((l) => l.id), + type: stageT, + loc, + }, + loc, + }, + ], + loc, + }); + } + return { + closure: { + kind: "call", + callee: maker, + args: give ? [value, cell!] : [value], + type: stageT, + loc, + }, + out, + }; +} + function lowerIterableSource( lowerer: Lowerer, node: ts.Expression, @@ -4356,7 +4880,7 @@ function lowerIterableSource( interface PipelineStagesLowering { args: IrExpr[]; last: IrType; - dest: 0 | 1 | 2; + dest: 0 | 1 | 2 | 3; destPromise: (IrType & { kind: "promise" }) | null; wrap: (call: IrExpr) => IrExpr; } @@ -4377,6 +4901,7 @@ function lowerPipelineStages( loc: SrcLoc, preLowered?: ReadonlyMap, dynSource?: IrExpr, + plan?: DynStagePlan, ): PipelineStagesLowering { if (stageArgs.length > 52) { lowerer.noLowering( @@ -4390,7 +4915,7 @@ function lowerPipelineStages( const helperArgs: IrExpr[] = []; let positions = 0; let mask = 0; - let dest: 0 | 1 | 2 = 0; + let dest: 0 | 1 | 2 | 3 = 0; let destPromise: (IrType & { kind: "promise" }) | null = null; // What the next function stage receives: null before the first stage, // else the previous stream's type or the previous stage's generator. @@ -4399,17 +4924,69 @@ function lowerPipelineStages( // before all stages, so when a stage precedes a stream the arguments // are bound to locals first, keeping JavaScript's evaluation order. const argOrder: { list: IrExpr[]; index: number }[] = []; + // An untyped stage followed by an untyped function stage hands its value + // over through one cell per call (DynStageHandoff). + let handoffCell: IrLocal | null = null; + const cellRef = (): IrExpr => { + handoffCell ??= lowerer.declareHiddenLocal("%pipelineHandoff", DYN); + return varRef(handoffCell.id, DYN, loc); + }; + let takes = false; for (let j = 0; j < stageArgs.length; j++) { const arg = stageArgs[j]!; + const givesToNext = plan?.fns.has(j + 1) === true; if (j === 0 && dynSource !== undefined) { - const source = lowerDynIterableSource(lowerer, dynSource, loc); + const source = lowerDynSourceStage( + lowerer, + dynSource, + loc, + givesToNext ? cellRef() : undefined, + ); + takes = givesToNext; if (source === null) lowerer.noLowering("pipeline over this source", arg); + // a source function takes { signal } + mask |= 1 << stages.length; positions += 1; argOrder.push({ list: stages, index: stages.length }); stages.push(source!.closure); srcT = source!.out; continue; } + if (plan?.fns.has(j) === true) { + // a checked-dynamic stage that is a function at run time + const isLast = j === stageArgs.length - 1; + const take = takes; + const give = givesToNext && !isLast; + takes = give; + const lowered = lowerDynFunctionStage( + lowerer, + plan.bound.get(j)!, + srcT!, + j, + isLast, + loc, + take || give ? { cell: cellRef(), take, give } : undefined, + ); + mask |= 1 << stages.length; + positions += 2 ** j; + argOrder.push({ list: stages, index: stages.length }); + stages.push(lowered.closure); + if (isLast) { + // its result is classified at run time: a promise settles the + // pipeline's value, and an async iterable is pumped into the + // PassThrough + const genT = dynStageGenType(); + const helpers = fromGenHelpers(lowerer, genT, loc); + if (!helpers) lowerer.noLowering("a pipeline destination with this generator type", arg); + helperArgs.push(helpers![0]!, helpers![1]!, helpers![2]!); + dest = 3; + destPromise = { kind: "promise", inner: DYN }; + srcT = null; + } else { + srcT = lowered.out; + } + continue; + } // compose's head and tail functions arrive as Duplex.from streams const fn = preLowered?.has(j) ? null : pipelineStageFunction(lowerer, arg); const syncSourceFn = @@ -4423,7 +5000,9 @@ function lowerPipelineStages( !preLowered?.has(j) && (syncSourceFn || (fn === null && iterableSourceType(lowerer, arg) !== null)) ) { - const source = lowerIterableSource(lowerer, arg, syncSourceFn ? fn : null, loc); + const source = + (syncSourceFn ? undefined : plan?.preSource) ?? + lowerIterableSource(lowerer, arg, syncSourceFn ? fn : null, loc); if (source === null) lowerer.noLowering("pipeline over this source", arg); positions += 1; argOrder.push({ list: stages, index: stages.length }); @@ -4448,7 +5027,12 @@ function lowerPipelineStages( } let stream: IrExpr; try { - stream = preLowered?.get(j) ?? lowerStreamArg(lowerer, arg, "pipeline"); + const bound = plan?.bound.get(j); + stream = + preLowered?.get(j) ?? + (bound !== undefined + ? streamArgOf(lowerer, arg, bound, "pipeline") + : lowerStreamArg(lowerer, arg, "pipeline")); } catch (e) { if (e instanceof StreamArgTypeThrow) lowerer.noLowering("pipeline over this value", arg); throw e; @@ -4527,6 +5111,8 @@ function lowerPipelineStages( ); } helperArgs.push(pipelineBoxHelper(lowerer, destPromise!, loc)); + } else if (dest === 3) { + helperArgs.push(dynStageIterHelper(lowerer, loc)); } const argDecls: IrStmt[] = []; const firstStage = argOrder.findIndex((a) => a.list === stages); @@ -4538,6 +5124,15 @@ function lowerPipelineStages( a.list[a.index] = varRef(local.id, init.type, init.loc); } } + if (handoffCell !== null) { + const cell: IrLocal = handoffCell; + argDecls.unshift({ + kind: "varDecl", + localId: cell.id, + init: { kind: "dynObjLit", fields: [], type: DYN, loc }, + loc, + }); + } const bindArgs = (call: IrExpr): IrExpr => argDecls.length === 0 ? call diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 56f899525..a75efc001 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -1,9 +1,17 @@ +import { unionNarrowFailureMessage } from "./union-narrow.js"; +import { + compactDeclOrigins, + inPackageFile, + recordOrigin, + slotOrigin, + type RecordOrigin, +} from "./decl-origin.js"; import { isDeclBoundaryFile } from "../decl-projection.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; import { moduleNamespaceObject, moduleNamespaceValue } from "./module-namespace-elements.js"; import { keyedHelperName } from "./stable-names.js"; import { positionTag, StructuralPositions } from "./structural-positions.js"; -import { onlyForwarded, stripTypeWrappers } from "./typed-arguments.js"; +import { bindingSymbolOfRead, onlyForwarded, stripTypeWrappers } from "./typed-arguments.js"; import { needsGeneralNodeStreams, noteUntypedStreamFlow, @@ -119,6 +127,7 @@ import { deferredReadHelper, widthNarrowHelper, } from "./coercions/unions.js"; +import { assertedConversion } from "./coercions/asserted.js"; import { lowerUtilTypeValue, pathModuleValue, @@ -128,7 +137,7 @@ import { devirtualizeVirtualCalls } from "./devirtualize.js"; import { foldRegexSpanReads } from "./regex-span-reads.js"; import { RuntimeOptionalProvenance } from "./runtime-optional-provenance.js"; import { presentAtUse, reassignedBindings } from "./runtime-optional-guards.js"; -import { NarrowingStability } from "./narrowing-stability.js"; +import { NarrowingStability, widenedReceiver } from "./narrowing-stability.js"; import { ArrayElementStates } from "./runtime-optional-elements.js"; import { indexReadInBounds } from "./runtime-optional-bounds.js"; import { ArrayOwnership } from "./runtime-optional-ownership.js"; @@ -204,7 +213,14 @@ import { isBundledShimForwarder } from "../package-shims.js"; import { erasedTypeOnlyImport } from "../type-only-imports.js"; import { literalValues } from "../literal-values.js"; import { discriminantOwners, literalUnionArm, type UnionLiteral } from "../union-discriminants.js"; -import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; +import type { + ScrDiagnostic, + TypeFailure, + TypeFailureFamily, + TypeFailureSlot, +} from "../../diagnostics/diagnostic.js"; +import { declarationGroupKey, typeFailureOf } from "./type-failure.js"; +import { dynamicSlotDiag, type DynamicSlotEntry } from "../../diagnostics/dynamic-slots.js"; import { unsupportedAnyOperationDiag, blockedBindingUseDiag, @@ -460,6 +476,7 @@ import { genericLocalClassBinding, } from "./generic-functions.js"; import { bindingNeverReassigned } from "./binding-analysis.js"; +import { isMessageListenerCall } from "./expressions/message-listeners.js"; import type { InstantiationPath } from "./instantiation-path.js"; import { bodyReadsArguments } from "../arguments-usage.js"; import { completeArgs, wrappedUndefined, undefinedArgFor } from "./call-arguments.js"; @@ -713,7 +730,42 @@ interface GenericDemandOwner { classDemands: ClassInfo[]; } -export class PoisonError extends Error {} +/** A slot the localized-failure commit points turned into a checked + * dynamic slot (SC6104): where it is, what its declared type is, and the + * diagnostic it would have failed with. */ +export interface DynamicSlot { + loc: SrcLoc; + failure: TypeFailure; + code: string; + reason: string; +} + +/** Localized failure: rounds of the declaration flush retry. */ +const MAX_DECLARATION_RETRIES = 16; + +/** The slot kinds the declaration flush retry degrades (stage L2). */ +const DECLARATION_SLOTS: ReadonlySet = new Set([ + "field", + "param", + "return", + "local", + "global", + "loop", +]); + +export class PoisonError extends Error { + /** Set when a type with no static lowering poisoned a slot (badType and + * the dyn class field policy): what failed, for the localized-failure + * commit points. Report-only in stage L0. */ + readonly failure: TypeFailure | undefined; + /** The checker type that failed (with `failure`). */ + readonly failedType: ts.Type | undefined; + constructor(failure?: TypeFailure, failedType?: ts.Type) { + super(); + this.failure = failure; + this.failedType = failedType; + } +} /** A function-shaped value that exists only in the compiler's static * projection graph. Declared-function aliases point at the original @@ -828,6 +880,9 @@ export interface LowerResult { /** Performance warnings (SC61xx): the likely-hot functions the source * leaves untyped (lowering/dynamic-tier.ts). Never failures. */ performance?: ScrDiagnostic[]; + /** Localized failure: the checked dynamic slots the commit points + * created (each also an SC6104 in `performance`), present when any. */ + dynamicSlots?: DynamicSlotEntry[]; /** Functions lowering placed in the dynamic tier (ir/tiers.ts), present * when there are any. */ dynamicFunctions?: DynamicFunction[]; @@ -939,6 +994,13 @@ export interface LowererMode { /** Symbols whose deferred diagnostics reachable emit already flushed — * the remainder must not report them a second time. */ alreadyFlushed?: ReadonlySet; + /** Localized failure: the types reachable emit degraded (the coverage + * remainder maps them alike, so the declarations reachable emit + * registered with dynamic slots collect there too). */ + degradedTypes?: ReadonlyMap; + /** Localized failure: the declarations reachable emit registered with + * dynamic slots (the coverage remainder registers them too). */ + retriedSymbols?: ReadonlySet; /** LowerOptions.externalTypes, threaded through every lowering pass. */ externalTypes?: ReadonlyMap; /** LowerOptions.externalTypeSpecifiersByFile, shared by every pass. */ @@ -1243,6 +1305,22 @@ export function lowerToIr( timing("dynamic-tier-scan", { found: hotDynamic === null ? 0 : 1 }); if (hotDynamic !== null) result = { ...result, performance: [hotDynamic] }; } + // Localized failure (SC6104): one performance warning per dynamic slot. + if (resultLowerer.dynamicSlots.length > 0) { + const dynamicSlots: DynamicSlotEntry[] = resultLowerer.dynamicSlots.map((slot) => ({ + loc: slot.loc, + name: slot.failure.name, + slot: slot.failure.slot, + declaredType: slot.failure.declaredType, + code: slot.code, + reason: slot.reason, + })); + result = { + ...result, + performance: [...(result.performance ?? []), ...dynamicSlots.map(dynamicSlotDiag)], + dynamicSlots, + }; + } // Contract types (@scriptc/types): written annotations, checked against // the IR by the compile pipeline (contracts/check.ts). // Acyclic object types name the IR unit the program lowered them to; a @@ -1330,6 +1408,8 @@ export function lowerToIr( reachable, remainder: true, alreadyFlushed: resultLowerer.flushedSymbols, + degradedTypes: resultLowerer.degradedTypes, + retriedSymbols: resultLowerer.retriedSymbols, inlineRequireCycleCalls, targetPlatform, ffiImports, @@ -2248,6 +2328,10 @@ export class Lowerer { ts.Type, { tsBindings: Map; instance: ts.Type }[] >(); + /** Values an erased type assertion produced, with the asserted type + * (`x as T` keeps x's representation): a slot that no ordinary + * conversion reaches converts them as asserted (assertedConversion). */ + readonly assertedValues = new Map(); /** Non-null while an IMPLICIT-ANY instance body lowers (npm-static JS; * see generic-functions.ts): bound param symbol → * the call site's checker type, consulted by typeOf for identifier @@ -2477,6 +2561,69 @@ export class Lowerer { return fallback ?? null; } + /** An object or array literal in an untyped context: its contextual type + * is `unknown` or `any` from a slot the program declares (a variable, + * field, parameter, return type or `unknown[]` element; a library + * parameter typed `any` only through a type argument), it is thrown, or + * it is a member of such a literal, and it holds data only (no methods, + * accessors or function members). Its value is built as a dynamic + * object: no type the program wrote gives it a native layout, and a later + * checked cast adopts it (dyn.adoptShared), so every holder keeps one + * identity. A library parameter the declarations type `any` or `unknown` + * itself (`console.log`, `JSON.stringify`) only reads the value and keeps + * the native literal. */ + untypedLiteralContext(x: ts.Expression): boolean { + if (isJsSourceFile(ts.getSourceFileOfNode(x))) return false; + // Data only: a literal with methods, accessors or function members is an + // object with behavior, which keeps its native layout (and boxed form). + if ( + ts.isObjectLiteralExpression(x) && + x.properties.nodes.some((p) => { + if (!ts.isPropertyAssignment(p)) + return !ts.isShorthandPropertyAssignment(p) && !ts.isSpreadAssignment(p); + let init: ts.Expression = p.initializer; + while (ts.isParenthesizedExpression(init)) init = init.expression; + return ( + ts.isFunctionExpression(init) || ts.isArrowFunction(init) || ts.isClassExpression(init) + ); + }) + ) + return false; + let node: ts.Node = x; + while (node.parent !== undefined && ts.isParenthesizedExpression(node.parent)) + node = node.parent; + const parent = node.parent; + if (parent === undefined) return false; + if (ts.isThrowStatement(parent)) return true; + if (ts.isPropertyAssignment(parent) && parent.initializer === node) { + const literal = parent.parent; + if (ts.isObjectLiteralExpression(literal)) return this.untypedLiteralContext(literal); + } + if (ts.isArrayLiteralExpression(parent) && this.untypedLiteralContext(parent)) return true; + const contextual = this.checker.getContextualType(x); + if (!contextual || !(contextual.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown))) + return false; + if (ts.isCallExpression(parent) || ts.isNewExpression(parent)) { + const signature = this.checker.getResolvedSignature(parent); + const decl = signature?.declaration; + if (decl && this.isStdlibFile(ts.getSourceFileOfNode(decl))) { + const index = (parent.arguments?.nodes ?? []).indexOf(node as ts.Expression); + const params = (decl as { parameters?: { nodes: readonly ts.ParameterDeclaration[] } }) + .parameters?.nodes; + const param = params?.[Math.min(index, params.length - 1)]; + let annotation = param?.type; + if (annotation && ts.isArrayTypeNode(annotation)) annotation = annotation.elementType; + if ( + !annotation || + annotation.kind === ts.SyntaxKind.AnyKeyword || + annotation.kind === ts.SyntaxKind.UnknownKeyword + ) + return false; + } + } + return true; + } + /** A fresh TypeScript data literal (`{ message: "m" }`) passed to a * checked-dynamic parameter of a callee implemented in JavaScript, whose * contract lets it reshape the object: a call through an untyped value, an @@ -3696,6 +3843,53 @@ export class Lowerer { * the fallthroughs report the inherited-blocker cascade (SC2004) * instead of misattributing the reference. */ readonly blockedBindings = new Set(); + /** Localized failure, report-only: the failure group (type-failure.ts) + * of each declaration whose rejection reached the build — a flushed + * deferred declaration, or a binding of a poisoned declaration + * statement. */ + readonly rejectedGroups = new Map(); + /** One frame per statement lowerStmts is lowering: the failure groups + * its lowering touched. A statement that then poisons attributes its + * diagnostics to them (ScrDiagnostic.cause). */ + readonly failureTouches: Set[] = []; + /** Localized failure: checker types (by id) whose strict mapping failed + * at a committed slot, with their lenient mapping. Filled only at the + * commit points (degradeAtCommit), so a program that compiles today + * never fills it. */ + readonly degradedTypes = new Map(); + /** The dynamic slots the commit points created: one SC6104 each. */ + readonly dynamicSlots: DynamicSlot[] = []; + /** Declarations retryDeferred registered. */ + readonly retriedSymbols = new Set(); + /** LowererMode.retriedSymbols. */ + readonly priorRetriedSymbols: ReadonlySet; + /** Bodies run() skipped because their declaration had not registered. */ + readonly skippedBodies = new Set(); + /** Declarations retryDeferred registered, in order: run() lowers the + * bodies it had skipped. */ + readonly retriedDeclarations: ts.Node[] = []; + /** Diagnostics of lowering that a statement's retry does not repeat (a + * function-local class's member bodies, lowered once and kept): + * dropAttemptDiags keeps them. */ + readonly loweredOnceDiags = new Set(); + /** Deferred classes whose static field initializers' place in %init + * has been lowered (retryDeferredAtDeclaration): a later flush must not + * register them, since their initializers would never run. */ + readonly staticInitsPassed = new Set(); + /** Deferred declarations' collections, for the flush retry + * (retryDeferred). */ + readonly deferredCollectors = new Map< + ts.Symbol, + { symbolOf: () => ts.Symbol | undefined; collect: () => void } + >(); + /** True while a flush retry collects a declaration: the dyn class + * field policy (dynFieldPolicyFence) is lifted for reached classes. */ + liftDynFieldPolicy = false; + /** Collection diagnostics that a flush retry may simply run again (a + * derived class whose base had not registered yet). */ + readonly retriableDiags = new Set(); + /** The checker type behind each type failure badType committed. */ + readonly failedTypes = new Map(); /** True while collectProgram runs: resolution helpers must not flush * deferred diagnostics (collection itself resolves symbols — extends * clauses — and collection order must not decide what reports). */ @@ -3749,18 +3943,88 @@ export class Lowerer { /** Bumps the per-file attribution counter (no-op unless provenance is * active and this is the emit/discovery lane — mirror the CALLER's * suppressStats guard, this method only gates remainder). */ - bumpFileStat(file: string, kind: "total" | "failed"): void { + bumpFileStat(file: string, kind: "total" | "failed", by = 1): void { if (this.remainder || !provenanceActive()) return; let s = this.statsByFile.get(file); if (!s) this.statsByFile.set(file, (s = { total: 0, failed: 0 })); - if (kind === "total") s.total += 1; - else if (kind === "failed") s.failed += 1; + if (kind === "total") s.total += by; + else if (kind === "failed") s.failed += by; } /** The program holds the declaration-typed boundary's projection * (decl-projection.ts): user-file record and array values crossing into * dynamic slots (package parameters) box as live capsules, so a value * that comes back through a boundary cast keeps its identity. */ + /** Origins of package-declared types checked in this program + * (decl-origin.ts), by IrModule.declOrigins index. */ + readonly declOrigins: string[] = []; + private readonly declOriginIds = new Map(); + private readonly packageFileMemo = new Map(); + private readonly slotOriginMemo = new Map(); + /** The value whose slot coercion is being lowered, while that value is + * in a package file: union extractions built for it carry the slot's + * origin in their TypeError. */ + coercionOriginNode: ts.Node | null = null; + /** Per user file, each declaration-boundary cast's index in source + * order (narrowing.ts recordedBoundaryOrigin). */ + readonly boundaryCastOrder = new Map>(); + /** The union type of that value: only its own extraction is the slot's. */ + coercionOriginUnion: string | null = null; + /** While a record projection into a package-declared type builds, the + * origin of the field being converted. */ + coercionFieldOrigin: string | null = null; + private readonly recordOriginMemo = new Map(); + + /** The package-declared record type the current coercion converts to. */ + coercionRecordOrigin(): RecordOrigin | null { + const node = this.coercionOriginNode; + if (node === null) return null; + let origin = this.recordOriginMemo.get(node); + if (origin === undefined) { + try { + origin = recordOrigin(this.checker, node); + } catch { + origin = null; + } + this.recordOriginMemo.set(node, origin); + } + return origin; + } + + declOriginIndex(origin: string): number { + let id = this.declOriginIds.get(origin); + if (id === undefined) { + id = this.declOrigins.length; + this.declOrigins.push(origin); + this.declOriginIds.set(origin, id); + } + return id; + } + + /** The origin of the package-declared slot a value flows into. */ + slotOriginOf(node: ts.Node): string | null { + let origin = this.slotOriginMemo.get(node); + if (origin === undefined) { + try { + origin = slotOrigin(this.checker, node); + } catch { + origin = null; + } + this.slotOriginMemo.set(node, origin); + } + return origin; + } + + private inPackageFileOf(node: ts.Node): boolean { + const sf = ts.getSourceFileOfNode(node); + let inside = this.packageFileMemo.get(sf); + if (inside === undefined) { + inside = inPackageFile(node); + this.packageFileMemo.set(sf, inside); + } + return inside; + } + private declBoundaryMemo: boolean | null = null; get declBoundary(): boolean { this.declBoundaryMemo ??= this.program.getSourceFileNames().some(isDeclBoundaryFile); @@ -3778,6 +4042,9 @@ export class Lowerer { ) { // The mode's options, sorted by name; the values derived from them follow. this.alreadyFlushed = mode.alreadyFlushed ?? new Set(); + if (mode.degradedTypes) + for (const [id, degraded] of mode.degradedTypes) this.degradedTypes.set(id, degraded); + this.priorRetriedSymbols = mode.retriedSymbols ?? new Set(); this.externalTypes = mode.externalTypes ?? new Map(); this.ffiBindingSymbols = mode.ffiBindingSymbols ?? null; this.ffiImports = mode.ffiImports ?? []; @@ -3813,6 +4080,10 @@ export class Lowerer { resolveTypeParam: this.typeParamResolver, resolveTypeParamTs: this.typeParamTsResolver, resolveBoundAliasTs: this.boundAliasTsResolver, + degradedType: (type) => { + const id = (type as { id?: number }).id; + return id === undefined ? null : (this.degradedTypes.get(id)?.lenient ?? null); + }, isErasing: () => this.erasingDepth > 0, genericClassInstance: (decl, ref, mapArgument) => this.genericClassInstanceType(decl, ref, mapArgument), @@ -4845,7 +5116,14 @@ export class Lowerer { } collectProgram(parts: FileParts[]): void { - return collectProgram(this, parts); + collectProgram(this, parts); + // The coverage remainder registers the declarations reachable emit + // registered with dynamic slots (retryDeferred), so it neither reports + // them as unreached nor blocks their uses. + for (const symbol of this.priorRetriedSymbols) { + const diags = this.deferredDiags.get(symbol); + if (diags !== undefined) this.retryDeferred(symbol, diags); + } } /** Names every file's %init and registers the run-once guard globals @@ -5838,6 +6116,22 @@ export class Lowerer { }; if (!stringRead(e.left) && !stringRead(e.right)) return null; if (!primitive(e.left) || !primitive(e.right)) return null; + // A missing read is undefined, and `+` yields a number only when + // neither operand is a string: an operand that is always a string + // (`"x-" + PAD[i]`) makes the result a string. Only literals, + // templates and sums with such an operand are always strings; a + // binding or call can hold a missing read. + const definitelyString = (part: ts.Expression): boolean => { + const p = peel(part); + if (ts.isStringLiteral(p) || ts.isNoSubstitutionTemplateLiteral(p)) return true; + if (ts.isTemplateExpression(p)) return true; + return ( + ts.isBinaryExpression(p) && + p.operatorToken.kind === ts.SyntaxKind.PlusToken && + (definitelyString(p.left) || definitelyString(p.right)) + ); + }; + if (definitelyString(e.left) || definitelyString(e.right)) return null; return { kind: "union", unionId: this.unions.intern([F64, STRING]) }; }; /** The union of primitive result types, or null when any is missing or @@ -6509,19 +6803,20 @@ export class Lowerer { node.initializer !== undefined; const collectDynamicViews = (): Set => { const keys = new Set(); - const dynamicValue = (node: ts.Expression): boolean => { - // A binding's declared type says whether it holds dynamic data; a - // flow type such as `Array.isArray(x)`'s `any[]` only narrows it. + // A binding's declared type says whether it holds dynamic data; a flow + // type such as `Array.isArray(x)`'s `any[]` only narrows it. + const typedNode = (node: ts.Expression): ts.Node => { const e = peel(node); const symbol = ts.isIdentifier(e) ? symbolOf(e) : null; const declaration = symbol ? symbol.valueDeclaration : undefined; - const declared = - declaration && + return declaration && (ts.isVariableDeclaration(declaration) || ts.isParameter(declaration)) && ts.isIdentifier(declaration.name) - ? declaration.name - : null; - const type = typeOrNull(declared ?? node); + ? declaration.name + : node; + }; + const dynamicValue = (node: ts.Expression): boolean => { + const type = typeOrNull(typedNode(node)); if (type === null) return true; if ((type.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) return true; if (!this.checker.isArrayType(type) || this.checker.isTupleType(type)) return false; @@ -6542,57 +6837,64 @@ export class Lowerer { collect(field.type, depth + 1); } else if (type.kind === "promise") collect(type.inner, depth + 1); }; - const target = (type: ts.Type | null | undefined): void => { + const convert = (type: ts.Type | null | undefined): void => { // An unanswerable target may be any array. if (type === null) keys.add("*"); else if (type) collect(this.mapTypeOf(type), 0); }; - const visit = (node: ts.Node): void => { - if ((ts.isAsExpression(node) || ts.isTypeAssertion(node)) && dynamicValue(node.expression)) - target(typeOrNull(node)); - else if ( - ts.isVariableDeclaration(node) && - node.type && - node.initializer && - dynamicValue(node.initializer) - ) - target(typeOrNull(node.name)); - else if (initializedBinding(node) && dynamicValue(node.initializer!)) - target(typeOrNull(node.name ?? node)); - else if ( - ts.isBinaryExpression(node) && - assignmentOperators.has(node.operatorToken.kind) && - dynamicValue(node.right) - ) - target(typeOrNull(node.left)); - else if (ts.isReturnStatement(node) && node.expression && dynamicValue(node.expression)) - target(contextualOrNull(node.expression)); - else if (ts.isYieldExpression(node) && node.expression && dynamicValue(node.expression)) - target(contextualOrNull(node.expression)); - else if (ts.isCallExpression(node) || ts.isNewExpression(node)) { + /** Each value `node` converts, with the type it converts to. */ + const conversions = ( + node: ts.Node, + judge: (value: ts.Expression, target: () => ts.Type | null | undefined) => void, + ): void => { + if (ts.isAsExpression(node) || ts.isTypeAssertion(node)) + judge(node.expression, () => typeOrNull(node)); + else if (ts.isVariableDeclaration(node) && node.type && node.initializer) + judge(node.initializer, () => typeOrNull(node.name)); + else if (initializedBinding(node)) + judge(node.initializer!, () => typeOrNull(node.name ?? node)); + else if (ts.isBinaryExpression(node) && assignmentOperators.has(node.operatorToken.kind)) + judge(node.right, () => typeOrNull(node.left)); + else if ((ts.isReturnStatement(node) || ts.isYieldExpression(node)) && node.expression) { + const value = node.expression; + judge(value, () => contextualOrNull(value)); + } else if (ts.isCallExpression(node) || ts.isNewExpression(node)) { for (const arg of node.arguments?.nodes ?? []) - if (!ts.isSpreadElement(arg) && dynamicValue(arg)) target(contextualOrNull(arg)); - } else if (ts.isPropertyAssignment(node) && dynamicValue(node.initializer)) - target(contextualOrNull(node.initializer)); - else if (ts.isArrayLiteralExpression(node)) { + if (!ts.isSpreadElement(arg)) judge(arg, () => contextualOrNull(arg)); + } else if (ts.isPropertyAssignment(node)) { + const value = node.initializer; + judge(value, () => contextualOrNull(value)); + } else if (ts.isArrayLiteralExpression(node)) { for (const element of node.elements.nodes) - if ( - !ts.isSpreadElement(element) && - !ts.isOmittedExpression(element) && - dynamicValue(element) - ) - target(contextualOrNull(element)); - } else if ( - ts.isArrowFunction(node) && - !ts.isBlock(node.body) && - dynamicValue(node.body as ts.Expression) - ) - target(contextualOrNull(node.body as ts.Expression)); + if (!ts.isSpreadElement(element) && !ts.isOmittedExpression(element)) + judge(element, () => contextualOrNull(element)); + } else if (ts.isArrowFunction(node) && !ts.isBlock(node.body)) { + const body = node.body as ts.Expression; + judge(body, () => contextualOrNull(body)); + } }; + // The values' types in one batch: these bodies' reachability is not + // known yet. + const typed: ts.Node[] = []; + for (const sf of sourceFiles) + ts.walkPreorder(sf, (node) => { + panicSafe(() => conversions(node, (value) => typed.push(typedNode(value))), undefined); + return undefined; + }); + this.checker.prefetchClassCollection(typed, []); for (const sf of sourceFiles) ts.walkPreorder(sf, (node) => { // A checker panic hides this conversion's target: any array. - if (!panicSafe(() => (visit(node), true), false)) keys.add("*"); + const judged = panicSafe( + () => ( + conversions(node, (value, target) => { + if (dynamicValue(value)) convert(target()); + }), + true + ), + false, + ); + if (!judged) keys.add("*"); return undefined; }); return keys; @@ -7029,6 +7331,12 @@ export class Lowerer { node.operatorToken.kind === ts.SyntaxKind.EqualsToken ) values.push(node.right); + // A for-of loop reads its iterable's array keys. + if (ts.isForOfStatement(node)) values.push(node.expression); + // A property read in such a slot reads its declared type through + // its receiver's (widenedType). + const receiver = widenedReceiver(node); + if (receiver !== null) values.push(receiver); }); this.checker.prefetchCollectionTypes(values); scanSites.set(sf, nodes); @@ -7608,7 +7916,8 @@ export class Lowerer { siteBase.set(sf, siteCount); siteCount += sitesOf(sf).length; } - siteDependencies = new SiteDependencies(siteCount); + // Each function's returns are one more site, after the files' sites. + siteDependencies = new SiteDependencies(siteCount + functionDeclBySymbol.size); let fullPass = true; for (let changed = true, first = true; changed || !fullPass; first = false) { fullPass = first || !changed; @@ -7622,8 +7931,14 @@ export class Lowerer { } } for (const sf of sourceFiles) if (scanFile(sf)) changed = true; + let returnSite = siteCount; for (const [symbol, decl] of functionDeclBySymbol) { - if (!scanReturns(symbol, decl)) continue; + const site = returnSite++; + if (!fullPass && !siteDependencies.isStale(site)) continue; + siteDependencies.begin(site); + const returnsChanged = scanReturns(symbol, decl); + siteDependencies.end(returnsChanged); + if (!returnsChanged) continue; siteDependencies.changed(symbol); changed = true; } @@ -8082,9 +8397,12 @@ export class Lowerer { if (sig && !this.wantBody(sig.name)) continue; const fn = this.lowerFunction(decl); if (fn) functions.push(fn); - else if (this.countsSkips()) { - this.countSkippedBody(decl); - if (this.remainder) this.reportBlockedBody(decl); + else { + this.skippedBodies.add(decl); + if (this.countsSkips()) { + this.countSkippedBody(decl); + if (this.remainder) this.reportBlockedBody(decl); + } } } for (const decl of fp.classDecls) { @@ -8093,7 +8411,10 @@ export class Lowerer { functions.push( ...this.withClosureRoot(info.def.name, () => this.lowerClassMembers(info)), ); - else if (this.countsSkips()) this.countSkippedBody(decl); + else { + this.skippedBodies.add(decl); + if (this.countsSkips()) this.countSkippedBody(decl); + } } } @@ -8128,15 +8449,38 @@ export class Lowerer { let gc = 0; let gi = 0; let es = 0; + let rd = 0; const classDispatch = new ClassDynamicDispatch(); let dispatchChanged = !this.remainder; while ( dispatchChanged || + rd < this.retriedDeclarations.length || ec < this.exprClasses.length || gc < this.genericClassInstances.length || gi < this.instantiationQueue.length || es < this.emitSpecQueue.length ) { + // Localized failure: declarations a flush registered with dynamic + // slots (retryDeferred) after the loops above skipped their bodies. + while (rd < this.retriedDeclarations.length) { + const decl = this.retriedDeclarations[rd++]!; + if (!this.skippedBodies.delete(decl)) continue; + if (this.countsSkips()) this.uncountSkippedBody(decl); + if (ts.isFunctionDeclaration(decl)) { + const fn = this.lowerFunction(decl); + if (fn) functions.push(fn); + else { + this.skippedBodies.add(decl); + if (this.countsSkips()) this.countSkippedBody(decl); + } + } else if (ts.isClassDeclaration(decl)) { + const info = this.classes.get(this.classNamer(decl)); + if (info) + functions.push( + ...this.withClosureRoot(info.def.name, () => this.lowerClassMembers(info)), + ); + } + } while (ec < this.exprClasses.length) { functions.push(...this.lowerClassMembers(this.exprClasses[ec++]!)); } @@ -8149,13 +8493,10 @@ export class Lowerer { // generic method's this/super fence, a fenced parameter // default): the diagnostic is recorded — the instance skips // like a signature-blocked function (lowerFunction's rule). - try { - functions.push( - this.withClosureRoot(inst.name, () => this.lowerGenericInstance(info, inst)), - ); - } catch (e) { - if (!(e instanceof PoisonError)) throw e; - } + const fn = this.lowerUnitDegrading(() => + this.withClosureRoot(inst.name, () => this.lowerGenericInstance(info, inst)), + ); + if (fn !== null) functions.push(fn); } // Emit-override specializations queued by the emit sites above (a // body can queue more — the super-forward chain — and generic @@ -8327,6 +8668,7 @@ export class Lowerer { ...(this.foreignElementKeys.size > 0 ? { foreignElementKeys: [...this.foreignElementKeys].sort() } : {}), + ...(this.declOrigins.length > 0 ? { declOrigins: [...this.declOrigins] } : {}), ...(this.threadSlotKeys.size > 0 ? { threadSlotKeys: [...this.threadSlotKeys].sort() } : {}), @@ -8338,6 +8680,7 @@ export class Lowerer { ...(this.ffiImports.length > 0 ? { ffiImports: [...this.ffiImports] } : {}), }; if (module) sanitizeUnregisteredClassTypes(module, (name) => this.classes.has(name)); + if (module?.declOrigins !== undefined) compactDeclOrigins(module); if (module) checkCycleBindingAccesses(this, module); if (module) bindCompletionCallbacks(module); // Execution tiers from the final bodies (ir/tiers.ts). Assigned with or @@ -8416,6 +8759,22 @@ export class Lowerer { * percentage would overstate how much of the program compiles. Count * every statement the body holds as attempted and failed, in addition to * the skipped-function tally. */ + /** countSkippedBody undone: a flush registered the declaration after + * its body was skipped, and the body lowers now. */ + private uncountSkippedBody(decl: ts.Node): void { + this.stats.functionsSkipped--; + if (this.suppressStats) return; + const n = countBodyStatements(decl); + if (n === 0) return; + this.stats.statementsTotal -= n; + this.stats.statementsFailed -= n; + const file = ts.getSourceFileOfNode(decl).fileName; + for (let i = 0; i < n; i++) { + this.bumpFileStat(file, "total", -1); + this.bumpFileStat(file, "failed", -1); + } + } + private countSkippedBody(decl: ts.Node): void { this.stats.functionsSkipped++; if (this.suppressStats) return; @@ -8739,6 +9098,44 @@ export class Lowerer { register(`%${cName}.set:${prop}`, [], () => this.throwingSetterFn(info, prop)); } }; + // Localized failure: a declaration a flush registered with dynamic + // slots (retryDeferred) registers its units like a class expression. + this.onDeclarationRegistered = (decl: ts.Node): void => { + if (ts.isFunctionDeclaration(decl)) { + if (!decl.body) return; + const symbol = declSymbolOf(this, decl); + const sig = symbol ? this.fnSigsBySymbol.get(symbol) : undefined; + if (!sig || units.has(sig.name)) return; + units.set(sig.name, { + order: unitOrder++, + roots: functionRoots(decl), + lower: () => this.lowerFunction(decl), + }); + if (reachable.has(sig.name)) queue.push(sig.name); + metadataPriority.set(sig.name, [3, expressionMetadataOrder++]); + } else if (ts.isClassDeclaration(decl)) { + const info = this.classes.get(this.classNamer(decl)); + if (!info) return; + if (!info.generic) { + if (!units.has(`%${info.def.name}.constructor`)) this.onExprClassCollected?.(info); + return; + } + // A generic class FAMILY's only units are its statics; its + // instantiations lower in the instance drain. + if (!info.staticMethods) return; + for (const [name, entry] of info.staticMethods) { + const unit = `%${info.def.name}.static:${name}`; + if (units.has(unit)) continue; + units.set(unit, { + order: unitOrder++, + roots: functionRoots(entry.member), + lower: () => lowerStaticMethod(this, info, name), + }); + if (reachable.has(unit)) queue.push(unit); + metadataPriority.set(unit, [3, expressionMetadataOrder++]); + } + } + }; const onEdge = this.onEdge; for (const name of collectionEdges) onEdge(name); // Generic instances queued by the bodies above lower here (an instance @@ -8794,17 +9191,16 @@ export class Lowerer { this.genericDemandPriority.set(inst, ref); // Body-level poisons skip the instance after retaining the // diagnostic and every edge fired before poisoning. - try { - instanceFunctions.push( - this.withGenericDemandOwner(owner, () => - this.shapes.withDeclaredOrderPriority(ref, () => - this.withClosureRoot(inst.name, () => this.lowerGenericInstance(info, inst)), - ), + const fn = this.lowerUnitDegrading(() => + this.withGenericDemandOwner(owner, () => + this.shapes.withDeclaredOrderPriority(ref, () => + this.withClosureRoot(inst.name, () => this.lowerGenericInstance(info, inst)), ), - ); + ), + ); + if (fn !== null) { + instanceFunctions.push(fn); relieveHeapPressure(); - } catch (e) { - if (!(e instanceof PoisonError)) throw e; } } } @@ -9176,7 +9572,14 @@ export class Lowerer { this.diagSink.push(d); return; } - if (pending) this.commitFlush(pending.symbol, pending.source); + // A probe's copy of a declaration's flushed diagnostics, committed + // after a flush retry registered the declaration: nothing to report. + if (pending && this.retriedSymbols.has(pending.symbol)) return; + if (pending) { + this.commitFlush(pending.symbol, pending.source); + const group = this.rejectedGroups.get(pending.symbol); + if (group !== undefined) d.rejects ??= group; + } // One site, one report: some declarations map a type twice (module // globals pre-register before their initializers lower) — an exact // duplicate (code + span + message) adds noise, not information. @@ -9231,6 +9634,166 @@ export class Lowerer { return (this.typeOf(node).flags & ts.TypeFlags.Any) !== 0; } + /** badType's commit: the diagnostic carries the structured type + * failure (report-only in stage L0), and so does the poison. */ + private typeFail( + node: ts.Node, + type: ts.Type, + family: TypeFailureFamily, + diag: ScrDiagnostic, + ): never { + const failure = typeFailureOf(node, this.checker.typeToString(type), family); + diag.typeFailure = failure; + this.failedTypes.set(failure, type); + this.pushDiag(diag); + throw new PoisonError(failure, type); + } + + /** Localized failure's commit point (design §3.3): `e` poisoned a unit + * whose failure is about to enter the build. When it is a degradable type + * failure at a slot kind in `slots`, the failing type joins + * degradedTypes (its lenient mapping is the checked dynamic, whole slot), + * the failure's own diagnostic leaves the build, and the caller lowers + * the unit again. Returns the slot to record once that retry succeeds, + * or null to commit as today. Never inside a capture buffer (a probe or a + * deferred collection) and never under --tier=static-only. */ + degradeAtCommit(e: unknown, slots: ReadonlySet): DynamicSlot | null { + if (this.tierMode !== "auto" || this.diagSink !== null) return null; + if (!(e instanceof PoisonError)) return null; + const failure = e.failure; + const type = e.failedType; + if (failure === undefined || type === undefined) return null; + if (!failure.degradable || failure.family === "dyn-field-policy") return null; + if (!slots.has(failure.slot)) return null; + const id = (type as { id?: number }).id; + // Already degraded and still failing: no progress, commit as today. + if (id === undefined || this.degradedTypes.has(id)) return null; + let committed: ScrDiagnostic | undefined; + for (let i = this.diags.length - 1; i >= 0; i--) { + if (this.diags[i]!.typeFailure !== failure) continue; + committed = this.diags[i]; + this.diags.splice(i, 1); + } + if (committed === undefined) return null; + this.degradedTypes.set(id, { lenient: DYN, code: committed.code, reason: committed.message }); + return { loc: committed.loc, failure, code: committed.code, reason: committed.message }; + } + + /** A declaration slot whose type maps through degradedTypes is a + * dynamic slot too, though only the first such slot failed: record it + * (outside probes; the lookup is skipped while nothing is degraded). */ + noteDegradedSlot(node: ts.Node, type: ts.Type): void { + if (this.diagSink !== null) return; + const degraded = this.degradedTypes.get((type as { id?: number }).id ?? -1); + if (degraded === undefined) return; + const failure = typeFailureOf(node, this.checker.typeToString(type), "unsupported"); + if (failure.slot === "expression" || failure.slot === "member") return; + this.noteDynamicSlot({ + loc: locOf(node), + failure, + code: degraded.code, + reason: degraded.reason, + }); + } + + /** Localized failure, statement commit point for declarations a probe + * flushed: a statement committing the flushed diagnostics of a deferred + * declaration (inside a capture buffer the flush could not retry) reopens + * it here, outside every probe. When the declaration's diagnostics are + * all retriable, they leave the build, the declaration retries + * (retryDeferred), and on success the caller lowers the statement again. + * Returns whether some declaration registered. */ + reopenFlushedAtCommit(diagsBefore: number): boolean { + if (this.tierMode !== "auto" || this.diagSink !== null) return false; + const groups = new Set(); + for (let i = diagsBefore; i < this.diags.length; i++) { + const group = this.diags[i]!.rejects; + if (group !== undefined && !group.startsWith("statement @")) groups.add(group); + } + if (groups.size === 0) return false; + let registered = false; + for (const [symbol, group] of [...this.rejectedGroups]) { + if (!groups.has(group) || !this.deferredCollectors.has(symbol)) continue; + const committed = this.diags.filter((d) => d.rejects === group); + this.diags.splice(0, this.diags.length, ...this.diags.filter((d) => d.rejects !== group)); + this.flushedSymbols.delete(symbol); + this.rejectedGroups.delete(symbol); + if (this.retryDeferred(symbol, committed)) { + registered = true; + continue; + } + // Not retriable: the declaration's diagnostics commit as before. + const last = this.deferredDiags.get(symbol) ?? committed; + this.deferredDiags.delete(symbol); + this.flushedSymbols.add(symbol); + this.noteRejected(symbol, last); + for (const d of last) this.pushDiag(d); + } + return registered; + } + + /** A failed attempt's own diagnostics leave the build before its unit + * lowers again; flushed declaration diagnostics (`rejects`) stay, since + * the retry will not flush them a second time. */ + dropAttemptDiags(diagsBefore: number): void { + const kept = this.diags + .slice(diagsBefore) + .filter((d) => d.rejects !== undefined || this.loweredOnceDiags.has(d)); + this.diags.length = diagsBefore; + this.diags.push(...kept); + } + + /** Lowers one unit whose body-level poison skips it (a generic + * instance), with localized failure's commit point: a degradable type + * failure at one of its declaration slots degrades the type and the unit + * lowers again (at most MAX_DECLARATION_RETRIES times). Null when it + * still poisons; its diagnostics are then committed as before. */ + lowerUnitDegrading(lower: () => T): T | null { + const slots: DynamicSlot[] = []; + for (let attempt = 0; ; attempt++) { + const mark = this.diags.length; + try { + const result = lower(); + for (const slot of slots) this.noteDynamicSlot(slot); + return result; + } catch (e) { + if (!(e instanceof PoisonError)) throw e; + if (attempt >= MAX_DECLARATION_RETRIES) return null; + const slot = this.degradeAtCommit(e, DECLARATION_SLOTS); + if (slot === null) return null; + slots.push(slot); + this.dropAttemptDiags(mark); + } + } + } + + /** Records a dynamic slot whose unit lowered with it (one per site). */ + noteDynamicSlot(slot: DynamicSlot): void { + const seen = this.dynamicSlots.some( + (s) => + s.loc.file === slot.loc.file && + s.loc.start === slot.loc.start && + s.loc.end === slot.loc.end, + ); + if (!seen) this.dynamicSlots.push(slot); + } + + /** The SC1090 refusal of checked-dynamic class fields in TypeScript + * (classFieldStoresDyn): a policy fence, not a missing representation. + * A flush retry lifts it (liftDynFieldPolicy), so a reached class with + * an 'any' field registers. */ + dynFieldPolicyFence(node: ts.Node, feature: string): never { + const diag = unsupportedDiag("SC1090", locOf(node), feature); + const failure = typeFailureOf( + node, + this.checker.typeToString(this.typeOf(node)), + "dyn-field-policy", + ); + diag.typeFailure = failure; + this.pushDiag(diag); + throw new PoisonError(failure, this.typeOf(node)); + } + badType(node: ts.Node, type: ts.Type): never { const widened = this.checker.getBaseTypeOfLiteralType(type); // Types declared by the ADOPTED @types/node (Buffer, NodeJS.Timeout, @@ -9242,8 +9805,12 @@ export class Lowerer { // applies to node-typed values. const typeSym = widened.getAliasSymbol() ?? widened.getSymbol(); if (this.nodeTypesOnlySymbol(typeSym)) { - this.pushDiag(noLoweringDiag(this.checker.typeToString(type), locOf(node), undefined, true)); - throw new PoisonError(); + this.typeFail( + node, + type, + "node-types", + noLoweringDiag(this.checker.typeToString(type), locOf(node), undefined, true), + ); } // Engine-backed ambient TYPES in a STATIC build report the per-site // SC2012 rather than the generic supported-types recitation. Native @@ -9254,10 +9821,12 @@ export class Lowerer { (FETCH_AMBIENT_TYPES as readonly string[]).includes(ts.symbolName(typeSym)) && this.isStdlibSymbol(typeSym) ) { - this.pushDiag( + this.typeFail( + node, + type, + "fetch-ambient", unsupportedNativeApiDiag(`a value of type '${ts.symbolName(typeSym)}'`, locOf(node)), ); - throw new PoisonError(); } // ENUM OBJECTS (`typeof e` — a numeric enum's type carries the // reverse-map index signature, a string enum's just its members): @@ -9266,14 +9835,16 @@ export class Lowerer { // as the per-site identifier fence — never the index-signature // recitation. if (typeSym && (typeSym.flags & ts.SymbolFlags.Enum) !== 0) { - this.pushDiag( + this.typeFail( + node, + type, + "enum-object", unsupportedDiag( "SC1090", locOf(node), `enum objects as values ('${ts.symbolName(typeSym)}' — member reads like '${ts.symbolName(typeSym)}.X' compile to constants; the object itself has no runtime representation)`, ), ); - throw new PoisonError(); } // Would-be records with INDEX SIGNATURES (`Record`, // `{ [k: string]: T }`) get the index-signature fence (SC2006) naming @@ -9303,19 +9874,26 @@ export class Lowerer { this.isStdlibFile(ts.getSourceFileOfNode(d)), ) ) { - this.pushDiag(noLoweringDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "stdlib-index", + noLoweringDiag(this.checker.typeToString(type), locOf(node)), + ); } - this.pushDiag(indexSignatureTypeDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "index-signature", + indexSignatureTypeDiag(this.checker.typeToString(type), locOf(node)), + ); } // Attribute an unsupported package type to its package. { const pkg = this.npmPackageOf(widened); if (pkg) { - this.pushDiag(unsupportedPackageValueDiag(pkg, locOf(node))); - throw new PoisonError(); + this.typeFail(node, type, "package", unsupportedPackageValueDiag(pkg, locOf(node))); } } // A type that keeps a GENERIC call signature (`(x: T) => T` slots, @@ -9331,8 +9909,12 @@ export class Lowerer { this.checker.getCallSignatures(p).some((s) => s.getTypeParameters().length > 0), ) ) { - this.pushDiag(genericSignatureTypeDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "generic-signature", + genericSignatureTypeDiag(this.checker.typeToString(type), locOf(node)), + ); } } // STANDARD-LIBRARY nominal provenance, decided once: interface/class @@ -9352,16 +9934,24 @@ export class Lowerer { // generic overload set keeps the monomorphization story: a value of a // multi-signature type has no single compiled signature to hold. if (!stdlibNominal && this.checker.getCallSignatures(widened).length > 1) { - this.pushDiag(overloadedSignatureTypeDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "overloads", + overloadedSignatureTypeDiag(this.checker.typeToString(type), locOf(node)), + ); } // INTERSECTIONS that resolved to no lowering (SC2008). The ones that // compile never get here: member intersections intern through the // record path, callable hybrids map to '%call' records, and pinned // mixin instantiations resolve by chain structure. if (widened.isIntersectionType()) { - this.pushDiag(intersectionTypeDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "intersection", + intersectionTypeDiag(this.checker.typeToString(type), locOf(node)), + ); } // SUPPORTED shapes over a component outside its slot (SC2009): the // Map/Set domains, array/tuple elements, union arms, function @@ -9373,8 +9963,12 @@ export class Lowerer { { const detail = describeComponentBlocker(widened, this.typeCtx); if (detail !== null) { - this.pushDiag(componentTypeDiag(this.checker.typeToString(type), detail, locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "component", + componentTypeDiag(this.checker.typeToString(type), detail, locOf(node)), + ); } } // STANDARD-LIBRARY nominal types with no lowering at all: the SC2020 @@ -9404,26 +9998,36 @@ export class Lowerer { TextDecoder: "TextDecoder instances support recognized labels, fatal/ignoreBOM option literals, and streaming UTF-8/UTF-16/single-byte decode; runtime option objects remain unsupported", }; - this.pushDiag( + this.typeFail( + node, + type, + "stdlib-nominal", noLoweringDiag( this.checker.typeToString(type), locOf(node), typeHints[ts.symbolNameOf(stdlibOwnSym) ?? ""], ), ); - throw new PoisonError(); } // USER record shapes blocked by ONE member (SC2009's record arm): // name the member and its type. { const detail = describeRecordMemberBlocker(widened, this.typeCtx); if (detail !== null) { - this.pushDiag(componentTypeDiag(this.checker.typeToString(type), detail, locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "record-member", + componentTypeDiag(this.checker.typeToString(type), detail, locOf(node)), + ); } } - this.pushDiag(unsupportedTypeDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "unsupported", + unsupportedTypeDiag(this.checker.typeToString(type), locOf(node)), + ); } /** The lib fence (SC2020): a reached use of standard-library surface @@ -9857,6 +10461,38 @@ export class Lowerer { ); }; + private structuredCloneReadCounts: { workerData: number; messageListeners: number } | null = null; + + /** How often the program reads `workerData` (every expression spelling it, + * counted conservatively) and registers a `"message"` listener: a + * structured clone read at one site only has no other holder + * (narrowing.ts freshDynSource). Computed once per lowering pass. */ + structuredCloneReads(): { workerData: number; messageListeners: number } { + if (this.structuredCloneReadCounts) return this.structuredCloneReadCounts; + const counts = { workerData: 0, messageListeners: 0 }; + const visit = (node: ts.Node): void => { + if (ts.isIdentifier(node) && node.text === "workerData") { + const parent = node.parent; + const declaresName = + parent !== undefined && + (ts.isImportSpecifier(parent) || + ts.isExportSpecifier(parent) || + ((ts.isPropertyAssignment(parent) || + ts.isPropertyDeclaration(parent) || + ts.isPropertySignature(parent) || + ts.isMethodDeclaration(parent)) && + parent.name === node)); + if (!declaresName) counts.workerData++; + } else if (ts.isCallExpression(node) && isMessageListenerCall(node)) { + counts.messageListeners++; + } + ts.forEachChild(node, visit); + }; + for (const sf of this.moduleOrder) ts.forEachChild(sf, visit); + this.structuredCloneReadCounts = counts; + return counts; + } + irTypeOf(node: ts.Node): IrType { const t = this.typeOf(node); // JS inference can leave mutable fields, accessor parameters and @@ -9871,6 +10507,7 @@ export class Lowerer { // f64 — but must not: pre-empt the mapping so the JS fallback below // answers instead. const mapped = neverTaintedJsType(this, node, t) ? null : this.mapTypeOf(t); + if (mapped !== null && this.degradedTypes.size > 0) this.noteDegradedSlot(node, t); if (!mapped) { const argumentsType = jsArgumentsFunctionType(this, t); if (argumentsType !== null) return argumentsType; @@ -10612,6 +11249,8 @@ export class Lowerer { if (!typeEquals(expr.type, expected)) { const w = this.widthCoerce(expr, expected); if (w) return w; + const asserted = assertedConversion(this, expr, expected); + if (asserted) return asserted; // A UNIT (null/undefined) flowing into a plain non-nullable slot // the checker approved (`null!` casts, non-strict assignments): // the stranded-source stance (divergence 38) without a union in @@ -10630,7 +11269,7 @@ export class Lowerer { if (helper) { return { kind: "call", callee: helper, args: [expr], type: expected, loc: expr.loc }; } - return expr; + return assertedConversion(this, expr, expected) ?? expr; } if (expr.type.kind === "void") { // A void CALL RESULT flowing into a union with an undefined arm @@ -10969,9 +11608,51 @@ export class Lowerer { * wrap implicitly (coerceToExpected), then the exact-type fence runs * (SC2002 for record shapes, SC2003 for unions). Every slot-directed * lowering goes through here (via lowerExprExpecting) or calls this - * directly when the expression was already lowered. */ + * directly when the expression was already lowered. In package files, a + * check this coercion creates names the package declaration that types + * the slot (decl-origin.ts). */ coerceInto(node: ts.Node, expr: IrExpr, expected: IrType): IrExpr { + if (!this.inPackageFileOf(node)) return this.coerceIntoSlot(node, expr, expected); + const outer = this.coercionOriginNode; + const outerUnion = this.coercionOriginUnion; + this.coercionOriginNode = node; + this.coercionOriginUnion = expr.type.kind === "union" ? expr.type.unionId : null; + let out: IrExpr; + try { + out = this.coerceIntoSlot(node, expr, expected); + } finally { + this.coercionOriginNode = outer; + this.coercionOriginUnion = outerUnion; + } + if (out !== expr && out.kind === "dynCheck" && out.origin === undefined) { + const origin = this.slotOriginOf(node); + if (origin !== null) return { ...out, origin: this.declOriginIndex(origin) }; + } + return out; + } + + private coerceIntoSlot(node: ts.Node, expr: IrExpr, expected: IrType): IrExpr { refuseInferredCallbackDrops(this, node, expr.type, expected); + // A union into one of its record arms extracts that arm, trusting the + // checker to have excluded the others. A built-in object arm the checker + // still types the site with (`c ?? new Map()` into an interface the + // untyped Map satisfies only through `any`) holds a value at run time, + // and no conversion makes a record of it, so the flow is refused like + // the arm's value would be. + if ( + expr.type.kind === "union" && + expected.kind === "record" && + this.armTag(expr.type.unionId, expected) >= 0 + ) { + const site = this.mapTypeOf(this.typeOf(node)); + const siteArms = + site?.kind === "union" ? (this.unions.get(site.unionId)?.arms ?? []) : site ? [site] : []; + for (const arm of this.unions.get(expr.type.unionId)?.arms ?? []) { + if (arm.kind !== "map" && arm.kind !== "set") continue; + if (!siteArms.some((s) => typeEquals(s, arm))) continue; + this.requireExactShape(node, arm, expected); + } + } // An assertion's scalar check whose value goes straight back into a // checked-dynamic slot (`process.loadEnvFile({} as unknown as string)`) // keeps the original value: JavaScript erases assertions, so the check @@ -11134,6 +11815,25 @@ export class Lowerer { } } + // A value the checker types `never` at this site (a read narrowed past + // every member of its union) exists only when an assertion lied about + // its type, so its layout need not fit the slot: it is evaluated, then a + // TypeError is thrown, as checked extraction of an impossible arm does. + if (!typeEquals(e.type, expected) && (this.typeOf(node).flags & ts.TypeFlags.Never) !== 0) { + return { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: e, loc: e.loc }], + result: nodeThrowExpr( + 1, + "", + unionNarrowFailureMessage(e.type, (t) => this.fmt(t)), + expected, + e.loc, + ), + type: expected, + loc: e.loc, + }; + } this.requireExactShape(node, e.type, expected); return e; } @@ -11457,6 +12157,7 @@ export class Lowerer { !this.mapTypeOf(this.typeOf(x)) || this.excessJsContextualProperty(x, parts) || this.reshapeableContextualLiteral(x, parts) || + this.untypedLiteralContext(x) || holdsEvolvingArray(this, x) ) { return lowerDynObjectLiteral(this, x); @@ -12050,6 +12751,7 @@ export class Lowerer { const list = this.deferredDiags.get(symbol) ?? []; list.push(...sink); this.deferredDiags.set(symbol, list); + this.deferredCollectors.set(symbol, { symbolOf, collect }); return symbol; } finally { this.diagSink = previousSink; @@ -12066,20 +12768,48 @@ export class Lowerer { this.reportDeferred(symbol); } + /** A statement's lowering used a declaration: if that declaration was + * rejected, the statement's own failure (if any) is its cascade. */ + touchRejected(symbol: ts.Symbol): void { + const group = this.rejectedGroups.get(symbol); + if (group === undefined) return; + const frame = this.failureTouches[this.failureTouches.length - 1]; + if (frame !== undefined) frame.add(group); + } + + /** Records a declaration whose deferred diagnostics reached the build: + * its failure group, on the group's diagnostics too. */ + private noteRejected(symbol: ts.Symbol, diags: readonly ScrDiagnostic[]): void { + let group = this.rejectedGroups.get(symbol); + if (group === undefined) { + group = declarationGroupKey(symbol); + this.rejectedGroups.set(symbol, group); + } + for (const d of diags) d.rejects ??= group; + } + /** flushDeferred without the collection-pass guard: for declarations * whose diagnostics must report eagerly (class statements with * declaration-time code, extended bases — collectClassShapeDeferring). */ reportDeferred(symbol: ts.Symbol): void { const diags = this.deferredDiags.get(symbol); - if (!diags) return; + if (!diags) { + this.touchRejected(symbol); + return; + } if (this.alreadyFlushed.has(symbol)) { this.deferredDiags.delete(symbol); // the emit pass reported these return; } if (this.diagSink === null) { + if (this.retryDeferred(symbol, diags)) return; + // A failed retry leaves the last attempt's diagnostics deferred. + const committed = this.deferredDiags.get(symbol) ?? diags; this.deferredDiags.delete(symbol); this.flushedSymbols.add(symbol); - for (const d of diags) this.pushDiag(d); + this.noteRejected(symbol, committed); + for (const d of committed) this.pushDiag(d); + this.touchRejected(symbol); return; } // A capture buffer is active (a probe, or another declaration's @@ -12096,6 +12826,7 @@ export class Lowerer { * (the declaration counts as flushed for the coverage remainder). */ private commitFlush(symbol: ts.Symbol, source: ScrDiagnostic): void { this.flushedSymbols.add(symbol); + this.noteRejected(symbol, [source]); const list = this.deferredDiags.get(symbol); if (!list) return; const index = list.indexOf(source); @@ -12103,9 +12834,132 @@ export class Lowerer { if (list.length === 0) this.deferredDiags.delete(symbol); } - flushDeferredClass(className: string): void { + /** Flushes a deferred class known by its IR name, and answers the class + * when the flush registered it (retryDeferred). */ + flushDeferredClass(className: string): ClassInfo | undefined { const symbol = this.deferredClassByName.get(className); if (symbol) this.flushDeferred(symbol); + return this.classes.get(className); + } + + /** Localized failure's declaration commit point (design §3.3): a + * reference is flushing a deferred declaration into the build. When + * every deferred diagnostic is a degradable type failure at one of the + * declaration's slots, the failing types degrade and the collection + * runs again (up to MAX_DECLARATION_RETRIES rounds, each degrading the + * types the previous round still failed on). On success the + * declaration registers, each degraded slot is an SC6104, and nothing + * reports. Otherwise the last round's diagnostics stay deferred for the + * caller to commit. */ + private retryDeferred(symbol: ts.Symbol, diags: readonly ScrDiagnostic[]): boolean { + if (this.tierMode !== "auto") return false; + if (this.staticInitsPassed.has(symbol)) return false; + const collector = this.deferredCollectors.get(symbol); + if (collector === undefined) return false; + const slots: DynamicSlot[] = []; + // The types this retry degraded (not those of retries its collections + // ran), for retryDeferredAtDeclaration to restore when it fails. + const degraded: number[] = []; + try { + return this.retryDeferredRounds(symbol, collector, diags, slots, degraded); + } finally { + this.retryDegraded = degraded; + } + } + + private retryDeferredRounds( + symbol: ts.Symbol, + collector: { symbolOf: () => ts.Symbol | undefined; collect: () => void }, + diags: readonly ScrDiagnostic[], + slots: DynamicSlot[], + degraded: number[], + ): boolean { + let current = diags; + for (let round = 0; round < MAX_DECLARATION_RETRIES; round++) { + const failing: { diag: ScrDiagnostic; failure: TypeFailure; id: number }[] = []; + let retriable = false; + for (const diag of current) { + if (this.retriableDiags.has(diag)) { + retriable = true; + continue; + } + const failure = diag.typeFailure; + // The dyn class field policy lifts during the retry; nothing to + // degrade. + if (failure?.family === "dyn-field-policy") { + retriable = true; + continue; + } + const type = failure === undefined ? undefined : this.failedTypes.get(failure); + const id = (type as { id?: number } | undefined)?.id; + if ( + failure === undefined || + id === undefined || + !failure.degradable || + !DECLARATION_SLOTS.has(failure.slot) || + this.degradedTypes.has(id) + ) + return false; + failing.push({ diag, failure, id }); + } + // A round makes progress by degrading a type, or (once) by running + // a collection that only waited for another declaration. + if (failing.length === 0 && !(retriable && round === 0)) return false; + for (const { diag, failure, id } of failing) { + this.degradedTypes.set(id, { lenient: DYN, code: diag.code, reason: diag.message }); + degraded.push(id); + slots.push({ loc: diag.loc, failure, code: diag.code, reason: diag.message }); + } + this.deferredDiags.delete(symbol); + const lifted = this.liftDynFieldPolicy; + this.liftDynFieldPolicy = true; + let registered: boolean; + try { + registered = this.collectDeferring(collector.symbolOf, collector.collect) === null; + } finally { + this.liftDynFieldPolicy = lifted; + } + if (registered) { + this.retriedSymbols.add(symbol); + this.deferredCollectors.delete(symbol); + for (const [name, deferred] of this.deferredClassByName) + if (deferred === symbol) this.deferredClassByName.delete(name); + for (const slot of slots) this.noteDynamicSlot(slot); + for (const decl of symbol.declarations) + if (ts.isFunctionDeclaration(decl) || ts.isClassDeclaration(decl)) { + this.retriedDeclarations.push(decl); + this.onDeclarationRegistered?.(decl); + } + return true; + } + current = this.deferredDiags.get(symbol) ?? []; + } + return false; + } + + private retryDegraded: number[] = []; + + /** A deferred class with static field initializers is reached where its + * statement evaluates: Node runs the initializers there, referenced or + * not. Its file's %init calls this at that place, which retries the class + * as a flush would, so the initializers lower in place. A failed retry + * reports nothing and leaves the class deferred as it was, and from here + * on no flush registers it (staticInitsPassed): it reports instead. + * Returns whether the class has registered. */ + retryDeferredAtDeclaration(symbol: ts.Symbol): boolean { + const diags = this.deferredDiags.get(symbol); + let registered = !this.deferredCollectors.has(symbol); + if (!registered && diags !== undefined) { + this.retryDegraded = []; + registered = this.retryDeferred(symbol, diags); + if (!registered) { + for (const id of this.retryDegraded) this.degradedTypes.delete(id); + this.deferredDiags.set(symbol, [...diags]); + } + this.retryDegraded = []; + } + this.staticInitsPassed.add(symbol); + return registered; } collectSignature(decl: ts.FunctionDeclaration): void { @@ -12384,6 +13238,9 @@ export class Lowerer { * bodies as worklist units (the units map is otherwise built before * lowering starts). Null in the emit pass. */ onExprClassCollected: ((info: ClassInfo) => void) | null = null; + /** emitReachable's unit registration for a declaration retryDeferred + * registered after collection. */ + onDeclarationRegistered: ((decl: ts.Node) => void) | null = null; /** Mixin functions (`(Base: T) => class extends Base {…}`) by their * function-like node: recognized shape, or null for checked @@ -12562,6 +13419,8 @@ export class Lowerer { /* ── scoping and captures ─────────────────────────────────────────── */ declareLocal(nameNode: ts.Node, name: string, type: IrType, mutable: boolean): IrLocal { + if (type.kind === "dyn" && this.degradedTypes.size > 0) + this.noteDegradedSlot(nameNode, this.typeOf(nameNode)); const symbol = this.checker.getSymbolAtLocation(nameNode); const local = declareContextLocal( this.ctx, @@ -12639,7 +13498,7 @@ export class Lowerer { const fn = decl.parent; const symbol = this.checker.getSymbolAtLocation(nameNode); const isRead = (id: ts.Identifier): boolean => - id !== nameNode && this.checker.getSymbolAtLocation(id) === symbol; + id !== nameNode && bindingSymbolOfRead(this, id) === symbol; const pure = fn !== undefined && symbol !== undefined && onlyForwarded(this, fn, isRead); // A forwarded-only tuple is never built: a tuple with optional // positions (`Parameters<(n?: T) => R>`, whose length varies per call) @@ -13221,8 +14080,8 @@ export class Lowerer { return lowerStmts(this, stmts); } - noteBlockedBindings(stmt: ts.Statement): void { - return noteBlockedBindings(this, stmt); + noteBlockedBindings(stmt: ts.Statement, diags?: readonly ScrDiagnostic[]): void { + return noteBlockedBindings(this, stmt, diags); } isBlockedBinding(symbol: ts.Symbol | null): boolean { diff --git a/packages/compiler/src/frontend/lowering/nan-coded-reads.ts b/packages/compiler/src/frontend/lowering/nan-coded-reads.ts index 65efe5b2a..24c981bd3 100644 --- a/packages/compiler/src/frontend/lowering/nan-coded-reads.ts +++ b/packages/compiler/src/frontend/lowering/nan-coded-reads.ts @@ -249,6 +249,10 @@ export class NanCodedReads { private readonly returns = new Map(); /** Identifiers by text across the analyzed files. */ private readonly names = new Map(); + /** The identifiers of each name, grouped by the symbol they resolve to + * (referencesOf), once `names` is complete. */ + private readonly references = new Map>(); + private named = false; private readonly symbolCache = new Map(); private readonly pneCache = new Map(); private readonly pneReturnCache = new Map(); @@ -326,6 +330,7 @@ export class NanCodedReads { } }); } + this.named = true; // A proven-present read is a plain number; only the rest may be // undefined. The proof needs every reference, so it runs after the walk. for (const [read, elem] of candidates) @@ -879,8 +884,8 @@ export class NanCodedReads { else if (!decl.initializer) slot.sources.push(null); } } else return null; - for (const id of this.names.get(nameNode.text) ?? []) { - if (id === nameNode || this.symbolOf(id) !== symbol) continue; + for (const id of this.referencesOf(symbol, nameNode.text)) { + if (id === nameNode) continue; const write = this.writeOf(id); if (write !== "read") slot.reassigned = true; if (write === "read") slot.uses.push(id); @@ -895,6 +900,26 @@ export class NanCodedReads { return slot; } + /** The identifiers named `text` that resolve to `symbol`, in walk order. + * Each name's identifiers are resolved once, not once per binding. */ + private referencesOf(symbol: ts.Symbol, text: string): readonly ts.Identifier[] { + const ids = this.names.get(text) ?? []; + if (!this.named) return ids.filter((id) => this.symbolOf(id) === symbol); + let bySymbol = this.references.get(text); + if (bySymbol === undefined) { + bySymbol = new Map(); + for (const id of ids) { + const resolved = this.symbolOf(id); + if (resolved === null) continue; + const list = bySymbol.get(resolved); + if (list === undefined) bySymbol.set(resolved, [id]); + else list.push(id); + } + this.references.set(text, bySymbol); + } + return bySymbol.get(symbol) ?? []; + } + /** How an identifier reference touches its binding. */ private writeOf(id: ts.Identifier): "read" | "arith" | "unknown" | ts.Expression { let n: ts.Node = id; diff --git a/packages/compiler/src/frontend/lowering/narrowing-stability.ts b/packages/compiler/src/frontend/lowering/narrowing-stability.ts index 63e503221..d0c9b04a6 100644 --- a/packages/compiler/src/frontend/lowering/narrowing-stability.ts +++ b/packages/compiler/src/frontend/lowering/narrowing-stability.ts @@ -67,13 +67,12 @@ export class NarrowingStability { // Program-level queries before the program's files are split concern // declarations, never narrowed reads in function bodies. if (this.files === null) return null; - const position = widePosition(node); - if (position === null) return null; const cached = this.widened.get(node); if (cached !== undefined) return cached; + const position = widePosition(node); let result: ts.Type | null = null; try { - result = this.computeWidened(node, position); + if (position !== null) result = this.computeWidened(node, position); } finally { this.widened.set(node, result); } @@ -916,6 +915,15 @@ const INERT_CONSTRUCTORS = new Set([ "BigUint64Array", ]); +/** The receiver whose type widenedType reads for `node`: a property read in + * a slot that accepts its declared type (an argument, a return, a typed + * binding or literal). Whole-program analyses batch these receivers. */ +export function widenedReceiver(node: ts.Node): ts.Expression | null { + return ts.isPropertyAccessExpression(node) && widePosition(node) === "context" + ? node.expression + : null; +} + /** Where a stale narrowed read can keep its declared representation: the * receiver of `?.`, an absence/type/truthiness test, or a slot whose * contextual type may accept the declared type. */ diff --git a/packages/compiler/src/frontend/lowering/surfaces.ts b/packages/compiler/src/frontend/lowering/surfaces.ts index cf0428b60..3520e3aea 100644 --- a/packages/compiler/src/frontend/lowering/surfaces.ts +++ b/packages/compiler/src/frontend/lowering/surfaces.ts @@ -2223,8 +2223,8 @@ export const BUILTIN_MODULE_FN_ALIASES: Record< mkdirSync: ["fs.mkdirModeSync", "fs.mkdirRecursiveSync", "fs.mkdirRecursiveModeSync"], // The { recursive, force } and maxRetries/retryDelay option lowerings. rmSync: ["fs.rmOptsSync", "fs.rmRetrySync"], - // The { withFileTypes: true } Dirent form. - readdirSync: ["fs.readdirTypesSync"], + // The { withFileTypes: true } Dirent form and the { recursive: true } walk. + readdirSync: ["fs.readdirTypesSync", "fs.readdirRecursiveSync"], // The { throwIfNoEntry } probe form. statSync: ["fs.statProbeSync"], // realpathSync and realpathSync.native read as function values. diff --git a/packages/compiler/src/frontend/lowering/type-failure.ts b/packages/compiler/src/frontend/lowering/type-failure.ts new file mode 100644 index 000000000..3502a9bbf --- /dev/null +++ b/packages/compiler/src/frontend/lowering/type-failure.ts @@ -0,0 +1,195 @@ +/* Localized failure, stage L0 (report-only). + * + * A type with no static lowering poisons the slot that holds it (a field, + * a parameter, a return, a local), and today the poison rejects the whole + * declaration around the slot: a class, a signature, a declaration + * statement. Every later use of that declaration then fails on its own + * line. The localized-failure design turns such slots into checked + * dynamic slots at the points where the build would otherwise commit the + * error. + * + * This module holds the bookkeeping that stage measures without changing + * any lowering: where a failing type sits (its slot), whether a dynamic + * slot would keep the program's meaning (design §3.7), and the keys that + * tie a cascade diagnostic to the rejected declaration it inherits from. + * The data rides the committed diagnostics as report-only fields + * (ScrDiagnostic.typeFailure, .rejects, .cause); tier-analysis.ts turns it + * into the would-be dynamic slot report of `--explain-dynamic`. */ +import type { + TypeFailure, + TypeFailureFamily, + TypeFailureSlot, +} from "../../diagnostics/diagnostic.js"; +import * as ts from "../ts7/adapter.js"; + +/** Families a checked-dynamic slot can hold (design §3.7). The others + * name values the compiler cannot produce (opaque stdlib and @types/node + * types, engine-backed ambient types), package value types that scope B + * routes to Node islands, enum objects with no runtime value, and generic + * implementations without an erased instance. */ +const DEGRADABLE_FAMILIES: ReadonlySet = new Set([ + "index-signature", + "overloads", + "intersection", + "component", + "record-member", + "unsupported", + "dyn-field-policy", +]); + +function nameText(name: ts.Node | undefined): string | null { + if (name === undefined) return null; + if (ts.isIdentifier(name) || ts.isPrivateIdentifier(name)) return name.text; + if (ts.isStringLiteral(name) || ts.isNumericLiteral(name)) return name.text; + return ts.getTextOfNode(name); +} + +function isParameterProperty(param: ts.ParameterDeclaration): boolean { + return ( + ts.isConstructorDeclaration(param.parent) && + (param.modifiers?.nodes ?? []).some( + (m) => + m.kind === ts.SyntaxKind.PublicKeyword || + m.kind === ts.SyntaxKind.PrivateKeyword || + m.kind === ts.SyntaxKind.ProtectedKeyword || + m.kind === ts.SyntaxKind.ReadonlyKeyword, + ) + ); +} + +function variableSlot(decl: ts.VariableDeclaration): TypeFailureSlot { + const list = decl.parent; + const holder = list?.parent; + if ( + holder !== undefined && + (ts.isForOfStatement(holder) || ts.isForInStatement(holder) || ts.isForStatement(holder)) + ) + return "loop"; + if (holder !== undefined && ts.isVariableStatement(holder) && ts.isSourceFile(holder.parent)) + return "global"; + return "local"; +} + +function functionLabel(fn: ts.Node): string { + if (ts.isConstructorDeclaration(fn)) return "constructor"; + if (ts.isFunctionLikeDeclaration(fn)) { + const own = nameText(fn.name); + if (own !== null) return own; + const parent = fn.parent; + if ( + parent !== undefined && + (ts.isVariableDeclaration(parent) || + ts.isPropertyAssignment(parent) || + ts.isPropertyDeclaration(parent)) + ) { + const bound = nameText(parent.name); + if (bound !== null) return bound; + } + } + return ""; +} + +/** The slot a badType blame node denotes. A declaration's own node, its + * name and its type annotation name the declaration's slot; anything + * reached through an expression is an expression slot (the type of a + * cast, a literal, a temporary). */ +export function typeFailureSlot(node: ts.Node): { slot: TypeFailureSlot; name: string } { + let current: ts.Node = node; + for (let depth = 0; depth < 64; depth++) { + if (ts.isParameter(current)) { + const name = nameText(current.name) ?? ""; + return { slot: isParameterProperty(current) ? "field" : "param", name }; + } + if (ts.isPropertyDeclaration(current)) + return { slot: "field", name: nameText(current.name) ?? "" }; + if (ts.isVariableDeclaration(current)) + return { slot: variableSlot(current), name: nameText(current.name) ?? "" }; + if (ts.isBindingElement(current)) { + let owner: ts.Node | undefined = current.parent; + while (owner !== undefined && !ts.isVariableDeclaration(owner) && !ts.isParameter(owner)) + owner = owner.parent; + const slot: TypeFailureSlot = + owner === undefined + ? "local" + : ts.isParameter(owner) + ? "param" + : variableSlot(owner as ts.VariableDeclaration); + return { slot, name: nameText(current.name) ?? "" }; + } + if (ts.isPropertySignature(current) || ts.isMethodSignatureDeclaration(current)) + return { slot: "member", name: nameText(current.name) ?? "" }; + if (ts.isFunctionLikeDeclaration(current)) + return { slot: "return", name: functionLabel(current) }; + if (ts.isCatchClause(current)) return { slot: "local", name: "" }; + const parent: ts.Node | undefined = current.parent; + if (parent === undefined || ts.isSourceFile(parent)) break; + // Climb from a declaration's name or type annotation, or from inside a + // type node; any other position is an expression's own type. + const declared = + (ts.isParameter(parent) || + ts.isPropertyDeclaration(parent) || + ts.isVariableDeclaration(parent) || + ts.isBindingElement(parent) || + ts.isPropertySignature(parent) || + ts.isMethodSignatureDeclaration(parent) || + ts.isFunctionLikeDeclaration(parent)) && + ((parent as { name?: ts.Node }).name === current || + (parent as { type?: ts.Node }).type === current); + if (!declared && !ts.isTypeNode(current) && !ts.isTypeNode(parent)) break; + current = parent; + } + const text = ts.getTextOfNode(node); + return { slot: "expression", name: text.length > 60 ? `${text.slice(0, 57)}...` : text }; +} + +/** Program code outside node_modules, in TypeScript: the code holes apply + * to (design §3.3, §3.8). JavaScript keeps its dynamic fallback and its + * runtime fences; npm packages keep island routing. */ +function programTypeScript(node: ts.Node): boolean { + const sf = ts.getSourceFileOfNode(node); + const file = sf.fileName; + if (file.includes("/node_modules/")) return false; + if (sf.isDeclarationFile) return false; + return !/\.(?:[cm]?js|jsx)$/.test(file); +} + +export function typeFailureOf( + node: ts.Node, + declaredType: string, + family: TypeFailureFamily, +): TypeFailure { + const { slot, name } = typeFailureSlot(node); + return { + slot, + name, + declaredType, + family, + degradable: DEGRADABLE_FAMILIES.has(family) && programTypeScript(node), + }; +} + +function locKey(node: ts.Node): string { + const sf = ts.getSourceFileOfNode(node); + return `${sf.fileName}:${ts.tokenStartOfNode(node)}`; +} + +/** The failure-group key of a deferred declaration (a class shape or a + * function signature whose collection poisoned). */ +export function declarationGroupKey(symbol: ts.Symbol): string { + const decl = symbol.declarations[0]; + const name = ts.symbolName(symbol); + if (decl === undefined) return `declaration ${name}`; + const kind = + ts.isClassDeclaration(decl) || ts.isClassExpression(decl) + ? "class" + : ts.isFunctionLikeDeclaration(decl) + ? "function" + : "declaration"; + return `${kind} ${name} @${locKey(decl)}`; +} + +/** The failure-group key of a poisoned declaration statement whose + * bindings are blocked. */ +export function statementGroupKey(stmt: ts.Statement): string { + return `statement @${locKey(stmt)}`; +} diff --git a/packages/compiler/src/frontend/lowering/typed-arguments.ts b/packages/compiler/src/frontend/lowering/typed-arguments.ts index 2fbd83482..e837d59e6 100644 --- a/packages/compiler/src/frontend/lowering/typed-arguments.ts +++ b/packages/compiler/src/frontend/lowering/typed-arguments.ts @@ -2,7 +2,15 @@ import * as ts from "../ts7/adapter.js"; import { bodyReadsArguments } from "../arguments-usage.js"; import { isJsSourceFile, locOf } from "../program.js"; import { restTupleElementTypes, unitOnlyUnion } from "../type-mapper.js"; -import { DYN, type IrExpr, type IrLocal, type IrStmt, type IrType, arrayOf } from "../../ir/ir.js"; +import { + DYN, + type IrExpr, + type IrLocal, + type IrParam, + type IrStmt, + type IrType, + arrayOf, +} from "../../ir/ir.js"; import type { Lowerer } from "./lowerer.js"; import { captureContextArguments } from "./function-context.js"; import { isThisParameter, type ParamShape } from "./call-signatures.js"; @@ -156,7 +164,7 @@ export function declareTypedArguments( lowerer: Lowerer, node: ts.FunctionExpression, shapes: readonly ParamShape[], - params: { localId: string; name: string; type: IrType }[], + params: IrParam[], prologue: IrStmt[], ): void { const loc = locOf(node); @@ -187,6 +195,15 @@ export function declareTypedArguments( }); } +/** The binding an identifier reads: its symbol, or for the name of a + * shorthand property (`{ xs }`) the variable whose value the property takes. */ +export function bindingSymbolOfRead(lowerer: Lowerer, id: ts.Identifier): ts.Symbol | undefined { + const parent = id.parent; + if (parent && ts.isShorthandPropertyAssignment(parent) && parent.name === id) + return lowerer.checker.getShorthandAssignmentValueSymbol(parent); + return lowerer.checker.getSymbolAtLocation(id); +} + /** Whether every read of a tuple binding in `fn` (identified by `isRoot`) * only forwards it: `f.apply(t, x)`, `g(...x)` into a program function, * `x.length`, or a const alias whose own reads only forward. Reads inside @@ -254,7 +271,7 @@ export function onlyForwarded( const alias = lowerer.checker.getSymbolAtLocation(p.name); const aliasName = p.name; if (alias) { - roots.push((id) => id !== aliasName && lowerer.checker.getSymbolAtLocation(id) === alias); + roots.push((id) => id !== aliasName && bindingSymbolOfRead(lowerer, id) === alias); return undefined; } } diff --git a/packages/compiler/src/frontend/lowering/union-narrow.ts b/packages/compiler/src/frontend/lowering/union-narrow.ts index 99cca9941..f9a858fb6 100644 --- a/packages/compiler/src/frontend/lowering/union-narrow.ts +++ b/packages/compiler/src/frontend/lowering/union-narrow.ts @@ -1,4 +1,5 @@ import { BOOL, STRING, isUnitType, typeEquals } from "../../ir/ir.js"; +import { renderOriginNote } from "./decl-origin.js"; import type { IrExpr, IrFunction, IrStmt, IrType, IrUnionDef, SrcLoc } from "../../ir/ir.js"; /** The TypeError message for a checked extraction that finds `arm`. */ @@ -35,6 +36,11 @@ export function buildUnionNarrow( /** Converts another arm's payload into the target (a record arm that is * a width subtype of a record target), or null to throw. */ convert: ((tag: number, payload: IrExpr) => IrExpr | null) | null = null, + /** The target's package-declared origin (decl-origin.ts): each arm's + * message names the declaration and the patch admitting that arm. */ + origin: string | null = null, + /** The program targets Windows (the patch command's quoting). */ + windows = false, ): IrFunction | null { if (isUnitType(target) || target.kind === "void") return null; const targetTag = from.arms.findIndex((arm) => typeEquals(arm, target)); @@ -87,7 +93,19 @@ export function buildUnionNarrow( args: [ { kind: "strLit", - value: failureMessage ?? unionNarrowFailureMessage(arm, formatType), + value: + (failureMessage ?? unionNarrowFailureMessage(arm, formatType)) + + (origin === null + ? "" + : renderOriginNote( + origin, + isUnitType(arm) + ? arm.kind === "undefinedT" + ? "undefined" + : "null" + : formatType(arm), + windows, + )), type: STRING, loc, }, diff --git a/packages/compiler/src/frontend/program-analysis.ts b/packages/compiler/src/frontend/program-analysis.ts index 475942908..b20f5d5e5 100644 --- a/packages/compiler/src/frontend/program-analysis.ts +++ b/packages/compiler/src/frontend/program-analysis.ts @@ -40,6 +40,12 @@ export class ProgramAnalysis { SourceFile, ReadonlyMap >(); + /** Per file: its identifiers by text, each with its position in the + * file's preorder walk (fileIdentifiers). */ + readonly fileIdentifiers = new Map< + SourceFile, + ReadonlyMap + >(); clear(): void { this.createRequireReasons.clear(); @@ -53,5 +59,6 @@ export class ProgramAnalysis { this.requireEarlyReferences.clear(); this.requireFirstRunnable.clear(); this.splitRequireBindings.clear(); + this.fileIdentifiers.clear(); } } diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 9c933e52f..a4ab89b43 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -141,8 +141,13 @@ import { typeOnlyReachedFiles, } from "./type-only-imports.js"; import { moduleSourceCandidates } from "./module-source-candidates.js"; -import { planDeclProjection, type DeclProjection } from "./decl-projection.js"; +import { + planDeclProjection, + resetDeclBoundaryOrigins, + type DeclProjection, +} from "./decl-projection.js"; import { configureTypePatches, packageSourceText } from "./type-patches/layer.js"; +import { resetDeclOriginVersions } from "./lowering/decl-origin.js"; import { reportShapes } from "./shape-inference/index.js"; import { inferredJsDiagnosticSuppressed } from "./inferred-js-diagnostics.js"; import { @@ -1111,6 +1116,8 @@ export function loadProgram( // JavaScript, even when named. A registry entry's `sourceVersions` // (chalk 6) compile from their own source. configurePackageShims(entryPath); + resetDeclBoundaryOrigins(); + resetDeclOriginVersions(); // The project's type patches (type-patches/layer.ts), read by every host // this load creates; nothing at all without `scriptc.typePatches`. configureTypePatches(entryPath, (path, text, kind) => services.parse(path, text, kind)); @@ -2569,24 +2576,36 @@ export function isCreateRequireBinding7(program: ts.Program, callee: ts.Identifi /** How a use of a createRequire loader lowers: * - "static": a literal call or a literal resolution query, which erases * the loader value. - * - "deferred": a call with a computed specifier or another argument shape, - * or `resolve` / `resolve.paths` read as a value, inside the try block - * of a try statement in the same function. The call throws a runtime - * fence, and the resolve value is a function that throws one when called - * (module-resolution.ts), so the statement's catch clause handles them - * (packages that look for optional native addons). - * - "computed": the same shapes outside a try block. + * - "runtime": in JavaScript source, a call with a computed specifier, a + * require.resolve call with a computed request, or `resolve` read as a + * value. The runtime resolves the specifier like Node + * (module-resolution.ts, scr_require.c). + * - "deferred": in JavaScript source, inside the try block of a try + * statement in the same function, the other shapes: spread arguments, + * require.resolve with another argument count, and `resolve.paths` with + * a computed request or read as a value. The call throws a runtime + * fence, and the resolve.paths value is a function that throws one when + * called (module-resolution.ts), so the statement's catch clause handles + * them (packages that look for optional native addons). + * - "computed": the "runtime" uses in TypeScript source, and the + * "deferred" shapes outside a try block. * - "escaping": the loader itself as a value, writes, optional calls, and * other members. */ const COMPUTED_CREATE_REQUIRE_REASON = - "its createRequire result is used with a computed specifier, or its resolve as a value, outside a try block (static requires need literal specifiers)"; + "its createRequire result is used with a computed specifier in TypeScript, or with spread arguments or resolve.paths outside a try block (static requires need literal specifiers)"; -type CreateRequireUse7 = "static" | "deferred" | "computed" | "escaping"; +type CreateRequireUse7 = "static" | "runtime" | "deferred" | "computed" | "escaping"; function createRequireUse7(expr: ts.Expression): CreateRequireUse7 { + const use = createRequireUseShape7(expr); + return use === "runtime" && !isJsSourceFile(ts.getSourceFileOfNode(expr)) ? "computed" : use; +} + +function createRequireUseShape7(expr: ts.Expression): CreateRequireUse7 { let use: ts.Node = expr; while (ts.isParenthesizedExpression(use.parent)) use = use.parent; let member = false; + let paths = false; if ( ts.isPropertyAccessExpression(use.parent) && use.parent.expression === use && @@ -2600,16 +2619,26 @@ function createRequireUse7(expr: ts.Expression): CreateRequireUse7 { use.parent.expression === use && use.parent.name.text === "paths" && !use.parent.questionDotToken - ) + ) { use = use.parent; + paths = true; + } } const call = use.parent; if (ts.isCallExpression(call) && call.expression === use) { if (call.questionDotToken) return "escaping"; - if (call.arguments.nodes.length === 1 && ts.isStringLiteralLike(call.arguments.nodes[0]!)) - return "static"; + const args = call.arguments.nodes; + if (args.length === 1 && ts.isStringLiteralLike(args[0]!)) return "static"; + if ( + !paths && + !args.some((arg) => ts.isSpreadElement(arg)) && + !(member && (args.length === 0 || args.length > 2)) + ) + return "runtime"; } else if (!member || isAssignmentTarget7(use)) { return "escaping"; + } else if (!paths) { + return "runtime"; } return insideTryBlock7(use) ? "deferred" : "computed"; } diff --git a/packages/compiler/src/frontend/tier-analysis.ts b/packages/compiler/src/frontend/tier-analysis.ts index b75e491a8..0baedd7dd 100644 --- a/packages/compiler/src/frontend/tier-analysis.ts +++ b/packages/compiler/src/frontend/tier-analysis.ts @@ -29,7 +29,12 @@ * Attribution is syntactic and approximate: the site's innermost node, the * operand it acts on, and that operand's declaration. Stage S6 replaces it * with requests the failing lowering names exactly. */ -import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import type { + ScrDiagnostic, + TypeFailureFamily, + TypeFailureSlot, +} from "../diagnostics/diagnostic.js"; +import { isWarningCode } from "../diagnostics/render.js"; import type { IrFunction, SrcLoc } from "../ir/ir.js"; import { dynamicOperationSites, type DynamicFunction } from "../ir/tiers.js"; import * as ts from "./ts7/adapter.js"; @@ -85,8 +90,70 @@ export interface TierDynamicFunction extends DynamicFunction { statements: number; } +/** The localized-failure stage whose slot positions include a slot kind + * (design §3.6): locals and globals degrade first, declarations next, + * members of declared types with push-down, expression types last. */ +export type LocalizedFailureStage = "L1" | "L2" | "L3" | "L4"; + +export function slotStage(slot: TypeFailureSlot): LocalizedFailureStage { + switch (slot) { + case "local": + case "global": + case "loop": + return "L1"; + case "field": + case "param": + case "return": + return "L2"; + case "member": + return "L3"; + case "expression": + return "L4"; + } +} + +/** A committed type failure that would become a checked dynamic slot + * (SC6104) instead of an error. */ +export interface WouldBeSlot { + loc: SrcLoc; + slot: TypeFailureSlot; + name: string; + declaredType: string; + family: TypeFailureFamily; + code: string; + message: string; + stage: LocalizedFailureStage; + /** The declaration the failure rejects today (a failure-group key), if + * any: a class, a signature, or a declaration statement. */ + rejects: string | null; + /** Other committed diagnostics that inherit from this slot's group and + * clear once every group they inherit from degrades (with push-down and + * expression slots, the full design). */ + cascade: number; +} + +/** Localized failure, report-only (stage L0): what the committed errors + * would become. Two scenarios: through stage L2 (declaration slots) and + * through L4 (members and expressions too). */ +export interface LocalizedFailureReport { + /** Reached errors (warnings excluded). */ + errors: number; + /** Type failures, by whether a checked dynamic slot keeps meaning. */ + typeFailures: { degradable: number; refused: number }; + slots: WouldBeSlot[]; + /** Diagnostics attributed to a rejected declaration they used. */ + cascades: number; + /** Errors that disappear (degraded slots plus cleared cascades). */ + clearedThroughL2: number; + clearedThroughL4: number; + /** The errors that remain through L4, by code. */ + remainingByCode: Record; +} + export interface TierReport { mode: TierMode; + /** Localized failure (report-only). */ + localizedFailure: LocalizedFailureReport; /** Reached source statements that perform a dynamic operation. */ statementsDynamic: number; dynamicFunctions: TierDynamicFunction[]; @@ -576,6 +643,119 @@ function demotedUnits(requests: readonly DemotionRequest[]): DemotedUnit[] { ); } +/* ── localized failure ───────────────────────────────────────────────── */ + +/** A diagnostic's failure-group bookkeeping (lowerer: badType, deferred + * flushes, blocked statements) turned into the would-be report. A group + * (a rejected declaration or blocked statement) clears in a scenario when + * each of its diagnostics either degrades (a degradable type failure at a + * slot the scenario covers) or is a cascade whose every cause clears. */ +export function localizedFailureReport(reached: readonly ScrDiagnostic[]): LocalizedFailureReport { + const errors = reached.filter((d) => !isWarningCode(d.code)); + const groups = new Map(); + for (const d of errors) { + if (d.rejects === undefined) continue; + const list = groups.get(d.rejects); + if (list === undefined) groups.set(d.rejects, [d]); + else list.push(d); + } + const stages: Record<"L2" | "L4", ReadonlySet> = { + L2: new Set(["L1", "L2"]), + L4: new Set(["L1", "L2", "L3", "L4"]), + }; + const clears = (scenario: "L2" | "L4") => { + const covered = stages[scenario]; + const memo = new Map(); + const degrades = (d: ScrDiagnostic): boolean => + d.typeFailure !== undefined && + d.typeFailure.degradable && + covered.has(slotStage(d.typeFailure.slot)); + const causeClears = (d: ScrDiagnostic): boolean => + d.cause !== undefined && d.cause.length > 0 && d.cause.every((g) => groupClears(g)); + const groupClears = (group: string): boolean => { + const known = memo.get(group); + if (known !== undefined) return known; + memo.set(group, false); // a cycle never clears + const members = groups.get(group) ?? []; + const ok = members.length > 0 && members.every((d) => degrades(d) || causeClears(d)); + memo.set(group, ok); + return ok; + }; + return { degrades, causeClears, groupClears }; + }; + const l2 = clears("L2"); + const l4 = clears("L4"); + const cleared = (c: ReturnType, d: ScrDiagnostic) => + c.degrades(d) || c.causeClears(d); + // Each slot's cascade: diagnostics whose causes (transitively) include + // the slot's group and that clear in the full scenario. + const reachesGroup = (d: ScrDiagnostic, group: string, seen: Set): boolean => { + for (const g of d.cause ?? []) { + if (g === group) return true; + if (seen.has(g)) continue; + seen.add(g); + for (const m of groups.get(g) ?? []) if (reachesGroup(m, group, seen)) return true; + } + return false; + }; + const slots: WouldBeSlot[] = []; + let degradable = 0; + let refused = 0; + for (const d of errors) { + const f = d.typeFailure; + if (f === undefined) continue; + if (!f.degradable) { + refused++; + continue; + } + degradable++; + const group = d.rejects ?? null; + let cascade = 0; + if (group !== null) + for (const e of errors) + if ( + e !== d && + e.typeFailure === undefined && + cleared(l4, e) && + reachesGroup(e, group, new Set()) + ) + cascade++; + slots.push({ + loc: d.loc, + slot: f.slot, + name: f.name, + declaredType: f.declaredType, + family: f.family, + code: d.code, + message: d.message, + stage: slotStage(f.slot), + rejects: group, + cascade, + }); + } + slots.sort( + (a, b) => + b.cascade - a.cascade || a.loc.file.localeCompare(b.loc.file) || a.loc.start - b.loc.start, + ); + const remainingByCode: Record = {}; + let clearedThroughL2 = 0; + let clearedThroughL4 = 0; + for (const d of errors) { + if (cleared(l2, d)) clearedThroughL2++; + if (cleared(l4, d)) clearedThroughL4++; + else remainingByCode[d.code] = (remainingByCode[d.code] ?? 0) + 1; + } + return { + errors: errors.length, + typeFailures: { degradable, refused }, + slots, + cascades: errors.filter((d) => d.cause !== undefined && d.typeFailure === undefined).length, + clearedThroughL2, + clearedThroughL4, + remainingByCode, + }; +} + export interface TierAnalysisInput { mode: TierMode; /** The dynamic-tier IR functions of the reached program. */ @@ -624,6 +804,7 @@ export function analyzeTiers(program: ts.Program, input: TierAnalysisInput): Tie const dynamic = dynamicFunctionReports(program, input.dynamicFunctions); return { mode: input.mode, + localizedFailure: localizedFailureReport(input.reached), statementsDynamic: dynamic.statements, dynamicFunctions: dynamic.functions, requests, diff --git a/packages/compiler/src/frontend/ts7/ast-node.test.ts b/packages/compiler/src/frontend/ts7/ast-node.test.ts index 8b391e622..b650e7725 100644 --- a/packages/compiler/src/frontend/ts7/ast-node.test.ts +++ b/packages/compiler/src/frontend/ts7/ast-node.test.ts @@ -473,6 +473,29 @@ test("named child access shares node and list identities without materializing s expect(() => file.namedChild(another, "name")).toThrow("another source file"); }); +test("named child getters and text keep answering the first read", () => { + let materialized = 0; + const file = new AstFile(decoded.get("main.ts")!.bytes, () => { + materialized++; + }); + let index = 1; + while (file.wire.kind(index) !== AstKind.FunctionDeclaration) index++; + const node = file.node(index); + const name = node.name; + expect(name?.index).toBe(file.wire.namedChild(index, "name")); + const after = materialized; + expect(node.name).toBe(name); + expect(node.childNode("name")).toBe(name); + expect(file.node(name!.index)).toBe(name); + expect(node.typeParameters).toBeUndefined(); + expect(materialized).toBe(after); + const text = name!.text; + expect(text).toBe(file.wire.text(name!.index)); + expect(name!.text).toBe(text); + expect(node.text).toBeUndefined(); + expect(node.text).toBeUndefined(); +}); + test("warming a named child preserves lazy failures in later malformed sibling links", () => { const bytes = decoded.get("main.ts")!.bytes; const wire = new AstFile(bytes).wire; diff --git a/packages/compiler/src/frontend/ts7/ast-node.ts b/packages/compiler/src/frontend/ts7/ast-node.ts index 4c7726c00..c49dceffb 100644 --- a/packages/compiler/src/frontend/ts7/ast-node.ts +++ b/packages/compiler/src/frontend/ts7/ast-node.ts @@ -83,6 +83,9 @@ export interface AstFileOptions { writeTargets?: boolean; } +const NO_CHILD_INDICES = new Uint32Array(0); +const NO_CHILD_NODES: AstNode[] = []; + export class AstFile { readonly wire: AstWireFile; readonly root: AstNode; @@ -94,7 +97,9 @@ export class AstFile { private readonly structuredNodes = new Map(); private readonly strings = new Map(); private lines: number[] | undefined; - private childIndices: Uint32Array | undefined; + /** Resolved named children, eight slots per node; empty until the first + * named child is read. */ + private childIndices: Uint32Array = NO_CHILD_INDICES; /** The root source file's name (AstNode.fileName). */ private rootFileName: string | undefined; @@ -164,8 +169,8 @@ export class AstFile { const mask = node.data >>> 30 === 0 ? node.data & 0xff : 0xff; if ((mask & (1 << order)) === 0) return 0; const slot = node.index * 8 + order; - const indices = this.childIndices; - if (indices !== undefined) { + let indices = this.childIndices; + if (indices.length !== 0) { const cached = indices[slot]!; if (cached !== 0) return cached; } @@ -173,9 +178,11 @@ export class AstFile { // The pinned wire has at most eight named children per node. One packed // table avoids per-node arrays; allocate it only after successful access // so malformed siblings still fail lazily in the checked wire decoder. - if (this.childIndices === undefined) - this.childIndices = new Uint32Array(this.wire.nodeCount * 8); - this.childIndices[slot] = index; + if (indices.length === 0) { + indices = new Uint32Array(this.wire.nodeCount * 8); + this.childIndices = indices; + } + indices[slot] = index; return index; } @@ -320,6 +327,13 @@ export class AstNode { readonly data: number; private parentResolved = false; private parentCache: AstNode | undefined; + /** The node's text, read once: the frontend asks the same identifiers + * for their text many times. */ + private textResolved = false; + private textCache: string | undefined; + /** Named child nodes by wire slot, filled as they are read; empty until + * the first one is. Slots the child mask leaves empty are never stored. */ + private childNodes: AstNode[] = NO_CHILD_NODES; constructor( readonly file: AstFile, @@ -351,7 +365,11 @@ export class AstNode { return this.parentCache; } get text(): string | undefined { - return this.file.wire.text(this.index); + if (!this.textResolved) { + this.textCache = this.file.wire.text(this.index); + this.textResolved = true; + } + return this.textCache; } get rawText(): string | undefined { return this.file.wire.rawText(this.index); @@ -363,8 +381,21 @@ export class AstNode { } /** childNode by wire slot, as the generated getters select it. */ childNodeAt(order: number): AstNode | undefined { + let nodes = this.childNodes; + if (order >= 0 && order < nodes.length) { + const cached = nodes[order]; + if (cached !== undefined) return cached; + } const index = this.file.namedChildAt(this, order); - return index === 0 ? undefined : this.file.node(index); + if (index === 0) return undefined; + const child = this.file.node(index); + if (nodes.length === 0) { + const mask = this.data >>> 30 === 0 ? this.data & 0xff : 0xff; + nodes = new Array(32 - Math.clz32(mask)); + this.childNodes = nodes; + } + nodes[order] = child; + return child; } childListAt(order: number): AstNodeList | undefined { const index = this.file.namedChildAt(this, order); diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts index 83abaeddc..6d0c0651d 100644 --- a/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts @@ -4,10 +4,14 @@ import { join } from "node:path"; import { afterAll, afterEach, expect, test } from "vitest"; import type { SourceFile } from "./ast-types.js"; import { + prebuiltLibDiagnosticsText, + readPrebuiltLibDiagnostics, + recordPrebuiltLibDiagnostics, reusedKeptLibDiagnostics, reusedLibDiagnostics, setLibDiagnosticsStore, type LibDiagnosticsStore, + type PrebuiltLibDiagnostics, } from "./lib-diagnostics.js"; import type { Ts7DiagnosticData } from "./session-schema.generated.js"; import type { Ts7CheckedFileDiagnosticData, Ts7SessionProgram } from "./session.js"; @@ -24,7 +28,10 @@ const main = join(dir, "main.ts"); const util = join(dir, "\u{1F600}.ts"); const other = join(dir, "�.ts"); -afterEach(() => setLibDiagnosticsStore(null)); +afterEach(() => { + setLibDiagnosticsStore(null); + recordPrebuiltLibDiagnostics(null); +}); afterAll(() => { closeSourceParser(); rmSync(dir, { recursive: true, force: true }); @@ -309,3 +316,69 @@ test("a script that uses an answer stored without global names collects and stor expect(store.entries.size).toBe(1); expect(JSON.parse([...store.entries.values()][0]!).globals).not.toBeNull(); }); + +/** The content keys a full check of the program records, with checker release `checker`. */ +function recorded( + checker: string, + semantic: Map, +): PrebuiltLibDiagnostics { + const keys = new Set(); + recordPrebuiltLibDiagnostics({ checker, record: (key) => keys.add(key) }); + setLibDiagnosticsStore(memoryStore(), "build-time checker"); + reusedLibDiagnostics(sessionProgram({ semantic }), names, supplied, moduleFile); + recordPrebuiltLibDiagnostics(null); + return { checker, keys }; +} + +test("a prebuilt answer stands in for checking the supplied files in an empty store", () => { + const semantic = new Map([[main, [diagnostic(main, 5)]]]); + const prebuilt = recorded("1.0.0", semantic); + expect(prebuilt.keys.size).toBe(1); + // Another installation: its own store, its own checker identity. + const store = memoryStore(); + setLibDiagnosticsStore(store, "installed checker", () => prebuilt); + const program = sessionProgram({ semantic }); + expect(reusedLibDiagnostics(program, names, supplied, moduleFile)).toEqual({ + semantic: [diagnostic(main, 5)], + global: [], + }); + expect(program.asked).toEqual([`checked ${main}`]); + // The answer is stored, so the next build reads it from the store. + expect(store.entries.size).toBe(1); + const next = sessionProgram({ semantic }); + setLibDiagnosticsStore(store, "installed checker", () => { + throw new Error("the store answers first"); + }); + reusedLibDiagnostics(next, names, supplied, moduleFile); + expect(next.asked).toEqual([`checked ${main}`]); +}); + +test("prebuilt answers apply only to the same checker release and file contents", () => { + const semantic = new Map([[main, []]]); + const prebuilt = recorded("1.0.0", semantic); + const other: PrebuiltLibDiagnostics = { checker: "1.0.1", keys: prebuilt.keys }; + setLibDiagnosticsStore(memoryStore(), "installed checker", () => other); + const release = sessionProgram({ semantic }); + reusedLibDiagnostics(release, names, supplied, moduleFile); + expect(release.asked[0]).toBe("getGlobalDiagnostics"); + writeFileSync(fallback, fallbackText + "// edited\n"); + setLibDiagnosticsStore(memoryStore(), "installed checker", () => prebuilt); + const edited = sessionProgram({ semantic }); + reusedLibDiagnostics(edited, names, supplied, moduleFile); + expect(edited.asked[0]).toBe("getGlobalDiagnostics"); + writeFileSync(fallback, fallbackText); +}); + +test("a full check that reports diagnostics in a supplied file records no prebuilt answer", () => { + const prebuilt = recorded("1.0.0", new Map([[lib, [diagnostic(lib, 1)]]])); + expect(prebuilt.keys.size).toBe(0); +}); + +test("prebuilt answer files round-trip, and a malformed one is no answer", () => { + const path = join(dir, "lib-diagnostics.json"); + writeFileSync(path, prebuiltLibDiagnosticsText("1.0.0", ["b", "a", "b"])); + expect(readPrebuiltLibDiagnostics(path)).toEqual({ checker: "1.0.0", keys: new Set(["a", "b"]) }); + writeFileSync(path, '{ "schema": "other", "checker": "1.0.0", "keys": [] }'); + expect(readPrebuiltLibDiagnostics(path)).toBeNull(); + expect(readPrebuiltLibDiagnostics(join(dir, "missing.json"))).toBeNull(); +}); diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts index c3e563fef..ca1f71fc2 100644 --- a/packages/compiler/src/frontend/ts7/lib-diagnostics.ts +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts @@ -17,9 +17,17 @@ * which a program of modules does not otherwise read (about 4 MB at the * peak of a small build). An answer stored by such a program leaves them * out; the first program with scripts that uses the answer collects them - * and stores them with it. */ + * and stores them with it. + * + * A native distribution also ships prebuilt answers: the content keys of + * the supplied file sets the checker reports no diagnostics in, recorded when + * the distribution was built. A build whose store has no answer uses a + * prebuilt one when the checker is the distribution's own and the files' + * content key is among them, and stores it, so builds with an empty cache + * skip the check of the supplied files too. */ import { createHash } from "node:crypto"; -import { statSync } from "node:fs"; +import { readFileSync, statSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; import type { Node, SourceFile } from "./ast-types.js"; import type { Ts7DiagnosticData } from "./session-schema.generated.js"; import type { Ts7SessionProgram } from "./session.js"; @@ -32,15 +40,108 @@ export interface LibDiagnosticsStore { write(key: string, text: string): void; } -const SCHEMA = "scriptc.lib-diagnostics.v2"; +/** The answers a distribution ships: the content keys (contentKey) of the + * supplied file sets whose full check reported no diagnostics with the + * checker release `checker`. */ +export interface PrebuiltLibDiagnostics { + checker: string; + keys: ReadonlySet; +} -let configured: { store: LibDiagnosticsStore; checker: string } | null = null; +const SCHEMA = "scriptc.lib-diagnostics.v2"; +const PREBUILT_SCHEMA = "scriptc.lib-diagnostics-prebuilt.v1"; + +let configured: { + store: LibDiagnosticsStore; + checker: string; + /** Read on the first store miss; null when there are none. */ + prebuilt: (() => PrebuiltLibDiagnostics | null) | null; +} | null = null; /** Has later builds of this process reuse the diagnostics of the supplied * declaration files through `store`, for the checker identified by - * `checker` (its executable's identity), or stop reusing them (null). */ -export function setLibDiagnosticsStore(store: LibDiagnosticsStore | null, checker = ""): void { - configured = store === null ? null : { store, checker }; + * `checker` (its executable's identity), or stop reusing them (null). + * `prebuilt` reads the answers the distribution ships for that checker. */ +export function setLibDiagnosticsStore( + store: LibDiagnosticsStore | null, + checker = "", + prebuilt: (() => PrebuiltLibDiagnostics | null) | null = null, +): void { + if (store === null) { + configured = null; + return; + } + let loaded: PrebuiltLibDiagnostics | null | undefined; + configured = { + store, + checker, + prebuilt: + prebuilt === null ? null : () => (loaded === undefined ? (loaded = prebuilt()) : loaded), + }; +} + +let prebuiltRecorder: { checker: string; record: (key: string) => void } | null = null; + +/** Has later full checks that report no diagnostics in the supplied files + * pass their content key (contentKey, for the checker release `checker`) to + * `record`, or stop (null). A distribution build collects its prebuilt + * answers this way. */ +export function recordPrebuiltLibDiagnostics( + recorder: { checker: string; record: (key: string) => void } | null, +): void { + prebuiltRecorder = recorder; +} + +/** Reads a prebuilt answers file (`{ schema, checker, keys }`); null when + * it is missing or malformed. */ +export function readPrebuiltLibDiagnostics(path: string): PrebuiltLibDiagnostics | null { + try { + const value = JSON.parse(readFileSync(path, "utf8")) as { + schema?: unknown; + checker?: unknown; + keys?: unknown; + }; + if ( + value.schema !== PREBUILT_SCHEMA || + typeof value.checker !== "string" || + !Array.isArray(value.keys) || + !value.keys.every((key) => typeof key === "string") + ) + return null; + return { checker: value.checker, keys: new Set(value.keys as string[]) }; + } catch { + return null; + } +} + +/** A distribution's prebuilt answers at `path`, for its tsc-ts at `checker` + * (`/lib/tsc-ts`): null unless their checker release is the + * version of the package the executable belongs to. */ +export function distributionPrebuiltLibDiagnostics( + path: string, + checker: string, +): PrebuiltLibDiagnostics | null { + const prebuilt = readPrebuiltLibDiagnostics(path); + return prebuilt !== null && checkerPackageVersion(checker) === prebuilt.checker ? prebuilt : null; +} + +/** The version of the tsc-ts package whose executable is `checker` + * (`/lib/tsc-ts`), or null when it cannot be read. */ +export function checkerPackageVersion(checker: string): string | null { + try { + const pkg = JSON.parse( + readFileSync(join(dirname(dirname(checker)), "package.json"), "utf8"), + ) as { version?: unknown }; + return typeof pkg.version === "string" ? pkg.version : null; + } catch { + return null; + } +} + +/** The text of a prebuilt answers file holding `keys` for the checker + * release `checker`. */ +export function prebuiltLibDiagnosticsText(checker: string, keys: Iterable): string { + return `${JSON.stringify({ schema: PREBUILT_SCHEMA, checker, keys: [...new Set(keys)].sort() }, null, 1)}\n`; } /** A checker executable's identity for the stored diagnostics' key: its @@ -120,7 +221,7 @@ function reuse( const options = { ...program.getCompilerOptions(), configFilePath: undefined }; const key = entryKey(checker, options, libs); if (key === null) return null; - const stored = readEntry(store, key, libs); + const stored = readEntry(store, key, libs) ?? prebuiltEntry(store, key, options, libs); // An answer stored without the supplied files' global names: a program // with scripts collects them and stores them with the answer. let globals = stored?.globals ?? null; @@ -152,6 +253,10 @@ function reuse( const files: Record = {}; for (const name of libs) files[name] = program.diagnosticData("getSemanticDiagnostics", name); writeEntry(store, key, names, files); + if (prebuiltRecorder !== null && Object.values(files).every((list) => list.length === 0)) { + const content = contentKey(prebuiltRecorder.checker, options, libs); + if (content !== null) prebuiltRecorder.record(content); + } } } return { semantic, global }; @@ -330,6 +435,42 @@ function entryKey(checker: string, options: unknown, libs: readonly string[]): s .digest("hex"); } +/** The supplied files' answer from the distribution's prebuilt answers, + * stored under `key` for later builds: every file without diagnostics, when + * they hold the files' content key; null otherwise. */ +function prebuiltEntry( + store: LibDiagnosticsStore, + key: string, + options: unknown, + libs: readonly string[], +): { files: Map; globals: Set | null } | null { + const prebuilt = configured?.prebuilt?.() ?? null; + if (prebuilt === null || prebuilt.keys.size === 0) return null; + const content = contentKey(prebuilt.checker, options, libs); + if (content === null || !prebuilt.keys.has(content)) return null; + const files: Record = {}; + for (const name of libs) files[name] = []; + writeEntry(store, key, null, files); + return { files: new Map(Object.entries(files)), globals: null }; +} + +/** A supplied file set's key independent of where the files are installed: + * the checker release, the compiler options and each file's name and + * content digest, in program order; null when a file cannot be read. */ +function contentKey(checker: string, options: unknown, libs: readonly string[]): string | null { + const files: [string, string][] = []; + for (const name of libs) { + try { + files.push([basename(name), createHash("sha256").update(readFileSync(name)).digest("hex")]); + } catch { + return null; + } + } + return createHash("sha256") + .update(JSON.stringify({ schema: PREBUILT_SCHEMA, checker, options, files })) + .digest("hex"); +} + function readEntry( store: LibDiagnosticsStore, key: string, diff --git a/packages/compiler/src/frontend/tsc-codes.ts b/packages/compiler/src/frontend/tsc-codes.ts index a2831f1d1..a30945be7 100644 --- a/packages/compiler/src/frontend/tsc-codes.ts +++ b/packages/compiler/src/frontend/tsc-codes.ts @@ -160,7 +160,12 @@ export const JS_ANY_OPERATOR_CODES: ReadonlySet = new Set([2362, 2363, 2 * whose types come entirely from inference (checkJs + JSDoc) and whose JS * relaxation stance applies. */ export function isJsSourceFileName(fileName: string): boolean { - // Only the final few code units can match. Keep the same expression so - // unusual filenames retain its classification. - return /\.(js|mjs|cjs|jsx)$/.test(fileName.slice(-6)); + // The same classification as /\.(js|mjs|cjs|jsx)$/, without a regular + // expression: this runs for most type queries. + return ( + fileName.endsWith(".js") || + fileName.endsWith(".mjs") || + fileName.endsWith(".cjs") || + fileName.endsWith(".jsx") + ); } diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 4b1bf2a42..59ae4887d 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1282,6 +1282,11 @@ export interface TypeMapperCtx { * checker type its signature instantiates, or null. Consulted only when * the symbolic form does not map. */ resolveBoundAliasTs?: TypeParamTsResolver; + /** Localized failure: the lenient mapping of a type whose strict mapping + * failed at a committed slot (Lowerer.degradedTypes), or null. Consulted + * only after the strict mapping (and its bound-alias retry) returns null, + * at every depth, so every occurrence of a degraded type maps alike. */ + degradedType?: (type: ts.Type) => IrType | null; /** GENERIC program classes (monomorphization by flow): the mapping of a * concrete instantiation reference (`Box`) — the Lowerer * registers/reuses the instantiation (`Box%0`) and answers its object @@ -1856,6 +1861,16 @@ export function mapType(type: ts.Type, ctx: TypeMapperCtx): IrType | null { if (mapped !== null) contextResolutions++; } } + // Localized failure: a type that failed at a committed slot maps to its + // lenient form. It counts as a context resolution so neither it nor a + // type containing it is memoized: the strict form may map elsewhere. + if (mapped === null && ctx.degradedType) { + const lenient = ctx.degradedType(type); + if (lenient !== null) { + mapped = lenient; + contextResolutions++; + } + } if ( mapped !== null && memoKey !== null && @@ -5297,10 +5312,12 @@ function mapCallSignature(sig: ts.Signature, ctx: TypeMapperCtx): IrType | null // A parameter a TypeScript program types `any` takes whatever the // caller passes: the checked-dynamic slot, as an `any`-typed binding // already is (dynFallbackType). Declaration files keep their paths. - // Only function declarations, spelled function types and lambda - // parameters whose default is the only source of their type (`(n = - // lookup() ?? 5) => ...` with an `any` lookup): other lambdas and - // methods in object literals keep their existing checked-dynamic story. + // Only function declarations, spelled function types, method and call + // signatures of interfaces and type literals (`execute(params: any): + // Promise`), and lambda parameters whose default is the only source + // of their type (`(n = lookup() ?? 5) => ...` with an `any` lookup): + // other lambdas and methods in object literals keep their existing + // checked-dynamic story. if ( !pt && !rest && @@ -5309,6 +5326,8 @@ function mapCallSignature(sig: ts.Signature, ctx: TypeMapperCtx): IrType | null decl.parent !== undefined && (ts.isFunctionDeclaration(decl.parent) || ts.isFunctionTypeNode(decl.parent) || + ts.isMethodSignatureDeclaration(decl.parent) || + ts.isCallSignatureDeclaration(decl.parent) || ((ts.isArrowFunction(decl.parent) || ts.isFunctionExpression(decl.parent)) && decl.initializer !== undefined && decl.type === undefined && diff --git a/packages/compiler/src/frontend/type-patches/command.ts b/packages/compiler/src/frontend/type-patches/command.ts new file mode 100644 index 000000000..dfa05ea1e --- /dev/null +++ b/packages/compiler/src/frontend/type-patches/command.ts @@ -0,0 +1,413 @@ +/* `scriptc types patch` and `scriptc types list`: write and inspect the + * project's type patches (package.json scriptc.typePatches). + * + * scriptc types patch [--range ] [--dry-run] + * scriptc types list + * + * `patch` finds the installed package from the working directory, shows + * every declaration the selector matches (in the package's declaration + * files, its JSDoc and its @types package) with its type before and after, + * and adds the patch to the nearest package.json under a range covering + * the installed version (`^` unless --range says otherwise, or an + * existing entry's range when one covers it). Nothing is written when the + * selector matches nothing. `list` shows each configured patch, the + * installed copies it applies to and what it matches. */ + +import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { dirname, join, relative } from "node:path"; +import { parseProjectConfig, type TypePatchEntry } from "../../project-config.js"; +import type { TypePatchParse } from "./layer.js"; +import { patchFile, type PatchMatch } from "./locate.js"; +import { parseSelector, parseTypeText, type Selector, type TypePatchOp } from "./selector.js"; +import { isValidRange, satisfies } from "./semver.js"; + +export type TypePatchCommandRequest = + | { + action: "patch"; + cwd: string; + pkg: string; + selector: string; + type: string; + range: string | null; + dryRun: boolean; + } + | { action: "list"; cwd: string }; + +export interface TypePatchCommandResult { + stdout: string; + stderr: string; + exitCode: number; +} + +interface PackageCopy { + root: string; + version: string | null; + /** True for an `@types/` twin. */ + twin: boolean; +} + +const PATCHABLE = /\.(?:[cm]?js|d\.[cm]?ts|[cm]?ts)$/; + +function nearestManifest(cwd: string): string | null { + for (let dir = cwd; ;) { + const manifest = join(dir, "package.json"); + if (existsSync(manifest)) return manifest; + const parent = dirname(dir); + if (parent === dir) return null; + dir = parent; + } +} + +function versionOf(root: string): string | null { + try { + const parsed = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as { + version?: unknown; + }; + return typeof parsed.version === "string" ? parsed.version : null; + } catch { + return null; + } +} + +function typesName(pkg: string): string { + return pkg.startsWith("@") ? `@types/${pkg.slice(1).replace("/", "__")}` : `@types/${pkg}`; +} + +/** The copy of `pkg` (and of its @types twin) Node resolution finds from + * `cwd`. A twin is versioned by the runtime package beside it. */ +function installedCopies(cwd: string, pkg: string): PackageCopy[] { + const out: PackageCopy[] = []; + let runtimeVersion: string | null = null; + let twinFound = false; + for (let dir = cwd; ;) { + const modules = join(dir, "node_modules"); + if (runtimeVersion === null && existsSync(join(modules, pkg, "package.json"))) { + const root = join(modules, pkg); + runtimeVersion = versionOf(root); + out.push({ root, version: runtimeVersion, twin: false }); + } + if (!twinFound && existsSync(join(modules, typesName(pkg), "package.json"))) { + twinFound = true; + out.push({ root: join(modules, typesName(pkg)), version: null, twin: true }); + } + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + for (const copy of out) if (copy.twin) copy.version = runtimeVersion ?? versionOf(copy.root); + return out; +} + +/** The files of a package copy a patch can apply to (nested packages + * excluded), package-relative. */ +function packageFiles(root: string): string[] { + const out: string[] = []; + const walk = (dir: string): void => { + let names: string[]; + try { + names = readdirSync(dir).sort(); + } catch { + return; + } + for (const name of names) { + if (name === "node_modules" || name.startsWith(".")) continue; + const path = join(dir, name); + let isDir = false; + try { + isDir = statSync(path).isDirectory(); + } catch { + continue; + } + if (isDir) walk(path); + else if (PATCHABLE.test(name)) out.push(relative(root, path).split("\\").join("/")); + } + }; + walk(root); + return out; +} + +interface FileMatch extends PatchMatch { + file: string; +} + +/** Every declaration a selector matches in a copy, and the problems where + * it cannot apply (`always`: a problem even where the selector matched + * elsewhere, such as an ambiguous replacement, which leaves those + * declarations as they are). */ +function matchCopy( + copy: PackageCopy, + selector: Selector, + op: TypePatchOp, + parse: TypePatchParse, +): { matches: FileMatch[]; problems: { message: string; always: boolean }[] } { + const words = + selector.root === "export=" + ? ["exports", "export"] + : selector.root === "default" + ? ["default"] + : [selector.root]; + const matches: FileMatch[] = []; + const problems: { message: string; always: boolean }[] = []; + for (const rel of packageFiles(copy.root)) { + if (selector.file !== null && selector.file !== rel) continue; + const path = join(copy.root, rel); + let text: string; + try { + text = readFileSync(path, "utf8"); + } catch { + continue; + } + if (!words.some((w) => text.includes(w))) continue; + let result; + try { + result = patchFile(rel, text, parse(path, text, /\.[cm]?ts$/.test(rel) ? "ts" : "js"), [ + { id: 0, selector, op }, + ]); + } catch { + continue; + } + for (const m of result.matches) matches.push({ ...m, file: rel }); + for (const p of result.problems) + problems.push({ message: `${p.message} (${rel})`, always: p.always === true }); + } + return { matches, problems }; +} + +function shownCopy(copy: PackageCopy, base: string, pkg: string): string { + const name = copy.twin ? typesName(pkg) : pkg; + return `${name} ${copy.version ?? "(no version)"} (${relative(base, copy.root).split("\\").join("/")})`; +} + +function describe(m: FileMatch): string { + if (m.redundant) return ` ${m.file}:${m.line} ${m.before} (already declared so)`; + return ` ${m.file}:${m.line} ${m.before} -> ${m.after}`; +} + +/** The indentation a JSON text uses (two spaces when it cannot tell). */ +function indentOf(text: string): string { + const m = /^[{[]\s*\n([ \t]+)\S/.exec(text); + return m === null ? " " : m[1]!; +} + +function isObject(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +/** JSON text of `value` with `path` set to the string `leaf` (objects + * created along the way, the leaf replaced if present), keeping key order + * and indenting like JSON.stringify(value, null, indent). The parsed value + * itself is only read. */ +function renderJson( + value: unknown, + indent: string, + at: string, + path: readonly string[], + leaf: string, +): string { + const inner = at + indent; + if (path.length > 0 && !isObject(value)) return renderJson({}, indent, at, path, leaf); + if (isObject(value)) { + const keys = Object.keys(value); + const parts: string[] = []; + const head = path[0]; + for (const k of keys) { + const child = + head === k + ? path.length === 1 + ? JSON.stringify(leaf) + : renderJson(value[k], indent, inner, path.slice(1), leaf) + : renderJson(value[k], indent, inner, [], leaf); + parts.push(`${inner}${JSON.stringify(k)}: ${child}`); + } + if (head !== undefined && !keys.includes(head)) { + const child = + path.length === 1 + ? JSON.stringify(leaf) + : renderJson({}, indent, inner, path.slice(1), leaf); + parts.push(`${inner}${JSON.stringify(head)}: ${child}`); + } + return parts.length === 0 ? "{}" : `{\n${parts.join(",\n")}\n${at}}`; + } + if (Array.isArray(value)) { + const items: unknown[] = value; + if (items.length === 0) return "[]"; + return `[\n${items.map((item) => `${inner}${renderJson(item, indent, inner, [], leaf)}`).join(",\n")}\n${at}]`; + } + return JSON.stringify(value) ?? "null"; +} + +function patchCommand( + req: Extract, + parse: TypePatchParse, +): TypePatchCommandResult { + const fail = (message: string): TypePatchCommandResult => ({ + stdout: "", + stderr: `scriptc: ${message}\n`, + exitCode: 1, + }); + const selector = parseSelector(req.selector); + if ("error" in selector) return fail(`'${req.selector}' is not a selector: it ${selector.error}`); + const op = parseTypeText(req.type); + if ("error" in op) return fail(`the type '${req.type}' ${op.error}`); + if (req.range !== null && !isValidRange(req.range)) + return fail(`'${req.range}' is not a semver range`); + const manifest = nearestManifest(req.cwd); + if (manifest === null) return fail("no package.json found from the working directory"); + const base = dirname(manifest); + const copies = installedCopies(req.cwd, req.pkg); + const runtime = copies.find((c) => !c.twin) ?? copies[0]; + if (runtime === undefined) return fail(`${req.pkg} is not installed here`); + const version = runtime.version; + const lines: string[] = []; + let matched = 0; + let changed = 0; + const problems: { message: string; always: boolean }[] = []; + for (const copy of copies) { + const found = matchCopy(copy, selector, op, parse); + problems.push(...found.problems); + if (found.matches.length === 0) continue; + lines.push(shownCopy(copy, base, req.pkg)); + for (const m of found.matches) { + lines.push(describe(m)); + matched++; + if (!m.redundant) changed++; + } + } + if (matched === 0) { + const why = + problems.length > 0 + ? problems.map((p) => p.message).join("\n") + : `'${req.selector}' matches no declaration`; + return fail(`${why}\nnothing was written`); + } + // A declaration the patch cannot apply to where it also matched (an + // ambiguous replacement) would stay as it is. + const unapplied = problems.filter((p) => p.always); + if (unapplied.length > 0) + return fail(`${unapplied.map((p) => p.message).join("\n")}\nnothing was written`); + const text = readFileSync(manifest, "utf8"); + // The compiler applies every entry whose range covers the installed + // version, and a later entry wins on the same selector. Without a range, + // the patch goes to the last covering entry that patches the selector + // already, else to the first covering entry; with one, to the entry of + // that range. Otherwise a new entry, which comes last, takes it. + const entries = parseProjectConfig(manifest, text).typePatches.filter( + (e: TypePatchEntry) => e.pkg === req.pkg, + ); + const covers = (e: TypePatchEntry): boolean => version !== null && satisfies(version, e.range); + const declares = (e: TypePatchEntry): boolean => + e.declarations.some((d) => d.selector === req.selector); + // The declarations may say this already while a covering entry patches + // the selector to something else, which the build would keep using. + const overridden = entries.some( + (e) => + covers(e) && e.declarations.some((d) => d.selector === req.selector && d.type !== req.type), + ); + if (changed === 0 && !overridden) { + lines.push("the declarations already say this; nothing was written"); + return { stdout: `${lines.join("\n")}\n`, stderr: "", exitCode: 0 }; + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return fail(`${relative(req.cwd, manifest) || "package.json"} is not valid JSON`); + } + const existing = + req.range === null + ? (entries.findLast((e) => covers(e) && declares(e)) ?? entries.find(covers)) + : entries.find((e) => e.range === req.range); + if (existing !== undefined && covers(existing)) { + const later = entries + .slice(entries.indexOf(existing) + 1) + .find((e) => covers(e) && declares(e)); + if (later !== undefined) + return fail( + `scriptc.typePatches["${later.key}"] also patches '${req.selector}' for ${req.pkg} ${version} and takes precedence over scriptc.typePatches["${existing.key}"]\nnothing was written`, + ); + } + const key = + existing?.key ?? `${req.pkg}@${req.range ?? (version === null ? "*" : `^${version}`)}`; + const where = `scriptc.typePatches["${key}"].declarations["${req.selector}"]`; + if (req.dryRun) { + lines.push(`would write ${relative(req.cwd, manifest) || "package.json"} ${where}`); + return { stdout: `${lines.join("\n")}\n`, stderr: "", exitCode: 0 }; + } + const trailing = text.endsWith("\n") ? "\n" : ""; + writeFileSync( + manifest, + renderJson( + parsed, + indentOf(text), + "", + ["scriptc", "typePatches", key, "declarations", req.selector], + req.type, + ) + trailing, + ); + lines.push(`wrote ${relative(req.cwd, manifest) || "package.json"} ${where}`); + return { stdout: `${lines.join("\n")}\n`, stderr: "", exitCode: 0 }; +} + +function listCommand(cwd: string, parse: TypePatchParse): TypePatchCommandResult { + const manifest = nearestManifest(cwd); + if (manifest === null) + return { + stdout: "", + stderr: "scriptc: no package.json found from the working directory\n", + exitCode: 1, + }; + const base = dirname(manifest); + const config = parseProjectConfig(manifest, readFileSync(manifest, "utf8")); + const lines: string[] = []; + for (const d of config.diagnostics) + if (d.message.includes("typePatches")) lines.push(`warning ${d.code}: ${d.message}`); + if (config.typePatches.length === 0) { + lines.push("no type patches"); + return { stdout: `${lines.join("\n")}\n`, stderr: "", exitCode: 0 }; + } + for (const entry of config.typePatches) { + lines.push(entry.key); + const copies = installedCopies(cwd, entry.pkg); + const covered = copies.filter((c) => c.version !== null && satisfies(c.version, entry.range)); + if (copies.length === 0) lines.push(` not installed`); + else if (covered.length === 0) + lines.push( + ` does not apply to the installed ${entry.pkg} ${copies[0]!.version ?? "(no version)"}`, + ); + for (const decl of entry.declarations) { + const selector = parseSelector(decl.selector) as Selector; + const op = parseTypeText(decl.type) as TypePatchOp; + const found: FileMatch[] = []; + for (const copy of covered) { + for (const m of matchCopy(copy, selector, op, parse).matches) + found.push({ + ...m, + file: `${relative(base, copy.root).split("\\").join("/")}/${m.file}`, + }); + } + const status = + covered.length === 0 + ? "" + : found.length === 0 + ? " (matches nothing)" + : found.every((m) => m.redundant) + ? " (already declared so)" + : ""; + lines.push(` ${decl.selector}: ${decl.type}${status}`); + for (const m of found) + if (!m.redundant) lines.push(` ${m.file}:${m.line} ${m.before} -> ${m.after}`); + } + for (const [target, local] of entry.files) + lines.push(` file ${target}: ${relative(base, local).split("\\").join("/")}`); + if (entry.types !== null) + lines.push(` types: ${relative(base, entry.types).split("\\").join("/")}`); + } + return { stdout: `${lines.join("\n")}\n`, stderr: "", exitCode: 0 }; +} + +export function runTypePatchCommand( + req: TypePatchCommandRequest, + parse: TypePatchParse, +): TypePatchCommandResult { + return req.action === "patch" ? patchCommand(req, parse) : listCommand(req.cwd, parse); +} diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index c58edcdca..3f8e95524 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,3 +1,9 @@ +import { nodeFrontendServices } from "./frontend/services-node.js"; +import { + runTypePatchCommand, + type TypePatchCommandRequest, + type TypePatchCommandResult, +} from "./frontend/type-patches/command.js"; import { optimizationField, type NativeOptimization } from "./backend/optimization.js"; import { targetCpuField, targetCpuTargetRefusal } from "./backend/target-cpu.js"; import { compilationTiming, type CompilationTiming, type TimingClock } from "./timing.js"; @@ -48,6 +54,7 @@ export type { import { compilePackedLibrary } from "./backend/library-pack.js"; import { node24FetchCompatProfile } from "./compat/fetch-profile.js"; import { libFnSignatures } from "./ir/builtin-signatures.js"; +import { type FunctionLayout, parseFunctionLayout } from "./backend/llvm/function-layout.js"; import { InternalCompilerError } from "./errors.js"; import { frontendProcessExecArgv } from "./frontend-process-args.js"; import { type ChildProcess, spawn } from "node:child_process"; @@ -57,6 +64,7 @@ import { copyFile, mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node: import { tmpdir } from "node:os"; import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { gunzipSync } from "node:zlib"; import { clearCcCaches, configuredTargetPlatform, @@ -197,6 +205,15 @@ export { } from "./backend/external-c.js"; export { warmNativeCaches } from "./backend/warm-cache.js"; export { mergeNativeProfiles as mergeProfiles } from "./backend/native-codegen.js"; + +/** `scriptc types patch|list` with the Node frontend's parser. */ +export async function typePatches( + request: TypePatchCommandRequest, +): Promise { + return runTypePatchCommand(request, (path, text, kind) => + nodeFrontendServices().parse(path, text, kind), + ); +} export { collectGarbageUnderHeapPressure } from "./heap-collection.js"; export { ANDROID_MIN_API, @@ -514,6 +531,14 @@ function performanceWarnings(sink: CompileWarningSink | undefined): { return performance.length === 0 ? {} : { performance }; } +/** The profile match of a --profile-use build's program object, for its + * artifact and its executable cache entry, which a cache hit reports again. */ +function profileMatchField(object: NativeProgramObject | null): { + profileMatch?: NativeProfileMatch; +} { + return object?.profileMatch === undefined ? {} : { profileMatch: object.profileMatch }; +} + function withCompileWarnings( result: T, sink: CompileWarningSink, @@ -648,12 +673,14 @@ async function compileExecutableNative( ...(features.streamDyn ? { streamDyn: true } : {}), ...(features.handleProps ? { handleProps: true } : {}), ...(features.declAdopt ? { declAdopt: true } : {}), + ...(features.typeOrigins ? { typeOrigins: true } : {}), ...(features.dynForAwait ? { dynForAwait: true } : {}), ...(features.eventsOnceSignal ? { eventsOnceSignal: true } : {}), ...(features.mime ? { mime: true } : {}), ...(features.utilTiming ? { utilTiming: true } : {}), ...(features.dynBuffer ? { dynBuffer: true } : {}), ...(features.builtinValues ? { builtinValues: true } : {}), + ...(features.computedRequire ? { computedRequire: true } : {}), events: features.events, emitter: features.emitter, symbol: features.symbol, @@ -776,6 +803,37 @@ async function readNativeProfile( return { use: { path, digest: createHash("sha256").update(bytes).digest("hex") } }; } +/** The digest of the request's function layout file, which keys the + * executable cache: the layout changes the code's order, not its frontend + * input. */ +async function functionLayoutKey(opts: CompileRequestOptions): Promise { + if (opts.functionLayout === undefined) return undefined; + const path = resolve(opts.functionLayout); + let bytes: Buffer; + try { + bytes = await readFile(path); + } catch (err) { + throw new Error( + `cannot read function layout ${path}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + return createHash("sha256").update(bytes).digest("hex"); +} + +/** The request's function layout. */ +async function readFunctionLayout( + opts: CompileRequestOptions, +): Promise<{ functionLayout?: FunctionLayout }> { + if (opts.functionLayout === undefined) return {}; + const path = resolve(opts.functionLayout); + const bytes = await readFile(path); + return { + functionLayout: parseFunctionLayout( + (path.endsWith(".gz") ? gunzipSync(bytes) : bytes).toString("utf8"), + ), + }; +} + function usesPrecompiledRuntimePack(opts: CompileRequestOptions, backend: "llvm"): boolean { if (backend !== "llvm" || opts.sanitize === true) return false; return nativeCodegenTarget() !== null; @@ -849,6 +907,7 @@ async function prepareExecutableInput( llvm = emitLlvmModuleSource(mod, { targetTriple: process.env["SCRIPTC_TARGET"] ?? "", ...debugOptions, + ...(await readFunctionLayout(opts)), pointerBits: buildPlatform === "wasi" ? 32 : 64, wasi: buildPlatform === "wasi", runtimeAbiMarker: outputKind === "obj", @@ -923,6 +982,7 @@ async function prepareExecutableInput( llvmSource = emitLlvmModuleSource(mod, { targetTriple: process.env["SCRIPTC_TARGET"] ?? "", ...debugOptions, + ...(await readFunctionLayout(opts)), pointerBits: buildPlatform === "wasi" ? 32 : 64, wasi: buildPlatform === "wasi", runtimeAbiMarker: useRuntimePack, @@ -934,6 +994,7 @@ async function prepareExecutableInput( ...inlineRcOption(opts.optimization, opts.sanitize === true), ...(opts.detachedExit === true ? { detachedExit: true } : {}), ...targetCpuField(opts.targetCpu), + ...(opts.cpuDispatch === true ? { cpuDispatch: true } : {}), }); } catch (err) { if (!(err instanceof LlvmUnsupportedError)) throw err; @@ -1365,6 +1426,19 @@ async function compileTracked( sourceTexts: new Map(), }; } + if (opts.cpuDispatch === true && outputKind !== "exe") { + return { + ok: false, + diagnostics: [ + nativeCodegenDiag( + "SC3002", + "--cpu-dispatch is only supported for executable output", + entryPath, + ), + ], + sourceTexts: new Map(), + }; + } if (opts.nativeLinkInfo === true && outputKind !== "obj") { return { ok: false, @@ -1532,9 +1606,11 @@ async function compileTracked( platform: buildPlatform, }); let profile: NativeProfileOptions | undefined; + let functionLayoutDigest: string | undefined; try { if (profileRefusal !== null) throw new Error(profileRefusal); profile = await readNativeProfile(opts); + functionLayoutDigest = await functionLayoutKey(opts); } catch (err) { return { ok: false, @@ -1568,7 +1644,9 @@ async function compileTracked( ...(opts.windowsSubsystem === "gui" ? { windowsSubsystem: "gui" as const } : {}), ...(opts.detachedExit === true ? { detachedExit: true as const } : {}), ...targetCpuField(opts.targetCpu), + ...(opts.cpuDispatch === true ? { cpuDispatch: true as const } : {}), ...(profile === undefined ? {} : { profile: nativeProfileKey(profile) }), + ...(functionLayoutDigest === undefined ? {} : { functionLayout: functionLayoutDigest }), ...checkerField(), ...(youngNurseryRequested() ? { youngNursery: true as const } : {}), npmStatic: opts.npmStatic ?? null, @@ -1643,6 +1721,7 @@ async function compileTracked( path: opts.outPath, translationUnitPath: earlyHit.llvmPath, backend: earlyHit.native.backend, + ...(earlyHit.profileMatch === undefined ? {} : { profileMatch: earlyHit.profileMatch }), }, binaryPath: opts.outPath, llvmPath: earlyHit.llvmPath, @@ -1720,6 +1799,7 @@ async function compileTracked( executableRestored: true, nativeDependencies: dependencies, frontend: earlyHit.frontend, + ...profileMatchField(nativeProgramObject), }); }, nativeProgramObject?.partitionPaths, @@ -1765,6 +1845,7 @@ async function compileTracked( path: opts.outPath, translationUnitPath: earlyHit.llvmPath, backend: earlyHit.native.backend, + ...profileMatchField(nativeProgramObject), }, binaryPath: opts.outPath, llvmPath: earlyHit.llvmPath, @@ -1856,6 +1937,7 @@ async function compileTracked( frontend: frontendInputs.snapshot(), ...(irSourcePath === undefined ? {} : { irPath: irSourcePath }), ...performanceWarnings(warningSink), + ...profileMatchField(nativeProgramObject), }); publishedExecutable = true; }, @@ -1875,6 +1957,7 @@ async function compileTracked( frontend: frontendInputs.snapshot(), ...(irSourcePath === undefined ? {} : { irPath: irSourcePath }), ...performanceWarnings(warningSink), + ...profileMatchField(nativeProgramObject), }).catch(() => undefined); } } catch (err) { @@ -1903,7 +1986,6 @@ async function compileTracked( } await pruneBuildCache(cacheRoot); timing("complete"); - const profileMatch = nativeProgramObject?.profileMatch; return { ok: true, artifact: { @@ -1911,7 +1993,7 @@ async function compileTracked( path: opts.outPath, translationUnitPath: llvmPath, backend, - ...(profileMatch === undefined ? {} : { profileMatch }), + ...profileMatchField(nativeProgramObject), }, binaryPath: opts.outPath, llvmPath, diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index b0cc96687..709f72d45 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -92,6 +92,9 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "child.stdioStream", "stream.unboxDyn", "stream.dynIterableSource", + "stream.dynSourceValue", + "stream.unboxDynStage", + "stream.dynTransformResult", "stream.handleStream", "stream.pipeDyn", "stream.unpipeDyn", @@ -703,6 +706,8 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "process.builtinId", "process.builtinModule", "process.builtinUnsupported", + "module.requireComputed", + "module.resolveComputed", "fs.callbackValue", "fs.callbackCall", "dyn.keyDelete", @@ -760,6 +765,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "fs.rmSync", "fs.rmdirSync", "fs.readdirSync", + "fs.readdirRecursiveSync", "fs.readdirTypesSync", "fs.globSync", "fs.globTypesSync", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index e230932ba..0be10a37a 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -385,6 +385,18 @@ function buildLibFnSignatures() { "module.parent": { argTypes: [F64], result: F64 }, "module.loaded": { argTypes: [F64], result: BOOL }, "module.cacheGet": { argTypes: [STRING], result: F64 }, + /** createRequire's require(id) and require.resolve(request, options) + * with a specifier computed at run time (scr_require.c): the arguments, + * then the requiring file, the call's "file:line" and the builtin ids. + * Both resolve like Node at run time. requireComputed always throws + * (Node's argument errors or MODULE_NOT_FOUND), or ends the program with + * an SC2020 refusal when the specifier resolves; resolveComputed answers + * the resolved id or real path. */ + "module.requireComputed": { argTypes: [DYN, STRING, STRING, arrayOf(STRING)], result: DYN }, + "module.resolveComputed": { + argTypes: [DYN, DYN, STRING, STRING, arrayOf(STRING)], + result: STRING, + }, "module.cacheHas": { argTypes: [STRING], result: BOOL }, "module.cacheKeys": { argTypes: [], result: arrayOf(STRING) }, "timers.setTimeout": { argTypes: [{ kind: "func", params: [], ret: VOID }, F64], result: VOID }, @@ -580,6 +592,9 @@ function buildLibFnSignatures() { "fs.rmSync": { argTypes: [STRING], result: VOID }, "fs.rmdirSync": { argTypes: [STRING], result: VOID }, "fs.readdirSync": { argTypes: [STRING], result: arrayOf(STRING) }, + /** `fs.readdirSync(path, { recursive: true })`: every entry below the + * path, relative to it, in Node's queue order. */ + "fs.readdirRecursiveSync": { argTypes: [STRING], result: arrayOf(STRING) }, // Result is the call site's Dirent record array (VOID is the // structure-checked sentinel, the os.networkInterfaces pattern). /** `fs.readdirSync(path, { withFileTypes: true })` — Dirent rows over @@ -2870,9 +2885,27 @@ function buildLibFnSignatures() { result: { kind: "object", className: "%Duplex" }, }, /** Whether pipeline() pumps a checked-dynamic first stage as an - * iterable (a string, or a value with an iterator method that is not a - * stream with pipe() and on()) instead of piping it as a stream. */ + * iterable (a string, a value with an iterator method that is not a + * stream with pipe() and on(), a { readable, writable } pair, or a + * source function's result) instead of piping it as a stream. */ "stream.dynIterableSource": { argTypes: [DYN], result: BOOL }, + /** The value pipeline() pumps for a checked-dynamic first stage: a + * source function's result (the flag set), which must be iterable, else + * a { readable, writable } pair's readable side or the value itself. */ + "stream.dynSourceValue": { argTypes: [DYN, BOOL], result: DYN }, + /** An untyped pipeline function stage's result before the last stage, + * which must be async iterable: ERR_INVALID_RETURN_VALUE naming the + * stage ("transform[i]") otherwise. Answers the value (+1). */ + "stream.dynTransformResult": { argTypes: [DYN, STRING], result: DYN }, + /** A checked-dynamic pipeline() stage taken as a stream: a native stream + * (+1), or Duplex.from of a value that is not stream-like or a function + * (an iterable), which the pipeline uses without piping into it. Other + * values throw: Duplex.from's ERR_INVALID_ARG_TYPE, or the given + * not-supported message. */ + "stream.unboxDynStage": { + argTypes: [DYN, STRING], + result: { kind: "object", className: "%Duplex" }, + }, /** The Transform a Cipheriv or Decipheriv reads and writes as a stream * (+1, created on first use and owned by the cipher). */ "stream.handleStream": { @@ -3967,6 +4000,12 @@ function buildLibFnSignatures() { * Throws the path TypeError, changing nothing, when a converted object * is not plain data (a prototype, accessors, symbols, frozen state). */ "dyn.declAdopt": { argTypes: [DYN, DYN], result: VOID }, + /** dyn.declAdopt for every other checked cast of a dynamic value to a + * record or array (`value as T`): a shared source object or array + * becomes a live capsule of the value built from it. A node that is not + * plain data, or that the value holds twice, keeps its separate copy. + * Never throws: the conversion already succeeded. */ + "dyn.adoptShared": { argTypes: [DYN, DYN], result: VOID }, "dyn.cloneMissing": { argTypes: [], result: DYN }, /** structuredClone with a NON-EMPTY transfer array of static values: * nothing static is transferable, so the call always throws Node's @@ -5237,10 +5276,12 @@ export type IrLibFn = keyof ReturnType; /** The function-stage runs of a pipeline of `total` stages that a stream * follows (positions bit j: stage j is a function), and an async * generator destination's run (flags bits 1-2 = 1, pumped into the - * PassThrough pipeline() answers): each pumps its last generator through - * one (next, read, return) triple. */ + * PassThrough pipeline() answers, or 3, an untyped destination whose + * result may be pumped there): each pumps its last generator through one + * (next, read, return) triple. */ export function pipelinePumpedRuns(total: number, positions: number, flags: number): number { - let runs = ((flags >> 1) & 3) === 1 ? 1 : 0; + const dest = (flags >> 1) & 3; + let runs = dest === 1 || dest === 3 ? 1 : 0; for (let j = 0; j + 1 < total; j++) { if (Math.floor(positions / 2 ** j) % 2 === 1 && Math.floor(positions / 2 ** (j + 1)) % 2 === 0) runs++; diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index e97025e82..4101ae2fd 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -223,6 +223,7 @@ const MODULE_FIELDS: Record>, Role> = { "threadSlotKeys[]": "other", "foreignElementKeys[]": "other", "freezableArrays[]": "other", + "declOrigins[]": "other", }; const loc = { file: "references.ts", start: 0, end: 0 }; diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index c946d799f..f82e71d4a 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1063,6 +1063,11 @@ export interface IrModule { * closures boxed for package code return records and arrays as live * capsules, so a value that comes back keeps its identity. */ declBoundary?: true; + /** Origins of package-declared types checked by `dynCheck.origin` + * (frontend/lowering/decl-origin.ts): fields separated by \x1f — + * package, version, suggested range, selector, where it is declared, + * declared type. */ + declOrigins?: string[]; /** The program constructs LifetimeRegion (@scriptc/experimental): every * ProgramLifetime instance lives in a counted region (the region.* * intrinsics; backend/llvm/regions.ts). */ @@ -4040,12 +4045,17 @@ export type IrExpr = * unshared source is simply consumed. Values must be plain data * (ordinary objects and arrays without accessors, prototypes, symbols * or frozen state), otherwise the cast throws the path TypeError. */ + /** `origin`: the target type comes from an npm package's declaration; + * an index into `IrModule.declOrigins`. A failing check's TypeError then + * names the declaration and the type patch that would admit the value + * (scr_type_origin.c). */ | { kind: "dynCheck"; value: IrExpr; preserveRefs?: true; adopt?: true; exact?: true; + origin?: number; type: IrType; loc: SrcLoc; }; diff --git a/packages/compiler/src/ir/receiver-readers.ts b/packages/compiler/src/ir/receiver-readers.ts index f5270aa7d..067c474ca 100644 --- a/packages/compiler/src/ir/receiver-readers.ts +++ b/packages/compiler/src/ir/receiver-readers.ts @@ -1,6 +1,7 @@ import { isBorrowSafeMath } from "./analysis.js"; import type { IrExpr, IrFunction, IrLibFn, IrStmt } from "./ir.js"; import { eachNodeOfKinds, type IrNodeIndex } from "./node-index.js"; +import { everyExprChild, everyStmtChild } from "./traverse.js"; /** Expression kinds that run no user code of their own and never consult * the ambient receiver stack. Calls are admitted here and resolved through @@ -151,6 +152,7 @@ function opaqueOperand(e: IrExpr): boolean { * this set needs no receiver bracket. */ export class AmbientReceiverReaders { private readers: Set | null = null; + private byMethod: Map | null = null; constructor( private readonly functions: ReadonlyMap, @@ -163,8 +165,63 @@ export class AmbientReceiverReaders { return !this.compute().has(fn.name); } - private compute(): Set { - if (this.readers) return this.readers; + /** A signature adapter body that observes the ambient receiver only by + * forwarding it, unchanged, to the original it captures (a callValue on + * that capture whose receiver is the plain `this` read). Such an adapter + * ignores the receiver exactly when its original does. */ + forwardsOnly(fn: IrFunction): boolean { + if (fn.async || fn.generator) return false; + const original = fn.captures?.[0]?.localId; + if (original === undefined) return false; + const readers = this.compute(); + const edge = (name: string): boolean => this.functions.has(name) && !readers.has(name); + const stmt = (node: IrStmt): boolean => + RECEIVER_FREE_STMTS.has(node.kind) && everyStmtChild(node, expr, stmt); + const expr = (node: IrExpr): boolean => { + if (node.kind === "callValue" && node.receiver !== undefined) { + const receiver = node.receiver; + return ( + receiver.kind === "libCall" && + receiver.fn === "dyn.this" && + receiver.args.length === 0 && + node.callee.kind === "varRef" && + node.callee.localId === original && + node.args.every(expr) + ); + } + return this.admits(node, edge) && everyExprChild(node, expr, stmt); + }; + return fn.body.every(stmt); + } + + /** Whether one expression node, apart from its children, leaves the + * ambient receiver alone; `edge` admits a direct callee. */ + private admits(expr: IrExpr, edge: (name: string) => boolean): boolean { + if (!RECEIVER_FREE_EXPRS.has(expr.kind)) return false; + switch (expr.kind) { + case "call": + return edge(expr.callee); + case "new": + return edge(`%${expr.className}.constructor`); + case "virtualCall": { + // Any class's override may run: admit only when every + // same-named method is itself receiver-free. + const targets = this.methods().get(expr.method) ?? []; + return targets.length > 0 && targets.every(edge); + } + case "libCall": + return isBorrowSafeMath(expr.fn) || RECEIVER_FREE_LIB_FNS.has(expr.fn); + case "toString": + return !opaqueOperand(expr.operand); + default: { + const operands = dataIntrinsicOperands(expr); + return operands === null || !operands.some(opaqueOperand); + } + } + } + + private methods(): Map { + if (this.byMethod) return this.byMethod; const byMethod = new Map(); for (const name of this.functions.keys()) { const dot = name.lastIndexOf("."); @@ -174,6 +231,12 @@ export class AmbientReceiverReaders { list.push(name); byMethod.set(method, list); } + this.byMethod = byMethod; + return byMethod; + } + + private compute(): Set { + if (this.readers) return this.readers; const readers = new Set(); const callers = new Map(); const scan = this.nodes?.scanExcept(TRIVIALLY_RECEIVER_FREE); @@ -185,34 +248,11 @@ export class AmbientReceiverReaders { callees.add(name); return true; }; - const expr = (expr: IrExpr): boolean => { - if (!RECEIVER_FREE_EXPRS.has(expr.kind)) return false; - switch (expr.kind) { - case "call": - return edge(expr.callee); - case "new": - return edge(`%${expr.className}.constructor`); - case "virtualCall": { - // Any class's override may run: admit only when every - // same-named method is itself receiver-free. - const targets = byMethod.get(expr.method) ?? []; - return targets.length > 0 && targets.every(edge); - } - case "libCall": - return isBorrowSafeMath(expr.fn) || RECEIVER_FREE_LIB_FNS.has(expr.fn); - case "toString": - return !opaqueOperand(expr.operand); - default: { - const operands = dataIntrinsicOperands(expr); - return operands === null || !operands.some(opaqueOperand); - } - } - }; if (!reads) eachNodeOfKinds(fn, scan, (node, statement) => { const free = statement ? RECEIVER_FREE_STMTS.has((node as IrStmt).kind) - : expr(node as IrExpr); + : this.admits(node as IrExpr, edge); if (!free) reads = true; return free; }); diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 494c4e3e8..a4c1e8c35 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -278,6 +278,9 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "stream.stdioWritable", "stream.unboxDyn", "stream.dynIterableSource", + "stream.dynSourceValue", + "stream.dynTransformResult", + "stream.unboxDynStage", "stream.handleStream", "stream.unpipeDyn", "stream.workerStdio", @@ -314,6 +317,7 @@ const READABLE_DYN_ASYNC_LIB_FNS: ReadonlySet = new Set([ "stream.duplexFromAsyncFn", "stream.duplexFromGenSignal", "stream.duplexFromAsyncFnSignal", + "stream.unboxDynStage", ]); /** libCall name prefixes of the Readable entry points that await through the dyn-async unit. */ @@ -963,7 +967,13 @@ const RUNTIME_FEATURE_RULES = { /** The declaration-typed boundary's adoption (scr_decl_adopt.c): its * casts, and closures boxed in a boundary program, which adopt their * arguments. */ - declAdopt: rule({ fns: ["dyn.declAdopt"], initial: (mod) => mod.declBoundary === true }), + declAdopt: rule({ + fns: ["dyn.declAdopt", "dyn.adoptShared"], + initial: (mod) => mod.declBoundary === true, + }), + // Checked conversions into package-declared types name the declaration + // in their TypeError (scr_type_origin.c). + typeOrigins: rule({ initial: (mod) => (mod.declOrigins?.length ?? 0) > 0 }), /** for await over checked-dynamic values (scr_dyn_for_await.c). */ dynForAwait: rule({ fns: ["dyn.asyncIteratorOpen", "dyn.asyncIteratorWasSync"] }), /** Buffer.prototype methods called on checked-dynamic values @@ -973,6 +983,9 @@ const RUNTIME_FEATURE_RULES = { fns: ["buffer.prototypeMember"], expr: (node, _tables) => node.kind === "dynInvoke" && DYN_BUFFER_METHODS.has(node.method), }), + /** createRequire's require and require.resolve with specifiers computed + * at run time (scr_require.c). */ + computedRequire: rule({ fns: ["module.requireComputed", "module.resolveComputed"] }), /** Builtin functions read as values (scr_builtin_values.c). */ builtinValues: rule({ fns: [ @@ -1261,9 +1274,11 @@ function noRuntimeFeatures(): RuntimeFeatures { streamDyn: false, handleProps: false, declAdopt: false, + typeOrigins: false, dynForAwait: false, dynBuffer: false, builtinValues: false, + computedRequire: false, symbol: false, tls: false, tlsCa: false, diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 45ebf5c79..51a426d23 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -127,6 +127,12 @@ export function validateModule(mod: IrModule): IrValidationError[] { errors.push({ message: "worker execution mode must be a boolean", loc: moduleLoc }); if (mod.declBoundary !== undefined && mod.declBoundary !== true) errors.push({ message: "declBoundary must be true when present", loc: moduleLoc }); + const declOrigins = mod.declOrigins; + if ( + declOrigins !== undefined && + (!Array.isArray(declOrigins) || declOrigins.some((o) => typeof o !== "string")) + ) + errors.push({ message: "declOrigins must be an array of strings", loc: moduleLoc }); if (mod.untypedIntegrity !== undefined && mod.untypedIntegrity !== true) errors.push({ message: "untypedIntegrity must be true when present", loc: moduleLoc }); if (mod.lifetimeRegions !== undefined && mod.lifetimeRegions !== true) @@ -1001,6 +1007,7 @@ export function validateModule(mod: IrModule): IrValidationError[] { mod.workers === true, mod.lifetimeRegions === true, new Set(mod.threadSlotKeys ?? []), + mod.declOrigins?.length ?? 0, ); } return errors; @@ -1020,6 +1027,7 @@ function validateFunction( workers: boolean, lifetimeRegions: boolean, threadSlotKeys: ReadonlySet, + declOriginCount: number, ): void { const locals = new Map(fn.locals.map((l) => [l.id, l])); const err = (message: string, loc: SrcLoc) => @@ -5761,7 +5769,7 @@ function validateFunction( } const lastIsFunction = Math.floor(positions! / 2 ** (total - 1)) % 2 === 1; const helperCount = 3 * pipelinePumpedRuns(total, positions!, flags!); - const groups = helperCount + (destKind === 2 ? 1 : 0) + (flags! & 32 ? 2 : 0); + const groups = helperCount + (destKind >= 2 ? 1 : 0) + (flags! & 32 ? 2 : 0); const cbArgs = e.fn === "sp.pipelineStagesOpts" ? 1 + (flags! & 64 ? 1 : 0) @@ -5776,7 +5784,7 @@ function validateFunction( total > 52 || positions! >= 2 ** total || functions !== k || - destKind === 3 || + (destKind === 3 && e.fn === "stream.composeStages") || (destKind !== 0) !== lastIsFunction || total < 2 || e.args.length !== 5 + n! + k! + groups + cbArgs @@ -5795,7 +5803,7 @@ function validateFunction( err(`libCall ${e.fn} stage function ${i} must be a func`, e.loc); } at2 += k!; - const helpers = helperCount + (destKind === 2 ? 1 : 0); + const helpers = helperCount + (destKind >= 2 ? 1 : 0); for (let i = at2; i < at2 + helpers; i++) { if (e.args[i]?.type.kind !== "func") err(`libCall ${e.fn} helper ${i} must be a func`, e.loc); @@ -6154,6 +6162,11 @@ function validateFunction( case "dynCheck": { checkExpr(e.value); expectType(e.value, DYN, "dynCheck operand"); + if ( + e.origin !== undefined && + (!Number.isInteger(e.origin) || e.origin < 0 || e.origin >= declOriginCount) + ) + err(`dynCheck origin ${e.origin} is not an index of declOrigins`, e.loc); // The target drives the emitted validator/builder: non-dyn, // non-void, JSON-representable (closures/class instances can never // be found inside a JSON dyn — the frontend rejects those casts). diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index b3de05846..4209ecf0e 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { mkdtempSync, rmSync } from "node:fs"; +import { appendFileSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { runCli } from "../cli/command.js"; @@ -12,6 +12,7 @@ import { mergeNativeProfiles } from "../backend/native-tools.js"; import { setDeclarationRoot } from "../frontend/dts-paths.js"; import { setFrontendDisposedHook } from "../frontend/dispose-hook.js"; import { FrontendServices } from "../frontend/services.js"; +import { runTypePatchCommand } from "../frontend/type-patches/command.js"; import { createNativeTs7Api } from "../frontend/ts7/native-api.js"; import { resolveProvenanceSourcesWithParser } from "../frontend/provenance-core.js"; import { setCompilerReleaseVersion } from "../library/sidecar.js"; @@ -19,6 +20,9 @@ import { NativeCompiler } from "./compiler.js"; import { NATIVE_STRING_BYTES, openNativeCache } from "./cache.js"; import { checkerExecutableIdentity, + checkerPackageVersion, + distributionPrebuiltLibDiagnostics, + recordPrebuiltLibDiagnostics, setLibDiagnosticsStore, } from "../frontend/ts7/lib-diagnostics.js"; import { setCheckerFeatureStore } from "../frontend/ts7/checker-features.js"; @@ -67,7 +71,23 @@ async function main(): Promise { write: (key, text) => store.write("lib-diagnostics", key, text), }, checkerExecutableIdentity(toolchain.ts7Executable), + toolchain.prebuiltLibDiagnostics === undefined + ? null + : () => + distributionPrebuiltLibDiagnostics( + toolchain.prebuiltLibDiagnostics!, + toolchain.ts7Executable, + ), ); + // A distribution build records its prebuilt lib diagnostics (build-native-cli.mts). + const recordPath = process.env["SCRIPTC_LIB_DIAGNOSTICS_RECORD"]; + const checkerVersion = + recordPath === undefined ? null : checkerPackageVersion(toolchain.ts7Executable); + if (recordPath !== undefined && checkerVersion !== null) + recordPrebuiltLibDiagnostics({ + checker: checkerVersion, + record: (key) => appendFileSync(recordPath, `${key}\n`), + }); setCheckerFeatureStore({ read: (key) => store.read("checker-features", key, NATIVE_STRING_BYTES)?.toString("utf8") ?? null, @@ -169,6 +189,19 @@ async function main(): Promise { } return { cacheRoot: cache.root, profiles }; }, + typePatches: async (request) => { + const toolchain = getCompiler().toolchain; + const services = new FrontendServices((options) => + createNativeTs7Api({ ...options, executable: toolchain.ts7Executable }), + ); + try { + return runTypePatchCommand(request, (path, source, kind) => + services.parse(path, source, kind), + ); + } finally { + services.close(); + } + }, mergeProfiles: async (outputPath, inputPaths, sparse) => { const toolchain = getCompiler().toolchain; mergeNativeProfiles({ diff --git a/packages/compiler/src/native/compiler.test.ts b/packages/compiler/src/native/compiler.test.ts index a085356a0..c9636ea76 100644 --- a/packages/compiler/src/native/compiler.test.ts +++ b/packages/compiler/src/native/compiler.test.ts @@ -107,7 +107,7 @@ test.each([ }); vi.mocked(emitNativeObject).mockImplementation((options) => { writeFileSync(options.outputPath, "object"); - return [options.outputPath]; + return { outputs: [options.outputPath], profileMatch: null }; }); const cache = new NativeCache(join(directory, "cache")); const key = contentDigest("executable link"); @@ -250,7 +250,7 @@ test.each([false, true])( expect(emitted).toContain("define i32 @main"); if (fail) throw new Error("codegen failed"); writeFileSync(options.outputPath, "new object"); - return [options.outputPath]; + return { outputs: [options.outputPath], profileMatch: null }; }); const target = MACOS_ARM64_TARGET; const toolchain: NativeToolchain = { @@ -341,7 +341,7 @@ test.skipIf(process.platform === "win32").each([false, true])( }); vi.mocked(emitNativeObject).mockImplementation((options) => { writeFileSync(options.outputPath, "object"); - return [options.outputPath]; + return { outputs: [options.outputPath], profileMatch: null }; }); vi.mocked(openNativeExecutableCache).mockReturnValue(null); const links: string[][] = []; @@ -378,3 +378,84 @@ test.skipIf(process.platform === "win32").each([false, true])( if (lldFails) expect(links[1]).not.toContain("-fuse-ld=lld"); }, ); + +test.skipIf(process.platform === "win32")( + "a --profile-use executable reports the helper's profile match", + () => { + const directory = mkdtempSync(join("/tmp", "scriptc-native-profile-")); + directories.push(directory); + const clang = join(directory, "clang"); + writeFileSync(clang, "#!/bin/sh\n"); + chmodSync(clang, 0o755); + const entry = join(directory, "entry.ts"); + const output = join(directory, "program"); + const profile = join(directory, "app.profdata"); + writeFileSync(profile, "indexed profile"); + const module: IrModule = { + irVersion: 17, + sourceFile: entry, + entry: "%main", + functions: [ + { + name: "%main", + params: [], + locals: [], + body: [], + returnType: VOID, + loc: { file: entry, start: 0, end: 0 }, + }, + ], + }; + vi.mocked(runNativeFrontend).mockReturnValue({ + preflight: [], + entryText: () => "", + entryExports: () => new Map(), + entryContract: () => { + throw new Error("unexpected contract request"); + }, + sourceTexts: () => new Map(), + npmStatic: [], + npmImportSites: new Map(), + dispose: () => {}, + lower: () => ({ + module, + diagnostics: [], + runtimeFences: [], + stats: { statementsTotal: 0, statementsFailed: 0, functionsSkipped: 0 }, + }), + }); + const match = { annotated: 120, mismatched: 7, missing: 40, unknownPartitions: 0 }; + vi.mocked(emitNativeObject).mockImplementation((options) => { + expect(options.profile?.use?.path).toBe(profile); + writeFileSync(options.outputPath, "object"); + return { outputs: [options.outputPath], profileMatch: match }; + }); + vi.mocked(openNativeExecutableCache).mockReturnValue(null); + vi.mocked(execFileSync).mockImplementation((_command, args) => { + const list = args as string[]; + writeFileSync(list[list.indexOf("-o") + 1]!, "executable"); + return ""; + }); + const target = LINUX_X64_GNU_TARGET; + const compiler = new NativeCompiler({ + compilerVersion: "0.0.0", + target, + helper: target.helper, + ts7Executable: "unused", + helperExecutable: "unused", + helperPackageRoot: "unused", + runtimePackRoot: "unused", + linker: clang, + linkerArgs: [], + dsymutil: "unused", + }); + const result = compiler.compile(entry, { + outDir: directory, + outPath: output, + backend: "llvm", + optimization: "speed", + profileUse: profile, + }); + expect(result).toMatchObject({ ok: true, artifact: { kind: "exe", profileMatch: match } }); + }, +); diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index 0b56b0a53..ea3772db3 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -21,7 +21,7 @@ import type { CompileRequestResult, } from "../compile-types.js"; import { LlvmUnsupportedError } from "../backend/llvm/emitter.js"; -import { executableLinkInputs } from "../backend/link-plan-core.js"; +import { executableLinkInputs, systemLibraryLinkerArgs } from "../backend/link-plan-core.js"; import { formatNativeLinkInfo } from "../backend/native-link-info-core.js"; import { emitNativeObject, @@ -63,6 +63,8 @@ import { contentDigest, fileDigest, partsDigest, type NativeCache } from "./cach import { nativeProfileKey, nativeProfileRefusal, + parseNativeProfileMatch, + type NativeProfileMatch, type NativeProfileOptions, } from "../backend/native-profile.js"; import { splitLlvmLibraryProgram } from "../backend/llvm/split.js"; @@ -78,6 +80,18 @@ function partitionCacheKey(key: string, index: number): string { return index === 0 ? key : contentDigest(`${key}:${index}`); } +/** The object cache entry that holds a cached program's profile match. */ +function profileMatchCacheKey(key: string): string { + return contentDigest(`${key}:profile-match`); +} + +/** A program's native objects, and how a --profile-use build's profile + * matched its functions. */ +interface NativeObjects { + paths: string[]; + profileMatch?: NativeProfileMatch; +} + function failure(error: unknown, entry: string, sources: Map): CompileFailure { return { ok: false, @@ -186,7 +200,7 @@ export class NativeCompiler { partitions = 1, profile?: NativeProfileOptions, llvmDigest: () => string = () => fileDigest(input), - ): string[] { + ): NativeObjects { const toolchain = this.toolchain; const outputs = nativePartitionPaths(output, partitions); const helperOptions = { @@ -223,7 +237,20 @@ export class NativeCompiler { verifyNativeHelper(helperOptions); if (nativeFileIdentity(toolchain.helperExecutable) === identity) { cached.forEach((bytes, index) => writeFileSync(outputs[index]!, bytes!)); - return outputs; + if (profile?.use === undefined) return { paths: outputs }; + // Without the cached match, every partition's counts are unknown. + const match = this.cache.read("object", profileMatchCacheKey(key)); + return { + paths: outputs, + profileMatch: (match === null + ? null + : parseNativeProfileMatch(match.toString("utf8"))) ?? { + annotated: 0, + mismatched: 0, + missing: 0, + unknownPartitions: outputs.length, + }, + }; } key = null; } @@ -235,7 +262,7 @@ export class NativeCompiler { this.cache === null || identity === null || partitions < 2 ? null : this.cache.partitionDirectory(identity); - emitNativeObject({ + const { profileMatch } = emitNativeObject({ executable: toolchain.helperExecutable, packageRoot: toolchain.helperPackageRoot, compilerVersion: toolchain.compilerVersion, @@ -252,15 +279,30 @@ export class NativeCompiler { }); if (this.cache !== null && key !== null) { try { - if (nativeFileIdentity(toolchain.helperExecutable) === identity) + if (nativeFileIdentity(toolchain.helperExecutable) === identity) { + // The match goes in first: a reader that finds every partition finds it too. + if (profileMatch !== null) + this.cache.write( + "object", + profileMatchCacheKey(key), + `${JSON.stringify({ + profile: { + annotated: profileMatch.annotated, + mismatched: profileMatch.mismatched, + missing: profileMatch.missing, + unknown_partitions: profileMatch.unknownPartitions, + }, + })}\n`, + ); outputs.forEach((path, index) => this.cache!.write("object", partitionCacheKey(key!, index), readFileSync(path)), ); + } } catch { /* An updated helper or unavailable cache only prevents reuse. */ } } - return outputs; + return profileMatch === null ? { paths: outputs } : { paths: outputs, profileMatch }; } private emitProgramObject( @@ -272,7 +314,7 @@ export class NativeCompiler { library: boolean, profile?: NativeProfileOptions, llvmDigest?: () => string, - ): string[] { + ): NativeObjects { // Library archives merge dev shards relocatably; executables link the // helper's program partitions directly. const llvm = @@ -303,7 +345,7 @@ export class NativeCompiler { } const merged = localizeNativeLibrary(this.toolchain, stage, objects, [], split.publicSymbols); renameSync(merged, output); - return [output]; + return { paths: [output] }; } compile(entry: string, options: CompileRequestOptions): CompileRequestResult { @@ -427,6 +469,9 @@ export class NativeCompiler { const kept = executableCache?.keep(output) ?? false; const restored = kept || (executableCache?.restore(stagedOutput) ?? false); timing(restored ? "executable-cache-hit" : "executable-cache-miss"); + // The executable cache leaves out --profile-use builds, so this is set + // whenever the build has a profile to report. + let profileMatch: NativeProfileMatch | undefined; if (!restored) { // A missing or invalid runtime pack fails before code generation. if (outputKind === "exe") selectPack(); @@ -440,8 +485,8 @@ export class NativeCompiler { object, ]); requireNativeArtifact(object); - } else if (outputKind === "exe") - programObjects = this.emitProgramObject( + } else if (outputKind === "exe") { + const objects = this.emitProgramObject( llvmInput, object, entry, @@ -451,7 +496,9 @@ export class NativeCompiler { profile, digestLlvm, ); - else + programObjects = objects.paths; + profileMatch = objects.profileMatch; + } else this.emitObject( llvmInput, object, @@ -527,7 +574,7 @@ export class NativeCompiler { ...lld, ...plan.driverFlags, ...plan.inputs, - ...plan.systemLibraries.map((name) => `-l${name}`), + ...systemLibraryLinkerArgs(target, plan.systemLibraries), "-o", stagedOutput, ]; @@ -578,7 +625,13 @@ export class NativeCompiler { rmSync(output + ".dSYM", { recursive: true, force: true }); return { ok: true, - artifact: { kind: "exe", path: output, translationUnitPath: llvmPath, backend: "llvm" }, + artifact: { + kind: "exe", + path: output, + translationUnitPath: llvmPath, + backend: "llvm", + ...(profileMatch === undefined ? {} : { profileMatch }), + }, binaryPath: output, llvmPath, backend: "llvm", diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index 749027b80..dc6cdafa3 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -438,6 +438,7 @@ export function prepareNativeExecutable( ...inlineRcOption(options.optimization, options.sanitize === true), ...(options.detachedExit === true && outputKind === "exe" ? { detachedExit: true } : {}), ...targetCpuField(options.targetCpu), + ...(options.cpuDispatch === true && outputKind === "exe" ? { cpuDispatch: true } : {}), ...(options.optimization === "dev" && !options.strip ? { debugSources: prepared.sourceTexts } : {}), diff --git a/packages/compiler/src/native/sanitizer.test.ts b/packages/compiler/src/native/sanitizer.test.ts index c28f7fce4..df2bdf69b 100644 --- a/packages/compiler/src/native/sanitizer.test.ts +++ b/packages/compiler/src/native/sanitizer.test.ts @@ -81,12 +81,14 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")( streamDyn: false, handleProps: false, declAdopt: false, + typeOrigins: false, dynForAwait: false, eventsOnceSignal: false, mime: false, utilTiming: false, dynBuffer: false, builtinValues: false, + computedRequire: false, warningReplace: false, stdioExtended: false, brotli: false, diff --git a/packages/compiler/src/native/toolchain.test.ts b/packages/compiler/src/native/toolchain.test.ts index ad4570c0c..c64bce76b 100644 --- a/packages/compiler/src/native/toolchain.test.ts +++ b/packages/compiler/src/native/toolchain.test.ts @@ -54,6 +54,40 @@ test("native toolchain installation paths relocate with the manifest", () => { } }); +test("prebuilt lib diagnostics apply while the checker is the distribution's tsc-ts", () => { + const root = realpathSync( + mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-tools-")), + ); + const path = join(root, "toolchain.json"); + const manifest: NativeToolchainManifest = { + schema: "scriptc.native-toolchain.v1", + compiler_version: "1.2.3", + target: "macos-arm64", + ts7: "typescript/tsc", + tsc_ts: "tsc-ts/lib/tsc-ts", + llvm_package: "llvm", + runtime_pack: "runtime", + linker: "tools/linker", + linker_args: [], + dsymutil: "dsymutil", + lib_diagnostics: "declarations/lib-diagnostics.json", + }; + try { + writeFileSync(path, JSON.stringify(manifest)); + expect(loadNativeToolchain(path).prebuiltLibDiagnostics).toBe( + join(root, "declarations/lib-diagnostics.json"), + ); + // tsgo checks without them. + expect( + loadNativeToolchain(path, { SCRIPTC_CHECKER: "tsgo" }).prebuiltLibDiagnostics, + ).toBeUndefined(); + writeFileSync(path, JSON.stringify({ ...manifest, lib_diagnostics: "" })); + expect(() => loadNativeToolchain(path)).toThrow(); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + test("native toolchain assets resolve from the physical manifest through package links", () => { const root = realpathSync( mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-linked-tools-")), diff --git a/packages/compiler/src/native/toolchain.ts b/packages/compiler/src/native/toolchain.ts index 5abbb329e..c9167ba14 100644 --- a/packages/compiler/src/native/toolchain.ts +++ b/packages/compiler/src/native/toolchain.ts @@ -30,6 +30,9 @@ export interface NativeToolchain { declarationsRoot?: string; comptimeExecutable?: string; wasiNodeRunner?: string; + /** The prebuilt lib diagnostics the distribution ships, present while the + * checker is the distribution's own tsc-ts (lib-diagnostics.ts). */ + prebuiltLibDiagnostics?: string; } /** Distribution paths are relative to this manifest, so moving an installed @@ -54,6 +57,8 @@ export interface NativeToolchainManifest { relocatable_linker?: string; comptime?: string; wasi_node_runner?: string; + /** Answers for the supplied declaration files, computed with `tsc_ts`. */ + lib_diagnostics?: string; } function pathFrom(root: string, value: string): string { @@ -120,6 +125,7 @@ export function loadNativeToolchain( "relocatable_linker", "comptime", "wasi_node_runner", + "lib_diagnostics", ]) { if (value[name] !== undefined && (typeof value[name] !== "string" || value[name] === "")) throw new Error(`native toolchain requires a nonempty string for ${name}`); @@ -221,5 +227,11 @@ export function loadNativeToolchain( ...(manifest.wasi_node_runner === undefined ? {} : { wasiNodeRunner: pathFrom(root, manifest.wasi_node_runner) }), + // The prebuilt answers are the distribution's tsc-ts's. + ...(manifest.lib_diagnostics === undefined || + manifest.tsc_ts === undefined || + checkerBackend(env) === "tsgo" + ? {} + : { prebuiltLibDiagnostics: pathFrom(root, manifest.lib_diagnostics) }), }; } diff --git a/packages/compiler/test/callback-pending-emission.test.ts b/packages/compiler/test/callback-pending-emission.test.ts index 3b2ed3823..7e7996c83 100644 --- a/packages/compiler/test/callback-pending-emission.test.ts +++ b/packages/compiler/test/callback-pending-emission.test.ts @@ -40,9 +40,11 @@ try { const module = deserializeModule(readFileSync(output, "utf8")); expect(validateModule(module)).toEqual([]); const llvm = emitLlvmModule(module, {}); + // The fallback calls the closure's entry, after giving it an undefined + // receiver when it reads one, so its edge comes from receiver.done then. const guards = [ ...llvm.matchAll( - /(%t\d+) = phi i1 \[ false, %callback\.direct\d+ \], \[ true, %callback\.indirect\d+ \]/g, + /(%t\d+) = phi i1 \[ false, %callback\.direct\d+ \], \[ true, %(?:callback\.indirect|receiver\.done)\d+ \]/g, ), ]; // The quiet every and map callbacks; not the throwing one. diff --git a/packages/compiler/test/canon-normalize.test.ts b/packages/compiler/test/canon-normalize.test.ts new file mode 100644 index 000000000..ac25924f3 --- /dev/null +++ b/packages/compiler/test/canon-normalize.test.ts @@ -0,0 +1,273 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { canonEqual, canonKey, Normalizer, type Canon } from "../src/frontend/canon.js"; +import { loadProgram } from "../src/frontend/program-node.js"; +import * as ts from "../src/frontend/ts7/adapter.js"; + +let directory: string; +let load: ReturnType; +let checker: ts.TypeChecker; +const aliases = new Map(); +const asTypes = new Map(); +const valueTypes = new Map(); +const typeParams = new Map(); + +beforeAll(() => { + directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-canon-")); + const entry = join(directory, "input.ts"); + writeFileSync( + entry, + ` + // #2228: an intersection that restates an optional array member. + export interface BuildOptions { cwd: string; contextFiles?: Array<{ path: string; content: string }> } + export type Normalized = BuildOptions & { contextFiles: Array<{ path: string; content: string }> }; + export type Same = { x: number[] } & { x: number[] }; + // Unions flatten, drop never, read void as undefined, and dedupe. + export type Flat = string | (number | undefined) | never | void | string; + export type Flag = true | false | 0; + // Branded primitive. + export type Id = string & { readonly brand: "id" }; + // #2241/#2246: a recursive interface with a generic method. + export interface TreeNode { name: string; child?: TreeNode; send(message: T): string } + // #2227: a generic alias spelled over a generic body's own parameters. + export type Catalog = { groups: G[]; provider: P }; + export function cast(x: unknown) { + return x as Catalog; + } + // Narrowed type parameters. + export function present(x: T & {}) { + return x; + } + export const sampleNumber = 1 as number; + export const sampleText = "a" as string; + export const sampleMaybe = undefined as string | undefined; + export class Point { x = 0 } + export type Points = readonly Point[]; + export type Pair = [string, number]; + // Two type parameters named T, in different functions. + export function first(x: unknown) { + return x as T[]; + } + export function second(x: unknown) { + return x as readonly T[]; + } + // Distinct callables, and arrays of them. + export type Calls = ((a: string) => void) & ((a: number) => void); + export type CallArrays = ((a: string) => void)[] | ((a: number) => void)[]; + // A dictionary with a named property. + export type Dict = { [key: string]: number } & { size: number }; + // Recursive array and tuple aliases. + export type Nested = number | Nested[]; + export type Chain = [number, Chain] | null; + // Optional and rest tuple elements. + export type Loose = [string, number?, ...boolean[]]; + // Arrays of two instantiations of one generic class. + export type Maps = Map[] | Map[]; + `, + ); + load = loadProgram(entry); + checker = load.program.getTypeChecker(); + ts.walkPreorder(load.entry, (node) => { + if (ts.isTypeAliasDeclaration(node) || ts.isInterfaceDeclaration(node)) + aliases.set( + node.name.text, + checker.getTypeAtLocation(ts.isTypeAliasDeclaration(node) ? node.type : node.name), + ); + if (ts.isAsExpression(node)) { + let fn: ts.Node | undefined = node.parent; + while (fn !== undefined && !ts.isFunctionDeclaration(fn)) fn = fn.parent; + const params = + fn !== undefined && ts.isFunctionDeclaration(fn) + ? (fn.typeParameters?.nodes ?? []).map((p) => checker.getSymbolAtLocation(p.name)!) + : []; + asTypes.set(ts.getTextOfNode(node.type), { + type: checker.getTypeAtLocation(node.type), + params, + }); + } + if (ts.isTypeParameterDeclaration(node) && ts.isFunctionDeclaration(node.parent)) + typeParams.set( + `${node.parent.name?.text}.${node.name.text}`, + checker.getSymbolAtLocation(node.name)!, + ); + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name)) + valueTypes.set(node.name.text, checker.getTypeAtLocation(node.name)); + if ( + ts.isParameter(node) && + node.type !== undefined && + ts.isFunctionDeclaration(node.parent) && + node.parent.name?.text === "present" + ) + asTypes.set("present", { + type: checker.getTypeAtLocation(node.type), + params: (node.parent.typeParameters?.nodes ?? []).map((p) => + checker.getSymbolAtLocation(p.name)!, + ), + }); + }); +}); +afterAll(() => { + load?.dispose(); + if (directory) rmSync(directory, { recursive: true, force: true }); +}); + +const key = (c: Canon) => canonKey(c); + +describe("canon", () => { + test("intersections that restate members resolve to one object (#2228)", () => { + const n = new Normalizer(checker); + expect(key(n.canon(aliases.get("Same")!))).toBe("{ x: number[] }"); + expect(key(n.canon(aliases.get("Normalized")!))).toBe( + "{ cwd: string; contextFiles: { path: string; content: string }[] }", + ); + }); + + test("unions are flat, never-free and deduplicated", () => { + const n = new Normalizer(checker); + expect(key(n.canon(aliases.get("Flat")!))).toBe("string | number | undefined"); + expect(key(n.canon(aliases.get("Flag")!))).toBe("boolean | 0"); + }); + + test("a branded primitive is the primitive", () => { + expect(key(new Normalizer(checker).canon(aliases.get("Id")!))).toBe("string"); + }); + + test("recursion closes on the node under construction (#2241, #2246)", () => { + const n = new Normalizer(checker); + const tree = n.canon(aliases.get("TreeNode")!); + expect(tree.k).toBe("obj"); + if (tree.k !== "obj") return; + const child = tree + .props() + .find((p) => p.name === "child")! + .type(); + expect(child.k).toBe("union"); + if (child.k !== "union") return; + expect(child.arms).toContain(tree); + const send = tree + .props() + .find((p) => p.name === "send")! + .type(); + expect(send.k === "obj" && send.call()[0]!.generic).toBe(true); + }); + + test("a generic alias in a generic body substitutes the environment (#2227)", () => { + const n = new Normalizer(checker); + const { type, params } = asTypes.get("Catalog")!; + expect(key(n.canon(type))).toBe("{ groups: param G[]; provider: param P }"); + const bound = n.canon( + type, + new Map([ + [params[0]!, valueTypes.get("sampleNumber")!], + [params[1]!, valueTypes.get("sampleText")!], + ]), + ); + expect(key(bound)).toBe("{ groups: number[]; provider: string }"); + expect(bound.k === "obj" && bound.alias?.name).toBe("Catalog"); + expect(bound.k === "obj" && bound.alias?.tsType).toBe(type); + }); + + test("a narrowed parameter drops its unit arms", () => { + const n = new Normalizer(checker); + const { type, params } = asTypes.get("present")!; + expect(key(n.canon(type, new Map([[params[0]!, valueTypes.get("sampleMaybe")!]])))).toBe( + "string", + ); + }); + + test("type parameters with the same name are distinct in the intern key", () => { + const n = new Normalizer(checker); + const { type } = asTypes.get("T[]")!; + const firstT = typeParams.get("first.T")!; + const secondT = typeParams.get("second.T")!; + const num = valueTypes.get("sampleNumber")!; + const text = valueTypes.get("sampleText")!; + expect( + key( + n.canon( + type, + new Map([ + [firstT, num], + [secondT, text], + ]), + ), + ), + ).toBe("number[]"); + expect( + key( + n.canon( + type, + new Map([ + [firstT, text], + [secondT, num], + ]), + ), + ), + ).toBe("string[]"); + }); + + test("callables and arrays of callables with different parameters stay distinct", () => { + const n = new Normalizer(checker); + const calls = n.canon(aliases.get("Calls")!); + expect(calls.k === "obj" && calls.call().length).toBe(2); + const arms = n.canon(aliases.get("CallArrays")!); + expect(arms.k === "union" && arms.arms.length).toBe(2); + expect(key(arms)).toBe("{ (string) => void }[] | { (number) => void }[]"); + }); + + test("intersecting a dictionary with a named property keeps the index signature", () => { + const dict = new Normalizer(checker).canon(aliases.get("Dict")!); + expect(key(dict)).toBe("{ [string]: number; size: number }"); + }); + + test("recursive array and tuple aliases close on themselves", () => { + const n = new Normalizer(checker); + const nested = n.canon(aliases.get("Nested")!); + expect(nested.k).toBe("union"); + if (nested.k !== "union") return; + // The array closes on itself through the union of its element. + const array = nested.arms.find((a) => a.k === "array"); + const elem = array?.k === "array" ? array.elem : undefined; + expect(elem?.k === "union" && elem.arms.includes(array!)).toBe(true); + const chain = n.canon(aliases.get("Chain")!); + expect(chain.k).toBe("union"); + if (chain.k !== "union") return; + const tuple = chain.arms.find((a) => a.k === "tuple"); + const rest = tuple?.k === "tuple" ? tuple.elems[1]!.type : undefined; + expect(rest?.k === "union" && rest.arms.includes(tuple!)).toBe(true); + }); + + test("a failed comparison withdraws the pairs it assumed equal", () => { + const array = (p: "number" | "string"): Canon => ({ + k: "array", + elem: { k: "prim", p }, + readonly: false, + }); + // number[] fails against string[] before it matches number[]; that + // attempt must not make string[] match later. + const a: Canon = { k: "union", arms: [array("number"), array("number")] }; + const b: Canon = { k: "union", arms: [array("string"), array("number")] }; + expect(canonEqual(a, b)).toBe(false); + expect(canonEqual(b, a)).toBe(false); + }); + + test("instantiations of one generic class stay distinct", () => { + const maps = new Normalizer(checker).canon(aliases.get("Maps")!); + expect(maps.k === "union" && maps.arms.length).toBe(2); + expect(key(maps)).toBe("Map[] | Map[]"); + }); + + test("optional and rest tuple elements keep their flags", () => { + expect(key(new Normalizer(checker).canon(aliases.get("Loose")!))).toBe( + "[string, number | undefined?, ...boolean]", + ); + }); + + test("classes are nominal; arrays and tuples keep their elements", () => { + const n = new Normalizer(checker); + expect(key(n.canon(aliases.get("Points")!))).toBe("readonly Point[]"); + expect(key(n.canon(aliases.get("Pair")!))).toBe("[string, number]"); + }); +}); diff --git a/packages/compiler/test/chunked-emission.test.ts b/packages/compiler/test/chunked-emission.test.ts index 47e82cf9e..5d54ff804 100644 --- a/packages/compiler/test/chunked-emission.test.ts +++ b/packages/compiler/test/chunked-emission.test.ts @@ -95,7 +95,8 @@ test("a Promise.all tuple adapter requested in a range is defined", async () => throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); const ir = await readFile(outPath, "utf8"); const serial = text(emitLlvmModuleSource(deserializeModule(ir), {})); - expect(serial).toContain("define internal void @sc_all_tuple_0("); + // The adapter is named from its key (contentSymbol). + expect(serial).toMatch(/define internal void @sc_all_tuple_\d+\(/); const chunked = text(emitLlvmModuleSource(deserializeModule(ir), { functionChunks: 2 })); expect(chunked === serial).toBe(true); } finally { diff --git a/packages/compiler/test/constant-choice-emission.test.ts b/packages/compiler/test/constant-choice-emission.test.ts new file mode 100644 index 000000000..bbc54001a --- /dev/null +++ b/packages/compiler/test/constant-choice-emission.test.ts @@ -0,0 +1,64 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../src/index.js"; +import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; + +test("a chain of literal stores is one store of a table entry, indexed as on every path", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-constant-choices-")); + try { + const entry = join(dir, "main.ts"); + await writeFile( + entry, + ` +function bases(arr: Uint8Array): void { + for (let i = 0; i < arr.length; ++i) { + const x = arr[i]; + if (x === 97) arr[i] = 0; + else if (x === 116) arr[i] = 1; + else if (x === 99) arr[i] = 2; + else arr[i] = 3; + } +} +function wide(flag: boolean): string { + const a = new Uint8Array(4); + let j = 0; + if (flag) j = 4294967297; + if (j === 0) a[j] = 11; + else if (j === 1) a[j] = 22; + else if (j === 2) a[j] = 33; + else a[j] = 44; + return a.join(","); +} +const sequence = new Uint8Array([97, 116, 99, 103]); +bases(sequence); +console.log(sequence.join(","), wide(true), wide(false)); +`, + ); + const outPath = join(dir, "main.ir.json"); + const lowered = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!lowered.ok) + throw new Error(lowered.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const ll = emitLlvmModule(deserializeModule(await readFile(outPath, "utf8"))); + const body = (name: string): string => + ll.match( + new RegExp(`define internal [^\\n]+ @sc_[bf]+_${name}\\([^\\n]*\\) #0 \\{[\\s\\S]*?\\n\\}`), + )![0]; + + // The cases' span, 97 to 116, with the final else's 3 between the cases. + expect(ll).toMatch( + /@sc_choice_\w+ = private unnamed_addr constant \[20 x i32\] \[i32 0, i32 3, i32 2, (i32 3, ){16}i32 1\]/, + ); + expect(body("bases")).toMatch(/load i32, ptr %\S+\n/); + + // The index is the compared local. Inside the first arm its range is + // [0, 0], but the one store also runs for 4294967297, so the index + // converts as the local's whole range allows (to i64), not to i32. + const wide = body("wide"); + expect(wide).toMatch(/@sc_choice_\w+, i64 0, i64/); + expect(wide).not.toMatch(/fptosi double %\S+ to i32/); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/counted-loop-emission.test.ts b/packages/compiler/test/counted-loop-emission.test.ts index d7ebc97fa..e9a79a71a 100644 --- a/packages/compiler/test/counted-loop-emission.test.ts +++ b/packages/compiler/test/counted-loop-emission.test.ts @@ -244,3 +244,102 @@ test("worker programs poll no loop whose proven ranges bound its runs inside ano await rm(dir, { recursive: true, force: true }); } }); + +test("worker programs read and write shared elements at a strip-mined loop's counter with plain accesses", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-counted-shared-")); + try { + const entry = join(dir, "shared.ts"); + await writeFile( + entry, + [ + 'import { isMainThread, Worker } from "node:worker_threads";', + "export function scale(a: Float64Array, b: Float64Array, n: number): void {", + " for (let i = 0; i < n; i++) b[i] = a[i]! * 2;", + "}", + "export function codes(a: Uint8Array): void {", + " for (let i = 0; i < a.length; ++i) a[i] = a[i] === 97 ? 0 : 3;", + "}", + "export function shift(a: Uint8Array): void {", + " for (let i = 0; i < a.length - 1; i++) a[i] = a[i + 1]!;", + "}", + "export function rows(a: Int32Array, n: number): number {", + " let t = 0;", + " for (let i = 0; i < n; i++) for (let j = 0; j < n; j++) t += a[i]! + a[j]!;", + " return t;", + "}", + "export function waits(a: Int32Array, out: Uint8Array, n: number): number {", + " let spins = 0;", + " for (let i = 0; i < n; i++) {", + " out[i] = 1;", + " while (a[i] === 0) spins++;", + " do spins++; while (a[i] === 1);", + " for (let k = 0; a[i]! > k; k++) spins++;", + " for (let k = 0; k < 3; k += a[i]!) spins++;", + " }", + " return spins;", + "}", + "export function spin(a: Int32Array): number {", + " let n = 0;", + " while (a[0] === 0 && n < 10) n++;", + " return n;", + "}", + "const a = new Float64Array(new SharedArrayBuffer(64)).fill(1);", + "const b = new Float64Array(8);", + "scale(a, b, 8);", + "const c = new Uint8Array(new SharedArrayBuffer(8)).fill(97);", + "codes(c);", + "shift(c);", + "const d = new Int32Array(new SharedArrayBuffer(16)).fill(1);", + "console.log(b[7], c[0], rows(d, 4), spin(d), waits(new Int32Array(new SharedArrayBuffer(16)).fill(2), new Uint8Array(new SharedArrayBuffer(4)), 4));", + "if (!isMainThread) new Worker(new URL(import.meta.url));", + ].join("\n"), + ); + const outPath = join(dir, "shared.ir.json"); + const result = await compile(entry, { outPath, outDir: dir, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const ll = emitLlvmModule(deserializeModule(await readFile(outPath, "utf8"))); + const bodies = (name: string): string => + [ + ...ll.matchAll( + new RegExp( + `define internal [^\\n]+ @sc_b?f_${name}(?:\\.i32)?\\([^\\n]*\\n([\\s\\S]*?)\\n\\}`, + "g", + ), + ), + ] + .map((match) => match[1]!) + .join("\n"); + // Element loads (the poll's flag read is atomic too, from a global). + const elementLoads = (body: string): number => + body.match(/load atomic \w+, ptr %/g)?.length ?? 0; + // Elements at the counter of a strip-mined loop: plain loads and stores. + for (const name of ["scale", "codes"]) { + const body = bodies(name); + expect(body).toMatch(/^loop\.poll\d+:$/m); + expect(elementLoads(body)).toBe(0); + expect(body).not.toContain("store atomic"); + } + // `a[i + 1]` is not at the counter: its read stays atomic, the store + // at the counter is plain. + const shift = bodies("shift"); + expect(shift).toMatch(/load atomic i8, ptr %/); + expect(shift).not.toContain("store atomic"); + // The outer counter read inside the inner loop reads one element on every + // inner iteration: it stays atomic; the inner counter's read is plain. + const rows = bodies("rows"); + expect(rows).toMatch(/load atomic i32, ptr %/); + // A loop that is not counted keeps atomic reads. + expect(bodies("spin")).toMatch(/load atomic i32, ptr %/); + // A nested loop that tests the outer counter's element in its condition + // or steps by it reads it again on every iteration: each read stays + // atomic, so another thread's write ends the wait. + const waits = bodies("waits"); + expect(waits).toMatch(/^loop\.poll\d+:$/m); + expect(waits.match(/load atomic i32, ptr %/g)?.length).toBeGreaterThanOrEqual(4); + // The outer loop's write at its counter stays plain (speculation counters + // are atomic stores to globals). + expect(waits).not.toMatch(/store atomic i8 \S+, ptr %/); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/cpu-dispatch-emission.test.ts b/packages/compiler/test/cpu-dispatch-emission.test.ts new file mode 100644 index 000000000..170c28ede --- /dev/null +++ b/packages/compiler/test/cpu-dispatch-emission.test.ts @@ -0,0 +1,149 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { beforeAll, expect, test } from "vitest"; +import { compile, deserializeModule, type IrModule } from "../src/index.js"; +import { emitLlvmModule, type LlvmTargetOptions } from "../src/backend/llvm/emitter.js"; + +const program = ` +function dot(a: Float64Array, b: Float64Array): number { + let s = 0; + for (let i = 0; i < a.length; i++) s += a[i]! * b[i]!; + return s; +} +function scale(a: Float64Array, k: number): void { + for (let i = 0; i < a.length; i++) a[i] = a[i]! * k; +} +function positive(x: number): boolean { + return x > 0; +} +function check(x: number): number { + if (x < 0) throw new RangeError("negative"); + return Math.sqrt(x); +} +async function later(x: number): Promise { + await null; + return dot(new Float64Array([x]), new Float64Array([x])); +} +const a = new Float64Array(64).map((_, i) => i * 0.5); +scale(a, 1.25); +console.log(dot(a, a), [1, -2, 3].filter(positive).length); +try { + console.log(check(dot(a, a)), check(-1)); +} catch (error) { + console.log((error as Error).message); +} +later(3).then((value) => console.log(value)); +`; + +let mod: IrModule; + +beforeAll(async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-cpu-dispatch-")); + try { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "main.ir.json"); + await writeFile(entry, program); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + mod = deserializeModule(await readFile(outPath, "utf8")); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}, 120_000); + +const x86 = { targetTriple: "x86_64-unknown-linux-gnu", unwindAbi: true } as const; + +function emit(options: LlvmTargetOptions): string { + return emitLlvmModule(mod, options); +} + +/** Each definition's text by name. */ +function definitions(ll: string): Map { + const out = new Map(); + for (const m of ll.matchAll(/^define [^\n]*?@([\w.$-]+)\([^]*?^}/gm)) out.set(m[1]!, m[0]); + return out; +} + +test("a dispatching x86-64 build gives each program function a baseline and an x86-64-v3 copy", () => { + const ll = emit({ ...x86, cpuDispatch: true }); + const defs = definitions(ll); + const cloned = [...defs.keys()] + .filter((name) => name.endsWith(".v3")) + .map((name) => name.slice(0, -3)); + expect(cloned.length).toBeGreaterThan(4); + for (const name of ["sc_bf_dot", "sc_bf_scale", "sc_f_positive", "sc_f_check"]) + expect( + cloned.some((c) => c.startsWith(name)), + name, + ).toBe(true); + const dispatched = cloned.filter((name) => defs.has(name)); + for (const name of cloned) { + const v3 = defs.get(`${name}.v3`)!; + expect(defs.get(`${name}.base`), name).toBeDefined(); + expect(v3.split("\n", 1)[0]).toMatch(/ #[34]( |$)/); + // The function's own name, where the module still uses it, dispatches + // on the level. + const dispatcher = defs.get(name); + if (dispatcher === undefined) { + expect(ll, name).not.toMatch(new RegExp(`@${name.replace(/[.$]/g, "\\$&")}(?![\\w.$-])`)); + continue; + } + expect(dispatcher).toContain("load i32, ptr @sc_cpu_level"); + expect(dispatcher).toContain(`musttail call`); + expect(dispatcher).toContain(`@${name}.v3(`); + expect(dispatcher).toContain(`@${name}.base(`); + // A copy calls the other cloned functions' copies of its own tier. + for (const call of v3.matchAll(/@([\w.$-]+)\(/g)) { + const callee = call[1]!; + expect(cloned.includes(callee), `${name}.v3 calls ${callee}`).toBe(false); + if (callee.endsWith(".base")) throw new Error(`${name}.v3 calls ${callee}`); + } + for (const call of defs.get(`${name}.base`)!.matchAll(/@([\w.$-]+)\(/g)) + expect(call[1]!.endsWith(".v3"), `${name}.base calls ${call[1]}`).toBe(false); + } + // The program's entry (called from main) and dot (called from the + // coroutine body) keep their dispatcher. Functions only the copies call + // have none, so their copies have the callers the original had. + expect(dispatched).toContain("sc_f__x25_main"); + expect(dispatched).toContain("sc_bf_dot"); + for (const name of ["sc_bf_scale", "sc_f_positive", "sc_f_check"]) + expect( + dispatched.filter((d) => d.startsWith(name)), + name, + ).toEqual([]); + // Coroutine bodies keep one body. + const coroutines = [...defs].filter(([, text]) => / #1 /.test(text.split("\n", 1)[0]!)); + expect(coroutines.length).toBeGreaterThan(0); + for (const [name] of coroutines) expect(defs.has(`${name}.v3`), name).toBe(false); + // Functions used as values are referenced by their dispatching name. + expect(ll).not.toMatch(/ptr @[\w.$-]+\.(v3|base)(?!\()/); + expect(ll).toMatch(/^attributes #3 = \{ sanitize_address "target-cpu"="x86-64-v3" \}$/m); + // Unsanitized builds leave sanitize_address out of the copies' groups too. + expect(emit({ ...x86, cpuDispatch: true, sanitizeAttributes: false })).toMatch( + /^attributes #3 = \{ "target-cpu"="x86-64-v3" \}$/m, + ); + // main sets the level before anything else runs. + expect(defs.get("main")).toMatch( + /^entry:\n(?: {2}call void @scr_runtime_abi\w*\(\)\n)? call void @sc_cpu_level_init\(\)$/m, + ); + // No copy contracts floating-point operations. + expect(ll).not.toMatch( + /\b(?:fadd|fsub|fmul|fdiv|frem|call) (?:[a-z]+ )*(?:contract|fast|afn|reassoc)\b/, + ); +}); + +test("builds without dispatch, for other targets or for a target level emit no copies", () => { + const plain = emit(x86); + expect(plain).not.toContain("@sc_cpu_level"); + expect(plain).not.toMatch(/\.v3\(/); + expect(emit({ ...x86, cpuDispatch: false })).toBe(plain); + const v3 = emit({ ...x86, targetCpu: "x86-64-v3" }); + expect(emit({ ...x86, targetCpu: "x86-64-v3", cpuDispatch: true })).toBe(v3); + for (const options of [ + { targetTriple: "aarch64-unknown-linux-gnu" }, + { targetTriple: "wasm32-wasi", wasi: true, pointerBits: 32 as const }, + ]) + expect(emit({ ...options, cpuDispatch: true })).toBe(emit(options)); +}); diff --git a/packages/compiler/test/decl-origin-quoting.test.ts b/packages/compiler/test/decl-origin-quoting.test.ts new file mode 100644 index 000000000..40b351440 --- /dev/null +++ b/packages/compiler/test/decl-origin-quoting.test.ts @@ -0,0 +1,73 @@ +/* The suggested `scriptc types patch` command in a declared-type note + * (frontend/lowering/decl-origin.ts) passes each value it carries as one + * argument. Package names, selectors and types come from package + * declarations. Run through sh with printf in place of scriptc, the POSIX + * form prints each value exactly and creates no file. The Windows form + * double-quotes values that cmd and PowerShell keep as written there, and + * leaves the arguments out when either shell would expand or escape one. */ +import { execFile } from "node:child_process"; +import { mkdtemp, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { patchSuggestion, shellArgument } from "../src/frontend/lowering/decl-origin.js"; + +const exec = promisify(execFile); + +const hostile = [ + "Profile.a;touch semicolon", + "Profile.$(touch subst)", + "Profile.`touch backtick`", + "Profile.it's \"quoted\" \\ end'", + "Profile.a&&touch and|touch pipe>out", + "Profile.a\nb", +]; + +test("plain values stay bare and types are quoted", () => { + expect(patchSuggestion("kit", "Profile.email", "| undefined", false)).toBe( + "if that declaration is wrong, patch it: scriptc types patch kit Profile.email '| undefined'", + ); + expect(patchSuggestion("@scope/kit", "Terminal.columns()", "| undefined", true)).toBe( + 'if that declaration is wrong, patch it: scriptc types patch @scope/kit "Terminal.columns()" "| undefined"', + ); +}); + +test.skipIf(process.platform === "win32")( + "the POSIX form passes each value as one argument", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-decl-origin-")); + try { + for (const selector of hostile) { + const pkg = "evil;touch pkg"; + const command = patchSuggestion(pkg, selector, "| undefined", false).replace( + "if that declaration is wrong, patch it: scriptc types patch", + "printf '%s\\n'", + ); + const { stdout } = await exec("sh", ["-c", command], { cwd: dir }); + expect(stdout).toBe(`${pkg}\n${selector}\n| undefined\n`); + expect(await readdir(dir)).toEqual([]); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, +); + +test("the Windows form double-quotes values only where both shells keep them", () => { + expect(shellArgument("Profile.a;b&c|df", true)).toBe('"Profile.a;b&c|df"'); + for (const value of [ + "Profile.$(x)", + "Profile.`x`", + 'Profile."x"', + "Profile.%PATH%", + "Profile.!x!", + "Profile.“x”", + "Profile.‘x’", + "Profile.a\nb", + ]) + expect(shellArgument(value, true), value).toBeNull(); + expect(patchSuggestion("kit", "Profile.$(x)", "| undefined", true)).toBe( + "if that declaration is wrong, patch it with scriptc types patch", + ); +}); diff --git a/packages/compiler/test/function-layout.test.ts b/packages/compiler/test/function-layout.test.ts new file mode 100644 index 000000000..ecf4af7d9 --- /dev/null +++ b/packages/compiler/test/function-layout.test.ts @@ -0,0 +1,183 @@ +import { execFileSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import { expect, test, vi } from "vitest"; +import { compile } from "../src/index.js"; +import { + functionLayoutKey, + layoutTierOf, + parseFunctionLayout, +} from "../src/backend/llvm/function-layout.js"; + +const source = ` +function first(n: number): number { return n + 1; } +function second(n: number): number { return n * 2; } +function third(n: number): number { return n - 3; } +console.log(first(Number(process.argv.length)), second(4), third(5)); +`; + +async function emitLlvm(layout?: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-function-layout-")); + try { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "main.ll"); + await writeFile(entry, source); + let functionLayout: string | undefined; + if (layout !== undefined) { + functionLayout = join(dir, "layout.gz"); + await writeFile(functionLayout, gzipSync(layout)); + } + const result = await compile(entry, { + outDir: dir, + outPath, + outputKind: "llvm", + ...(functionLayout === undefined ? {} : { functionLayout }), + }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + return await readFile(outPath, "utf8"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +/** Each defined function's name with the partition group line it follows. */ +function definitions(llvm: string): { name: string; group: string }[] { + const out: { name: string; group: string }[] = []; + let group = ""; + for (const line of llvm.split("\n")) { + if (line.startsWith("; scriptc-group")) group = line.slice("; scriptc-group".length).trim(); + const name = /^define [^@]*@([\w.$]+)\(/.exec(line)?.[1]; + if (name !== undefined) out.push({ name, group }); + } + return out; +} + +test("a layout writes the functions it lists first, by tier, in their tier's group", async () => { + const plain = definitions(await emitLlvm()).filter((d) => /_(first|second|third)$/.test(d.name)); + expect(plain.map((d) => d.name.replace(/^sc_b?f_/, ""))).toEqual(["first", "second", "third"]); + expect(plain.every((d) => !d.group.startsWith("layout-"))).toBe(true); + + const names = new Map(plain.map((d) => [d.name.replace(/^sc_b?f_/, ""), d.name])); + const laid = definitions(await emitLlvm(`2 ${names.get("first")}\n1 ${names.get("third")}\n`)); + const index = (fn: string) => laid.findIndex((d) => d.name === names.get(fn)); + expect(index("third")).toBeLessThan(index("first")); + expect(index("first")).toBeLessThan(index("second")); + expect(laid[index("third")]!.group).toBe("layout-1"); + expect(laid[index("first")]!.group).toBe("layout-2"); + expect(laid[index("second")]!.group).not.toMatch(/^layout-/); + // Listed functions precede every unlisted definition in the function section. + const firstUnlisted = laid.findIndex( + (d, i) => i > index("third") && !d.group.startsWith("layout-"), + ); + expect(firstUnlisted).toBeGreaterThan(index("first")); +}); + +test("layout keys leave out module numbers", () => { + expect(functionLayoutKey("sc_bf__x25_m447_lowerCall")).toBe( + functionLayoutKey("sc_bf__x25_m12_lowerCall"), + ); + expect(functionLayoutKey("sc_f__x25_init_253")).toBe(functionLayoutKey("sc_f__x25_init_7")); + expect(functionLayoutKey("sc_bf__x25_m3_Lexer_scan")).not.toBe( + functionLayoutKey("sc_bf__x25_m3_Lexer_peek"), + ); +}); + +test("a function listed twice keeps its first tier, and malformed lines are skipped", () => { + const layout = parseFunctionLayout("1 sc_f_a\n2 sc_f_a\n2 sc_f_b\nnot a line\n0 sc_f_c\n\n"); + expect([...layout]).toEqual([ + ["sc_f_a", 1], + ["sc_f_b", 2], + ]); + const text = + "define internal void @sc_f_b() {\n ret void\n}\n\ndefine void @sc_f_a() {\n ret void\n}"; + expect(layoutTierOf(layout, text)).toBe(1); + expect(layoutTierOf(layout, "define void @sc_f_z() {\n ret void\n}")).toBeUndefined(); +}); + +test("an executable built with a layout that lists helpers and functions runs unchanged", async () => { + const program = ` +class Point { constructor(public x: number, public y: number) {} } +function norm(p: Point): number { return Math.sqrt(p.x * p.x + p.y * p.y); } +const points = [new Point(3, 4), new Point(6, 8)]; +console.log(points.map(norm).join(","), points[0]!.x === 3 ? "three" : "other"); +`; + const dir = await mkdtemp(join(tmpdir(), "scriptc-function-layout-exe-")); + try { + const entry = join(dir, "main.ts"); + await writeFile(entry, program); + const llvmPath = join(dir, "main.ll"); + const emitted = await compile(entry, { outDir: dir, outPath: llvmPath, outputKind: "llvm" }); + expect(emitted.ok, JSON.stringify(emitted.diagnostics)).toBe(true); + // The generated helpers written before the functions, and one function, + // in two tiers. + const names = definitions(await readFile(llvmPath, "utf8")).map((d) => d.name); + const helpers = names.slice( + 0, + names.findIndex((name) => /^sc_b?f_/.test(name)), + ); + expect(helpers.length).toBeGreaterThan(0); + const norm = names.find((name) => /^sc_b?f_norm$/.test(name))!; + const layout = join(dir, "layout"); + await writeFile(layout, [`1 ${norm}`, ...helpers.map((name) => `2 ${name}`)].join("\n")); + const laid = join(dir, "laid.ll"); + const laidOut = await compile(entry, { + outDir: dir, + outPath: laid, + outputKind: "llvm", + functionLayout: layout, + }); + expect(laidOut.ok, JSON.stringify(laidOut.diagnostics)).toBe(true); + const order = definitions(await readFile(laid, "utf8")); + expect(order.filter((d) => d.group === "layout-2").map((d) => d.name)).toEqual(helpers); + expect(order.find((d) => d.name === norm)!.group).toBe("layout-1"); + const exe = join(dir, "main"); + const built = await compile(entry, { + outputKind: "exe", + outDir: dir, + outPath: exe, + functionLayout: layout, + }); + expect(built.ok, JSON.stringify(built.diagnostics)).toBe(true); + expect(execFileSync(exe, { encoding: "utf8" })).toBe("5,10 three\n"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("a changed function layout misses the executable cache", async () => { + vi.stubEnv("SCRIPTC_NO_CACHE", undefined); + const dir = await mkdtemp(join(tmpdir(), "scriptc-function-layout-cache-")); + try { + vi.stubEnv("SCRIPTC_CACHE_DIR", join(dir, "cache")); + const entry = join(dir, "main.ts"); + await writeFile(entry, source); + const exe = join(dir, "main"); + const llvm = join(dir, "main.ll"); + const build = async (functionLayout?: string) => { + const built = await compile(entry, { + outputKind: "exe", + outDir: dir, + outPath: exe, + ...(functionLayout === undefined ? {} : { functionLayout }), + }); + expect(built.ok, JSON.stringify(built.diagnostics)).toBe(true); + return definitions(await readFile(llvm, "utf8")); + }; + const plain = await build(); + const third = plain.find((d) => /^sc_b?f_third$/.test(d.name))!.name; + const first = plain.find((d) => /^sc_b?f_first$/.test(d.name))!.name; + const layout = join(dir, "layout"); + await writeFile(layout, `1 ${third}\n`); + expect((await build(layout)).find((d) => d.name === third)!.group).toBe("layout-1"); + // The same path with other contents is another layout. + await writeFile(layout, `1 ${first}\n`); + const edited = await build(layout); + expect(edited.find((d) => d.name === first)!.group).toBe("layout-1"); + expect(edited.find((d) => d.name === third)!.group).not.toMatch(/^layout-/); + } finally { + vi.unstubAllEnvs(); + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/llvm-remainder.test.ts b/packages/compiler/test/llvm-remainder.test.ts index 9e6cb02e7..e3f585144 100644 --- a/packages/compiler/test/llvm-remainder.test.ts +++ b/packages/compiler/test/llvm-remainder.test.ts @@ -90,7 +90,7 @@ console.log(remainder(5, 2.5), signed(-7.5), maybeZero(7.5, 2.5)); expect(maybeZero).toContain("rem.float"); expect(ll).not.toContain("@llvm.fptosi.sat"); expect(ll).not.toContain("@fmod"); - expect(ll).toContain("call double @pow("); + expect(ll).toContain("call double @scr_math_pow("); // Where truncation is an integer round trip, a dividend below 2^62 takes // the i64 remainder by the power of two and adds back its fraction; a diff --git a/packages/compiler/test/localized-failure-retries.test.ts b/packages/compiler/test/localized-failure-retries.test.ts new file mode 100644 index 000000000..6bf1b1e23 --- /dev/null +++ b/packages/compiler/test/localized-failure-retries.test.ts @@ -0,0 +1,70 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +async function compileSource(source: string) { + const directory = mkdtempSync(join(tmpdir(), "scriptc-localized-")); + try { + const entry = join(directory, "main.ts"); + writeFileSync(entry, source); + return await compile(entry, { + outDir: directory, + outPath: join(directory, "program.ir"), + outputKind: "ir", + }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +// A statement that lowers again after a later declarator degrades a type +// reuses the member bodies of a function-local class it lowered the first +// time, so the diagnostics of those bodies still report. +test("a degrading retry keeps the diagnostics of a local class lowered before it", async () => { + const result = await compileSource(`type Progress = (done: number, total: number) => void; +type Callbacks = Progress | AbortSignal | Map string> | Set; +function make(): number { + const K = class { + m(): number { + const a = [1, 2]; + (a as any).foo = 1; + return a.length; + } + }, + size = (cb: Callbacks): number => (cb instanceof Set ? cb.size : 0); + return new K().m() + size(new Set([1])); +} +console.log(make()); +`); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected the local class's diagnostic"); + expect(result.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "SC1090", + message: expect.stringContaining("assignment to non-variables"), + }), + ]), + ); +}); + +// Without the failing statement, the same program compiles: the arrow's +// parameter is a dynamic slot. +test("the same program without the local class's failure compiles", async () => { + const result = await compileSource(`type Progress = (done: number, total: number) => void; +type Callbacks = Progress | AbortSignal | Map string> | Set; +function make(): number { + const K = class { + m(): number { + return 2; + } + }, + size = (cb: Callbacks): number => (cb instanceof Set ? cb.size : 0); + return new K().m() + size(new Set([1])); +} +console.log(make()); +`); + expect(result.ok, JSON.stringify(result.ok ? [] : result.diagnostics)).toBe(true); +}); diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index a57d96f01..bdad62203 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v45"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v46"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v45"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v46"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/speculative-loop-emission.test.ts b/packages/compiler/test/speculative-loop-emission.test.ts index d26a0c96c..dcec6d8af 100644 --- a/packages/compiler/test/speculative-loop-emission.test.ts +++ b/packages/compiler/test/speculative-loop-emission.test.ts @@ -183,3 +183,47 @@ console.log(key(new Uint8Array([1, 2, 3, 0]), 6)); expect(fast).not.toMatch(/ashr i64 %/); expect(fast).not.toContain("@llvm.smul.with.overflow.i64"); }); + +test("a speculative loop reads its views' header fields once, before its copies", async () => { + // A view held by a const local or by a parameter is the same view in + // every iteration. Its data pointer is read before the branch to the fast + // copy, and no copy (ordinary, fast, or the one a failed check resumes + // in) reads it again. + const llvm = await emitLlvm(` +let last = 42; +function fill(array: ArrayBuffer, count: number): void { + const buf = new Float64Array(array); + let l = last; + for (let i = 0; i < count; i++) { + l = (l * 3877 + 29573) % 139968; + buf[i] = l; + } + last = l; +} +function fillInto(buf: Float64Array, count: number): void { + let l = last; + for (let i = 0; i < count; i++) { + l = (l * 3877 + 29573) % 139968; + buf[i] = l; + } + last = l; +} +const array = new ArrayBuffer(8 * 16); +fill(array, 16); +const view = new Float64Array(8); +fillInto(view, 8); +console.log(new Float64Array(array)[15], view[7], last); +`); + const dataReads = /getelementptr inbounds %ScrBytes, ptr %\w+, i64 0, i32 3$/gm; + for (const name of ["fill", "fillInto"]) { + const fn = + new RegExp(`^define internal void @sc_bf_${name}(?:\\.i32)?\\([^]*?^}`, "m").exec( + llvm, + )?.[0] ?? ""; + const branch = /^ {2}br i1 %\w+, label %spec\.fast\d+, label %spec\.slow\d+$/m.exec(fn); + expect(branch, name).not.toBeNull(); + const copies = fn.slice(branch!.index); + expect(copies.match(dataReads), name).toBeNull(); + expect(fn.slice(0, branch!.index).match(dataReads)?.length, name).toBeGreaterThan(0); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5025-lifetime-region-shared-candidate.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5025-lifetime-region-shared-candidate.ts.json new file mode 100644 index 000000000..73b06cc54 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5025-lifetime-region-shared-candidate.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5025-lifetime-region-shared-candidate.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6311-typed-array-integer-products.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6311-typed-array-integer-products.ts.json new file mode 100644 index 000000000..d3134396a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6311-typed-array-integer-products.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6311-typed-array-integer-products.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7508-pipeline-untyped-source-functions.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7508-pipeline-untyped-source-functions.mjs.json new file mode 100644 index 000000000..4027063d4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7508-pipeline-untyped-source-functions.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7508-pipeline-untyped-source-functions.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7509-pipeline-untyped-function-stages.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7509-pipeline-untyped-function-stages.mjs.json new file mode 100644 index 000000000..028a5c1c6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7509-pipeline-untyped-function-stages.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7509-pipeline-untyped-function-stages.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7510-pipeline-untyped-later-iterables.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7510-pipeline-untyped-later-iterables.mjs.json new file mode 100644 index 000000000..be18c801a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7510-pipeline-untyped-later-iterables.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7510-pipeline-untyped-later-iterables.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7511-pipeline-untyped-stages.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7511-pipeline-untyped-stages.cjs.json new file mode 100644 index 000000000..2353cc44c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7511-pipeline-untyped-stages.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7511-pipeline-untyped-stages.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7512-pipeline-any-stages.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7512-pipeline-any-stages.ts.json new file mode 100644 index 000000000..acea0e98d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7512-pipeline-any-stages.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7512-pipeline-any-stages.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7513-pipeline-untyped-stage-errors.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7513-pipeline-untyped-stage-errors.mjs.json new file mode 100644 index 000000000..39a5dba57 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7513-pipeline-untyped-stage-errors.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7513-pipeline-untyped-stage-errors.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7717-math-fdlibm.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7717-math-fdlibm.ts.json new file mode 100644 index 000000000..5c790ff61 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7717-math-fdlibm.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7717-math-fdlibm.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7718-math-fdlibm.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7718-math-fdlibm.cjs.json new file mode 100644 index 000000000..201049b54 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7718-math-fdlibm.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7718-math-fdlibm.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8008-string-index-paths.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8008-string-index-paths.ts.json new file mode 100644 index 000000000..5e2082ab9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8008-string-index-paths.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8008-string-index-paths.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8014-foreign-element-callbacks.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8014-foreign-element-callbacks.ts.json new file mode 100644 index 000000000..7754e8cb8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8014-foreign-element-callbacks.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8014-foreign-element-callbacks.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8210-require-computed-missing/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8210-require-computed-missing/main.ts.json new file mode 100644 index 000000000..659ee27aa --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8210-require-computed-missing/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8210-require-computed-missing/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8211-require-resolve-computed/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8211-require-resolve-computed/main.ts.json new file mode 100644 index 000000000..44f6cb75f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8211-require-resolve-computed/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8211-require-resolve-computed/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8212-native-helper-probe-no-display/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8212-native-helper-probe-no-display/main.ts.json new file mode 100644 index 000000000..2a83db362 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8212-native-helper-probe-no-display/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8212-native-helper-probe-no-display/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8230-function-record-dispatch.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8230-function-record-dispatch.ts.json new file mode 100644 index 000000000..1670b97b4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8230-function-record-dispatch.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8230-function-record-dispatch.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8231-function-table-shapes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8231-function-table-shapes.ts.json new file mode 100644 index 000000000..8f1963ef5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8231-function-table-shapes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8231-function-table-shapes.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8232-optional-function-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8232-optional-function-values.ts.json new file mode 100644 index 000000000..671364bc6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8232-optional-function-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8232-optional-function-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8332-worker-shared-counted-loops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8332-worker-shared-counted-loops.ts.json new file mode 100644 index 000000000..57308033d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8332-worker-shared-counted-loops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8332-worker-shared-counted-loops.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/boxed-record-identity.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/boxed-record-identity.ts.json new file mode 100644 index 000000000..759521f3e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/boxed-record-identity.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/boxed-record-identity.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/checked-union-record-assertions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/checked-union-record-assertions.ts.json new file mode 100644 index 000000000..ff77d1a95 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/checked-union-record-assertions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/checked-union-record-assertions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/constant-choice-chains.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/constant-choice-chains.ts.json new file mode 100644 index 000000000..059a21156 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/constant-choice-chains.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/constant-choice-chains.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-fresh.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-fresh.ts.json new file mode 100644 index 000000000..e1643bcab --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-fresh.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/dyn-cast-identity-fresh.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-holders.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-holders.ts.json new file mode 100644 index 000000000..e1a06ecc5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-holders.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/dyn-cast-identity-holders.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-workers.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-workers.ts.json new file mode 100644 index 000000000..e8ecb066e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-workers.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/dyn-cast-identity-workers.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity.ts.json new file mode 100644 index 000000000..fdf360d21 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/dyn-cast-identity.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-literal-identity.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-literal-identity.ts.json new file mode 100644 index 000000000..41fde2c81 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-literal-identity.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/dyn-literal-identity.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fs-readdir-recursive.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fs-readdir-recursive.ts.json new file mode 100644 index 000000000..c045eadf1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fs-readdir-recursive.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/fs-readdir-recursive.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-interface-void-calls.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-interface-void-calls.ts.json new file mode 100644 index 000000000..a5c69b714 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-interface-void-calls.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/generic-interface-void-calls.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-cycles.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-cycles.ts.json new file mode 100644 index 000000000..7df4fd38f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-cycles.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-cycles.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-declarations.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-declarations.ts.json new file mode 100644 index 000000000..0530e8e13 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-declarations.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-declarations.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-forward-capture.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-forward-capture.ts.json new file mode 100644 index 000000000..f77570166 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-forward-capture.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-forward-capture.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-function-slots.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-function-slots.ts.json new file mode 100644 index 000000000..88890074d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-function-slots.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-function-slots.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-held-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-held-values.ts.json new file mode 100644 index 000000000..2327ce301 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-held-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-held-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-locals.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-locals.ts.json new file mode 100644 index 000000000..9a45d0613 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-locals.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-locals.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-pi-shapes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-pi-shapes.ts.json new file mode 100644 index 000000000..d1603f029 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-pi-shapes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-pi-shapes.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-registered-statics.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-registered-statics.ts.json new file mode 100644 index 000000000..b3d375ea6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-registered-statics.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-registered-statics.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/method-signature-any-params.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/method-signature-any-params.ts.json new file mode 100644 index 000000000..b3f7ed165 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/method-signature-any-params.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/method-signature-any-params.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/never-narrowed-literal-argument.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/never-narrowed-literal-argument.ts.json new file mode 100644 index 000000000..55b7b5d8a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/never-narrowed-literal-argument.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/never-narrowed-literal-argument.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/rest-tuple-shorthand-property.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/rest-tuple-shorthand-property.ts.json new file mode 100644 index 000000000..ca2d5289c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/rest-tuple-shorthand-property.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/rest-tuple-shorthand-property.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-loop-view-headers.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-loop-view-headers.ts.json new file mode 100644 index 000000000..9fcddc195 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-loop-view-headers.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/speculative-loop-view-headers.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-plus-element-read-return.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-plus-element-read-return.ts.json new file mode 100644 index 000000000..1e78a1802 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-plus-element-read-return.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/string-plus-element-read-return.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/symbol-keyed-recursive-record.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/symbol-keyed-recursive-record.ts.json new file mode 100644 index 000000000..e46feaab2 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/symbol-keyed-recursive-record.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/symbol-keyed-recursive-record.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/undeclared-key-optional-chain/main.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/undeclared-key-optional-chain/main.cjs.json new file mode 100644 index 000000000..252e5eb99 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/undeclared-key-optional-chain/main.cjs.json @@ -0,0 +1,18 @@ +{ + "order": [ + "/tests/corpus/undeclared-key-optional-chain/main.cjs" + ], + "diags": [ + { + "code": "SC1010", + "message": "the 'hostilekit' module is not supported yet", + "loc": { + "file": "/tests/corpus/undeclared-key-optional-chain/main.cjs", + "start": 289, + "end": 329 + }, + "milestone": "M4", + "hint": "relative imports (./file, ../dir/file), package.json-mediated project imports (#alias via the imports field, self-name references via exports), statically compiled npm packages, and the built-in fs, fs/promises, path, os, url, crypto, zlib, child_process, net, http, tls, https, http2, dgram, dns, dns/promises, util, util/types, string_decoder, querystring, readline, events, stream, stream/promises, stream/consumers, stream/web, buffer, assert, assert/strict, worker_threads, cluster, tty, async_hooks, timers, timers/promises, diagnostics_channel, perf_hooks, module, v8, and console modules (bare or node:-prefixed) and node:test (node:-prefixed only, like in Node) are supported" + } + ] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-clone-empty-array-overrides.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-clone-empty-array-overrides.ts.json new file mode 100644 index 000000000..cb11c5098 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-clone-empty-array-overrides.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/union-clone-empty-array-overrides.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-arm.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-arm.ts.json new file mode 100644 index 000000000..a29a9d99a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-arm.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/union-record-kept-fields-arm.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-no-discriminant.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-no-discriminant.ts.json new file mode 100644 index 000000000..f40acc997 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-no-discriminant.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/union-record-kept-fields-no-discriminant.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-tuple-element-read.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-tuple-element-read.ts.json new file mode 100644 index 000000000..bf23de474 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-tuple-element-read.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/union-tuple-element-read.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/var-object-filled-later.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/var-object-filled-later.js.json new file mode 100644 index 000000000..26329147f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/var-object-filled-later.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/var-object-filled-later.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/promise-all-generic-mixed-union.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/promise-all-generic-mixed-union.ts.json new file mode 100644 index 000000000..2732b9539 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/promise-all-generic-mixed-union.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/promise-all-generic-mixed-union.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/untyped-map-into-interface.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/untyped-map-into-interface.ts.json new file mode 100644 index 000000000..1d39d436f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/untyped-map-into-interface.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/untyped-map-into-interface.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-keyed-construction/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-keyed-construction/main.ts.json new file mode 100644 index 000000000..5817aa040 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-keyed-construction/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/types-keyed-construction/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts.json new file mode 100644 index 000000000..8c56898f2 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/type-patches-command.test.ts b/packages/compiler/test/type-patches-command.test.ts new file mode 100644 index 000000000..2ac8e1c9d --- /dev/null +++ b/packages/compiler/test/type-patches-command.test.ts @@ -0,0 +1,198 @@ +/* `scriptc types patch` and `scriptc types list` + * (frontend/type-patches/command.ts) on a copy of the type patch fixture: + * matches are shown before anything is written, the patch lands in the + * nearest package.json under a range covering the installed version (or + * an existing entry that covers it), and nothing is written when the + * selector matches nothing or the build already uses the requested type. */ +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, test } from "vitest"; +import { nodeFrontendServices } from "../src/frontend/services-node.js"; +import { runTypePatchCommand } from "../src/frontend/type-patches/command.js"; + +const fixture = join(import.meta.dirname, "../../../tests/fixtures/type-patches"); +const parse = (path: string, text: string, kind: "ts" | "js") => + nodeFrontendServices().parse(path, text, kind); +let root: string; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "scriptc-types-command-")); + cpSync(fixture, root, { recursive: true }); + writeFileSync( + join(root, "package.json"), + JSON.stringify({ name: "app", private: true, type: "module" }, null, 2) + "\n", + ); +}); +afterEach(() => rmSync(root, { recursive: true, force: true })); + +const patch = (pkg: string, selector: string, type: string, extra: object = {}) => + runTypePatchCommand( + { action: "patch", cwd: root, pkg, selector, type, range: null, dryRun: false, ...extra }, + parse, + ); + +test("patch shows the matches and writes the entry", () => { + const result = patch("termctx", "OutputContext.width", "| undefined"); + expect(result).toEqual({ + stdout: + "termctx 1.2.0 (node_modules/termctx)\n" + + " index.js:24 number -> number | undefined\n" + + `wrote package.json scriptc.typePatches["termctx@^1.2.0"].declarations["OutputContext.width"]\n`, + stderr: "", + exitCode: 0, + }); + expect(JSON.parse(readFileSync(join(root, "package.json"), "utf8"))).toEqual({ + name: "app", + private: true, + type: "module", + scriptc: { + typePatches: { + "termctx@^1.2.0": { declarations: { "OutputContext.width": "| undefined" } }, + }, + }, + }); + // A second patch joins the entry whose range covers the installed copy. + expect(patch("termctx", "Terminal.columns()", "| undefined").exitCode).toBe(0); + const text = readFileSync(join(root, "package.json"), "utf8"); + expect(text.endsWith("}\n")).toBe(true); + expect(JSON.parse(text).scriptc.typePatches).toEqual({ + "termctx@^1.2.0": { + declarations: { "OutputContext.width": "| undefined", "Terminal.columns()": "| undefined" }, + }, + }); +}); + +test("--range and --dry-run", () => { + const dry = patch("ms", "index.js:parse()", "| undefined", { dryRun: true, range: "2.1.x" }); + expect(dry.stdout).toBe( + "ms 2.1.3 (node_modules/ms)\n" + + " index.js:44 Number -> Number | undefined\n" + + `would write package.json scriptc.typePatches["ms@2.1.x"].declarations["index.js:parse()"]\n`, + ); + expect(JSON.parse(readFileSync(join(root, "package.json"), "utf8")).scriptc).toBeUndefined(); +}); + +test("nothing is written for a selector that matches nothing or is already so", () => { + const missing = patch("termctx", "OutputContext.height", "| undefined"); + expect(missing.exitCode).toBe(1); + expect(missing.stderr).toBe( + "scriptc: 'OutputContext.height' matches no declaration\nnothing was written\n", + ); + const redundant = patch("ms", "index.js:fmtShort(ms)", "| Number"); + expect(redundant).toMatchObject({ exitCode: 0 }); + expect(redundant.stdout).toContain("the declarations already say this; nothing was written\n"); + expect(patch("absent-package", "A.b", "| undefined").stderr).toBe( + "scriptc: absent-package is not installed here\n", + ); + expect(patch("termctx", "1bad", "| undefined").stderr).toContain("is not a selector"); + expect(JSON.parse(readFileSync(join(root, "package.json"), "utf8")).scriptc).toBeUndefined(); +}); + +test("a type the declarations already say replaces a covering patch that says another", () => { + const manifest = join(root, "package.json"); + writeFileSync( + manifest, + JSON.stringify( + { + name: "app", + scriptc: { + typePatches: { + "termctx@^1.2.0": { declarations: { "OutputContext.width": "| undefined" } }, + }, + }, + }, + null, + 2, + ) + "\n", + ); + const result = patch("termctx", "OutputContext.width", "number"); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain( + `wrote package.json scriptc.typePatches["termctx@^1.2.0"].declarations["OutputContext.width"]\n`, + ); + expect(JSON.parse(readFileSync(manifest, "utf8")).scriptc.typePatches).toEqual({ + "termctx@^1.2.0": { declarations: { "OutputContext.width": "number" } }, + }); + expect(patch("termctx", "OutputContext.width", "number").stdout).toContain( + "the declarations already say this; nothing was written\n", + ); +}); + +test("list shows each patch and what it matches", () => { + writeFileSync(join(root, "package.json"), readFileSync(join(fixture, "package.json"))); + const listed = runTypePatchCommand({ action: "list", cwd: root }, parse); + expect(listed.stdout).toBe( + [ + "termctx@^1.0.0", + " OutputContext.width: | undefined", + " node_modules/termctx/index.js:24 number -> number | undefined", + " Terminal.columns(): | undefined", + " node_modules/termctx/index.js:14 number -> number | undefined", + "ms@^2.1.0", + " index.js:parse(): | undefined", + " node_modules/ms/index.js:44 Number -> Number | undefined", + " export=(): | undefined", + " node_modules/ms/index.js:22 String|Number -> String|Number | undefined", + "", + ].join("\n"), + ); + writeFileSync(join(root, "stale.json"), ""); + writeFileSync( + join(root, "package.json"), + JSON.stringify({ + scriptc: { typePatches: { "termctx@^2.0.0": { declarations: { "A.b": "| undefined" } } } }, + }), + ); + expect(runTypePatchCommand({ action: "list", cwd: root }, parse).stdout).toBe( + "termctx@^2.0.0\n does not apply to the installed termctx 1.2.0\n A.b: | undefined\n", + ); +}); + +test("nothing is written when a replacement is ambiguous in one of the files it matches", () => { + const pkg = join(root, "node_modules/overloads"); + mkdirSync(pkg, { recursive: true }); + writeFileSync( + join(pkg, "package.json"), + JSON.stringify({ name: "overloads", version: "1.0.0", types: "index.d.ts" }), + ); + writeFileSync(join(pkg, "index.d.ts"), "export declare function pick(a: string): string;\n"); + writeFileSync( + join(pkg, "more.d.ts"), + "export declare function pick(a: string): string;\nexport declare function pick(a: number): string;\n", + ); + const result = patch("overloads", "pick()", "string | undefined"); + expect(result.exitCode).toBe(1); + expect(result.stderr).toBe( + "scriptc: 'pick()' matches 2 declarations in more.d.ts; a replacement must name one " + + "(widen them, or replace the declaration file) (more.d.ts)\nnothing was written\n", + ); + expect(JSON.parse(readFileSync(join(root, "package.json"), "utf8")).scriptc).toBeUndefined(); +}); + +test("the patch goes to the covering entry the compiler applies", () => { + const manifest = join(root, "package.json"); + const entries = { + "termctx@^1.0.0": { declarations: { "OutputContext.width": "| null" } }, + "termctx@1.x": { declarations: { "OutputContext.width": "| null" } }, + }; + writeFileSync(manifest, JSON.stringify({ name: "app", scriptc: { typePatches: entries } })); + // Both ranges cover termctx 1.2.0, and the later entry wins on the same + // selector: it takes the patch. + const result = patch("termctx", "OutputContext.width", "| undefined"); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain( + `wrote package.json scriptc.typePatches["termctx@1.x"].declarations["OutputContext.width"]\n`, + ); + expect(JSON.parse(readFileSync(manifest, "utf8")).scriptc.typePatches).toEqual({ + "termctx@^1.0.0": { declarations: { "OutputContext.width": "| null" } }, + "termctx@1.x": { declarations: { "OutputContext.width": "| undefined" } }, + }); + // Asked for the earlier range, the later entry would still win. + const earlier = patch("termctx", "OutputContext.width", "| undefined", { range: "^1.0.0" }); + expect(earlier.exitCode).toBe(1); + expect(earlier.stderr).toBe( + `scriptc: scriptc.typePatches["termctx@1.x"] also patches 'OutputContext.width' for termctx 1.2.0 ` + + `and takes precedence over scriptc.typePatches["termctx@^1.0.0"]\nnothing was written\n`, + ); +}); diff --git a/packages/compiler/test/worker-shared-loop-version-emission.test.ts b/packages/compiler/test/worker-shared-loop-version-emission.test.ts index 7eebcc2e4..ad7ec1ae9 100644 --- a/packages/compiler/test/worker-shared-loop-version-emission.test.ts +++ b/packages/compiler/test/worker-shared-loop-version-emission.test.ts @@ -56,11 +56,13 @@ if (isMainThread) { // The guard tests the bounds first, then whether the view is shared. expect(fn).toMatch(/^bytes\.version\.owned\d*:/m); expect(fn).toMatch(/^bytes\.version\.fast\.shared\d*:/m); - // The shared copy reads with no bounds test, and shared-safe: a view - // with shared storage loads atomically. + // The shared copy reads with no bounds test. Each copy is a counted loop + // that polls between blocks, so its read at the counter is a plain load + // (plainSharedElement in expr-bytes.ts). const shared = blocks(fn, "bytes.version.fast.shared"); expect(shared).not.toMatch(/icmp (ult|uge|ugt) i64 %\w+, %\w+/); - expect(fn).toMatch(/load atomic double, ptr %\w+ monotonic/); + expect(fn).toMatch(/load double, ptr %\w+/); + expect(fn).not.toMatch(/load atomic double/); }); test("a program without workers keeps two copies of the loop", async () => { diff --git a/packages/llvm-linux-arm64-gnu/scripts/build.mjs b/packages/llvm-linux-arm64-gnu/scripts/build.mjs index 603a77d39..1036d7c6e 100644 --- a/packages/llvm-linux-arm64-gnu/scripts/build.mjs +++ b/packages/llvm-linux-arm64-gnu/scripts/build.mjs @@ -1,9 +1,9 @@ #!/usr/bin/env node -import { nativeTargetCmakeArgs } from "../../../native/llvm-codegen/targets.mjs"; -import { execFileSync } from "node:child_process"; -import { chmodSync, copyFileSync, mkdirSync, readFileSync } from "node:fs"; +import { buildLinuxHelper } from "../../../native/llvm-codegen/linux-helper.mjs"; +import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; + const root = dirname(dirname(fileURLToPath(import.meta.url))); const repo = fileURLToPath(new URL("../../..", import.meta.url)); if (process.platform !== "linux" || process.arch !== "arm64") { @@ -11,30 +11,10 @@ if (process.platform !== "linux" || process.arch !== "arm64") { process.exit(0); } const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); -const build = join(repo, "node_modules/.cache/scriptc-llvm-linux-arm64-gnu"); -execFileSync( - "cmake", - [ - "-S", - join(repo, "native/llvm-codegen"), - "-B", - build, - "-G", - "Ninja", - `-DLLVM_DIR=${process.env.LLVM_DIR ?? "/usr/lib/llvm-22/lib/cmake/llvm"}`, - `-DSCRIPTC_PACKAGE_VERSION=${manifest.version}`, - "-DSCRIPTC_DEFAULT_TARGET=aarch64-unknown-linux-gnu", - "-DSCRIPTC_DEFAULT_DATA_LAYOUT=e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32", - ...nativeTargetCmakeArgs, - "-DSCRIPTC_STATIC_LINUX_BUILD=ON", - "-DCMAKE_EXE_LINKER_FLAGS=-static", - "-DCMAKE_BUILD_TYPE=Release", - ], - { stdio: "inherit" }, -); -execFileSync("cmake", ["--build", build, "--target", "scriptc-llvm-codegen"], { stdio: "inherit" }); -mkdirSync(join(root, "bin"), { recursive: true }); -const output = join(root, "bin", "scriptc-llvm-codegen"); -copyFileSync(join(build, "scriptc-llvm-codegen"), output); -execFileSync("strip", [output], { stdio: "inherit" }); -chmodSync(output, 0o755); +buildLinuxHelper({ + arch: "arm64", + libc: "gnu", + version: manifest.version, + build: join(repo, "node_modules/.cache/scriptc-llvm-linux-arm64-gnu"), + output: join(root, "bin", "scriptc-llvm-codegen"), +}); diff --git a/packages/llvm-linux-arm64-musl/scripts/build.mjs b/packages/llvm-linux-arm64-musl/scripts/build.mjs index 3b9c49983..187459b2a 100644 --- a/packages/llvm-linux-arm64-musl/scripts/build.mjs +++ b/packages/llvm-linux-arm64-musl/scripts/build.mjs @@ -1,9 +1,9 @@ #!/usr/bin/env node -import { nativeTargetCmakeArgs } from "../../../native/llvm-codegen/targets.mjs"; -import { execFileSync } from "node:child_process"; -import { chmodSync, copyFileSync, mkdirSync, readFileSync } from "node:fs"; +import { buildLinuxHelper } from "../../../native/llvm-codegen/linux-helper.mjs"; +import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; + const root = dirname(dirname(fileURLToPath(import.meta.url))); const repo = fileURLToPath(new URL("../../..", import.meta.url)); if (process.platform !== "linux" || process.arch !== "arm64") { @@ -11,32 +11,10 @@ if (process.platform !== "linux" || process.arch !== "arm64") { process.exit(0); } const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); -const build = join(repo, "node_modules/.cache/scriptc-llvm-linux-arm64-musl"); -execFileSync( - "cmake", - [ - "-S", - join(repo, "native/llvm-codegen"), - "-B", - build, - "-G", - "Ninja", - `-DCMAKE_C_COMPILER=${process.env.CC ?? "cc"}`, - `-DCMAKE_CXX_COMPILER=${process.env.CXX ?? "c++"}`, - "-DSCRIPTC_STATIC_LINUX_BUILD=ON", - "-DCMAKE_EXE_LINKER_FLAGS=-static", - `-DLLVM_DIR=${process.env.LLVM_DIR ?? "/usr/lib/llvm-22/lib/cmake/llvm"}`, - `-DSCRIPTC_PACKAGE_VERSION=${manifest.version}`, - "-DSCRIPTC_DEFAULT_TARGET=aarch64-unknown-linux-musl", - "-DSCRIPTC_DEFAULT_DATA_LAYOUT=e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32", - ...nativeTargetCmakeArgs, - "-DCMAKE_BUILD_TYPE=Release", - ], - { stdio: "inherit" }, -); -execFileSync("cmake", ["--build", build, "--target", "scriptc-llvm-codegen"], { stdio: "inherit" }); -mkdirSync(join(root, "bin"), { recursive: true }); -const output = join(root, "bin", "scriptc-llvm-codegen"); -copyFileSync(join(build, "scriptc-llvm-codegen"), output); -execFileSync("strip", [output], { stdio: "inherit" }); -chmodSync(output, 0o755); +buildLinuxHelper({ + arch: "arm64", + libc: "musl", + version: manifest.version, + build: join(repo, "node_modules/.cache/scriptc-llvm-linux-arm64-musl"), + output: join(root, "bin", "scriptc-llvm-codegen"), +}); diff --git a/packages/llvm-linux-x64-gnu/scripts/build.mjs b/packages/llvm-linux-x64-gnu/scripts/build.mjs index 5f10c9652..f70fdfa43 100644 --- a/packages/llvm-linux-x64-gnu/scripts/build.mjs +++ b/packages/llvm-linux-x64-gnu/scripts/build.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node -import { nativeTargetCmakeArgs } from "../../../native/llvm-codegen/targets.mjs"; -import { execFileSync } from "node:child_process"; -import { chmodSync, copyFileSync, mkdirSync, readFileSync } from "node:fs"; +import { buildLinuxHelper } from "../../../native/llvm-codegen/linux-helper.mjs"; +import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -12,30 +11,10 @@ if (process.platform !== "linux" || process.arch !== "x64") { process.exit(0); } const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); -const build = join(repo, "node_modules/.cache/scriptc-llvm-linux-x64-gnu"); -execFileSync( - "cmake", - [ - "-S", - join(repo, "native/llvm-codegen"), - "-B", - build, - "-G", - "Ninja", - `-DLLVM_DIR=${process.env.LLVM_DIR ?? "/usr/lib/llvm-22/lib/cmake/llvm"}`, - `-DSCRIPTC_PACKAGE_VERSION=${manifest.version}`, - "-DSCRIPTC_DEFAULT_TARGET=x86_64-unknown-linux-gnu", - "-DSCRIPTC_DEFAULT_DATA_LAYOUT=e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", - ...nativeTargetCmakeArgs, - "-DSCRIPTC_STATIC_LINUX_BUILD=ON", - "-DCMAKE_EXE_LINKER_FLAGS=-static", - "-DCMAKE_BUILD_TYPE=Release", - ], - { stdio: "inherit" }, -); -execFileSync("cmake", ["--build", build, "--target", "scriptc-llvm-codegen"], { stdio: "inherit" }); -mkdirSync(join(root, "bin"), { recursive: true }); -const output = join(root, "bin", "scriptc-llvm-codegen"); -copyFileSync(join(build, "scriptc-llvm-codegen"), output); -execFileSync("strip", [output], { stdio: "inherit" }); -chmodSync(output, 0o755); +buildLinuxHelper({ + arch: "x64", + libc: "gnu", + version: manifest.version, + build: join(repo, "node_modules/.cache/scriptc-llvm-linux-x64-gnu"), + output: join(root, "bin", "scriptc-llvm-codegen"), +}); diff --git a/packages/llvm-linux-x64-musl/scripts/build.mjs b/packages/llvm-linux-x64-musl/scripts/build.mjs index 20987e0e5..bb5a0ec7e 100644 --- a/packages/llvm-linux-x64-musl/scripts/build.mjs +++ b/packages/llvm-linux-x64-musl/scripts/build.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node -import { nativeTargetCmakeArgs } from "../../../native/llvm-codegen/targets.mjs"; -import { execFileSync } from "node:child_process"; -import { chmodSync, copyFileSync, mkdirSync, readFileSync } from "node:fs"; +import { buildLinuxHelper } from "../../../native/llvm-codegen/linux-helper.mjs"; +import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -12,32 +11,10 @@ if (process.platform !== "linux" || process.arch !== "x64") { process.exit(0); } const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); -const build = join(repo, "node_modules/.cache/scriptc-llvm-linux-x64-musl"); -execFileSync( - "cmake", - [ - "-S", - join(repo, "native/llvm-codegen"), - "-B", - build, - "-G", - "Ninja", - `-DCMAKE_C_COMPILER=${process.env.CC ?? "cc"}`, - `-DCMAKE_CXX_COMPILER=${process.env.CXX ?? "c++"}`, - "-DSCRIPTC_STATIC_LINUX_BUILD=ON", - "-DCMAKE_EXE_LINKER_FLAGS=-static", - `-DLLVM_DIR=${process.env.LLVM_DIR ?? "/usr/lib/llvm-22/lib/cmake/llvm"}`, - `-DSCRIPTC_PACKAGE_VERSION=${manifest.version}`, - "-DSCRIPTC_DEFAULT_TARGET=x86_64-unknown-linux-musl", - "-DSCRIPTC_DEFAULT_DATA_LAYOUT=e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", - ...nativeTargetCmakeArgs, - "-DCMAKE_BUILD_TYPE=Release", - ], - { stdio: "inherit" }, -); -execFileSync("cmake", ["--build", build, "--target", "scriptc-llvm-codegen"], { stdio: "inherit" }); -mkdirSync(join(root, "bin"), { recursive: true }); -const output = join(root, "bin", "scriptc-llvm-codegen"); -copyFileSync(join(build, "scriptc-llvm-codegen"), output); -execFileSync("strip", [output], { stdio: "inherit" }); -chmodSync(output, 0o755); +buildLinuxHelper({ + arch: "x64", + libc: "musl", + version: manifest.version, + build: join(repo, "node_modules/.cache/scriptc-llvm-linux-x64-musl"), + output: join(root, "bin", "scriptc-llvm-codegen"), +}); diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 3305d69de..6006e73ac 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -348,6 +348,11 @@ async function build() { "artifacts/licenses/v8-number-radix.txt", "BSD-3-Clause", ], + [ + join(vendorRoot, "v8-ieee754", "LICENSE"), + "artifacts/licenses/v8-ieee754.txt", + "BSD-3-Clause", + ], [join(vendorRoot, "unicode", "LICENSE"), "artifacts/licenses/unicode.txt", "Unicode-3.0"], [join(zlib, "LICENSE"), "artifacts/licenses/zlib.txt", "Zlib"], [join(vendorRoot, "brotli", "LICENSE"), "artifacts/licenses/brotli.txt", "MIT"], @@ -366,7 +371,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 45, marker: "scr_runtime_abi_v45" }, + runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 49e5896b6..16178e851 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -52,6 +52,10 @@ export const BASE_RUNTIME_SOURCES = [ "scr_alloc.c", // crypto.getCipherInfo's table; it links only into programs that call it. "scr_cipher_info.c", + // Math's transcendental functions as V8 computes them, and on glibc the + // exact libm functions that let a program link without libm.so. + "scr_ieee754.c", + "scr_glibc_math.c", ]; export const OPTIONAL_RUNTIME_SOURCES = [ @@ -68,6 +72,8 @@ export const OPTIONAL_RUNTIME_SOURCES = [ ["scr_bytes_dyn.c", all("dynBuffer", "dynInvoke")], // Builtin functions read as values in JavaScript dependencies. ["scr_builtin_values.c", "builtinValues"], + // createRequire's computed require and require.resolve. + ["scr_require.c", "computedRequire"], ["scr_dc.c", "dc"], // The dynamic tier's jsval values (two-tier design I3): linked only into // programs whose IR has dynamic-tier operations. @@ -136,6 +142,7 @@ export const OPTIONAL_RUNTIME_SOURCES = [ // Own properties JavaScript adds to children. ["scr_dyn_handle_props.c", "handleProps"], ["scr_decl_adopt.c", "declAdopt"], + ["scr_type_origin.c", "typeOrigins"], // for await over checked-dynamic values. ["scr_dyn_for_await.c", "dynForAwait"], ["scr_dyn_for_await_stream.c", all("dynForAwait", "stream", "dynAsync")], diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 8c0863b04..357ba7337 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -447,6 +447,11 @@ async function build() { "artifacts/licenses/v8-number-radix.txt", "BSD-3-Clause", ], + [ + join(vendorRoot, "v8-ieee754", "LICENSE"), + "artifacts/licenses/v8-ieee754.txt", + "BSD-3-Clause", + ], [join(zlib, "LICENSE"), "artifacts/licenses/zlib.txt", "Zlib"], [join(vendorRoot, "brotli", "LICENSE"), "artifacts/licenses/brotli.txt", "MIT"], [join(mbedtls, "LICENSE"), "artifacts/licenses/mbedtls.txt", "Apache-2.0"], @@ -469,7 +474,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 45, marker: "scr_runtime_abi_v45" }, + runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_alloc.c b/packages/runtime/src/scr_alloc.c index a27df378a..087c926f7 100644 --- a/packages/runtime/src/scr_alloc.c +++ b/packages/runtime/src/scr_alloc.c @@ -2116,6 +2116,99 @@ void scr_heap_stats(double *total, double *used) { *used = (double)(held - (idle < held ? idle : held) + sys_used); } +/* scr_heap_trim: returns the heap pages that hold only freed memory to the + * system after a phase's data died (scr_collect_cycles_phase). The + * size-class slices keep freed blocks on their class's free list, which only + * later blocks of the same class reuse, so a program whose next phase + * allocates other sizes would otherwise grow its resident memory on top of + * the freed heap. */ +#if defined(__GLIBC__) && !defined(SCR_HEAP_SANITIZER) +#include +#endif +#if SCR_SMALL_ALLOC && !defined(SCR_WORKERS) && defined(__linux__) && defined(MADV_DONTNEED) +#include +#include + +/* Returns the pages of class c's slice below its frontier that hold only + * free blocks (MADV_DONTNEED), and relinks the class's other free blocks in + * address order. A block wholly inside a returned page leaves the free list: + * the class carves new blocks past its frontier instead, and the slice's + * reservation is far larger than what a program fills. */ +static void scr_sa_return_free_pages(unsigned c, size_t page) { + size_t sz = scr_sa_class_size(c); + char *start = (char *)scr_sa.base + ((size_t)c << scr_sa.shift); + char *top = scr_sa.bump[c]; + size_t nblocks = (size_t)(top - start) / sz; + if (!nblocks || !scr_sa.free[c]) return; + size_t words = (nblocks + 63) / 64; + uint64_t *bits = mmap(NULL, words * 8, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + if (bits == MAP_FAILED) return; + for (ScrSaBlock *b = scr_sa.free[c]; b; b = b->next) { + size_t k = (size_t)((char *)b - start) / sz; + bits[k / 64] |= (uint64_t)1 << (k % 64); + } +#define SCR_SA_FREE_BIT(k) ((bits[(k) / 64] >> ((k) % 64)) & 1) + /* Mark the returned pages' blocks by clearing their bits after the + * madvise; a page is returned when every block overlapping it is free. */ + uintptr_t first = ((uintptr_t)start + page - 1) & ~(uintptr_t)(page - 1); + uintptr_t last = (uintptr_t)top & ~(uintptr_t)(page - 1); + char *run = NULL; + for (uintptr_t pg = first; pg <= last; pg += page) { + bool empty = false; + if (pg < last) { + size_t k0 = (size_t)((char *)pg - start) / sz; + size_t k1 = (size_t)((char *)pg + page - 1 - start) / sz; + if (k1 >= nblocks) k1 = nblocks - 1; + empty = true; + for (size_t k = k0; k <= k1 && empty; k++) empty = SCR_SA_FREE_BIT(k); + } + if (empty && !run) run = (char *)pg; + if (!empty && run) { + (void)madvise(run, (size_t)((char *)pg - run), MADV_DONTNEED); + /* Blocks wholly inside the returned pages are dropped. */ + size_t k0 = ((size_t)(run - start) + sz - 1) / sz; + size_t k1 = (size_t)((char *)pg - start) / sz; + for (size_t k = k0; k < k1 && (char *)start + (k + 1) * sz <= (char *)pg; k++) + bits[k / 64] &= ~((uint64_t)1 << (k % 64)); + run = NULL; + } + } + /* The kept free blocks, in address order. */ + ScrSaBlock *head = NULL, **tail = &head; + for (size_t w = 0; w < words; w++) + for (uint64_t m = bits[w]; m; m &= m - 1) { + ScrSaBlock *b = (ScrSaBlock *)(start + (w * 64 + (size_t)__builtin_ctzll(m)) * sz); + *tail = b; + tail = &b->next; + } + *tail = NULL; + scr_sa.free[c] = head; +#undef SCR_SA_FREE_BIT + (void)munmap(bits, words * 8); +} +#endif + +void scr_heap_trim(void) { +#ifdef SCR_SA_TRIM + /* A worker runtime returns the calling thread's arena pages as a parking + * thread does; the arena stays the thread's own. */ + if (scr_sa_arena >= 0 && scr_sa_trim_worth()) { + scr_sa_trim(); + return; + } +#elif SCR_SMALL_ALLOC && !defined(SCR_WORKERS) && defined(__linux__) && defined(MADV_DONTNEED) + if (scr_sa.span) { + long page = sysconf(_SC_PAGESIZE); + for (unsigned c = 0; c < SCR_SA_NCLASS; c++) + if (scr_sa.lim[c]) scr_sa_return_free_pages(c, page > 0 ? (size_t)page : 4096); + } +#endif +#if defined(__GLIBC__) && !defined(SCR_HEAP_SANITIZER) + /* Large blocks: the system allocator's free pages too. */ + (void)malloc_trim(0); +#endif +} + /* ── program-lifetime region ──────────────────────────────────────────── * Instances of compact classes (llvm/classes.ts LlClassMeta.compact) in * programs built for Linux without sanitizers (runtimes with the size-class diff --git a/packages/runtime/src/scr_bytes.c b/packages/runtime/src/scr_bytes.c index 6399683fc..6bb56195a 100644 --- a/packages/runtime/src/scr_bytes.c +++ b/packages/runtime/src/scr_bytes.c @@ -4141,8 +4141,8 @@ static bool scr_bytes_rw_check(const ScrBytes *b, double offset, size_t width) { * exactly Node — and writes zeros downstream. */ static bool scr_bytes_check_int(const ScrBytes *b, double value, double offset, size_t width, bool sign) { - double max = sign ? exp2((double)(8 * width - 1)) - 1 : exp2((double)(8 * width)) - 1; - double min = sign ? -exp2((double)(8 * width - 1)) : 0; + double max = sign ? ldexp(1.0, (int)(8 * width - 1)) - 1 : ldexp(1.0, (int)(8 * width)) - 1; + double min = sign ? -ldexp(1.0, (int)(8 * width - 1)) : 0; if (value > max || value < min) { char recv[48]; scr_num_received(value, recv); diff --git a/packages/runtime/src/scr_bytes_dyn.c b/packages/runtime/src/scr_bytes_dyn.c index e390ddfce..5ce36bdac 100644 --- a/packages/runtime/src/scr_bytes_dyn.c +++ b/packages/runtime/src/scr_bytes_dyn.c @@ -44,8 +44,8 @@ static size_t scr_bytes_dyn_width(ScrBytesNumKind kind) { /* Node's checkInt range test and message, before the offset is looked at. */ static bool scr_bytes_dyn_value_in_range(double value, size_t width, bool sign) { - double max = sign ? exp2((double)(8 * width - 1)) - 1 : exp2((double)(8 * width)) - 1; - double min = sign ? -exp2((double)(8 * width - 1)) : 0; + double max = sign ? ldexp(1.0, (int)(8 * width - 1)) - 1 : ldexp(1.0, (int)(8 * width)) - 1; + double min = sign ? -ldexp(1.0, (int)(8 * width - 1)) : 0; if (!(value > max || value < min)) return true; char recv[48]; scr_num_received(value, recv); diff --git a/packages/runtime/src/scr_closure.h b/packages/runtime/src/scr_closure.h index 9644722ce..cfca90313 100644 --- a/packages/runtime/src/scr_closure.h +++ b/packages/runtime/src/scr_closure.h @@ -98,7 +98,7 @@ typedef struct ScrClosure { * (like the dyn→closure edge): a cycle through it is merely never * collected. */ ScrBox *props; - uint32_t function_kind; /* low bits: 0 ordinary, 1 generator, 2 async, 3 async generator; bit 2: ordinary own prototype; bit 3: checked callable adapter in caps[0] */ + uint32_t function_kind; /* low bits: 0 ordinary, 1 generator, 2 async, 3 async generator; bit 2: ordinary own prototype; bit 3: checked callable adapter in caps[0]; bit 4: SCR_FN_RECEIVER_FREE */ /* The root's `length` as boxed values report it; meaningful only when * identity is set (the root's own boxes carry their arity). */ uint32_t identity_length; @@ -117,6 +117,13 @@ typedef struct ScrClosure { ScrBox *caps[]; } ScrClosure; +/* function_kind bit 4: the body never observes the ambient receiver (the + * `this` an unbound call binds), so compiled callers skip the receiver + * bracket around an indirect call. Set only by the compiler; the kind + * proper is function_kind & SCR_FN_KIND_MASK. */ +#define SCR_FN_RECEIVER_FREE 16u +#define SCR_FN_KIND_MASK 15u + ScrClosure *scr_closure_new(void *fn, size_t ncaps); /* +1; caller fills caps with +1 box refs */ static inline ScrClosure *scr_closure_retain(ScrClosure *c) { diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 67d4aca15..18c715cb2 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v45(void) {} +void scr_runtime_abi_v46(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index 206463e18..92392897a 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v45(void); +void scr_runtime_abi_v46(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 0d9df7106..d421e8e45 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -1474,7 +1474,10 @@ static void scr_scan_black(void *obj, uintptr_t depth) { const ScrCycType *type = scr_cyc_type(h); bool acyclic = type->trace == scr_cyc_no_children; if (!scr_cyc_is_buffered(h)) h->buf = acyclic ? SCR_CYC_ACYCLIC : 0; - if (scr_cyc_gen(h) < SCR_CYC_OLD) scr_cyc_push_work(&scr_promote, obj); + /* Such a candidate also keeps its generation: the generation names the + * buffer its slot is in (scr_cyc_on_dead), so it is promoted only once a + * pass takes it. */ + if (scr_cyc_gen(h) < SCR_CYC_OLD && !scr_cyc_is_buffered(h)) scr_cyc_push_work(&scr_promote, obj); if (acyclic) return; if (depth == SCR_CYC_WALK_DEPTH) { scr_cyc_push_work(&scr_restore_pending, obj); @@ -2350,7 +2353,9 @@ void scr_collect_cycles(void) { * finding anything, and the death rule's backoff then start over from the * heap the sweep leaves, as after a productive pass. Otherwise the next * phase's garbage would accumulate until the heap grew to that widened - * fraction. */ + * fraction. The freed memory then goes back to the system (scr_heap_trim): + * the next phase allocates other sizes than the dead data held, so it would + * reuse little of it and grow its resident memory on top. */ void scr_collect_cycles_phase(void) { scr_collect_cycles(); for (unsigned gen = SCR_CYC_MATURE; gen < SCR_CYC_NGENS; gen++) { @@ -2358,6 +2363,7 @@ void scr_collect_cycles_phase(void) { scr_cyc_growth_streak[gen] = 0; } scr_cyc_death_shift = 0; + scr_heap_trim(); } /* Worker TLS becomes unreachable when its OS thread returns. Release the diff --git a/packages/runtime/src/scr_decl_adopt.c b/packages/runtime/src/scr_decl_adopt.c index 7372826ab..fc1c4456c 100644 --- a/packages/runtime/src/scr_decl_adopt.c +++ b/packages/runtime/src/scr_decl_adopt.c @@ -15,7 +15,11 @@ * value has no prototype, accessors, symbol keys or frozen state) and * collects the shared ones; it throws the path TypeError before anything * changes. Phase two converts the shared nodes in place - * (scr_dyn_decl_become, scr_json.c). */ + * (scr_dyn_decl_become, scr_json.c). + * + * Every other checked cast of a dynamic value to a record or array runs the + * same pass leniently (scr_dyn_adopt_shared): a node that cannot become a + * capsule keeps the cast's copy. */ #include "scr_runtime.h" #include @@ -31,6 +35,9 @@ typedef struct { typedef struct { ScrDeclAdopt *items; size_t len, cap; + /* A checked cast outside the boundary (scr_dyn_adopt_shared): a node + * that cannot become a capsule keeps the copy instead of throwing. */ + bool lenient; } ScrDeclAdoptList; static void scr_decl_adopt_push(ScrDeclAdoptList *list, ScrDyn *node, ScrDyn *capsule) { @@ -44,7 +51,7 @@ static void scr_decl_adopt_push(ScrDeclAdoptList *list, ScrDyn *node, ScrDyn *ca list->len++; } -static bool scr_decl_adopt_plain(const ScrDyn *n, const ScrDynPath *path) { +static bool scr_decl_adopt_plain(const ScrDyn *n, const ScrDynPath *path, bool lenient) { const char *want = NULL; if (n->kind == SCR_DYN_OBJ) { if (n->prototype || n->null_proto || n->v.obj.class_name) @@ -72,7 +79,7 @@ static bool scr_decl_adopt_plain(const ScrDyn *n, const ScrDynPath *path) { want = "a plain array (this value is frozen, sealed or not extensible)"; } if (!want) return true; - scr_dyn_check_fail(path, want, n); + if (!lenient) scr_dyn_check_fail(path, want, n); return false; } @@ -82,6 +89,14 @@ static ScrDyn *scr_decl_capsule_ptr_holder(const ScrDeclAdoptList *list, const S return NULL; } +/* Whether the walk follows a child: plain objects and arrays. A boxed + * native record or array is already one object with its native value; a + * cast to another layout copies it (the copy keeps a cached identity per + * capsule, scr_dyn_typed_ref_cache_cast). */ +static bool scr_decl_adopt_follows(const ScrDyn *child) { + return child && (child->kind == SCR_DYN_OBJ || child->kind == SCR_DYN_ARR); +} + /* `capsule` is owned by the call: pushed or released. */ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsule, const ScrDynPath *path, unsigned depth) { @@ -90,15 +105,16 @@ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsu scr_dyn_release(capsule); return true; } - if (!scr_decl_adopt_plain(n, path)) { + if (!scr_decl_adopt_plain(n, path, list->lenient)) { scr_dyn_release(capsule); - return false; + return list->lenient; } if (n->rc > 1 && n->rc != SIZE_MAX) { ScrDyn *seen = scr_decl_capsule_ptr_holder(list, n); if (seen) { bool same = seen->v.typed_ref.ptr == capsule->v.typed_ref.ptr; scr_dyn_release(capsule); + if (!same && list->lenient) return true; if (!same) scr_dyn_check_fail(path, "a value that is not also stored elsewhere in this value (scriptc builds a separate native copy for each place)", n); return same; @@ -112,7 +128,7 @@ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsu if (!ops->has(ptr, i)) continue; const ScrStr *name = ops->names[i]; ScrDyn *child = scr_dyn_obj_get(n, name->data, name->len); - if (!child || (child->kind != SCR_DYN_OBJ && child->kind != SCR_DYN_ARR)) continue; + if (!scr_decl_adopt_follows(child)) continue; ScrDynPath p = { path, name->data, 0 }; if (!scr_decl_adopt_walk(list, child, ops->get(ptr, i), &p, depth + 1)) { if (!(n->rc > 1 && n->rc != SIZE_MAX)) scr_dyn_release(capsule); @@ -135,7 +151,7 @@ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsu size_t len = n->v.arr.len < view->v.arr.len ? n->v.arr.len : view->v.arr.len; for (size_t i = 0; i < len; i++) { ScrDyn *child = n->v.arr.items[i], *element = view->v.arr.items[i]; - if (!child || !element || (child->kind != SCR_DYN_OBJ && child->kind != SCR_DYN_ARR)) continue; + if (!element || !scr_decl_adopt_follows(child)) continue; ScrDynPath p = { path, NULL, i }; if (!scr_decl_adopt_walk(list, child, scr_dyn_retain(element), &p, depth + 1)) { scr_dyn_release(view); @@ -150,7 +166,7 @@ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsu const ScrDynTypedArrayOps *ops = capsule->v.typed_ref.array; for (size_t i = 0; i < n->v.arr.len; i++) { ScrDyn *child = n->v.arr.items[i]; - if (!child || (child->kind != SCR_DYN_OBJ && child->kind != SCR_DYN_ARR)) continue; + if (!scr_decl_adopt_follows(child)) continue; ScrDynPath p = { path, NULL, i }; if (!scr_decl_adopt_walk(list, child, ops->get(ptr, (double)i), &p, depth + 1)) { if (!(n->rc > 1 && n->rc != SIZE_MAX)) scr_dyn_release(capsule); @@ -162,8 +178,8 @@ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsu return true; } -void scr_dyn_decl_adopt(ScrDyn *source, ScrDyn *capsule) { - ScrDeclAdoptList list = { NULL, 0, 0 }; +static void scr_decl_adopt_run(ScrDyn *source, ScrDyn *capsule, bool lenient) { + ScrDeclAdoptList list = { NULL, 0, 0, lenient }; bool ok = scr_decl_adopt_walk(&list, source, scr_dyn_retain(capsule), NULL, 0); for (size_t i = 0; i < list.len; i++) { ScrDeclAdopt *item = &list.items[i]; @@ -172,3 +188,16 @@ void scr_dyn_decl_adopt(ScrDyn *source, ScrDyn *capsule) { } free(list.items); } + +void scr_dyn_decl_adopt(ScrDyn *source, ScrDyn *capsule) { + scr_decl_adopt_run(source, capsule, false); +} + +/* Any other checked cast of a dynamic value to a record or array + * (`value as T`): the same adoption, so a source that something else still + * holds keeps one identity with the built value. A node that is not plain + * data, or that the value holds in two places, keeps the separate copy the + * cast built (the conversion already succeeded; nothing throws here). */ +void scr_dyn_adopt_shared(ScrDyn *source, ScrDyn *capsule) { + scr_decl_adopt_run(source, capsule, true); +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 2d916a825..550aaf311 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -393,6 +393,12 @@ const ScrDynTypedRecordOps *scr_dyn_typed_ref_live_record(const ScrDyn *d); * holds. Both borrowed; a node that is not plain data throws the path * TypeError before anything changes. */ void scr_dyn_decl_adopt(ScrDyn *source, ScrDyn *capsule); +void scr_dyn_adopt_shared(ScrDyn *source, ScrDyn *capsule); +/* A checked conversion into a type an npm package declares failed + * (scr_type_origin.c, linked by programs with such checks): the pending + * TypeError's message gains the declaration and the type patch that would + * admit the value. `origin` is the compiler's origin string. */ +void scr_type_origin_note(const char *origin); /* Prototype members of a compiled class seen as `unknown`, emitted per * class by the compiler: the string names of methods, getters and setters @@ -1548,6 +1554,9 @@ ScrDyn *scr_dyn_builtin_stream_object(struct ScrStream *s); ScrDyn *scr_dyn_builtin_stream_view(struct ScrStream *s); struct ScrStream *scr_dyn_builtin_stream_check(const ScrDyn *value, double kind); struct ScrStream *scr_dyn_builtin_stream_unbox(const ScrDyn *value, const ScrStr *message); +/* stream.unboxDynStage in those programs (scr_stream_unbox_dyn_stage's + * contract, also taking constructor-function streams). */ +struct ScrStream *scr_dyn_builtin_stream_unbox_stage(const ScrDyn *value, const ScrStr *message); /* stream.dynNative in those programs: scr_stream_dyn_native's contract, * also taking constructor-function streams that keep the runtime's write * and end (+1 or NULL). */ diff --git a/packages/runtime/src/scr_dyn_builtin_stream.c b/packages/runtime/src/scr_dyn_builtin_stream.c index 2875cfa06..52c1e7269 100644 --- a/packages/runtime/src/scr_dyn_builtin_stream.c +++ b/packages/runtime/src/scr_dyn_builtin_stream.c @@ -1752,3 +1752,11 @@ ScrStream *scr_dyn_builtin_stream_unbox(const ScrDyn *value, const ScrStr *messa } return scr_stream_unbox_dyn(value, message); } + +/* stream.unboxDynStage in programs that link this unit: a + * constructor-function stream as scr_dyn_builtin_stream_unbox takes it, + * any other value as scr_stream_unbox_dyn_stage does. */ +ScrStream *scr_dyn_builtin_stream_unbox_stage(const ScrDyn *value, const ScrStr *message) { + if (bcs_of(value) && value->kind == SCR_DYN_OBJ) return scr_dyn_builtin_stream_unbox(value, message); + return scr_stream_unbox_dyn_stage(value, message); +} diff --git a/packages/runtime/src/scr_glibc_math.c b/packages/runtime/src/scr_glibc_math.c new file mode 100644 index 000000000..2ef2ed2e7 --- /dev/null +++ b/packages/runtime/src/scr_glibc_math.c @@ -0,0 +1,271 @@ +/* The libm functions that glibc executables reach without calling a Math + * transcendental: floor, ceil, trunc and round (LLVM's rounding intrinsics + * on x86-64 before SSE4.1, and the runtime's own calls), fmod (the % + * operator), sqrt (Math.sqrt's call for a negative argument), and ldexp, + * scalbn and frexp, which glibc exports from both libc and libm (the link + * would otherwise take them from libm). Each is exactly specified, so these + * definitions answer what glibc's do, the NaN payloads of NaN inputs + * included. Defining them in the executable lets a program that uses no + * other libm function link without libm.so (-lm is linked as needed), + * which saves loading it at startup. Library builds keep the embedder's + * libm. */ + +#include +#include + +#if defined(__GLIBC__) && !defined(SCR_LIB) + +/* The sanitizer runtime calls some of these functions while it initializes, + * and the dynamic loader calls the resolvers before that: none of the code + * here carries sanitizer instrumentation. no_sanitize("thread") alone keeps + * ThreadSanitizer's function entry and exit calls, which crash before its + * runtime initializes; disable_sanitizer_instrumentation removes them. */ +#pragma clang attribute push(__attribute__((no_sanitize("address", "undefined", "thread"), disable_sanitizer_instrumentation)), apply_to = function) + +static inline uint64_t sgm_bits(double d) { + uint64_t bits; + memcpy(&bits, &d, sizeof bits); + return bits; +} + +static inline double sgm_double(uint64_t bits) { + double d; + memcpy(&d, &bits, sizeof d); + return d; +} + +#define SGM_SIGN (UINT64_C(1) << 63) + +/* Hidden: the executable's own references bind to these definitions, which + * are not exported (libc.so and libm.so define the same names), so section + * GC drops the ones a program does not call. */ +#define SGM_DEFINE __attribute__((visibility("hidden"))) + +/* Values of magnitude below 2^52 round through an integer conversion: the + * truncated value, stepped by one when the rounding direction needs it, with + * the value's sign so that a zero result keeps it (ceil(-0.5) is -0). */ +static inline int sgm_small(double x) { return __builtin_fabs(x) < 0x1p52; } + +/* The unbiased exponent; 1024 for the infinities and NaN. Values with an + * exponent of 52 or more are integers already. */ +static inline int sgm_exponent(uint64_t bits) { return (int)((bits >> 52) & 0x7FF) - 1023; } + +static double sgm_floor(double x) { + if (sgm_small(x)) { + double t = (double)(int64_t)x; + return __builtin_copysign(t > x ? t - 1.0 : t, x); + } + uint64_t bits = sgm_bits(x); + int e = sgm_exponent(bits); + if (e >= 52) return e == 1024 ? x + x : x; + if (e < 0) { + if ((bits << 1) == 0) return x; + return bits & SGM_SIGN ? -1.0 : 0.0; + } + uint64_t fraction = (UINT64_C(1) << (52 - e)) - 1; + if ((bits & fraction) == 0) return x; + if (bits & SGM_SIGN) bits += fraction; + return sgm_double(bits & ~fraction); +} + +static double sgm_ceil(double x) { + if (sgm_small(x)) { + double t = (double)(int64_t)x; + return __builtin_copysign(t < x ? t + 1.0 : t, x); + } + uint64_t bits = sgm_bits(x); + int e = sgm_exponent(bits); + if (e >= 52) return e == 1024 ? x + x : x; + if (e < 0) { + if ((bits << 1) == 0) return x; + return bits & SGM_SIGN ? -0.0 : 1.0; + } + uint64_t fraction = (UINT64_C(1) << (52 - e)) - 1; + if ((bits & fraction) == 0) return x; + if (!(bits & SGM_SIGN)) bits += fraction; + return sgm_double(bits & ~fraction); +} + +static double sgm_trunc(double x) { + if (sgm_small(x)) return __builtin_copysign((double)(int64_t)x, x); + uint64_t bits = sgm_bits(x); + int e = sgm_exponent(bits); + if (e >= 52) return e == 1024 ? x + x : x; + if (e < 0) return sgm_double(bits & SGM_SIGN); + uint64_t fraction = (UINT64_C(1) << (52 - e)) - 1; + return sgm_double(bits & ~fraction); +} + +#if defined(__x86_64__) +#include +#include + +/* glibc selects SSE4.1's rounding instruction for floor, ceil and trunc when + * the CPU has it, and so do these: the same results, with no call into the + * integer path. */ +__attribute__((target("sse4.1"))) static double sgm_floor_sse41(double x) { + return _mm_cvtsd_f64(_mm_floor_sd(_mm_set_sd(x), _mm_set_sd(x))); +} +__attribute__((target("sse4.1"))) static double sgm_ceil_sse41(double x) { + return _mm_cvtsd_f64(_mm_ceil_sd(_mm_set_sd(x), _mm_set_sd(x))); +} +__attribute__((target("sse4.1"))) static double sgm_trunc_sse41(double x) { + return _mm_cvtsd_f64(_mm_round_sd(_mm_set_sd(x), _mm_set_sd(x), _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)); +} + +static int sgm_has_sse41(void) { + unsigned eax, ebx, ecx, edx; + return __get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx & bit_SSE4_1) != 0; +} + +typedef double (*SgmUnary)(double); +static SgmUnary sgm_resolve_floor(void) { return sgm_has_sse41() ? sgm_floor_sse41 : sgm_floor; } +static SgmUnary sgm_resolve_ceil(void) { return sgm_has_sse41() ? sgm_ceil_sse41 : sgm_ceil; } +static SgmUnary sgm_resolve_trunc(void) { return sgm_has_sse41() ? sgm_trunc_sse41 : sgm_trunc; } +SGM_DEFINE double floor(double x) __attribute__((ifunc("sgm_resolve_floor"))); +SGM_DEFINE double ceil(double x) __attribute__((ifunc("sgm_resolve_ceil"))); +SGM_DEFINE double trunc(double x) __attribute__((ifunc("sgm_resolve_trunc"))); +#else +SGM_DEFINE double floor(double x) { return sgm_floor(x); } +SGM_DEFINE double ceil(double x) { return sgm_ceil(x); } +SGM_DEFINE double trunc(double x) { return sgm_trunc(x); } +#endif + +/* Math.sqrt is LLVM's sqrt libcall: the square root instruction inline, and + * a call to sqrt for a negative argument, where libm would set errno. This + * definition answers that call with the same instruction. */ +SGM_DEFINE double sqrt(double x) { +#if defined(__x86_64__) + return _mm_cvtsd_f64(_mm_sqrt_sd(_mm_set_sd(x), _mm_set_sd(x))); +#elif defined(__aarch64__) + double r; + __asm__("fsqrt %d0, %d1" : "=w"(r) : "w"(x)); + return r; +#else + return __builtin_sqrt(x); +#endif +} + +/* Halfway cases round away from zero. */ +SGM_DEFINE double round(double x) { + uint64_t bits = sgm_bits(x); + int e = sgm_exponent(bits); + if (e >= 52) return e == 1024 ? x + x : x; + if (e < 0) { + uint64_t sign = bits & SGM_SIGN; + return e == -1 ? sgm_double(sign | sgm_bits(1.0)) : sgm_double(sign); + } + uint64_t fraction = (UINT64_C(1) << (52 - e)) - 1; + if ((bits & fraction) == 0) return x; + bits += UINT64_C(1) << (51 - e); + return sgm_double(bits & ~fraction); +} + +/* The significand of a finite nonzero double as an integer below 2^53, and + * its exponent so that the value's magnitude is significand * 2^exponent. */ +static inline uint64_t sgm_significand(uint64_t bits, int *exponent) { + int biased = (int)((bits >> 52) & 0x7FF); + uint64_t significand = bits & ((UINT64_C(1) << 52) - 1); + if (biased == 0) { + *exponent = -1074; + return significand; + } + *exponent = biased - 1075; + return significand | (UINT64_C(1) << 52); +} + +/* x - n*y for the integer n = trunc(x/y), which is exact: the remainder + * comes from shift-and-subtract over the significands. NaN, an infinite x + * and a zero y answer (x*y)/(x*y), as glibc's fmod does. */ +SGM_DEFINE double fmod(double x, double y) { + uint64_t xb = sgm_bits(x), yb = sgm_bits(y); + uint64_t xmag = xb & ~SGM_SIGN, ymag = yb & ~SGM_SIGN; + uint64_t infinity = UINT64_C(0x7FF0000000000000); + if (ymag == 0 || xmag >= infinity || ymag > infinity) return (x * y) / (x * y); + if (xmag < ymag) return x; + if (xmag == ymag) return 0.0 * x; + int ex, ey; + uint64_t mx = sgm_significand(xb, &ex); + uint64_t my = sgm_significand(yb, &ey); + /* Normalize both to 53 significant bits, so the loop below shifts one + * exponent step at a time. */ + while (mx < (UINT64_C(1) << 52)) { + mx <<= 1; + ex--; + } + while (my < (UINT64_C(1) << 52)) { + my <<= 1; + ey--; + } + for (; ex > ey; ex--) { + if (mx >= my) mx -= my; + if (mx == 0) return 0.0 * x; + mx <<= 1; + } + if (mx >= my) mx -= my; + if (mx == 0) return 0.0 * x; + /* The remainder is mx * 2^ey, below y: rebuild it as a double. */ + while (mx < (UINT64_C(1) << 52)) { + mx <<= 1; + ey--; + } + uint64_t result; + if (ey + 1075 >= 1) { + result = ((uint64_t)(ey + 1075) << 52) | (mx & ((UINT64_C(1) << 52) - 1)); + } else { + result = mx >> (1 - (ey + 1075)); + } + return sgm_double(result | (xb & SGM_SIGN)); +} + +/* x * 2^n, rounded once. Exponents beyond one double's range scale in + * steps; the step toward the subnormals stays 53 bits above the smallest + * normal so that only the last multiplication rounds. */ +SGM_DEFINE double scalbn(double x, int n) { + if (n > 1023) { + x *= 0x1p1023; + n -= 1023; + if (n > 1023) { + x *= 0x1p1023; + n -= 1023; + if (n > 1023) n = 1023; + } + } else if (n < -1022) { + x *= 0x1p-1022 * 0x1p53; + n += 1022 - 53; + if (n < -1022) { + x *= 0x1p-1022 * 0x1p53; + n += 1022 - 53; + if (n < -1022) n = -1022; + } + } + return x * sgm_double((uint64_t)(0x3FF + n) << 52); +} + +SGM_DEFINE double ldexp(double x, int n) { return scalbn(x, n); } + +/* The significand in [0.5, 1) and the power of two: x = significand * 2^e. + * Zero, the infinities and NaN answer themselves with e = 0. */ +SGM_DEFINE double frexp(double x, int *e) { + uint64_t bits = sgm_bits(x); + int biased = (int)((bits >> 52) & 0x7FF); + if (biased == 0) { + if ((bits << 1) == 0) { + *e = 0; + return x; + } + x = frexp(x * 0x1p64, e); + *e -= 64; + return x; + } + if (biased == 0x7FF) { + *e = 0; + return x; + } + *e = biased - 1022; + return sgm_double((bits & ~(UINT64_C(0x7FF) << 52)) | (UINT64_C(0x3FE) << 52)); +} + +#pragma clang attribute pop + +#endif diff --git a/packages/runtime/src/scr_ieee754.c b/packages/runtime/src/scr_ieee754.c new file mode 100644 index 000000000..798bcbcbc --- /dev/null +++ b/packages/runtime/src/scr_ieee754.c @@ -0,0 +1,2730 @@ +// The following is adapted from fdlibm (http://www.netlib.org/fdlibm). +// +// ==================================================== +// Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +// +// Developed at SunSoft, a Sun Microsystems, Inc. business. +// Permission to use, copy, modify, and distribute this +// software is freely granted, provided that this notice +// is preserved. +// ==================================================== +// +// The original source code covered by the above license above has been +// modified significantly by Google Inc. +// Copyright 2016 the V8 project authors. All rights reserved. + +/* Math's transcendental functions as V8 computes them: a C port of V8's + * src/base/ieee754.cc (Node v24.22.0's deps/v8), V8's adaptation of + * fdlibm. Node answers Math.sin, Math.exp, Math.cbrt and the other + * functions here through that file on every platform, so the compiled + * Math members call scr_ieee754_* and print the values Node prints. The + * functions are transcribed unchanged; the port renames the entry points, + * gives the kernels internal linkage, and replaces V8's helpers with C + * (bit casts, wraparound arithmetic, NaN and infinity constants). V8's + * BSD license is in ../vendor/v8-ieee754/LICENSE. + * + * Math.pow is not here: V8's math::pow calls the platform pow + * (scr_math_pow, scr_lib.c). */ + +#include +#include +#include + +#include "scr_ieee754.h" + +/* Floating-point contraction follows Node's build of this file. Node for + * macOS on arm64 is compiled by clang, which fuses a multiply and an add + * within one expression into a fused multiply-add; the results differ from + * the unfused ones in the last bit for some inputs. Other builds round each + * operation on its own, as Node's x86-64 builds do. */ +#if defined(__APPLE__) && defined(__aarch64__) +#pragma STDC FP_CONTRACT ON +#else +#pragma STDC FP_CONTRACT OFF +#endif + +static inline uint64_t scr_fd_bits(double d) { + uint64_t bits; + memcpy(&bits, &d, sizeof bits); + return bits; +} + +static inline double scr_fd_double(uint64_t bits) { + double d; + memcpy(&d, &bits, sizeof d); + return d; +} + +/* floor for __kernel_rem_pio2's nonnegative integer-part values, which are + * below 2^53: exact, with no libm call. */ +static inline double scr_fd_floor(double x) { + double t = (double)(int64_t)x; + return t > x ? t - 1.0 : t; +} + +/* + * The original fdlibm code used statements like: + * n0 = ((*(int*)&one)>>29)^1; * index of high word * + * ix0 = *(n0+(int*)&x); * high word of x * + * ix1 = *((1-n0)+(int*)&x); * low word of x * + * to dig two 32 bit words out of the 64 bit IEEE floating point + * value. That is non-ANSI, and, moreover, the gcc instruction + * scheduler gets it wrong. We instead use the following macros. + * Unlike the original code, we determine the endianness at compile + * time, not at run time; I don't see much benefit to selecting + * endianness at run time. + */ + +/* Get two 32 bit ints from a double. */ + +#define EXTRACT_WORDS(ix0, ix1, d) \ + do { \ + uint64_t bits = scr_fd_bits(d); \ + (ix0) = bits >> 32; \ + (ix1) = bits & 0xFFFFFFFFu; \ + } while (0) + +/* Get the more significant 32 bit int from a double. */ + +#define GET_HIGH_WORD(i, d) \ + do { \ + uint64_t bits = scr_fd_bits(d); \ + (i) = bits >> 32; \ + } while (0) + +/* Get the less significant 32 bit int from a double. */ + +#define GET_LOW_WORD(i, d) \ + do { \ + uint64_t bits = scr_fd_bits(d); \ + (i) = bits & 0xFFFFFFFFu; \ + } while (0) + +/* Set a double from two 32 bit ints. */ + +#define INSERT_WORDS(d, ix0, ix1) \ + do { \ + uint64_t bits = 0; \ + bits |= (uint64_t)(ix0) << 32; \ + bits |= (uint32_t)(ix1); \ + (d) = scr_fd_double(bits); \ + } while (0) + +/* Set the more significant 32 bits of a double from an int. */ + +#define SET_HIGH_WORD(d, v) \ + do { \ + uint64_t bits = scr_fd_bits(d); \ + bits &= 0x00000000FFFFFFFF; \ + bits |= (uint64_t)(v) << 32; \ + (d) = scr_fd_double(bits); \ + } while (0) + +/* Set the less significant 32 bits of a double from an int. */ + +#define SET_LOW_WORD(d, v) \ + do { \ + uint64_t bits = scr_fd_bits(d); \ + bits &= 0xFFFFFFFF00000000; \ + bits |= (uint32_t)(v); \ + (d) = scr_fd_double(bits); \ + } while (0) + +static int32_t __ieee754_rem_pio2(double x, double* y); +static int __kernel_rem_pio2(double* x, double* y, int e0, int nx, int prec, + const int32_t* ipio2); +static double __kernel_cos(double x, double y); +static double __kernel_sin(double x, double y, int iy); + +/* __ieee754_rem_pio2(x,y) + * + * return the remainder of x rem pi/2 in y[0]+y[1] + * use __kernel_rem_pio2() + */ +static int32_t __ieee754_rem_pio2(double x, double *y) { + /* + * Table of constants for 2/pi, 396 Hex digits (476 decimal) of 2/pi + */ + static const int32_t two_over_pi[] = { + 0xA2F983, 0x6E4E44, 0x1529FC, 0x2757D1, 0xF534DD, 0xC0DB62, 0x95993C, + 0x439041, 0xFE5163, 0xABDEBB, 0xC561B7, 0x246E3A, 0x424DD2, 0xE00649, + 0x2EEA09, 0xD1921C, 0xFE1DEB, 0x1CB129, 0xA73EE8, 0x8235F5, 0x2EBB44, + 0x84E99C, 0x7026B4, 0x5F7E41, 0x3991D6, 0x398353, 0x39F49C, 0x845F8B, + 0xBDF928, 0x3B1FF8, 0x97FFDE, 0x05980F, 0xEF2F11, 0x8B5A0A, 0x6D1F6D, + 0x367ECF, 0x27CB09, 0xB74F46, 0x3F669E, 0x5FEA2D, 0x7527BA, 0xC7EBE5, + 0xF17B3D, 0x0739F7, 0x8A5292, 0xEA6BFB, 0x5FB11F, 0x8D5D08, 0x560330, + 0x46FC7B, 0x6BABF0, 0xCFBC20, 0x9AF436, 0x1DA9E3, 0x91615E, 0xE61B08, + 0x659985, 0x5F14A0, 0x68408D, 0xFFD880, 0x4D7327, 0x310606, 0x1556CA, + 0x73A8C9, 0x60E27B, 0xC08C6B, + }; + + static const int32_t npio2_hw[] = { + 0x3FF921FB, 0x400921FB, 0x4012D97C, 0x401921FB, 0x401F6A7A, 0x4022D97C, + 0x4025FDBB, 0x402921FB, 0x402C463A, 0x402F6A7A, 0x4031475C, 0x4032D97C, + 0x40346B9C, 0x4035FDBB, 0x40378FDB, 0x403921FB, 0x403AB41B, 0x403C463A, + 0x403DD85A, 0x403F6A7A, 0x40407E4C, 0x4041475C, 0x4042106C, 0x4042D97C, + 0x4043A28C, 0x40446B9C, 0x404534AC, 0x4045FDBB, 0x4046C6CB, 0x40478FDB, + 0x404858EB, 0x404921FB, + }; + + /* + * invpio2: 53 bits of 2/pi + * pio2_1: first 33 bit of pi/2 + * pio2_1t: pi/2 - pio2_1 + * pio2_2: second 33 bit of pi/2 + * pio2_2t: pi/2 - (pio2_1+pio2_2) + * pio2_3: third 33 bit of pi/2 + * pio2_3t: pi/2 - (pio2_1+pio2_2+pio2_3) + */ + + static const double + zero = 0.00000000000000000000e+00, /* 0x00000000, 0x00000000 */ + half = 5.00000000000000000000e-01, /* 0x3FE00000, 0x00000000 */ + two24 = 1.67772160000000000000e+07, /* 0x41700000, 0x00000000 */ + invpio2 = 6.36619772367581382433e-01, /* 0x3FE45F30, 0x6DC9C883 */ + pio2_1 = 1.57079632673412561417e+00, /* 0x3FF921FB, 0x54400000 */ + pio2_1t = 6.07710050650619224932e-11, /* 0x3DD0B461, 0x1A626331 */ + pio2_2 = 6.07710050630396597660e-11, /* 0x3DD0B461, 0x1A600000 */ + pio2_2t = 2.02226624879595063154e-21, /* 0x3BA3198A, 0x2E037073 */ + pio2_3 = 2.02226624871116645580e-21, /* 0x3BA3198A, 0x2E000000 */ + pio2_3t = 8.47842766036889956997e-32; /* 0x397B839A, 0x252049C1 */ + + double z, w, t, r, fn; + double tx[3]; + int32_t e0, i, j, nx, n, ix, hx; + uint32_t low; + + z = 0; + GET_HIGH_WORD(hx, x); /* high word of x */ + ix = hx & 0x7FFFFFFF; + if (ix <= 0x3FE921FB) { /* |x| ~<= pi/4 , no need for reduction */ + y[0] = x; + y[1] = 0; + return 0; + } + if (ix < 0x4002D97C) { /* |x| < 3pi/4, special case with n=+-1 */ + if (hx > 0) { + z = x - pio2_1; + if (ix != 0x3FF921FB) { /* 33+53 bit pi is good enough */ + y[0] = z - pio2_1t; + y[1] = (z - y[0]) - pio2_1t; + } else { /* near pi/2, use 33+33+53 bit pi */ + z -= pio2_2; + y[0] = z - pio2_2t; + y[1] = (z - y[0]) - pio2_2t; + } + return 1; + } else { /* negative x */ + z = x + pio2_1; + if (ix != 0x3FF921FB) { /* 33+53 bit pi is good enough */ + y[0] = z + pio2_1t; + y[1] = (z - y[0]) + pio2_1t; + } else { /* near pi/2, use 33+33+53 bit pi */ + z += pio2_2; + y[0] = z + pio2_2t; + y[1] = (z - y[0]) + pio2_2t; + } + return -1; + } + } + if (ix <= 0x413921FB) { /* |x| ~<= 2^19*(pi/2), medium size */ + t = __builtin_fabs(x); + n = (int32_t)(t * invpio2 + half); + fn = (double)(n); + r = t - fn * pio2_1; + w = fn * pio2_1t; /* 1st round good to 85 bit */ + if (n < 32 && ix != npio2_hw[n - 1]) { + y[0] = r - w; /* quick check no cancellation */ + } else { + uint32_t high; + j = ix >> 20; + y[0] = r - w; + GET_HIGH_WORD(high, y[0]); + i = j - ((high >> 20) & 0x7FF); + if (i > 16) { /* 2nd iteration needed, good to 118 */ + t = r; + w = fn * pio2_2; + r = t - w; + w = fn * pio2_2t - ((t - r) - w); + y[0] = r - w; + GET_HIGH_WORD(high, y[0]); + i = j - ((high >> 20) & 0x7FF); + if (i > 49) { /* 3rd iteration need, 151 bits acc */ + t = r; /* will cover all possible cases */ + w = fn * pio2_3; + r = t - w; + w = fn * pio2_3t - ((t - r) - w); + y[0] = r - w; + } + } + } + y[1] = (r - y[0]) - w; + if (hx < 0) { + y[0] = -y[0]; + y[1] = -y[1]; + return -n; + } else { + return n; + } + } + /* + * all other (large) arguments + */ + if (ix >= 0x7FF00000) { /* x is inf or NaN */ + y[0] = y[1] = x - x; + return 0; + } + /* set z = scalbn(|x|,ilogb(x)-23) */ + GET_LOW_WORD(low, x); + SET_LOW_WORD(z, low); + e0 = (ix >> 20) - 1046; /* e0 = ilogb(z)-23; */ + SET_HIGH_WORD(z, ix - (int32_t)((uint32_t)(e0) << 20)); + for (i = 0; i < 2; i++) { + tx[i] = (double)((int32_t)(z)); + z = (z - tx[i]) * two24; + } + tx[2] = z; + nx = 3; + while (tx[nx - 1] == zero) nx--; /* skip zero term */ + n = __kernel_rem_pio2(tx, y, e0, nx, 2, two_over_pi); + if (hx < 0) { + y[0] = -y[0]; + y[1] = -y[1]; + return -n; + } + return n; +} + +/* __kernel_cos( x, y ) + * kernel cos function on [-pi/4, pi/4], pi/4 ~ 0.785398164 + * Input x is assumed to be bounded by ~pi/4 in magnitude. + * Input y is the tail of x. + * + * Algorithm + * 1. Since cos(-x) = cos(x), we need only to consider positive x. + * 2. if x < 2^-27 (hx<0x3E400000 0), return 1 with inexact if x!=0. + * 3. cos(x) is approximated by a polynomial of degree 14 on + * [0,pi/4] + * 4 14 + * cos(x) ~ 1 - x*x/2 + C1*x + ... + C6*x + * where the remez error is + * + * | 2 4 6 8 10 12 14 | -58 + * |cos(x)-(1-.5*x +C1*x +C2*x +C3*x +C4*x +C5*x +C6*x )| <= 2 + * | | + * + * 4 6 8 10 12 14 + * 4. let r = C1*x +C2*x +C3*x +C4*x +C5*x +C6*x , then + * cos(x) = 1 - x*x/2 + r + * since cos(x+y) ~ cos(x) - sin(x)*y + * ~ cos(x) - x*y, + * a correction term is necessary in cos(x) and hence + * cos(x+y) = 1 - (x*x/2 - (r - x*y)) + * For better accuracy when x > 0.3, let qx = |x|/4 with + * the last 32 bits mask off, and if x > 0.78125, let qx = 0.28125. + * Then + * cos(x+y) = (1-qx) - ((x*x/2-qx) - (r-x*y)). + * Note that 1-qx and (x*x/2-qx) is EXACT here, and the + * magnitude of the latter is at least a quarter of x*x/2, + * thus, reducing the rounding error in the subtraction. + */ +static inline double __kernel_cos(double x, double y) { + static const double + one = 1.00000000000000000000e+00, /* 0x3FF00000, 0x00000000 */ + C1 = 4.16666666666666019037e-02, /* 0x3FA55555, 0x5555554C */ + C2 = -1.38888888888741095749e-03, /* 0xBF56C16C, 0x16C15177 */ + C3 = 2.48015872894767294178e-05, /* 0x3EFA01A0, 0x19CB1590 */ + C4 = -2.75573143513906633035e-07, /* 0xBE927E4F, 0x809C52AD */ + C5 = 2.08757232129817482790e-09, /* 0x3E21EE9E, 0xBDB4B1C4 */ + C6 = -1.13596475577881948265e-11; /* 0xBDA8FAE9, 0xBE8838D4 */ + + double a, iz, z, r, qx; + int32_t ix; + GET_HIGH_WORD(ix, x); + ix &= 0x7FFFFFFF; /* ix = |x|'s high word*/ + if (ix < 0x3E400000) { /* if x < 2**27 */ + if ((int)(x) == 0) return one; /* generate inexact */ + } + z = x * x; + r = z * (C1 + z * (C2 + z * (C3 + z * (C4 + z * (C5 + z * C6))))); + if (ix < 0x3FD33333) { /* if |x| < 0.3 */ + return one - (0.5 * z - (z * r - x * y)); + } else { + if (ix > 0x3FE90000) { /* x > 0.78125 */ + qx = 0.28125; + } else { + INSERT_WORDS(qx, ix - 0x00200000, 0); /* x/4 */ + } + iz = 0.5 * z - qx; + a = one - qx; + return a - (iz - (z * r - x * y)); + } +} + +/* __kernel_rem_pio2(x,y,e0,nx,prec,ipio2) + * double x[],y[]; int e0,nx,prec; int ipio2[]; + * + * __kernel_rem_pio2 return the last three digits of N with + * y = x - N*pi/2 + * so that |y| < pi/2. + * + * The method is to compute the integer (mod 8) and fraction parts of + * (2/pi)*x without doing the full multiplication. In general we + * skip the part of the product that are known to be a huge integer ( + * more accurately, = 0 mod 8 ). Thus the number of operations are + * independent of the exponent of the input. + * + * (2/pi) is represented by an array of 24-bit integers in ipio2[]. + * + * Input parameters: + * x[] The input value (must be positive) is broken into nx + * pieces of 24-bit integers in double precision format. + * x[i] will be the i-th 24 bit of x. The scaled exponent + * of x[0] is given in input parameter e0 (i.e., x[0]*2^e0 + * match x's up to 24 bits. + * + * Example of breaking a double positive z into x[0]+x[1]+x[2]: + * e0 = ilogb(z)-23 + * z = scalbn(z,-e0) + * for i = 0,1,2 + * x[i] = floor(z) + * z = (z-x[i])*2**24 + * + * + * y[] output result in an array of double precision numbers. + * The dimension of y[] is: + * 24-bit precision 1 + * 53-bit precision 2 + * 64-bit precision 2 + * 113-bit precision 3 + * The actual value is the sum of them. Thus for 113-bit + * precison, one may have to do something like: + * + * long double t,w,r_head, r_tail; + * t = (long double)y[2] + (long double)y[1]; + * w = (long double)y[0]; + * r_head = t+w; + * r_tail = w - (r_head - t); + * + * e0 The exponent of x[0] + * + * nx dimension of x[] + * + * prec an integer indicating the precision: + * 0 24 bits (single) + * 1 53 bits (double) + * 2 64 bits (extended) + * 3 113 bits (quad) + * + * ipio2[] + * integer array, contains the (24*i)-th to (24*i+23)-th + * bit of 2/pi after binary point. The corresponding + * floating value is + * + * ipio2[i] * 2^(-24(i+1)). + * + * External function: + * double scalbn(), floor(); + * + * + * Here is the description of some local variables: + * + * jk jk+1 is the initial number of terms of ipio2[] needed + * in the computation. The recommended value is 2,3,4, + * 6 for single, double, extended,and quad. + * + * jz local integer variable indicating the number of + * terms of ipio2[] used. + * + * jx nx - 1 + * + * jv index for pointing to the suitable ipio2[] for the + * computation. In general, we want + * ( 2^e0*x[0] * ipio2[jv-1]*2^(-24jv) )/8 + * is an integer. Thus + * e0-3-24*jv >= 0 or (e0-3)/24 >= jv + * Hence jv = max(0,(e0-3)/24). + * + * jp jp+1 is the number of terms in PIo2[] needed, jp = jk. + * + * q[] double array with integral value, representing the + * 24-bits chunk of the product of x and 2/pi. + * + * q0 the corresponding exponent of q[0]. Note that the + * exponent for q[i] would be q0-24*i. + * + * PIo2[] double precision array, obtained by cutting pi/2 + * into 24 bits chunks. + * + * f[] ipio2[] in floating point + * + * iq[] integer array by breaking up q[] in 24-bits chunk. + * + * fq[] final product of x*(2/pi) in fq[0],..,fq[jk] + * + * ih integer. If >0 it indicates q[] is >= 0.5, hence + * it also indicates the *sign* of the result. + * + */ +static int __kernel_rem_pio2(double *x, double *y, int e0, int nx, int prec, + const int32_t *ipio2) { + /* Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + */ + static const int init_jk[] = {2, 3, 4, 6}; /* initial value for jk */ + + static const double PIo2[] = { + 1.57079625129699707031e+00, /* 0x3FF921FB, 0x40000000 */ + 7.54978941586159635335e-08, /* 0x3E74442D, 0x00000000 */ + 5.39030252995776476554e-15, /* 0x3CF84698, 0x80000000 */ + 3.28200341580791294123e-22, /* 0x3B78CC51, 0x60000000 */ + 1.27065575308067607349e-29, /* 0x39F01B83, 0x80000000 */ + 1.22933308981111328932e-36, /* 0x387A2520, 0x40000000 */ + 2.73370053816464559624e-44, /* 0x36E38222, 0x80000000 */ + 2.16741683877804819444e-51, /* 0x3569F31D, 0x00000000 */ + }; + + static const double + zero = 0.0, + one = 1.0, + two24 = 1.67772160000000000000e+07, /* 0x41700000, 0x00000000 */ + twon24 = 5.96046447753906250000e-08; /* 0x3E700000, 0x00000000 */ + + int32_t jz, jx, jv, jp, jk, carry, n, iq[20], i, j, k, m, q0, ih; + double z, fw, f[20], fq[20], q[20]; + + /* initialize jk*/ + jk = init_jk[prec]; + jp = jk; + + /* determine jx,jv,q0, note that 3>q0 */ + jx = nx - 1; + jv = (e0 - 3) / 24; + if (jv < 0) jv = 0; + q0 = e0 - 24 * (jv + 1); + + /* set up f[0] to f[jx+jk] where f[jx+jk] = ipio2[jv+jk] */ + j = jv - jx; + m = jx + jk; + for (i = 0; i <= m; i++, j++) { + f[i] = (j < 0) ? zero : (double)(ipio2[j]); + } + + /* compute q[0],q[1],...q[jk] */ + for (i = 0; i <= jk; i++) { + for (j = 0, fw = 0.0; j <= jx; j++) fw += x[j] * f[jx + i - j]; + q[i] = fw; + } + + jz = jk; +recompute: + /* distill q[] into iq[] reversingly */ + for (i = 0, j = jz, z = q[jz]; j > 0; i++, j--) { + fw = (double)((int32_t)(twon24 * z)); + iq[i] = (int32_t)(z - two24 * fw); + z = q[j - 1] + fw; + } + + /* compute n */ + z = scalbn(z, q0); /* actual value of z */ + z -= 8.0 * scr_fd_floor(z * 0.125); /* trim off integer >= 8 */ + n = (int32_t)(z); + z -= (double)(n); + ih = 0; + if (q0 > 0) { /* need iq[jz-1] to determine n */ + i = (iq[jz - 1] >> (24 - q0)); + n += i; + iq[jz - 1] -= i << (24 - q0); + ih = iq[jz - 1] >> (23 - q0); + } else if (q0 == 0) { + ih = iq[jz - 1] >> 23; + } else if (z >= 0.5) { + ih = 2; + } + + if (ih > 0) { /* q > 0.5 */ + n += 1; + carry = 0; + for (i = 0; i < jz; i++) { /* compute 1-q */ + j = iq[i]; + if (carry == 0) { + if (j != 0) { + carry = 1; + iq[i] = 0x1000000 - j; + } + } else { + iq[i] = 0xFFFFFF - j; + } + } + if (q0 > 0) { /* rare case: chance is 1 in 12 */ + switch (q0) { + case 1: + iq[jz - 1] &= 0x7FFFFF; + break; + case 2: + iq[jz - 1] &= 0x3FFFFF; + break; + } + } + if (ih == 2) { + z = one - z; + if (carry != 0) z -= scalbn(one, q0); + } + } + + /* check if recomputation is needed */ + if (z == zero) { + j = 0; + for (i = jz - 1; i >= jk; i--) j |= iq[i]; + if (j == 0) { /* need recomputation */ + for (k = 1; jk >= k && iq[jk - k] == 0; k++) { + /* k = no. of terms needed */ + } + + for (i = jz + 1; i <= jz + k; i++) { /* add q[jz+1] to q[jz+k] */ + f[jx + i] = ipio2[jv + i]; + for (j = 0, fw = 0.0; j <= jx; j++) fw += x[j] * f[jx + i - j]; + q[i] = fw; + } + jz += k; + goto recompute; + } + } + + /* chop off zero terms */ + if (z == 0.0) { + jz -= 1; + q0 -= 24; + while (iq[jz] == 0) { + jz--; + q0 -= 24; + } + } else { /* break z into 24-bit if necessary */ + z = scalbn(z, -q0); + if (z >= two24) { + fw = (double)((int32_t)(twon24 * z)); + iq[jz] = z - two24 * fw; + jz += 1; + q0 += 24; + iq[jz] = fw; + } else { + iq[jz] = z; + } + } + + /* convert integer "bit" chunk to floating-point value */ + fw = scalbn(one, q0); + for (i = jz; i >= 0; i--) { + q[i] = fw * iq[i]; + fw *= twon24; + } + + /* compute PIo2[0,...,jp]*q[jz,...,0] */ + for (i = jz; i >= 0; i--) { + for (fw = 0.0, k = 0; k <= jp && k <= jz - i; k++) fw += PIo2[k] * q[i + k]; + fq[jz - i] = fw; + } + + /* compress fq[] into y[] */ + switch (prec) { + case 0: + fw = 0.0; + for (i = jz; i >= 0; i--) fw += fq[i]; + y[0] = (ih == 0) ? fw : -fw; + break; + case 1: + case 2: + fw = 0.0; + for (i = jz; i >= 0; i--) fw += fq[i]; + y[0] = (ih == 0) ? fw : -fw; + fw = fq[0] - fw; + for (i = 1; i <= jz; i++) fw += fq[i]; + y[1] = (ih == 0) ? fw : -fw; + break; + case 3: /* painful */ + for (i = jz; i > 0; i--) { + fw = fq[i - 1] + fq[i]; + fq[i] += fq[i - 1] - fw; + fq[i - 1] = fw; + } + for (i = jz; i > 1; i--) { + fw = fq[i - 1] + fq[i]; + fq[i] += fq[i - 1] - fw; + fq[i - 1] = fw; + } + for (fw = 0.0, i = jz; i >= 2; i--) fw += fq[i]; + if (ih == 0) { + y[0] = fq[0]; + y[1] = fq[1]; + y[2] = fw; + } else { + y[0] = -fq[0]; + y[1] = -fq[1]; + y[2] = -fw; + } + } + return n & 7; +} + +/* __kernel_sin( x, y, iy) + * kernel sin function on [-pi/4, pi/4], pi/4 ~ 0.7854 + * Input x is assumed to be bounded by ~pi/4 in magnitude. + * Input y is the tail of x. + * Input iy indicates whether y is 0. (if iy=0, y assume to be 0). + * + * Algorithm + * 1. Since sin(-x) = -sin(x), we need only to consider positive x. + * 2. if x < 2^-27 (hx<0x3E400000 0), return x with inexact if x!=0. + * 3. sin(x) is approximated by a polynomial of degree 13 on + * [0,pi/4] + * 3 13 + * sin(x) ~ x + S1*x + ... + S6*x + * where + * + * |sin(x) 2 4 6 8 10 12 | -58 + * |----- - (1+S1*x +S2*x +S3*x +S4*x +S5*x +S6*x )| <= 2 + * | x | + * + * 4. sin(x+y) = sin(x) + sin'(x')*y + * ~ sin(x) + (1-x*x/2)*y + * For better accuracy, let + * 3 2 2 2 2 + * r = x *(S2+x *(S3+x *(S4+x *(S5+x *S6)))) + * then 3 2 + * sin(x) = x + (S1*x + (x *(r-y/2)+y)) + */ +static inline double __kernel_sin(double x, double y, int iy) { + static const double + half = 5.00000000000000000000e-01, /* 0x3FE00000, 0x00000000 */ + S1 = -1.66666666666666324348e-01, /* 0xBFC55555, 0x55555549 */ + S2 = 8.33333333332248946124e-03, /* 0x3F811111, 0x1110F8A6 */ + S3 = -1.98412698298579493134e-04, /* 0xBF2A01A0, 0x19C161D5 */ + S4 = 2.75573137070700676789e-06, /* 0x3EC71DE3, 0x57B1FE7D */ + S5 = -2.50507602534068634195e-08, /* 0xBE5AE5E6, 0x8A2B9CEB */ + S6 = 1.58969099521155010221e-10; /* 0x3DE5D93A, 0x5ACFD57C */ + + double z, r, v; + int32_t ix; + GET_HIGH_WORD(ix, x); + ix &= 0x7FFFFFFF; /* high word of x */ + if (ix < 0x3E400000) { /* |x| < 2**-27 */ + if ((int)(x) == 0) return x; + } /* generate inexact */ + z = x * x; + v = z * x; + r = S2 + z * (S3 + z * (S4 + z * (S5 + z * S6))); + if (iy == 0) { + return x + v * (S1 + z * r); + } else { + return x - ((z * (half * y - v * r) - y) - v * S1); + } +} + +/* __kernel_tan( x, y, k ) + * kernel tan function on [-pi/4, pi/4], pi/4 ~ 0.7854 + * Input x is assumed to be bounded by ~pi/4 in magnitude. + * Input y is the tail of x. + * Input k indicates whether tan (if k=1) or + * -1/tan (if k= -1) is returned. + * + * Algorithm + * 1. Since tan(-x) = -tan(x), we need only to consider positive x. + * 2. if x < 2^-28 (hx<0x3E300000 0), return x with inexact if x!=0. + * 3. tan(x) is approximated by an odd polynomial of degree 27 on + * [0,0.67434] + * 3 27 + * tan(x) ~ x + T1*x + ... + T13*x + * where + * + * |tan(x) 2 4 26 | -59.2 + * |----- - (1+T1*x +T2*x +.... +T13*x )| <= 2 + * | x | + * + * Note: tan(x+y) = tan(x) + tan'(x)*y + * ~ tan(x) + (1+x*x)*y + * Therefore, for better accuracy in computing tan(x+y), let + * 3 2 2 2 2 + * r = x *(T2+x *(T3+x *(...+x *(T12+x *T13)))) + * then + * 3 2 + * tan(x+y) = x + (T1*x + (x *(r+y)+y)) + * + * 4. For x in [0.67434,pi/4], let y = pi/4 - x, then + * tan(x) = tan(pi/4-y) = (1-tan(y))/(1+tan(y)) + * = 1 - 2*(tan(y) - (tan(y)^2)/(1+tan(y))) + */ +static double __kernel_tan(double x, double y, int iy) { + static const double xxx[] = { + 3.33333333333334091986e-01, /* 3FD55555, 55555563 */ + 1.33333333333201242699e-01, /* 3FC11111, 1110FE7A */ + 5.39682539762260521377e-02, /* 3FABA1BA, 1BB341FE */ + 2.18694882948595424599e-02, /* 3F9664F4, 8406D637 */ + 8.86323982359930005737e-03, /* 3F8226E3, E96E8493 */ + 3.59207910759131235356e-03, /* 3F6D6D22, C9560328 */ + 1.45620945432529025516e-03, /* 3F57DBC8, FEE08315 */ + 5.88041240820264096874e-04, /* 3F4344D8, F2F26501 */ + 2.46463134818469906812e-04, /* 3F3026F7, 1A8D1068 */ + 7.81794442939557092300e-05, /* 3F147E88, A03792A6 */ + 7.14072491382608190305e-05, /* 3F12B80F, 32F0A7E9 */ + -1.85586374855275456654e-05, /* BEF375CB, DB605373 */ + 2.59073051863633712884e-05, /* 3EFB2A70, 74BF7AD4 */ + /* one */ 1.00000000000000000000e+00, /* 3FF00000, 00000000 */ + /* pio4 */ 7.85398163397448278999e-01, /* 3FE921FB, 54442D18 */ + /* pio4lo */ 3.06161699786838301793e-17 /* 3C81A626, 33145C07 */ + }; +#define one xxx[13] +#define pio4 xxx[14] +#define pio4lo xxx[15] +#define T xxx + + double z, r, v, w, s; + int32_t ix, hx; + + GET_HIGH_WORD(hx, x); /* high word of x */ + ix = hx & 0x7FFFFFFF; /* high word of |x| */ + if (ix < 0x3E300000) { /* x < 2**-28 */ + if ((int)(x) == 0) { /* generate inexact */ + uint32_t low; + GET_LOW_WORD(low, x); + if (((ix | low) | (iy + 1)) == 0) { + return one / __builtin_fabs(x); + } else { + if (iy == 1) { + return x; + } else { /* compute -1 / (x+y) carefully */ + double a, t; + + z = w = x + y; + SET_LOW_WORD(z, 0); + v = y - (z - x); + t = a = -one / w; + SET_LOW_WORD(t, 0); + s = one + t * z; + return t + a * (s + t * v); + } + } + } + } + if (ix >= 0x3FE59428) { /* |x| >= 0.6744 */ + if (hx < 0) { + x = -x; + y = -y; + } + z = pio4 - x; + w = pio4lo - y; + x = z + w; + y = 0.0; + } + z = x * x; + w = z * z; + /* + * Break x^5*(T[1]+x^2*T[2]+...) into + * x^5(T[1]+x^4*T[3]+...+x^20*T[11]) + + * x^5(x^2*(T[2]+x^4*T[4]+...+x^22*[T12])) + */ + r = T[1] + w * (T[3] + w * (T[5] + w * (T[7] + w * (T[9] + w * T[11])))); + v = z * + (T[2] + w * (T[4] + w * (T[6] + w * (T[8] + w * (T[10] + w * T[12]))))); + s = z * x; + r = y + z * (s * (r + v) + y); + r += T[0] * s; + w = x + r; + if (ix >= 0x3FE59428) { + v = iy; + return (1 - ((hx >> 30) & 2)) * (v - 2.0 * (x - (w * w / (w + v) - r))); + } + if (iy == 1) { + return w; + } else { + /* + * if allow error up to 2 ulp, simply return + * -1.0 / (x+r) here + */ + /* compute -1.0 / (x+r) accurately */ + double a, t; + z = w; + SET_LOW_WORD(z, 0); + v = r - (z - x); /* z+v = r+x */ + t = a = -1.0 / w; /* a = -1.0/w */ + SET_LOW_WORD(t, 0); + s = 1.0 + t * z; + return t + a * (s + t * v); + } + +#undef one +#undef pio4 +#undef pio4lo +#undef T +} + +/* acos(x) + * Method : + * acos(x) = pi/2 - asin(x) + * acos(-x) = pi/2 + asin(x) + * For |x|<=0.5 + * acos(x) = pi/2 - (x + x*x^2*R(x^2)) (see asin.c) + * For x>0.5 + * acos(x) = pi/2 - (pi/2 - 2asin(sqrt((1-x)/2))) + * = 2asin(sqrt((1-x)/2)) + * = 2s + 2s*z*R(z) ...z=(1-x)/2, s=sqrt(z) + * = 2f + (2c + 2s*z*R(z)) + * where f=hi part of s, and c = (z-f*f)/(s+f) is the correction term + * for f so that f+c ~ sqrt(z). + * For x<-0.5 + * acos(x) = pi - 2asin(sqrt((1-|x|)/2)) + * = pi - 0.5*(s+s*z*R(z)), where z=(1-|x|)/2,s=sqrt(z) + * + * Special cases: + * if x is NaN, return x itself; + * if |x|>1, return NaN with invalid signal. + * + * Function needed: sqrt + */ +double scr_ieee754_acos(double x) { + static const double + one = 1.00000000000000000000e+00, /* 0x3FF00000, 0x00000000 */ + pi = 3.14159265358979311600e+00, /* 0x400921FB, 0x54442D18 */ + pio2_hi = 1.57079632679489655800e+00, /* 0x3FF921FB, 0x54442D18 */ + pio2_lo = 6.12323399573676603587e-17, /* 0x3C91A626, 0x33145C07 */ + pS0 = 1.66666666666666657415e-01, /* 0x3FC55555, 0x55555555 */ + pS1 = -3.25565818622400915405e-01, /* 0xBFD4D612, 0x03EB6F7D */ + pS2 = 2.01212532134862925881e-01, /* 0x3FC9C155, 0x0E884455 */ + pS3 = -4.00555345006794114027e-02, /* 0xBFA48228, 0xB5688F3B */ + pS4 = 7.91534994289814532176e-04, /* 0x3F49EFE0, 0x7501B288 */ + pS5 = 3.47933107596021167570e-05, /* 0x3F023DE1, 0x0DFDF709 */ + qS1 = -2.40339491173441421878e+00, /* 0xC0033A27, 0x1C8A2D4B */ + qS2 = 2.02094576023350569471e+00, /* 0x40002AE5, 0x9C598AC8 */ + qS3 = -6.88283971605453293030e-01, /* 0xBFE6066C, 0x1B8D0159 */ + qS4 = 7.70381505559019352791e-02; /* 0x3FB3B8C5, 0xB12E9282 */ + + double z, p, q, r, w, s, c, df; + int32_t hx, ix; + GET_HIGH_WORD(hx, x); + ix = hx & 0x7FFFFFFF; + if (ix >= 0x3FF00000) { /* |x| >= 1 */ + uint32_t lx; + GET_LOW_WORD(lx, x); + if (((ix - 0x3FF00000) | lx) == 0) { /* |x|==1 */ + if (hx > 0) + return 0.0; /* acos(1) = 0 */ + else + return pi + 2.0 * pio2_lo; /* acos(-1)= pi */ + } + return __builtin_nans(""); // acos(|x|>1) is NaN + } + if (ix < 0x3FE00000) { /* |x| < 0.5 */ + if (ix <= 0x3C600000) return pio2_hi + pio2_lo; /*if|x|<2**-57*/ + z = x * x; + p = z * (pS0 + z * (pS1 + z * (pS2 + z * (pS3 + z * (pS4 + z * pS5))))); + q = one + z * (qS1 + z * (qS2 + z * (qS3 + z * qS4))); + r = p / q; + return pio2_hi - (x - (pio2_lo - x * r)); + } else if (hx < 0) { /* x < -0.5 */ + z = (one + x) * 0.5; + p = z * (pS0 + z * (pS1 + z * (pS2 + z * (pS3 + z * (pS4 + z * pS5))))); + q = one + z * (qS1 + z * (qS2 + z * (qS3 + z * qS4))); + s = __builtin_sqrt(z); + r = p / q; + w = r * s - pio2_lo; + return pi - 2.0 * (s + w); + } else { /* x > 0.5 */ + z = (one - x) * 0.5; + s = __builtin_sqrt(z); + df = s; + SET_LOW_WORD(df, 0); + c = (z - df * df) / (s + df); + p = z * (pS0 + z * (pS1 + z * (pS2 + z * (pS3 + z * (pS4 + z * pS5))))); + q = one + z * (qS1 + z * (qS2 + z * (qS3 + z * qS4))); + r = p / q; + w = r * s + c; + return 2.0 * (df + w); + } +} + +/* acosh(x) + * Method : + * Based on + * acosh(x) = log [ x + sqrt(x*x-1) ] + * we have + * acosh(x) := log(x)+ln2, if x is large; else + * acosh(x) := log(2x-1/(sqrt(x*x-1)+x)) if x>2; else + * acosh(x) := log1p(t+sqrt(2.0*t+t*t)); where t=x-1. + * + * Special cases: + * acosh(x) is NaN with signal if x<1. + * acosh(NaN) is NaN without signal. + */ +double scr_ieee754_acosh(double x) { + static const double + one = 1.0, + ln2 = 6.93147180559945286227e-01; /* 0x3FE62E42, 0xFEFA39EF */ + double t; + int32_t hx; + uint32_t lx; + EXTRACT_WORDS(hx, lx, x); + if (hx < 0x3FF00000) { /* x < 1 */ + return __builtin_nans(""); + } else if (hx >= 0x41B00000) { /* x > 2**28 */ + if (hx >= 0x7FF00000) { /* x is inf of NaN */ + return x + x; + } else { + return scr_ieee754_log(x) + ln2; /* acosh(huge)=log(2x) */ + } + } else if (((hx - 0x3FF00000) | lx) == 0) { + return 0.0; /* acosh(1) = 0 */ + } else if (hx > 0x40000000) { /* 2**28 > x > 2 */ + t = x * x; + return scr_ieee754_log(2.0 * x - one / (x + __builtin_sqrt(t - one))); + } else { /* 10.98 + * asin(x) = pi/2 - 2*(s+s*z*R(z)) + * = pio2_hi - (2*(s+s*z*R(z)) - pio2_lo) + * For x<=0.98, let pio4_hi = pio2_hi/2, then + * f = hi part of s; + * c = sqrt(z) - f = (z-f*f)/(s+f) ...f+c=sqrt(z) + * and + * asin(x) = pi/2 - 2*(s+s*z*R(z)) + * = pio4_hi+(pio4-2s)-(2s*z*R(z)-pio2_lo) + * = pio4_hi+(pio4-2f)-(2s*z*R(z)-(pio2_lo+2c)) + * + * Special cases: + * if x is NaN, return x itself; + * if |x|>1, return NaN with invalid signal. + */ +double scr_ieee754_asin(double x) { + static const double + one = 1.00000000000000000000e+00, /* 0x3FF00000, 0x00000000 */ + huge = 1.000e+300, + pio2_hi = 1.57079632679489655800e+00, /* 0x3FF921FB, 0x54442D18 */ + pio2_lo = 6.12323399573676603587e-17, /* 0x3C91A626, 0x33145C07 */ + pio4_hi = 7.85398163397448278999e-01, /* 0x3FE921FB, 0x54442D18 */ + /* coefficient for R(x^2) */ + pS0 = 1.66666666666666657415e-01, /* 0x3FC55555, 0x55555555 */ + pS1 = -3.25565818622400915405e-01, /* 0xBFD4D612, 0x03EB6F7D */ + pS2 = 2.01212532134862925881e-01, /* 0x3FC9C155, 0x0E884455 */ + pS3 = -4.00555345006794114027e-02, /* 0xBFA48228, 0xB5688F3B */ + pS4 = 7.91534994289814532176e-04, /* 0x3F49EFE0, 0x7501B288 */ + pS5 = 3.47933107596021167570e-05, /* 0x3F023DE1, 0x0DFDF709 */ + qS1 = -2.40339491173441421878e+00, /* 0xC0033A27, 0x1C8A2D4B */ + qS2 = 2.02094576023350569471e+00, /* 0x40002AE5, 0x9C598AC8 */ + qS3 = -6.88283971605453293030e-01, /* 0xBFE6066C, 0x1B8D0159 */ + qS4 = 7.70381505559019352791e-02; /* 0x3FB3B8C5, 0xB12E9282 */ + + double t, w, p, q, c, r, s; + int32_t hx, ix; + + t = 0; + GET_HIGH_WORD(hx, x); + ix = hx & 0x7FFFFFFF; + if (ix >= 0x3FF00000) { /* |x|>= 1 */ + uint32_t lx; + GET_LOW_WORD(lx, x); + if (((ix - 0x3FF00000) | lx) == 0) { /* asin(1)=+-pi/2 with inexact */ + return x * pio2_hi + x * pio2_lo; + } + return __builtin_nans(""); // asin(|x|>1) is NaN + } else if (ix < 0x3FE00000) { /* |x|<0.5 */ + if (ix < 0x3E400000) { /* if |x| < 2**-27 */ + if (huge + x > one) return x; /* return x with inexact if x!=0*/ + } else { + t = x * x; + } + p = t * (pS0 + t * (pS1 + t * (pS2 + t * (pS3 + t * (pS4 + t * pS5))))); + q = one + t * (qS1 + t * (qS2 + t * (qS3 + t * qS4))); + w = p / q; + return x + x * w; + } + /* 1> |x|>= 0.5 */ + w = one - __builtin_fabs(x); + t = w * 0.5; + p = t * (pS0 + t * (pS1 + t * (pS2 + t * (pS3 + t * (pS4 + t * pS5))))); + q = one + t * (qS1 + t * (qS2 + t * (qS3 + t * qS4))); + s = __builtin_sqrt(t); + if (ix >= 0x3FEF3333) { /* if |x| > 0.975 */ + w = p / q; + t = pio2_hi - (2.0 * (s + s * w) - pio2_lo); + } else { + w = s; + SET_LOW_WORD(w, 0); + c = (t - w * w) / (s + w); + r = p / q; + p = 2.0 * s * r - (pio2_lo - 2.0 * c); + q = pio4_hi - 2.0 * w; + t = pio4_hi - (p - q); + } + if (hx > 0) + return t; + else + return -t; +} +/* asinh(x) + * Method : + * Based on + * asinh(x) = sign(x) * log [ |x| + sqrt(x*x+1) ] + * we have + * asinh(x) := x if 1+x*x=1, + * := sign(x)*(log(x)+ln2)) for large |x|, else + * := sign(x)*log(2|x|+1/(|x|+sqrt(x*x+1))) if|x|>2, else + * := sign(x)*log1p(|x| + x^2/(1 + sqrt(1+x^2))) + */ +double scr_ieee754_asinh(double x) { + static const double + one = 1.00000000000000000000e+00, /* 0x3FF00000, 0x00000000 */ + ln2 = 6.93147180559945286227e-01, /* 0x3FE62E42, 0xFEFA39EF */ + huge = 1.00000000000000000000e+300; + + double t, w; + int32_t hx, ix; + GET_HIGH_WORD(hx, x); + ix = hx & 0x7FFFFFFF; + if (ix >= 0x7FF00000) return x + x; /* x is inf or NaN */ + if (ix < 0x3E300000) { /* |x|<2**-28 */ + if (huge + x > one) return x; /* return x inexact except 0 */ + } + if (ix > 0x41B00000) { /* |x| > 2**28 */ + w = scr_ieee754_log(__builtin_fabs(x)) + ln2; + } else if (ix > 0x40000000) { /* 2**28 > |x| > 2.0 */ + t = __builtin_fabs(x); + w = scr_ieee754_log(2.0 * t + one / (__builtin_sqrt(x * x + one) + t)); + } else { /* 2.0 > |x| > 2**-28 */ + t = x * x; + w = scr_ieee754_log1p(__builtin_fabs(x) + t / (one + __builtin_sqrt(one + t))); + } + if (hx > 0) { + return w; + } else { + return -w; + } +} + +/* atan(x) + * Method + * 1. Reduce x to positive by atan(x) = -atan(-x). + * 2. According to the integer k=4t+0.25 chopped, t=x, the argument + * is further reduced to one of the following intervals and the + * arctangent of t is evaluated by the corresponding formula: + * + * [0,7/16] atan(x) = t-t^3*(a1+t^2*(a2+...(a10+t^2*a11)...) + * [7/16,11/16] atan(x) = atan(1/2) + atan( (t-0.5)/(1+t/2) ) + * [11/16.19/16] atan(x) = atan( 1 ) + atan( (t-1)/(1+t) ) + * [19/16,39/16] atan(x) = atan(3/2) + atan( (t-1.5)/(1+1.5t) ) + * [39/16,INF] atan(x) = atan(INF) + atan( -1/t ) + * + * Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + */ +double scr_ieee754_atan(double x) { + static const double atanhi[] = { + 4.63647609000806093515e-01, /* atan(0.5)hi 0x3FDDAC67, 0x0561BB4F */ + 7.85398163397448278999e-01, /* atan(1.0)hi 0x3FE921FB, 0x54442D18 */ + 9.82793723247329054082e-01, /* atan(1.5)hi 0x3FEF730B, 0xD281F69B */ + 1.57079632679489655800e+00, /* atan(inf)hi 0x3FF921FB, 0x54442D18 */ + }; + + static const double atanlo[] = { + 2.26987774529616870924e-17, /* atan(0.5)lo 0x3C7A2B7F, 0x222F65E2 */ + 3.06161699786838301793e-17, /* atan(1.0)lo 0x3C81A626, 0x33145C07 */ + 1.39033110312309984516e-17, /* atan(1.5)lo 0x3C700788, 0x7AF0CBBD */ + 6.12323399573676603587e-17, /* atan(inf)lo 0x3C91A626, 0x33145C07 */ + }; + + static const double aT[] = { + 3.33333333333329318027e-01, /* 0x3FD55555, 0x5555550D */ + -1.99999999998764832476e-01, /* 0xBFC99999, 0x9998EBC4 */ + 1.42857142725034663711e-01, /* 0x3FC24924, 0x920083FF */ + -1.11111104054623557880e-01, /* 0xBFBC71C6, 0xFE231671 */ + 9.09088713343650656196e-02, /* 0x3FB745CD, 0xC54C206E */ + -7.69187620504482999495e-02, /* 0xBFB3B0F2, 0xAF749A6D */ + 6.66107313738753120669e-02, /* 0x3FB10D66, 0xA0D03D51 */ + -5.83357013379057348645e-02, /* 0xBFADDE2D, 0x52DEFD9A */ + 4.97687799461593236017e-02, /* 0x3FA97B4B, 0x24760DEB */ + -3.65315727442169155270e-02, /* 0xBFA2B444, 0x2C6A6C2F */ + 1.62858201153657823623e-02, /* 0x3F90AD3A, 0xE322DA11 */ + }; + + static const double one = 1.0, huge = 1.0e300; + + double w, s1, s2, z; + int32_t ix, hx, id; + + GET_HIGH_WORD(hx, x); + ix = hx & 0x7FFFFFFF; + if (ix >= 0x44100000) { /* if |x| >= 2^66 */ + uint32_t low; + GET_LOW_WORD(low, x); + if (ix > 0x7FF00000 || (ix == 0x7FF00000 && (low != 0))) + return x + x; /* NaN */ + if (hx > 0) + return atanhi[3] + *(volatile double *)(&atanlo[3]); + else + return -atanhi[3] - *(volatile double *)(&atanlo[3]); + } + if (ix < 0x3FDC0000) { /* |x| < 0.4375 */ + if (ix < 0x3E400000) { /* |x| < 2^-27 */ + if (huge + x > one) return x; /* raise inexact */ + } + id = -1; + } else { + x = __builtin_fabs(x); + if (ix < 0x3FF30000) { /* |x| < 1.1875 */ + if (ix < 0x3FE60000) { /* 7/16 <=|x|<11/16 */ + id = 0; + x = (2.0 * x - one) / (2.0 + x); + } else { /* 11/16<=|x|< 19/16 */ + id = 1; + x = (x - one) / (x + one); + } + } else { + if (ix < 0x40038000) { /* |x| < 2.4375 */ + id = 2; + x = (x - 1.5) / (one + 1.5 * x); + } else { /* 2.4375 <= |x| < 2^66 */ + id = 3; + x = -1.0 / x; + } + } + } + /* end of argument reduction */ + z = x * x; + w = z * z; + /* break sum from i=0 to 10 aT[i]z**(i+1) into odd and even poly */ + s1 = z * (aT[0] + + w * (aT[2] + w * (aT[4] + w * (aT[6] + w * (aT[8] + w * aT[10]))))); + s2 = w * (aT[1] + w * (aT[3] + w * (aT[5] + w * (aT[7] + w * aT[9])))); + if (id < 0) { + return x - x * (s1 + s2); + } else { + z = atanhi[id] - ((x * (s1 + s2) - atanlo[id]) - x); + return (hx < 0) ? -z : z; + } +} + +/* atan2(y,x) + * Method : + * 1. Reduce y to positive by atan2(y,x)=-atan2(-y,x). + * 2. Reduce x to positive by (if x and y are unexceptional): + * ARG (x+iy) = arctan(y/x) ... if x > 0, + * ARG (x+iy) = pi - arctan[y/(-x)] ... if x < 0, + * + * Special cases: + * + * ATAN2((anything), NaN ) is NaN; + * ATAN2(NAN , (anything) ) is NaN; + * ATAN2(+-0, +(anything but NaN)) is +-0 ; + * ATAN2(+-0, -(anything but NaN)) is +-pi ; + * ATAN2(+-(anything but 0 and NaN), 0) is +-pi/2; + * ATAN2(+-(anything but INF and NaN), +INF) is +-0 ; + * ATAN2(+-(anything but INF and NaN), -INF) is +-pi; + * ATAN2(+-INF,+INF ) is +-pi/4 ; + * ATAN2(+-INF,-INF ) is +-3pi/4; + * ATAN2(+-INF, (anything but,0,NaN, and INF)) is +-pi/2; + * + * Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + */ +double scr_ieee754_atan2(double y, double x) { + static volatile double tiny = 1.0e-300; + static const double + zero = 0.0, + pi_o_4 = 7.8539816339744827900E-01, /* 0x3FE921FB, 0x54442D18 */ + pi_o_2 = 1.5707963267948965580E+00, /* 0x3FF921FB, 0x54442D18 */ + pi = 3.1415926535897931160E+00; /* 0x400921FB, 0x54442D18 */ + static volatile double pi_lo = + 1.2246467991473531772E-16; /* 0x3CA1A626, 0x33145C07 */ + + double z; + int32_t k, m, hx, hy, ix, iy; + uint32_t lx, ly; + + EXTRACT_WORDS(hx, lx, x); + ix = hx & 0x7FFFFFFF; + EXTRACT_WORDS(hy, ly, y); + iy = hy & 0x7FFFFFFF; + if (((ix | ((lx | ((int32_t)(0u - (uint32_t)(lx)))) >> 31)) > 0x7FF00000) || + ((iy | ((ly | ((int32_t)(0u - (uint32_t)(ly)))) >> 31)) > 0x7FF00000)) { + return x + y; /* x or y is NaN */ + } + if ((((int32_t)((uint32_t)(hx) - (uint32_t)(0x3FF00000))) | lx) == 0) { + return scr_ieee754_atan(y); /* x=1.0 */ + } + m = ((hy >> 31) & 1) | ((hx >> 30) & 2); /* 2*sign(x)+sign(y) */ + + /* when y = 0 */ + if ((iy | ly) == 0) { + switch (m) { + case 0: + case 1: + return y; /* atan(+-0,+anything)=+-0 */ + case 2: + return pi + tiny; /* atan(+0,-anything) = pi */ + case 3: + return -pi - tiny; /* atan(-0,-anything) =-pi */ + } + } + /* when x = 0 */ + if ((ix | lx) == 0) return (hy < 0) ? -pi_o_2 - tiny : pi_o_2 + tiny; + + /* when x is INF */ + if (ix == 0x7FF00000) { + if (iy == 0x7FF00000) { + switch (m) { + case 0: + return pi_o_4 + tiny; /* atan(+INF,+INF) */ + case 1: + return -pi_o_4 - tiny; /* atan(-INF,+INF) */ + case 2: + return 3.0 * pi_o_4 + tiny; /*atan(+INF,-INF)*/ + case 3: + return -3.0 * pi_o_4 - tiny; /*atan(-INF,-INF)*/ + } + } else { + switch (m) { + case 0: + return zero; /* atan(+...,+INF) */ + case 1: + return -zero; /* atan(-...,+INF) */ + case 2: + return pi + tiny; /* atan(+...,-INF) */ + case 3: + return -pi - tiny; /* atan(-...,-INF) */ + } + } + } + /* when y is INF */ + if (iy == 0x7FF00000) return (hy < 0) ? -pi_o_2 - tiny : pi_o_2 + tiny; + + /* compute y/x */ + k = (iy - ix) >> 20; + if (k > 60) { /* |y/x| > 2**60 */ + z = pi_o_2 + 0.5 * pi_lo; + m &= 1; + } else if (hx < 0 && k < -60) { + z = 0.0; /* 0 > |y|/x > -2**-60 */ + } else { + z = scr_ieee754_atan(__builtin_fabs(y / x)); /* safe to do y/x */ + } + switch (m) { + case 0: + return z; /* atan(+,+) */ + case 1: + return -z; /* atan(-,+) */ + case 2: + return pi - (z - pi_lo); /* atan(+,-) */ + default: /* case 3 */ + return (z - pi_lo) - pi; /* atan(-,-) */ + } +} + +/* cos(x) + * Return cosine function of x. + * + * kernel function: + * __kernel_sin ... sine function on [-pi/4,pi/4] + * __kernel_cos ... cosine function on [-pi/4,pi/4] + * __ieee754_rem_pio2 ... argument reduction routine + * + * Method. + * Let S,C and T denote the sin, cos and tan respectively on + * [-PI/4, +PI/4]. Reduce the argument x to y1+y2 = x-k*pi/2 + * in [-pi/4 , +pi/4], and let n = k mod 4. + * We have + * + * n sin(x) cos(x) tan(x) + * ---------------------------------------------------------- + * 0 S C T + * 1 C -S -1/T + * 2 -S -C T + * 3 -C S -1/T + * ---------------------------------------------------------- + * + * Special cases: + * Let trig be any of sin, cos, or tan. + * trig(+-INF) is NaN, with signals; + * trig(NaN) is that NaN; + * + * Accuracy: + * TRIG(x) returns trig(x) nearly rounded + */ +double scr_ieee754_cos(double x) { + double y[2], z = 0.0; + int32_t n, ix; + + /* High word of x. */ + GET_HIGH_WORD(ix, x); + + /* |x| ~< pi/4 */ + ix &= 0x7FFFFFFF; + if (ix <= 0x3FE921FB) { + return __kernel_cos(x, z); + } else if (ix >= 0x7FF00000) { + /* cos(Inf or NaN) is NaN */ + return x - x; + } else { + /* argument reduction needed */ + n = __ieee754_rem_pio2(x, y); + switch (n & 3) { + case 0: + return __kernel_cos(y[0], y[1]); + case 1: + return -__kernel_sin(y[0], y[1], 1); + case 2: + return -__kernel_cos(y[0], y[1]); + default: + return __kernel_sin(y[0], y[1], 1); + } + } +} + +/* exp(x) + * Returns the exponential of x. + * + * Method + * 1. Argument reduction: + * Reduce x to an r so that |r| <= 0.5*ln2 ~ 0.34658. + * Given x, find r and integer k such that + * + * x = k*ln2 + r, |r| <= 0.5*ln2. + * + * Here r will be represented as r = hi-lo for better + * accuracy. + * + * 2. Approximation of exp(r) by a special rational function on + * the interval [0,0.34658]: + * Write + * R(r**2) = r*(exp(r)+1)/(exp(r)-1) = 2 + r*r/6 - r**4/360 + ... + * We use a special Remes algorithm on [0,0.34658] to generate + * a polynomial of degree 5 to approximate R. The maximum error + * of this polynomial approximation is bounded by 2**-59. In + * other words, + * R(z) ~ 2.0 + P1*z + P2*z**2 + P3*z**3 + P4*z**4 + P5*z**5 + * (where z=r*r, and the values of P1 to P5 are listed below) + * and + * | 5 | -59 + * | 2.0+P1*z+...+P5*z - R(z) | <= 2 + * | | + * The computation of exp(r) thus becomes + * 2*r + * exp(r) = 1 + ------- + * R - r + * r*R1(r) + * = 1 + r + ----------- (for better accuracy) + * 2 - R1(r) + * where + * 2 4 10 + * R1(r) = r - (P1*r + P2*r + ... + P5*r ). + * + * 3. Scale back to obtain exp(x): + * From step 1, we have + * exp(x) = 2^k * exp(r) + * + * Special cases: + * exp(INF) is INF, exp(NaN) is NaN; + * exp(-INF) is 0, and + * for finite argument, only exp(0)=1 is exact. + * + * Accuracy: + * according to an error analysis, the error is always less than + * 1 ulp (unit in the last place). + * + * Misc. info. + * For IEEE double + * if x > 7.09782712893383973096e+02 then exp(x) overflow + * if x < -7.45133219101941108420e+02 then exp(x) underflow + * + * Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + */ +double scr_ieee754_exp(double x) { + static const double + one = 1.0, + halF[2] = {0.5, -0.5}, + o_threshold = 7.09782712893383973096e+02, /* 0x40862E42, 0xFEFA39EF */ + u_threshold = -7.45133219101941108420e+02, /* 0xC0874910, 0xD52D3051 */ + ln2HI[2] = {6.93147180369123816490e-01, /* 0x3FE62E42, 0xFEE00000 */ + -6.93147180369123816490e-01}, /* 0xBFE62E42, 0xFEE00000 */ + ln2LO[2] = {1.90821492927058770002e-10, /* 0x3DEA39EF, 0x35793C76 */ + -1.90821492927058770002e-10}, /* 0xBDEA39EF, 0x35793C76 */ + invln2 = 1.44269504088896338700e+00, /* 0x3FF71547, 0x652B82FE */ + P1 = 1.66666666666666019037e-01, /* 0x3FC55555, 0x5555553E */ + P2 = -2.77777777770155933842e-03, /* 0xBF66C16C, 0x16BEBD93 */ + P3 = 6.61375632143793436117e-05, /* 0x3F11566A, 0xAF25DE2C */ + P4 = -1.65339022054652515390e-06, /* 0xBEBBBD41, 0xC5D26BF1 */ + P5 = 4.13813679705723846039e-08, /* 0x3E663769, 0x72BEA4D0 */ + E = 2.718281828459045; /* 0x4005BF0A, 0x8B145769 */ + + static volatile double + huge = 1.0e+300, + twom1000 = 9.33263618503218878990e-302, /* 2**-1000=0x01700000,0*/ + two1023 = 8.988465674311579539e307; /* 0x1p1023 */ + + double y, hi = 0.0, lo = 0.0, c, t, twopk; + int32_t k = 0, xsb; + uint32_t hx; + + GET_HIGH_WORD(hx, x); + xsb = (hx >> 31) & 1; /* sign bit of x */ + hx &= 0x7FFFFFFF; /* high word of |x| */ + + /* filter out non-finite argument */ + if (hx >= 0x40862E42) { /* if |x|>=709.78... */ + if (hx >= 0x7FF00000) { + uint32_t lx; + GET_LOW_WORD(lx, x); + if (((hx & 0xFFFFF) | lx) != 0) + return x + x; /* NaN */ + else + return (xsb == 0) ? x : 0.0; /* exp(+-inf)={inf,0} */ + } + if (x > o_threshold) return huge * huge; /* overflow */ + if (x < u_threshold) return twom1000 * twom1000; /* underflow */ + } + + /* argument reduction */ + if (hx > 0x3FD62E42) { /* if |x| > 0.5 ln2 */ + if (hx < 0x3FF0A2B2) { /* and |x| < 1.5 ln2 */ + /* TODO(rtoy): We special case exp(1) here to return the correct + * value of E, as the computation below would get the last bit + * wrong. We should probably fix the algorithm instead. + */ + if (x == 1.0) return E; + hi = x - ln2HI[xsb]; + lo = ln2LO[xsb]; + k = 1 - xsb - xsb; + } else { + k = (int)(invln2 * x + halF[xsb]); + t = k; + hi = x - t * ln2HI[0]; /* t*ln2HI is exact here */ + lo = t * ln2LO[0]; + } + x = hi - lo; + } else if (hx < 0x3E300000) { /* when |x|<2**-28 */ + if (huge + x > one) return one + x; /* trigger inexact */ + } else { + k = 0; + } + + /* x is now in primary range */ + t = x * x; + if (k >= -1021) { + INSERT_WORDS( + twopk, + 0x3FF00000 + (int32_t)((uint32_t)(k) << 20), 0); + } else { + INSERT_WORDS(twopk, 0x3FF00000 + ((uint32_t)(k + 1000) << 20), + 0); + } + c = x - t * (P1 + t * (P2 + t * (P3 + t * (P4 + t * P5)))); + if (k == 0) { + return one - ((x * c) / (c - 2.0) - x); + } else { + y = one - ((lo - (x * c) / (2.0 - c)) - hi); + } + if (k >= -1021) { + if (k == 1024) return y * 2.0 * two1023; + return y * twopk; + } else { + return y * twopk * twom1000; + } +} + +/* + * Method : + * 1.Reduced x to positive by atanh(-x) = -atanh(x) + * 2.For x>=0.5 + * 1 2x x + * atanh(x) = --- * log(1 + -------) = 0.5 * log1p(2 * --------) + * 2 1 - x 1 - x + * + * For x<0.5 + * atanh(x) = 0.5*log1p(2x+2x*x/(1-x)) + * + * Special cases: + * atanh(x) is NaN if |x| > 1 with signal; + * atanh(NaN) is that NaN with no signal; + * atanh(+-1) is +-INF with signal. + * + */ +double scr_ieee754_atanh(double x) { + static const double one = 1.0, huge = 1e300; + static const double zero = 0.0; + + double t; + int32_t hx, ix; + uint32_t lx; + EXTRACT_WORDS(hx, lx, x); + ix = hx & 0x7FFFFFFF; + if ((ix | ((lx | ((int32_t)(0u - (uint32_t)(lx)))) >> 31)) > 0x3FF00000) { + /* |x|>1 */ + return __builtin_nans(""); + } + if (ix == 0x3FF00000) { + return x > 0 ? __builtin_inf() + : -__builtin_inf(); + } + if (ix < 0x3E300000 && (huge + x) > zero) return x; /* x<2**-28 */ + SET_HIGH_WORD(x, ix); + if (ix < 0x3FE00000) { /* x < 0.5 */ + t = x + x; + t = 0.5 * scr_ieee754_log1p(t + t * x / (one - x)); + } else { + t = 0.5 * scr_ieee754_log1p((x + x) / (one - x)); + } + if (hx >= 0) + return t; + else + return -t; +} + +/* log(x) + * Return the logrithm of x + * + * Method : + * 1. Argument Reduction: find k and f such that + * x = 2^k * (1+f), + * where sqrt(2)/2 < 1+f < sqrt(2) . + * + * 2. Approximation of log(1+f). + * Let s = f/(2+f) ; based on log(1+f) = log(1+s) - log(1-s) + * = 2s + 2/3 s**3 + 2/5 s**5 + ....., + * = 2s + s*R + * We use a special Reme algorithm on [0,0.1716] to generate + * a polynomial of degree 14 to approximate R The maximum error + * of this polynomial approximation is bounded by 2**-58.45. In + * other words, + * 2 4 6 8 10 12 14 + * R(z) ~ Lg1*s +Lg2*s +Lg3*s +Lg4*s +Lg5*s +Lg6*s +Lg7*s + * (the values of Lg1 to Lg7 are listed in the program) + * and + * | 2 14 | -58.45 + * | Lg1*s +...+Lg7*s - R(z) | <= 2 + * | | + * Note that 2s = f - s*f = f - hfsq + s*hfsq, where hfsq = f*f/2. + * In order to guarantee error in log below 1ulp, we compute log + * by + * log(1+f) = f - s*(f - R) (if f is not too large) + * log(1+f) = f - (hfsq - s*(hfsq+R)). (better accuracy) + * + * 3. Finally, log(x) = k*ln2 + log(1+f). + * = k*ln2_hi+(f-(hfsq-(s*(hfsq+R)+k*ln2_lo))) + * Here ln2 is split into two floating point number: + * ln2_hi + ln2_lo, + * where n*ln2_hi is always exact for |n| < 2000. + * + * Special cases: + * log(x) is NaN with signal if x < 0 (including -INF) ; + * log(+INF) is +INF; log(0) is -INF with signal; + * log(NaN) is that NaN with no signal. + * + * Accuracy: + * according to an error analysis, the error is always less than + * 1 ulp (unit in the last place). + * + * Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + */ +double scr_ieee754_log(double x) { + static const double /* -- */ + ln2_hi = 6.93147180369123816490e-01, /* 3fe62e42 fee00000 */ + ln2_lo = 1.90821492927058770002e-10, /* 3dea39ef 35793c76 */ + two54 = 1.80143985094819840000e+16, /* 43500000 00000000 */ + Lg1 = 6.666666666666735130e-01, /* 3FE55555 55555593 */ + Lg2 = 3.999999999940941908e-01, /* 3FD99999 9997FA04 */ + Lg3 = 2.857142874366239149e-01, /* 3FD24924 94229359 */ + Lg4 = 2.222219843214978396e-01, /* 3FCC71C5 1D8E78AF */ + Lg5 = 1.818357216161805012e-01, /* 3FC74664 96CB03DE */ + Lg6 = 1.531383769920937332e-01, /* 3FC39A09 D078C69F */ + Lg7 = 1.479819860511658591e-01; /* 3FC2F112 DF3E5244 */ + + static const double zero = 0.0; + + double hfsq, f, s, z, R, w, t1, t2, dk; + int32_t k, hx, i, j; + uint32_t lx; + + EXTRACT_WORDS(hx, lx, x); + + k = 0; + if (hx < 0x00100000) { /* x < 2**-1022 */ + if (((hx & 0x7FFFFFFF) | lx) == 0) { + return -__builtin_inf(); /* log(+-0)=-inf */ + } + if (hx < 0) { + return __builtin_nans(""); /* log(-#) = NaN */ + } + k -= 54; + x *= two54; /* subnormal number, scale up x */ + GET_HIGH_WORD(hx, x); + } + if (hx >= 0x7FF00000) return x + x; + k += (hx >> 20) - 1023; + hx &= 0x000FFFFF; + i = (hx + 0x95F64) & 0x100000; + SET_HIGH_WORD(x, hx | (i ^ 0x3FF00000)); /* normalize x or x/2 */ + k += (i >> 20); + f = x - 1.0; + if ((0x000FFFFF & (2 + hx)) < 3) { /* -2**-20 <= f < 2**-20 */ + if (f == zero) { + if (k == 0) { + return zero; + } else { + dk = (double)(k); + return dk * ln2_hi + dk * ln2_lo; + } + } + R = f * f * (0.5 - 0.33333333333333333 * f); + if (k == 0) { + return f - R; + } else { + dk = (double)(k); + return dk * ln2_hi - ((R - dk * ln2_lo) - f); + } + } + s = f / (2.0 + f); + dk = (double)(k); + z = s * s; + i = hx - 0x6147A; + w = z * z; + j = 0x6B851 - hx; + t1 = w * (Lg2 + w * (Lg4 + w * Lg6)); + t2 = z * (Lg1 + w * (Lg3 + w * (Lg5 + w * Lg7))); + i |= j; + R = t2 + t1; + if (i > 0) { + hfsq = 0.5 * f * f; + if (k == 0) + return f - (hfsq - s * (hfsq + R)); + else + return dk * ln2_hi - ((hfsq - (s * (hfsq + R) + dk * ln2_lo)) - f); + } else { + if (k == 0) + return f - s * (f - R); + else + return dk * ln2_hi - ((s * (f - R) - dk * ln2_lo) - f); + } +} + +/* double log1p(double x) + * + * Method : + * 1. Argument Reduction: find k and f such that + * 1+x = 2^k * (1+f), + * where sqrt(2)/2 < 1+f < sqrt(2) . + * + * Note. If k=0, then f=x is exact. However, if k!=0, then f + * may not be representable exactly. In that case, a correction + * term is need. Let u=1+x rounded. Let c = (1+x)-u, then + * log(1+x) - log(u) ~ c/u. Thus, we proceed to compute log(u), + * and add back the correction term c/u. + * (Note: when x > 2**53, one can simply return log(x)) + * + * 2. Approximation of log1p(f). + * Let s = f/(2+f) ; based on log(1+f) = log(1+s) - log(1-s) + * = 2s + 2/3 s**3 + 2/5 s**5 + ....., + * = 2s + s*R + * We use a special Reme algorithm on [0,0.1716] to generate + * a polynomial of degree 14 to approximate R The maximum error + * of this polynomial approximation is bounded by 2**-58.45. In + * other words, + * 2 4 6 8 10 12 14 + * R(z) ~ Lp1*s +Lp2*s +Lp3*s +Lp4*s +Lp5*s +Lp6*s +Lp7*s + * (the values of Lp1 to Lp7 are listed in the program) + * and + * | 2 14 | -58.45 + * | Lp1*s +...+Lp7*s - R(z) | <= 2 + * | | + * Note that 2s = f - s*f = f - hfsq + s*hfsq, where hfsq = f*f/2. + * In order to guarantee error in log below 1ulp, we compute log + * by + * log1p(f) = f - (hfsq - s*(hfsq+R)). + * + * 3. Finally, log1p(x) = k*ln2 + log1p(f). + * = k*ln2_hi+(f-(hfsq-(s*(hfsq+R)+k*ln2_lo))) + * Here ln2 is split into two floating point number: + * ln2_hi + ln2_lo, + * where n*ln2_hi is always exact for |n| < 2000. + * + * Special cases: + * log1p(x) is NaN with signal if x < -1 (including -INF) ; + * log1p(+INF) is +INF; log1p(-1) is -INF with signal; + * log1p(NaN) is that NaN with no signal. + * + * Accuracy: + * according to an error analysis, the error is always less than + * 1 ulp (unit in the last place). + * + * Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + * + * Note: Assuming log() return accurate answer, the following + * algorithm can be used to compute log1p(x) to within a few ULP: + * + * u = 1+x; + * if(u==1.0) return x ; else + * return log(u)*(x/(u-1.0)); + * + * See HP-15C Advanced Functions Handbook, p.193. + */ +double scr_ieee754_log1p(double x) { + static const double /* -- */ + ln2_hi = 6.93147180369123816490e-01, /* 3fe62e42 fee00000 */ + ln2_lo = 1.90821492927058770002e-10, /* 3dea39ef 35793c76 */ + two54 = 1.80143985094819840000e+16, /* 43500000 00000000 */ + Lp1 = 6.666666666666735130e-01, /* 3FE55555 55555593 */ + Lp2 = 3.999999999940941908e-01, /* 3FD99999 9997FA04 */ + Lp3 = 2.857142874366239149e-01, /* 3FD24924 94229359 */ + Lp4 = 2.222219843214978396e-01, /* 3FCC71C5 1D8E78AF */ + Lp5 = 1.818357216161805012e-01, /* 3FC74664 96CB03DE */ + Lp6 = 1.531383769920937332e-01, /* 3FC39A09 D078C69F */ + Lp7 = 1.479819860511658591e-01; /* 3FC2F112 DF3E5244 */ + + static const double zero = 0.0; + + double hfsq, f, c, s, z, R, u; + int32_t k, hx, hu, ax; + + GET_HIGH_WORD(hx, x); + ax = hx & 0x7FFFFFFF; + + k = 1; + if (hx < 0x3FDA827A) { /* 1+x < sqrt(2)+ */ + if (ax >= 0x3FF00000) { /* x <= -1.0 */ + if (x == -1.0) + return -__builtin_inf(); /* log1p(-1)=+inf */ + else + return __builtin_nans(""); // log1p(x<-1)=NaN + } + if (ax < 0x3E200000) { /* |x| < 2**-29 */ + if (two54 + x > zero /* raise inexact */ + && ax < 0x3C900000) /* |x| < 2**-54 */ + return x; + else + return x - x * x * 0.5; + } + if (hx > 0 || hx <= (int32_t)(0xBFD2BEC4)) { + k = 0; + f = x; + hu = 1; + } /* sqrt(2)/2- <= 1+x < sqrt(2)+ */ + } + if (hx >= 0x7FF00000) return x + x; + if (k != 0) { + if (hx < 0x43400000) { + u = 1.0 + x; + GET_HIGH_WORD(hu, u); + k = (hu >> 20) - 1023; + c = (k > 0) ? 1.0 - (u - x) : x - (u - 1.0); /* correction term */ + c /= u; + } else { + u = x; + GET_HIGH_WORD(hu, u); + k = (hu >> 20) - 1023; + c = 0; + } + hu &= 0x000FFFFF; + /* + * The approximation to sqrt(2) used in thresholds is not + * critical. However, the ones used above must give less + * strict bounds than the one here so that the k==0 case is + * never reached from here, since here we have committed to + * using the correction term but don't use it if k==0. + */ + if (hu < 0x6A09E) { /* u ~< sqrt(2) */ + SET_HIGH_WORD(u, hu | 0x3FF00000); /* normalize u */ + } else { + k += 1; + SET_HIGH_WORD(u, hu | 0x3FE00000); /* normalize u/2 */ + hu = (0x00100000 - hu) >> 2; + } + f = u - 1.0; + } + hfsq = 0.5 * f * f; + if (hu == 0) { /* |f| < 2**-20 */ + if (f == zero) { + if (k == 0) { + return zero; + } else { + c += k * ln2_lo; + return k * ln2_hi + c; + } + } + R = hfsq * (1.0 - 0.66666666666666666 * f); + if (k == 0) + return f - R; + else + return k * ln2_hi - ((R - (k * ln2_lo + c)) - f); + } + s = f / (2.0 + f); + z = s * s; + R = z * (Lp1 + + z * (Lp2 + z * (Lp3 + z * (Lp4 + z * (Lp5 + z * (Lp6 + z * Lp7)))))); + if (k == 0) + return f - (hfsq - s * (hfsq + R)); + else + return k * ln2_hi - ((hfsq - (s * (hfsq + R) + (k * ln2_lo + c))) - f); +} + +/* + * k_log1p(f): + * Return log(1+f) - f for 1+f in ~[sqrt(2)/2, sqrt(2)]. + * + * The following describes the overall strategy for computing + * logarithms in base e. The argument reduction and adding the final + * term of the polynomial are done by the caller for increased accuracy + * when different bases are used. + * + * Method : + * 1. Argument Reduction: find k and f such that + * x = 2^k * (1+f), + * where sqrt(2)/2 < 1+f < sqrt(2) . + * + * 2. Approximation of log(1+f). + * Let s = f/(2+f) ; based on log(1+f) = log(1+s) - log(1-s) + * = 2s + 2/3 s**3 + 2/5 s**5 + ....., + * = 2s + s*R + * We use a special Reme algorithm on [0,0.1716] to generate + * a polynomial of degree 14 to approximate R The maximum error + * of this polynomial approximation is bounded by 2**-58.45. In + * other words, + * 2 4 6 8 10 12 14 + * R(z) ~ Lg1*s +Lg2*s +Lg3*s +Lg4*s +Lg5*s +Lg6*s +Lg7*s + * (the values of Lg1 to Lg7 are listed in the program) + * and + * | 2 14 | -58.45 + * | Lg1*s +...+Lg7*s - R(z) | <= 2 + * | | + * Note that 2s = f - s*f = f - hfsq + s*hfsq, where hfsq = f*f/2. + * In order to guarantee error in log below 1ulp, we compute log + * by + * log(1+f) = f - s*(f - R) (if f is not too large) + * log(1+f) = f - (hfsq - s*(hfsq+R)). (better accuracy) + * + * 3. Finally, log(x) = k*ln2 + log(1+f). + * = k*ln2_hi+(f-(hfsq-(s*(hfsq+R)+k*ln2_lo))) + * Here ln2 is split into two floating point number: + * ln2_hi + ln2_lo, + * where n*ln2_hi is always exact for |n| < 2000. + * + * Special cases: + * log(x) is NaN with signal if x < 0 (including -INF) ; + * log(+INF) is +INF; log(0) is -INF with signal; + * log(NaN) is that NaN with no signal. + * + * Accuracy: + * according to an error analysis, the error is always less than + * 1 ulp (unit in the last place). + * + * Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + */ + +static const double Lg1 = 6.666666666666735130e-01, /* 3FE55555 55555593 */ + Lg2 = 3.999999999940941908e-01, /* 3FD99999 9997FA04 */ + Lg3 = 2.857142874366239149e-01, /* 3FD24924 94229359 */ + Lg4 = 2.222219843214978396e-01, /* 3FCC71C5 1D8E78AF */ + Lg5 = 1.818357216161805012e-01, /* 3FC74664 96CB03DE */ + Lg6 = 1.531383769920937332e-01, /* 3FC39A09 D078C69F */ + Lg7 = 1.479819860511658591e-01; /* 3FC2F112 DF3E5244 */ + +/* + * We always inline k_log1p(), since doing so produces a + * substantial performance improvement (~40% on amd64). + */ +static inline double k_log1p(double f) { + double hfsq, s, z, R, w, t1, t2; + + s = f / (2.0 + f); + z = s * s; + w = z * z; + t1 = w * (Lg2 + w * (Lg4 + w * Lg6)); + t2 = z * (Lg1 + w * (Lg3 + w * (Lg5 + w * Lg7))); + R = t2 + t1; + hfsq = 0.5 * f * f; + return s * (hfsq + R); +} + +/* + * Return the base 2 logarithm of x. See e_log.c and k_log.h for most + * comments. + * + * This reduces x to {k, 1+f} exactly as in e_log.c, then calls the kernel, + * then does the combining and scaling steps + * log2(x) = (f - 0.5*f*f + k_log1p(f)) / ln2 + k + * in not-quite-routine extra precision. + */ +double scr_ieee754_log2(double x) { + static const double + two54 = 1.80143985094819840000e+16, /* 0x43500000, 0x00000000 */ + ivln2hi = 1.44269504072144627571e+00, /* 0x3FF71547, 0x65200000 */ + ivln2lo = 1.67517131648865118353e-10; /* 0x3DE705FC, 0x2EEFA200 */ + + double f, hfsq, hi, lo, r, val_hi, val_lo, w, y; + int32_t i, k, hx; + uint32_t lx; + + EXTRACT_WORDS(hx, lx, x); + + k = 0; + if (hx < 0x00100000) { /* x < 2**-1022 */ + if (((hx & 0x7FFFFFFF) | lx) == 0) { + return -__builtin_inf(); /* log(+-0)=-inf */ + } + if (hx < 0) { + return __builtin_nans(""); /* log(-#) = NaN */ + } + k -= 54; + x *= two54; /* subnormal number, scale up x */ + GET_HIGH_WORD(hx, x); + } + if (hx >= 0x7FF00000) return x + x; + if (hx == 0x3FF00000 && lx == 0) return 0.0; /* log(1) = +0 */ + k += (hx >> 20) - 1023; + hx &= 0x000FFFFF; + i = (hx + 0x95F64) & 0x100000; + SET_HIGH_WORD(x, hx | (i ^ 0x3FF00000)); /* normalize x or x/2 */ + k += (i >> 20); + y = (double)(k); + f = x - 1.0; + hfsq = 0.5 * f * f; + r = k_log1p(f); + + /* + * f-hfsq must (for args near 1) be evaluated in extra precision + * to avoid a large cancellation when x is near sqrt(2) or 1/sqrt(2). + * This is fairly efficient since f-hfsq only depends on f, so can + * be evaluated in parallel with R. Not combining hfsq with R also + * keeps R small (though not as small as a true `lo' term would be), + * so that extra precision is not needed for terms involving R. + * + * Compiler bugs involving extra precision used to break Dekker's + * theorem for spitting f-hfsq as hi+lo, unless double_t was used + * or the multi-precision calculations were avoided when double_t + * has extra precision. These problems are now automatically + * avoided as a side effect of the optimization of combining the + * Dekker splitting step with the clear-low-bits step. + * + * y must (for args near sqrt(2) and 1/sqrt(2)) be added in extra + * precision to avoid a very large cancellation when x is very near + * these values. Unlike the above cancellations, this problem is + * specific to base 2. It is strange that adding +-1 is so much + * harder than adding +-ln2 or +-log10_2. + * + * This uses Dekker's theorem to normalize y+val_hi, so the + * compiler bugs are back in some configurations, sigh. And I + * don't want to used double_t to avoid them, since that gives a + * pessimization and the support for avoiding the pessimization + * is not yet available. + * + * The multi-precision calculations for the multiplications are + * routine. + */ + hi = f - hfsq; + SET_LOW_WORD(hi, 0); + lo = (f - hi) - hfsq + r; + val_hi = hi * ivln2hi; + val_lo = (lo + hi) * ivln2lo + lo * ivln2hi; + + /* spadd(val_hi, val_lo, y), except for not using double_t: */ + w = y + val_hi; + val_lo += (y - w) + val_hi; + val_hi = w; + + return val_lo + val_hi; +} + +/* + * Return the base 10 logarithm of x + * + * Method : + * Let log10_2hi = leading 40 bits of log10(2) and + * log10_2lo = log10(2) - log10_2hi, + * ivln10 = 1/log(10) rounded. + * Then + * n = ilogb(x), + * if(n<0) n = n+1; + * x = scalbn(x,-n); + * log10(x) := n*log10_2hi + (n*log10_2lo + ivln10*log(x)) + * + * Note 1: + * To guarantee log10(10**n)=n, where 10**n is normal, the rounding + * mode must set to Round-to-Nearest. + * Note 2: + * [1/log(10)] rounded to 53 bits has error .198 ulps; + * log10 is monotonic at all binary break points. + * + * Special cases: + * log10(x) is NaN if x < 0; + * log10(+INF) is +INF; log10(0) is -INF; + * log10(NaN) is that NaN; + * log10(10**N) = N for N=0,1,...,22. + */ +double scr_ieee754_log10(double x) { + static const double + two54 = 1.80143985094819840000e+16, /* 0x43500000, 0x00000000 */ + ivln10 = 4.34294481903251816668e-01, + log10_2hi = 3.01029995663611771306e-01, /* 0x3FD34413, 0x509F6000 */ + log10_2lo = 3.69423907715893078616e-13; /* 0x3D59FEF3, 0x11F12B36 */ + + double y; + int32_t i, k, hx; + uint32_t lx; + + EXTRACT_WORDS(hx, lx, x); + + k = 0; + if (hx < 0x00100000) { /* x < 2**-1022 */ + if (((hx & 0x7FFFFFFF) | lx) == 0) { + return -__builtin_inf(); /* log(+-0)=-inf */ + } + if (hx < 0) { + return __builtin_nan(""); /* log(-#) = NaN */ + } + k -= 54; + x *= two54; /* subnormal number, scale up x */ + GET_HIGH_WORD(hx, x); + GET_LOW_WORD(lx, x); + } + if (hx >= 0x7FF00000) return x + x; + if (hx == 0x3FF00000 && lx == 0) return 0.0; /* log(1) = +0 */ + k += (hx >> 20) - 1023; + + i = (k & 0x80000000) >> 31; + hx = (hx & 0x000FFFFF) | ((0x3FF - i) << 20); + y = k + i; + SET_HIGH_WORD(x, hx); + SET_LOW_WORD(x, lx); + + double z = y * log10_2lo + ivln10 * scr_ieee754_log(x); + return z + y * log10_2hi; +} + +/* expm1(x) + * Returns exp(x)-1, the exponential of x minus 1. + * + * Method + * 1. Argument reduction: + * Given x, find r and integer k such that + * + * x = k*ln2 + r, |r| <= 0.5*ln2 ~ 0.34658 + * + * Here a correction term c will be computed to compensate + * the error in r when rounded to a floating-point number. + * + * 2. Approximating expm1(r) by a special rational function on + * the interval [0,0.34658]: + * Since + * r*(exp(r)+1)/(exp(r)-1) = 2+ r^2/6 - r^4/360 + ... + * we define R1(r*r) by + * r*(exp(r)+1)/(exp(r)-1) = 2+ r^2/6 * R1(r*r) + * That is, + * R1(r**2) = 6/r *((exp(r)+1)/(exp(r)-1) - 2/r) + * = 6/r * ( 1 + 2.0*(1/(exp(r)-1) - 1/r)) + * = 1 - r^2/60 + r^4/2520 - r^6/100800 + ... + * We use a special Reme algorithm on [0,0.347] to generate + * a polynomial of degree 5 in r*r to approximate R1. The + * maximum error of this polynomial approximation is bounded + * by 2**-61. In other words, + * R1(z) ~ 1.0 + Q1*z + Q2*z**2 + Q3*z**3 + Q4*z**4 + Q5*z**5 + * where Q1 = -1.6666666666666567384E-2, + * Q2 = 3.9682539681370365873E-4, + * Q3 = -9.9206344733435987357E-6, + * Q4 = 2.5051361420808517002E-7, + * Q5 = -6.2843505682382617102E-9; + * z = r*r, + * with error bounded by + * | 5 | -61 + * | 1.0+Q1*z+...+Q5*z - R1(z) | <= 2 + * | | + * + * expm1(r) = exp(r)-1 is then computed by the following + * specific way which minimize the accumulation rounding error: + * 2 3 + * r r [ 3 - (R1 + R1*r/2) ] + * expm1(r) = r + --- + --- * [--------------------] + * 2 2 [ 6 - r*(3 - R1*r/2) ] + * + * To compensate the error in the argument reduction, we use + * expm1(r+c) = expm1(r) + c + expm1(r)*c + * ~ expm1(r) + c + r*c + * Thus c+r*c will be added in as the correction terms for + * expm1(r+c). Now rearrange the term to avoid optimization + * screw up: + * ( 2 2 ) + * ({ ( r [ R1 - (3 - R1*r/2) ] ) } r ) + * expm1(r+c)~r - ({r*(--- * [--------------------]-c)-c} - --- ) + * ({ ( 2 [ 6 - r*(3 - R1*r/2) ] ) } 2 ) + * ( ) + * + * = r - E + * 3. Scale back to obtain expm1(x): + * From step 1, we have + * expm1(x) = either 2^k*[expm1(r)+1] - 1 + * = or 2^k*[expm1(r) + (1-2^-k)] + * 4. Implementation notes: + * (A). To save one multiplication, we scale the coefficient Qi + * to Qi*2^i, and replace z by (x^2)/2. + * (B). To achieve maximum accuracy, we compute expm1(x) by + * (i) if x < -56*ln2, return -1.0, (raise inexact if x!=inf) + * (ii) if k=0, return r-E + * (iii) if k=-1, return 0.5*(r-E)-0.5 + * (iv) if k=1 if r < -0.25, return 2*((r+0.5)- E) + * else return 1.0+2.0*(r-E); + * (v) if (k<-2||k>56) return 2^k(1-(E-r)) - 1 (or exp(x)-1) + * (vi) if k <= 20, return 2^k((1-2^-k)-(E-r)), else + * (vii) return 2^k(1-((E+2^-k)-r)) + * + * Special cases: + * expm1(INF) is INF, expm1(NaN) is NaN; + * expm1(-INF) is -1, and + * for finite argument, only expm1(0)=0 is exact. + * + * Accuracy: + * according to an error analysis, the error is always less than + * 1 ulp (unit in the last place). + * + * Misc. info. + * For IEEE double + * if x > 7.09782712893383973096e+02 then expm1(x) overflow + * + * Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + */ +double scr_ieee754_expm1(double x) { + static const double + one = 1.0, + tiny = 1.0e-300, + o_threshold = 7.09782712893383973096e+02, /* 0x40862E42, 0xFEFA39EF */ + ln2_hi = 6.93147180369123816490e-01, /* 0x3FE62E42, 0xFEE00000 */ + ln2_lo = 1.90821492927058770002e-10, /* 0x3DEA39EF, 0x35793C76 */ + invln2 = 1.44269504088896338700e+00, /* 0x3FF71547, 0x652B82FE */ + /* Scaled Q's: Qn_here = 2**n * Qn_above, for R(2*z) where z = hxs = + x*x/2: */ + Q1 = -3.33333333333331316428e-02, /* BFA11111 111110F4 */ + Q2 = 1.58730158725481460165e-03, /* 3F5A01A0 19FE5585 */ + Q3 = -7.93650757867487942473e-05, /* BF14CE19 9EAADBB7 */ + Q4 = 4.00821782732936239552e-06, /* 3ED0CFCA 86E65239 */ + Q5 = -2.01099218183624371326e-07; /* BE8AFDB7 6E09C32D */ + + static volatile double huge = 1.0e+300; + + double y, hi, lo, c, t, e, hxs, hfx, r1, twopk; + int32_t k, xsb; + uint32_t hx; + + GET_HIGH_WORD(hx, x); + xsb = hx & 0x80000000; /* sign bit of x */ + hx &= 0x7FFFFFFF; /* high word of |x| */ + + /* filter out huge and non-finite argument */ + if (hx >= 0x4043687A) { /* if |x|>=56*ln2 */ + if (hx >= 0x40862E42) { /* if |x|>=709.78... */ + if (hx >= 0x7FF00000) { + uint32_t low; + GET_LOW_WORD(low, x); + if (((hx & 0xFFFFF) | low) != 0) + return x + x; /* NaN */ + else + return (xsb == 0) ? x : -1.0; /* exp(+-inf)={inf,-1} */ + } + if (x > o_threshold) return huge * huge; /* overflow */ + } + if (xsb != 0) { /* x < -56*ln2, return -1.0 with inexact */ + if (x + tiny < 0.0) /* raise inexact */ + return tiny - one; /* return -1 */ + } + } + + /* argument reduction */ + if (hx > 0x3FD62E42) { /* if |x| > 0.5 ln2 */ + if (hx < 0x3FF0A2B2) { /* and |x| < 1.5 ln2 */ + if (xsb == 0) { + hi = x - ln2_hi; + lo = ln2_lo; + k = 1; + } else { + hi = x + ln2_hi; + lo = -ln2_lo; + k = -1; + } + } else { + k = invln2 * x + ((xsb == 0) ? 0.5 : -0.5); + t = k; + hi = x - t * ln2_hi; /* t*ln2_hi is exact here */ + lo = t * ln2_lo; + } + x = hi - lo; + c = (hi - x) - lo; + } else if (hx < 0x3C900000) { /* when |x|<2**-54, return x */ + t = huge + x; /* return x with inexact flags when x!=0 */ + return x - (t - (huge + x)); + } else { + k = 0; + } + + /* x is now in primary range */ + hfx = 0.5 * x; + hxs = x * hfx; + r1 = one + hxs * (Q1 + hxs * (Q2 + hxs * (Q3 + hxs * (Q4 + hxs * Q5)))); + t = 3.0 - r1 * hfx; + e = hxs * ((r1 - t) / (6.0 - x * t)); + if (k == 0) { + return x - (x * e - hxs); /* c is 0 */ + } else { + INSERT_WORDS( + twopk, + 0x3FF00000 + (int32_t)((uint32_t)(k) << 20), + 0); /* 2^k */ + e = (x * (e - c) - c); + e -= hxs; + if (k == -1) return 0.5 * (x - e) - 0.5; + if (k == 1) { + if (x < -0.25) + return -2.0 * (e - (x + 0.5)); + else + return one + 2.0 * (x - e); + } + if (k <= -2 || k > 56) { /* suffice to return exp(x)-1 */ + y = one - (e - x); + // TODO(mvstanton): is this replacement for the hex float + // sufficient? + // if (k == 1024) y = y*2.0*0x1p1023; + if (k == 1024) + y = y * 2.0 * 8.98846567431158e+307; + else + y = y * twopk; + return y - one; + } + t = one; + if (k < 20) { + SET_HIGH_WORD(t, 0x3FF00000 - (0x200000 >> k)); /* t=1-2^-k */ + y = t - (e - x); + y = y * twopk; + } else { + SET_HIGH_WORD(t, ((0x3FF - k) << 20)); /* 2^-k */ + y = x - (e + t); + y += one; + y = y * twopk; + } + } + return y; +} + +double scr_ieee754_cbrt(double x) { + static const uint32_t + B1 = 715094163, /* B1 = (1023-1023/3-0.03306235651)*2**20 */ + B2 = 696219795; /* B2 = (1023-1023/3-54/3-0.03306235651)*2**20 */ + + /* |1/cbrt(x) - p(x)| < 2**-23.5 (~[-7.93e-8, 7.929e-8]). */ + static const double P0 = 1.87595182427177009643, /* 0x3FFE03E6, 0x0F61E692 */ + P1 = -1.88497979543377169875, /* 0xBFFE28E0, 0x92F02420 */ + P2 = 1.621429720105354466140, /* 0x3FF9F160, 0x4A49D6C2 */ + P3 = -0.758397934778766047437, /* 0xBFE844CB, 0xBEE751D9 */ + P4 = 0.145996192886612446982; /* 0x3FC2B000, 0xD4E4EDD7 */ + + int32_t hx; + double r, s, t = 0.0, w; + uint32_t sign; + uint32_t high, low; + + EXTRACT_WORDS(hx, low, x); + sign = hx & 0x80000000; /* sign= sign(x) */ + hx ^= sign; + if (hx >= 0x7FF00000) return (x + x); /* cbrt(NaN,INF) is itself */ + + /* + * Rough cbrt to 5 bits: + * cbrt(2**e*(1+m) ~= 2**(e/3)*(1+(e%3+m)/3) + * where e is integral and >= 0, m is real and in [0, 1), and "/" and + * "%" are integer division and modulus with rounding towards minus + * infinity. The RHS is always >= the LHS and has a maximum relative + * error of about 1 in 16. Adding a bias of -0.03306235651 to the + * (e%3+m)/3 term reduces the error to about 1 in 32. With the IEEE + * floating point representation, for finite positive normal values, + * ordinary integer division of the value in bits magically gives + * almost exactly the RHS of the above provided we first subtract the + * exponent bias (1023 for doubles) and later add it back. We do the + * subtraction virtually to keep e >= 0 so that ordinary integer + * division rounds towards minus infinity; this is also efficient. + */ + if (hx < 0x00100000) { /* zero or subnormal? */ + if ((hx | low) == 0) return (x); /* cbrt(0) is itself */ + SET_HIGH_WORD(t, 0x43500000); /* set t= 2**54 */ + t *= x; + GET_HIGH_WORD(high, t); + INSERT_WORDS(t, sign | ((high & 0x7FFFFFFF) / 3 + B2), 0); + } else { + INSERT_WORDS(t, sign | (hx / 3 + B1), 0); + } + + /* + * New cbrt to 23 bits: + * cbrt(x) = t*cbrt(x/t**3) ~= t*P(t**3/x) + * where P(r) is a polynomial of degree 4 that approximates 1/cbrt(r) + * to within 2**-23.5 when |r - 1| < 1/10. The rough approximation + * has produced t such than |t/cbrt(x) - 1| ~< 1/32, and cubing this + * gives us bounds for r = t**3/x. + * + * Try to optimize for parallel evaluation as in k_tanf.c. + */ + r = (t * t) * (t / x); + t = t * ((P0 + r * (P1 + r * P2)) + ((r * r) * r) * (P3 + r * P4)); + + /* + * Round t away from zero to 23 bits (sloppily except for ensuring that + * the result is larger in magnitude than cbrt(x) but not much more than + * 2 23-bit ulps larger). With rounding towards zero, the error bound + * would be ~5/6 instead of ~4/6. With a maximum error of 2 23-bit ulps + * in the rounded t, the infinite-precision error in the Newton + * approximation barely affects third digit in the final error + * 0.667; the error in the rounded t can be up to about 3 23-bit ulps + * before the final error is larger than 0.667 ulps. + */ + uint64_t bits = scr_fd_bits(t); + bits = (bits + 0x80000000) & 0xFFFFFFFFC0000000ULL; + t = scr_fd_double(bits); + + /* one step Newton iteration to 53 bits with error < 0.667 ulps */ + s = t * t; /* t*t is exact */ + r = x / s; /* error <= 0.5 ulps; |r| < |t| */ + w = t + t; /* t+t is exact */ + r = (r - t) / (w + r); /* r-t is exact; w+r ~= 3*t */ + t = t + t * r; /* error <= 0.5 + 0.5/3 + epsilon */ + + return (t); +} + +/* sin(x) + * Return sine function of x. + * + * kernel function: + * __kernel_sin ... sine function on [-pi/4,pi/4] + * __kernel_cos ... cose function on [-pi/4,pi/4] + * __ieee754_rem_pio2 ... argument reduction routine + * + * Method. + * Let S,C and T denote the sin, cos and tan respectively on + * [-PI/4, +PI/4]. Reduce the argument x to y1+y2 = x-k*pi/2 + * in [-pi/4 , +pi/4], and let n = k mod 4. + * We have + * + * n sin(x) cos(x) tan(x) + * ---------------------------------------------------------- + * 0 S C T + * 1 C -S -1/T + * 2 -S -C T + * 3 -C S -1/T + * ---------------------------------------------------------- + * + * Special cases: + * Let trig be any of sin, cos, or tan. + * trig(+-INF) is NaN, with signals; + * trig(NaN) is that NaN; + * + * Accuracy: + * TRIG(x) returns trig(x) nearly rounded + */ +double scr_ieee754_sin(double x) { + double y[2], z = 0.0; + int32_t n, ix; + + /* High word of x. */ + GET_HIGH_WORD(ix, x); + + /* |x| ~< pi/4 */ + ix &= 0x7FFFFFFF; + if (ix <= 0x3FE921FB) { + return __kernel_sin(x, z, 0); + } else if (ix >= 0x7FF00000) { + /* sin(Inf or NaN) is NaN */ + return x - x; + } else { + /* argument reduction needed */ + n = __ieee754_rem_pio2(x, y); + switch (n & 3) { + case 0: + return __kernel_sin(y[0], y[1], 1); + case 1: + return __kernel_cos(y[0], y[1]); + case 2: + return -__kernel_sin(y[0], y[1], 1); + default: + return -__kernel_cos(y[0], y[1]); + } + } +} + +/* tan(x) + * Return tangent function of x. + * + * kernel function: + * __kernel_tan ... tangent function on [-pi/4,pi/4] + * __ieee754_rem_pio2 ... argument reduction routine + * + * Method. + * Let S,C and T denote the sin, cos and tan respectively on + * [-PI/4, +PI/4]. Reduce the argument x to y1+y2 = x-k*pi/2 + * in [-pi/4 , +pi/4], and let n = k mod 4. + * We have + * + * n sin(x) cos(x) tan(x) + * ---------------------------------------------------------- + * 0 S C T + * 1 C -S -1/T + * 2 -S -C T + * 3 -C S -1/T + * ---------------------------------------------------------- + * + * Special cases: + * Let trig be any of sin, cos, or tan. + * trig(+-INF) is NaN, with signals; + * trig(NaN) is that NaN; + * + * Accuracy: + * TRIG(x) returns trig(x) nearly rounded + */ +double scr_ieee754_tan(double x) { + double y[2], z = 0.0; + int32_t n, ix; + + /* High word of x. */ + GET_HIGH_WORD(ix, x); + + /* |x| ~< pi/4 */ + ix &= 0x7FFFFFFF; + if (ix <= 0x3FE921FB) { + return __kernel_tan(x, z, 1); + } else if (ix >= 0x7FF00000) { + /* tan(Inf or NaN) is NaN */ + return x - x; /* NaN */ + } else { + /* argument reduction needed */ + n = __ieee754_rem_pio2(x, y); + /* 1 -> n even, -1 -> n odd */ + return __kernel_tan(y[0], y[1], 1 - ((n & 1) << 1)); + } +} + +/* + * ES6 draft 09-27-13, section 20.2.2.12. + * Math.cosh + * Method : + * mathematically cosh(x) if defined to be (exp(x)+exp(-x))/2 + * 1. Replace x by |x| (cosh(x) = cosh(-x)). + * 2. + * [ exp(x) - 1 ]^2 + * 0 <= x <= ln2/2 : cosh(x) := 1 + ------------------- + * 2*exp(x) + * + * exp(x) + 1/exp(x) + * ln2/2 <= x <= 22 : cosh(x) := ------------------- + * 2 + * 22 <= x <= lnovft : cosh(x) := exp(x)/2 + * lnovft <= x <= ln2ovft: cosh(x) := exp(x/2)/2 * exp(x/2) + * ln2ovft < x : cosh(x) := huge*huge (overflow) + * + * Special cases: + * cosh(x) is |x| if x is +INF, -INF, or NaN. + * only cosh(0)=1 is exact for finite x. + */ +double scr_ieee754_cosh(double x) { + static const double KCOSH_OVERFLOW = 710.4758600739439; + static const double one = 1.0, half = 0.5; + static volatile double huge = 1.0e+300; + + int32_t ix; + + /* High word of |x|. */ + GET_HIGH_WORD(ix, x); + ix &= 0x7FFFFFFF; + + // |x| in [0,0.5*log2], return 1+expm1(|x|)^2/(2*exp(|x|)) + if (ix < 0x3FD62E43) { + double t = scr_ieee754_expm1(__builtin_fabs(x)); + double w = one + t; + // For |x| < 2^-55, cosh(x) = 1 + if (ix < 0x3C800000) return w; + return one + (t * t) / (w + w); + } + + // |x| in [0.5*log2, 22], return (exp(|x|)+1/exp(|x|)/2 + if (ix < 0x40360000) { + double t = scr_ieee754_exp(__builtin_fabs(x)); + return half * t + half / t; + } + + // |x| in [22, log(maxdouble)], return half*exp(|x|) + if (ix < 0x40862E42) return half * scr_ieee754_exp(__builtin_fabs(x)); + + // |x| in [log(maxdouble), overflowthreshold] + if (__builtin_fabs(x) <= KCOSH_OVERFLOW) { + double w = scr_ieee754_exp(half * __builtin_fabs(x)); + double t = half * w; + return t * w; + } + + /* x is INF or NaN */ + if (ix >= 0x7FF00000) return x * x; + + // |x| > overflowthreshold. + return huge * huge; +} + +/* + * ES6 draft 09-27-13, section 20.2.2.30. + * Math.sinh + * Method : + * mathematically sinh(x) if defined to be (exp(x)-exp(-x))/2 + * 1. Replace x by |x| (sinh(-x) = -sinh(x)). + * 2. + * E + E/(E+1) + * 0 <= x <= 22 : sinh(x) := --------------, E=expm1(x) + * 2 + * + * 22 <= x <= lnovft : sinh(x) := exp(x)/2 + * lnovft <= x <= ln2ovft: sinh(x) := exp(x/2)/2 * exp(x/2) + * ln2ovft < x : sinh(x) := x*shuge (overflow) + * + * Special cases: + * sinh(x) is |x| if x is +Infinity, -Infinity, or NaN. + * only sinh(0)=0 is exact for finite x. + */ +double scr_ieee754_sinh(double x) { + static const double KSINH_OVERFLOW = 710.4758600739439, + TWO_M28 = + 3.725290298461914e-9, // 2^-28, empty lower half + LOG_MAXD = 709.7822265625; // 0x40862E42 00000000, empty lower half + static const double shuge = 1.0e307; + + double h = (x < 0) ? -0.5 : 0.5; + // |x| in [0, 22]. return sign(x)*0.5*(E+E/(E+1)) + double ax = __builtin_fabs(x); + if (ax < 22) { + // For |x| < 2^-28, sinh(x) = x + if (ax < TWO_M28) return x; + double t = scr_ieee754_expm1(ax); + if (ax < 1) { + return h * (2 * t - t * t / (t + 1)); + } + return h * (t + t / (t + 1)); + } + // |x| in [22, log(maxdouble)], return 0.5 * exp(|x|) + if (ax < LOG_MAXD) return h * scr_ieee754_exp(ax); + // |x| in [log(maxdouble), overflowthreshold] + // overflowthreshold = 710.4758600739426 + if (ax <= KSINH_OVERFLOW) { + double w = scr_ieee754_exp(0.5 * ax); + double t = h * w; + return t * w; + } + // |x| > overflowthreshold or is NaN. + // Return Infinity of the appropriate sign or NaN. + return x * shuge; +} + +/* Tanh(x) + * Return the Hyperbolic Tangent of x + * + * Method : + * x -x + * e - e + * 0. tanh(x) is defined to be ----------- + * x -x + * e + e + * 1. reduce x to non-negative by tanh(-x) = -tanh(x). + * 2. 0 <= x < 2**-28 : tanh(x) := x with inexact if x != 0 + * -t + * 2**-28 <= x < 1 : tanh(x) := -----; t = expm1(-2x) + * t + 2 + * 2 + * 1 <= x < 22 : tanh(x) := 1 - -----; t = expm1(2x) + * t + 2 + * 22 <= x <= INF : tanh(x) := 1. + * + * Special cases: + * tanh(NaN) is NaN; + * only tanh(0)=0 is exact for finite argument. + */ +double scr_ieee754_tanh(double x) { + static const volatile double tiny = 1.0e-300; + static const double one = 1.0, two = 2.0, huge = 1.0e300; + double t, z; + int32_t jx, ix; + + GET_HIGH_WORD(jx, x); + ix = jx & 0x7FFFFFFF; + + /* x is INF or NaN */ + if (ix >= 0x7FF00000) { + if (jx >= 0) + return one / x + one; /* tanh(+-inf)=+-1 */ + else + return one / x - one; /* tanh(NaN) = NaN */ + } + + /* |x| < 22 */ + if (ix < 0x40360000) { /* |x|<22 */ + if (ix < 0x3E300000) { /* |x|<2**-28 */ + if (huge + x > one) return x; /* tanh(tiny) = tiny with inexact */ + } + if (ix >= 0x3FF00000) { /* |x|>=1 */ + t = scr_ieee754_expm1(two * __builtin_fabs(x)); + z = one - two / (t + two); + } else { + t = scr_ieee754_expm1(-two * __builtin_fabs(x)); + z = -t / (t + two); + } + /* |x| >= 22, return +-1 */ + } else { + z = one - tiny; /* raise inexact flag */ + } + return (jx >= 0) ? z : -z; +} + +#undef EXTRACT_WORDS +#undef GET_HIGH_WORD +#undef GET_LOW_WORD +#undef INSERT_WORDS +#undef SET_HIGH_WORD +#undef SET_LOW_WORD + diff --git a/packages/runtime/src/scr_ieee754.h b/packages/runtime/src/scr_ieee754.h new file mode 100644 index 000000000..7dddf9889 --- /dev/null +++ b/packages/runtime/src/scr_ieee754.h @@ -0,0 +1,26 @@ +#ifndef SCR_IEEE754_H +#define SCR_IEEE754_H + +/* Math's transcendental functions as V8 computes them (scr_ieee754.c). */ +double scr_ieee754_acos(double x); +double scr_ieee754_acosh(double x); +double scr_ieee754_asin(double x); +double scr_ieee754_asinh(double x); +double scr_ieee754_atan(double x); +double scr_ieee754_atan2(double y, double x); +double scr_ieee754_atanh(double x); +double scr_ieee754_cbrt(double x); +double scr_ieee754_cos(double x); +double scr_ieee754_cosh(double x); +double scr_ieee754_exp(double x); +double scr_ieee754_expm1(double x); +double scr_ieee754_log(double x); +double scr_ieee754_log10(double x); +double scr_ieee754_log1p(double x); +double scr_ieee754_log2(double x); +double scr_ieee754_sin(double x); +double scr_ieee754_sinh(double x); +double scr_ieee754_tan(double x); +double scr_ieee754_tanh(double x); + +#endif diff --git a/packages/runtime/src/scr_js_value.c b/packages/runtime/src/scr_js_value.c index 0313fb0cd..1e2021de1 100644 --- a/packages/runtime/src/scr_js_value.c +++ b/packages/runtime/src/scr_js_value.c @@ -209,7 +209,7 @@ static double scr_js_number_arithmetic(double a, double b, const ScrStr *operati case '/': return a / b; case '%': return fmod(a, b); default: - if (operation->len == 2) return (fabs(a) == 1 && !isfinite(b)) ? NAN : pow(a, b); + if (operation->len == 2) return scr_math_pow(a, b); return a * b; } } diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index e19ea65cc..4c8cce6b9 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -2564,7 +2564,7 @@ static void scr_dyn_record_view_freeze(ScrDyn *view) { void scr_record_freeze(void *record) { size_t *rc = record; - if (*rc != SIZE_MAX) *rc |= SCR_RC_FROZEN; + if (!scr_rc_immortal(record)) *rc |= SCR_RC_FROZEN; } void scr_record_throw_frozen(ScrStr *key, ScrStr *owner, double operation) { @@ -6908,7 +6908,7 @@ ScrDyn *scr_dyn_class_base_prototype(ScrDyn *constructor) { default: break; } if (constructor->kind != SCR_DYN_FUNC || constructor->v.fn.class_obj || - scr_closure_identity(constructor->v.fn.clo)->function_kind != 4) { + (scr_closure_identity(constructor->v.fn.clo)->function_kind & SCR_FN_KIND_MASK) != 4) { static const char message[] = "Class extends value is not a native ordinary constructor"; scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); return NULL; @@ -7056,7 +7056,8 @@ void scr_dyn_name_class_prototype(ScrDyn *prototype, const ScrStr *name) { /* Ordinary native function constructors retain their live prototype table. * A constructor may replace the fresh receiver with another object. */ ScrDyn *scr_dyn_construct(const ScrDyn *callee, const ScrDyn *args, const ScrStr *what) { - if (callee->kind != SCR_DYN_FUNC || callee->v.fn.class_obj || scr_closure_identity(callee->v.fn.clo)->function_kind != 4) + if (callee->kind != SCR_DYN_FUNC || callee->v.fn.class_obj || + (scr_closure_identity(callee->v.fn.clo)->function_kind & SCR_FN_KIND_MASK) != 4) return scr_bytes_construct(callee, args, what); ScrDyn *prototype = scr_dyn_fn_get(callee, "prototype", 9); if (!prototype) return NULL; @@ -9191,7 +9192,9 @@ bool scr_dyn_is_callable(const ScrDyn *value) { int scr_dyn_function_kind(const ScrDyn *value) { while (value->kind == SCR_DYN_PROXY) value = value->v.proxy.target; - return value->kind == SCR_DYN_FUNC ? scr_closure_identity(value->v.fn.clo)->function_kind : 0; + return value->kind == SCR_DYN_FUNC + ? (int)(scr_closure_identity(value->v.fn.clo)->function_kind & SCR_FN_KIND_MASK) + : 0; } void scr_dyn_adopt_identity(ScrClosure *adapter, const ScrDyn *value) { @@ -11157,9 +11160,8 @@ ScrDyn *scr_dyn_arithmetic(const ScrDyn *left, const ScrDyn *right, const ScrStr if (result) { out = scr_dyn_new_bigint(result); scr_bigint_release(result); } } else { double a = l->v.num, b = r->v.num; - // ECMAScript differs from libm for ±1 ** ±Infinity and 1 ** NaN. double result = op == '-' ? a - b : op == '/' ? a / b : op == '%' ? fmod(a, b) - : operation->len == 2 ? ((fabs(a) == 1 && !isfinite(b)) ? NAN : pow(a, b)) : a * b; + : operation->len == 2 ? scr_math_pow(a, b) : a * b; out = scr_dyn_new_num(result); } scr_dyn_release(l); diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 1a625cdf1..928400da3 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -6344,6 +6344,56 @@ ScrArr *scr_fs_readdir(ScrStr *path) { return arr; } +/* fs.readdirSync(path, { recursive: true }): Node's readdirSyncRecursive. + * Directories are listed in queue order: the base, then each directory in + * the order its entry was listed. Each entry's path is path.join of its + * directory and name; the result holds it relative to the base + * (path.relative). An entry stat(2) reports as a directory, a symlink to + * one included, joins the queue. */ +ScrArr *scr_fs_readdir_recursive(ScrStr *base) { + if (!scr_fs_path_ok(base, "path")) return NULL; + ScrArr *out = scr_arr_new(SCR_ELEM_STR, 0); + size_t len = 1, cap = 8; + ScrStr **queue = malloc(cap * sizeof *queue); + if (!queue) scr_trap("scriptc: out of memory\n"); + queue[0] = scr_str_retain(base); + bool failed = false; + for (size_t at = 0; at < len && !failed; at++) { + ScrScandir *scan = scr_fs_scandir(queue[at]); + if (!scan) { + failed = true; + break; + } + for (size_t i = 0; i < scr_fs_scandir_count(scan); i++) { + ScrArr *parts = scr_arr_new(SCR_ELEM_STR, 2); + scr_arr_push_ref(parts, scr_str_retain(queue[at])); + scr_arr_push_ref(parts, scr_fs_scandir_name(scan, i)); + ScrStr *full = scr_path_join(parts); + scr_arr_release(parts); + scr_arr_push_ref(out, scr_path_relative(base, full)); + struct stat st; + if (stat(full->data, &st) == 0 && S_ISDIR(st.st_mode)) { + if (len == cap) { + cap *= 2; + queue = realloc(queue, cap * sizeof *queue); + if (!queue) scr_trap("scriptc: out of memory\n"); + } + queue[len++] = full; + } else { + scr_str_release(full); + } + } + scr_fs_scandir_free(scan); + } + for (size_t i = 0; i < len; i++) scr_str_release(queue[i]); + free(queue); + if (failed) { + scr_arr_release(out); + return NULL; + } + return out; +} + /* ── the withFileTypes scandir snapshot ────────────────────────────── * One readdir pass capturing name + entry kind for the emitter's Dirent * assembly (scr_runtime.h has the contract). Kinds are libuv's UV_DIRENT @@ -6828,11 +6878,17 @@ double scr_math_round(double x) { return (r == 0.0 && x < 0.0) ? -0.0 : r; } -/* IEEE pow(±1, ±Infinity) and pow(1, NaN) return 1; ECMAScript requires - * NaN. The other special cases, including signed zero and odd integer - * powers, follow libm. */ +/* V8's math::pow: the platform pow, as Node's Math.pow and ** call it, + * after the cases V8 answers itself. IEEE pow(±1, ±Infinity) and + * pow(1, NaN) return 1; ECMAScript requires NaN. A NaN base becomes the + * quiet NaN, and the exponents 2 and 0.5 answer base * base and + * sqrt(base + 0) (+Infinity for an infinite base), as V8's optimizing + * compilers do; pow can differ from those in the last bit. */ double scr_math_pow(double base, double exponent) { if (isnan(exponent) || (isinf(exponent) && fabs(base) == 1.0)) return NAN; + if (isnan(base)) base = NAN; + if (exponent == 2.0) return base * base; + if (exponent == 0.5) return isinf(base) ? INFINITY : sqrt(base + 0.0); /* 2 ** n for an integer n in the double range is an exact power of two, * the value pow returns; its bits are the exponent alone. */ if (base == 2.0 && exponent >= -1074.0 && exponent <= 1023.0) { @@ -12815,8 +12871,15 @@ ScrStr *scr_num_to_string_radix_f64(double value, double requested) { bool negative = value < 0; value = fabs(value); double integer = floor(value), fraction = value - integer; - double tolerance = 0.5 * (nextafter(value, INFINITY) - value); - if (tolerance <= 0) tolerance = nextafter(0, INFINITY); + /* The next double above the positive finite value: its bits plus one + * (Infinity above the largest double). */ + uint64_t value_bits; + memcpy(&value_bits, &value, sizeof value_bits); + value_bits += 1; + double next; + memcpy(&next, &value_bits, sizeof next); + double tolerance = 0.5 * (next - value); + if (tolerance <= 0) tolerance = 0x1p-1074; if (fraction >= tolerance) { buffer[end++] = '.'; do { diff --git a/packages/runtime/src/scr_lib.h b/packages/runtime/src/scr_lib.h index 7ba9e2907..e01bc7b23 100644 --- a/packages/runtime/src/scr_lib.h +++ b/packages/runtime/src/scr_lib.h @@ -293,6 +293,7 @@ void scr_fs_mkdir(ScrStr *path); void scr_fs_rm(ScrStr *path); void scr_fs_rmdir(ScrStr *path); ScrArr *scr_fs_readdir(ScrStr *path); /* names (byte order on POSIX, as libuv sorts), no "."/".." */ +ScrArr *scr_fs_readdir_recursive(ScrStr *path); /* relative paths of every entry below path, Node's queue order */ /* The fs option forms (same throw discipline). mkdir_recursive is Node's * recursive algorithm (existing-dir target is a no-op; errors report diff --git a/packages/runtime/src/scr_path.h b/packages/runtime/src/scr_path.h index 9600b5e95..de5e85491 100644 --- a/packages/runtime/src/scr_path.h +++ b/packages/runtime/src/scr_path.h @@ -1,5 +1,6 @@ -/* scriptc runtime — node:path (scr_path.c) and path globs (scr_glob.c). - * Included by scr_runtime.h, which says where new declarations go. */ +/* scriptc runtime — node:path (scr_path.c), path globs (scr_glob.c), and + * module resolution at run time (scr_require.c). Included by scr_runtime.h, + * which says where new declarations go. */ #ifndef SCR_PATH_H #define SCR_PATH_H @@ -89,4 +90,18 @@ void scr_fs_glob_callback(ScrDyn *args); ScrArr *scr_fsp_glob_strings(const ScrDyn *pattern, const ScrDyn *options); ScrDyn *scr_fsp_glob_error(void); +/* createRequire's require(id) and require.resolve(request, options) with a + * specifier computed at run time (scr_require.c). Both resolve with Node + * 24's CommonJS algorithm from `importer`, the requiring file; `builtins` + * lists the builtin ids (process.getBuiltinModule's table), and `site` is + * the call's "file:line" for the SC2020 refusal. All arguments are + * borrowed. scr_require_computed always leaves an exception pending and + * returns NULL: Node's argument errors, or MODULE_NOT_FOUND when nothing + * resolves; a specifier that resolves ends the program with the refusal. + * scr_require_resolve answers the resolved id or real path (+1), or NULL + * with Node's error pending. */ +ScrDyn *scr_require_computed(ScrDyn *id, ScrStr *importer, ScrStr *site, ScrArr *builtins); +ScrStr *scr_require_resolve(ScrDyn *request, ScrDyn *options, ScrStr *importer, ScrStr *site, + ScrArr *builtins); + #endif /* SCR_PATH_H */ diff --git a/packages/runtime/src/scr_publish.c b/packages/runtime/src/scr_publish.c index afbe4d6af..8d11d3806 100644 --- a/packages/runtime/src/scr_publish.c +++ b/packages/runtime/src/scr_publish.c @@ -128,8 +128,10 @@ bool scr_pub_mark(ScrPub *ctx, void *obj, bool cyc, int kind) { size_t *rc = obj; if (SCR_RC_PUBLISHED(obj) || ctx->refusal) return false; bool compact = (kind & SCR_PUB_COMPACT) != 0; - /* Both log the low 32 bits: a count, or a compact header word. */ - bool class_id = compact || (kind & SCR_PUB_CLASS_ID) != 0; + /* Both log the low 32 bits: a count, or a compact header word. An + * object with a layout id in its rc word (scr_rc_layout_id) keeps it, as + * a class instance keeps its class id. */ + bool class_id = compact || (kind & SCR_PUB_CLASS_ID) != 0 || scr_rc_layout_id(obj) != 0; kind &= ~(SCR_PUB_CLASS_ID | SCR_PUB_COMPACT); if (ctx->logged == ctx->log_cap) ctx->log = scr_pub_grow(ctx->log, &ctx->log_cap, sizeof *ctx->log, ctx->log_inline); diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 171e420c2..5a705054a 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -8,13 +8,27 @@ /* ── reference counts ───────────────────────────────────────────────── * Every heap object's first member is its `size_t rc` word. On 64-bit - * targets the reference count is the word's low 32 bits: an emitted - * instance of a class in a program-rooted hierarchy keeps its class id (its - * class's preorder number + 1) in the high 32 bits (llvm/classes.ts - * classIdRcWord), and no other object's count reaches 2^32. A count of SCR_RC_IMMORTAL marks - * an immortal object; for every object but those class instances the whole - * word is then SIZE_MAX. Code that reads or writes the count of an object of - * any kind goes through SCR_RC_COUNT. */ + * targets the word is + * + * bits 0-31 the reference count (SCR_RC_COUNT) + * bit 62 F: a frozen record (SCR_RC_FROZEN) + * bit 63 P: a program-lifetime record (llvm/classes.ts + * programLifetimeRc) + * bits 32-61 the layout id (scr_rc_layout_id) + * + * and no object's count reaches 2^32. The layout id names the object's + * layout in the single object model (scr_layout.h); 0 means the object + * never needed one. An emitted instance of a class in a program-rooted + * hierarchy keeps its class id (its class's preorder number + 1) there + * (llvm/classes.ts classIdRcWord): a class id is the layout id of its + * class's canonical layout. A count of SCR_RC_IMMORTAL marks an immortal + * object. Publication and the runtime's own immortal objects write the + * whole word as SIZE_MAX when the object has no layout id, and write only + * the count when it has one, so an immortal object keeps its layout id + * (scr_rc_immortal tests the count; a high half of all ones reads as no + * layout id). Code that reads or writes the count of an object of any + * kind goes through SCR_RC_COUNT; code that may see an object with a layout + * id tests immortality with scr_rc_immortal, never against SIZE_MAX. */ #if SIZE_MAX > UINT32_MAX typedef uint32_t ScrRcCount; #ifdef __BYTE_ORDER__ @@ -58,25 +72,47 @@ typedef size_t ScrRcCount; #define SCR_RC_FROZEN ((size_t)1 << 30) #define SCR_RC_REFS(obj) (SCR_RC_COUNT(obj) == SCR_RC_IMMORTAL ? SCR_RC_IMMORTAL : SCR_RC_COUNT(obj) & ~SCR_RC_FROZEN) #endif -static inline bool scr_rc_frozen(const void *obj) { - size_t word = *(const size_t *)obj; - return word != SIZE_MAX && (word & SCR_RC_FROZEN) != 0; +/* Whether `obj`, an object whose rc word is a count (not a compact class + * instance), is immortal: its count is SCR_RC_IMMORTAL, whatever its high + * half holds. */ +static inline bool scr_rc_immortal(const void *obj) { + return SCR_RC_COUNT(obj) == SCR_RC_IMMORTAL; } -/* The class id in the high half of `obj`'s rc word, or 0 for an object - * without one: the high half of any other object's word is 0, 0xffffffff - * (immortal), 0x80000000 (program-lifetime) or 0x40000000 (a frozen record), - * and a class id is never one of these. */ -static inline uint32_t scr_rc_class_id(const void *obj) { +static inline bool scr_rc_frozen(const void *obj) { + return !scr_rc_immortal(obj) && (*(const size_t *)obj & SCR_RC_FROZEN) != 0; +} + +/* The high half of the rc word on 64-bit targets: the program-lifetime + * bit, the frozen bit and the layout id (see the layout above). 32-bit + * targets have no high half; their objects keep a layout id elsewhere when + * they need one (scr_layout.h). */ +#define SCR_RC_HIGH_LIFETIME UINT32_C(0x80000000) +#define SCR_RC_HIGH_FROZEN UINT32_C(0x40000000) +#define SCR_RC_LAYOUT_ID_MASK UINT32_C(0x3fffffff) +/* Layout ids are below this bound: 2^30 - 1 is the id bits of an all-ones + * high half, which reads as no id. */ +#define SCR_RC_LAYOUT_ID_LIMIT SCR_RC_LAYOUT_ID_MASK + +/* The layout id in the high half of `obj`'s rc word, or 0 for an object + * without one. The whole-word immortal encoding (SIZE_MAX) has no id. */ +static inline uint32_t scr_rc_layout_id(const void *obj) { #if SIZE_MAX > UINT32_MAX uint32_t high = (uint32_t)(*(const size_t *)obj >> 32); - return high == UINT32_MAX || high == UINT32_C(0x80000000) || high == UINT32_C(0x40000000) ? 0 : high; + return high == UINT32_MAX ? 0 : high & SCR_RC_LAYOUT_ID_MASK; #else (void)obj; return 0; #endif } +/* The class id in the high half of `obj`'s rc word, or 0 for an object + * without one. A class id is the layout id of the class's canonical + * layout, and no object of any other kind has a layout id yet. */ +static inline uint32_t scr_rc_class_id(const void *obj) { + return scr_rc_layout_id(obj); +} + /* ── cycle collection (scr_cycle.c) ─────────────────────────────────── * Reference counting alone cannot free cycles, so every object that can * participate in one — capture boxes, heap closures, unions, promises, and @@ -696,6 +732,10 @@ static inline void scr_cyc_dead(void *obj) { * live heap — do not put it on a per-turn path. */ void scr_collect_cycles(void); void scr_collect_cycles_phase(void); +/* Returns the heap pages that hold only freed memory to the system: the + * calling thread's size-class pages whose blocks are all free, and the + * system allocator's free pages (scr_alloc.c). */ +void scr_heap_trim(void); /* Sets the growth cap SCR_CYCLE_GROWTH_CAP sets (scr_cycle.c), for a program * that knows its own memory profile; the environment variable takes * precedence. Call it before the program allocates much. */ diff --git a/packages/runtime/src/scr_rc.test.c b/packages/runtime/src/scr_rc.test.c new file mode 100644 index 000000000..159235ce9 --- /dev/null +++ b/packages/runtime/src/scr_rc.test.c @@ -0,0 +1,47 @@ +#undef NDEBUG +#include "scr_rc.h" +#include +#include + +/* The rc word's encodings (scr_rc.h): the count in the low half, and the + * program-lifetime bit, the frozen bit and the layout id in the high half. + * Immortality is a count of all ones; a layout id survives it. */ +static size_t word(uint32_t high, uint32_t count) { + return ((size_t)high << 32) | count; +} + +int main(void) { +#if SIZE_MAX > UINT32_MAX + /* An ordinary object: no layout id, not frozen, not immortal. */ + size_t plain = word(0, 3); + assert(scr_rc_layout_id(&plain) == 0 && !scr_rc_frozen(&plain) && !scr_rc_immortal(&plain)); + /* The whole-word immortal encoding has no layout id and is not frozen. */ + size_t immortal = SIZE_MAX; + assert(scr_rc_layout_id(&immortal) == 0 && scr_rc_immortal(&immortal) && !scr_rc_frozen(&immortal)); + /* A program-lifetime record (llvm/classes.ts programLifetimeRc) and a + * frozen record have no layout id. */ + size_t lifetime = word(SCR_RC_HIGH_LIFETIME, 0); + assert(scr_rc_layout_id(&lifetime) == 0 && !scr_rc_immortal(&lifetime)); + size_t frozen = word(SCR_RC_HIGH_FROZEN, 2); + assert(scr_rc_layout_id(&frozen) == 0 && scr_rc_frozen(&frozen)); + /* A class instance keeps its class id, which is its layout id, beside + * its count, also once immortal, frozen or program-lifetime. */ + size_t instance = word(7, 1); + assert(scr_rc_layout_id(&instance) == 7 && scr_rc_class_id(&instance) == 7); + SCR_RC_COUNT(&instance) = SCR_RC_IMMORTAL; + assert(scr_rc_immortal(&instance) && scr_rc_layout_id(&instance) == 7 && !scr_rc_frozen(&instance)); + size_t frozen_id = word(SCR_RC_HIGH_FROZEN | 9, 1); + assert(scr_rc_layout_id(&frozen_id) == 9 && scr_rc_frozen(&frozen_id)); + size_t lifetime_id = word(SCR_RC_HIGH_LIFETIME | 11, UINT32_C(0x80000000)); + assert(scr_rc_layout_id(&lifetime_id) == 11 && !scr_rc_immortal(&lifetime_id)); + /* The largest layout id. */ + size_t top = word(SCR_RC_LAYOUT_ID_LIMIT - 1, 1); + assert(scr_rc_layout_id(&top) == SCR_RC_LAYOUT_ID_LIMIT - 1); +#else + size_t plain = 3, immortal = SIZE_MAX; + assert(scr_rc_layout_id(&plain) == 0 && !scr_rc_immortal(&plain)); + assert(scr_rc_immortal(&immortal) && !scr_rc_frozen(&immortal)); +#endif + puts("rc word encodings"); + return 0; +} diff --git a/packages/runtime/src/scr_rc.test.ts b/packages/runtime/src/scr_rc.test.ts new file mode 100644 index 000000000..181d0beba --- /dev/null +++ b/packages/runtime/src/scr_rc.test.ts @@ -0,0 +1,36 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; + +const exec = promisify(execFile); + +/* The rc word helpers are inline in scr_rc.h, so the test links alone. */ +test.skipIf(process.platform === "win32")( + "the rc word keeps the count, the lifetime and frozen bits and the layout id apart", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-rc-")); + try { + const bin = join(dir, "rc"); + await exec("clang", [ + "-std=c11", + "-O1", + "-g", + "-fsanitize=address,undefined", + join(import.meta.dirname, "scr_rc.test.c"), + "-o", + bin, + ]); + const result = await exec(bin, [], { + timeout: 60000, + env: { ...process.env, UBSAN_OPTIONS: "halt_on_error=1" }, + }); + expect(result.stdout).toBe("rc word encodings\n"); + expect(result.stderr).toBe(""); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, +); diff --git a/packages/runtime/src/scr_regex.c b/packages/runtime/src/scr_regex.c index 37f94bd65..4296ce791 100644 --- a/packages/runtime/src/scr_regex.c +++ b/packages/runtime/src/scr_regex.c @@ -2469,6 +2469,146 @@ static int scr_unicode_normalize(uint32_t **pdst, const uint32_t *src, int len, return (int)count; } +/* Normalization quick check (UAX #15): a string whose every code point is + * inert for the form is already in that form. A code point is inert when + * normalizing it alone leaves it unchanged, its canonical combining class + * is 0, and (for the composing forms) no composition can take it as its + * second element. The last condition holds in the ranges below: none holds + * a code point with combining class 0 that composes with a preceding one + * (those are Hangul jamo vowels and trailing consonants, U+3099/U+309A, + * and vowel signs of Indic and Southeast Asian scripts above U+0900), so a + * code point outside them takes the full algorithm. Inert flags are + * computed through unicode_normalize, 256 code points at a time, when a + * block is first seen. */ +#define SCR_NQC_BLOCKS 0x200 +typedef struct { + uint8_t known[4][SCR_NQC_BLOCKS]; + uint8_t inert[4][SCR_NQC_BLOCKS][32]; + uint8_t starter[4][SCR_NQC_BLOCKS][32]; /* class 0, never a second element */ + struct { + uint32_t cp; + uint8_t len; + char bytes[15]; + } single[4][256]; /* a lone code point's normalized UTF-8 */ +} ScrNormalizeQuick; +static SCR_TL ScrNormalizeQuick *scr_nqc; + +static void scr_nqc_free(void) { + free(scr_nqc); + scr_nqc = NULL; +} + +static bool scr_nqc_in_ranges(uint32_t c) { + return c < 0x900 || (c >= 0x2000 && c < 0x3099) || (c >= 0x309B && c < 0xA000) || + (c >= 0xAC00 && c < 0xD7A4) || (c >= 0xF900 && c < 0x10000) || + (c >= 0x1F000 && c < 0x1FB00); +} + +/* Bit 0: inert. Bit 1: a starter, with class 0 and never the second element + * of a composition, so nothing before it reorders or composes with it. */ +/* Whether c has class 0 and lies in the ranges above. U+0345 is the one + * code point of class 240, which the reordering test cannot tell from + * class 0; a mark of class 1..239 after U+0345 moves before it. */ +static bool scr_nqc_class0(uint32_t c) { + if (c == 0x345 || !scr_nqc_in_ranges(c)) return false; + uint32_t pair[2] = {0x345, c}; + uint32_t *out = NULL; + int n = unicode_normalize(&out, pair, 2, UNICODE_NFD, NULL, scr_normalize_realloc); + bool starter = n >= 2 && out[0] == 0x345; + free(out); + return starter; +} + +static unsigned scr_nqc_compute(uint32_t c, UnicodeNormalizationEnum mode) { + if (!scr_nqc_class0(c)) return 0; + uint32_t *out = NULL; + int n = unicode_normalize(&out, &c, 1, mode, NULL, scr_normalize_realloc); + bool same = n == 1 && out[0] == c; + free(out); + if (same) return 3u; + /* A starter's decomposition must begin with one too (U+FF9E's + * compatibility decomposition is U+3099, which composes with the code + * point before it). */ + UnicodeNormalizationEnum decompose = + mode == UNICODE_NFKC || mode == UNICODE_NFKD ? UNICODE_NFKD : UNICODE_NFD; + out = NULL; + n = unicode_normalize(&out, &c, 1, decompose, NULL, scr_normalize_realloc); + bool leads = n >= 1 && scr_nqc_class0(out[0]); + free(out); + return leads ? 2u : 0u; +} + +/* scr_nqc_compute's bits for c, from its block's flags. */ +static unsigned scr_nqc_class(uint32_t c, UnicodeNormalizationEnum mode) { + if (c >= (uint32_t)SCR_NQC_BLOCKS << 8) return 0; + if (!scr_nqc) { + scr_nqc = calloc(1, sizeof *scr_nqc); + if (!scr_nqc) return 0; + scr_atexit(scr_nqc_free); + } + uint32_t block = c >> 8; + uint8_t *inert = scr_nqc->inert[mode][block]; + uint8_t *starter = scr_nqc->starter[mode][block]; + if (!scr_nqc->known[mode][block]) { + for (uint32_t i = 0; i < 256; i++) { + unsigned bits = scr_nqc_compute((block << 8) | i, mode); + if (bits & 1) inert[i >> 3] |= (uint8_t)(1u << (i & 7)); + if (bits & 2) starter[i >> 3] |= (uint8_t)(1u << (i & 7)); + } + scr_nqc->known[mode][block] = 1; + } + unsigned bit = c & 0xff; + return ((inert[bit >> 3] >> (bit & 7)) & 1) | (((starter[bit >> 3] >> (bit & 7)) & 1) << 1); +} + +/* Normalize the code points of data[from, to) and append them. */ +static void scr_nqc_segment(ScrJsonBuf *b, const char *data, size_t from, size_t to, + UnicodeNormalizationEnum mode) { + size_t first_step; + uint32_t first = scr_case_decode(data + from, &first_step); + if (first_step == to - from && scr_nqc) { + /* One code point (a precomposed letter between inert ones): its + * normalized bytes repeat, so they are kept per form. */ + __typeof__(scr_nqc->single[0][0]) *slot = &scr_nqc->single[mode][first & 0xff]; + if (slot->cp != first || slot->len == 0) { + uint32_t *output = NULL; + int count = unicode_normalize(&output, &first, 1, mode, NULL, scr_normalize_realloc); + if (count < 0) scr_regex_oom(); + ScrJsonBuf one; + scr_jb_init(&one); + for (int i = 0; i < count; i++) scr_case_put_cp(&one, output[i]); + free(output); + ScrStr *bytes = scr_jb_finish(&one); + if (bytes->len == 0 || bytes->len > sizeof slot->bytes) { + scr_jb_write(b, bytes->data, bytes->len); + scr_str_release(bytes); + return; + } + slot->cp = first; + slot->len = (uint8_t)bytes->len; + memcpy(slot->bytes, bytes->data, bytes->len); + scr_str_release(bytes); + } + scr_jb_write(b, slot->bytes, slot->len); + return; + } + uint32_t local[32]; + uint32_t *input = to - from <= 32 ? local : malloc((to - from) * sizeof *input); + if (!input) scr_regex_oom(); + int length = 0; + for (size_t offset = from; offset < to;) { + size_t step; + input[length++] = scr_case_decode(data + offset, &step); + offset += step; + } + uint32_t *output = NULL; + int count = scr_unicode_normalize(&output, input, length, mode); + if (input != local) free(input); + if (count < 0) scr_regex_oom(); + for (int i = 0; i < count; i++) scr_case_put_cp(b, output[i]); + free(output); +} + ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { UnicodeNormalizationEnum mode; if (form->len == 3 && !memcmp(form->data, "NFC", 3)) mode = UNICODE_NFC; @@ -2483,23 +2623,58 @@ ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { /* Every ASCII string is already normalized in all four forms. Validate * the form first: even an empty subject must reject an invalid form. */ if (scr_text_ascii(s)) return scr_str_retain((ScrStr *)s); - if (s->len > INT32_MAX / sizeof(uint32_t)) scr_regex_oom(); - uint32_t *input = malloc((s->len + 1) * sizeof(uint32_t)); - if (!input) scr_regex_oom(); - int length = 0; + bool inert = true; for (size_t offset = 0; offset < s->len;) { + if ((unsigned char)s->data[offset] < 0x80) { + offset++; + continue; + } size_t step; - input[length++] = scr_case_decode(s->data + offset, &step); + uint32_t c = scr_case_decode(s->data + offset, &step); offset += step; + if (!(scr_nqc_class(c, mode) & 1)) { + inert = false; + break; + } } - uint32_t *output = NULL; - int count = scr_unicode_normalize(&output, input, length, mode); - free(input); - if (count < 0) scr_regex_oom(); + if (inert) return scr_str_retain((ScrStr *)s); + if (s->len > INT32_MAX / sizeof(uint32_t)) scr_regex_oom(); + /* An inert code point bounds reordering and composition, and so does any + * other starter (class 0, never a composition's second element), so the + * text normalizes segment by segment. A segment starts at a starter: runs + * of inert code points copy as they are, and a segment with any other + * code point goes through scr_unicode_normalize. */ ScrJsonBuf buffer; scr_jb_init(&buffer); - for (int i = 0; i < count; i++) scr_case_put_cp(&buffer, output[i]); - free(output); + size_t seg = 0, last_inert = 0; + bool plain = true; + for (size_t offset = 0; offset < s->len;) { + unsigned char lead = (unsigned char)s->data[offset]; + size_t step = 1; + unsigned kind = 3; + if (lead >= 0x80) kind = scr_nqc_class(scr_case_decode(s->data + offset, &step), mode); + if (kind & 1) { + if (!plain) { + scr_nqc_segment(&buffer, s->data, seg, offset, mode); + seg = offset; + plain = true; + } + last_inert = offset; + } else if (kind & 2) { + if (plain) scr_jb_write(&buffer, s->data + seg, offset - seg); + else scr_nqc_segment(&buffer, s->data, seg, offset, mode); + seg = offset; + plain = false; + } else if (plain) { + size_t start = last_inert > seg ? last_inert : seg; + scr_jb_write(&buffer, s->data + seg, start - seg); + seg = start; + plain = false; + } + offset += step; + } + if (plain) scr_jb_write(&buffer, s->data + seg, s->len - seg); + else scr_nqc_segment(&buffer, s->data, seg, s->len, mode); return scr_jb_finish(&buffer); } diff --git a/packages/runtime/src/scr_require.c b/packages/runtime/src/scr_require.c new file mode 100644 index 000000000..d1ede2186 --- /dev/null +++ b/packages/runtime/src/scr_require.c @@ -0,0 +1,902 @@ +/* createRequire's require(id) and require.resolve(request) with a specifier + * computed at run time (module-resolution.ts lowers both calls here). + * + * The compiled module graph is fixed at build time, so neither call loads a + * module. Both resolve the specifier with Node 24's CommonJS algorithm + * (lib/internal/modules/cjs/loader.js: Module._resolveFilename and + * Module._findPath, with package.json "main", "exports" and self-reference) + * against the file system at run time: + * + * - require.resolve returns what Node returns: the builtin's id, or the real + * path of the resolved file. It throws Node's MODULE_NOT_FOUND error when + * nothing resolves. + * - require throws the same MODULE_NOT_FOUND error when nothing resolves, so + * packages that probe for optional files take their absent branch as in + * Node. When the specifier resolves (a builtin, or a file such as a native + * addon), Node would load the module, which a compiled program cannot do: + * the program ends with an SC2020 refusal naming the module. The refusal is + * not an exception, so a `catch` clause cannot take a fallback that Node + * would not take. + * + * Resolution steps whose Node result this file does not reproduce (package + * "imports", invalid package configs or export targets, unexported subpaths, + * percent-encoded export targets, an export target that names no file, a + * "main" that names no file) end the program with the same SC2020 refusal, + * naming the step. */ +#include "scr_runtime.h" + +#include +#include +#include + +typedef struct { + ScrStr *request; /* borrowed */ + ScrStr *importer; /* borrowed: the requiring file */ + ScrArr *builtins; /* borrowed: Node 24's require-able builtin ids */ + /* Set when resolution reaches a step it does not reproduce; the caller + * refuses with this reason. Static text. */ + const char *refusal; +} ScrRequire; + +static ScrStr *req_cstr(const char *s) { return scr_str_new(s, strlen(s)); } + +static bool req_has_prefix(const ScrStr *s, const char *prefix) { + size_t n = strlen(prefix); + return s->len >= n && memcmp(s->data, prefix, n) == 0; +} + +static bool req_has_suffix(const ScrStr *s, const char *suffix) { + size_t n = strlen(suffix); + return s->len >= n && memcmp(s->data + s->len - n, suffix, n) == 0; +} + +static bool req_eq(const ScrStr *s, const char *text) { + size_t n = strlen(text); + return s->len == n && memcmp(s->data, text, n) == 0; +} + +static ScrStr *req_concat(const ScrStr *a, const char *b, size_t b_len) { + ScrJsonBuf out; + scr_jb_init(&out); + scr_jb_write(&out, a->data, a->len); + scr_jb_write(&out, b, b_len); + return scr_jb_finish(&out); +} + +static ScrStr *req_resolve2(ScrStr *a, ScrStr *b) { + ScrArr *parts = scr_arr_new(SCR_ELEM_STR, 2); + scr_arr_push_ref(parts, scr_str_retain(a)); + scr_arr_push_ref(parts, scr_str_retain(b)); +#ifdef _WIN32 + ScrStr *out = scr_path_win32_resolve(parts); +#else + ScrStr *out = scr_path_resolve(parts); +#endif + scr_arr_release(parts); + return out; +} + +static ScrStr *req_dirname(ScrStr *path) { +#ifdef _WIN32 + return scr_path_win32_dirname(path); +#else + return scr_path_dirname(path); +#endif +} + +static bool req_is_absolute(ScrStr *path) { +#ifdef _WIN32 + return scr_path_win32_is_absolute(path); +#else + return scr_path_is_absolute(path); +#endif +} + +/* Node's internalModuleStat: 0 for a file, 1 for a directory, -1 when the + * path does not exist or cannot be read. Never throws. */ +static int req_stat(ScrStr *path) { + ScrStats *stats = scr_fs_stat_probe(path, false, false); + if (stats == NULL) { + if (scr_exc_pending()) scr_exc_clear(); + return -1; + } + int kind = scr_stats_is_dir(stats) ? 1 : 0; + scr_stats_release(stats); + return kind; +} + +/* tryFile: the real path of `path` when it is a file (+1), else NULL. */ +static ScrStr *req_try_file(ScrStr *path) { + if (req_stat(path) != 0) return NULL; + ScrStr *real = scr_fs_realpath(path); + if (real != NULL) return real; + if (scr_exc_pending()) scr_exc_clear(); + return scr_str_retain(path); +} + +/* tryExtensions with Module._extensions' keys, in registration order. */ +static ScrStr *req_try_extensions(ScrStr *base) { + static const char *const extensions[] = { ".js", ".json", ".node" }; + for (size_t i = 0; i < sizeof extensions / sizeof extensions[0]; i++) { + ScrStr *candidate = req_concat(base, extensions[i], strlen(extensions[i])); + ScrStr *found = req_try_file(candidate); + scr_str_release(candidate); + if (found != NULL) return found; + } + return NULL; +} + +/* ── package.json ────────────────────────────────────────────────────── */ + +/* The package.json at `pjson` parsed (+1), or NULL when there is none. + * A file Node rejects as an invalid package config (unparsable JSON, a + * non-object document, a non-string "name" or "type") sets the refusal. */ +static ScrDyn *req_read_package(ScrRequire *r, ScrStr *pjson) { + if (req_stat(pjson) != 0) return NULL; + ScrStr *text = scr_fs_read_file(pjson); + if (text == NULL) { + if (scr_exc_pending()) scr_exc_clear(); + return NULL; + } + /* Node's reader skips a leading UTF-8 byte order mark. */ + if (text->len >= 3 && (unsigned char)text->data[0] == 0xEF && + (unsigned char)text->data[1] == 0xBB && (unsigned char)text->data[2] == 0xBF) { + ScrStr *body = scr_str_new(text->data + 3, text->len - 3); + scr_str_release(text); + text = body; + } + ScrDyn *pkg = scr_json_parse(text); + scr_str_release(text); + if (pkg == NULL) { + if (scr_exc_pending()) scr_exc_clear(); + r->refusal = "a package.json that is not valid JSON"; + return NULL; + } + const ScrDyn *name = pkg->kind == SCR_DYN_OBJ ? scr_dyn_obj_get(pkg, "name", 4) : NULL; + const ScrDyn *type = pkg->kind == SCR_DYN_OBJ ? scr_dyn_obj_get(pkg, "type", 4) : NULL; + if (pkg->kind != SCR_DYN_OBJ || (name != NULL && name->kind != SCR_DYN_STR) || + (type != NULL && type->kind != SCR_DYN_STR)) { + scr_dyn_release(pkg); + r->refusal = "an invalid package.json"; + return NULL; + } + return pkg; +} + +/* A string field of a parsed package.json (borrowed), or NULL. */ +static ScrStr *req_package_string(const ScrDyn *pkg, const char *key) { + const ScrDyn *value = scr_dyn_obj_get(pkg, key, strlen(key)); + return value != NULL && value->kind == SCR_DYN_STR ? value->v.str : NULL; +} + +/* "exports" or "imports" as Node keeps them: a string, an object or an + * array (borrowed); NULL for every other value. */ +static const ScrDyn *req_package_map(const ScrDyn *pkg, const char *key) { + const ScrDyn *value = scr_dyn_obj_get(pkg, key, strlen(key)); + if (value == NULL) return NULL; + return value->kind == SCR_DYN_STR || value->kind == SCR_DYN_OBJ || value->kind == SCR_DYN_ARR ? value : NULL; +} + +/* tryPackage: a directory's "main" entry, then its index files. */ +static ScrStr *req_try_package(ScrRequire *r, ScrStr *dir) { + ScrStr *pjson_name = req_cstr("package.json"); + ScrStr *pjson = req_resolve2(dir, pjson_name); + scr_str_release(pjson_name); + ScrDyn *pkg = req_read_package(r, pjson); + scr_str_release(pjson); + if (r->refusal != NULL) return NULL; + ScrStr *main = pkg != NULL ? req_package_string(pkg, "main") : NULL; + ScrStr *index_name = req_cstr("index"); + ScrStr *index = req_resolve2(dir, index_name); + ScrStr *found = NULL; + if (main == NULL) { + found = req_try_extensions(index); + } else { + ScrStr *filename = req_resolve2(dir, main); + found = req_try_file(filename); + if (found == NULL) found = req_try_extensions(filename); + if (found == NULL) { + ScrStr *main_index = req_resolve2(filename, index_name); + found = req_try_extensions(main_index); + scr_str_release(main_index); + } + scr_str_release(filename); + if (found == NULL) { + /* Node falls back to the index with a DEP0128 warning, or throws an + * error that names the "main" entry. */ + r->refusal = "a package.json \"main\" entry that names no file"; + } + } + scr_str_release(index); + scr_str_release(index_name); + if (pkg != NULL) scr_dyn_release(pkg); + return found; +} + +/* ── package exports (lib/internal/modules/esm/resolve.js) ───────────── */ + +/* A target or subpath segment Node rejects or warns about: "", ".", "..", + * "node_modules" in any case, and their percent-encoded spellings. The + * caller refuses '%' separately, so the plain spellings suffice. */ +static bool req_invalid_segment(const char *s, size_t len) { + size_t start = 0; + for (size_t i = 0; i <= len; i++) { + if (i < len && s[i] != '/' && s[i] != '\\') continue; + size_t n = i - start; + const char *seg = s + start; + if (n == 0 && i < len) return true; + if ((n == 1 && seg[0] == '.') || (n == 2 && seg[0] == '.' && seg[1] == '.')) return true; + if (n == 12) { + static const char nm[] = "node_modules"; + bool same = true; + for (size_t k = 0; k < 12 && same; k++) { + char c = seg[k]; + if (c >= 'A' && c <= 'Z') c = (char)(c - 'A' + 'a'); + same = c == nm[k]; + } + if (same) return true; + } + start = i + 1; + } + return false; +} + +/* Characters whose URL parsing changes the path Node resolves: percent + * escapes, backslashes, query and fragment delimiters, and the control + * characters the URL parser strips. */ +static bool req_url_sensitive(const char *s, size_t len) { + for (size_t i = 0; i < len; i++) { + unsigned char c = (unsigned char)s[i]; + if (c == '%' || c == '\\' || c == '?' || c == '#' || c < 0x20 || c == 0x7f) return true; + } + return false; +} + +typedef enum { REQ_TARGET_PATH, REQ_TARGET_NULL, REQ_TARGET_UNDEFINED, REQ_TARGET_INVALID } ScrRequireTarget; + +static bool req_condition(const ScrStr *key) { + return req_eq(key, "default") || req_eq(key, "require") || req_eq(key, "node") || + req_eq(key, "node-addons") || req_eq(key, "module-sync"); +} + +static bool req_array_index(const ScrStr *key) { + if (key->len == 0 || key->len > 10) return false; + if (key->len > 1 && key->data[0] == '0') return false; + double n = 0; + for (size_t i = 0; i < key->len; i++) { + if (key->data[i] < '0' || key->data[i] > '9') return false; + n = n * 10 + (key->data[i] - '0'); + } + return n < 4294967295.0; +} + +/* resolvePackageTarget for an "exports" target. `pkg_dir` is the package + * directory, `subpath` the part the target receives. A path result is in + * `*out` (+1, not yet checked against the file system). */ +static ScrRequireTarget req_resolve_target(ScrRequire *r, ScrStr *pkg_dir, const ScrDyn *target, + const char *subpath, size_t subpath_len, bool pattern, + ScrStr **out) { + if (target->kind == SCR_DYN_STR) { + ScrStr *t = target->v.str; + if (subpath_len != 0 && !pattern && !req_has_suffix(t, "/")) return REQ_TARGET_INVALID; + if (!req_has_prefix(t, "./")) return REQ_TARGET_INVALID; + if (req_url_sensitive(t->data, t->len) || req_url_sensitive(subpath, subpath_len) || + req_invalid_segment(t->data + 2, t->len - 2) || + (subpath_len != 0 && req_invalid_segment(subpath, subpath_len))) { + r->refusal = "a package \"exports\" target that Node rejects, warns about or decodes"; + return REQ_TARGET_INVALID; + } + if (pattern && memchr(pkg_dir->data, '*', pkg_dir->len) != NULL) { + /* Node substitutes the pattern in the whole URL, directory included. */ + r->refusal = "a package \"exports\" pattern in a directory whose path contains '*'"; + return REQ_TARGET_INVALID; + } + ScrJsonBuf path; + scr_jb_init(&path); + scr_jb_write(&path, pkg_dir->data, pkg_dir->len); + const char *rest = t->data + 1; /* keeps the '/' after "." */ + size_t rest_len = t->len - 1; + if (pattern) { + for (size_t i = 0; i < rest_len; i++) { + if (rest[i] == '*') + scr_jb_write(&path, subpath, subpath_len); + else + scr_jb_putc(&path, rest[i]); + } + } else { + scr_jb_write(&path, rest, rest_len); + scr_jb_write(&path, subpath, subpath_len); + } + ScrStr *joined = scr_jb_finish(&path); +#ifdef _WIN32 + *out = scr_path_win32_normalize(joined); +#else + *out = scr_path_normalize(joined); +#endif + scr_str_release(joined); + return REQ_TARGET_PATH; + } + if (target->kind == SCR_DYN_ARR) { + double n = scr_dyn_arr_len(target); + if (n == 0) return REQ_TARGET_NULL; + ScrRequireTarget last = REQ_TARGET_UNDEFINED; + for (double i = 0; i < n; i++) { + ScrDyn *item = scr_dyn_arr_at(target, i); + ScrRequireTarget result = req_resolve_target(r, pkg_dir, item, subpath, subpath_len, pattern, out); + scr_dyn_release(item); + if (r->refusal != NULL) return REQ_TARGET_INVALID; + if (result == REQ_TARGET_PATH) return result; + if (result == REQ_TARGET_INVALID) last = REQ_TARGET_INVALID; + else if (result == REQ_TARGET_NULL) last = REQ_TARGET_NULL; + } + return last == REQ_TARGET_UNDEFINED ? REQ_TARGET_UNDEFINED : last; + } + if (target->kind == SCR_DYN_OBJ) { + ScrDyn *keys = scr_dyn_obj_own_keys(target); + double n = scr_dyn_arr_len(keys); + ScrRequireTarget result = REQ_TARGET_UNDEFINED; + for (double i = 0; i < n; i++) { + ScrDyn *key = scr_dyn_arr_at(keys, i); + bool index = req_array_index(key->v.str); + scr_dyn_release(key); + if (index) { + r->refusal = "numeric keys in a package's \"exports\""; + scr_dyn_release(keys); + return REQ_TARGET_INVALID; + } + } + for (double i = 0; i < n && result == REQ_TARGET_UNDEFINED; i++) { + ScrDyn *key = scr_dyn_arr_at(keys, i); + if (req_condition(key->v.str)) { + const ScrDyn *value = scr_dyn_obj_get(target, key->v.str->data, key->v.str->len); + result = req_resolve_target(r, pkg_dir, value, subpath, subpath_len, pattern, out); + if (r->refusal != NULL) result = REQ_TARGET_INVALID; + } + scr_dyn_release(key); + } + scr_dyn_release(keys); + return result; + } + if (target->kind == SCR_DYN_NULL) return REQ_TARGET_NULL; + return REQ_TARGET_INVALID; +} + +/* patternKeyCompare(a, b) === 1: `b` is the better pattern match. */ +static bool req_pattern_better(const ScrStr *a, const ScrStr *b) { + const char *as = a ? memchr(a->data, '*', a->len) : NULL; + const char *bs = memchr(b->data, '*', b->len); + size_t a_len = a ? a->len : 0; + size_t base_a = as ? (size_t)(as - a->data) + 1 : a_len; + size_t base_b = bs ? (size_t)(bs - b->data) + 1 : b->len; + if (base_a > base_b) return false; + if (base_b > base_a) return true; + if (as == NULL) return true; + if (bs == NULL) return false; + return b->len > a_len; +} + +/* packageExportsResolve for `subpath` ("." or "./..."), then + * finalizeEsmResolution's file check. +1 real path, or NULL with the + * refusal set. */ +static ScrStr *req_exports_resolve(ScrRequire *r, ScrStr *pkg_dir, const ScrDyn *exports, ScrStr *subpath) { + bool sugar; + if (exports->kind != SCR_DYN_OBJ) { + sugar = true; + } else { + sugar = false; + ScrDyn *keys = scr_dyn_obj_own_keys(exports); + double n = scr_dyn_arr_len(keys); + for (double i = 0; i < n; i++) { + ScrDyn *key = scr_dyn_arr_at(keys, i); + bool conditional = key->v.str->len == 0 || key->v.str->data[0] != '.'; + scr_dyn_release(key); + if (i == 0) { + sugar = conditional; + } else if (sugar != conditional) { + scr_dyn_release(keys); + r->refusal = "a package \"exports\" object that mixes subpaths and conditions"; + return NULL; + } + } + scr_dyn_release(keys); + } + if (exports->kind == SCR_DYN_STR && exports->v.str->len > 0 && + (exports->v.str->data[0] == '[' || exports->v.str->data[0] == '{')) { + r->refusal = "a package \"exports\" string that Node parses as JSON"; + return NULL; + } + ScrStr *path = NULL; + ScrRequireTarget result = REQ_TARGET_UNDEFINED; + bool has_star = memchr(subpath->data, '*', subpath->len) != NULL; + if (sugar) { + if (req_eq(subpath, ".")) + result = req_resolve_target(r, pkg_dir, exports, "", 0, false, &path); + } else { + const ScrDyn *exact = has_star || req_has_suffix(subpath, "/") + ? NULL + : scr_dyn_obj_get(exports, subpath->data, subpath->len); + if (exact != NULL) { + result = req_resolve_target(r, pkg_dir, exact, "", 0, false, &path); + } else { + ScrStr *best = NULL; + size_t best_start = 0, best_end = 0; + ScrDyn *keys = scr_dyn_obj_own_keys(exports); + double n = scr_dyn_arr_len(keys); + for (double i = 0; i < n; i++) { + ScrDyn *key_value = scr_dyn_arr_at(keys, i); + ScrStr *key = key_value->v.str; + const char *star = memchr(key->data, '*', key->len); + size_t at = star ? (size_t)(star - key->data) : 0; + if (star != NULL && subpath->len >= at && memcmp(subpath->data, key->data, at) == 0) { + size_t trailer = key->len - at - 1; + bool single = memchr(star + 1, '*', trailer) == NULL; + if (subpath->len >= key->len && + memcmp(subpath->data + subpath->len - trailer, star + 1, trailer) == 0 && + req_pattern_better(best, key) && single) { + if (best) scr_str_release(best); + best = scr_str_retain(key); + best_start = at; + best_end = subpath->len - trailer; + } + } + scr_dyn_release(key_value); + } + scr_dyn_release(keys); + if (best != NULL) { + if (req_has_suffix(subpath, "/")) { + r->refusal = "a package \"exports\" pattern matched by a subpath ending in '/'"; + } else { + const ScrDyn *target = scr_dyn_obj_get(exports, best->data, best->len); + result = req_resolve_target(r, pkg_dir, target, subpath->data + best_start, + best_end - best_start, true, &path); + } + scr_str_release(best); + } + } + } + if (r->refusal != NULL) { + if (path) scr_str_release(path); + return NULL; + } + if (result != REQ_TARGET_PATH) { + r->refusal = result == REQ_TARGET_INVALID ? "an invalid package \"exports\" target" + : "a subpath that the package's \"exports\" do not export"; + return NULL; + } + ScrStr *found = req_try_file(path); + scr_str_release(path); + if (found == NULL) r->refusal = "a package \"exports\" target that names no file"; + return found; +} + +/* ── Module._findPath ────────────────────────────────────────────────── */ + +/* EXPORTS_PATTERN: /^((?:@[^/\\%]+\/)?[^./\\%][^/\\%]*)(\/.*)?$/ — the + * package name's length in `request`, or 0 when it does not match. */ +static size_t req_package_name_len(const ScrStr *request) { + const char *s = request->data; + size_t len = request->len, i = 0; + if (len > 0 && s[0] == '@') { + i = 1; + size_t start = i; + while (i < len && s[i] != '/' && s[i] != '\\' && s[i] != '%') i++; + if (i == start || i >= len || s[i] != '/') i = 0; + else i++; + } + if (i >= len || s[i] == '.' || s[i] == '/' || s[i] == '\\' || s[i] == '%') return 0; + while (i < len && s[i] != '/' && s[i] != '\\' && s[i] != '%') i++; + if (i < len && s[i] != '/') return 0; + return i; +} + +/* resolveExports: a package under `nm_dir` whose package.json has + * "exports". +1 real path; NULL when the package has no "exports" (or the + * refusal is set). */ +static ScrStr *req_resolve_exports(ScrRequire *r, ScrStr *nm_dir) { + size_t name_len = req_package_name_len(r->request); + if (name_len == 0) return NULL; + ScrStr *name = scr_str_new(r->request->data, name_len); + ScrStr *pkg_dir = req_resolve2(nm_dir, name); + scr_str_release(name); + ScrStr *pjson = req_concat(pkg_dir, "/package.json", 13); + ScrDyn *pkg = req_read_package(r, pjson); + scr_str_release(pjson); + ScrStr *found = NULL; + const ScrDyn *exports = pkg != NULL ? req_package_map(pkg, "exports") : NULL; + if (exports != NULL) { + ScrJsonBuf sub; + scr_jb_init(&sub); + scr_jb_putc(&sub, '.'); + scr_jb_write(&sub, r->request->data + name_len, r->request->len - name_len); + ScrStr *subpath = scr_jb_finish(&sub); + found = req_exports_resolve(r, pkg_dir, exports, subpath); + scr_str_release(subpath); + } + if (pkg != NULL) scr_dyn_release(pkg); + scr_str_release(pkg_dir); + return found; +} + +/* isRelative in Module._findPath: ".", "..", and "./" or "../" prefixes. */ +static bool req_is_relative(const ScrStr *request) { + if (req_eq(request, ".") || req_eq(request, "..")) return true; + if (req_has_prefix(request, "./") || req_has_prefix(request, "../")) return true; +#ifdef _WIN32 + if (req_has_prefix(request, ".\\") || req_has_prefix(request, "..\\")) return true; +#endif + return false; +} + +static bool req_trailing_slash(const ScrStr *request) { + const char *s = request->data; + size_t n = request->len; + if (n == 0) return false; + if (s[n - 1] == '/') return true; + if (s[n - 1] != '.') return false; + if (n == 1 || s[n - 2] == '/') return true; + return s[n - 2] == '.' && (n == 2 || s[n - 3] == '/'); +} + +/* Module._findPath over `paths` (borrowed strings). +1 real path or NULL. */ +static ScrStr *req_find_path(ScrRequire *r, ScrStr *const *paths, size_t count) { + ScrStr *request = r->request; + bool absolute = req_is_absolute(request); + ScrStr *empty = NULL; + if (absolute) { + empty = scr_str_new("", 0); + paths = ∅ + count = 1; + } + bool trailing = req_trailing_slash(request); + bool inside = true; + if (req_is_relative(request)) { +#ifdef _WIN32 + ScrStr *normalized = scr_path_win32_normalize(request); +#else + ScrStr *normalized = scr_path_normalize(request); +#endif + inside = !req_has_prefix(normalized, ".."); + scr_str_release(normalized); + } + ScrStr *found = NULL; + for (size_t i = 0; i < count && found == NULL && r->refusal == NULL; i++) { + ScrStr *dir = paths[i]; + if (inside && dir->len > 0 && req_stat(dir) < 1) continue; + if (!absolute) { + found = req_resolve_exports(r, dir); + if (found != NULL || r->refusal != NULL) break; + } + ScrStr *base = req_resolve2(dir, request); + int kind = req_stat(base); + if (!trailing) { + if (kind == 0) found = req_try_file(base); + if (found == NULL) found = req_try_extensions(base); + } + if (found == NULL && kind == 1) found = req_try_package(r, base); + scr_str_release(base); + } + if (empty) scr_str_release(empty); + return found; +} + +/* Module._nodeModulePaths(from) followed by Module.globalPaths' entries + * that come from the environment (NODE_PATH, then the home directory's + * .node_modules and .node_libraries). Node also searches the lib/node + * directory of its own installation prefix, which a compiled program does + * not have. Fills `out` (+1 each) and answers the count. */ +static size_t req_lookup_paths(ScrStr *from, ScrStr ***out) { + size_t cap = 16, n = 0; + ScrStr **paths = malloc(cap * sizeof *paths); + if (!paths) scr_trap("scriptc: out of memory\n"); +#define REQ_PUSH(value) \ + do { \ + if (n == cap) { \ + cap *= 2; \ + ScrStr **grown = realloc(paths, cap * sizeof *paths); \ + if (!grown) scr_trap("scriptc: out of memory\n"); \ + paths = grown; \ + } \ + paths[n++] = (value); \ + } while (0) + ScrStr *dot = scr_str_new(".", 1); + ScrStr *abs = req_resolve2(from, dot); + scr_str_release(dot); + static const char nm[] = "node_modules"; + const char *s = abs->data; +#ifdef _WIN32 + if (abs->len >= 2 && s[abs->len - 1] == '\\' && s[abs->len - 2] == ':') { + REQ_PUSH(req_concat(abs, "node_modules", 12)); + } else { + for (long i = (long)abs->len - 1, p = 0, last = (long)abs->len; i >= 0; --i) { + char c = s[i]; + if (c == '\\' || c == '/' || c == ':') { + if (p != 12) { + ScrStr *prefix = scr_str_new(s, (size_t)last); + REQ_PUSH(req_concat(prefix, "\\node_modules", 13)); + scr_str_release(prefix); + } + last = i; + p = 0; + } else if (p != -1) { + p = (p < 12 && nm[11 - p] == c) ? p + 1 : -1; + } + } + } +#else + if (abs->len == 1 && s[0] == '/') { + REQ_PUSH(scr_str_new("/node_modules", 13)); + } else { + for (long i = (long)abs->len - 1, p = 0, last = (long)abs->len; i >= 0; --i) { + char c = s[i]; + if (c == '/') { + if (p != 12) { + ScrStr *prefix = scr_str_new(s, (size_t)last); + REQ_PUSH(req_concat(prefix, "/node_modules", 13)); + scr_str_release(prefix); + } + last = i; + p = 0; + } else if (p != -1) { + p = (p < 12 && nm[11 - p] == c) ? p + 1 : -1; + } + } + REQ_PUSH(scr_str_new("/node_modules", 13)); + } +#endif + scr_str_release(abs); +#ifdef _WIN32 + const char *node_path = scr_getenv("NODE_PATH"); + const char *home = scr_getenv("USERPROFILE"); + const char delimiter = ';'; +#else + const char *node_path = scr_getenv("NODE_PATH"); + const char *home = scr_getenv("HOME"); + const char delimiter = ':'; +#endif + if (node_path != NULL) { + for (const char *p = node_path; *p;) { + const char *end = strchr(p, delimiter); + size_t len = end ? (size_t)(end - p) : strlen(p); + if (len > 0) REQ_PUSH(scr_str_new(p, len)); + p += len; + if (*p) p++; + } + } + if (home != NULL && *home) { + ScrStr *home_dir = req_cstr(home); + static const char *const globals[] = { ".node_modules", ".node_libraries" }; + for (size_t i = 0; i < 2; i++) { + ScrStr *name = req_cstr(globals[i]); + REQ_PUSH(req_resolve2(home_dir, name)); + scr_str_release(name); + } + scr_str_release(home_dir); + } +#undef REQ_PUSH + *out = paths; + return n; +} + +/* trySelf: the nearest package.json above the importer (Node stops below + * the root and at a node_modules directory) resolves the request through + * its own "exports" when its "name" prefixes it. */ +static ScrStr *req_try_self(ScrRequire *r) { + ScrStr *dir = req_dirname(r->importer); + ScrStr *found = NULL; + for (;;) { + ScrStr *parent = req_dirname(dir); + bool root = scr_str_eq(parent, dir); + scr_str_release(parent); + if (root) break; + ScrStr *dot = scr_str_new(".", 1); + ScrStr *abs = req_resolve2(dir, dot); + scr_str_release(dot); + bool boundary = req_has_suffix(abs, "/node_modules") +#ifdef _WIN32 + || req_has_suffix(abs, "\\node_modules") +#endif + ; + scr_str_release(abs); + if (boundary) break; + ScrStr *pjson_name = req_cstr("package.json"); + ScrStr *pjson = req_resolve2(dir, pjson_name); + scr_str_release(pjson_name); + ScrDyn *pkg = req_read_package(r, pjson); + scr_str_release(pjson); + if (r->refusal != NULL) break; + if (pkg != NULL) { + ScrStr *name = req_package_string(pkg, "name"); + const ScrDyn *exports = req_package_map(pkg, "exports"); + if (req_has_prefix(r->request, "#") && req_package_map(pkg, "imports") != NULL) { + r->refusal = "a package \"imports\" specifier"; + } else if (name != NULL && exports != NULL) { + bool whole = scr_str_eq(name, r->request); + bool prefixed = r->request->len > name->len && + memcmp(r->request->data, name->data, name->len) == 0 && + r->request->data[name->len] == '/'; + if (whole || prefixed) { + ScrJsonBuf sub; + scr_jb_init(&sub); + scr_jb_putc(&sub, '.'); + scr_jb_write(&sub, r->request->data + name->len, r->request->len - name->len); + ScrStr *subpath = scr_jb_finish(&sub); + found = req_exports_resolve(r, dir, exports, subpath); + scr_str_release(subpath); + } + } + scr_dyn_release(pkg); + break; + } + ScrStr *next = req_dirname(dir); + scr_str_release(dir); + dir = next; + } + scr_str_release(dir); + return found; +} + +/* The builtin id `request` names (+1), or NULL. */ +static ScrStr *req_builtin(ScrRequire *r) { + ScrDyn *id = scr_dyn_new_str(r->request); + ScrStr *bare = scr_process_builtin_id(id, r->builtins); + scr_dyn_release(id); + if (bare == NULL || bare->len == 0) { + if (bare) scr_str_release(bare); + return NULL; + } + scr_str_release(bare); + return scr_str_retain(r->request); +} + +/* Module._resolveFilename: +1 resolved id or path; NULL when nothing + * resolves (or the refusal is set). */ +static ScrStr *req_resolve_filename(ScrRequire *r) { + ScrStr *builtin = req_builtin(r); + if (builtin != NULL) return builtin; + ScrStr *found = req_try_self(r); + if (found != NULL || r->refusal != NULL) return found; + bool lookup = r->request->len == 0 || r->request->data[0] != '.' || + (r->request->len > 1 && r->request->data[1] != '.' && r->request->data[1] != '/' +#ifdef _WIN32 + && r->request->data[1] != '\\' +#endif + ); + if (lookup) { + ScrStr *dir = req_dirname(r->importer); + ScrStr **paths = NULL; + size_t count = req_lookup_paths(dir, &paths); + scr_str_release(dir); + found = req_find_path(r, paths, count); + for (size_t i = 0; i < count; i++) scr_str_release(paths[i]); + free(paths); + } else { + ScrStr *dir = req_dirname(r->importer); + found = req_find_path(r, &dir, 1); + scr_str_release(dir); + } + return found; +} + +/* ── entry points ────────────────────────────────────────────────────── */ + +/* Ends the program with the SC2020 refusal: it names the native addon, the + * builtin or file the specifier resolves to, or the resolution step, then + * the program exits with status 1 as after an uncaught error, running the + * 'exit' listeners. */ +static void req_refuse(ScrRequire *r, const char *call, const ScrStr *found, bool builtin, ScrStr *site) { + ScrJsonBuf text; + scr_jb_init(&text); + if (r->refusal != NULL) { + scr_jb_puts(&text, call); + scr_jb_puts(&text, " of '"); + scr_jb_write(&text, r->request->data, r->request->len); + scr_jb_puts(&text, "' reaches "); + scr_jb_puts(&text, r->refusal); + scr_jb_puts(&text, ", which scriptc does not resolve at run time"); + } else if (builtin) { + scr_jb_puts(&text, call); + scr_jb_puts(&text, " of the builtin '"); + scr_jb_write(&text, found->data, found->len); + scr_jb_puts(&text, "' with a computed specifier is not supported; require it with a string literal"); + } else if (req_has_suffix(found, ".node")) { + scr_jb_puts(&text, "native addon '"); + scr_jb_write(&text, found->data, found->len); + scr_jb_puts(&text, "' requires Node's addon runtime, which is unavailable in scriptc"); + } else { + scr_jb_puts(&text, call); + scr_jb_puts(&text, " of '"); + scr_jb_write(&text, r->request->data, r->request->len); + scr_jb_puts(&text, "' resolves to '"); + scr_jb_write(&text, found->data, found->len); + scr_jb_puts(&text, "', which is not part of the compiled program: the module graph is fixed at build time"); + } + scr_jb_puts(&text, " [SC2020 at "); + scr_jb_write(&text, site->data, site->len); + scr_jb_puts(&text, "]\n"); + ScrStr *message = scr_jb_finish(&text); + fflush(stdout); + fputs("scriptc: ", stderr); + fwrite(message->data, 1, message->len, stderr); + fflush(stderr); + scr_str_release(message); + scr_process_exit(1); +} + +static void req_throw(const char *code, const char *prefix, ScrRequire *r, bool stack) { + ScrJsonBuf text; + scr_jb_init(&text); + scr_jb_puts(&text, prefix); + scr_jb_write(&text, r->request->data, r->request->len); + if (stack) { + scr_jb_puts(&text, "'\nRequire stack:\n- "); + scr_jb_write(&text, r->importer->data, r->importer->len); + } + ScrStr *message = scr_jb_finish(&text); + scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, code); + scr_str_release(message); +} + +ScrDyn *scr_require_computed(ScrDyn *id, ScrStr *importer, ScrStr *site, ScrArr *builtins) { + if (id->kind != SCR_DYN_STR) { + scr_dyn_arg_type_fail("id", "of type string", id); + return NULL; + } + if (id->v.str->len == 0) { + static const char message[] = "The argument 'id' must be a non-empty string. Received ''"; + scr_throw_error_msg_code(SCR_ERR_TYPE, message, sizeof message - 1, "ERR_INVALID_ARG_VALUE"); + return NULL; + } + ScrRequire r = { id->v.str, importer, builtins, NULL }; + ScrStr *builtin = req_builtin(&r); + if (builtin == NULL && req_has_prefix(r.request, "node:")) { + req_throw("ERR_UNKNOWN_BUILTIN_MODULE", "No such built-in module: ", &r, false); + return NULL; + } + ScrStr *found = builtin != NULL ? builtin : req_resolve_filename(&r); + if (found == NULL && r.refusal == NULL) { + req_throw("MODULE_NOT_FOUND", "Cannot find module '", &r, true); + return NULL; + } + req_refuse(&r, "createRequire's require", found, builtin != NULL, site); + if (found) scr_str_release(found); + return NULL; +} + +/* require.resolve's options: Node reads `paths` and `conditions` from an + * object; other values select the default lookup. */ +static const char *req_options_refusal(const ScrDyn *options) { + switch (options->kind) { + case SCR_DYN_UNDEF: + case SCR_DYN_NULL: + case SCR_DYN_NUM: + case SCR_DYN_BOOL: + case SCR_DYN_STR: + return NULL; + case SCR_DYN_OBJ: { + const ScrDyn *paths = scr_dyn_obj_get(options, "paths", 5); + const ScrDyn *conditions = scr_dyn_obj_get(options, "conditions", 10); + if (paths != NULL && paths->kind != SCR_DYN_UNDEF) return "require.resolve's paths option"; + if (conditions != NULL && conditions->kind != SCR_DYN_UNDEF) return "require.resolve's conditions option"; + return NULL; + } + default: + return "require.resolve options that are not a plain object"; + } +} + +ScrStr *scr_require_resolve(ScrDyn *request, ScrDyn *options, ScrStr *importer, ScrStr *site, + ScrArr *builtins) { + if (request->kind != SCR_DYN_STR) { + scr_dyn_arg_type_fail("request", "of type string", request); + return NULL; + } + ScrRequire r = { request->v.str, importer, builtins, req_options_refusal(options) }; + ScrStr *found = r.refusal == NULL ? req_resolve_filename(&r) : NULL; + if (r.refusal != NULL) { + if (found) scr_str_release(found); + req_refuse(&r, "createRequire's require.resolve", NULL, false, site); + return NULL; + } + if (found == NULL) req_throw("MODULE_NOT_FOUND", "Cannot find module '", &r, true); + return found; +} diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index bf01a4a9b..f19f3918b 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -31,6 +31,7 @@ #include "scr_string_methods.h" #include "scr_qs.h" #include "scr_array.h" +#include "scr_ieee754.h" #include "scr_regex.h" #include "scr_map.h" #include "scr_closure.h" diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 201d6e34b..fb416e5c3 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -93,6 +93,10 @@ struct ScrStreamState { bool is_transform; /* the writable side routes through transform/flush */ bool allow_half_open; bool auto_destroy, emit_close; + /* Duplex.from of a pipeline() argument that is not a stream: the + * pipeline takes it as that stage's stream without wiring the stage + * (scr_stream_unbox_dyn_stage) */ + bool pipeline_body; /* shared lifecycle */ bool destroyed, destroy_calling, error_emitted, close_emitted, error_scheduled, close_scheduled; @@ -3850,7 +3854,9 @@ typedef ScrPromise *(*ScrPipeNextFn)(ScrClosure *, ScrGen *); typedef ScrDyn *(*ScrPipeReadFn)(ScrClosure *, void *); typedef ScrDyn *(*ScrPipeBoxFn)(ScrClosure *, ScrPromise *); -enum { SCR_PIPE_DEST_STREAM, SCR_PIPE_DEST_GEN, SCR_PIPE_DEST_PROMISE }; +/* SCR_PIPE_DEST_DYN: an untyped destination function's result, a + * checked-dynamic value the pipeline classifies (scr_pipe_dest_dyn_kind) */ +enum { SCR_PIPE_DEST_STREAM, SCR_PIPE_DEST_GEN, SCR_PIPE_DEST_PROMISE, SCR_PIPE_DEST_DYN }; /* flags bit 7: the callback is typed code's (stream.pipelineStagesDyn) */ enum { SCR_PIPE_TYPED_CB = 128 }; @@ -3868,7 +3874,9 @@ typedef struct { * async generator destination's PassThrough included), in pipeline * order, or NULL */ ScrClosure **helpers; - ScrClosure *box; /* SCR_PIPE_DEST_PROMISE: boxes its promise as a dyn promise */ + /* SCR_PIPE_DEST_PROMISE: boxes its promise as a dyn promise; + * SCR_PIPE_DEST_DYN: answers the async generator over a dyn iterable */ + ScrClosure *box; } ScrPipeStages; static bool scr_pipe_is_stage(const ScrPipeStages *fs, size_t j) { @@ -4234,6 +4242,71 @@ static void scr_pipe_dest_promise(ScrStreamPipeCtx *ctx, ScrPromise *p /*moves*/ scr_promise_release(scr_async_spawn_after(d->boxed->v.promise, &scr_pipe_dest_entry, d)); } +/* An untyped destination's promise: scr_pipe_dest_promise over a promise + * that already holds dyn values. */ +static void scr_pipe_dest_dyn_promise(ScrStreamPipeCtx *ctx, ScrDyn *boxed /*moves*/, size_t pi, + bool end) { + ScrPipeDest *d = malloc(sizeof *d); + if (!d) scr_stream_oom(); + d->ctx = scr_stream_pipectx_retain_v(ctx); + d->typed = scr_promise_retain(boxed->v.promise); + d->boxed = boxed; + d->pi = pi; + d->end = end; + scr_promise_release(scr_async_spawn_after(d->boxed->v.promise, &scr_pipe_dest_entry, d)); +} + +typedef ScrGen *(*ScrPipeIterFn)(ScrClosure *, ScrDyn *); +static bool scr_stream_dyn_has_method(const ScrDyn *value, const char *key); +static bool scr_stream_dyn_transform_like(const ScrDyn *value); +static void scr_stream_throw_web_stage(void); +static bool scr_stream_dyn_stage_returned_stream(const ScrDyn *value); +static void scr_stream_throw_invalid_return(const char *expected, const char *name, + const ScrDyn *got); + +/* What pipelineImpl does with an untyped destination function's result: + * a promise settles the pipeline's value (the `then` branch), and an + * async iterable, or a { readable, writable } pair's readable side, is + * pumped into the PassThrough. *out answers the promise or that + * iterable's async generator (+1). Other values throw + * ERR_INVALID_RETURN_VALUE, and thenables that are not promises and web + * streams throw the not-supported Error; -1 then. */ +enum { SCR_PIPE_DYN_PROMISE, SCR_PIPE_DYN_ITER }; +static int scr_pipe_dest_dyn_kind(const ScrPipeStages *fs, ScrDyn *r, void **out) { + if (r->kind == SCR_DYN_PROMISE) { + *out = scr_dyn_retain(r); + return SCR_PIPE_DYN_PROMISE; + } + if (r->kind == SCR_DYN_OBJ && scr_stream_dyn_has_method(r, "then")) { + static const char msg[] = + "pipeline destinations that return a thenable other than a promise are not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, msg, sizeof msg - 1, "SC2020"); + return -1; + } + if (scr_exc_pending() || scr_stream_dyn_stage_returned_stream(r)) return -1; + ScrDyn *read = NULL; + int kind = scr_dyn_iterable_kind(r); + if (kind < 0) return -1; + if (r->kind == SCR_DYN_HANDLE && + (r->v.handle.tag == SCR_DYNH_WEB_STREAM || r->v.handle.tag == SCR_DYNH_WEB_TRANSFORM)) { + scr_stream_throw_web_stage(); + return -1; + } + if (kind == 2) { + read = scr_dyn_retain(r); + } else if (scr_stream_dyn_transform_like(r)) { + read = scr_dyn_obj_read(r, "readable", 8); + } + if (!read) { + if (!scr_exc_pending()) scr_stream_throw_invalid_return("AsyncIterable or Promise", "destination", r); + return -1; + } + ScrGen *g = ((ScrPipeIterFn)fs->box->fn)(fs->box, read); + if (!g) return -1; + *out = g; + return SCR_PIPE_DYN_ITER; +} + static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosure *cb, ScrStreamErrInv inv, bool end_opt, ScrDyn *signal, ScrAbortAddFn add, const ScrPipeStages *fs, @@ -4294,8 +4367,18 @@ static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosu : (void *)scr_stream_retain(ret); ret_gen = NULL; void *out = scr_pipe_call_stage(fs, k++, src); + int dyn_kind = -1; + if (!scr_exc_pending() && dest && fs->dest == SCR_PIPE_DEST_DYN) { + ScrDyn *r = out ? out : scr_dyn_retain(scr_dyn_undefined()); + out = NULL; + dyn_kind = scr_pipe_dest_dyn_kind(fs, r, &out); + scr_dyn_release(r); + } if (scr_exc_pending()) { - if (out && dest) scr_promise_release(out); + if (out && dest && fs->dest == SCR_PIPE_DEST_DYN) { + if (dyn_kind == SCR_PIPE_DYN_ITER) scr_gen_release(out); + else scr_dyn_release(out); + } else if (out && dest) scr_promise_release(out); else if (out) scr_gen_release(out); scr_stream_pipectx_drop(ctx); scr_stream_pipectx_release_v(ctx); @@ -4315,9 +4398,12 @@ static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosu size_t pi = ctx->n++; ctx->streams[pi] = pt; ctx->finish_count++; - if (fs->dest == SCR_PIPE_DEST_GEN) { + if (fs->dest == SCR_PIPE_DEST_GEN || + (fs->dest == SCR_PIPE_DEST_DYN && dyn_kind == SCR_PIPE_DYN_ITER)) { /* pumpToNode(ret, pt, finish, { end }) */ scr_pump_start(ctx, out, fs->helpers + 3 * h++, pt, end); + } else if (fs->dest == SCR_PIPE_DEST_DYN) { + scr_pipe_dest_dyn_promise(ctx, out, pi, end); } else { scr_pipe_dest_promise(ctx, out, fs->box, pi, end); } @@ -4332,6 +4418,20 @@ static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosu continue; } ScrStream *s = streams[i]; + if (s->st->pipeline_body) { + /* ret = Duplex.from(stream) for an argument that is not a stream: + * no destroyer or 'error' listener, and the previous stage is + * neither piped nor pumped into it */ + s->st->pipeline_body = false; + if (ret_gen) { + scr_gen_release(ret_gen); + ret_gen = NULL; + h++; + } + ret = s; + i++; + continue; + } bool writing = j > 0; if (reading && !scr_pipe_is_stage(fs, j + 1) && (streams[i + 1]->st->close_emitted || streams[i + 1]->st->destroyed)) { @@ -5212,13 +5312,31 @@ static void scr_sp_value_inv(ScrClosure *cb, ScrError *err, ScrDyn *value, ScrPr scr_promise_release(p); } +/* scr_sp_value_inv for an untyped destination (SCR_PIPE_DEST_DYN), whose + * result promise carries a dyn value: when the destination returned an + * async iterable (no promise), or its promise carries no payload, the + * result is undefined, as in Node. */ +static void scr_sp_dyn_value_inv(ScrClosure *cb, ScrError *err, ScrDyn *value, + ScrPromise *value_p) { + if (err != NULL || (value_p != NULL && scr_promise_payload_kind(value_p) != SCR_EXC_NONE)) { + scr_sp_value_inv(cb, err, value, value_p); + return; + } + ScrPromise *p = scr_box_get_ref(cb->caps[0]); /* +1 */ + scr_promise_fulfill_ref(p, scr_dyn_retain(scr_dyn_undefined()), scr_dyn_retain_v, + scr_dyn_release_v, NULL); + scr_promise_release(p); +} + ScrPromise *scr_sp_pipeline_stages(double n, ScrStream **streams, double count, ScrClosure **stages, double positions, double flags, double mask, ScrClosure **helpers, ScrClosure *box, ScrDyn *signal, ScrClosure *abort) { ScrPromise *p = scr_promise_new(); ScrPipeStages fs = scr_pipe_stages_of(count, stages, positions, flags, mask, helpers, box, signal); - ScrPipeValInv vi = fs.dest != SCR_PIPE_DEST_STREAM ? &scr_sp_value_inv : NULL; + ScrPipeValInv vi = fs.dest == SCR_PIPE_DEST_STREAM ? NULL + : fs.dest == SCR_PIPE_DEST_DYN ? &scr_sp_dyn_value_inv + : &scr_sp_value_inv; ScrStream *dst = scr_stream_pipeline_ex((size_t)n, streams, scr_sp_watcher(p), &scr_sp_settle_inv, true, NULL, NULL, &fs, abort, vi); if (dst) scr_stream_release(dst); @@ -5270,7 +5388,9 @@ ScrPromise *scr_sp_pipeline_stages_opts(double n, ScrStream **streams, double co ScrPromise *p = scr_promise_new(); if (!scr_sp_opts_signal(p, &outer)) return p; ScrPipeStages fs = scr_pipe_stages_of(count, stages, positions, flags, mask, helpers, box, signal); - ScrPipeValInv vi = fs.dest != SCR_PIPE_DEST_STREAM ? &scr_sp_value_inv : NULL; + ScrPipeValInv vi = fs.dest == SCR_PIPE_DEST_STREAM ? NULL + : fs.dest == SCR_PIPE_DEST_DYN ? &scr_sp_dyn_value_inv + : &scr_sp_value_inv; ScrStream *dst = scr_stream_pipeline_ex((size_t)n, streams, scr_sp_watcher(p), &scr_sp_settle_inv, end, outer, add, &fs, abort, vi); if (dst) scr_stream_release(dst); @@ -8593,10 +8713,90 @@ ScrStream *scr_stream_unbox_dyn(const ScrDyn *value, const ScrStr *message) { return NULL; } -/* pipeline.js's first stage: a value with pipe() and on() methods is a - * readable node stream, which the stage ladder tests before isIterable; - * a string, or a value with a Symbol.asyncIterator or Symbol.iterator - * method, is an iterable the pipeline pumps into the next stage. */ +/* Whether a checked-dynamic object's member is of type "object". */ +static bool scr_stream_dyn_member_object(const ScrDyn *value, const char *key) { + ScrDyn *m = scr_dyn_obj_read(value, key, strlen(key)); + if (!m) return false; + ScrStr *t = scr_dyn_typeof(m); + bool object = t->len == 6 && memcmp(t->data, "object", 6) == 0; + scr_str_release(t); + scr_dyn_release(m); + return object; +} + +/* pipeline.js's isTransformStream for a value that is not a stream: an + * object whose readable and writable members are both of type "object". */ +static bool scr_stream_dyn_transform_like(const ScrDyn *value) { + if (value->kind == SCR_DYN_HANDLE) return value->v.handle.tag == SCR_DYNH_WEB_TRANSFORM; + return value->kind == SCR_DYN_OBJ && scr_stream_dyn_member_object(value, "readable") && + !scr_exc_pending() && scr_stream_dyn_member_object(value, "writable"); +} + +/* Whether a checked-dynamic object has a function member `key`. */ +static bool scr_stream_dyn_has_method(const ScrDyn *value, const char *key) { + ScrDyn *m = scr_dyn_obj_read(value, key, strlen(key)); + if (!m) return false; + bool fn = m->kind == SCR_DYN_FUNC; + scr_dyn_release(m); + return fn; +} + +/* pipeline.js's isNodeStream and isWebStream for a checked-dynamic value + * that is not a native stream: a stream-like object or a web stream, which + * the pipeline would pipe or pump into and has no lowering for. */ +static bool scr_stream_dyn_stream_like(const ScrDyn *value) { + if (value->kind == SCR_DYN_HANDLE) { + int tag = value->v.handle.tag; + return tag == SCR_DYNH_WEB_STREAM || tag == SCR_DYNH_WEB_TRANSFORM || tag == SCR_DYNH_WEB_WRITABLE; + } + if (value->kind != SCR_DYN_OBJ) return false; + static const char *const state_keys[] = {"_readableState", "_writableState"}; + for (size_t i = 0; i < 2; i++) { + ScrDyn *v = scr_dyn_obj_read(value, state_keys[i], strlen(state_keys[i])); + bool set = v && scr_dyn_truthy(v); + if (v) scr_dyn_release(v); + if (set) return true; + } + bool on = scr_stream_dyn_has_method(value, "on"); + if (on && (scr_stream_dyn_has_method(value, "write") || scr_stream_dyn_has_method(value, "pipe"))) + return true; + if (scr_stream_dyn_has_method(value, "pipeThrough") && + scr_stream_dyn_has_method(value, "getReader") && scr_stream_dyn_has_method(value, "cancel")) + return true; + if (scr_stream_dyn_has_method(value, "getWriter") && scr_stream_dyn_has_method(value, "abort")) + return true; + return scr_stream_dyn_transform_like(value); +} + +/* A checked-dynamic pipeline() stage that the pipeline takes as a stream + * (the stream.unboxDynStage libCall): a boxed native stream or a stream + * handle answers its stream (+1). Any other value that is not stream-like + * or a function takes pipelineImpl's last branch, Duplex.from(stream): + * an iterable answers a readable Duplex the pipeline uses without piping + * the previous stage into it, and values Duplex.from rejects throw its + * ERR_INVALID_ARG_TYPE. Stream-like objects, web streams and functions + * throw `message` as the not-supported Error. */ +ScrStream *scr_stream_unbox_dyn_stage(const ScrDyn *value, const ScrStr *message) { + ScrStream *stream = scr_stream_of_handle(value); + if (!stream) stream = scr_stream_boxed(value); + if (stream) return stream; + if (value->kind == SCR_DYN_FUNC || scr_stream_dyn_stream_like(value)) { + if (!scr_exc_pending()) + scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, "SC2020"); + return NULL; + } + if (scr_exc_pending()) return NULL; + ScrStream *d = scr_stream_duplex_from_dyn((ScrDyn *)value); + if (d) d->st->pipeline_body = true; + return d; +} + +/* pipeline.js's first stage: a function is a source function, which the + * pipeline calls; a value with pipe() and on() methods is a readable node + * stream, which the stage ladder tests before isIterable; a string, a + * value with a Symbol.asyncIterator or Symbol.iterator method, or a + * { readable, writable } pair is an iterable the pipeline pumps into the + * next stage. */ bool scr_stream_dyn_iterable_source(const ScrDyn *value) { ScrStream *stream = scr_stream_of_handle(value); if (!stream) stream = scr_stream_boxed(value); @@ -8604,7 +8804,8 @@ bool scr_stream_dyn_iterable_source(const ScrDyn *value) { scr_stream_release(stream); return false; } - if (value->kind == SCR_DYN_STR) return true; + if (value->kind == SCR_DYN_STR || value->kind == SCR_DYN_FUNC) return true; + if (value->kind == SCR_DYN_HANDLE && value->v.handle.tag == SCR_DYNH_WEB_TRANSFORM) return true; if (value->kind == SCR_DYN_OBJ) { ScrDyn *pipe = scr_dyn_obj_read(value, "pipe", 4); if (!pipe) return false; @@ -8617,6 +8818,79 @@ bool scr_stream_dyn_iterable_source(const ScrDyn *value) { scr_dyn_release(on); if (streamish) return false; } + if (scr_stream_dyn_transform_like(value)) return true; + if (scr_exc_pending()) return false; } return scr_dyn_iterable_kind(value) > 0; } + +/* Untyped web streams as pipeline stages have no lowering. */ +static void scr_stream_throw_web_stage(void) { + static const char msg[] = "pipeline stages that are untyped web streams are not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, msg, sizeof msg - 1, "SC2020"); +} + +/* A native stream returned by an untyped pipeline function, which Node + * pipes or passes on as the stream: throws the not-supported Error. */ +static bool scr_stream_dyn_stage_returned_stream(const ScrDyn *value) { + ScrStream *stream = scr_stream_of_handle(value); + if (!stream) stream = scr_stream_boxed(value); + if (!stream) return false; + scr_stream_release(stream); + static const char msg[] = "pipeline function stages that return a stream are not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, msg, sizeof msg - 1, "SC2020"); + return true; +} + +/* Node's ERR_INVALID_RETURN_VALUE TypeError: "Expected to be + * returned from the "" function but got ." */ +static void scr_stream_throw_invalid_return(const char *expected, const char *name, + const ScrDyn *got) { + char detail[160]; + const char *type = scr_dyn_specific_type(got, detail, sizeof detail); + char msg[384]; + int n = snprintf(msg, sizeof msg, "Expected %s to be returned from the \"%s\" function but got %s.", + expected, name, type); + if (n < 0) n = 0; + if ((size_t)n >= sizeof msg) n = (int)sizeof msg - 1; + scr_throw_error_msg_code(SCR_ERR_TYPE, msg, (size_t)n, "ERR_INVALID_RETURN_VALUE"); +} + +/* An untyped pipeline function stage's result before the last stage + * (pipelineImpl's `isIterable(ret, true)` check): an async iterable + * answers itself (+1); other values throw ERR_INVALID_RETURN_VALUE naming + * the stage, as `label` ("transform[i]") does. */ +ScrDyn *scr_stream_dyn_transform_result(const ScrDyn *value, const ScrStr *label) { + if (scr_stream_dyn_stage_returned_stream(value)) return NULL; + int kind = scr_dyn_iterable_kind(value); + if (kind < 0) return NULL; + if (kind != 2) { + scr_stream_throw_invalid_return("AsyncIterable", label->data, value); + return NULL; + } + return scr_dyn_retain((ScrDyn *)value); +} + +/* The value a checked-dynamic first stage pumps (pipelineImpl's i === 0 + * branch): for `returned`, a source function's result, which must be + * iterable; otherwise a { readable, writable } pair's readable side, or + * the value itself. +1, or NULL with an exception pending. */ +ScrDyn *scr_stream_dyn_source_value(const ScrDyn *value, bool returned) { + if (!returned) { + if (value->kind == SCR_DYN_HANDLE && value->v.handle.tag == SCR_DYNH_WEB_TRANSFORM) { + scr_stream_throw_web_stage(); + return NULL; + } + if (scr_stream_dyn_transform_like(value)) return scr_dyn_obj_read(value, "readable", 8); + if (scr_exc_pending()) return NULL; + return scr_dyn_retain((ScrDyn *)value); + } + if (scr_stream_dyn_stage_returned_stream(value)) return NULL; + int kind = scr_dyn_iterable_kind(value); + if (kind < 0) return NULL; + if (kind == 0) { + scr_stream_throw_invalid_return("Iterable, AsyncIterable or Stream", "source", value); + return NULL; + } + return scr_dyn_retain((ScrDyn *)value); +} diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index 7acad0eeb..c10978298 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -149,6 +149,18 @@ ScrStream *scr_stream_unbox_dyn(const struct ScrDyn *value, const ScrStr *messag * to pump rather than as a stream (false with an exception pending when * reading the value's members throws). */ bool scr_stream_dyn_iterable_source(const struct ScrDyn *value); +/* A checked-dynamic pipeline() stage the pipeline takes as a stream: a + * native stream (+1), or Duplex.from of a value that is not stream-like + * or a function, which the pipeline uses without piping into it; NULL + * with an exception otherwise (`message` for the unsupported shapes). */ +ScrStream *scr_stream_unbox_dyn_stage(const struct ScrDyn *value, const ScrStr *message); +/* The iterable pipeline() pumps for a checked-dynamic first stage: a + * source function's result when `returned`, else a { readable, writable } + * pair's readable side or the value (+1, or NULL with an exception). */ +struct ScrDyn *scr_stream_dyn_source_value(const struct ScrDyn *value, bool returned); +/* An untyped function stage's result before the last pipeline stage: the + * async iterable itself (+1), or NULL with ERR_INVALID_RETURN_VALUE. */ +struct ScrDyn *scr_stream_dyn_transform_result(const struct ScrDyn *value, const ScrStr *label); /* Handles that are Transforms (ScrDynHandleOps.transform: Cipheriv and * Decipheriv). scr_stream_of_handle: the handle's Transform (+1), built on * first use, or NULL when the value is not such a handle. diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 64a4772e5..71d629941 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -83,6 +83,12 @@ typedef struct { size_t cu; /* UTF-16 code-unit offset, always a code-point boundary */ size_t cb; /* matching UTF-8 byte offset, never a continuation byte */ } ScrSidxPoint; +/* One allocation: the 32-bit base of every 64th unit follows the deltas. */ +typedef struct { + size_t n; /* the UTF-16 length the offsets describe */ + uint32_t *base; /* byte offset of every 64th unit */ + uint8_t delta[]; /* each unit's byte offset past its block's base */ +} ScrSidxOffsets; typedef struct ScrSidx { const ScrStr *s; /* NULL = empty slot */ size_t u16len; /* SCR_U16_UNKNOWN until the whole current string */ @@ -94,10 +100,12 @@ typedef struct ScrSidx { size_t indexed_cb; bool no_more_points; /* metadata allocation failed/overflowed: fail open */ bool points_complete; /* points cover every stride of indexed prefix */ + uint32_t walks; /* index walks before the offsets exist */ uint16_t *units; /* the UTF-16 code units, once reads warrant them */ - size_t nunits; - size_t reads; /* indexed reads that missed the unit array */ -} ScrSidx; + uint32_t nunits; /* units and offsets need at most 1 << 24 bytes */ + uint32_t reads; /* indexed reads that missed the unit array */ + ScrSidxOffsets *offsets; /* every unit's byte offset, once walks warrant */ +} ScrSidx; /* 128 bytes on 64-bit targets: every lookup clears or copies one */ /* Keep short-string cursor traffic out of the sparse cache. A short-lived * one-byte receiver can be far more common than a large indexed one; sharing * the round-robin slots would otherwise rebuild a warm index every few calls. @@ -136,6 +144,7 @@ static void scr_sidx_clear(ScrSidx *e, unsigned *live) { scr_str_scan.n = 0; } free(e->units); + if (e->offsets) free(e->offsets); /* rare: skip the call on the hot clear */ memset(e, 0, sizeof(*e)); } @@ -337,6 +346,9 @@ static void scr_sidx_concat_append(const ScrStr *s, size_t oldlen) { e->units = NULL; e->nunits = 0; e->reads = 0; + free(e->offsets); /* and so do the offsets */ + e->offsets = NULL; + e->walks = 0; } } } @@ -1576,6 +1588,8 @@ static void scr_sidx_ascii_run(const ScrStr *s, ScrSidx *e) { /* How far past its end a forward access may grow the cached ASCII run, and * how far beyond the access the growth looks ahead. */ #define SCR_SIDX_ASCII_REACH ((size_t)4096) +/* How far from the hot cursor an index walks from it. */ +#define SCR_SIDX_NEAR_UNITS ((size_t)64) #define SCR_SIDX_ASCII_AHEAD ((size_t)256) /* Grow the cached ASCII run forward until it covers byte `want` (plus a @@ -1624,8 +1638,118 @@ static inline size_t scr_u16_to_byte_c(const ScrStr *s, ScrSidx *e, size_t u16, return scr_u16_to_byte_walk(s, e, u16, mid); } +/* Index walks on one long non-ASCII receiver (slices, substrings and + * indexed reads taken across it) each step through bytes from the nearest + * known position. Once those walks would have paid for one pass over the + * receiver (a walk per 32 bytes, at least 64 walks), a receiver from + * SCR_SIDX_MIN_BYTES to SCR_SIDX_OFFSETS_MAX bytes keeps every unit's byte + * offset: a 32-bit base per 64 units and one byte per unit past it (64 units + * span at most 192 bytes). The low half of an astral character has its high + * half's offset. The offsets are freed with the entry, and an append drops + * them (scr_sidx_concat_append). */ +#define SCR_SIDX_OFFSET_BLOCK ((size_t)64) +#define SCR_SIDX_OFFSETS_MAX ((size_t)1 << 24) +static bool scr_sidx_offsets(const ScrStr *s, ScrSidx *e) { + if (e->offsets) return true; + if (s->len < SCR_SIDX_MIN_BYTES || s->len > SCR_SIDX_OFFSETS_MAX) return false; + if (++e->walks < 64 || (size_t)e->walks * 32 < s->len) return false; + size_t n = scr_sidx_len(s, e); + if (s->ascii || e->u16len == s->len) return false; + size_t ndelta = (n + 1 + 3) & ~(size_t)3; + ScrSidxOffsets *o = + malloc(sizeof *o + ndelta + (n / SCR_SIDX_OFFSET_BLOCK + 1) * sizeof(uint32_t)); + if (!o) { + e->walks = 0; + return false; + } + o->n = n; + o->base = (uint32_t *)(o->delta + ndelta); + uint32_t *base = o->base; + uint8_t *delta = o->delta; + size_t cu = 0, cb = 0; + while (cb < s->len && cu < n) { + size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]); + for (size_t w = seq == 4 ? 2 : 1; w > 0 && cu < n; w--, cu++) { + if (cu % SCR_SIDX_OFFSET_BLOCK == 0) base[cu / SCR_SIDX_OFFSET_BLOCK] = (uint32_t)cb; + delta[cu] = (uint8_t)(cb - base[cu / SCR_SIDX_OFFSET_BLOCK]); + } + cb += seq; + } + e->offsets = o; + return true; +} + +static inline size_t scr_sidx_offset_at(const ScrSidx *e, size_t u16) { + return e->offsets->base[u16 / SCR_SIDX_OFFSET_BLOCK] + e->offsets->delta[u16]; +} + +/* Walk from a known code-point boundary (cu, cb) to UTF-16 index u16 and + * leave the hot cursor there. Same contract as scr_u16_to_byte_walk. */ +static size_t scr_sidx_walk_from(const ScrStr *s, ScrSidx *e, size_t cu, size_t cb, size_t u16, + bool *mid, bool refresh_run) { + while (cu > u16) { + /* Eight bytes starting inside a sequence can reach back at most three + * more bytes to its lead. Such a span contains at most nine UTF-16 + * units (seven ASCII bytes plus an astral pair), so it cannot overshoot + * a target at least nine units away. */ + if (cu - u16 >= 9 && cb >= 8) { + size_t start = cb - 8; + while (start > 0 && ((unsigned char)s->data[start] & 0xC0) == 0x80) + start--; + cu -= scr_utf16_units_span(s->data + start, cb - start, NULL); + cb = start; + } else { + scr_sidx_back(s, &cu, &cb); + } + } + bool m = false; + while (cu < u16 && cb < s->len) { + if (u16 - cu >= 9 && s->len - cb >= 8) { + size_t end = cb + 8; + while (end < s->len && ((unsigned char)s->data[end] & 0xC0) == 0x80) + end++; + cu += scr_utf16_units_span(s->data + cb, end - cb, NULL); + cb = end; + continue; + } + SCR_SIDX_STEP(); + size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]); + size_t w = seq == 4 ? 2 : 1; + if (cu + w > u16) { /* u16 lands between the halves of an astral char */ + m = true; + break; + } + cu += w; + cb += seq; + } + e->cu = cu; + e->cb = cb; + if (refresh_run && cb < s->len && (unsigned char)s->data[cb] < 0x80) + scr_sidx_ascii_run(s, e); + *mid = m; + return cb; +} + static __attribute__((noinline)) size_t scr_u16_to_byte_walk(const ScrStr *s, ScrSidx *e, size_t u16, bool *mid) { + if (scr_sidx_offsets(s, e)) { + *mid = false; + if (u16 >= e->offsets->n) return s->len; + size_t cb = scr_sidx_offset_at(e, u16); + size_t cu = u16; + if (u16 > 0 && scr_sidx_offset_at(e, u16 - 1) == cb) { + *mid = true; /* the low half of an astral character */ + cu = u16 - 1; + } + e->cu = cu; + e->cb = cb; + return cb; + } + /* Within a short distance of the hot cursor (a scanner + * taking slices near its position), walk from that cursor: no index + * search, extension or ASCII-run growth is cheaper. */ + if (e->cb <= s->len && scr_sidx_abs_diff(e->cu, u16) <= SCR_SIDX_NEAR_UNITS) + return scr_sidx_walk_from(s, e, e->cu, e->cb, u16, mid, false); /* A short step forward from the hot cursor (a sequential scan of * non-ASCII text) walks from the cursor itself. */ if (u16 >= e->cu && u16 - e->cu <= 8) { @@ -1665,48 +1789,7 @@ static __attribute__((noinline)) size_t scr_u16_to_byte_walk(const ScrStr *s, Sc } scr_sidx_materialize_identity_prefix(s, e); ScrSidxPoint near = scr_sidx_near_u16(s, e, u16); - size_t cu = near.cu, cb = near.cb; - while (cu > u16) { - /* Eight bytes starting inside a sequence can reach back at most three - * more bytes to its lead. Such a span contains at most nine UTF-16 - * units (seven ASCII bytes plus an astral pair), so it cannot overshoot - * a target at least nine units away. */ - if (cu - u16 >= 9 && cb >= 8) { - size_t start = cb - 8; - while (start > 0 && ((unsigned char)s->data[start] & 0xC0) == 0x80) - start--; - cu -= scr_utf16_units_span(s->data + start, cb - start, NULL); - cb = start; - } else { - scr_sidx_back(s, &cu, &cb); - } - } - bool m = false; - while (cu < u16 && cb < s->len) { - if (u16 - cu >= 9 && s->len - cb >= 8) { - size_t end = cb + 8; - while (end < s->len && ((unsigned char)s->data[end] & 0xC0) == 0x80) - end++; - cu += scr_utf16_units_span(s->data + cb, end - cb, NULL); - cb = end; - continue; - } - SCR_SIDX_STEP(); - size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]); - size_t w = seq == 4 ? 2 : 1; - if (cu + w > u16) { /* u16 lands between the halves of an astral char */ - m = true; - break; - } - cu += w; - cb += seq; - } - e->cu = cu; - e->cb = cb; - if (cb < s->len && (unsigned char)s->data[cb] < 0x80) - scr_sidx_ascii_run(s, e); - *mid = m; - return cb; + return scr_sidx_walk_from(s, e, near.cu, near.cb, u16, mid, true); } static size_t scr_byte_to_u16_walk(const ScrStr *s, ScrSidx *e, size_t byte_off); @@ -1875,10 +1958,17 @@ static uint16_t *scr_str_units(const ScrStr *s, size_t *n) { * megabytes, which scanners read in full. The array is freed with the * entry, and an append drops it (scr_sidx_concat_append). */ #define SCR_SIDX_UNITS_MAX ((size_t)1 << 24) +static void scr_sidx_take_units(const ScrStr *s, ScrSidx *e) { + size_t n = 0; + e->units = scr_str_units(s, &n); + e->nunits = (uint32_t)n; +} + static bool scr_sidx_units(const ScrStr *s, ScrSidx *e) { if (!e->units) { - if (++e->reads < 32 || e->reads * 16 < s->len || s->len > SCR_SIDX_UNITS_MAX) return false; - e->units = scr_str_units(s, &e->nunits); + if (s->len > SCR_SIDX_UNITS_MAX || ++e->reads < 32 || (size_t)e->reads * 16 < s->len) + return false; + scr_sidx_take_units(s, e); } /* Publish the scan's receiver for emitted reads (scr_runtime.h). */ if (scr_str_scan.s != s) { @@ -1905,7 +1995,7 @@ void scr_str_scan_units_for(ScrStr *s, double from, double to) { (void)scr_sidx_units(s, e); return; } - e->units = scr_str_units(s, &e->nunits); + scr_sidx_take_units(s, e); } if (scr_str_scan.s != s) { scr_str_scan.s = s; @@ -1980,8 +2070,9 @@ double scr_str_last_index_of_from(ScrStr *s, ScrStr *needle, double position) { size_t start_byte = scr_u16_to_byte_c(s, e, start16, &mid); size_t max_start = s->len - needle->len; if (start_byte < max_start) max_start = start_byte; + char first = needle->data[0]; for (size_t i = max_start + 1; i-- > 0;) { - if (memcmp(s->data + i, needle->data, needle->len) == 0) { + if (s->data[i] == first && memcmp(s->data + i, needle->data, needle->len) == 0) { return (double)scr_byte_to_u16_c(s, e, i); } } diff --git a/packages/runtime/src/scr_type_origin.c b/packages/runtime/src/scr_type_origin.c new file mode 100644 index 000000000..6ea872304 --- /dev/null +++ b/packages/runtime/src/scr_type_origin.c @@ -0,0 +1,222 @@ +/* Declared-type origins on checked-conversion TypeErrors, linked only by + * programs that check values against types an npm package declares + * (runtime feature typeOrigins). + * + * A checked conversion into a type that comes from a package's + * declarations (its .d.ts, @types package or JSDoc) throws the usual + * `expected at , got ` TypeError when the value does + * not fit. The compiler passes the declaration's origin to the conversion + * site's cold exception path, which calls scr_type_origin_note: the + * pending TypeError's message gains two lines naming the declaration and + * the type patch (package.json scriptc.typePatches) that would make the + * declaration admit the value, as the `scriptc types patch` command that + * writes it. The hot path is unchanged, and so is every + * other error. + * + * The origin is one string of fields separated by \x1f: package, version, + * suggested range (which the command chooses itself), the selector of the + * declared slot, where it is declared, and the declared type. */ +#include "scr_runtime.h" + +#include +#include + +typedef struct { + char *data; + size_t len, cap; +} ScrOriginBuf; + +static void scr_origin_put(ScrOriginBuf *b, const char *s, size_t n) { + if (b->len + n + 1 > b->cap) { + size_t cap = b->cap ? b->cap : 256; + while (b->len + n + 1 > cap) cap *= 2; + char *grown = realloc(b->data, cap); + if (!grown) scr_trap("scriptc: out of memory\n"); + b->data = grown; + b->cap = cap; + } + memcpy(b->data + b->len, s, n); + b->len += n; + b->data[b->len] = 0; +} + +static void scr_origin_puts(ScrOriginBuf *b, const char *s) { scr_origin_put(b, s, strlen(s)); } + +/* One argument of the suggested `scriptc types patch` command, quoted as + * the shell passes it as written (decl-origin.ts's shellArgument quotes the + * same way): bare when it has only characters no shell treats specially; + * otherwise, for a POSIX shell, in single quotes with each ' as '\'', and + * on Windows (cmd and PowerShell), in double quotes when it has none of the + * characters either shell expands, escapes or ends a string with there. + * Package names, selectors and types come from package declarations. + * Returns false, writing nothing, when no quoting is inert. */ +static bool scr_origin_bare(const char *s, size_t n) { + if (n == 0) return false; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + bool plain = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || + strchr("_@+=:,./-", c) != NULL; +#ifndef _WIN32 + plain = plain || c == '%'; +#endif + if (!plain || c == 0) return false; + } + return true; +} + +static bool scr_origin_quotable(const char *s, size_t n) { +#ifdef _WIN32 + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (c < 0x20 || c == 0x7f || (c != 0 && strchr("\"%!^$`", c) != NULL)) return false; + /* PowerShell also takes the typographic quotes U+2018-U+201F (UTF-8 + * E2 80 98..9F) as quotes. */ + if (c == 0xe2 && i + 2 < n && (unsigned char)s[i + 1] == 0x80 && + (unsigned char)s[i + 2] >= 0x98 && (unsigned char)s[i + 2] <= 0x9f) + return false; + } +#else + (void)s; + (void)n; +#endif + return true; +} + +static void scr_origin_put_arg(ScrOriginBuf *b, const char *s, size_t n) { + if (scr_origin_bare(s, n)) { + scr_origin_put(b, s, n); + return; + } +#ifdef _WIN32 + scr_origin_puts(b, "\""); + scr_origin_put(b, s, n); + scr_origin_puts(b, "\""); +#else + scr_origin_puts(b, "'"); + for (size_t i = 0; i < n; i++) { + if (s[i] == '\'') scr_origin_puts(b, "'\\''"); + else scr_origin_put(b, s + i, 1); + } + scr_origin_puts(b, "'"); +#endif +} + +/* Field `index` of the origin string (not terminated: [*out, *out+len)). */ +static size_t scr_origin_field(const char *origin, int index, const char **out) { + const char *p = origin; + for (int i = 0; i < index; i++) { + const char *sep = strchr(p, '\x1f'); + if (!sep) { + *out = ""; + return 0; + } + p = sep + 1; + } + const char *end = strchr(p, '\x1f'); + *out = p; + return end ? (size_t)(end - p) : strlen(p); +} + +/* The TypeScript type a `got` kind of the check message suggests adding. */ +static const char *scr_origin_arm(const char *kind, size_t len) { + if (len == 5 && memcmp(kind, "array", 5) == 0) return "unknown[]"; + if (len == 8 && memcmp(kind, "function", 8) == 0) return "Function"; + return NULL; /* the kind itself (undefined, null, string, number, ...) */ +} + +void scr_type_origin_note(const char *origin) { + ScrExcCell *cell = scr_exc_current_cell(); + if (cell->kind != SCR_EXC_OBJ || cell->payload == NULL) return; + ScrError *e = (ScrError *)cell->payload; + if (e->vt != &scr_error_vts[SCR_ERR_TYPE] || e->message == NULL) return; + const char *msg = e->message->data; + size_t mlen = e->message->len; + if (memchr(msg, '\n', mlen) != NULL) return; /* already noted */ + /* `expected at , got `: the path extends the + * declared slot's selector, the kind is what the patch admits. */ + const char *path = NULL, *got = NULL; + size_t path_len = 0, got_len = 0; + if (mlen > 9 && memcmp(msg, "expected ", 9) == 0) { + for (size_t i = mlen; i-- > 7;) { + if (memcmp(msg + i - 6, ", got ", 6) == 0) { + got = msg + i; + got_len = mlen - i; + for (size_t j = i - 6; j-- > 4;) { + if (memcmp(msg + j - 4, " at $", 5) == 0) { + path = msg + j; + path_len = i - 6 - j; + break; + } + } + break; + } + } + } + const char *pkg, *version, *selector, *where, *declared; + size_t pkg_len = scr_origin_field(origin, 0, &pkg); + size_t version_len = scr_origin_field(origin, 1, &version); + size_t selector_len = scr_origin_field(origin, 3, &selector); + size_t where_len = scr_origin_field(origin, 4, &where); + size_t declared_len = scr_origin_field(origin, 5, &declared); + + ScrOriginBuf b = {0}; + scr_origin_put(&b, msg, mlen); + scr_origin_puts(&b, "\n "); + scr_origin_put(&b, pkg, pkg_len); + if (version_len > 0) { + scr_origin_puts(&b, "@"); + scr_origin_put(&b, version, version_len); + } + scr_origin_puts(&b, " declares "); + scr_origin_put(&b, selector, selector_len); + if (declared_len > 0) { + scr_origin_puts(&b, " as "); + scr_origin_put(&b, declared, declared_len); + } + scr_origin_puts(&b, " ("); + scr_origin_put(&b, where, where_len); + scr_origin_puts(&b, ")\n "); + ScrOriginBuf sel = {0}, type = {0}; + scr_origin_put(&sel, selector, selector_len); + /* $[3].email extends the selector as [].email: elements of an array + * share one declared type. */ + for (size_t i = 1; path != NULL && i < path_len; i++) { + if (path[i] == '[') { + scr_origin_puts(&sel, "[]"); + while (i < path_len && path[i] != ']') i++; + } else { + scr_origin_put(&sel, path + i, 1); + } + } + scr_origin_puts(&type, "| "); + if (got != NULL) { + const char *arm = scr_origin_arm(got, got_len); + if (arm != NULL) scr_origin_puts(&type, arm); + else scr_origin_put(&type, got, got_len); + } else { + scr_origin_puts(&type, "undefined"); + } + if (scr_origin_quotable(pkg, pkg_len) && scr_origin_quotable(sel.data, sel.len) && + scr_origin_quotable(type.data, type.len)) { + scr_origin_puts(&b, "if that declaration is wrong, patch it: scriptc types patch "); + scr_origin_put_arg(&b, pkg, pkg_len); + scr_origin_puts(&b, " "); + scr_origin_put_arg(&b, sel.data, sel.len); + scr_origin_puts(&b, " "); + scr_origin_put_arg(&b, type.data, type.len); + } else { + scr_origin_puts(&b, "if that declaration is wrong, patch it with scriptc types patch"); + } + free(sel.data); + free(type.data); + + ScrStr *next = scr_str_new(b.data, b.len); + free(b.data); + scr_str_release(e->message); + e->message = next; + if (e->stack != NULL) { + /* The cached `stack` starts with the message. */ + scr_str_release(e->stack); + e->stack = NULL; + } +} diff --git a/packages/runtime/src/scr_type_origin.test.c b/packages/runtime/src/scr_type_origin.test.c new file mode 100644 index 000000000..bc6477d23 --- /dev/null +++ b/packages/runtime/src/scr_type_origin.test.c @@ -0,0 +1,24 @@ +/* Built by scr_type_origin.test.ts: a pending TypeError gains the + * declaration note for an origin whose package name and selector carry + * shell syntax; the test runs the suggested command through sh. */ +#include "scr_runtime.h" + +#include +#include + +int main(int argc, char **argv) { + if (argc != 3) return 2; + /* package, version, range, selector, where, declared type */ + char origin[1024]; + snprintf(origin, sizeof origin, "%s\x1f" "1.0.0\x1f^1.0.0\x1f%s\x1findex.d.ts:3\x1fstring", argv[1], + argv[2]); + const char *text = "expected string at $, got undefined"; + ScrExcCell *cell = scr_exc_current_cell(); + cell->kind = SCR_EXC_OBJ; + cell->payload = scr_error_new(SCR_ERR_TYPE, scr_str_new(text, strlen(text))); + scr_type_origin_note(origin); + ScrError *e = cell->payload; + fwrite(e->message->data, 1, e->message->len, stdout); + fputc('\n', stdout); + return 0; +} diff --git a/packages/runtime/src/scr_type_origin.test.ts b/packages/runtime/src/scr_type_origin.test.ts new file mode 100644 index 000000000..b31195deb --- /dev/null +++ b/packages/runtime/src/scr_type_origin.test.ts @@ -0,0 +1,69 @@ +/* The suggested `scriptc types patch` command on a checked-conversion + * TypeError (scr_type_origin.c) passes each value it carries as one + * argument: package names and selectors come from package declarations, + * and a command copied from the message must not run anything they spell. + * Run through sh with printf in place of scriptc, the command prints the + * package, the selector and the type exactly, and creates no file. */ +import { execFile } from "node:child_process"; +import { mkdtemp, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { EXECUTABLE_RUNTIME_SOURCES } from "../../compiler/src/backend/native-toolchain.js"; + +const exec = promisify(execFile); + +const hostile = [ + ["kit", "Profile.email"], + ["evil;touch pkg", "Profile.a;touch semicolon"], + ["kit", "Profile.$(touch subst)"], + ["kit", "Profile.`touch backtick`"], + ["kit", "Profile.it's \"quoted\" \\ end'"], + ["kit", "Profile.a&&touch and|touch pipe>out"], +] as const; + +test.skipIf(process.platform === "win32")( + "the suggested patch command passes declaration values as single arguments", + { timeout: 300_000 }, + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-type-origin-")); + try { + const bin = join(dir, "origin"); + await exec("clang", [ + "-std=c11", + "-O1", + "-w", + "-pthread", + ...(process.platform === "linux" ? ["-D_GNU_SOURCE"] : []), + join(import.meta.dirname, "scr_type_origin.test.c"), + join(import.meta.dirname, "scr_type_origin.c"), + ...EXECUTABLE_RUNTIME_SOURCES.map((name) => join(import.meta.dirname, name)), + ...(process.platform === "linux" ? ["-lm"] : []), + "-o", + bin, + ]); + const sandbox = join(dir, "run"); + await exec("mkdir", [sandbox]); + const commands: string[] = []; + for (const [pkg, selector] of hostile) { + const { stdout } = await exec(bin, [pkg, selector]); + const line = stdout.split("\n").find((l) => l.includes("scriptc types patch")); + expect(line, stdout).toBeDefined(); + const command = line!.slice(line!.indexOf("scriptc types patch")); + commands.push(command); + const printed = await exec( + "sh", + ["-c", command.replace("scriptc types patch", "printf '%s\\n'")], + { cwd: sandbox }, + ); + expect(printed.stdout).toBe(`${pkg}\n${selector}\n| undefined\n`); + expect(await readdir(sandbox)).toEqual([]); + } + // Values without shell syntax stay bare. + expect(commands[0]).toBe("scriptc types patch kit Profile.email '| undefined'"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, +); diff --git a/packages/runtime/test/heap-trim.test.ts b/packages/runtime/test/heap-trim.test.ts new file mode 100644 index 000000000..f2ec405fe --- /dev/null +++ b/packages/runtime/test/heap-trim.test.ts @@ -0,0 +1,43 @@ +import { execFile } from "node:child_process"; +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; + +const execFileAsync = promisify(execFile); +const testDir = import.meta.dirname; + +// scr_heap_trim returns the size-class pages that hold only free blocks, in +// the single-threaded allocator and in the per-thread arenas of worker +// runtimes while another thread allocates. +test.runIf(process.platform === "linux")( + "heap trim returns free pages and keeps live blocks", + async () => { + const buildDir = join(testDir, "build"); + await mkdir(buildDir, { recursive: true }); + for (const [name, flags] of [ + ["test_heap_trim", []], + ["test_heap_trim_workers", ["-DSCR_WORKERS", "-pthread"]], + ] as const) { + const bin = join(buildDir, name); + await execFileAsync("clang", [ + "-std=c11", + "-D_GNU_SOURCE", + "-O1", + "-Wall", + "-Wextra", + "-fsanitize=undefined", + ...flags, + "-o", + bin, + join(testDir, "test_heap_trim.c"), + join(testDir, "../src/scr_alloc.c"), + ]); + const run = await execFileAsync(bin, [], { + env: { ...process.env, UBSAN_OPTIONS: "halt_on_error=1" }, + }); + expect(run.stdout).toBe("heap trim checks passed\n"); + } + }, + 120_000, +); diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index dc48abd26..08eaecad4 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -1314,6 +1314,44 @@ static void check_limited_pass(void) { check(freed - before == 4 * RING, "the live ring was not collected once dead"); } +/* A limited pass that reaches a live candidate it did not take leaves the + * candidate in its slot and in its generation, which names the slot's + * buffer: when the candidate dies afterwards, it leaves that buffer, and no + * later pass meets its freed block. */ +static void check_limited_pass_keeps_untaken_generation(void) { + enum { RING = 16 }; + if (configured_nursery_threshold() < 8) return; + scr_collect_cycles(); + size_t before = freed; + static ScrCycType cyc_type = { &node_trace, &node_free, 0 }; + Node *shared = scr_cyc_alloc(sizeof(*shared), &cyc_type); + shared->rc = 1; /* its owner */ + /* A dead ring the filter takes points at it. */ + Node *dead = make_ring(RING); + dead->other = shared; + shared->rc++; + /* It loses a reference and becomes a candidate the filter refuses. */ + shared->rc++; + release_live(shared); + release_live(dead); + ScrCycHdr *h = scr_cyc_hdr(shared); + unsigned gen = scr_cyc_gen(h); + check(scr_cyc_is_buffered(h), "the shared node is not a candidate"); + take(dead); + check(scr_cyc_collect_set(&taken, SCR_CYC_NURSERY) == RING, + "limited pass did not free the taken ring"); + check(scr_cyc_is_buffered(h) && scr_cyc_gen(h) == gen, + "limited pass moved a candidate it did not take out of its generation"); + memset(taken_slots, 0, sizeof taken_slots); + /* Its owner lets go: it dies directly, out of its buffer. */ + check(shared->rc == 1, "the shared node kept a reference from the freed ring"); + shared->rc--; + scr_cyc_on_dead(shared); + node_free(shared); + scr_collect_cycles(); + check(freed - before == RING + 1, "the passes after the limited one freed something else"); +} + /* A full pass whose fullest candidate buffer is the old generation's (more * entries than a pass keeps between passes), with nursery candidates * beside it: every dead ring in either buffer is reclaimed, the live rings @@ -1593,6 +1631,7 @@ int main(void) { check_allocation_growth_runs_passes(); check_publication_takes_candidates(); check_limited_pass(); + check_limited_pass_keeps_untaken_generation(); check_large_old_buffer_with_nursery_candidates(); check_unjudged_backlog_follows_growth(); check_acyclic_objects(); diff --git a/packages/runtime/test/test_heap_trim.c b/packages/runtime/test/test_heap_trim.c new file mode 100644 index 000000000..afd140430 --- /dev/null +++ b/packages/runtime/test/test_heap_trim.c @@ -0,0 +1,145 @@ +/* scr_heap_trim: after most blocks of a class are freed, the pages holding + * only free blocks go back to the system, the blocks still allocated keep + * their contents, and the class serves new blocks that overlap none of + * them. Built with and without SCR_WORKERS; with it, another thread + * allocates and frees its own blocks throughout, and frees blocks of the + * trimming thread before the trim (remote frees). */ +#include "../src/scr_runtime.h" + +#include +#include +#include +#include +#include +#ifdef SCR_WORKERS +#include +#include +#endif + +#define COUNT 400000 +#define SIZE 48 + +_Noreturn void scr_trap(const char *msg) { + fputs(msg, stderr); + exit(EXIT_FAILURE); +} + +/* One block kept per KEEP_PAGES pages of blocks, whatever the page size: a + * kept block pins its page, so the others hold only free blocks. */ +#define KEEP_PAGES 4 +static size_t keep_every; + +static void check(bool condition, const char *message) { + if (condition) return; + fprintf(stderr, "heap trim test failed: %s\n", message); + exit(EXIT_FAILURE); +} + +/* Resident bytes of this process. */ +static size_t resident(void) { + FILE *f = fopen("/proc/self/statm", "r"); + check(f != NULL, "open /proc/self/statm"); + unsigned long size = 0, pages = 0; + check(fscanf(f, "%lu %lu", &size, &pages) == 2, "read /proc/self/statm"); + fclose(f); + return (size_t)pages * (size_t)sysconf(_SC_PAGESIZE); +} + +#ifdef SCR_WORKERS +static atomic_bool churning = true; + +/* Allocates, fills and frees blocks of several sizes until told to stop. */ +static void *churn(void *arg) { + (void)arg; + void *held[256] = {0}; + for (size_t round = 0; atomic_load(&churning); round++) { + size_t slot = round % 256, size = 16 + (round % 7) * 24; + if (held[slot]) scr_mem_free(held[slot]); + held[slot] = scr_mem_alloc(size); + check(held[slot] != NULL, "allocate on the other thread"); + memset(held[slot], 0x33, size); + } + for (size_t i = 0; i < 256; i++) + if (held[i]) scr_mem_free(held[i]); + return NULL; +} + +/* Frees `count` blocks starting at `first` from another thread. */ +typedef struct { + unsigned char **blocks; + size_t first, count; +} RemoteFree; + +static void *free_remotely(void *arg) { + RemoteFree *r = arg; + for (size_t i = r->first; i < r->first + r->count; i++) + if (i % keep_every != 0) { + scr_mem_free(r->blocks[i]); + r->blocks[i] = NULL; + } + return NULL; +} +#endif + +static int compare(const void *a, const void *b) { + uintptr_t x = *(const uintptr_t *)a, y = *(const uintptr_t *)b; + return x < y ? -1 : x > y; +} + +int main(void) { + keep_every = KEEP_PAGES * ((size_t)sysconf(_SC_PAGESIZE) / SIZE + 1); +#ifdef SCR_WORKERS + pthread_t churner; + check(pthread_create(&churner, NULL, churn, NULL) == 0, "start the other thread"); +#endif + unsigned char **blocks = malloc(COUNT * sizeof *blocks); + check(blocks != NULL, "block table"); + for (size_t i = 0; i < COUNT; i++) { + blocks[i] = scr_mem_alloc(SIZE); + check(blocks[i] != NULL, "allocate"); + memset(blocks[i], (int)(i & 0xff), SIZE); + } + size_t kept = 0; + uintptr_t *keep = malloc((COUNT / keep_every + 1) * sizeof *keep); + check(keep != NULL, "keep table"); +#ifdef SCR_WORKERS + RemoteFree remote = {blocks, 0, COUNT / 4}; + pthread_t freer; + check(pthread_create(&freer, NULL, free_remotely, &remote) == 0, "start the freeing thread"); + check(pthread_join(freer, NULL) == 0, "join the freeing thread"); +#endif + for (size_t i = 0; i < COUNT; i++) { + if (i % keep_every == 0) { + keep[kept++] = (uintptr_t)blocks[i]; + continue; + } + if (blocks[i]) scr_mem_free(blocks[i]); + } + size_t before = resident(); + scr_heap_trim(); + size_t after = resident(); + check(before > after && before - after >= (size_t)8 << 20, "the free pages go back to the system"); + for (size_t i = 0; i < COUNT; i += keep_every) + for (size_t j = 0; j < SIZE; j++) check(blocks[i][j] == (unsigned char)(i & 0xff), "kept blocks keep their contents"); + qsort(keep, kept, sizeof *keep, compare); + for (size_t i = 0; i < COUNT; i++) { + if (i % keep_every == 0) continue; + unsigned char *p = scr_mem_alloc(SIZE); + check(p != NULL, "allocate after the trim"); + uintptr_t key = (uintptr_t)p; + check(bsearch(&key, keep, kept, sizeof *keep, compare) == NULL, "a new block is not a kept one"); + memset(p, 0x5a, SIZE); + blocks[i] = p; + } + for (size_t i = 0; i < COUNT; i += keep_every) + for (size_t j = 0; j < SIZE; j++) check(blocks[i][j] == (unsigned char)(i & 0xff), "kept blocks survive new allocations"); + for (size_t i = 0; i < COUNT; i++) scr_mem_free(blocks[i]); +#ifdef SCR_WORKERS + atomic_store(&churning, false); + check(pthread_join(churner, NULL) == 0, "join the other thread"); +#endif + free(keep); + free(blocks); + printf("heap trim checks passed\n"); + return 0; +} diff --git a/packages/runtime/vendor/README.md b/packages/runtime/vendor/README.md index 5f75b683b..53238410e 100644 --- a/packages/runtime/vendor/README.md +++ b/packages/runtime/vendor/README.md @@ -16,6 +16,13 @@ The x86_64 and AArch64 register-save/restore and `makecontext` logic is adapted The double-to-shortest-decimal core of Ryū (d2s and its headers only — the float and fixed/exponent variants are not vendored). scr_number.c textually `#include`s d2s.c and calls its internal `d2d` digit generator; ECMA-262 digit *placement* stays in scr_number.c. The only modification to upstream files: `#include "ryu/x.h"` flattened to `#include "x.h"` in d2s.c and d2s_intrinsics.h so the directory is self-contained. To update, re-fetch the listed files at the new commit and re-apply that include flattening. +## v8-ieee754/ (derived source) + +- Project: https://github.com/v8/v8, `src/base/ieee754.cc` as Node v24.22.0 bundles it (`deps/v8`, V8 13.6.233.17) +- License: BSD-3-Clause (V8), with fdlibm's notice (see v8-ieee754/LICENSE) + +V8's adaptation of fdlibm, the code behind Node's Math.sin, Math.exp, Math.cbrt and the other transcendental functions on every platform. `src/scr_ieee754.c` is a C port of the file, so compiled Math members print the values Node prints. The function bodies are unchanged; the port renames the entry points (`scr_ieee754_*`), gives the kernels internal linkage, and replaces V8's C++ helpers (bit casts, wraparound arithmetic, NaN and infinity constants) with C. `legacy::pow` is not ported: Node's Math.pow calls the platform pow. To update, re-fetch `deps/v8/src/base/ieee754.cc` at the new Node version and repeat the same conversion. + ## quickjs-ng/ - Project: https://github.com/quickjs-ng/quickjs diff --git a/packages/runtime/vendor/v8-ieee754/LICENSE b/packages/runtime/vendor/v8-ieee754/LICENSE new file mode 100644 index 000000000..9258b08c2 --- /dev/null +++ b/packages/runtime/vendor/v8-ieee754/LICENSE @@ -0,0 +1,75 @@ +This license applies to all parts of V8 that are not externally +maintained libraries. The externally maintained libraries used by V8 +are: + + - PCRE test suite, located in + test/mjsunit/third_party/regexp-pcre/regexp-pcre.js. This is based on the + test suite from PCRE-7.3, which is copyrighted by the University + of Cambridge and Google, Inc. The copyright notice and license + are embedded in regexp-pcre.js. + + - Layout tests, located in test/mjsunit/third_party/object-keys. These are + based on layout tests from webkit.org which are copyrighted by + Apple Computer, Inc. and released under a 3-clause BSD license. + + - Strongtalk assembler, the basis of the files assembler-arm-inl.h, + assembler-arm.cc, assembler-arm.h, assembler-ia32-inl.h, + assembler-ia32.cc, assembler-ia32.h, assembler-x64-inl.h, + assembler-x64.cc, assembler-x64.h, assembler.cc and assembler.h. + This code is copyrighted by Sun Microsystems Inc. and released + under a 3-clause BSD license. + + - Valgrind client API header, located at third_party/valgrind/valgrind.h + This is released under the BSD license. + + - The Wasm C/C++ API headers, located at third_party/wasm-api/wasm.{h,hh} + This is released under the Apache license. The API's upstream prototype + implementation also formed the basis of V8's implementation in + src/wasm/c-api.cc. + +These libraries have their own licenses; we recommend you read them, +as their terms may differ from the terms below. + +Further license information can be found in LICENSE files located in +sub-directories. + +Copyright 2014, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +src/scr_ieee754.c is a port of V8's src/base/ieee754.cc, which V8 adapted +from fdlibm. fdlibm's notice: + +==================================================== +Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + +Developed at SunSoft, a Sun Microsystems, Inc. business. +Permission to use, copy, modify, and distribute this +software is freely granted, provided that this notice +is preserved. +==================================================== diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5fc32620a..f4a645778 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -110,8 +110,8 @@ importers: specifier: workspace:* version: link:../runtime tsc-ts: - specifier: 0.0.9 - version: 0.0.9 + specifier: 0.0.10 + version: 0.0.10 typescript: specifier: 7.0.2 version: 7.0.2 @@ -1360,8 +1360,8 @@ packages: cpu: [x64] os: [win32] - '@scriptc/threads@0.4.2': - resolution: {integrity: sha512-nR9FUxpOpgtTmJu6dejttElBL+JJTAZzJ2DZM+eNoujmLx9VpdlAVdgj3cu/MDsTUg/1sIUBq1ME1csC/Aa6yg==} + '@scriptc/threads@0.4.5': + resolution: {integrity: sha512-wHYrCItM68xYAwjnvM6VrRLDWd9+lAEQNjKh6PyLA8RdE7D8ts/VMoTFLmXRu2uvRywJuY18WPSCoEFRf3qfiA==} engines: {node: '>=24.0.0'} '@sinclair/typebox@0.25.24': @@ -1377,42 +1377,42 @@ packages: '@ts-morph/common@0.11.1': resolution: {integrity: sha512-7hWZS0NRpEsNV8vWJzg7FEz6V8MaLNeJOmwmghqUXTpzk16V1LLZhdo+4QvE/+zv4cVci0OviuJFnqhEfoV3+g==} - '@tsc-ts/darwin-arm64@0.0.9': - resolution: {integrity: sha512-aIjSulef3DIEX7F3RRWLXyHWFgjXiVffBkrfE83ay/tpbsQaniu16cFxYGYNaqqVGft/oivecpdTDBbpPlsb8Q==} + '@tsc-ts/darwin-arm64@0.0.10': + resolution: {integrity: sha512-mRctKijCzWRxJgxiVAdsk+x9WB7kZz9GBbJ4eVP95bJFkIz9D11uw3xC0m6jWYNoaShF6zn48eLMTYOyJNxBdA==} cpu: [arm64] os: [darwin] - '@tsc-ts/darwin-x64@0.0.9': - resolution: {integrity: sha512-LHE74tzIEDtrEN8ENvCH/ejnnZjT0D2Vaz2HmggcvJudgr0W300CcwGf2Kdfw8MVBBk6JN0HPF9+a5AUUtFZGg==} + '@tsc-ts/darwin-x64@0.0.10': + resolution: {integrity: sha512-oV78/mi3ot8CG3BA9QxrsamvNXutpschgae+qV5skU4gAqimmBsBHy/MWTf9SCssvtWeqrX4uZvcwE0h/AJ+MQ==} cpu: [x64] os: [darwin] - '@tsc-ts/linux-arm64-gnu@0.0.9': - resolution: {integrity: sha512-dNiQIKApLcKIJPzb3pk2gfGrrgmOWa+q3/qjrx5ia/SzIWz0+lENlVvHiIn9ukqDacDE30FW8xHzby8Kyd+10A==} + '@tsc-ts/linux-arm64-gnu@0.0.10': + resolution: {integrity: sha512-O5msxFydn9r64QVm9WDWG8CKkXLjXWo12XeJWDncHlQYK80SoM4T8/C1UTDLtb6A2P+NQ5a+Xq4AaHmNb8VIqg==} cpu: [arm64] os: [linux] libc: [glibc] - '@tsc-ts/linux-arm64-musl@0.0.9': - resolution: {integrity: sha512-DolAqHtr1GUyG03Ch8+sJzpX1G+xzazAmj18kABzuOPre/6njrSWnmqJmtXQWvhwLBBEjajIkUJ4V+t5qigsrg==} + '@tsc-ts/linux-arm64-musl@0.0.10': + resolution: {integrity: sha512-v9N5HuqcEW97MzwzLqPNAN5tfAcXIAlLRtp7BSJJd5VKfBu6DplqNZzwCH+//NuAQOtWGSr/Dq565JFPmDRt5w==} cpu: [arm64] os: [linux] libc: [musl] - '@tsc-ts/linux-x64-gnu@0.0.9': - resolution: {integrity: sha512-leANVZ4tKz3myBSVogb173ttbAurPOOGjy1WeBlRhkVJ7zOPMiyzJi3cPOAK/4O8RcxXOsCiC2AfKPKDYM8bEw==} + '@tsc-ts/linux-x64-gnu@0.0.10': + resolution: {integrity: sha512-sRTPB/50uAJSpMjlA3sc9GTAySnLwIlf1txR8eHlcVpKE6q0QgWvEkKqCSGtFXkn8IQ8z8O68jrAk9WjmRTaYQ==} cpu: [x64] os: [linux] libc: [glibc] - '@tsc-ts/linux-x64-musl@0.0.9': - resolution: {integrity: sha512-a4FW16QXkrwNHAr4Qo/qDA5w9LxCJNrYbpqiz3TJ1sePTqvW/JTJc14tjx6avhW4mIaQp8AB52m7TqNAuDMm4Q==} + '@tsc-ts/linux-x64-musl@0.0.10': + resolution: {integrity: sha512-U4qZwP0JzrR/l4kzhh+w+y6o2guEEinC/wminfXXDhez2DtjNTOrM1GxMPfZ7JkCIcKEruL7iCC2telapFk5LQ==} cpu: [x64] os: [linux] libc: [musl] - '@tsc-ts/win32-x64-msvc@0.0.9': - resolution: {integrity: sha512-phThYnmOinlvY4dS4DEl1L9t/mFLiHR1zL5plSUfMB5NizA/jESfgMGRu+HW3uwOpCETSxgsuTNAXO7UuEIHIQ==} + '@tsc-ts/win32-x64-msvc@0.0.10': + resolution: {integrity: sha512-TsiK6k0ljFW3pNR5lvF7SL0nv3aGxr31ZbBCfIXcBrqAvpumINOy/G0ESfTvZWx8llwOQt3PxBS414gF/J0tFA==} cpu: [x64] os: [win32] @@ -2577,8 +2577,8 @@ packages: ts-toolbelt@6.15.5: resolution: {integrity: sha512-FZIXf1ksVyLcfr7M317jbB67XFJhOO1YqdTcuGaq9q5jLUoTikukZ+98TPjKiP2jC5CgmYdWWYs0s2nLSU0/1A==} - tsc-ts@0.0.9: - resolution: {integrity: sha512-7fDfC/o1YoITG6sSXj4ZznijQYg8AfoCagmYvwG1BFLR/hz4Lwnq7jqDwcZfoDuPYTkrs8jHUTAVG1turniJHA==} + tsc-ts@0.0.10: + resolution: {integrity: sha512-oF5hZbeNgJXwJ868adnvBAihxYUwra1rj6Sa2+UhuhW477t5MmGnOsZgGTSGz511uTfuUxDz6e/gLIzvvScbGw==} engines: {node: '>=20'} hasBin: true @@ -3439,7 +3439,7 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.62.2': optional: true - '@scriptc/threads@0.4.2': {} + '@scriptc/threads@0.4.5': {} '@sinclair/typebox@0.25.24': {} @@ -3454,25 +3454,25 @@ snapshots: mkdirp: 1.0.4 path-browserify: 1.0.1 - '@tsc-ts/darwin-arm64@0.0.9': + '@tsc-ts/darwin-arm64@0.0.10': optional: true - '@tsc-ts/darwin-x64@0.0.9': + '@tsc-ts/darwin-x64@0.0.10': optional: true - '@tsc-ts/linux-arm64-gnu@0.0.9': + '@tsc-ts/linux-arm64-gnu@0.0.10': optional: true - '@tsc-ts/linux-arm64-musl@0.0.9': + '@tsc-ts/linux-arm64-musl@0.0.10': optional: true - '@tsc-ts/linux-x64-gnu@0.0.9': + '@tsc-ts/linux-x64-gnu@0.0.10': optional: true - '@tsc-ts/linux-x64-musl@0.0.9': + '@tsc-ts/linux-x64-musl@0.0.10': optional: true - '@tsc-ts/win32-x64-msvc@0.0.9': + '@tsc-ts/win32-x64-msvc@0.0.10': optional: true '@tybys/wasm-util@0.10.3': @@ -4813,17 +4813,17 @@ snapshots: ts-toolbelt@6.15.5: {} - tsc-ts@0.0.9: + tsc-ts@0.0.10: dependencies: - '@scriptc/threads': 0.4.2 + '@scriptc/threads': 0.4.5 optionalDependencies: - '@tsc-ts/darwin-arm64': 0.0.9 - '@tsc-ts/darwin-x64': 0.0.9 - '@tsc-ts/linux-arm64-gnu': 0.0.9 - '@tsc-ts/linux-arm64-musl': 0.0.9 - '@tsc-ts/linux-x64-gnu': 0.0.9 - '@tsc-ts/linux-x64-musl': 0.0.9 - '@tsc-ts/win32-x64-msvc': 0.0.9 + '@tsc-ts/darwin-arm64': 0.0.10 + '@tsc-ts/darwin-x64': 0.0.10 + '@tsc-ts/linux-arm64-gnu': 0.0.10 + '@tsc-ts/linux-arm64-musl': 0.0.10 + '@tsc-ts/linux-x64-gnu': 0.0.10 + '@tsc-ts/linux-x64-musl': 0.0.10 + '@tsc-ts/win32-x64-msvc': 0.0.10 tslib@2.8.1: optional: true diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c9f873c4e..b678a8730 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,15 +7,15 @@ minimumReleaseAgeExclude: - "effect@4.0.0" - "@effect/platform-node@4.0.0" - "@effect/platform-node-shared@4.0.0" - - "@tsc-ts/darwin-arm64@0.0.9" - - "@tsc-ts/darwin-x64@0.0.9" - - "@tsc-ts/linux-arm64-gnu@0.0.9" - - "@tsc-ts/linux-arm64-musl@0.0.9" - - "@tsc-ts/linux-x64-gnu@0.0.9" - - "@tsc-ts/linux-x64-musl@0.0.9" - - "@tsc-ts/win32-x64-msvc@0.0.9" - - "tsc-ts@0.0.9" - - "@scriptc/threads@0.4.2" + - "@tsc-ts/darwin-arm64@0.0.10" + - "@tsc-ts/darwin-x64@0.0.10" + - "@tsc-ts/linux-arm64-gnu@0.0.10" + - "@tsc-ts/linux-arm64-musl@0.0.10" + - "@tsc-ts/linux-x64-gnu@0.0.10" + - "@tsc-ts/linux-x64-musl@0.0.10" + - "@tsc-ts/win32-x64-msvc@0.0.10" + - "tsc-ts@0.0.10" + - "@scriptc/threads@0.4.5" allowBuilds: esbuild: true overrides: diff --git a/scripts/bench-runtime.mjs b/scripts/bench-runtime.mjs index d89a4e759..0845d57b8 100644 --- a/scripts/bench-runtime.mjs +++ b/scripts/bench-runtime.mjs @@ -42,6 +42,8 @@ const { values } = parseArgs({ "candidate-optimization": { type: "string", default: "release" }, "baseline-target-cpu": { type: "string" }, "candidate-target-cpu": { type: "string" }, + "baseline-cpu-dispatch": { type: "boolean", default: false }, + "candidate-cpu-dispatch": { type: "boolean", default: false }, json: { type: "string" }, "no-lock": { type: "boolean", default: false }, node: { type: "boolean", default: true }, @@ -54,6 +56,7 @@ if (values.help) { [--workloads=a,b] [--runs=15] [--warmup=2] [--timeout=120] [--layouts=1] [--json=] [--baseline-optimization=release] [--candidate-optimization=release] [--baseline-target-cpu=] [--candidate-target-cpu=] + [--baseline-cpu-dispatch] [--candidate-cpu-dispatch] [--no-lock] [--no-node] [--keep] Each checkout must have a built CLI (packages/cli/dist/bootstrap.js) and native @@ -74,8 +77,9 @@ Workload names are the file names in benchmarks/runtime/workloads/*.json; for ex contenders, e.g. --baseline=. --candidate-optimization=speed compares speed with release. --baseline-target-cpu/--candidate-target-cpu pass --target-cpu to each contender's builds, e.g. --baseline=. --candidate-target-cpu=x86-64-v3 compares -x86-64-v3 code with the baseline. cold_build_ms records each contender's first, -uncached build.`); +x86-64-v3 code with the baseline. --baseline-cpu-dispatch/--candidate-cpu-dispatch +pass --cpu-dispatch, and SCRIPTC_CPU_LEVEL in the environment reaches every timed +run. cold_build_ms records each contender's first, uncached build.`); process.exit(0); } const runs = Number(values.runs); @@ -444,7 +448,7 @@ function revision(root) { }); return r.stdout.trim() + (dirty.stdout.trim() ? "+dirty" : ""); } -function build(label, root, workload, layout, optimization, targetCpu) { +function build(label, root, workload, layout, optimization, targetCpu, cpuDispatch) { const cli = cliFor(root); const out = join(work, label, layout === 0 ? workload.name : `${workload.name}.layout${layout}`); mkdirSync(dirname(out), { recursive: true }); @@ -466,6 +470,7 @@ function build(label, root, workload, layout, optimization, targetCpu) { entry, `--optimization=${optimization}`, ...(targetCpu === undefined ? [] : [`--target-cpu=${targetCpu}`]), + ...(cpuDispatch ? ["--cpu-dispatch"] : []), "-o", out, ], @@ -492,6 +497,7 @@ const contenders = [ root: resolve(values.baseline), optimization: values["baseline-optimization"], targetCpu: values["baseline-target-cpu"], + cpuDispatch: values["baseline-cpu-dispatch"], }, ] : []), @@ -500,6 +506,7 @@ const contenders = [ root: resolve(values.candidate), optimization: values["candidate-optimization"], targetCpu: values["candidate-target-cpu"], + cpuDispatch: values["candidate-cpu-dispatch"], }, ]; const report = { @@ -606,7 +613,7 @@ try { entry.results[c.label] = result; result.binaries = []; for (let layout = 0; layout < layouts && result.status === undefined; layout++) { - const built = build(c.label, c.root, w, layout, c.optimization, c.targetCpu); + const built = build(c.label, c.root, w, layout, c.optimization, c.targetCpu, c.cpuDispatch); result.build_ms += Math.round(built.buildMs); if (layout === 0) result.cold_build_ms = Math.round(built.buildMs); if (!built.ok) { diff --git a/scripts/bootstrap-seed.mjs b/scripts/bootstrap-seed.mjs index f0993f126..6bce075f5 100644 --- a/scripts/bootstrap-seed.mjs +++ b/scripts/bootstrap-seed.mjs @@ -25,6 +25,19 @@ export function verifyBootstrapSeed(directory, revision, sanitize) { } } +/** The environment a bootstrap seed builds with, over the inherited one. The + * bootstrap compares the seed's LLVM with the native compiler's own emission, + * which has no function layout. The layout only orders the functions, so the + * seed is built without it. */ +export function seedBuildEnvironment(sanitize) { + return { + SCRIPTC_SAN: sanitize ? "1" : "", + SCRIPTC_NATIVE_EMIT_IR: "1", + SCRIPTC_NATIVE_OPTIMIZATION: sanitize ? "dev" : "release", + SCRIPTC_NATIVE_LAYOUT: "", + }; +} + async function main() { const directory = process.env.SCRIPTC_BOOTSTRAP_SEED_DIRECTORY; const revision = process.env.GITHUB_SHA; @@ -47,11 +60,7 @@ async function main() { { cwd: root, stdio: "inherit", - env: { - ...process.env, - SCRIPTC_NATIVE_EMIT_IR: "1", - SCRIPTC_NATIVE_OPTIMIZATION: sanitize ? "dev" : "release", - }, + env: { ...process.env, ...seedBuildEnvironment(sanitize) }, }, ); writeFileSync( diff --git a/scripts/build-llvm-dist.mjs b/scripts/build-llvm-dist.mjs new file mode 100644 index 000000000..9398b46a4 --- /dev/null +++ b/scripts/build-llvm-dist.mjs @@ -0,0 +1,96 @@ +#!/usr/bin/env node +/* Builds the LLVM distribution the Linux release helpers link + * (native/llvm-codegen/llvm-dist.mjs): LLVM_VERSION from its release source + * tarball, with ThinLTO and native/llvm-codegen/llvm.profdata.gz, the + * helper's targets, and static libraries only. clang and lld of the same + * version must be on PATH (LLVM's release archive for the host has both). + * Writes the install tree and, with --archive, a .tar.xz of it and its + * .sha256. The llvm-dist workflow runs it for each Linux architecture; it + * takes about 8 minutes on 32 cores and about an hour on 4. + * + * With --instrumented it builds the IR-instrumented libraries that + * scripts/refresh-llvm-profile.mjs collects the profile with instead. + * + * node scripts/build-llvm-dist.mjs --source --prefix + * [--archive ] [--work ] [--jobs N] [--instrumented] + */ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { cpus, tmpdir } from "node:os"; +import { basename, dirname, join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { llvmCmakeArgs, writeLlvmProfile } from "../native/llvm-codegen/llvm-dist.mjs"; + +const { values } = parseArgs({ + options: { + source: { type: "string" }, + prefix: { type: "string" }, + archive: { type: "string" }, + work: { type: "string" }, + jobs: { type: "string", default: String(cpus().length) }, + instrumented: { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + }, +}); +if (values.help || values.source === undefined || values.prefix === undefined) { + console.log( + "Usage: node scripts/build-llvm-dist.mjs --source --prefix [--archive ] [--work ] [--jobs N] [--instrumented]", + ); + process.exit(values.help ? 0 : 2); +} +if (process.platform !== "linux") throw new Error("the LLVM distribution is built on Linux"); +const source = resolve(values.source); +if (!existsSync(join(source, "llvm/CMakeLists.txt"))) + throw new Error(`${source} is not an llvm-project source tree`); +const prefix = resolve(values.prefix); +const work = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-llvm-dist-"))); +const build = join(work, values.instrumented ? "llvm-instrumented" : "llvm"); +mkdirSync(build, { recursive: true }); + +function run(command, args, log, env = process.env) { + console.log(`$ ${command} ${args.join(" ")}`); + const result = spawnSync(command, args, { encoding: "utf8", maxBuffer: 1 << 30, env }); + if (log !== undefined) writeFileSync(log, result.stdout + result.stderr); + if (result.status !== 0) { + process.stderr.write((result.stdout + result.stderr).slice(-8000)); + throw new Error(`${command} failed with status ${result.status}`); + } + return result.stdout + result.stderr; +} + +const profile = values.instrumented ? undefined : writeLlvmProfile(work); +run("cmake", llvmCmakeArgs({ source, build, prefix, profile, instrumented: values.instrumented })); +const log = join(work, "build.log"); +const output = run("ninja", ["-C", build, "-j", values.jobs, "install"], log); + +if (profile !== undefined) { + // clang reports, per file, how many of its functions the profile no longer + // matches; on a host other than the one it was collected on, that count + // shows how much of the profile still applies. + let files = 0, + functions = 0, + mismatched = 0; + for (const match of output.matchAll(/of (\d+) functions?, (\d+) ha(?:ve|s) mismatched data/g)) { + files++; + functions += Number(match[1]); + mismatched += Number(match[2]); + } + console.log( + files === 0 + ? "profile: no file reports functions with mismatched profile data" + : `profile: ${mismatched} of ${functions} functions in ${files} files have mismatched profile data (build log ${log})`, + ); +} + +if (values.archive !== undefined) { + const archive = resolve(values.archive); + mkdirSync(dirname(archive), { recursive: true }); + run("tar", ["-C", dirname(prefix), "-cJf", archive, basename(prefix)], undefined, { + ...process.env, + XZ_OPT: "-T0", + }); + const digest = createHash("sha256").update(readFileSync(archive)).digest("hex"); + writeFileSync(`${archive}.sha256`, `${digest} ${basename(archive)}\n`); + console.log(`wrote ${archive} (${digest})`); +} diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index 4d3f00ff5..e76065880 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -1,12 +1,14 @@ /** Build the production CLI seed and its relocatable compiler assets. */ -import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { basename, dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { gunzipSync } from "node:zlib"; import { compile } from "../packages/compiler/src/index.js"; import { NATIVE_TARGETS, nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js"; import { tsgoExecutable } from "../packages/compiler/src/frontend/ts7/rpc-api.js"; +import { prebuiltLibDiagnosticsText } from "../packages/compiler/src/frontend/ts7/lib-diagnostics.js"; import { stageNativeCliAssets } from "./native-cli-assets.mjs"; import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js"; @@ -35,6 +37,15 @@ const useProfile = !sanitize && requestedProfile !== "" && (requestedProfile !== undefined || existsSync(committedProfile)); +// Builds lay out the compiler's functions with the committed layout +// (scripts/refresh-cli-layout.mjs), or the one SCRIPTC_NATIVE_LAYOUT names; +// an empty SCRIPTC_NATIVE_LAYOUT, and instrumented builds, use none. +const committedLayout = join(root, "packages/compiler/native/cli.layout.gz"); +const requestedLayout = process.env["SCRIPTC_NATIVE_LAYOUT"]; +const functionLayout = + profileGenerate !== undefined || requestedLayout === "" + ? undefined + : (requestedLayout ?? (existsSync(committedLayout) ? committedLayout : undefined)); let target = nativeCodegenTarget(); if (target === null || target.platform === "wasi") throw new Error("a supported native host is required to build the compiler"); @@ -124,6 +135,7 @@ const result = await compile(join(root, "packages/compiler/src/native/cli.ts"), sanitize, ...(profileGenerate === undefined ? {} : { profileGenerate }), ...(profileUse === undefined ? {} : { profileUse }), + ...(functionLayout === undefined ? {} : { functionLayout }), emitIr: process.env["SCRIPTC_NATIVE_EMIT_IR"] === "1", }); if (!result.ok) @@ -181,4 +193,60 @@ const manifest: NativeToolchainManifest = { comptime: relative(bin, comptime), }; writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n"); +// The prebuilt lib diagnostics (frontend/ts7/lib-diagnostics.ts): the +// declaration file sets the distribution's tsc-ts reports no diagnostics in +// for programs without a tsconfig, recorded by the built compiler itself. A +// compiler this host cannot run ships without them. +const prebuiltKeys = recordPrebuiltLibDiagnostics(executable); +if (prebuiltKeys === null) { + console.warn( + "warning: the built compiler did not run here; it ships without prebuilt lib diagnostics", + ); +} else { + const prebuilt = join(lib, "declarations", "lib-diagnostics.json"); + const checker = ( + JSON.parse(readFileSync(join(lib, "tsc-ts", "package.json"), "utf8")) as { version: string } + ).version; + writeFileSync(prebuilt, prebuiltLibDiagnosticsText(checker, prebuiltKeys)); + manifest.lib_diagnostics = relative(bin, prebuilt); + writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n"); +} console.log(executable); + +/** The content keys the built compiler records building, each in an empty + * cache, a module without imports (the program it lowers) and a module with + * a type error (which also checks the program without scriptc's overrides); + * null when it cannot run. */ +function recordPrebuiltLibDiagnostics(compiler: string): string[] | null { + const work = mkdtempSync(join(tmpdir(), "scriptc-lib-diagnostics-")); + try { + const record = join(work, "keys.txt"); + const probes = [ + { name: "clean", source: "export {};\n", status: 0 }, + { name: "error", source: 'export const n: number = "x";\n', status: 1 }, + ]; + for (const probe of probes) { + const dir = join(work, probe.name); + mkdirSync(dir); + writeFileSync(join(dir, "main.ts"), probe.source); + const env: NodeJS.ProcessEnv = { + ...process.env, + SCRIPTC_CACHE_DIR: join(dir, "cache"), + SCRIPTC_LIB_DIAGNOSTICS_RECORD: record, + }; + delete env["SCRIPTC_TIMING"]; + const run = spawnSync( + compiler, + ["build", join(dir, "main.ts"), "--optimization", "dev", "-o", join(dir, "main")], + { cwd: dir, env, encoding: "utf8" }, + ); + if (run.status !== probe.status) return null; + } + if (!existsSync(record)) return null; + return readFileSync(record, "utf8") + .split("\n") + .filter((key) => key !== ""); + } finally { + rmSync(work, { recursive: true, force: true }); + } +} diff --git a/scripts/check-llvm-dist.mjs b/scripts/check-llvm-dist.mjs new file mode 100644 index 000000000..cce490c9a --- /dev/null +++ b/scripts/check-llvm-dist.mjs @@ -0,0 +1,105 @@ +#!/usr/bin/env node +/* Compares two LLVM helpers on the helper inputs of this checkout + * (scripts/llvm-helper-inputs.mjs): every input must give identical objects + * with both, and the script prints each input group's median wall time per + * helper over the rounds, with the candidate/baseline ratio. The llvm-dist + * workflow runs it with the helper linked against LLVM's release archive as + * the baseline and the one linked against the profiled ThinLTO distribution + * as the candidate, on each architecture it builds. Exits with status 1 when + * an object differs. + * + * node scripts/check-llvm-dist.mjs --baseline --candidate + * [--rounds N] [--work ] [--tsc-ts ] [--extra ] + */ +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import { parseArgs } from "node:util"; +import { captureHelperInputs, helperInputPrograms } from "./llvm-helper-inputs.mjs"; + +const { values } = parseArgs({ + options: { + baseline: { type: "string" }, + candidate: { type: "string" }, + rounds: { type: "string", default: "3" }, + work: { type: "string" }, + "tsc-ts": { type: "string" }, + extra: { type: "string", multiple: true, default: [] }, + help: { type: "boolean", default: false }, + }, +}); +if (values.help || values.baseline === undefined || values.candidate === undefined) { + console.log( + "Usage: node scripts/check-llvm-dist.mjs --baseline --candidate [--rounds N] [--work ] [--tsc-ts ] [--extra ]", + ); + process.exit(values.help ? 0 : 2); +} +const helpers = { baseline: resolve(values.baseline), candidate: resolve(values.candidate) }; +const work = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-llvm-check-"))); +const inputs = await captureHelperInputs( + helperInputPrograms({ + tscTs: values["tsc-ts"] === undefined ? undefined : resolve(values["tsc-ts"]), + extra: values.extra.map((dir) => resolve(dir)), + }), + join(work, "inputs"), +); + +function emit(helper, input, out) { + rmSync(out, { recursive: true, force: true }); + mkdirSync(out, { recursive: true }); + const start = performance.now(); + const result = spawnSync(helper, input.args(out), { encoding: "utf8" }); + const ms = performance.now() - start; + if (result.status !== 0) + throw new Error(`${helper} failed on ${input.name}: ${result.stdout}${result.stderr}`); + return ms; +} + +const times = new Map(); // `${group}|${helper}` -> per-round totals +const group = (name) => (name === "tsc-ts" ? name : name.split("-")[0]); +const rounds = Number(values.rounds); +let differing = 0; +for (let round = 0; round < rounds; round++) { + const order = round % 2 === 0 ? ["baseline", "candidate"] : ["candidate", "baseline"]; + for (const input of inputs) { + for (const which of order) { + const ms = emit(helpers[which], input, join(work, which)); + const key = `${group(input.name)}|${which}`; + const totals = times.get(key) ?? new Array(rounds).fill(0); + totals[round] += ms; + times.set(key, totals); + } + if (round === 0) { + for (const file of readdirSync(join(work, "baseline"))) { + const a = readFileSync(join(work, "baseline", file)); + const b = readFileSync(join(work, "candidate", file)); + if (!a.equals(b)) { + differing++; + console.log(`${input.name}: ${file} differs between the helpers`); + } + } + } + } +} + +const median = (xs) => { + const s = [...xs].sort((x, y) => x - y); + return s.length % 2 ? s[s.length >> 1] : (s[s.length / 2 - 1] + s[s.length / 2]) / 2; +}; +console.log(`${inputs.length} inputs, ${rounds} rounds; median wall time per group`); +console.log("| group | baseline | candidate | candidate/baseline |"); +console.log("| --- | ---: | ---: | ---: |"); +for (const name of [...new Set(inputs.map((input) => group(input.name)))]) { + const b = median(times.get(`${name}|baseline`)), + c = median(times.get(`${name}|candidate`)); + console.log( + `| ${name} | ${(b / 1000).toFixed(2)} s | ${(c / 1000).toFixed(2)} s | ${(c / b).toFixed(3)} |`, + ); +} +if (differing > 0) { + console.log(`${differing} objects differ between the helpers`); + process.exit(1); +} +console.log("every object is identical between the helpers"); diff --git a/scripts/llvm-helper-inputs.mjs b/scripts/llvm-helper-inputs.mjs new file mode 100644 index 000000000..160d7b50e --- /dev/null +++ b/scripts/llvm-helper-inputs.mjs @@ -0,0 +1,114 @@ +/* Inputs of the LLVM helper, captured once and replayed with any helper: + * scripts/refresh-llvm-profile.mjs trains the helper's profile on them and + * scripts/check-llvm-dist.mjs compares helpers on them. Each input is a + * program's LLVM text from `scriptc build --emit llvm` and the `emit` + * arguments the compiler passes for an executable build of it + * (native-codegen.ts), partitions included. The partition cache is left out, + * so every replay compiles every partition. */ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const EFFECT_FIXTURES = [ + "async-sleep", + "async-promise-typed", + "async-timeout", + "async-resume-timer", + "async-yield", +]; + +/** The programs: tsc-ts (when a checkout is given), five Effect fixtures, + * the runtime benchmark workloads, every 50th numbered corpus program, and + * `extra` directories of standalone programs. */ +export function helperInputPrograms({ tscTs, extra = [] } = {}) { + const programs = []; + if (tscTs !== undefined) + programs.push({ name: "tsc-ts", cwd: join(tscTs, "packages/tsc-ts"), entry: "src/main.ts" }); + for (const fixture of EFFECT_FIXTURES) + programs.push({ + name: `effect-${fixture}`, + cwd: root, + entry: `tests/fixtures/effect/${fixture}.ts`, + args: ["--npm-static", "effect,fast-check,pure-rand"], + }); + const suite = join(root, "benchmarks/runtime"); + for (const file of readdirSync(join(suite, "workloads")) + .filter((f) => f.endsWith(".json")) + .sort()) { + const { entry } = JSON.parse(readFileSync(join(suite, "workloads", file), "utf8")); + // Variants of a workload share its entry and differ only in arguments. + if (!programs.some((program) => program.cwd === suite && program.entry === entry)) + programs.push({ name: `wl-${file.slice(0, -5)}`, cwd: suite, entry }); + } + const corpus = join(root, "tests/corpus"); + for (const file of readdirSync(corpus).sort()) { + const number = /^(\d+)-.*\.ts$/.exec(file)?.[1]; + if (number !== undefined && Number(number) % 50 === 0) + programs.push({ name: `corpus-${file.slice(0, -3)}`, cwd: corpus, entry: file }); + } + for (const dir of extra) + for (const file of readdirSync(dir) + .filter((f) => f.endsWith(".ts")) + .sort()) + programs.push({ name: `extra-${file.slice(0, -3)}`, cwd: dir, entry: file }); + return programs; +} + +/** Writes `/.ll` for each program with this checkout's compiler + * and returns the inputs that compiled: `{ name, input, outputs, args(out) }`, + * where `args(out)` gives the helper's arguments writing into `out`. */ +export async function captureHelperInputs(programs, dir) { + const { nativeProgramPartitions } = await import( + join(root, "packages/compiler/dist/backend/native-codegen-core.js") + ); + const cli = join(root, "packages/cli/dist/main.js"); + const target = + process.arch === "arm64" ? "aarch64-unknown-linux-gnu" : "x86_64-unknown-linux-gnu"; + mkdirSync(dir, { recursive: true }); + const inputs = []; + for (const program of programs) { + const input = join(dir, `${program.name}.ll`); + if (!existsSync(input)) { + const built = spawnSync( + process.execPath, + [cli, "build", program.entry, "--emit", "llvm", "-o", input, ...(program.args ?? [])], + { cwd: program.cwd, encoding: "utf8" }, + ); + if (built.status !== 0) { + console.log(`skipped ${program.name}: the compiler refused it`); + continue; + } + } + const partitions = nativeProgramPartitions(statSync(input).size); + const source = join(program.cwd, program.entry); + inputs.push({ + name: program.name, + input, + partitions, + args: (out) => [ + "emit", + "--input", + input, + ...Array.from({ length: partitions }, (_, i) => [ + "--output", + join(out, `${program.name}.part${i}.o`), + ]).flat(), + "--filetype", + "obj", + "--target", + target, + "--opt-level", + "2", + "--relocation-model", + "pic", + "--diagnostic-format", + "json", + "--source-path", + source, + ], + }); + } + return inputs; +} diff --git a/scripts/refresh-cli-layout.mjs b/scripts/refresh-cli-layout.mjs new file mode 100644 index 000000000..cb396015f --- /dev/null +++ b/scripts/refresh-cli-layout.mjs @@ -0,0 +1,275 @@ +#!/usr/bin/env node +/* Refreshes packages/compiler/native/cli.layout.gz, the function layout that + * builds of the native compiler use (scripts/build-native-cli.mts). The + * layout lists the functions the compiler runs, in tiers: tier 1 for + * `--version` and rebuilds of unchanged programs, then the functions of the + * other builds by the share of training builds that run them (COLD_TIERS). + * The emitter writes the listed functions first, by tier, so the code a build + * runs sits together in the compiler's executable and a run maps fewer of its + * pages: a function nearly every build runs sits with the others that do, not + * among functions only a few programs need. + * + * Builds an instrumented native compiler for this Linux host (or reuses one + * with --instrumented), builds each training program cold and then again + * unchanged, and reads the functions each kind of run executed from the raw + * profiles. The training programs are the runtime benchmark workloads at + * `speed` and every seventh corpus program at the default optimization. A + * refresh takes about 15 minutes on 16 cores, most of it for the instrumented + * build. A stale layout lists some functions that no longer exist and misses + * new ones, which then keep their usual order. + * + * node scripts/refresh-cli-layout.mjs [--jobs 8] [--instrumented ] + */ +import { spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + appendFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { cpus, tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { constants, gzipSync, inflateSync } from "node:zlib"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const { values } = parseArgs({ + options: { + jobs: { type: "string", default: String(Math.max(1, Math.floor(cpus().length / 2))) }, + output: { type: "string", default: join(root, "packages/compiler/native/cli.layout.gz") }, + work: { type: "string" }, + instrumented: { type: "string" }, + help: { type: "boolean", default: false }, + }, +}); +if (values.help) { + console.log( + "Usage: node scripts/refresh-cli-layout.mjs [--jobs N] [--output ] [--work ] [--instrumented ]", + ); + process.exit(0); +} +if (process.platform !== "linux" || process.arch !== "x64") + throw new Error("the compiler layout is collected on Linux x64"); +const jobs = Number(values.jobs); +const output = resolve(values.output); +mkdirSync(dirname(output), { recursive: true }); +const work = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-cli-layout-"))); +const raw = join(work, "raw"); +const builds = join(work, "builds"); +rmSync(raw, { recursive: true, force: true }); +for (const dir of [raw, builds]) mkdirSync(dir, { recursive: true }); +const log = join(work, "training.log"); + +const cli = resolve(values.instrumented ?? join(work, "cli")); +if (values.instrumented === undefined) { + console.log(`building the instrumented compiler in ${cli}`); + const built = spawnSync( + process.execPath, + [ + "--max-old-space-size=8192", + "--import", + "tsx", + join(root, "scripts/build-native-cli.mts"), + cli, + ], + { + cwd: root, + stdio: "inherit", + env: { ...process.env, SCRIPTC_NATIVE_PROFILE_GENERATE: join(raw, "build-%4m.profraw") }, + }, + ); + if (built.status !== 0) throw new Error("the instrumented compiler did not build"); +} +const scriptc = join(cli, "bin/scriptc"); +// Phase timing shows whether a rebuild hit both caches. It is on for every +// run, so the rebuild's environment matches the first build's. +const baseEnv = { ...process.env, SCRIPTC_TIMING: "1" }; +delete baseEnv["SCRIPTC_LLVM_HELPER"]; + +/** Runs the instrumented compiler, writing its profile to `options.profile`. */ +function run(args, options) { + return new Promise((done) => { + const child = spawn(scriptc, args, { + cwd: options.cwd, + stdio: ["ignore", "pipe", "pipe"], + env: { + ...baseEnv, + SCRIPTC_CACHE_DIR: options.cache, + LLVM_PROFILE_FILE: options.profile, + }, + }); + let text = ""; + child.stdout.on("data", (chunk) => (text += chunk)); + child.stderr.on("data", (chunk) => (text += chunk)); + child.on("close", (code) => { + appendFileSync(log, `$ scriptc ${args.join(" ")}\n${text}exit ${code}\n`); + done({ ok: code === 0, text }); + }); + }); +} + +async function pool(tasks) { + let next = 0; + let passed = 0; + await Promise.all( + Array.from({ length: jobs }, async () => { + while (next < tasks.length) if (await tasks[next++]()) passed++; + }), + ); + return passed; +} + +const programNumber = (name) => Number.parseInt(name, 10); +const training = [ + ...readdirSync(join(root, "benchmarks/runtime/workloads")) + .filter((name) => name.endsWith(".ts")) + .map((name) => ({ + file: join(root, "benchmarks/runtime/workloads", name), + optimization: "speed", + })), + ...readdirSync(join(root, "tests/corpus")) + .filter((name) => /^\d+-.*\.ts$/.test(name) && programNumber(name) % 7 === 0) + .map((name) => ({ file: join(root, "tests/corpus", name), optimization: "release" })), +]; + +// Tier 1 collects each run that took the warm path from its own profile; a +// rebuild that missed a cache (it shows no `frontend-cache-hit` or +// `executable-cache-hit` phase) is left out. The first builds each write +// their own profile (cold--*), merged across the build's processes. +const warm = new Set(); +let skipped = 0; +async function warmRun(args, cache, name, cacheHits) { + const profile = join(raw, `warm-${name}.profraw`); + const { ok, text } = await run(args, { cwd: root, cache, profile }); + if (ok && cacheHits.every((phase) => text.includes(`"phase":"${phase}"`))) + for (const fn of executedFunctions(profile)) warm.add(fn); + else skipped++; + rmSync(profile, { force: true }); +} + +const started = Date.now(); +await warmRun(["--version"], join(work, "cache-version"), "version", []); +console.log(`training: ${training.length} programs, cold then unchanged, ${jobs} at a time`); +const passed = await pool( + training.map((program, index) => async () => { + const cache = join(work, `cache-${index}`); + const out = join(builds, `program-${index}`); + const args = ["build", program.file, `--optimization=${program.optimization}`, "-o", out]; + const { ok } = await run(args, { + cwd: root, + cache, + profile: join(raw, `cold-${index}-%1m.profraw`), + }); + if (ok) + await warmRun(args, cache, String(index), ["frontend-cache-hit", "executable-cache-hit"]); + rmSync(cache, { recursive: true, force: true }); + rmSync(out, { force: true }); + return ok; + }), +); +console.log( + `trained in ${Math.round((Date.now() - started) / 1000)} s; ${passed} of ${training.length} programs built, ${skipped} rebuilds left out of tier 1 (see ${log})`, +); + +/** Functions a raw profile (LLVM's format version 10) records as run: the + * names whose counters are not all zero, without the file prefix of + * internal functions. */ +function executedFunctions(path) { + const buf = readFileSync(path); + const u64 = (offset) => Number(buf.readBigUInt64LE(offset)); + if (buf.readBigUInt64LE(0) !== 0xff6c70726f667281n) + throw new Error(`${path} is not a 64-bit raw profile`); + const version = Number(buf.readBigUInt64LE(8) & 0xffffffffn); + if (version !== 10) throw new Error(`${path} has raw profile version ${version}, expected 10`); + const [binaryIdsBytes, records, beforeCounters, counters, afterCounters] = [2, 3, 4, 5, 6].map( + (field) => u64(field * 8), + ); + const [bitmapBytes, afterBitmap, namesBytes] = [7, 8, 9].map((field) => u64(field * 8)); + const valueKinds = u64(15 * 8) + 1; + const recordBytes = Math.ceil((6 * 8 + 4 + 2 * valueKinds + 4) / 8) * 8; + const dataStart = 16 * 8 + binaryIdsBytes; + const countersStart = dataStart + records * recordBytes + beforeCounters; + const namesStart = countersStart + counters * 8 + afterCounters + bitmapBytes + afterBitmap; + const countersDelta = buf.readBigInt64LE(10 * 8); + const names = new Map(); + let at = namesStart; + const uleb = () => { + let value = 0; + let scale = 1; + for (;;) { + const byte = buf[at++]; + value += (byte & 0x7f) * scale; + if ((byte & 0x80) === 0) return value; + scale *= 128; + } + }; + while (at < namesStart + namesBytes) { + const plain = uleb(); + const packed = uleb(); + if (plain === 0) break; + const bytes = + packed === 0 ? buf.subarray(at, at + plain) : inflateSync(buf.subarray(at, at + packed)); + at += packed === 0 ? plain : packed; + for (const name of bytes.toString("latin1").split("\x01")) + names.set(createHash("md5").update(name, "latin1").digest().readBigUInt64LE(0), name); + } + const ran = []; + for (let index = 0; index < records; index++) { + const record = dataStart + index * recordBytes; + const counterOffset = Number( + buf.readBigInt64LE(record + 16) - (countersDelta - BigInt(index * recordBytes)), + ); + const count = buf.readUInt32LE(record + 48); + for (let c = 0; c < count; c++) { + if (buf.readBigUInt64LE(countersStart + counterOffset + c * 8) !== 0n) { + const name = names.get(buf.readBigUInt64LE(record)); + if (name !== undefined) ran.push(name.slice(name.indexOf(";") + 1)); + break; + } + } + } + return ran; +} + +// The cold tiers, by the least share of training builds that run a function: +// tier 2 for functions at least 90% of the builds run, tier 3 for 60%, and so +// on; tier 7 holds the functions fewer than 3% of the builds run. +const COLD_TIERS = [0.9, 0.6, 0.3, 0.1, 0.03]; +const coldRuns = new Map(); +for (const name of readdirSync(raw)) { + const program = /^cold-(\d+)-/.exec(name)?.[1]; + if (program === undefined) continue; + const ran = coldRuns.get(program) ?? new Set(); + for (const fn of executedFunctions(join(raw, name))) ran.add(fn); + coldRuns.set(program, ran); +} +const runs = new Map(); +for (const ran of coldRuns.values()) for (const fn of ran) runs.set(fn, (runs.get(fn) ?? 0) + 1); +const coldTier = (count) => { + const share = count / coldRuns.size; + const index = COLD_TIERS.findIndex((least) => share >= least); + return 2 + (index < 0 ? COLD_TIERS.length : index); +}; +const lines = [ + ...[...warm].sort().map((name) => `1 ${name}`), + ...[...runs] + .filter(([name]) => !warm.has(name)) + .map(([name, count]) => [coldTier(count), name]) + .sort((a, b) => a[0] - b[0] || (a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0)) + .map(([tier, name]) => `${tier} ${name}`), +]; +writeFileSync(output, gzipSync(`${lines.join("\n")}\n`, { level: constants.Z_BEST_COMPRESSION })); +const perTier = new Map(); +for (const line of lines) + perTier.set(line.split(" ")[0], (perTier.get(line.split(" ")[0]) ?? 0) + 1); +console.log( + `wrote ${output}: ${[...perTier].map(([tier, count]) => `tier ${tier}: ${count}`).join(", ")} functions, from ${coldRuns.size} builds (${statSync(output).size} bytes)`, +); +rmSync(builds, { recursive: true, force: true }); +if (values.work === undefined) rmSync(work, { recursive: true, force: true }); diff --git a/scripts/refresh-llvm-profile.mjs b/scripts/refresh-llvm-profile.mjs new file mode 100644 index 000000000..48f730cd0 --- /dev/null +++ b/scripts/refresh-llvm-profile.mjs @@ -0,0 +1,112 @@ +#!/usr/bin/env node +/* Refreshes native/llvm-codegen/llvm.profdata.gz, the profile the release + * LLVM distribution and the Linux helpers' own sources are optimized with + * (native/llvm-codegen/llvm-dist.mjs). The profile follows LLVM's sources + * and the kind of code scriptc emits rather than scriptc's commits, so it + * needs a refresh when LLVM is bumped, or when the emitted code changes + * enough that the distribution check (scripts/check-llvm-dist.mjs) loses + * much of its gain. + * + * Builds LLVM with IR instrumentation (scripts/build-llvm-dist.mjs + * --instrumented) and an instrumented helper against it, then runs that + * helper on the training inputs (scripts/llvm-helper-inputs.mjs): tsc-ts, + * five Effect fixtures, the runtime benchmark workloads and every 50th + * corpus program. The merged profile is written gzip-compressed. Needs a + * built checkout on Linux x64, clang, lld and llvm-profdata of LLVM_VERSION + * on PATH, and the LLVM release sources. About 20 minutes on 32 cores. + * + * node scripts/refresh-llvm-profile.mjs --source + * --tsc-ts [--work ] [--jobs N] [--output ] + */ +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { cpus, tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { constants, gzipSync } from "node:zlib"; +import { llvmProfile } from "../native/llvm-codegen/llvm-dist.mjs"; +import { buildLinuxHelper } from "../native/llvm-codegen/linux-helper.mjs"; +import { captureHelperInputs, helperInputPrograms } from "./llvm-helper-inputs.mjs"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const { values } = parseArgs({ + options: { + source: { type: "string" }, + "tsc-ts": { type: "string" }, + work: { type: "string" }, + jobs: { type: "string", default: String(cpus().length) }, + output: { type: "string", default: llvmProfile }, + help: { type: "boolean", default: false }, + }, +}); +if (values.help || values.source === undefined || values["tsc-ts"] === undefined) { + console.log( + "Usage: node scripts/refresh-llvm-profile.mjs --source --tsc-ts [--work ] [--jobs N] [--output ]", + ); + process.exit(values.help ? 0 : 2); +} +if (process.platform !== "linux" || process.arch !== "x64") + throw new Error("the LLVM profile is collected on Linux x64"); +const work = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-llvm-profile-"))); +const prefix = join(work, "llvm-instrumented-install"); +const helper = join(work, "helper", "scriptc-llvm-codegen"); +const raw = join(work, "raw"); + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { stdio: "inherit", ...options }); + if (result.status !== 0) + throw new Error(`${command} ${args[0]} failed with status ${result.status}`); +} + +console.log(`building the instrumented LLVM in ${work}`); +run(process.execPath, [ + join(root, "scripts/build-llvm-dist.mjs"), + "--instrumented", + "--source", + resolve(values.source), + "--prefix", + prefix, + "--work", + work, + "--jobs", + values.jobs, +]); +const manifest = JSON.parse( + readFileSync(join(root, "packages/llvm-linux-x64-gnu/package.json"), "utf8"), +); +process.env.LLVM_DIR = join(prefix, "lib/cmake/llvm"); +process.env.SCRIPTC_LLVM_PROFILE_GENERATE = "1"; +delete process.env.SCRIPTC_LLVM_THINLTO; +buildLinuxHelper({ + arch: "x64", + libc: "gnu", + version: manifest.version, + build: join(work, "helper-build"), + output: helper, +}); + +const inputs = await captureHelperInputs( + helperInputPrograms({ tscTs: resolve(values["tsc-ts"]) }), + join(work, "inputs"), +); +rmSync(raw, { recursive: true, force: true }); +mkdirSync(raw, { recursive: true }); +const out = join(work, "objects"); +for (const input of inputs) { + rmSync(out, { recursive: true, force: true }); + mkdirSync(out, { recursive: true }); + const result = spawnSync(helper, input.args(out), { + env: { ...process.env, LLVM_PROFILE_FILE: join(raw, "helper-%p.profraw") }, + encoding: "utf8", + }); + console.log(`${input.name}: ${input.partitions} partitions, status ${result.status}`); + if (result.status !== 0) throw new Error(`${input.name}: ${result.stdout}${result.stderr}`); +} + +const merged = join(work, "llvm.profdata"); +run("llvm-profdata", ["merge", "-o", merged, ...readdirSync(raw).map((file) => join(raw, file))]); +const output = resolve(values.output); +mkdirSync(dirname(output), { recursive: true }); +writeFileSync(output, gzipSync(readFileSync(merged), { level: constants.Z_BEST_COMPRESSION })); +console.log(`wrote ${output} from ${inputs.length} inputs`); diff --git a/scripts/verify-native-cli.mjs b/scripts/verify-native-cli.mjs index 92de57938..310cc04a9 100644 --- a/scripts/verify-native-cli.mjs +++ b/scripts/verify-native-cli.mjs @@ -41,6 +41,13 @@ for (const name of [ throw new Error(`native toolchain needs a relative ${name}`); within(resolve(dirname(binary), hostPath(value))); } +// Optional: a compiler this host could not run while building ships without it. +if (toolchain.lib_diagnostics !== undefined) { + const value = toolchain.lib_diagnostics; + if (typeof value !== "string" || absolutePath(value)) + throw new Error("native toolchain needs a relative lib_diagnostics"); + within(resolve(dirname(binary), hostPath(value))); +} const pack = JSON.parse( readFileSync( resolve(dirname(binary), hostPath(toolchain.runtime_pack), "runtime-pack.json"), diff --git a/scripts/wait-npm-available.mjs b/scripts/wait-npm-available.mjs new file mode 100644 index 000000000..26d7db391 --- /dev/null +++ b/scripts/wait-npm-available.mjs @@ -0,0 +1,150 @@ +#!/usr/bin/env node +// Waits until every package of a release resolves on the npm registry. +// +// usage: wait-npm-available.mjs [--interval ] [--timeout ] +// +// Each line of the uploads file is `@`, optionally followed by +// the Unix time (seconds) at which npm accepted the upload. Every interval +// (120 seconds by default), each package that has not resolved yet is asked +// for with `npm view @ version --prefer-online`. A package +// resolves when npm prints its version. Each package is reported once it +// resolves, with the time from upload to the check that found it. +// +// Exits with 0 once every package resolves. When packages are still missing +// at the timeout (2 hours by default), prints their names and versions and +// exits with 1. The script only reads from the registry. +import { execFile } from "node:child_process"; +import { appendFileSync, readFileSync } from "node:fs"; +import { setTimeout as delay } from "node:timers/promises"; + +function usage() { + throw new Error( + "usage: wait-npm-available.mjs [--interval ] [--timeout ]", + ); +} + +const args = process.argv.slice(2); +let file; +let intervalSeconds = 120; +let timeoutSeconds = 2 * 60 * 60; +for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "--interval" || arg === "--timeout") { + const value = Number(args[++i]); + if (!Number.isFinite(value) || value <= 0) usage(); + if (arg === "--interval") intervalSeconds = value; + else timeoutSeconds = value; + } else if (file === undefined) { + file = arg; + } else { + usage(); + } +} +if (file === undefined) usage(); + +const packages = []; +for (const line of readFileSync(file, "utf8").split("\n")) { + const fields = line.trim().split(/\s+/); + if (fields[0] === "") continue; + const spec = fields[0]; + const at = spec.lastIndexOf("@"); + if (at <= 0 || at === spec.length - 1 || fields.length > 2) { + throw new Error(`malformed upload line: ${line}`); + } + const acceptedAt = fields[1] === undefined ? null : Number(fields[1]) * 1000; + if (acceptedAt !== null && !Number.isFinite(acceptedAt)) { + throw new Error(`malformed upload time: ${line}`); + } + packages.push({ spec, version: spec.slice(at + 1), acceptedAt, availableAt: null }); +} +if (packages.length === 0) throw new Error(`${file} names no packages`); + +function resolves(pkg) { + return new Promise((resolve) => { + execFile( + "npm", + ["view", pkg.spec, "version", "--prefer-online"], + { encoding: "utf8", timeout: 60_000 }, + (error, stdout, stderr) => { + if (error === null && stdout.trim() === pkg.version) { + resolve(true); + return; + } + const code = /npm error code (\S+)/.exec(stderr)?.[1]; + if (error !== null && code !== "E404") { + const reason = code ?? error.code ?? error.message.split("\n")[0]; + console.log(`${pkg.spec}: npm view failed (${reason}); checking again`); + } + resolve(false); + }, + ); + }); +} + +function duration(ms) { + const seconds = Math.max(0, Math.round(ms / 1000)); + const minutes = Math.floor(seconds / 60); + return minutes === 0 ? `${seconds}s` : `${minutes}m ${String(seconds % 60).padStart(2, "0")}s`; +} + +function uploadToAvailable(pkg) { + return pkg.acceptedAt === null + ? "upload time not recorded" + : duration(pkg.availableAt - pkg.acceptedAt); +} + +function summary(missing) { + const summaryFile = process.env.GITHUB_STEP_SUMMARY; + if (!summaryFile) return; + const rows = packages.map( + (pkg) => + `| \`${pkg.spec}\` | ${pkg.availableAt === null ? "missing" : uploadToAvailable(pkg)} |`, + ); + const heading = + missing.length === 0 + ? `Every package resolves on npm (${packages.length}).` + : `${missing.length} of ${packages.length} packages do not resolve on npm after ${duration(timeoutSeconds * 1000)}.`; + appendFileSync( + summaryFile, + [heading, "", "| Package | Upload to available |", "| --- | --- |", ...rows, ""].join("\n"), + ); +} + +const started = Date.now(); +const deadline = started + timeoutSeconds * 1000; +for (;;) { + const pending = packages.filter((pkg) => pkg.availableAt === null); + const checked = []; + // A few checks at a time keep each round short without flooding the registry. + for (let i = 0; i < pending.length; i += 8) { + const batch = pending.slice(i, i + 8); + const results = await Promise.all(batch.map(resolves)); + const now = Date.now(); + batch.forEach((pkg, j) => { + if (results[j]) { + pkg.availableAt = now; + checked.push(pkg); + } + }); + } + for (const pkg of checked) console.log(`${pkg.spec} is available (${uploadToAvailable(pkg)})`); + const missing = packages.filter((pkg) => pkg.availableAt === null); + if (missing.length === 0) { + console.log(`Every package resolves on npm (${packages.length}).`); + summary(missing); + process.exit(0); + } + const now = Date.now(); + if (now >= deadline) { + console.log( + `${missing.length} of ${packages.length} packages do not resolve on npm after ${duration(now - started)}:`, + ); + for (const pkg of missing) console.log(`::error::${pkg.spec} does not resolve on npm`); + summary(missing); + process.exit(1); + } + console.log( + `${missing.length} of ${packages.length} packages do not resolve yet; checking again in ${duration(Math.min(intervalSeconds * 1000, deadline - now))}`, + ); + await delay(Math.min(intervalSeconds * 1000, deadline - now)); +} diff --git a/tests/corpus/1110-math-members.ts b/tests/corpus/1110-math-members.ts index 80dd78209..5f07821c5 100644 --- a/tests/corpus/1110-math-members.ts +++ b/tests/corpus/1110-math-members.ts @@ -1,20 +1,17 @@ -// Math members execute in the island engine; results exit as ordinary -// static numbers. Members with exact IEEE results print directly. The -// transcendentals round through toFixed(9): the engine uses the system -// libm and Node uses V8's fdlibm port, which can differ in the last ulp -// (tan(1), asin(0.5), acos(0.5) do on macOS). +// Math members print the values Node prints, the transcendentals +// included: they are V8's fdlibm functions. console.log(Math.floor(7.6), Math.ceil(7.2), Math.round(7.5), Math.round(-7.5), Math.trunc(-7.9)); console.log(Math.abs(-3.5), Math.sign(-12), Math.sign(0), Math.sign(4.2)); console.log(Math.sqrt(144), Math.sqrt(2), Math.pow(2, 10), Math.pow(2, 0.5)); console.log(Math.hypot(3, 4), Math.min(2, -9), Math.max(2, -9)); console.log(Math.log2(1024), Math.log10(100000), Math.exp(0), Math.log(1)); console.log(Math.PI, Math.E); -console.log(Math.cbrt(27).toFixed(9), Math.exp(1).toFixed(9), Math.log(10).toFixed(9)); -console.log(Math.sin(1).toFixed(9), Math.cos(1).toFixed(9), Math.tan(1).toFixed(9)); -console.log(Math.asin(0.5).toFixed(9), Math.acos(0.5).toFixed(9)); -console.log(Math.atan(1).toFixed(9), Math.atan2(1, 1).toFixed(9), Math.atan2(-1, -1).toFixed(9)); -// Island results chain into static arithmetic like any number. +console.log(Math.cbrt(27), Math.exp(1), Math.log(10)); +console.log(Math.sin(1), Math.cos(1), Math.tan(1)); +console.log(Math.asin(0.5), Math.acos(0.5)); +console.log(Math.atan(1), Math.atan2(1, 1), Math.atan2(-1, -1)); +// Math results chain into static arithmetic like any number. const area = Math.PI * Math.pow(2, 2); console.log(area.toFixed(6), Math.floor(area) + 1); -// Non-finite edges: strict exits accept every engine number. +// Non-finite results. console.log(Math.log(0), Math.log(-1), Math.sqrt(-1), Math.pow(0, -1)); diff --git a/tests/corpus/5025-lifetime-region-shared-candidate.ts b/tests/corpus/5025-lifetime-region-shared-candidate.ts new file mode 100644 index 000000000..7b5084a05 --- /dev/null +++ b/tests/corpus/5025-lifetime-region-shared-candidate.ts @@ -0,0 +1,85 @@ +// LifetimeRegion (@scriptc/experimental): freeing a region whose objects +// own a cycle runs a limited cycle pass over what the region owned. That +// pass also reaches a shared object the region pointed at, which is a +// candidate of its own (it lost a reference) and stays referenced from +// outside. The shared object must stay a valid candidate after the pass: +// it dies afterwards, later passes run, and the program prints the same +// as under Node.js. +import type { ProgramLifetime } from "@scriptc/types"; +import { LifetimeRegion } from "@scriptc/experimental"; + +class Shared { + readonly label: string; + peers: Shared[] = []; + constructor(label: string) { + this.label = label; + } +} + +// Ordinary objects the region's instance owns: a cycle that points back +// into the region and at the shared object. +class Owned { + holder: Holder | undefined = undefined; + shared: Shared | undefined = undefined; + partner: Owned | undefined = undefined; +} + +class Holder implements ProgramLifetime { + owned: Owned | undefined = undefined; + readonly id: number; + constructor(id: number) { + this.id = id; + } +} + +// Cyclic garbage, so the collector runs passes of its own. +class Ring { + next: Ring | undefined = undefined; + readonly value: number; + constructor(value: number) { + this.value = value; + } +} + +function churn(n: number): number { + let total = 0; + for (let i = 0; i < n; i++) { + const a = new Ring(i); + const b = new Ring(i + 1); + a.next = b; + b.next = a; + total += a.value + b.next.value; + } + return total; +} + +function round(k: number): string { + let shared: Shared | undefined = new Shared(`shared ${k}`); + // A second reference, dropped below: the shared object becomes a candidate. + let keep: Shared[] = [shared]; + shared.peers.push(new Shared(`peer ${k}`)); + const region = new LifetimeRegion(); + region.run(() => { + const holder = new Holder(k); + const owned = new Owned(); + const partner = new Owned(); + owned.partner = partner; + partner.partner = owned; + owned.holder = holder; + owned.shared = shared; + holder.owned = owned; + }); + keep = []; + region.release(); + const label = shared.label + " / " + shared.peers[0]!.label + " / " + keep.length; + // The shared object dies, and the passes that follow read the candidate buffers. + shared = undefined; + churn(2000); + return label; +} + +for (let k = 0; k < 20; k++) { + const label = round(k); + if (k % 5 === 0) console.log(label); +} +console.log("churn", churn(10000)); diff --git a/tests/corpus/6311-typed-array-integer-products.ts b/tests/corpus/6311-typed-array-integer-products.ts new file mode 100644 index 000000000..28a074a4a --- /dev/null +++ b/tests/corpus/6311-typed-array-integer-products.ts @@ -0,0 +1,83 @@ +// Products and sums of typed-array element reads that only reach bitwise +// operators compute in 32-bit integers. Results must match Node for reads +// past the end (NaN, which converts to 0), negative factors, products that +// can be -0, sums that wrap past int32, and products near 2^53. + +const rgba = new Uint8Array([10, 20, 30, 255, 200, 100, 50, 0, 255, 255, 255, 128, 0, 0, 0, 1]); +const pixels = rgba.length / 4; +const gray = new Uint8Array(pixels); +for (let i = 0; i < gray.length; i++) { + const j = i * 4; + gray[i] = (rgba[j]! * 77 + rgba[j + 1]! * 150 + rgba[j + 2]! * 29) >> 8; +} +console.log("gray", gray.join(",")); + +// Reads past the end are NaN: the sum is NaN and converts to 0. +const tail: number[] = []; +for (let i = pixels - 2; i < pixels + 2; i++) { + const j = i * 4; + tail.push((rgba[j]! * 77 + rgba[j + 1]! * 150 + rgba[j + 2]! * 29) >> 8); + tail.push((rgba[j + 3]! * 3 - rgba[j]!) | 0); +} +console.log("tail", tail.join(",")); + +// Signed elements and negative factors, including products that are -0. +const signed = new Int8Array([0, -1, 1, -128, 127, 0, 5, -5]); +const mixed: number[] = []; +for (let i = 0; i < signed.length; i++) { + mixed.push((signed[i]! * -3) | 0); + mixed.push((signed[i]! * signed[(i + 1) % signed.length]! + signed[i]! * 1000) >> 1); + mixed.push(Object.is(signed[i]! * -1, -0) ? 1 : 0); +} +console.log("signed", mixed.join(",")); + +// Unsigned 16-bit products whose sum passes int32 and wraps. +const wide = new Uint16Array([65535, 65535, 1, 40000, 50000, 12345]); +const wrapped: number[] = []; +for (let i = 0; i + 1 < wide.length; i++) { + wrapped.push((wide[i]! * wide[i + 1]! + wide[i]! * wide[i]!) | 0); + wrapped.push((wide[i]! * wide[i + 1]!) >>> 0); + wrapped.push((wide[i]! * 65537 - wide[i + 1]!) ^ 0x5a5a); +} +console.log("uint16", wrapped.join(",")); + +// Unsigned 32-bit elements times constants just below and above 2^53. +const big = new Uint32Array([0xffffffff, 0x80000000, 3, 0]); +const near: number[] = []; +for (let i = 0; i < big.length; i++) { + near.push((big[i]! * 2097151) | 0); + near.push((big[i]! * 2097153) | 0); + near.push((big[i]! * 2097151 + big[(i + 1) % big.length]!) >>> 0); +} +console.log("uint32", near.join(",")); + +// A float element is never an exact integer. +const floats = new Float64Array([1.5, -2.25, 3]); +const fractional: number[] = []; +for (let i = 0; i < floats.length; i++) fractional.push((floats[i]! * 3 + floats[i]! * 2) | 0); +console.log("float", fractional.join(",")); + +// Clamped bytes and a sum used as a number keep their double values. +const clamped = new Uint8ClampedArray([0, 128, 255]); +let total = 0; +for (let i = 0; i < clamped.length; i++) { + total += clamped[i]! * 77 + clamped[i]! * 0.5; + total += (clamped[i]! * 77 + clamped[(i + 1) % 3]! * 150) >> 8; +} +console.log("clamped", total); + +// A larger image, so the loop runs its vectorized body. +const image = new Uint8Array(4 * 1000); +for (let i = 0; i < image.length; i++) image[i] = (i * 37 + (i >> 3)) & 255; +const luma = new Uint8Array(1000); +for (let i = 0; i < luma.length; i++) { + const j = i * 4; + luma[i] = (image[j]! * 77 + image[j + 1]! * 150 + image[j + 2]! * 29) >> 8; +} +let sum = 0; +let mix = 0; +for (let i = 0; i < luma.length; i++) { + sum += luma[i]!; + mix = (mix * 31 + luma[i]!) | 0; +} +console.log("luma", sum, mix); diff --git a/tests/corpus/7508-pipeline-untyped-source-functions.mjs b/tests/corpus/7508-pipeline-untyped-source-functions.mjs new file mode 100644 index 000000000..987221ffd --- /dev/null +++ b/tests/corpus/7508-pipeline-untyped-source-functions.mjs @@ -0,0 +1,85 @@ +// pipeline() over untyped first stages that are functions or +// { readable, writable } pairs: a source function is called with +// { signal } while the pipeline is set up and its iterable result is +// pumped; a result that is not iterable throws ERR_INVALID_RETURN_VALUE +// from the callback form and rejects the promise form; a pair pumps its +// readable side. +import { pipeline, Writable, Readable, PassThrough } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function pick(v) { + return v; +} +function sink(label) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + console.log(label, "write", JSON.stringify(String(chunk))); + cb(); + }, + final(cb) { + console.log(label, "final"); + cb(); + }, + }); +} + +const arraySource = pick(function (opts) { + console.log("array source called", typeof opts, typeof opts.signal, opts.signal.aborted); + return ["a1", "a2"]; +}); +const before = pipelineP(arraySource, sink("array")); +console.log("after pipeline() returned"); +await before; +console.log("array resolved"); + +await pipelineP(pick(() => "xy"), sink("string")); + +await pipelineP( + pick(async function* ({ signal }) { + console.log("generator source", typeof signal.aborted); + yield "g1"; + await new Promise((r) => setTimeout(r, 1)); + yield "g2"; + }), + pick(async function* (source) { + for await (const chunk of source) yield `[${chunk}]`; + }), + sink("generator"), +); + +await new Promise((resolve) => { + pipeline(pick(() => new Set(["s1", "s2"])), sink("set"), (err) => { + console.log("set done", err); + resolve(undefined); + }); +}); + +// a result that is not iterable +try { + pipeline(pick(() => 5), sink("five"), () => console.log("five callback")); +} catch (err) { + console.log("five threw", err.name, err.code, err.message); +} +try { + await pipelineP(pick(() => ({ value: 1 })), sink("object")); +} catch (err) { + console.log("object rejected", err.name, err.code, err.message); +} + +// a source function that throws +try { + await pipelineP( + pick(() => { + throw new Error("source failed"); + }), + sink("throws"), + ); +} catch (err) { + console.log("throws rejected", err.message); +} + +// a { readable, writable } pair pumps its readable side +const side = new PassThrough(); +await pipelineP(pick({ readable: Readable.from(["p1", "p2"]), writable: side }), sink("pair")); +console.log("done"); diff --git a/tests/corpus/7509-pipeline-untyped-function-stages.mjs b/tests/corpus/7509-pipeline-untyped-function-stages.mjs new file mode 100644 index 000000000..590334f90 --- /dev/null +++ b/tests/corpus/7509-pipeline-untyped-function-stages.mjs @@ -0,0 +1,117 @@ +// pipeline() with untyped stages after the first that are functions: a +// transform is called with the previous stream or generator and +// { signal } while the pipeline is set up, and must return an async +// iterable; an async function destination's value resolves the promise +// form and reaches the callback; an async generator destination feeds +// the PassThrough that pipeline() returns. Results of the wrong kind throw +// ERR_INVALID_RETURN_VALUE naming the stage. A stage after an untyped +// source function or transform receives that stage's own iterator, as +// Node's makeAsyncIterable answers an iterable itself. +import { pipeline, Writable, Readable, PassThrough } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function pick(v) { + return v; +} +function sink(label) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + console.log(label, "write", JSON.stringify(String(chunk))); + cb(); + }, + }); +} +const upper = pick(async function* (source, { signal }) { + console.log("upper called", source instanceof Readable, typeof signal.aborted); + for await (const chunk of source) yield String(chunk).toUpperCase(); +}); +const join = pick(async (source) => { + let text = ""; + for await (const chunk of source) text += chunk; + return text; +}); + +await pipelineP(Readable.from(["a", "b"]), upper, sink("transform")); +console.log("joined", await pipelineP(Readable.from(["x", "y"]), upper, join)); +console.log("two transforms", await pipelineP(pick(["m", "n"]), upper, upper, join)); + +await new Promise((resolve) => { + pipeline(Readable.from(["c1", "c2"]), upper, join, (err, value) => { + console.log("callback", err, value); + resolve(undefined); + }); +}); + +// an async generator destination +const wrap = pick(async function* (source) { + for await (const chunk of source) yield `<${chunk}>`; +}); +const out = pipeline(Readable.from(["p1", "p2"]), wrap, (err, value) => { + console.log("generator destination done", err, value); +}); +console.log("returns a PassThrough", out instanceof PassThrough); +for await (const chunk of out) console.log("passthrough", chunk); + +// an async generator destination in the promise form: the PassThrough +// takes its values and the promise resolves to undefined +const promised = await pipelineP(Readable.from(["q1", "q2"]), wrap); +console.log("generator destination promise", promised, typeof promised); + +// a destination whose promise rejects +try { + await pipelineP( + Readable.from(["r"]), + pick(async (source) => { + for await (const chunk of source) console.log("rejecting after", chunk); + throw new Error("destination failed"); + }), + ); +} catch (err) { + console.log("rejected", err.message); +} + +// results of the wrong kind +try { + await pipelineP(Readable.from(["i"]), pick(() => 5), sink("number")); +} catch (err) { + console.log("transform result", err.name, err.code, err.message); +} +try { + await pipelineP(Readable.from(["i"]), upper, pick(() => ["sync"]), sink("array")); +} catch (err) { + console.log("second transform result", err.code, err.message); +} +try { + pipeline(Readable.from(["i"]), pick(() => 7), () => console.log("never")); +} catch (err) { + console.log("destination result", err.name, err.code, err.message); +} +// the iterators handed from stage to stage +let transformIter = null; +const handed = await pipelineP( + pick(function* () { + yield "s1"; + yield "s2"; + }), + pick((source) => { + // a sync generator, not an async iterator over it + console.log( + "transform gets the source's iterator", + typeof source[Symbol.asyncIterator], + typeof source[Symbol.iterator], + ); + transformIter = (async function* () { + for await (const chunk of source) yield `${chunk}!`; + })(); + return transformIter; + }), + pick(async (source) => { + console.log("destination gets the transform's iterator", source === transformIter); + let text = ""; + for await (const chunk of source) text += chunk; + return text; + }), +); +console.log("handed", handed); +console.log("done"); diff --git a/tests/corpus/7510-pipeline-untyped-later-iterables.mjs b/tests/corpus/7510-pipeline-untyped-later-iterables.mjs new file mode 100644 index 000000000..7fe6e27b3 --- /dev/null +++ b/tests/corpus/7510-pipeline-untyped-later-iterables.mjs @@ -0,0 +1,64 @@ +// pipeline() with an untyped stage after the first that is not a stream or +// a function: Node takes Duplex.from of it, so an iterable becomes the +// readable stage the next stream is piped from, the stages before it are +// not read and are destroyed when the pipeline finishes, and a value +// Duplex.from rejects throws ERR_INVALID_ARG_TYPE. +import { pipeline, Writable, Readable, Duplex } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function pick(v) { + return v; +} +function sink(label) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + console.log(label, "write", JSON.stringify(String(chunk))); + cb(); + }, + }); +} +function watched(label, values) { + const source = Readable.from(values); + source.on("error", (err) => console.log(label, "error", err.code)); + source.on("close", () => console.log(label, "close", source.destroyed)); + return source; +} + +await pipelineP(watched("first", ["unread"]), pick(["z1", "z2"]), sink("array")); +console.log("array resolved"); +await new Promise((r) => setTimeout(r, 5)); + +await new Promise((resolve) => { + pipeline(watched("second", ["unread"]), pick("text"), sink("string"), (err) => { + console.log("string done", err); + resolve(undefined); + }); +}); +await new Promise((r) => setTimeout(r, 5)); + +// a generator stage before the iterable is never read +await pipelineP( + Readable.from(["s"]), + pick(async function* (source) { + console.log("generator stage called"); + for await (const chunk of source) yield chunk; + }), + pick(new Set(["set1", "set2"])), + sink("set"), +); +console.log("set resolved"); + +// the iterable as the last stage is what pipeline() returns +const last = pipeline(Readable.from(["u"]), pick(["tail"]), () => console.log("never")); +console.log(last instanceof Duplex, last.readable, last.writable); +for await (const chunk of last) console.log("last", chunk); + +for (const value of [42, { plain: true }, null]) { + try { + await pipelineP(Readable.from(["i"]), pick(value), sink("rejected")); + } catch (err) { + console.log(err.name, err.code, err.message); + } +} +console.log("done"); diff --git a/tests/corpus/7511-pipeline-untyped-stages.cjs b/tests/corpus/7511-pipeline-untyped-stages.cjs new file mode 100644 index 000000000..124372e12 --- /dev/null +++ b/tests/corpus/7511-pipeline-untyped-stages.cjs @@ -0,0 +1,50 @@ +// A CommonJS helper that passes its untyped parameters to pipeline(): the +// same call runs over streams, function stages and iterables, whichever +// each argument is at run time. +const { pipeline, Transform, Writable, Readable } = require("node:stream"); + +function run(label, source, middle, destination) { + return new Promise((resolve) => { + pipeline(source, middle, destination, (err, value) => { + console.log(label, "done", err ? err.code || err.message : err, value); + resolve(undefined); + }); + }); +} +function sink(label) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + console.log(label, "write", String(chunk)); + cb(); + }, + }); +} +const double = new Transform({ + objectMode: true, + transform(chunk, _enc, cb) { + cb(null, String(chunk) + String(chunk)); + }, +}); +async function* exclaim(source) { + for await (const chunk of source) yield `${chunk}!`; +} +async function count(source) { + let n = 0; + for await (const _ of source) n++; + return n; +} + +(async () => { + await run("streams", Readable.from(["a", "b"]), double, sink("streams")); + await run("function middle", Readable.from(["c"]), exclaim, sink("function middle")); + await run("function ends", () => ["d", "e", "f"], exclaim, count); + await run("iterable middle", Readable.from(["skipped"]), ["g"], sink("iterable middle")); + await run("generator into stream", ["h"], exclaim, new Transform({ + objectMode: true, + transform(chunk, _enc, cb) { + console.log("transform saw", chunk); + cb(null, chunk); + }, + }).on("data", (d) => console.log("data", d))); +})(); diff --git a/tests/corpus/7512-pipeline-any-stages.ts b/tests/corpus/7512-pipeline-any-stages.ts new file mode 100644 index 000000000..a2039a533 --- /dev/null +++ b/tests/corpus/7512-pipeline-any-stages.ts @@ -0,0 +1,45 @@ +// pipeline() in TypeScript with stages after the first typed any or +// unknown: functions run as function stages, streams are piped, and an +// iterable is taken as Duplex.from of it. +import { pipeline, Readable, Writable, PassThrough } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function sink(label: string): Writable { + return new Writable({ + objectMode: true, + write(chunk: unknown, _enc, cb) { + console.log(label, JSON.stringify(chunk)); + cb(); + }, + }); +} + +async function main(): Promise { + const shout: any = async function* (source: any) { + for await (const chunk of source) yield String(chunk).toUpperCase(); + }; + await pipelineP(Readable.from(["a", "b"]), shout, sink("function")); + + const total: unknown = async (source: any) => { + let sum = 0; + for await (const chunk of source) sum += Number(chunk); + return sum; + }; + const sum = await pipelineP(Readable.from(["1", "2", "3"]), total as any); + console.log("sum", sum); + + const relay: any = new PassThrough({ objectMode: true }); + await pipelineP(Readable.from([{ k: 1 }]), relay, sink("stream")); + + const values: any = ["v1", "v2"]; + await pipelineP(Readable.from(["unread"]), values, sink("iterable")); + + await new Promise((resolve) => { + pipeline(Readable.from(["x"]), shout, sink("callback"), (err: NodeJS.ErrnoException | null) => { + console.log("callback done", err == null); + resolve(); + }); + }); +} + +main().then(() => console.log("done")); diff --git a/tests/corpus/7513-pipeline-untyped-stage-errors.mjs b/tests/corpus/7513-pipeline-untyped-stage-errors.mjs new file mode 100644 index 000000000..ade4370b4 --- /dev/null +++ b/tests/corpus/7513-pipeline-untyped-stage-errors.mjs @@ -0,0 +1,86 @@ +// Untyped pipeline function stages next to typed stages: an untyped stage +// after a typed generator stage or a Transform, the { signal } an untyped +// stage receives aborting when the pipeline fails, and an untyped stage +// that throws while the pipeline is set up. +import { pipeline, Writable, Readable, Transform } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function pick(v) { + return v; +} +function sink(label) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + console.log(label, "write", JSON.stringify(String(chunk))); + cb(); + }, + }); +} + +await pipelineP( + Readable.from(["a", "b"]), + async function* (source) { + for await (const chunk of source) yield `${chunk}1`; + }, + pick(async function* (source) { + for await (const chunk of source) yield `${chunk}2`; + }), + sink("after generator"), +); + +await pipelineP( + Readable.from(["q"]), + new Transform({ + transform(chunk, _enc, cb) { + cb(null, chunk); + }, + }), + pick(async (source) => { + for await (const chunk of source) console.log("destination saw", String(chunk)); + }), +); + +let seen; +try { + await pipelineP( + Readable.from(["x", "y"]), + pick(async function* (source, { signal }) { + seen = signal; + signal.addEventListener("abort", () => console.log("signal abort event")); + for await (const chunk of source) yield chunk; + }), + new Writable({ + write(_chunk, _enc, cb) { + cb(new Error("write failed")); + }, + }), + ); +} catch (err) { + console.log("rejected", err.message, seen.aborted); +} + +try { + pipeline( + Readable.from(["t"]), + pick(() => { + throw new Error("stage threw"); + }), + sink("never"), + () => console.log("never called"), + ); +} catch (err) { + console.log("callback form threw", err.message); +} +try { + await pipelineP( + Readable.from(["t"]), + pick(() => { + throw new Error("stage threw again"); + }), + sink("never"), + ); +} catch (err) { + console.log("promise form rejected", err.message); +} +console.log("done"); diff --git a/tests/corpus/7515-net-socket-backpressure.ts b/tests/corpus/7515-net-socket-backpressure.ts index 260726a15..6524e6d54 100644 --- a/tests/corpus/7515-net-socket-backpressure.ts +++ b/tests/corpus/7515-net-socket-backpressure.ts @@ -7,6 +7,11 @@ // untyped code reads the same properties, and a write whose send fails // after the peer reset the connection answers false. An echo client that // waits for 'drain' when write answers false moves every byte. +// +// The server picks each connection's handling from `mode` when it accepts +// it. The lookup client's bytes fit in the kernel's buffers, so its 'drain' +// can come before the server accepts that connection; the program waits for +// the accept before it changes `mode`. import { connect, createServer, Socket } from "node:net"; import type { LookupAddress, LookupAllOptions, LookupOneOptions } from "node:dns"; @@ -113,6 +118,15 @@ function afterReset(port: number): Promise { let paused: Socket | null = null; let mode = "echo"; +let accepted: (() => void) | null = null; + +/** Resolves when the server accepts its next connection. */ +function nextConnection(): Promise { + return new Promise((resolve) => { + accepted = resolve; + }); +} + const server = createServer((socket: Socket) => { if (mode === "echo") socket.pipe(socket); else if (mode === "drain") socket.resume(); @@ -120,6 +134,11 @@ const server = createServer((socket: Socket) => { socket.pause(); paused = socket; } else socket.destroy(); + if (accepted !== null) { + const resolve = accepted; + accepted = null; + resolve(); + } }); server.listen(0, "127.0.0.1", async () => { @@ -127,7 +146,9 @@ server.listen(0, "127.0.0.1", async () => { const port = typeof addr === "object" && addr !== null ? addr.port : 0; await echo(port); mode = "drain"; + const lookupAccepted = nextConnection(); await lookupPending(port); + await lookupAccepted; mode = "pause"; await largeWrite(port); mode = "end"; diff --git a/tests/corpus/7717-math-fdlibm.ts b/tests/corpus/7717-math-fdlibm.ts new file mode 100644 index 000000000..91ce02adf --- /dev/null +++ b/tests/corpus/7717-math-fdlibm.ts @@ -0,0 +1,97 @@ +// Math's transcendental functions print the values Node prints: each line +// digests the result bits of one function over a few thousand inputs, so +// a difference in the last bit of any result changes the line. Sample +// values print in full, and pow's special cases follow V8's. +const bits = new Float64Array(1); +const words = new Uint32Array(bits.buffer); + +function digest(values: number[]): string { + let h1 = 0x811c9dc5; + let h2 = 0x01000193; + for (const v of values) { + // NaN prints as NaN; its bits are not part of the result. + bits[0] = Number.isNaN(v) ? 0 : v; + h1 = Math.imul(h1 ^ words[0], 0x01000193) >>> 0; + h2 = Math.imul(h2 ^ words[1], 0x01000193) >>> 0; + } + return `${h1}:${h2}`; +} + +let seed = 12345; +function next(): number { + seed = (seed * 1103515245 + 12345) % 2147483648; + return seed / 2147483648; +} + +const wide: number[] = []; +const unit: number[] = []; +const small: number[] = []; +for (let i = 0; i < 3000; i++) { + wide.push((next() - 0.5) * Math.pow(10, Math.floor(next() * 12) - 4)); + unit.push(next() * 2 - 1); + small.push((next() - 0.5) * 40); +} +const positive = wide.map((x) => Math.abs(x)); +const edges = [0, -0, 1, -1, 0.5, -0.5, 2, Math.PI, Math.E, 1e-300, 5e-324, 1e300, Infinity, -Infinity, NaN]; + +const unary: [string, (x: number) => number, number[]][] = [ + ["sin", Math.sin, wide], + ["cos", Math.cos, wide], + ["tan", Math.tan, wide], + ["asin", Math.asin, unit], + ["acos", Math.acos, unit], + ["atan", Math.atan, wide], + ["sinh", Math.sinh, small], + ["cosh", Math.cosh, small], + ["tanh", Math.tanh, small], + ["asinh", Math.asinh, wide], + ["acosh", Math.acosh, positive.map((x) => x + 1)], + ["atanh", Math.atanh, unit], + ["exp", Math.exp, small], + ["expm1", Math.expm1, small], + ["log", Math.log, positive], + ["log1p", Math.log1p, positive], + ["log2", Math.log2, positive], + ["log10", Math.log10, positive], + ["cbrt", Math.cbrt, wide], + ["sqrt", Math.sqrt, positive], +]; +for (const [name, fn, inputs] of unary) { + console.log(name, digest(inputs.map(fn)), digest(edges.map(fn)), fn(0.3), fn(0.75)); +} + +// Math.sin(i) and Math.cos(i * 0.37) for i below 4096: main's libm calls +// printed 284 of these 8192 values differently from Node. +const sines: number[] = []; +const cosines: number[] = []; +for (let i = 0; i < 4096; i++) { + sines.push(Math.sin(i)); + cosines.push(Math.cos(i * 0.37)); +} +console.log("sin(i)", digest(sines), "cos(i*0.37)", digest(cosines), Math.cos(2.59)); + +const atan2: number[] = []; +const pow: number[] = []; +for (let i = 0; i < wide.length; i++) { + atan2.push(Math.atan2(wide[i], small[i])); + pow.push(Math.pow(positive[i] % 50, small[i] / 4)); +} +console.log("atan2", digest(atan2), Math.atan2(2.5, 3), Math.atan2(-0, -1)); +console.log("pow", digest(pow), Math.pow(1.1, 0.3), 1.0000001 ** 1e7); + +// V8 answers these itself: x * x, the square root, and NaN for ±1 ** ±Infinity. +const squares = wide.map((x) => x ** 2); +const roots = positive.map((x) => Math.pow(x, 0.5)); +console.log("pow 2", digest(squares), "pow 0.5", digest(roots)); +// The same exponents when they are only known at run time. +const two = Number(process.argv[2] ?? "2"); +const half = two / 4; +console.log("pow runtime 2", digest(wide.map((x) => x ** two)), "0.5", digest(positive.map((x) => Math.pow(x, half)))); +console.log(Math.pow(-Infinity, 0.5), Math.pow(-0, 0.5), (-8) ** 0.5, 1 ** Infinity, (-1) ** -Infinity, NaN ** 0); +// Rounding is exact: Math.floor, Math.ceil and Math.trunc over the same +// inputs and over halves, signed zeros included. +const halves = wide.map((x, i) => Math.round(x * 4) / 4 - (i % 2 === 0 ? 0 : 0.5)); +console.log("floor", digest(wide.map(Math.floor)), digest(halves.map(Math.floor)), digest(edges.map(Math.floor))); +console.log("ceil", digest(wide.map(Math.ceil)), digest(halves.map(Math.ceil)), digest(edges.map(Math.ceil))); +console.log("trunc", digest(wide.map(Math.trunc)), digest(halves.map(Math.trunc)), Object.is(Math.ceil(-0.5), -0), Object.is(Math.floor(-0), -0)); +console.log(Math.hypot(3, 4), Math.hypot(1e200, 1e200), 7 % 2.5, -7.5 % 2, Math.floor(-0.5), Math.ceil(-0.5), Math.round(2.5), Math.trunc(-4.7)); diff --git a/tests/corpus/7718-math-fdlibm.cjs b/tests/corpus/7718-math-fdlibm.cjs new file mode 100644 index 000000000..35d26f7e8 --- /dev/null +++ b/tests/corpus/7718-math-fdlibm.cjs @@ -0,0 +1,12 @@ +// Math functions and ** over untyped values print the values Node prints +// (V8's fdlibm results, and V8's pow special cases). +const values = JSON.parse("[0.3, 0.75, 2.5, -7.25, 1e-8, 123456.789, 1e21]"); +const rows = []; +for (const x of values) { + rows.push([Math.sin(x), Math.cos(x), Math.tan(x), Math.exp(x / 100), Math.cbrt(x), Math.atan2(x, 3)].join(" ")); + rows.push([Math.log(Math.abs(x)), Math.log10(Math.abs(x)), Math.sinh(x / 1e5), Math.cosh(x / 1e5), Math.expm1(x / 1e6)].join(" ")); + rows.push([x ** 2, x ** 0.5, Math.abs(x) ** 0.3, x % 1.5].join(" ")); +} +console.log(rows.join("\n")); +const edges = JSON.parse('[1, -1, null]'); +console.log(edges[0] ** Infinity, edges[1] ** -Infinity, Number(edges[2]) ** 0.5, (-Infinity) ** 0.5); diff --git a/tests/corpus/8008-string-index-paths.ts b/tests/corpus/8008-string-index-paths.ts new file mode 100644 index 000000000..a9a748ad8 --- /dev/null +++ b/tests/corpus/8008-string-index-paths.ts @@ -0,0 +1,102 @@ +// UTF-16 index operations on long non-ASCII strings: slices and substrings +// walking forward and backward, interleaved on two strings, at astral +// characters and their halves, with negative and out-of-range indexes; +// lastIndexOf, startsWith and endsWith with positions; and normalize on +// text already in each form, composed and decomposed text, Hangul, +// compatibility characters and combining marks. + +const unit = "Ünïcödé tëxt with ß and 日本語 and émoji 😀 mixed in, "; +const big = unit.repeat(3000); +const other = "ascii words then 中文 and 🎉🎉 and ü, ".repeat(4000); + +function digest(text: string): number { + let h = 0; + for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) >>> 0; + return h; +} + +let forward = 0; +for (let i = 0; i + 30 < big.length; i += 7) { + forward = (forward * 33 + digest(big.slice(i, i + 20)) + digest(big.substring(i + 3, i + 9))) >>> 0; +} +let backward = 0; +for (let i = big.length - 1; i > 30; i -= 11) { + backward = (backward * 33 + digest(big.slice(i - 25, i)) + big.substring(i, i - 5).length) >>> 0; +} +let interleaved = 0; +for (let i = 0; i + 40 < other.length && i + 40 < big.length; i += 13) { + interleaved = (interleaved * 33 + digest(big.slice(i, i + 9)) + digest(other.slice(i + 1, i + 40))) >>> 0; +} +console.log(big.length, forward, backward, interleaved); + +// Indexes at astral characters and their halves. +const emojiAt = big.indexOf("😀", 100000); +console.log( + emojiAt, + JSON.stringify(big.slice(emojiAt, emojiAt + 1)), + JSON.stringify(big.slice(emojiAt + 1, emojiAt + 3)), + JSON.stringify(big.substring(emojiAt - 1, emojiAt + 1)), + big.charCodeAt(emojiAt + 1), + JSON.stringify(big.at(emojiAt + 1)), + JSON.stringify(big.charAt(emojiAt)), +); +console.log( + JSON.stringify(big.slice(-5)), + JSON.stringify(big.slice(-30, -20)), + big.slice(big.length - 3, big.length + 10).length, + JSON.stringify(big.substring(-10, 4)), + big.slice(5, 2).length, + JSON.stringify(big.slice(NaN, 3)), +); +for (const at of [0, 1, 52, 99999, 150000, big.length - 1, big.length, -1, -2, -big.length]) { + console.log(at, JSON.stringify(big.at(at)), JSON.stringify(big.charAt(at)), big.charCodeAt(at)); +} + +// lastIndexOf, startsWith and endsWith with positions. +console.log( + big.lastIndexOf("日本"), + big.lastIndexOf("日本", 100000), + big.lastIndexOf("😀", 50), + big.lastIndexOf("absent"), + big.lastIndexOf("é", 0), + big.lastIndexOf("Ü"), + big.lastIndexOf(""), + big.lastIndexOf("", 10), +); +let starts = 0; +let ends = 0; +for (let i = 0; i < big.length; i += 17) { + if (big.startsWith("tëxt", i)) starts++; + if (big.endsWith("😀", i)) ends++; +} +console.log(starts, ends, big.startsWith("Ü"), big.endsWith("in, "), big.startsWith("ünï"), big.endsWith("in, ", big.length - 1)); + +// normalize. +const samples = [ + "plain ascii", + "précomposé café naïve", + "décomposé café", + "日本語のテキスト 한국어 문장", + "각 각", + "ｆｕｌｌｗｉｄｔｈ ｶﾀｶﾅ fi ① ℌ ²", + "ậ ậ ͅ", + "Ω Å Å Ω K", + "が が か゚", + "emoji 😀 and \ud800 lone", + unit, +]; +for (const sample of samples) { + const forms = ["NFC", "NFD", "NFKC", "NFKD"].map((form) => sample.normalize(form)); + console.log( + forms.map((text) => `${text.length}:${digest(text)}`).join(" "), + forms.map((text) => text === sample).join(","), + ); +} +const longNfc = big.normalize(); +const longNfd = big.normalize("NFD"); +console.log(longNfc === big, longNfc.length, longNfd.length, digest(longNfd.slice(0, 100)), longNfd.normalize("NFC") === big); +try { + "x".normalize("NFX"); +} catch (error) { + console.log((error as Error).name, (error as Error).message); +} diff --git a/tests/corpus/8014-foreign-element-callbacks.ts b/tests/corpus/8014-foreign-element-callbacks.ts new file mode 100644 index 000000000..188a3119d --- /dev/null +++ b/tests/corpus/8014-foreign-element-callbacks.ts @@ -0,0 +1,118 @@ +// Callbacks over a number[] or string[] that holds elements written +// through an unknown[] view, when the callback never reads the element: +// it declares no element parameter, or never references the one it +// declares. The methods visit every element as in Node, and filter keeps +// a written element as it is. + +const numbers: number[] = [1, 2]; +const numbersView: unknown[] = numbers; +numbersView.push("x", undefined, { k: 1 }); +numbers.push(5); + +// Inline callbacks (fused loops). +console.log(numbers.filter(() => true)); +console.log(numbers.filter((_, i) => i % 2 === 0)); +console.log(numbers.map(() => 0)); +console.log(numbers.map((_, i) => i * 10)); +console.log(numbers.some(() => false), numbers.every((_v) => true)); +console.log(numbers.findIndex((_, i) => i === 2), numbers.findLastIndex((_, i) => i < 3)); +console.log(numbers.find(() => false), numbers.findLast((_, i) => i === 0)); +console.log(numbers.reduce((s) => s + 1, 0), numbers.reduce((s, v) => s + 1, 100)); +console.log(numbers.reduceRight((s, _v, i) => s + i, 0)); +let visits = 0; +numbers.forEach((_, i) => { + visits += i; +}); +console.log(visits); +console.log(numbers.filter((_, i) => i > 1).map((_, i) => i)); +console.log(numbers.flatMap(() => [7]), numbers.flatMap((_, i) => [i, -i])); +console.log(numbers.filter(() => true).length, typeof (numbers.filter(() => true) as unknown[])[2]); + +// Callback values (shared helpers). +const always = (): boolean => true; +const never = (): boolean => false; +const count = (s: number): number => s + 1; +console.log(numbers.filter(always)); +console.log(numbers.some(never), numbers.every(always), numbers.find(never)); +console.log(numbers.findIndex(never), numbers.reduce(count, 0)); +console.log(numbers.map(always)); + +// A kept written element is the same value in the result. +const kept = numbers.filter(always) as unknown[]; +console.log(kept[2] === "x", kept[3] === undefined, 3 in kept, kept[4], kept.length); + +// string[] the same way. +const words: string[] = ["a", "b"]; +const wordsView: unknown[] = words; +wordsView.push(7, null); +console.log(words.filter(() => true), words.map((_, i) => `#${i}`)); +console.log(words.filter((_w, i) => i !== 1), words.reduce((s) => s + 1, 0)); + +// Holes stay holes and are skipped as in Node. +const sparse: number[] = [1]; +const sparseView: unknown[] = sparse; +sparseView[3] = "z"; +console.log(sparse.filter(() => true), sparse.map((_, i) => i), sparse.findIndex((_, i) => i === 1)); + +// A parameter used only through a shorthand property is still read. +const plain = [1, 2, 3]; +console.log(plain.map((v) => ({ v })), plain.reduce((acc, v) => ({ sum: acc.sum + v, v }), { sum: 0, v: 0 })); +console.log(plain.filter((v) => [{ v }].length > 0 && v > 1)); + +// A callback that ignores its element still receives the element as it was +// before the callback ran: filter keeps it, and find returns it, even when +// the callback overwrites or removes the slot. +const changing: number[] = [1, 2, 3]; +const changingView: unknown[] = changing; +changingView.push("x"); +let at = 0; +console.log( + changing.filter(() => { + changingView[at] = at === 1 ? "w" : 100 + at; + at++; + return true; + }), + changing, +); +const found: number[] = [1, 2]; +(found as unknown[]).push("y"); +let step = 0; +console.log( + found.find(() => { + found[step] = 50; + step++; + return step === 2; + }), + found, +); +const shrinking: number[] = [4, 5]; +(shrinking as unknown[]).push("z", 6); +console.log( + shrinking.findLast(() => { + shrinking.length = 1; + return true; + }), + shrinking.length, +); +const cut: number[] = [7, 8, 9]; +(cut as unknown[]).push("q"); +console.log( + cut.filter((_v, i) => { + if (i === 1) cut.length = 1; + return true; + }), + cut.length, +); + +// Arrays proven to hold no undefined keep their elements in filter results +// when the predicate ignores them. +const dense = [1, 2, 3, 4]; +let calls = 0; +console.log( + dense.filter((_v, i) => { + calls++; + return i % 2 === 0; + }), + calls, + dense.filter((_v, i) => i > 1), +); diff --git a/tests/corpus/8210-require-computed-missing/main.ts b/tests/corpus/8210-require-computed-missing/main.ts new file mode 100644 index 000000000..685222157 --- /dev/null +++ b/tests/corpus/8210-require-computed-missing/main.ts @@ -0,0 +1,7 @@ +// A package that probes for optional files through createRequire with +// computed specifiers. A specifier that names no file throws Node's error, +// so the package takes its absent branch as in Node. +import { probeAddon, requireErrors } from "optional-probe"; + +console.log(probeAddon("clipboard")); +for (const line of requireErrors()) console.log(line); diff --git a/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.d.ts b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.d.ts new file mode 100644 index 000000000..4eb024b0c --- /dev/null +++ b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.d.ts @@ -0,0 +1,2 @@ +export declare function probeAddon(name: string): string; +export declare function requireErrors(): string[]; diff --git a/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.js b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.js new file mode 100644 index 000000000..c824302ba --- /dev/null +++ b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.js @@ -0,0 +1,46 @@ +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const importer = fileURLToPath(import.meta.url); +const load = createRequire(import.meta.url); +const here = dirname(importer); + +// Looks for a prebuilt addon in each packaging location. +export function probeAddon(name) { + const codes = []; + const dirs = [join(here, "prebuilds", `${process.platform}-${process.arch}`), join(here, "native"), here]; + for (const dir of dirs) { + try { + const addon = load(join(dir, `${name}.node`)); + return `loaded ${typeof addon}`; + } catch (error) { + codes.push(error.code); + } + } + return `absent: ${codes.join(", ")}`; +} + +function describe(spec) { + try { + load(spec); + return "loaded"; + } catch (error) { + return `${error.name} ${error.code}: ${error.message.replaceAll(importer, "")}`; + } +} + +export function requireErrors() { + const specs = [ + "./missing.js", + "./missing", + "../missing-directory/", + "scriptc-corpus-missing-package", + "@scriptc-corpus/missing/sub", + "node:not-a-builtin", + "", + 42, + undefined, + ]; + return specs.map((spec) => `${typeof spec === "string" ? JSON.stringify(spec) : String(spec)} -> ${describe(spec)}`); +} diff --git a/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/package.json b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/package.json new file mode 100644 index 000000000..50ad53129 --- /dev/null +++ b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/package.json @@ -0,0 +1 @@ +{ "name": "optional-probe", "version": "1.0.0", "type": "module", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/corpus/8211-require-resolve-computed/main.ts b/tests/corpus/8211-require-resolve-computed/main.ts new file mode 100644 index 000000000..ab94d0625 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/main.ts @@ -0,0 +1,9 @@ +// require.resolve with requests computed at run time, called directly and +// read as a value, resolves like Node: builtins, relative files with +// Node's extension search, package "main" and "exports" entries (a +// package.json that starts with a byte order mark included, written at run +// time), a package's own name, and MODULE_NOT_FOUND for requests that name +// nothing. +import { resolveAll } from "resolve-probe"; + +for (const line of resolveAll()) console.log(line); diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/node.js b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/node.js new file mode 100644 index 000000000..c414b6b26 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/node.js @@ -0,0 +1 @@ +module.exports = "node"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/other.js b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/other.js new file mode 100644 index 000000000..7f4090ad9 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/other.js @@ -0,0 +1 @@ +module.exports = "other"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/package.json b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/package.json new file mode 100644 index 000000000..25326e6e5 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/package.json @@ -0,0 +1 @@ +{ "name": "@scope/scoped", "version": "1.0.0", "exports": { "node": "./lib/node.js", "default": "./lib/other.js" } } diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/plain-index/index.js b/tests/corpus/8211-require-resolve-computed/node_modules/plain-index/index.js new file mode 100644 index 000000000..0d24e266f --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/plain-index/index.js @@ -0,0 +1 @@ +module.exports = "index"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/data.json b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/data.json new file mode 100644 index 000000000..6d94e13bd --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/data.json @@ -0,0 +1 @@ +{ "kind": "data" } diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.d.ts b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.d.ts new file mode 100644 index 000000000..1b8cc8a0e --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.d.ts @@ -0,0 +1 @@ +export declare function resolveAll(): string[]; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js new file mode 100644 index 000000000..8d581c90e --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js @@ -0,0 +1,71 @@ +import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { dirname, join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; + +const importer = fileURLToPath(import.meta.url); +const root = dirname(dirname(dirname(importer))); +const load = createRequire(import.meta.url); + +function show(resolved) { + return resolved.startsWith("/") || /^[A-Za-z]:\\/.test(resolved) ? relative(root, resolved).replaceAll("\\", "/") : resolved; +} + +function describe(resolve, request) { + try { + return show(resolve(request)); + } catch (error) { + return `${error.name} ${error.code}: ${error.message.replaceAll(importer, "")}`; + } +} + +const requests = [ + "fs", + "node:fs", + "node:test", + "./data.json", + "./data", + "./index", + "resolve-probe", + "resolve-probe/data", + "with-exports", + "with-exports/feature", + "with-exports/lib/helper", + "with-main", + "plain-index", + "@scope/scoped", + "./missing", + "scriptc-corpus-missing-package", + "", + 42, +]; + +// A package whose package.json starts with a byte order mark, written to a +// fresh directory and resolved by its absolute path: "main" is still read. +// The directory is named by its real path (the temporary directory is a +// symbolic link on macOS), so the resolved path is relative to it. +function resolveWithByteOrderMark() { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "scriptc-bom-"))); + try { + const pkg = join(dir, "with-bom"); + mkdirSync(join(pkg, "lib"), { recursive: true }); + const manifest = JSON.stringify({ name: "with-bom", main: "lib/entry.js" }); + writeFileSync(join(pkg, "package.json"), "\uFEFF" + manifest); + writeFileSync(join(pkg, "lib", "entry.js"), 'module.exports = "with-bom";\n'); + return relative(pkg, load.resolve(pkg)); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +export function resolveAll(options = {}) { + const resolve = options.resolve ?? load.resolve; + const label = (request) => (typeof request === "string" ? JSON.stringify(request) : String(request)); + return [ + ...requests.map((request) => `call ${label(request)} -> ${describe((r) => load.resolve(r), request)}`), + ...requests.slice(0, 4).map((request) => `value ${label(request)} -> ${describe(resolve, request)}`), + `options ${describe((r) => load.resolve(r, {}), "with-main")}`, + `byte order mark -> ${resolveWithByteOrderMark()}`, + ]; +} diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/package.json b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/package.json new file mode 100644 index 000000000..d0935f646 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/package.json @@ -0,0 +1 @@ +{ "name": "resolve-probe", "version": "1.0.0", "type": "module", "exports": { ".": { "types": "./index.d.ts", "default": "./index.js" }, "./data": "./data.json" } } diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/cjs.cjs b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/cjs.cjs new file mode 100644 index 000000000..2cd122bdc --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/cjs.cjs @@ -0,0 +1 @@ +module.exports = "cjs"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/esm.mjs b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/esm.mjs new file mode 100644 index 000000000..06c676e54 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/esm.mjs @@ -0,0 +1 @@ +export default "esm"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/feature.js b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/feature.js new file mode 100644 index 000000000..33535fc6f --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/feature.js @@ -0,0 +1 @@ +module.exports = "feature"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/helper.js b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/helper.js new file mode 100644 index 000000000..4f51f7bfa --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/helper.js @@ -0,0 +1 @@ +module.exports = "helper"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/main.js b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/main.js new file mode 100644 index 000000000..0470f8201 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/main.js @@ -0,0 +1 @@ +module.exports = "main"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/package.json b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/package.json new file mode 100644 index 000000000..09d74787c --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/package.json @@ -0,0 +1 @@ +{ "name": "with-exports", "version": "1.0.0", "main": "./main.js", "exports": { ".": { "import": "./esm.mjs", "require": "./cjs.cjs" }, "./feature": "./lib/feature.js", "./lib/*": "./lib/*.js", "./private/*": null } } diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-main/lib/entry.js b/tests/corpus/8211-require-resolve-computed/node_modules/with-main/lib/entry.js new file mode 100644 index 000000000..6555fae7c --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-main/lib/entry.js @@ -0,0 +1 @@ +module.exports = "entry"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-main/package.json b/tests/corpus/8211-require-resolve-computed/node_modules/with-main/package.json new file mode 100644 index 000000000..8c473085d --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-main/package.json @@ -0,0 +1 @@ +{ "name": "with-main", "version": "1.0.0", "main": "lib/entry" } diff --git a/tests/corpus/8212-native-helper-probe-no-display/main.ts b/tests/corpus/8212-native-helper-probe-no-display/main.ts new file mode 100644 index 000000000..a9e021017 --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/main.ts @@ -0,0 +1,18 @@ +// A terminal package that loads prebuilt native helpers the way pi-tui does: +// it builds candidate paths (through require.resolve read as a value) and +// requires each one inside a try block. On Linux without DISPLAY the +// clipboard helper is never loaded, and on other platforms no helper for +// the platform is installed, so the package uses its JavaScript fallback as +// in Node. +import { getNativeClipboard, getNativeModuleCandidates, isNativeModifierPressed } from "native-helper"; +import { relative } from "node:path"; + +delete process.env["DISPLAY"]; +console.log("clipboard", typeof getNativeClipboard()); +console.log("modifier", isNativeModifierPressed("shift")); +const root = import.meta.dirname; +for (const candidate of getNativeModuleCandidates("native/helper.node")) { + console.log("candidate", relative(root, candidate).replaceAll("\\", "/")); +} +const custom = getNativeModuleCandidates("native/helper.node", { resolvePackage: () => "/opt/helper/dist/index.js" }); +console.log("custom", custom[0]!.replaceAll("\\", "/")); diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.d.ts b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.d.ts new file mode 100644 index 000000000..8803ba552 --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.d.ts @@ -0,0 +1,11 @@ +export interface NativeHelper { + getText(): string; + getImage(): Uint8Array | undefined; + isModifierPressed?(key: string): boolean; +} +export declare function getNativeClipboard(): NativeHelper | undefined; +export declare function isNativeModifierPressed(key: string): boolean; +export declare function getNativeModuleCandidates( + nativePath: string, + options?: { moduleUrl?: string; resolvePackage?: (id: string) => string; execPath?: string }, +): string[]; diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.js b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.js new file mode 100644 index 000000000..874130379 --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.js @@ -0,0 +1,2 @@ +export { getNativeClipboard, isNativeModifierPressed } from "./native-platform.js"; +export { getNativeModuleCandidates } from "./native-module-path.js"; diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-module-path.js b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-module-path.js new file mode 100644 index 000000000..2a82b41f7 --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-module-path.js @@ -0,0 +1,20 @@ +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const moduleRequire = createRequire(import.meta.url); + +// Where a prebuilt helper can live: next to the installed package, next to +// this module, and next to the executable. +export function getNativeModuleCandidates(nativePath, options = {}) { + const moduleDir = dirname(fileURLToPath(options.moduleUrl ?? import.meta.url)); + const candidates = []; + try { + const entry = (options.resolvePackage ?? moduleRequire.resolve)("native-helper"); + candidates.push(join(dirname(entry), "..", nativePath)); + } catch { + // A standalone executable has no installed package. + } + candidates.push(join(moduleDir, "..", nativePath), join(moduleDir, nativePath)); + return Array.from(new Set(candidates)); +} diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-platform.js b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-platform.js new file mode 100644 index 000000000..9aaed1dc1 --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-platform.js @@ -0,0 +1,48 @@ +import { createRequire } from "node:module"; +import * as path from "node:path"; +import { getNativeModuleCandidates } from "./native-module-path.js"; + +const load = createRequire(import.meta.url); +const helpers = new Map(); + +function loadHelper(platform, suffix = "") { + const arch = process.arch; + if (arch !== "x64" && arch !== "arm64") return undefined; + const nativePath = path.join("native", platform, "prebuilds", `${platform}-${arch}`, `${platform}-helper${suffix}.node`); + if (helpers.has(nativePath)) return helpers.get(nativePath); + for (const modulePath of getNativeModuleCandidates(nativePath)) { + try { + const helper = load(modulePath); + if (typeof helper?.getText === "function") { + helpers.set(nativePath, helper); + return helper; + } + } catch { + // Try the next location. + } + } + helpers.set(nativePath, undefined); + return undefined; +} + +function getPlatformHelper() { + if (process.platform !== "darwin" && process.platform !== "win32") return undefined; + return loadHelper(process.platform); +} + +// The clipboard helper opens the X display, so it loads only when one is set. +export function getNativeClipboard() { + if (process.platform !== "linux") return getPlatformHelper(); + if (!process.env.DISPLAY) return undefined; + return loadHelper("linux", "-x11"); +} + +export function isNativeModifierPressed(key) { + const helper = getPlatformHelper(); + if (!helper?.isModifierPressed) return false; + try { + return helper.isModifierPressed(key) === true; + } catch { + return false; + } +} diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-arm64/linux-helper-x11.node b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-arm64/linux-helper-x11.node new file mode 100644 index 000000000..72d2c39bd --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-arm64/linux-helper-x11.node @@ -0,0 +1 @@ +placeholder for a prebuilt X11 clipboard helper diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-x64/linux-helper-x11.node b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-x64/linux-helper-x11.node new file mode 100644 index 000000000..72d2c39bd --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-x64/linux-helper-x11.node @@ -0,0 +1 @@ +placeholder for a prebuilt X11 clipboard helper diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/package.json b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/package.json new file mode 100644 index 000000000..475c1d5af --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/package.json @@ -0,0 +1 @@ +{ "name": "native-helper", "version": "1.0.0", "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts" } diff --git a/tests/corpus/8230-function-record-dispatch.ts b/tests/corpus/8230-function-record-dispatch.ts new file mode 100644 index 000000000..3acde5bd7 --- /dev/null +++ b/tests/corpus/8230-function-record-dispatch.ts @@ -0,0 +1,107 @@ +// Records of functions used as dispatch tables: keyed reads consumed at once +// (`t[k]!(…)`, `t[k]?.(…)`, a checked local), missing keys, handlers that +// declare fewer parameters than the table's type, handlers that replace, +// delete or re-add table entries (their own included) while they run, and +// key order after such mutation. +type Handler = (x: number, y: number) => number; + +let log: string[] = []; + +function make(tag: string): Handler { + let calls = 0; + return (x, y) => { + calls++; + log.push(tag + ":" + calls); + return x * 10 + y; + }; +} + +const table: Record = { + add: (x, y) => x + y, + neg: (x) => -x, + pow: (x, y) => x ** y, + made: make("made"), +}; + +function keys(): string[] { + return process.argv.length > 99 ? [] : ["add", "neg", "pow", "made", "missing", ""]; +} + +for (const k of keys()) { + const h = table[k] as Handler | undefined; + console.log(JSON.stringify(k), typeof h, h === undefined, h !== undefined && h === table[k]); + console.log(" optional", table[k]?.(3, 4) ?? "none"); + if (h) console.log(" checked", h(2, 5)); + try { + console.log(" asserted", table[k]!(5, 2)); + } catch (e) { + console.log(" asserted threw", (e as Error).name); + } +} + +// A handler that replaces its own entry while it runs: the replaced closure +// keeps running with its captured state. +function selfReplacing(): Handler { + const state = { hits: 0, label: "first" }; + return (x, y) => { + table["swap"] = make("second"); + state.hits++; + log.push(state.label + " still running after replace, hits " + state.hits); + return x + y + state.hits; + }; +} +table["swap"] = selfReplacing(); +console.log("swap 1", table["swap"]!(1, 2)); +console.log("swap 2", table["swap"]?.(1, 2)); +console.log("swap 3", table["swap"]!(1, 2)); + +// A handler that deletes its own entry. +function selfDeleting(name: string): Handler { + const captured = [name, "captured"]; + return (x) => { + delete table[name]; + log.push(captured.join(" ") + " after delete"); + return x; + }; +} +table["gone"] = selfDeleting("gone"); +const goneKey = keys()[0] === "add" ? "gone" : "add"; +console.log("gone", table[goneKey]?.(7, 0) ?? "none", table[goneKey]?.(7, 0) ?? "none"); + +// A checked local keeps its closure after the entry changes. +table["kept"] = make("kept-old"); +const kept = table["kept"] as Handler | undefined; +table["kept"] = make("kept-new"); +delete table["kept"]; +if (kept) console.log("kept", kept(4, 4)); +console.log("kept entry", table["kept"] === undefined); + +// Iteration while handlers add and delete keys: key order follows inserts. +const order: Record = { + b: (x) => x + 1, + a: (x) => { + delete order["c"]; + order["z"] = (v) => v * 2; + return x + 2; + }, + c: (x) => x + 3, + "1": (x) => x + 4, +}; +for (const k in order) console.log("in", k, order[k]?.(1, 0)); +console.log("keys", Object.keys(order).join(",")); +for (const k of Object.keys(order)) { + order[k + k] = (x) => x; + console.log("of", k, order[k]!(10, 0)); +} +console.log("keys", Object.keys(order).join(",")); + +// Reassigning the table binding itself from inside a handler. +let current: Record = { + go: (x) => { + current = { go: (y) => y * 100 }; + const local = [x, x]; + return local[0]! + local[1]!; + }, +}; +console.log("rebind", current["go"]!(3, 0), current["go"]!(3, 0)); +console.log(log.join("\n")); diff --git a/tests/corpus/8231-function-table-shapes.ts b/tests/corpus/8231-function-table-shapes.ts new file mode 100644 index 000000000..108c4cee0 --- /dev/null +++ b/tests/corpus/8231-function-table-shapes.ts @@ -0,0 +1,108 @@ +// Function dispatch through other table shapes: Map, arrays of +// handlers, object literals of methods (with and without `this`), handler +// mutation of the Map during calls, and adapters for handlers that declare +// fewer parameters than the table type. +type Op = (acc: number, arg: number) => number; + +const trace: string[] = []; +const byName = new Map(); +byName.set("add", (acc, arg) => acc + arg); +byName.set("half", (acc) => acc / 2); +byName.set("reset", () => { + byName.delete("reset"); + byName.set("reset", (acc) => acc - 1); + trace.push("reset replaced itself"); + return 0; +}); +byName.set("clear", (acc) => { + const keep = [acc, "before clear"]; + byName.clear(); + trace.push(keep.join(" ") + ", size " + byName.size); + return acc; +}); + +function program(): string[] { + return process.argv.length > 99 ? [] : ["add", "half", "nop", "reset", "reset", "add", "clear", "add"]; +} + +let acc = 8; +for (const name of program()) { + const handler = byName.get(name); + if (handler !== undefined) acc = handler(acc, 3); + else trace.push("no " + name); + console.log(name, acc, byName.get(name)?.(1, 1) ?? "-"); +} + +const byCode: Op[] = [(a, b) => a - b, (a) => a * 3, (a, b) => Math.max(a, b)]; +let total = 1; +for (let i = 0; i < 9; i++) { + total = byCode[i % 4]?.(total, i) ?? total + 100; + if (i % 4 < byCode.length) total = byCode[i % 4]!(total, 1); +} +console.log("array", total); + +const counter = { + base: 5, + bump(by: number): number { + this.base += by; + return this.base; + }, + twice(by: number): number { + return this.bump(by) + this.bump(by); + }, + plain(by: number): number { + return by * 2; + }, +}; +console.log("methods", counter.bump(1), counter.twice(2), counter.plain(3), counter["plain"](4)); + +// Methods that read `this`, called through computed keys and through a +// signature adapter (a method declaring fewer parameters than its type). +interface Account { + balance: number; + deposit: (amount: number, note: string) => number; + withdraw: (amount: number, note: string) => number; + peek: (amount: number, note: string) => number; +} +const account: Account = { + balance: 10, + deposit(amount) { + this.balance += amount; + return this.balance; + }, + withdraw(amount, note) { + this.balance -= amount; + trace.push("withdraw " + note); + return this.balance; + }, + peek: (amount) => amount, +}; +for (const op of ["deposit", "withdraw", "peek", "deposit"] as const) { + console.log(op, account[op](5, "n-" + op), account.balance); +} +const detached = account.peek; +console.log("detached", detached(3, "x")); + +interface Machine { + state: string; + on: Record string>; +} +const machine: Machine = { + state: "idle", + on: { + idle: (m, input) => (input > 0 ? "running" : m.state), + running: (m, input) => { + if (input < 0) { + m.on["stopped"] = () => "idle"; + return "stopped"; + } + return m.state; + }, + }, +}; +for (const input of [0, 1, 2, -1, 5, 3]) { + const next = machine.on[machine.state]?.(machine, input) ?? machine.state; + console.log("machine", machine.state, input, "->", next); + machine.state = next; +} +console.log(trace.join("\n")); diff --git a/tests/corpus/8232-optional-function-values.ts b/tests/corpus/8232-optional-function-values.ts new file mode 100644 index 000000000..5035c5f51 --- /dev/null +++ b/tests/corpus/8232-optional-function-values.ts @@ -0,0 +1,75 @@ +// Values typed `F | undefined` for a function type F: identity and +// equality (signature adapters compare as the function they adapt), typeof, +// truthiness, storage in arrays, records, class fields and Maps, optional +// parameters and returns. +type Unary = (x: number) => number; +type Binary = (x: number, y: number) => number; + +const double: Unary = (x) => x * 2; +const asBinary: Binary = double; +const maybe = (flag: boolean): Unary | undefined => (flag ? double : undefined); + +const a = maybe(true); +const b = maybe(true); +const none = maybe(false); +console.log(a === b, a === none, none === undefined, typeof a, typeof none, !!a, !!none); +console.log(a === double, (asBinary as unknown) === (double as unknown)); + +const slots: (Unary | undefined)[] = [double, undefined, (x) => x + 1]; +console.log(slots.map((f) => (f ? f(5) : "empty")).join(" "), slots.length); +console.log(slots.indexOf(double), slots.includes(undefined), slots.filter((f) => f !== undefined).length); + +interface Hooks { + before?: Unary; + after?: Unary; +} +const hooks: Hooks = { after: (x) => x - 1 }; +console.log(hooks.before?.(1) ?? "no before", hooks.after?.(1) ?? "no after", "before" in hooks); + +class Button { + onClick: Unary | undefined; + clicks = 0; + click(): number { + this.clicks++; + return this.onClick?.(this.clicks) ?? -1; + } +} +const button = new Button(); +console.log(button.click()); +button.onClick = (n) => n * 100; +console.log(button.click(), button.click()); +button.onClick = undefined; +console.log(button.click(), button.onClick === undefined); + +const registry = new Map([ + ["set", double], + ["unset", undefined], +]); +console.log(registry.get("set")?.(4), registry.get("unset")?.(4), registry.has("unset"), registry.get("other") === undefined); + +function run(x: number, transform?: Unary, combine?: Binary): number { + const t = transform ? transform(x) : x; + return combine === undefined ? t : combine(t, x); +} +console.log(run(3), run(3, double), run(3, undefined, (p, q) => p * q), run(3, (x) => -x, asBinary)); + +function pick(name: string): Binary | undefined { + switch (name) { + case "mul": + return (x, y) => x * y; + case "first": + return (x) => x; + default: + return undefined; + } +} +for (const name of ["mul", "first", "none"]) { + const f = pick(name); + console.log(name, f?.(6, 7), f === undefined ? "undefined" : typeof f); +} + +let late: Unary | undefined; +const callLate = (): number => late?.(9) ?? 0; +console.log(callLate()); +late = (x) => x + 0.5; +console.log(callLate()); diff --git a/tests/corpus/8332-worker-shared-counted-loops.ts b/tests/corpus/8332-worker-shared-counted-loops.ts new file mode 100644 index 000000000..010531c13 --- /dev/null +++ b/tests/corpus/8332-worker-shared-counted-loops.ts @@ -0,0 +1,128 @@ +// Counted loops over typed arrays backed by a SharedArrayBuffer in a program +// with workers: loops to a length and to a bound, ascending, descending and +// by two, reads and writes at the counter and next to it, nested loops, and +// each element type. The worker and the main thread run the same loops over +// the same shared memory, one after the other. +import { isMainThread, parentPort, Worker, workerData } from "node:worker_threads"; + +interface Shared { + bytes: Uint8Array; + shorts: Int16Array; + words: Uint32Array; + ints: Int32Array; + floats: Float32Array; + doubles: Float64Array; +} + +function codes(a: Uint8Array): void { + for (let i = 0; i < a.length; ++i) { + const x = a[i]; + if (x === 97) a[i] = 0; + else if (x === 116) a[i] = 1; + else if (x === 99) a[i] = 2; + else a[i] = 3; + } +} + +function checksum(a: Uint8Array): number { + let t = 0; + for (let i = 0; i < a.length; i++) t = (t * 31 + a[i]!) % 1000003; + return t; +} + +function shiftLeft(a: Uint8Array): void { + for (let i = 0; i < a.length - 1; i++) a[i] = a[i + 1]!; +} + +function descend(a: Int16Array, n: number): number { + let t = 0; + for (let i = n - 1; i >= 0; i--) { + a[i] = a[i]! - i * 3; + t += a[i]!; + } + return t; +} + +function everyOther(a: Uint32Array): number { + let t = 0; + for (let i = 0; i < a.length; i += 2) { + a[i] = a[i]! * 2 + 4000000000; + t += a[i]! % 1000; + } + return t; +} + +function grid(a: Int32Array, n: number): number { + let t = 0; + for (let i = 0; i < n; i++) for (let j = 0; j < n; j++) t += a[i]! * a[j]! - j; + return t; +} + +function mix(f: Float32Array, d: Float64Array, n: number): number { + for (let i = 0; i < n; i++) { + f[i] = f[i]! * 0.5 + i; + d[i] = d[i]! + f[i]! / 3; + } + let t = 0; + for (let i = 0; i < n; i++) t += d[i]!; + return Math.round(t * 1000) / 1000; +} + +function fill(s: Shared): void { + const letters = [97, 116, 99, 103]; + let seed = 7; + for (let i = 0; i < s.bytes.length; i++) { + seed = (seed * 1103515245 + 12345) % 2147483648; + s.bytes[i] = letters[(seed >> 16) % 4]!; + } + for (let i = 0; i < s.shorts.length; i++) s.shorts[i] = (i * 37) % 2000 - 1000; + for (let i = 0; i < s.words.length; i++) s.words[i] = i * 2654435761; + for (let i = 0; i < s.ints.length; i++) s.ints[i] = (i % 7) - 3; + for (let i = 0; i < s.floats.length; i++) s.floats[i] = i / 4; + for (let i = 0; i < s.doubles.length; i++) s.doubles[i] = i * 1.25; +} + +function run(where: string, s: Shared): string { + codes(s.bytes); + const coded = checksum(s.bytes); + shiftLeft(s.bytes); + const shifted = checksum(s.bytes); + const down = descend(s.shorts, s.shorts.length); + const other = everyOther(s.words); + const cells = grid(s.ints, 40); + const mixed = mix(s.floats, s.doubles, s.floats.length); + return `${where}: ${coded} ${shifted} ${down} ${other} ${cells} ${mixed} ${s.bytes[0]} ${s.words[2]}`; +} + +function views(buffers: SharedArrayBuffer[]): Shared { + return { + bytes: new Uint8Array(buffers[0]!), + shorts: new Int16Array(buffers[1]!), + words: new Uint32Array(buffers[2]!), + ints: new Int32Array(buffers[3]!), + floats: new Float32Array(buffers[4]!), + doubles: new Float64Array(buffers[5]!), + }; +} + +if (isMainThread) { + const buffers = [ + new SharedArrayBuffer(10007), + new SharedArrayBuffer(2 * 515), + new SharedArrayBuffer(4 * 333), + new SharedArrayBuffer(4 * 40), + new SharedArrayBuffer(4 * 129), + new SharedArrayBuffer(8 * 129), + ]; + const shared = views(buffers); + fill(shared); + const worker = new Worker(new URL(import.meta.url), { workerData: buffers }); + worker.on("message", (line: string) => { + console.log(line); + console.log(run("main", shared)); + fill(shared); + console.log(run("main again", shared)); + }); +} else { + parentPort!.postMessage(run("worker", views(workerData as SharedArrayBuffer[]))); +} diff --git a/tests/corpus/boxed-record-identity.ts b/tests/corpus/boxed-record-identity.ts new file mode 100644 index 000000000..91bb6c13c --- /dev/null +++ b/tests/corpus/boxed-record-identity.ts @@ -0,0 +1,68 @@ +// A native record, array or class instance seen as an untyped value is the +// same object: every box of it is one value, untyped writes and expandos +// show through typed reads and through other boxes, and it keys Map, Set +// and WeakMap entries as itself. + +type Point = { x: number; y: number; tag?: string }; +type Line = { a: Point; b: Point; pts: Point[] }; + +const p: Point = { x: 1, y: 2 }; +const u1: any = p; +const u2: any = p; +console.log("same box", u1 === u2, u1 === (p as unknown)); + +// Expandos show through another box, JSON and Object.keys, and survive the +// call that added them. +u1.extra = 5; +console.log("expando via other box", u2.extra, JSON.stringify(u2), Object.keys(u2).join(",")); +function tag(o: any): void { + o.note = "n"; +} +tag(p); +console.log("expando after call", (u2 as { note: string }).note); + +// Untyped writes reach typed reads, also of optional fields, and deletes. +u1.x = 10; +console.log("typed sees write", p.x); +u1.tag = "t"; +console.log("optional via dyn", p.tag, Object.keys(p).join(",")); +delete u1.tag; +console.log("delete optional", p.tag, "tag" in p, Object.keys(p).join(",")); + +// Nested records and arrays are the typed program's own objects. +const line: Line = { a: { x: 0, y: 0 }, b: { x: 1, y: 1 }, pts: [{ x: 2, y: 2 }] }; +const ul: any = line; +ul.a.x = 7; +ul.pts[0].y = 9; +ul.pts.push({ x: 3, y: 3 }); +console.log("nested", line.a.x, line.pts[0]!.y, line.pts.length, ul.a === line.a, ul.pts === line.pts); + +// Collection keys. +const m = new Map(); +m.set(p, 1); +console.log("map key", m.get(u1), m.has(p as unknown)); +const wm = new WeakMap(); +wm.set(u1, 2); +console.log("weak key", wm.get(p)); +const s = new Set([u1, u2, p]); +console.log("set size", s.size); + +// Class instances and arrays. +class Box { + v = 1; +} +const bx = new Box(); +const ub: any = bx; +ub.extra = 3; +const ub2: any = bx; +console.log("class expando", ub2.extra, ub === ub2, JSON.stringify(bx)); +const arr: number[] = [1, 2]; +const ua: any = arr; +ua.label = "L"; +const ua2: any = arr; +console.log("array expando", ua2.label, ua === ua2); + +// A box cast back to its own type is the original. +const back = u2 as Point; +back.y = 20; +console.log("cast back", back === p, p.y); diff --git a/tests/corpus/checked-union-record-assertions.ts b/tests/corpus/checked-union-record-assertions.ts new file mode 100644 index 000000000..f5d43cee1 --- /dev/null +++ b/tests/corpus/checked-union-record-assertions.ts @@ -0,0 +1,69 @@ +// Assertions whose target no static conversion reaches: a union asserted to +// the one layout some of its arms convert into, and records or arrays whose +// member types the assertion narrows. The value is checked against the +// asserted type; these programs assert truthfully, so they run as in Node. + +type Api = "alpha" | "beta" | (string & {}); + +interface Streams { + stream: (x: number) => string; + fetch?: (x: number) => string; +} + +interface Options { + name: string; + api?: Streams | Partial>; +} + +function isStreams(value: unknown): value is Streams { + return typeof value === "object" && value !== null && "stream" in value && typeof value.stream === "function"; +} + +function run(input: Options): string[] { + const single = input.api && isStreams(input.api) ? input.api : undefined; + const byApi = single || !input.api ? undefined : (input.api as Partial>); + const all = single ? [single] : Object.values(byApi ?? {}).filter((entry) => entry !== undefined); + return all.map((entry) => `${input.name}:${entry.stream(all.length)}`); +} + +const shared: Streams = { stream: (x) => `shared${x}` }; +console.log(run({ name: "single", api: shared })); +console.log(run<"alpha" | "beta">({ name: "map", api: { alpha: { stream: (x) => `alpha${x}` }, beta: shared } })); +console.log(run<"alpha">({ name: "none" })); + +type KnownKind = "alpha" | "beta" | "gamma"; + +interface AlphaOpts { + fast?: boolean; +} +interface BetaOpts { + depth?: number; +} + +interface Item { + id: string; + kind: TKind; + opts?: TKind extends "alpha" ? AlphaOpts : TKind extends "beta" ? BetaOpts : never; +} + +interface Other { + id: string; + kind: string; + size: number; +} + +const alpha: Item<"alpha"> = { id: "a1", kind: "alpha", opts: { fast: true } }; +const gamma: Item<"gamma"> = { id: "g1", kind: "gamma" }; +const other: Other = { id: "o1", kind: "other", size: 2 }; + +const everything: (Item | Other)[] = [alpha, gamma, other, { id: "g2", kind: "gamma" }]; +const gammas = everything.filter((entry) => entry.kind === "gamma") as Item<"gamma">[]; +console.log(gammas.length, gammas.map((entry) => `${entry.id}:${entry.opts === undefined}`).join(",")); + +const items: Item[] = [gamma, alpha]; +const first = items[0]! as Item<"gamma">; +console.log(first.id, first.kind, first.opts === undefined); + +const stored: { list: (Item | Other)[] } = { list: [gamma] }; +const restored = stored.list as Item<"gamma">[]; +console.log(JSON.stringify(restored)); diff --git a/tests/corpus/constant-choice-chains.ts b/tests/corpus/constant-choice-chains.ts new file mode 100644 index 000000000..02757d3e5 --- /dev/null +++ b/tests/corpus/constant-choice-chains.ts @@ -0,0 +1,128 @@ +// if/else-if chains that store one of several literals to one typed-array +// element, chosen by comparing one number with integer literals: the +// subject's values that no case names, repeated and negative cases, +// results that do not fit the element, and chains that keep their arms. + +function bases(arr: Uint8Array): void { + for (let i = 0; i < arr.length; ++i) { + const x = arr[i]; + if (x === 97) arr[i] = 0; + else if (x === 116) arr[i] = 1; + else if (x === 99) arr[i] = 2; + else arr[i] = 3; + } +} + +const sequence = new Uint8Array(64); +for (let i = 0; i < sequence.length; i++) sequence[i] = "actgACTGnx"[i % 10]!.charCodeAt(0); +bases(sequence); +console.log("bases", sequence.join("")); + +// A number subject: NaN, -0, fractions, infinities and values far outside the cases. +const subjects = [0, -0, 1, 2, 3, 4, -1, 0.5, 2.0000001, NaN, Infinity, -Infinity, 2 ** 40, -(2 ** 40), 1e300]; +function classify(out: Int16Array, at: number, v: number): void { + if (v === 0) out[at] = 10; + else if (v === 1) out[at] = -20; + else if (v === 2) out[at] = 30; + else if (v === 1) out[at] = 99; // never reached + else if (v === -1) out[at] = 40; + else out[at] = -50; +} +const classes = new Int16Array(subjects.length); +for (let i = 0; i < subjects.length; i++) classify(classes, i, subjects[i]!); +console.log("classes", classes.join(" ")); + +// The literal on the left, and results the element converts: 300 wraps in a +// Uint8Array, -1 is 255, 1.5 truncates, and a clamped array rounds and clamps. +function convert(u8: Uint8Array, c: Uint8ClampedArray, f: Float64Array, i: Int8Array, v: number): void { + if (5 === v) u8[0] = 300; + else if (6 === v) u8[0] = -1; + else if (7 === v) u8[0] = 1.5; + else u8[0] = 8589934599; // 2 ** 33 + 7 + if (v === 5) c[0] = 300; + else if (v === 6) c[0] = -1; + else if (v === 7) c[0] = 2.5; + else c[0] = 3.5; + if (v === 5) f[0] = 0.25; + else if (v === 6) f[0] = -0; + else if (v === 7) f[0] = 1e300; + else f[0] = NaN; + if (v === 5) i[0] = 128; + else if (v === 6) i[0] = -129; + else if (v === 7) i[0] = -128; + else i[0] = 127; +} +for (const v of [5, 6, 7, 8, NaN]) { + const u8 = new Uint8Array(1); + const c = new Uint8ClampedArray(1); + const f = new Float64Array(1); + const i = new Int8Array(1); + convert(u8, c, f, i, v); + console.log("convert", v, u8[0], c[0], Object.is(f[0], -0) ? "-0" : f[0], i[0]); +} + +// An index past the end ignores the store, whatever the chain chooses. +const short = new Uint8Array(2); +for (let k = 0; k < 4; k++) { + const v = k + 1; + if (v === 1) short[k] = 11; + else if (v === 2) short[k] = 22; + else if (v === 3) short[k] = 33; + else short[k] = 44; +} +console.log("short", short.join(",")); + +// The index is the compared local. Inside each arm the range analysis +// proves its value there; on the final else's path it is past the end, also +// past 2^32 and below -2^32, and the store is ignored. +function indexed(flag: boolean): string { + const a = new Uint8Array(4); + let j = 0; + if (flag) j = 4294967297; + if (j === 0) a[j] = 11; + else if (j === 1) a[j] = 22; + else if (j === 2) a[j] = 33; + else a[j] = 44; + return a.join(","); +} +function indexedLoop(): string { + const a = new Uint8Array(8); + for (let k = 0; k < 4; k++) { + let j = 1; + if (k === 3) j = 5; + else if (k === 2) j = 4294967298; + else if (k === 1) j = -4294967294; + if (j === 1) a[j] = 11; + else if (j === 2) a[j] = 22; + else if (j === 3) a[j] = 33; + else a[j] = 44; + } + return a.join(","); +} +console.log("indexed", indexed(false), indexed(true), indexedLoop()); + +// Chains that keep their arms: an arm with a second statement, no final +// else, a different index, and a subject that is not a local. +let calls = 0; +function count(): number { + calls++; + return 9; +} +const kept = new Uint8Array(8); +for (let k = 0; k < 8; k++) { + const v = k % 5; + if (v === 0) kept[k] = 1; + else if (v === 1) { + kept[k] = 2; + calls += 100; + } else if (v === 2) kept[k] = 3; + else kept[k] = 4; + if (v === 0) kept[0] += 1; + else if (v === 1) kept[0] += 2; + else if (v === 2) kept[0] += 3; + if (count() === v) kept[k] = 7; + else if (v === 3) kept[(k + 1) % 8] = 5; + else if (v === 4) kept[k] = 6; + else kept[k] = kept[k]!; +} +console.log("kept", kept.join(","), calls); diff --git a/tests/corpus/dyn-cast-identity-fresh.ts b/tests/corpus/dyn-cast-identity-fresh.ts new file mode 100644 index 000000000..f0ef0fd53 --- /dev/null +++ b/tests/corpus/dyn-cast-identity-fresh.ts @@ -0,0 +1,21 @@ +// Casts of values nothing else holds yet: a parse result and a response +// body. They convert outright; the results are ordinary records. + +type Point = { x: number; y: number }; +type Doc = { title: string; points: Point[] }; + +const parsed = JSON.parse('{"title":"t","points":[{"x":1,"y":2},{"x":3,"y":4}]}') as Doc; +parsed.points[0]!.x = 10; +parsed.points.push({ x: 5, y: 6 }); +console.log("parse", parsed.title, parsed.points.length, JSON.stringify(parsed.points[0])); + +const again = JSON.parse('{"x":1,"y":2}') as Point; +const other = JSON.parse('{"x":1,"y":2}') as Point; +console.log("separate parses", again === other, again.x === other.x); + +async function main(): Promise { + const body = (await new Response('{"title":"r","points":[{"x":7,"y":8}]}').json()) as Doc; + body.points[0]!.y = 80; + console.log("response", body.title, JSON.stringify(body.points)); +} +main(); diff --git a/tests/corpus/dyn-cast-identity-holders.ts b/tests/corpus/dyn-cast-identity-holders.ts new file mode 100644 index 000000000..55b311462 --- /dev/null +++ b/tests/corpus/dyn-cast-identity-holders.ts @@ -0,0 +1,62 @@ +// Checked casts of dynamic values held by parameters, fields, closures and +// containers keep one identity with the holder's value. + +type Point = { x: number; y: number }; + +// A parameter: the caller's value and the cast result are one object. +function asPoint(value: unknown): Point { + return value as Point; +} +const shared: unknown = JSON.parse('{"x":1,"y":2}'); +const fromParam = asPoint(shared); +fromParam.x = 5; +console.log("parameter", (shared as Point).x, asPoint(shared) === fromParam); + +// A field of a class instance. +class Holder { + data: unknown; + constructor(data: unknown) { + this.data = data; + } +} +const holder = new Holder(JSON.parse('{"x":3,"y":4}')); +const fromField = holder.data as Point; +fromField.y = 40; +console.log("field", JSON.stringify(holder.data), (holder.data as Point) === fromField); + +// A value captured by a closure. +const captured: unknown = JSON.parse('{"x":0,"y":0}'); +const bump = (): void => { + (captured as Point).x++; +}; +bump(); +bump(); +console.log("closure", (captured as Point).x); + +// Elements of an unknown[] container. +const list: unknown[] = [JSON.parse('{"x":1,"y":1}'), JSON.parse('{"x":2,"y":2}')]; +const second = list[1] as Point; +second.x = 20; +console.log("container", JSON.stringify(list), (list[1] as Point) === second); + +// A union target with a record arm. +const maybe: unknown = JSON.parse('{"x":7,"y":8}'); +const viaUnion = maybe as Point | undefined; +if (viaUnion) viaUnion.x = 70; +console.log("union", (maybe as Point).x); + +// A record whose field is an array, mutated through both casts. +type Bag = { items: number[] }; +const bagRaw: unknown = JSON.parse('{"items":[1]}'); +const bag = bagRaw as Bag; +const items = (bagRaw as { items: unknown }).items as number[]; +items.push(2); +bag.items.push(3); +console.log("record array", bag.items.join(","), items.length, bag.items === items); + +// A native value boxed as unknown and cast back to its own type. +const nativeList: number[] = [1, 2]; +const boxedList: unknown = nativeList; +const backList = boxedList as number[]; +backList.push(3); +console.log("native array", nativeList.length, backList === nativeList); diff --git a/tests/corpus/dyn-cast-identity-workers.ts b/tests/corpus/dyn-cast-identity-workers.ts new file mode 100644 index 000000000..9ee2e42b9 --- /dev/null +++ b/tests/corpus/dyn-cast-identity-workers.ts @@ -0,0 +1,23 @@ +// Structured clones a worker receives: workerData and message payloads. +// Two casts of workerData are one object; a message payload converts once. +import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads"; + +type Job = { name: string; point: { x: number; y: number }; tags: string[] }; + +if (isMainThread) { + const worker = new Worker(new URL(import.meta.url), { + workerData: { name: "job", point: { x: 1, y: 2 }, tags: ["a"] }, + }); + worker.on("message", (message: unknown) => { + const reply = message as Job; + reply.tags.push("main"); + console.log("reply", reply.name, reply.point.x, reply.tags.join(",")); + }); +} else { + const job = workerData as Job; + const same = workerData as Job; + job.point.x = 10; + same.tags.push("worker"); + console.log("worker", same.point.x, job.tags.join(","), job === same); + parentPort!.postMessage(job); +} diff --git a/tests/corpus/dyn-cast-identity.ts b/tests/corpus/dyn-cast-identity.ts new file mode 100644 index 000000000..5ac30c52f --- /dev/null +++ b/tests/corpus/dyn-cast-identity.ts @@ -0,0 +1,71 @@ +// A checked cast of a dynamic value to a record or array (`value as T`) +// keeps the value's identity: every cast of the same source, and the source +// itself, read and write one object. + +type Point = { x: number; y: number }; + +// Two casts of one parsed object alias each other and the source. +const raw: unknown = JSON.parse('{"x":1,"y":2}'); +const a = raw as Point; +const b = raw as Point; +a.x = 10; +b.y = 20; +console.log("aliasing", a.x, a.y, b.x, b.y); +console.log("identity", a === b, (raw as Point) === a); +console.log("source", JSON.stringify(raw)); + +// Nested objects keep their identity through the outer and inner casts. +type Outer = { name: string; inner: { v: number } }; +const nestedRaw: unknown = JSON.parse('{"name":"o","inner":{"v":1}}'); +const outer = nestedRaw as Outer; +const inner = (nestedRaw as { inner: unknown }).inner as { v: number }; +inner.v = 2; +console.log("nested", outer.inner.v, outer.inner === inner); +outer.inner.v = 3; +console.log("nested back", inner.v); + +// Arrays: pushes and element writes show through every cast. +const listRaw: unknown = JSON.parse("[1,2]"); +const first = listRaw as number[]; +const second = listRaw as number[]; +first.push(3); +second[0] = 9; +console.log("array", first.length, second.length, first[0], first === second, JSON.stringify(listRaw)); + +// Arrays of records. +type Item = { id: number; tags: string[] }; +const itemsRaw: unknown = JSON.parse('[{"id":1,"tags":["a"]},{"id":2,"tags":[]}]'); +const items = itemsRaw as Item[]; +const again = itemsRaw as Item[]; +items[0]!.tags.push("b"); +again[1]!.id = 7; +console.log("array of records", again[0]!.tags.join(","), items[1]!.id, items[0] === again[0]); + +// The source stays an ordinary object for untyped reads and writes. +const bagRaw: unknown = JSON.parse('{"y":2,"x":1,"extra":"e"}'); +const point = bagRaw as Point; +const bag = bagRaw as Record; +bag.z = 3; +console.log("added", JSON.stringify(bagRaw), Object.keys(bag).join(",")); +delete bag.extra; +console.log("deleted", JSON.stringify(bagRaw), JSON.stringify(point)); +for (const k in bag) console.log("key", k, bag[k]); +point.x = 5; +console.log("typed write", bag.x); + +// A native record boxed as unknown comes back as itself. +const native: Point = { x: 1, y: 2 }; +const boxed: unknown = native; +const unboxed = boxed as Point; +unboxed.x = 10; +console.log("native", native.x, unboxed === native); + +// A fresh parse result has no other holder. +const fresh = JSON.parse('{"x":4,"y":5}') as Point; +console.log("fresh", fresh.x + fresh.y); + +// A frozen source keeps working as a copy. +const frozen: unknown = Object.freeze(JSON.parse('{"x":1,"y":2}')); +const held = [frozen]; +const thawed = frozen as Point; +console.log("frozen", thawed.x, held.length, Object.isFrozen(frozen)); diff --git a/tests/corpus/dyn-literal-identity.ts b/tests/corpus/dyn-literal-identity.ts new file mode 100644 index 000000000..915b8f73e --- /dev/null +++ b/tests/corpus/dyn-literal-identity.ts @@ -0,0 +1,83 @@ +// Object and array literals in untyped contexts (an `unknown` or `any` +// slot, an `unknown[]` element, a thrown value, an `unknown`/`any` +// parameter) keep one identity through later checked casts. + +type Counter = { count: number }; +type Item = { name: string; tags: string[] }; + +// The original example: a write through the cast shows through the holder. +const holder: any = { count: 1 }; +const item = holder as Counter; +item.count++; +console.log("any holder", holder.count, item === holder); + +const box: unknown = { count: 10 }; +const viewA = box as Counter; +const viewB = box as Counter; +viewA.count += 5; +console.log("unknown holder", viewB.count, viewA === viewB, JSON.stringify(box)); + +// Elements of an unknown[] array literal. +const values: unknown[] = [ + { name: "a", tags: ["x"] }, + { name: "b", tags: [] }, +]; +const second = values[1] as Item; +second.tags.push("y"); +console.log("unknown[] element", JSON.stringify(values), (values[1] as Item) === second); + +// Nested literals inside an untyped literal. +const tree: unknown = { name: "root", children: [{ name: "leaf", tags: ["t"] }] }; +const root = tree as { name: string; children: Item[] }; +const leaf = (tree as { children: unknown[] }).children[0] as Item; +leaf.tags.push("u"); +console.log("nested", root.children[0]!.tags.join(","), root.children[0] === leaf); + +// A thrown literal. +try { + throw { code: "E_BUSY", attempts: [1] }; +} catch (error) { + const failure = error as { code: string; attempts: number[] }; + failure.attempts.push(2); + console.log("thrown", failure.code, (error as { attempts: number[] }).attempts.length, failure === error); +} + +// Literals passed to unknown and any parameters. +function remember(value: unknown): Counter { + const counter = value as Counter; + counter.count *= 2; + return value as Counter; +} +console.log("unknown parameter", remember({ count: 4 }).count); + +function relabel(value: any): Item { + const target = value as Item; + target.name = target.name.toUpperCase(); + return value; +} +const relabeled = relabel({ name: "pkg", tags: [] }); +console.log("any parameter", relabeled.name, relabeled.tags.length); + +// A literal returned as unknown, and one stored in an unknown field. +function make(): unknown { + return { count: 0 }; +} +const made = make(); +(made as Counter).count = 7; +console.log("returned", (made as Counter).count); + +class Slot { + value: unknown = { name: "slot", tags: ["s"] }; +} +const slot = new Slot(); +(slot.value as Item).tags.push("t"); +console.log("field", JSON.stringify(slot.value)); + +// Untyped literals keep JavaScript's own key order and extra keys. +const extra: unknown = { name: "e", z: 1, tags: [], a: 2 }; +const typed = extra as Item; +typed.tags.push("k"); +console.log("extra keys", JSON.stringify(extra), Object.keys(typed).join(",")); + +// Literals handed to library calls are unchanged. +console.log(JSON.stringify({ count: 3 }), [{ n: 1 }].length); diff --git a/tests/corpus/fs-readdir-recursive.ts b/tests/corpus/fs-readdir-recursive.ts new file mode 100644 index 000000000..39fd7f5cc --- /dev/null +++ b/tests/corpus/fs-readdir-recursive.ts @@ -0,0 +1,52 @@ +// readdirSync(path, { recursive: true }) lists every entry below the path, +// relative to it: the path's own entries first, then each directory's in the +// order its entry was listed. A symlink to a directory is followed. + +import { mkdirSync, mkdtempSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = mkdtempSync(join(tmpdir(), "readdir-recursive-")); +writeFileSync(join(root, "a.txt"), "a"); +mkdirSync(join(root, "b", "d"), { recursive: true }); +writeFileSync(join(root, "b", "c.txt"), "c"); +writeFileSync(join(root, "b", "d", "e.txt"), "e"); +mkdirSync(join(root, "empty")); +mkdirSync(join(root, "z")); +writeFileSync(join(root, "z", "y.txt"), "y"); +symlinkSync(join(root, "b", "d"), join(root, "link")); + +console.log(readdirSync(root, { recursive: true }).join(" ")); +console.log(readdirSync(root, { recursive: false }).join(" ")); +console.log(readdirSync(`${root}/b/`, { recursive: true }).join(" ")); + +let deep = false; +for (const flag of [false, true]) { + deep = flag; + console.log(deep, readdirSync(join(root, "z"), { recursive: deep }).length); +} +console.log(readdirSync(join(root, "empty"), { recursive: true }).length); + +// A flag expression typed `true` or `false` still evaluates, once. +let probes = 0; +function yes(): true { + probes++; + return true; +} +function no(): false { + probes++; + return false; +} +console.log( + readdirSync(join(root, "b"), { recursive: yes() }).length, + readdirSync(join(root, "b"), { recursive: no() }).length, + probes, +); + +try { + readdirSync(join(root, "missing"), { recursive: true }); +} catch (e) { + console.log((e as NodeJS.ErrnoException).code); +} + +rmSync(root, { recursive: true }); diff --git a/tests/corpus/generic-interface-void-calls.ts b/tests/corpus/generic-interface-void-calls.ts new file mode 100644 index 000000000..63c137c10 --- /dev/null +++ b/tests/corpus/generic-interface-void-calls.ts @@ -0,0 +1,65 @@ +// Calling a void generic method through an interface value that classes +// implement (the receiver is a checked-dynamic object, and the call +// dispatches by name to the class's erased method) discards the result, as +// JavaScript does, instead of converting it to `void`. + +interface Message { + content: string; + details?: T; +} + +interface Channel { + name: string; + publish(message: Message): void; + count(items: T[]): number; +} + +const delivered: string[] = []; + +class LogChannel implements Channel { + name: string; + constructor(name: string) { + this.name = name; + } + publish(message: Message): void { + delivered.push(`${this.name}:${message.content}:${message.details === undefined ? "-" : JSON.stringify(message.details)}`); + } + count(items: T[]): number { + return items.length; + } +} + +class LoudChannel extends LogChannel { + publish(message: Message): void { + super.publish({ content: message.content.toUpperCase(), details: message.details }); + } +} + +const channels: Channel[] = [new LogChannel("quiet"), new LoudChannel("loud")]; +for (const channel of channels) { + channel.publish({ content: "hello", details: [1, 2] }); + channel.publish({ content: "bare" }); + const result = channel.publish({ content: "typed", details: "x" }); + console.log(channel.name, result === undefined, channel.count(["a", "b", "c"])); +} +console.log(delivered.join(" | ")); + +// A class may implement the void member with a method that returns a value; +// JavaScript passes that value on. +class CountingChannel implements Channel { + name = "counting"; + sent = 0; + publish(message: Message): number { + this.sent++; + return message.content.length; + } + count(items: T[]): number { + return items.length * 2; + } +} + +const counted: Channel[] = [new CountingChannel(), new LogChannel("plain")]; +for (const channel of counted) { + console.log(channel.name, channel.publish({ content: "four" })); + console.log(`sent ${channel.publish({ content: "again", details: 2 })}`); +} diff --git a/tests/corpus/localized-failure-cycles.ts b/tests/corpus/localized-failure-cycles.ts new file mode 100644 index 000000000..2db499707 --- /dev/null +++ b/tests/corpus/localized-failure-cycles.ts @@ -0,0 +1,29 @@ +// Localized failure, stage L2: reference cycles through dynamic slots are +// collected (instance -> dynamic field -> capsule -> instance). +type Peer = Link | Map string> | AbortSignal | (() => number); + +class Link { + id: number; + peer: Peer | undefined; + constructor(id: number) { + this.id = id; + this.peer = undefined; + } + peerId(): number { + const p = this.peer; + return p instanceof Link ? p.id : -1; + } +} + +let total = 0; +for (let i = 0; i < 20000; i++) { + const a = new Link(i); + const b = new Link(i + 1); + a.peer = b; + b.peer = a; + total += a.peerId() - b.peerId(); +} +const keep = new Link(7); +keep.peer = () => keep.id * 2; +const f = keep.peer; +console.log(total, typeof f === "function" ? f() : 0, keep.peerId()); diff --git a/tests/corpus/localized-failure-declarations.ts b/tests/corpus/localized-failure-declarations.ts new file mode 100644 index 000000000..c4acdfbaa --- /dev/null +++ b/tests/corpus/localized-failure-declarations.ts @@ -0,0 +1,64 @@ +// Localized failure, stage L2: fields, parameters and returns whose +// declared type has no static lowering become checked dynamic slots, so +// their class or function still compiles. +interface ToolDefinition { + name: string; + execute(id: string, params: any): string; +} +type Progress = (done: number, total: number) => void; +type ProgressOrSignal = Progress | AbortSignal | Map string> | Set; + +class Session { + // A field: the class registers, the field holds the array by reference. + private tools: ToolDefinition[] = []; + state: any = { count: 0 }; + readonly label: string; + constructor(label: string) { + this.label = label; + } + add(tool: ToolDefinition): void { + this.tools.push(tool); + } + run(id: string, params: unknown): string[] { + const out: string[] = []; + for (const tool of this.tools) out.push(tool.execute(id, params)); + this.state.count = this.state.count + 1; + return out; + } + count(): number { + return this.tools.length; + } + // A return slot. + first(): ToolDefinition | ProgressOrSignal | undefined { + return this.tools[0]; + } +} + +// A parameter: the function registers and is called from typed code. +function listAll(dir: string, onProgress?: ProgressOrSignal): number { + if (typeof onProgress === "function") onProgress(1, dir.length); + else if (onProgress instanceof Set) return onProgress.size; + return dir.length; +} + +// A base class with a dynamic parameter, and an override. +class Base { + notify(target: ProgressOrSignal | undefined): string { + return target === undefined ? "base:none" : "base:" + typeof target; + } +} +class Derived extends Base { + override notify(target: ProgressOrSignal | undefined): string { + return "derived/" + super.notify(target); + } +} + +const echo: ToolDefinition = { name: "echo", execute: (id, params) => `${id}:${JSON.stringify(params)}` }; +const s = new Session("main"); +s.add(echo); +s.add({ name: "len", execute: (id) => String(id.length) }); +console.log(s.label, s.count(), s.run("abc", { k: 1 }).join(" | "), s.state.count); +console.log("same tool", s.first() === echo); +console.log(listAll("hello"), listAll("hello", (d, t) => console.log("progress", d, t)), listAll("x", new Set([1, 2]))); +const objs: Base[] = [new Base(), new Derived()]; +for (const o of objs) console.log(o.notify(undefined), o.notify((a, b) => a + b)); diff --git a/tests/corpus/localized-failure-forward-capture.ts b/tests/corpus/localized-failure-forward-capture.ts new file mode 100644 index 000000000..ba43a037d --- /dev/null +++ b/tests/corpus/localized-failure-forward-capture.ts @@ -0,0 +1,19 @@ +// Localized failure, stage L1: a statement that declares a binding an +// earlier closure captures, and then a binding whose declared type has no +// static lowering, still initializes the captured binding. +type Callbacks = ((done: number) => void) | AbortSignal | Map string> | Set; + +function run(): void { + const read = (): number => total; + const total = 3, + hook: Callbacks = new Set([1, 2]); + console.log(read(), hook instanceof Set, hook instanceof Set && hook.size); + let late = (): string => label; + let label = "first", + other: Callbacks = new Set([3]); + console.log(late(), other instanceof Set); + label = "second"; + late = (): string => label + "!"; + console.log(late()); +} +run(); diff --git a/tests/corpus/localized-failure-function-slots.ts b/tests/corpus/localized-failure-function-slots.ts new file mode 100644 index 000000000..0a7fdf641 --- /dev/null +++ b/tests/corpus/localized-failure-function-slots.ts @@ -0,0 +1,28 @@ +// Localized failure, stage L2: parameters and returns of arrows, function +// expressions and generic instances become checked dynamic slots. +type Hook = Map string> | AbortSignal | (() => number) | Set; + +interface Model { + id: T; + hook: Hook; +} + +function retry(label: string, request: () => Promise | Hook): T | string { + const r = request(); + return typeof r === "function" ? label + ":" + String(r()) : label + ":other"; +} + +function describe(model: Model): string { + return `${String(model.id)}/${model.hook instanceof Set ? model.hook.size : typeof model.hook}`; +} + +const run = (hook: Hook, n: number): string => (typeof hook === "function" ? String(hook() + n) : "set"); +const make = function (size: number): Hook { + return new Set(Array.from({ length: size }, (_, i) => `k${i}`)); +}; + +console.log(run(() => 40, 2), run(new Set(["a"]), 1)); +const h = make(3); +console.log(h instanceof Set ? h.size : -1); +console.log(retry("a", () => () => 7), retry("b", () => new Set())); +console.log(describe({ id: 5, hook: new Set(["x", "y"]) }), describe({ id: "s", hook: () => 1 })); diff --git a/tests/corpus/localized-failure-held-values.ts b/tests/corpus/localized-failure-held-values.ts new file mode 100644 index 000000000..ee52498c1 --- /dev/null +++ b/tests/corpus/localized-failure-held-values.ts @@ -0,0 +1,70 @@ +// Localized failure, stage L1: values of every kind held in a local or +// module global whose declared type has no static lowering keep their +// identity and behavior. +class Point { + x: number; + y: number; + constructor(x: number, y: number) { + this.x = x; + this.y = y; + } + norm(): number { + return Math.abs(this.x) + Math.abs(this.y); + } +} +interface Cfg { + name: string; + depth: number; +} +type Anything = + | Point + | Cfg + | number[] + | Map string> + | AbortSignal + | (() => number) + | Promise; + +const p = new Point(3, -4); +const cfg: Cfg = { name: "a", depth: 1 }; +const xs = [1, 2, 3]; +let counter = 0; +const tick = (): number => ++counter; + +// A module global holding a class instance. +let held: Anything = p; +if (held instanceof Point) { + console.log("point", held.norm(), held === p); + held.x = 10; +} +console.log("typed alias", p.x, p.norm()); + +// A record: mutation through the hole is visible in the typed alias. +held = cfg; +if (!(held instanceof Point) && !Array.isArray(held) && typeof held === "object" && "depth" in held) { + held.depth = 5; + console.log("record", held.name, held === cfg); +} +console.log("typed alias", cfg.depth, JSON.stringify(cfg), Object.keys(cfg).join(",")); + +// An array: pushes through the hole show in typed code. +held = xs; +if (Array.isArray(held)) { + held.push(4); + console.log("array", held.length, held === xs); +} +console.log("typed alias", xs.length, xs.join("+")); + +function main(): void { + // A local holding a closure, called through the hole. + let local: Anything = tick; + if (typeof local === "function") { + console.log("closure", local(), local(), counter, local === tick); + } + // A promise, awaited through the hole. + local = Promise.resolve(42); + if (local instanceof Promise) { + local.then((v) => console.log("promise", v)); + } +} +main(); diff --git a/tests/corpus/localized-failure-locals.ts b/tests/corpus/localized-failure-locals.ts new file mode 100644 index 000000000..418b86d87 --- /dev/null +++ b/tests/corpus/localized-failure-locals.ts @@ -0,0 +1,43 @@ +// Localized failure, stage L1: locals, loop variables and module globals +// whose declared type has no static lowering hold checked-dynamic values. +type Progress = (done: number, total: number) => void; +type Callbacks = Progress | AbortSignal | Map string> | Set; + +const report: Progress = (done, total) => console.log(`progress ${done}/${total}`); +const seen = new Set([1, 2, 3]); + +// A module global, written with typed values of several arms. +let current: Callbacks | undefined = report; + +function describe(label: string): void { + // A local of the same declared type. + const local: Callbacks | undefined = current; + if (local === undefined) { + console.log(label, "none"); + } else if (typeof local === "function") { + local(1, 2); + console.log(label, "function", local === report); + } else if (local instanceof Set) { + console.log(label, "set", local.size, local.has(2), local === seen); + local.add(4); + } else { + console.log(label, "other"); + } +} + +describe("first"); +current = seen; +describe("second"); +console.log("typed alias sees the write", seen.size, seen.has(4)); +current = undefined; +describe("third"); + +// A loop variable over an array whose element type does not compile. +const all: (Callbacks | number)[] = [report, seen, 7]; +for (const item of all) { + if (typeof item === "number") console.log("number", item + 1); + else if (typeof item === "function") item(3, 4); + else console.log("object", item instanceof Set); +} +all.push(9); +console.log(all.length); diff --git a/tests/corpus/localized-failure-pi-shapes.ts b/tests/corpus/localized-failure-pi-shapes.ts new file mode 100644 index 000000000..9879606eb --- /dev/null +++ b/tests/corpus/localized-failure-pi-shapes.ts @@ -0,0 +1,37 @@ +// Localized failure, stage L2: pi-shaped declarations. +interface Token { + text: string; +} +interface AgentMessage { + role: "user" | "assistant"; + content: string[]; +} +type CreateRuntime = (cwd: string) => Promise<{ session: Session; extra: Map string> | AbortSignal }>; + +class Session { + messages: AgentMessage[] = []; + // A field whose union mentions a stdlib type with no lowering: the class + // registers; constructing a WeakRef would still be refused. + cachedTokens?: WeakRef<{ source: string; tokens: Token[] }> | undefined; + rendererState: any = {}; + effectState: Record = {}; + private readonly createRuntime: CreateRuntime | undefined; + constructor(createRuntime: CreateRuntime | undefined) { + this.createRuntime = createRuntime; + } + hasRuntime(): boolean { + return this.createRuntime !== undefined; + } + clear(): AgentMessage[] { + // An inferred return over a spread clone. + return this.messages.map((message) => ({ ...message, content: [] })); + } +} + +const s = new Session(undefined); +const withRuntime = new Session(async (cwd) => ({ session: s, extra: new Map() })); +s.messages.push({ role: "user", content: ["hi"] }, { role: "assistant", content: ["yo", "!"] }); +s.rendererState.mode = "fast"; +s.effectState["k"] = 3; +console.log(s.hasRuntime(), s.cachedTokens === undefined, s.rendererState.mode, s.effectState["k"]); +console.log(JSON.stringify(s.clear()), s.messages[1]!.content.length, withRuntime.hasRuntime()); diff --git a/tests/corpus/localized-failure-registered-statics.ts b/tests/corpus/localized-failure-registered-statics.ts new file mode 100644 index 000000000..9ca49d4f2 --- /dev/null +++ b/tests/corpus/localized-failure-registered-statics.ts @@ -0,0 +1,46 @@ +// Localized failure, stage L2: a class that registers with dynamic slots +// runs its static field initializers where its statement evaluates, and a +// generic class's static methods compile once it registers. +type Progress = (done: number, total: number) => void; +type Callbacks = Progress | AbortSignal | Map string> | Set; + +function sideEffect(label: string): number { + console.log("static init", label); + return label.length; +} + +// Registers at its statement: the initializers run before the next line. +class Registry { + state: any = { n: 1 }; + static count = sideEffect("registry"); + static names: string[] = ["a", "b"]; + read(): number { + return this.state.n; + } +} + +console.log("after the class statement"); + +// Never referenced: its initializer runs all the same. +class Unused { + slot: any = null; + static ready = sideEffect("unused"); +} + +// A generic class whose static method has a dynamic parameter. +class Box { + value: T; + constructor(v: T) { + this.value = v; + } + static describe(target: Callbacks | undefined): string { + return target === undefined ? "none" : typeof target; + } + static label(): string { + return "box"; + } +} + +const r = new Registry(); +console.log(r.read(), Registry.count, Registry.names.join(",")); +console.log(Box.label(), Box.describe(undefined), Box.describe(new Set([1])), new Box(3).value); diff --git a/tests/corpus/method-signature-any-params.ts b/tests/corpus/method-signature-any-params.ts new file mode 100644 index 000000000..e505ae78c --- /dev/null +++ b/tests/corpus/method-signature-any-params.ts @@ -0,0 +1,39 @@ +// A method or call signature of an interface whose parameter is typed `any` +// (directly, or through an alias that resolves to `any`) takes whatever the +// caller passes, as a function type's `any` parameter already does: the +// interface compiles, and implementations receive the checked-dynamic value. + +type Loose = any; + +interface Tool { + name: string; + execute(id: string, params: any, extra?: Loose): Promise; + describe(value: Loose): string; +} + +interface Formatter { + (value: any): string; +} + +const tools: Tool[] = [ + { + name: "count", + execute: async (id, params) => `${id} has ${Object.keys(params ?? {}).length} keys`, + describe: (value) => `count(${Array.isArray(value) ? "array" : typeof value})`, + }, +]; + +const format: Formatter = (value) => (typeof value === "number" ? value.toFixed(1) : String(value)); + +async function main(): Promise { + const inputs: unknown[] = [{ a: 1, b: [2, 3] }, "text", 42, null, [1, 2]]; + for (const tool of tools) { + for (const input of inputs) { + console.log(tool.name, await tool.execute("call", input), tool.describe(input)); + } + console.log(tool.name, await tool.execute("extra", { k: true }, { more: "yes" })); + } + console.log(format(3), format("x"), format(undefined)); +} + +main().catch((e: unknown) => console.log("failed", e instanceof Error ? e.message : String(e))); diff --git a/tests/corpus/never-narrowed-literal-argument.ts b/tests/corpus/never-narrowed-literal-argument.ts new file mode 100644 index 000000000..a9bb5489f --- /dev/null +++ b/tests/corpus/never-narrowed-literal-argument.ts @@ -0,0 +1,46 @@ +// A string-literal union narrowed to `never` passes to a `never` parameter: +// the exhaustiveness-check idiom over if-chains and switches compiles, and the +// unreachable call never runs. + +function unreachable(_: never): never { + throw new Error("unreachable"); +} + +type Kind = "a" | "b"; +function viaIf(kind: Kind): number { + if (kind === "a") return 1; + if (kind === "b") return 2; + return unreachable(kind); +} + +type Shape = "circle" | "square" | "triangle"; +function corners(shape: Shape): number { + switch (shape) { + case "circle": + return 0; + case "square": + return 4; + case "triangle": + return 3; + default: + return unreachable(shape); + } +} + +function asserted(kind: Kind): string { + if (kind === "a" || kind === "b") return kind.toUpperCase(); + return unreachable(kind as never); +} + +type Level = 1 | 2; +function viaNumbers(level: Level): string { + if (level === 1) return "one"; + if (level === 2) return "two"; + return unreachable(level); +} + +const kinds: Kind[] = ["a", "b"]; +console.log(kinds.map(viaIf).join(","), viaIf(Date.now() > 0 ? "a" : "b")); +const shapes: Shape[] = ["circle", "square", "triangle"]; +console.log(shapes.map(corners).join(",")); +console.log(kinds.map(asserted).join(""), viaNumbers(1), viaNumbers(2)); diff --git a/tests/corpus/rest-tuple-shorthand-property.ts b/tests/corpus/rest-tuple-shorthand-property.ts new file mode 100644 index 000000000..d67afa443 --- /dev/null +++ b/tests/corpus/rest-tuple-shorthand-property.ts @@ -0,0 +1,23 @@ +// A rest parameter typed as a fixed tuple, read through a shorthand property +// (`{ xs }`), directly or through a const alias, is the tuple of the call's +// arguments. + +function wrap(...xs: [number]) { + return { xs }; +} +function pair(...args: [string, number]) { + const kept = args; + return { kept, size: args.length }; +} +function add(a: number, b: number): number { + return a + b; +} +function forward(...args: [number, number]): number { + return add(...args); +} + +const w = wrap(1); +console.log(w.xs[0], w.xs.length, JSON.stringify(w)); +const p = pair("a", 2); +console.log(p.kept[0], p.kept[1], p.size, JSON.stringify(p)); +console.log(forward(3, 4)); diff --git a/tests/corpus/speculative-loop-view-headers.ts b/tests/corpus/speculative-loop-view-headers.ts new file mode 100644 index 000000000..a1b92c1d9 --- /dev/null +++ b/tests/corpus/speculative-loop-view-headers.ts @@ -0,0 +1,36 @@ +// Speculative integer loops over typed-array views held by a const local +// and by a parameter. The fast copy, the copy a failed check resumes in and +// the ordinary copy all read and write the same views: a product that +// leaves the safe integers fails the fast copy's check partway through, +// and writes past the end are ignored. +function fill(buffer: ArrayBuffer, count: number, seed: number): number { + const view = new Float64Array(buffer); + let l = seed; + for (let i = 0; i < count; i++) { + l = (l * 3877 + 29573) % 139968; + view[i] = l; + } + return l; +} + +function grow(out: Int32Array, count: number, start: number): number { + let x = start; + for (let i = 0; i < count; i++) { + x = x * 3 + 1; + out[i] = x % 1000; + } + return x; +} + +function sum(view: Float64Array | Int32Array): number { + let s = 0; + for (const v of view) s += v; + return s; +} + +const buffer = new ArrayBuffer(8 * 64); +console.log(fill(buffer, 64, 42), sum(new Float64Array(buffer))); +console.log(fill(buffer, 100, 7), new Float64Array(buffer)[63]); +const out = new Int32Array(40); +console.log(grow(out, 60, 1), sum(out), out[0], out[39]); +console.log(grow(out, 10, 5), out.join(",")); diff --git a/tests/corpus/string-plus-element-read-return.ts b/tests/corpus/string-plus-element-read-return.ts new file mode 100644 index 000000000..ec07fa3d5 --- /dev/null +++ b/tests/corpus/string-plus-element-read-return.ts @@ -0,0 +1,42 @@ +// A function declared to return a string concatenates a string with an +// element of a readonly string array. A missing element reads as undefined, +// which `+` turns into "undefined" next to a string, so the result is always +// a string and its call can be used inline. + +const PAD: readonly string[] = ["00", "01", "02"]; + +function label(i: number): string { + return "x-" + PAD[i]; +} +function date(y: number, m: number, d: number): string { + return y + "-" + PAD[m] + "-" + PAD[d]; +} +function suffix(i: number): string { + return PAD[i] + "!"; +} + +console.log(label(1).length, label(1), label(7)); +console.log(date(2026, 1, 2).length, date(2026, 2, 9)); +console.log(suffix(0).toUpperCase(), suffix(5)); + +// Two possibly missing reads add to NaN when both are missing. +function pair(i: number, j: number): string { + return PAD[i] + PAD[j]; +} +const both = pair(8, 9); +console.log(pair(0, 1), typeof both, String(both)); + +// A missing element held in a binding or returned by a function is undefined, +// so two of them still add to NaN. +function at(i: number): string { + return PAD[i]; +} +function viaBinding(i: number, j: number): string { + const head = PAD[i]; + return head + PAD[j]; +} +function viaCall(i: number, j: number): string { + return at(i) + PAD[j]; +} +console.log(viaBinding(0, 1), String(viaBinding(5, 6)), typeof viaBinding(5, 6)); +console.log(viaCall(0, 2), String(viaCall(5, 6)), viaBinding(0, 9)); diff --git a/tests/corpus/symbol-keyed-recursive-record.ts b/tests/corpus/symbol-keyed-recursive-record.ts new file mode 100644 index 000000000..c26d7282e --- /dev/null +++ b/tests/corpus/symbol-keyed-recursive-record.ts @@ -0,0 +1,35 @@ +// A recursive record keyed by unique symbols, built from literals whose +// child arrays start empty, holds the nodes later pushed into those arrays. + +const kValue: unique symbol = Symbol("value"); +const kChildren: unique symbol = Symbol("children"); + +interface TreeNode { + [kChildren]: TreeNode[]; + [kValue]: number; +} + +function node(value: number): TreeNode { + return { [kValue]: value, [kChildren]: [] }; +} + +const root = node(0); +root[kChildren].push(node(1)); +root[kChildren].push(node(2)); +root[kChildren][0]![kChildren].push(node(3)); +console.log(root[kChildren].length, root[kChildren][0]![kValue], root[kChildren][1]![kValue]); + +function sum(n: TreeNode): number { + let total = n[kValue]; + for (const child of n[kChildren]) total += sum(child); + return total; +} +function depth(n: TreeNode): number { + return 1 + Math.max(0, ...n[kChildren].map(depth)); +} +console.log(sum(root), depth(root)); + +// A literal that starts with children. +const seeded: TreeNode = { [kValue]: 10, [kChildren]: [node(11)] }; +seeded[kChildren].push(node(12)); +console.log(seeded[kChildren].map((c) => c[kValue]).join(","), sum(seeded)); diff --git a/tests/corpus/undeclared-key-optional-chain/main.cjs b/tests/corpus/undeclared-key-optional-chain/main.cjs new file mode 100644 index 000000000..de7b3351a --- /dev/null +++ b/tests/corpus/undeclared-key-optional-chain/main.cjs @@ -0,0 +1,11 @@ +// A package typed from its JavaScript returns profiles without a key its +// declarations promise ("a;b"). An optional read of that key answers +// undefined for a profile, and for a missing one (the chain reads the +// nullish receiver as undefined), in presence tests and in values. +const { loadProfiles } = require("hostilekit"); +const profiles = loadProfiles(); +const present = profiles[0]; +const missing = profiles[5]; +console.log(present?.["a;b"] === undefined, present?.["a;b"] ?? "none", typeof present?.["a;b"]); +console.log(missing?.["a;b"] === undefined, missing?.["a;b"] ?? "none", typeof missing?.["a;b"]); +console.log(present["a;b"] === undefined, present.id); diff --git a/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.d.ts b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.d.ts new file mode 100644 index 000000000..4bf22dfea --- /dev/null +++ b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.d.ts @@ -0,0 +1,2 @@ +export type Profile = { id: number; "a;b": string }; +export declare function loadProfiles(): Profile[]; diff --git a/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.js b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.js new file mode 100644 index 000000000..316501256 --- /dev/null +++ b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.js @@ -0,0 +1,5 @@ +"use strict"; +function loadProfiles() { + return [{ id: 1 }]; +} +exports.loadProfiles = loadProfiles; diff --git a/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/package.json b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/package.json new file mode 100644 index 000000000..2fdbc5b41 --- /dev/null +++ b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/package.json @@ -0,0 +1 @@ +{ "name": "hostilekit", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/corpus/union-clone-empty-array-overrides.ts b/tests/corpus/union-clone-empty-array-overrides.ts new file mode 100644 index 000000000..98ddac59c --- /dev/null +++ b/tests/corpus/union-clone-empty-array-overrides.ts @@ -0,0 +1,92 @@ +// Spreading a union of records and overriding a shared field with an empty +// array literal keeps each value's own variant: the `[]` (typed `never[]`) +// becomes an empty array of the selected arm's element type, also when that +// arm's field is a union with one array arm (`string | Part[]`). + +interface TextPart { + type: "text"; + text: string; +} +interface ImagePart { + type: "image"; + data: string; +} +interface UserEntry { + role: "user"; + content: string | (TextPart | ImagePart)[]; + timestamp: number; +} +interface AssistantEntry { + role: "assistant"; + content: TextPart[]; + model: string; + timestamp: number; +} +interface ToolEntry { + role: "tool"; + toolCallId: string; + content: (TextPart | ImagePart)[]; + isError: boolean; + timestamp: number; +} +interface SystemEntry { + role: "system"; + content: string; + timestamp: number; +} +interface NoteEntry { + role: "note"; + text: string; + timestamp: number; +} +type Entry = UserEntry | AssistantEntry | ToolEntry | SystemEntry | NoteEntry; + +function repair(entry: Entry, clear: boolean): Entry[] { + if (entry.role === "system" && clear) return [{ ...entry, content: "" }]; + if ((entry.role === "user" || entry.role === "assistant" || entry.role === "tool") && clear) { + return [{ ...entry, content: [] }]; + } + return [entry]; +} + +interface Circle { + kind: "circle"; + tags: string[]; + radius: number; +} +interface Square { + kind: "square"; + tags: string[]; + side: number; +} +type Shape = Circle | Square; + +function cleared(shape: Shape): Shape { + return { ...shape, tags: [] }; +} + +const entries: Entry[] = [ + { role: "user", content: [{ type: "text", text: "hello" }], timestamp: 1 }, + { role: "assistant", content: [{ type: "text", text: "hi" }], model: "m", timestamp: 2 }, + { role: "tool", toolCallId: "t1", content: [{ type: "image", data: "AA==" }], isError: false, timestamp: 3 }, + { role: "note", text: "hi", timestamp: 4 }, + { role: "system", content: "be brief", timestamp: 5 }, + { role: "user", content: "kept", timestamp: 6 }, +]; +entries.forEach((entry, index) => { + for (const fixed of repair(entry, index % 2 === 0 || entry.role === "system")) { + console.log(fixed.role, JSON.stringify(fixed), fixed === entry); + } +}); +console.log(JSON.stringify(entries[0])); + +const shapes: Shape[] = [ + { kind: "circle", tags: ["round", "red"], radius: 2 }, + { kind: "square", tags: ["blue"], side: 3 }, +]; +for (const shape of shapes) { + const copy = cleared(shape); + copy.tags.push("fresh"); + const area = copy.kind === "circle" ? copy.radius * copy.radius * 3 : copy.side * copy.side; + console.log(copy.kind, area, copy.tags.join("+"), shape.tags.join("+"), copy !== shape); +} diff --git a/tests/corpus/union-record-kept-fields-arm.ts b/tests/corpus/union-record-kept-fields-arm.ts new file mode 100644 index 000000000..69887cf49 --- /dev/null +++ b/tests/corpus/union-record-kept-fields-arm.ts @@ -0,0 +1,56 @@ +// A record flowing into a discriminated union where one record arm holds it +// by dropping fields and another keeps every field but declares an optional +// field the record lacks. The record's discriminant selects the arm that keeps +// its fields. + +type Loc = { file: string; start: number }; + +type Expr = + | { kind: "ref"; type: number; loc: Loc } + | { + kind: "closure"; + fnName: string; + captures: string[]; + adapts?: true; + type: number; + loc: Loc; + } + | { kind: "call"; callee: string; args: Expr[]; type: number; loc: Loc }; + +function closureOf(name: string, loc: Loc): Expr { + const closure = { + kind: "closure" as const, + fnName: name, + captures: [], + type: 1, + loc, + }; + return closure; +} + +function describe(expr: Expr): string { + if (expr.kind === "closure") + return `closure ${expr.fnName}/${expr.captures.length} ${expr.adapts ?? "-"}`; + if (expr.kind === "call") + return `call ${expr.callee}(${expr.args.map(describe).join(", ")})`; + return `ref ${expr.type}`; +} + +const loc: Loc = { file: "a.ts", start: 3 }; +const first = closureOf("pick", loc); +console.log(first.kind, describe(first), JSON.stringify(first)); + +const exprs: Expr[] = [first]; +const captured = { + kind: "closure" as const, + fnName: "inner", + captures: ["x"], + type: 2, + loc, +}; +exprs.push(captured); +exprs.push({ kind: "ref", type: 7, loc }); +console.log( + describe({ kind: "call", callee: "run", args: exprs, type: 0, loc }), +); +console.log(exprs.map((e) => e.kind).join(",")); diff --git a/tests/corpus/union-record-kept-fields-no-discriminant.ts b/tests/corpus/union-record-kept-fields-no-discriminant.ts new file mode 100644 index 000000000..45370f152 --- /dev/null +++ b/tests/corpus/union-record-kept-fields-no-discriminant.ts @@ -0,0 +1,39 @@ +// A record flowing into a union without a discriminant where one record arm +// holds it by dropping fields and another keeps every field but declares an +// optional field the record lacks. The record converts into the arm that +// keeps its fields, and the optional field stays absent. + +type Loc = { file: string; start: number }; + +type Slot = + | { type: number; loc: Loc } + | { + fnName: string; + captures: string[]; + adapts?: true; + type: number; + loc: Loc; + }; + +const loc: Loc = { file: "a.ts", start: 3 }; + +function slotOf(name: string): Slot { + const slot = { fnName: name, captures: [], type: 1, loc }; + return slot; +} + +const slot = slotOf("stored"); +console.log("fnName" in slot, "adapts" in slot, JSON.stringify(slot)); +if ("fnName" in slot) + console.log(slot.fnName, slot.adapts, slot.captures.length); + +function describe(value: Slot): string { + return "fnName" in value + ? `${value.fnName}/${value.captures.length}` + : `type ${value.type}`; +} + +const inner = { fnName: "inner", captures: ["x"], type: 2, loc }; +const slots: Slot[] = [slot, { type: 7, loc }]; +slots.push(inner); +console.log(slots.map(describe).join(", "), JSON.stringify(slots[2])); diff --git a/tests/corpus/union-tuple-element-read.ts b/tests/corpus/union-tuple-element-read.ts new file mode 100644 index 000000000..141609d2b --- /dev/null +++ b/tests/corpus/union-tuple-element-read.ts @@ -0,0 +1,45 @@ +// Literal-index reads from a union of tuples whose element types differ: +// each tuple's own element is read, and the checker's narrowing of the +// tuple (or of the element) applies as usual. + +type Ok = readonly [true, T]; +type Err = readonly [false, E]; +type Result = Ok | Err; + +function pick(n: number): Result { + return n > 0 ? [true, n] : [false, "neg"]; +} + +for (const n of [1, -1, 5]) { + const r = pick(n); + console.log(r[0], r[1]); + if (r[0]) console.log(r[1] + 1); + else console.log(r[1].length); +} + +function describe(r: Result): string { + const value = r[1]; + return `${r[0] ? "ok" : "err"}:${typeof value}:${value}`; +} +console.log(describe(pick(2)), describe(pick(0))); + +function isOk(r: Result): boolean { + return r[0]; +} +function payload(r: Result): T | E { + return r[1]; +} +console.log(isOk(pick(3)), isOk(pick(-3)), payload(pick(4)), payload(pick(-4))); + +type Triple = readonly [string, number, boolean] | readonly [number, string, boolean]; +const triples: Triple[] = [ + ["a", 1, true], + [2, "b", false], +]; +for (const t of triples) console.log(typeof t[0], typeof t[1], t[2]); + +// A read through a single tuple view of the union. +function flag(r: Result): boolean { + return (r as readonly [boolean, T | E])[0]; +} +console.log(flag(pick(1)), flag(pick(-1))); diff --git a/tests/corpus/var-object-filled-later.js b/tests/corpus/var-object-filled-later.js new file mode 100644 index 000000000..009d8b54d --- /dev/null +++ b/tests/corpus/var-object-filled-later.js @@ -0,0 +1,48 @@ +// A function-scoped `var` initialized with an object literal that later +// assignments fill in: tsc types the binding by the members those +// assignments add, but the object only gains them after its initializer. +"use strict"; +function load(id) { + var p = {}; + console.log(p.count, "count" in p, Object.keys(p).length); + p.id = id; + p.count = id * 2; + p.label = "item-" + id; + return p; +} +const p = load(7); +console.log(p.id + p.count, p.label.toUpperCase(), JSON.stringify(p), Object.keys(p).join(",")); + +function partial(name) { + var entry = { name: name }; + entry.size = name.length; + entry.tags = [name, name + "!"]; + return entry; +} +const e = partial("abc"); +console.log(e.size * 10, e.tags.join("|"), JSON.stringify(e)); + +function nested(depth) { + var node = {}; + node.depth = depth; + node.child = depth > 0 ? nested(depth - 1) : null; + return node; +} +console.log(JSON.stringify(nested(2))); + +function rows(n) { + var out = []; + for (var i = 0; i < n; i++) { + var row = {}; + row.index = i; + row.square = i * i; + out.push(row); + } + return out; +} +console.log(rows(4).map(function (r) { return r.index + ":" + r.square; }).join(" ")); + +var top = {}; +top.ready = true; +top.total = rows(3).reduce(function (s, r) { return s + r.square; }, 0); +console.log(top.ready, top.total, JSON.stringify(top)); diff --git a/tests/coverage-fixtures/localized-failure.ts b/tests/coverage-fixtures/localized-failure.ts new file mode 100644 index 000000000..04a5b8037 --- /dev/null +++ b/tests/coverage-fixtures/localized-failure.ts @@ -0,0 +1,68 @@ +// The localized-failure report of `scriptc coverage --explain-dynamic`: +// type failures that would become checked dynamic slots, and the uses +// each one rejects today. +interface Tool { + name: string; + hooks: Map string> | AbortSignal | (() => void) | Set; +} +type Progress = (done: number, total: number) => void; +type Callbacks = Progress | AbortSignal | Map string> | Set; + +// A field: rejects the class, so every use of an instance fails. +class Session { + private tools: Tool[] = []; + count(): number { + return this.tools.length; + } + label(): string { + return "s"; + } +} + +// A parameter: rejects the function, so every call fails. +function listAll(dir: string, progress?: Callbacks): number { + return dir.length; +} + +// The dyn class field policy. +class Holder { + state: any = {}; + get(): number { + return 1; + } +} + +// A local: rejects its declaration statement, so its uses fail. +function pick(n: number): boolean { + const callbacks: Callbacks | undefined = n > 1 ? new Set() : undefined; + return callbacks === undefined; +} + +// Refused: a value the compiler cannot produce stays an error. +class Cache { + tokens: WeakRef = new WeakRef({}); + size(): number { + return 0; + } +} + +const s = new Session(); +console.log(s.count(), s.label()); +console.log(listAll("abc")); +console.log(new Holder().get()); +console.log(pick(1), new Cache().size()); + +// A class extending one the dyn class field policy rejects: its rejection +// and the uses of the derived class are that field's cascade. +class Base { + state: any = {}; + get(): number { + return 1; + } +} +class Derived extends Base { + twice(): number { + return 2; + } +} +console.log(new Derived().twice()); diff --git a/tests/diagnostics/promise-all-generic-mixed-union.ts b/tests/diagnostics/promise-all-generic-mixed-union.ts new file mode 100644 index 000000000..8324f6a23 --- /dev/null +++ b/tests/diagnostics/promise-all-generic-mixed-union.ts @@ -0,0 +1,8 @@ +// Promise.all over a generic array instantiated with a union of a value and +// a promise, with an element read of its result: the call is refused by name. +async function f(xs: V[]): Promise { + const values = await Promise.all(xs); + values[0]; +} +f>(["x"]); +export {}; diff --git a/tests/diagnostics/untyped-map-into-interface.ts b/tests/diagnostics/untyped-map-into-interface.ts new file mode 100644 index 000000000..ba5e26a90 --- /dev/null +++ b/tests/diagnostics/untyped-map-into-interface.ts @@ -0,0 +1,12 @@ +// An untyped `new Map()` satisfies a record interface only through `any`; +// its flow into the interface-typed result is refused at build time, like a +// typed Map is. +interface Cache { + get(key: string): number | undefined; + set(key: string, value: number): unknown; +} +function make(c?: Cache): Cache { + return c ?? new Map(); +} +console.log(make().get("a")); +export {}; diff --git a/tests/fixtures/npm-static/addon-probe-installed-cli.ts b/tests/fixtures/npm-static/addon-probe-installed-cli.ts new file mode 100644 index 000000000..5568ce086 --- /dev/null +++ b/tests/fixtures/npm-static/addon-probe-installed-cli.ts @@ -0,0 +1,4 @@ +import { loadInstalled } from "addon-probe"; + +console.log("before"); +console.log(loadInstalled(process.argv[2] === "module" ? "module" : "addon")); diff --git a/tests/fixtures/npm-static/node_modules/addon-probe/index.d.ts b/tests/fixtures/npm-static/node_modules/addon-probe/index.d.ts index a9987d787..ca349ec0e 100644 --- a/tests/fixtures/npm-static/node_modules/addon-probe/index.d.ts +++ b/tests/fixtures/npm-static/node_modules/addon-probe/index.d.ts @@ -1,3 +1,4 @@ export declare function probe(name: string): string; export declare function resolveHelper(name: string, resolvePackage?: (id: string) => string): string; +export declare function loadInstalled(kind: "addon" | "module"): string; export declare function locate(name: string): string; diff --git a/tests/fixtures/npm-static/node_modules/addon-probe/index.js b/tests/fixtures/npm-static/node_modules/addon-probe/index.js index 2cc423359..acd2bf814 100644 --- a/tests/fixtures/npm-static/node_modules/addon-probe/index.js +++ b/tests/fixtures/npm-static/node_modules/addon-probe/index.js @@ -27,6 +27,16 @@ export function resolveHelper(name, resolvePackage) { } } +// Requires a file that is installed: an addon, or a module that is not part +// of the compiled program. +export function loadInstalled(kind) { + try { + return typeof load(join(here, kind === "addon" ? "native/installed.node" : "installed.js")); + } catch { + return "fallback"; + } +} + let probes = 0; function candidate(name) { probes++; diff --git a/tests/fixtures/npm-static/node_modules/addon-probe/installed.js b/tests/fixtures/npm-static/node_modules/addon-probe/installed.js new file mode 100644 index 000000000..8476dc24e --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/addon-probe/installed.js @@ -0,0 +1 @@ +export const installed = true; diff --git a/tests/fixtures/npm-static/node_modules/addon-probe/native/installed.node b/tests/fixtures/npm-static/node_modules/addon-probe/native/installed.node new file mode 100644 index 000000000..3d1ca6ec7 --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/addon-probe/native/installed.node @@ -0,0 +1 @@ +placeholder for a prebuilt addon diff --git a/tests/fixtures/npm/cases/types-keyed-construction/main.ts b/tests/fixtures/npm/cases/types-keyed-construction/main.ts new file mode 100644 index 000000000..b163f3d1c --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/main.ts @@ -0,0 +1,12 @@ +// A plain-JavaScript package typed by an @types package builds its result +// key by key (`var p = {}; p.id = id; ...`): the object crosses the +// declaration boundary with every field, including the declared fields +// whose keys are not identifiers. +import { loadProfile, type Profile } from "profilekit"; + +const p: Profile = loadProfile(7); +console.log(p["a;b"].toUpperCase(), p["a;b"].length); +console.log(p["my-key"], p["my-key"].length, p["x y"].length); +console.log(p.id, p.count, JSON.stringify(p)); +const keys: string[] = Object.keys(p); +console.log(keys.join("|")); diff --git a/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/index.d.ts b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/index.d.ts new file mode 100644 index 000000000..2da797940 --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/index.d.ts @@ -0,0 +1,8 @@ +export interface Profile { + id: number; + "a;b": string; + "my-key": string; + "x y": string; + count: number; +} +export declare function loadProfile(id: number): Profile; diff --git a/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/package.json b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/package.json new file mode 100644 index 000000000..a7dd89b11 --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/package.json @@ -0,0 +1 @@ +{ "name": "@types/profilekit", "version": "1.0.0", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/index.js b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/index.js new file mode 100644 index 000000000..8d8df9336 --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/index.js @@ -0,0 +1,12 @@ +"use strict"; +// Plain JavaScript: the profile is built key by key. +function loadProfile(id) { + var p = {}; + p.id = id; + p["a;b"] = "semi" + id; + p["my-key"] = "dash"; + p["x y"] = "space"; + p.count = id * 2; + return p; +} +exports.loadProfile = loadProfile; diff --git a/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/package.json b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/package.json new file mode 100644 index 000000000..340a0a6c5 --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/package.json @@ -0,0 +1 @@ +{ "name": "profilekit", "version": "1.0.0", "main": "index.js" } diff --git a/tests/fixtures/npm/cases/types-keyed-construction/package.json b/tests/fixtures/npm/cases/types-keyed-construction/package.json new file mode 100644 index 000000000..24992c953 --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/package.json @@ -0,0 +1 @@ +{ "name": "types-keyed-construction-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts b/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts new file mode 100644 index 000000000..d069face6 --- /dev/null +++ b/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts @@ -0,0 +1,20 @@ +// A package typed from its JavaScript returns profiles without a key its +// declarations promise ("a;b"). A literal-key read of that key answers +// undefined, as Node's does; it never reads another field. +import { loadProfiles } from "hostilekit"; + +const profiles = loadProfiles(); +for (const p of profiles) { + console.log(p.id, p["a;b"] === undefined, typeof p["a;b"], "a;b" in p); + try { + console.log(p["a;b"].toUpperCase()); + } catch (e) { + console.log("upper threw", e instanceof TypeError); + } + try { + console.log(p["a;b"].length); + } catch (e) { + console.log("length threw", e instanceof TypeError); + } + console.log(p.id.toFixed(1), JSON.stringify(p)); +} diff --git a/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.d.ts b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.d.ts new file mode 100644 index 000000000..4bf22dfea --- /dev/null +++ b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.d.ts @@ -0,0 +1,2 @@ +export type Profile = { id: number; "a;b": string }; +export declare function loadProfiles(): Profile[]; diff --git a/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.js b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.js new file mode 100644 index 000000000..316501256 --- /dev/null +++ b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.js @@ -0,0 +1,5 @@ +"use strict"; +function loadProfiles() { + return [{ id: 1 }]; +} +exports.loadProfiles = loadProfiles; diff --git a/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/package.json b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/package.json new file mode 100644 index 000000000..2fdbc5b41 --- /dev/null +++ b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/package.json @@ -0,0 +1 @@ +{ "name": "hostilekit", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/types-undeclared-quoted-key/package.json b/tests/fixtures/npm/cases/types-undeclared-quoted-key/package.json new file mode 100644 index 000000000..afdb3cf5e --- /dev/null +++ b/tests/fixtures/npm/cases/types-undeclared-quoted-key/package.json @@ -0,0 +1 @@ +{ "name": "types-undeclared-quoted-key-case", "private": true, "type": "module" } diff --git a/tests/fixtures/type-patches/node_modules/gridctx/index.js b/tests/fixtures/type-patches/node_modules/gridctx/index.js new file mode 100644 index 000000000..44c282ad7 --- /dev/null +++ b/tests/fixtures/type-patches/node_modules/gridctx/index.js @@ -0,0 +1,43 @@ +"use strict"; +// termctx's Terminal under another name: panectx and gridctx declare the +// same OutputContext (one name, one shape) in two packages. + +class Terminal { + constructor(tty) { + this.tty = tty; + this.settings = { + columns: () => (this.tty ? 80 : undefined), + }; + } + + /** + * @returns {number} + */ + columns() { + return this.settings.columns(); + } + + /** + * @typedef OutputContext + * @type {object} + * @property {boolean} error + * @property {number} width + * + * @param {boolean} error + * @returns {OutputContext} + */ + outputContext(error) { + const width = this.settings.columns(); + return { error, width }; + } + + /** + * @param {OutputContext} context + * @returns {string} + */ + describe(context) { + return `width ${context.width}${context.error ? " (stderr)" : ""}`; + } +} + +module.exports = { Terminal }; diff --git a/tests/fixtures/type-patches/node_modules/gridctx/package.json b/tests/fixtures/type-patches/node_modules/gridctx/package.json new file mode 100644 index 000000000..36f6aba11 --- /dev/null +++ b/tests/fixtures/type-patches/node_modules/gridctx/package.json @@ -0,0 +1 @@ +{ "name": "gridctx", "version": "4.1.0", "main": "index.js" } diff --git a/tests/fixtures/type-patches/node_modules/panectx/index.js b/tests/fixtures/type-patches/node_modules/panectx/index.js new file mode 100644 index 000000000..44c282ad7 --- /dev/null +++ b/tests/fixtures/type-patches/node_modules/panectx/index.js @@ -0,0 +1,43 @@ +"use strict"; +// termctx's Terminal under another name: panectx and gridctx declare the +// same OutputContext (one name, one shape) in two packages. + +class Terminal { + constructor(tty) { + this.tty = tty; + this.settings = { + columns: () => (this.tty ? 80 : undefined), + }; + } + + /** + * @returns {number} + */ + columns() { + return this.settings.columns(); + } + + /** + * @typedef OutputContext + * @type {object} + * @property {boolean} error + * @property {number} width + * + * @param {boolean} error + * @returns {OutputContext} + */ + outputContext(error) { + const width = this.settings.columns(); + return { error, width }; + } + + /** + * @param {OutputContext} context + * @returns {string} + */ + describe(context) { + return `width ${context.width}${context.error ? " (stderr)" : ""}`; + } +} + +module.exports = { Terminal }; diff --git a/tests/fixtures/type-patches/node_modules/panectx/package.json b/tests/fixtures/type-patches/node_modules/panectx/package.json new file mode 100644 index 000000000..e73708257 --- /dev/null +++ b/tests/fixtures/type-patches/node_modules/panectx/package.json @@ -0,0 +1 @@ +{ "name": "panectx", "version": "2.0.0", "main": "index.js" } diff --git a/tests/fixtures/type-patches/same-name/main.ts b/tests/fixtures/type-patches/same-name/main.ts new file mode 100644 index 000000000..3cbfffc57 --- /dev/null +++ b/tests/fixtures/type-patches/same-name/main.ts @@ -0,0 +1,15 @@ +// panectx and gridctx each declare an OutputContext of the same shape: +// each failing check's TypeError names its own package's declaration +// (native only; Node never checks). +import { Terminal as PaneTerminal } from "panectx"; +import { Terminal as GridTerminal } from "gridctx"; + +const pane = new PaneTerminal(false); +const grid = new GridTerminal(false); +for (const read of [() => pane.outputContext(false).width, () => grid.outputContext(false).width]) { + try { + console.log(read()); + } catch (e) { + console.log((e as Error).message); + } +} diff --git a/tests/fixtures/type-patches/same-name/package.json b/tests/fixtures/type-patches/same-name/package.json new file mode 100644 index 000000000..fdf221cba --- /dev/null +++ b/tests/fixtures/type-patches/same-name/package.json @@ -0,0 +1 @@ +{ "name": "type-patches-same-name", "private": true, "type": "module" } diff --git a/tests/fixtures/type-patches/unpatched/main.ts b/tests/fixtures/type-patches/unpatched/main.ts new file mode 100644 index 000000000..89418d2d0 --- /dev/null +++ b/tests/fixtures/type-patches/unpatched/main.ts @@ -0,0 +1,13 @@ +// termctx's lying JSDoc without the project's type patches: each failing +// check's TypeError names the declaration and the patch that admits the +// value (native only; Node never checks). +import { Terminal } from "termctx"; + +const term = new Terminal(false); +for (const read of [() => term.columns(), () => term.outputContext(false).width]) { + try { + console.log(read()); + } catch (e) { + console.log((e as Error).message); + } +} diff --git a/tests/fixtures/type-patches/unpatched/package.json b/tests/fixtures/type-patches/unpatched/package.json new file mode 100644 index 000000000..b2d19d710 --- /dev/null +++ b/tests/fixtures/type-patches/unpatched/package.json @@ -0,0 +1 @@ +{ "name": "type-patches-unpatched", "private": true, "type": "module" } diff --git a/tests/harness/__snapshots__/coverage-explain-localized-failure.txt b/tests/harness/__snapshots__/coverage-explain-localized-failure.txt new file mode 100644 index 000000000..31a933b26 --- /dev/null +++ b/tests/harness/__snapshots__/coverage-explain-localized-failure.txt @@ -0,0 +1,19 @@ + dynamic tier (tier assignment, report-only; --tier=auto) + 5 functions already operate on dynamic values (4 statements) + Base.constructor tests/coverage-fixtures/localized-failure.ts:57:1: dynObjLit at tests/coverage-fixtures/localized-failure.ts:58:16 + Session.constructor tests/coverage-fixtures/localized-failure.ts:12:1: dynArrLit at tests/coverage-fixtures/localized-failure.ts:13:27 + Session.count tests/coverage-fixtures/localized-failure.ts:14:3: %dyn.class.property at tests/coverage-fixtures/localized-failure.ts:15:12 + Holder.constructor tests/coverage-fixtures/localized-failure.ts:28:1: dynObjLit at tests/coverage-fixtures/localized-failure.ts:29:16 + %dyn.class.property.0 tests/coverage-fixtures/localized-failure.ts:15:12: %dyn.class.readPropertiesOrEmpty at tests/coverage-fixtures/localized-failure.ts:15:12 + + 2 demotion requests (refusals and fences the dynamic tier would replace) + + 0 would-be dynamic slots (localized failure, report-only: type failures a checked dynamic slot would hold) + 2 errors: 0 clear with declaration slots (L1-L2), 0 with member and expression slots too (L3-L4); 1 cascade attributed; 1 type failure stays refused + class Cache tests/coverage-fixtures/localized-failure.ts:42:1 (1 request) + <- SC1090 reflection at tests/coverage-fixtures/localized-failure.ts:53:22: constructing through a class value whose class has no lowering (the class declaration itself was rejected — see its own diagnostic) is not supported yet + -> function turns dynamic + field Cache.tokens tests/coverage-fixtures/localized-failure.ts:43:3 (1 request) + <- SC2020 operation at tests/coverage-fixtures/localized-failure.ts:43:3: 'WeakRef' is part of the standard library types but has no scriptc lowering yet + -> function turns dynamic + function tests/coverage-fixtures/localized-failure.ts:1:1 (2 sites inside) \ No newline at end of file diff --git a/tests/harness/__snapshots__/promise-all-generic-mixed-union.ts.txt b/tests/harness/__snapshots__/promise-all-generic-mixed-union.ts.txt new file mode 100644 index 000000000..b0622d132 --- /dev/null +++ b/tests/harness/__snapshots__/promise-all-generic-mixed-union.ts.txt @@ -0,0 +1,8 @@ +promise-all-generic-mixed-union.ts:4:36 - error SC2020: 'Promise.all over this argument shape' is part of the standard library types but has no scriptc lowering yet (instantiating 'f' with | string>) + + 3 | async function f(xs: V[]): Promise { + 4 | const values = await Promise.all(xs); + | ^~ + 5 | values[0]; + + hint: an array of promises (Promise[]) is the supported form \ No newline at end of file diff --git a/tests/harness/__snapshots__/untyped-map-into-interface.ts.txt b/tests/harness/__snapshots__/untyped-map-into-interface.ts.txt new file mode 100644 index 000000000..03b54a136 --- /dev/null +++ b/tests/harness/__snapshots__/untyped-map-into-interface.ts.txt @@ -0,0 +1,8 @@ +untyped-map-into-interface.ts:9:10 - error SC2002: record shapes must match exactly or width-coerce: expected '{ get: (string) => number | undefined; set: (string, number) => unknown }', got 'Map' + + 8 | function make(c?: Cache): Cache { + 9 | return c ?? new Map(); + | ^~~~~~~~~~~~~~ + 10 | } + + hint: width subtyping compiles as a copy (each expected field copied off the source; extra fields drop) — this pair is outside the copy's rules; build a literal with exactly the expected fields instead \ No newline at end of file diff --git a/tests/harness/bootstrap-seed.test.ts b/tests/harness/bootstrap-seed.test.ts index 76b1f6de5..2da27baea 100644 --- a/tests/harness/bootstrap-seed.test.ts +++ b/tests/harness/bootstrap-seed.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { load } from "js-yaml"; import { expect, test } from "vitest"; -import { verifyBootstrapSeed } from "../../scripts/bootstrap-seed.mjs"; +import { seedBuildEnvironment, verifyBootstrapSeed } from "../../scripts/bootstrap-seed.mjs"; test("bootstrap artifacts require a complete seed from the same revision, lane, and path", () => { const directory = mkdtempSync(join(tmpdir(), "scriptc-seed-contract-")); @@ -36,6 +36,22 @@ test("bootstrap artifacts require a complete seed from the same revision, lane, } }); +test("seeds build without the function layout in both lanes, shared or not", () => { + for (const sanitize of [false, true]) + expect(seedBuildEnvironment(sanitize)).toEqual({ + SCRIPTC_SAN: sanitize ? "1" : "", + SCRIPTC_NATIVE_EMIT_IR: "1", + SCRIPTC_NATIVE_OPTIMIZATION: sanitize ? "dev" : "release", + SCRIPTC_NATIVE_LAYOUT: "", + }); + // The bootstrap test builds its own seed when CI shares none. + const driver = readFileSync( + new URL("./self-hosting-native-driver.test.ts", import.meta.url), + "utf8", + ); + expect(driver).toContain("...seedBuildEnvironment(sanitize)"); +}); + test("CI restores each seed only into its matching instrumented bootstrap lane", () => { const workflow = load( readFileSync(new URL("../../.github/workflows/ci.yml", import.meta.url), "utf8"), diff --git a/tests/harness/corpus-costs/7508-pipeline-untyped-source-functions.mjs.json b/tests/harness/corpus-costs/7508-pipeline-untyped-source-functions.mjs.json new file mode 100644 index 000000000..805df1394 --- /dev/null +++ b/tests/harness/corpus-costs/7508-pipeline-untyped-source-functions.mjs.json @@ -0,0 +1 @@ +{ "plain": 4900, "san": 6400 } diff --git a/tests/harness/corpus-costs/7509-pipeline-untyped-function-stages.mjs.json b/tests/harness/corpus-costs/7509-pipeline-untyped-function-stages.mjs.json new file mode 100644 index 000000000..93a0d0f3b --- /dev/null +++ b/tests/harness/corpus-costs/7509-pipeline-untyped-function-stages.mjs.json @@ -0,0 +1 @@ +{ "plain": 6100, "san": 8500 } diff --git a/tests/harness/corpus-costs/7510-pipeline-untyped-later-iterables.mjs.json b/tests/harness/corpus-costs/7510-pipeline-untyped-later-iterables.mjs.json new file mode 100644 index 000000000..e1920ef92 --- /dev/null +++ b/tests/harness/corpus-costs/7510-pipeline-untyped-later-iterables.mjs.json @@ -0,0 +1 @@ +{ "plain": 5100, "san": 8600 } diff --git a/tests/harness/corpus-costs/7511-pipeline-untyped-stages.cjs.json b/tests/harness/corpus-costs/7511-pipeline-untyped-stages.cjs.json new file mode 100644 index 000000000..eafd03265 --- /dev/null +++ b/tests/harness/corpus-costs/7511-pipeline-untyped-stages.cjs.json @@ -0,0 +1 @@ +{ "plain": 5300, "san": 6900 } diff --git a/tests/harness/corpus-costs/7512-pipeline-any-stages.ts.json b/tests/harness/corpus-costs/7512-pipeline-any-stages.ts.json new file mode 100644 index 000000000..ac0e8497d --- /dev/null +++ b/tests/harness/corpus-costs/7512-pipeline-any-stages.ts.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 6100 } diff --git a/tests/harness/corpus-costs/7513-pipeline-untyped-stage-errors.mjs.json b/tests/harness/corpus-costs/7513-pipeline-untyped-stage-errors.mjs.json new file mode 100644 index 000000000..f363efcf0 --- /dev/null +++ b/tests/harness/corpus-costs/7513-pipeline-untyped-stage-errors.mjs.json @@ -0,0 +1 @@ +{ "plain": 4700, "san": 6300 } diff --git a/tests/harness/corpus-costs/7717-math-fdlibm.ts.json b/tests/harness/corpus-costs/7717-math-fdlibm.ts.json new file mode 100644 index 000000000..63543b972 --- /dev/null +++ b/tests/harness/corpus-costs/7717-math-fdlibm.ts.json @@ -0,0 +1 @@ +{ "plain": 3357, "san": 4900 } diff --git a/tests/harness/corpus-costs/7718-math-fdlibm.cjs.json b/tests/harness/corpus-costs/7718-math-fdlibm.cjs.json new file mode 100644 index 000000000..281ed9b93 --- /dev/null +++ b/tests/harness/corpus-costs/7718-math-fdlibm.cjs.json @@ -0,0 +1 @@ +{ "plain": 3519, "san": 4613 } diff --git a/tests/harness/corpus-costs/8008-string-index-paths.ts.json b/tests/harness/corpus-costs/8008-string-index-paths.ts.json new file mode 100644 index 000000000..3a1b75cbc --- /dev/null +++ b/tests/harness/corpus-costs/8008-string-index-paths.ts.json @@ -0,0 +1 @@ +{ "plain": 4784, "san": 2792 } diff --git a/tests/harness/corpus-costs/8014-foreign-element-callbacks.ts.json b/tests/harness/corpus-costs/8014-foreign-element-callbacks.ts.json new file mode 100644 index 000000000..981bcbb55 --- /dev/null +++ b/tests/harness/corpus-costs/8014-foreign-element-callbacks.ts.json @@ -0,0 +1 @@ +{ "plain": 5089, "san": 8118 } diff --git a/tests/harness/corpus-costs/8210-require-computed-missing/main.ts.json b/tests/harness/corpus-costs/8210-require-computed-missing/main.ts.json new file mode 100644 index 000000000..ef439d096 --- /dev/null +++ b/tests/harness/corpus-costs/8210-require-computed-missing/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4500 } diff --git a/tests/harness/corpus-costs/8211-require-resolve-computed/main.ts.json b/tests/harness/corpus-costs/8211-require-resolve-computed/main.ts.json new file mode 100644 index 000000000..ef439d096 --- /dev/null +++ b/tests/harness/corpus-costs/8211-require-resolve-computed/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4500 } diff --git a/tests/harness/corpus-costs/8212-native-helper-probe-no-display/main.ts.json b/tests/harness/corpus-costs/8212-native-helper-probe-no-display/main.ts.json new file mode 100644 index 000000000..ef439d096 --- /dev/null +++ b/tests/harness/corpus-costs/8212-native-helper-probe-no-display/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4500 } diff --git a/tests/harness/corpus-costs/8230-function-record-dispatch.ts.json b/tests/harness/corpus-costs/8230-function-record-dispatch.ts.json new file mode 100644 index 000000000..1dd4b287a --- /dev/null +++ b/tests/harness/corpus-costs/8230-function-record-dispatch.ts.json @@ -0,0 +1 @@ +{ "plain": 4700, "san": 5800 } diff --git a/tests/harness/corpus-costs/8231-function-table-shapes.ts.json b/tests/harness/corpus-costs/8231-function-table-shapes.ts.json new file mode 100644 index 000000000..1dd4b287a --- /dev/null +++ b/tests/harness/corpus-costs/8231-function-table-shapes.ts.json @@ -0,0 +1 @@ +{ "plain": 4700, "san": 5800 } diff --git a/tests/harness/corpus-costs/8232-optional-function-values.ts.json b/tests/harness/corpus-costs/8232-optional-function-values.ts.json new file mode 100644 index 000000000..1dd4b287a --- /dev/null +++ b/tests/harness/corpus-costs/8232-optional-function-values.ts.json @@ -0,0 +1 @@ +{ "plain": 4700, "san": 5800 } diff --git a/tests/harness/corpus-costs/8332-worker-shared-counted-loops.ts.json b/tests/harness/corpus-costs/8332-worker-shared-counted-loops.ts.json new file mode 100644 index 000000000..6b0f0395b --- /dev/null +++ b/tests/harness/corpus-costs/8332-worker-shared-counted-loops.ts.json @@ -0,0 +1 @@ +{ "plain": 4011, "san": 2208 } diff --git a/tests/harness/corpus-costs/boxed-record-identity.ts.json b/tests/harness/corpus-costs/boxed-record-identity.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/boxed-record-identity.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/checked-union-record-assertions.ts.json b/tests/harness/corpus-costs/checked-union-record-assertions.ts.json new file mode 100644 index 000000000..fc3725628 --- /dev/null +++ b/tests/harness/corpus-costs/checked-union-record-assertions.ts.json @@ -0,0 +1 @@ +{ "plain": 14493, "san": 16666 } diff --git a/tests/harness/corpus-costs/constant-choice-chains.ts.json b/tests/harness/corpus-costs/constant-choice-chains.ts.json new file mode 100644 index 000000000..6562c9db3 --- /dev/null +++ b/tests/harness/corpus-costs/constant-choice-chains.ts.json @@ -0,0 +1 @@ +{ "plain": 3921, "san": 5592 } diff --git a/tests/harness/corpus-costs/dyn-cast-identity-fresh.ts.json b/tests/harness/corpus-costs/dyn-cast-identity-fresh.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/dyn-cast-identity-fresh.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/dyn-cast-identity-holders.ts.json b/tests/harness/corpus-costs/dyn-cast-identity-holders.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/dyn-cast-identity-holders.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/dyn-cast-identity-workers.ts.json b/tests/harness/corpus-costs/dyn-cast-identity-workers.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/dyn-cast-identity-workers.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/dyn-cast-identity.ts.json b/tests/harness/corpus-costs/dyn-cast-identity.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/dyn-cast-identity.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/dyn-literal-identity.ts.json b/tests/harness/corpus-costs/dyn-literal-identity.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/dyn-literal-identity.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/fs-readdir-recursive.ts.json b/tests/harness/corpus-costs/fs-readdir-recursive.ts.json new file mode 100644 index 000000000..ad3232154 --- /dev/null +++ b/tests/harness/corpus-costs/fs-readdir-recursive.ts.json @@ -0,0 +1 @@ +{ "plain": 3521, "san": 40657 } diff --git a/tests/harness/corpus-costs/generic-interface-void-calls.ts.json b/tests/harness/corpus-costs/generic-interface-void-calls.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/generic-interface-void-calls.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-cycles.ts.json b/tests/harness/corpus-costs/localized-failure-cycles.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-cycles.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-declarations.ts.json b/tests/harness/corpus-costs/localized-failure-declarations.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-declarations.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-forward-capture.ts.json b/tests/harness/corpus-costs/localized-failure-forward-capture.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-forward-capture.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-function-slots.ts.json b/tests/harness/corpus-costs/localized-failure-function-slots.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-function-slots.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-held-values.ts.json b/tests/harness/corpus-costs/localized-failure-held-values.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-held-values.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-locals.ts.json b/tests/harness/corpus-costs/localized-failure-locals.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-locals.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-pi-shapes.ts.json b/tests/harness/corpus-costs/localized-failure-pi-shapes.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-pi-shapes.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-registered-statics.ts.json b/tests/harness/corpus-costs/localized-failure-registered-statics.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-registered-statics.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/method-signature-any-params.ts.json b/tests/harness/corpus-costs/method-signature-any-params.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/method-signature-any-params.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/never-narrowed-literal-argument.ts.json b/tests/harness/corpus-costs/never-narrowed-literal-argument.ts.json new file mode 100644 index 000000000..1af81d8df --- /dev/null +++ b/tests/harness/corpus-costs/never-narrowed-literal-argument.ts.json @@ -0,0 +1 @@ +{ "plain": 2485, "san": 53450 } diff --git a/tests/harness/corpus-costs/rest-tuple-shorthand-property.ts.json b/tests/harness/corpus-costs/rest-tuple-shorthand-property.ts.json new file mode 100644 index 000000000..0358eaf91 --- /dev/null +++ b/tests/harness/corpus-costs/rest-tuple-shorthand-property.ts.json @@ -0,0 +1 @@ +{ "plain": 2199, "san": 6086 } diff --git a/tests/harness/corpus-costs/speculative-loop-view-headers.ts.json b/tests/harness/corpus-costs/speculative-loop-view-headers.ts.json new file mode 100644 index 000000000..47ead0063 --- /dev/null +++ b/tests/harness/corpus-costs/speculative-loop-view-headers.ts.json @@ -0,0 +1 @@ +{ "plain": 2400 } diff --git a/tests/harness/corpus-costs/string-plus-element-read-return.ts.json b/tests/harness/corpus-costs/string-plus-element-read-return.ts.json new file mode 100644 index 000000000..17a596f9c --- /dev/null +++ b/tests/harness/corpus-costs/string-plus-element-read-return.ts.json @@ -0,0 +1 @@ +{ "plain": 2298, "san": 3533 } diff --git a/tests/harness/corpus-costs/symbol-keyed-recursive-record.ts.json b/tests/harness/corpus-costs/symbol-keyed-recursive-record.ts.json new file mode 100644 index 000000000..917f53744 --- /dev/null +++ b/tests/harness/corpus-costs/symbol-keyed-recursive-record.ts.json @@ -0,0 +1 @@ +{ "plain": 2702, "san": 43257 } diff --git a/tests/harness/corpus-costs/undeclared-key-optional-chain/main.cjs.json b/tests/harness/corpus-costs/undeclared-key-optional-chain/main.cjs.json new file mode 100644 index 000000000..8cfabb847 --- /dev/null +++ b/tests/harness/corpus-costs/undeclared-key-optional-chain/main.cjs.json @@ -0,0 +1 @@ +{ "plain": 3000 } diff --git a/tests/harness/corpus-costs/union-clone-empty-array-overrides.ts.json b/tests/harness/corpus-costs/union-clone-empty-array-overrides.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/union-clone-empty-array-overrides.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/union-record-kept-fields-arm.ts.json b/tests/harness/corpus-costs/union-record-kept-fields-arm.ts.json new file mode 100644 index 000000000..3c750a2dc --- /dev/null +++ b/tests/harness/corpus-costs/union-record-kept-fields-arm.ts.json @@ -0,0 +1 @@ +{ "plain": 4793, "san": 7135 } diff --git a/tests/harness/corpus-costs/union-record-kept-fields-no-discriminant.ts.json b/tests/harness/corpus-costs/union-record-kept-fields-no-discriminant.ts.json new file mode 100644 index 000000000..5e891c40e --- /dev/null +++ b/tests/harness/corpus-costs/union-record-kept-fields-no-discriminant.ts.json @@ -0,0 +1 @@ +{ "plain": 4466, "san": 5253 } diff --git a/tests/harness/corpus-costs/union-tuple-element-read.ts.json b/tests/harness/corpus-costs/union-tuple-element-read.ts.json new file mode 100644 index 000000000..fcdd8c175 --- /dev/null +++ b/tests/harness/corpus-costs/union-tuple-element-read.ts.json @@ -0,0 +1 @@ +{ "plain": 2301, "san": 3138 } diff --git a/tests/harness/corpus-costs/var-object-filled-later.js.json b/tests/harness/corpus-costs/var-object-filled-later.js.json new file mode 100644 index 000000000..1e74b61e7 --- /dev/null +++ b/tests/harness/corpus-costs/var-object-filled-later.js.json @@ -0,0 +1 @@ +{ "plain": 3400 } diff --git a/tests/harness/coverage.test.ts b/tests/harness/coverage.test.ts index fde16a901..f8fa0da03 100644 --- a/tests/harness/coverage.test.ts +++ b/tests/harness/coverage.test.ts @@ -428,6 +428,63 @@ test("--explain-dynamic attributes each refusal to the unit the dynamic tier wou ); }); +test("--explain-dynamic lists the type failures localized failure would turn into dynamic slots", async () => { + const { coverage } = analyze(fixture("localized-failure.ts"), { explainDynamic: true }); + const report = coverage.tiers!.localizedFailure; + // Locals, fields and parameters degrade (stages L1 and L2), and the + // 'any' field compiles; only the refused WeakRef field and its + // construction remain. + expect(coverage.dynamicSlots?.map((s) => [s.slot, s.name])).toEqual([ + ["field", "tools"], + ["param", "progress"], + ["local", "callbacks"], + ]); + expect(report.errors).toBe(2); + expect(report.typeFailures).toEqual({ degradable: 0, refused: 1 }); + expect(report.slots).toEqual([]); + expect(report.cascades).toBe(1); + expect(report.remainingByCode).toEqual({ SC1090: 1, SC2020: 1 }); + const relative = JSON.parse( + JSON.stringify(coverage.tiers).replaceAll(repoRoot + "/", ""), + ) as typeof coverage.tiers; + await expect(renderExplainDynamic(relative!)).toMatchFileSnapshot( + "__snapshots__/coverage-explain-localized-failure.txt", + ); +}); + +test("a local or module global whose declared type does not compile becomes a dynamic slot", async () => { + const program = join(repoRoot, "tests/corpus/localized-failure-locals.ts"); + const { coverage, sourceTexts } = analyze(program); + expect(coverage.diagnostics).toEqual([]); + expect(coverage.dynamicSlots?.map((s) => [s.slot, s.name, s.declaredType, s.code])).toEqual([ + ["global", "current", "Callbacks | undefined", "SC2009"], + ["global", "all", "(number | Callbacks)[]", "SC2009"], + ["local", "local", "Callbacks | undefined", "SC2009"], + ]); + expect(coverage.performance?.filter((d) => d.code === "SC6104")).toHaveLength(3); + expect(coveragePasses(coverage, "blockers")).toBe(true); + expect(coveragePasses(coverage, "dynamic")).toBe(true); + expect(coveragePasses(coverage, "dynamic-slots")).toBe(false); + const envelope = coverageEnvelope(coverage, { compilerVersion: "test", sourceTexts }); + expect(envelope.dynamicSlots?.map((s) => [s.kind, s.name, s.line])).toEqual([ + ["global", "current", 10], + ["global", "all", 36], + ["local", "local", 14], + ]); + expect(envelope.diagnostics.filter((d) => d.code === "SC6104").map((d) => d.severity)).toEqual([ + "warning", + "warning", + "warning", + ]); + const text = renderCoverage(coverage, { sourceTexts, root: repoRoot }); + expect(text).toContain("dynamic slots 3"); + expect(text).toContain("module global 'current' (declared 'Callbacks | undefined')"); + // --tier=static-only keeps every refusal. + const strict = analyze(program, { tier: "static-only" }).coverage; + expect(strict.dynamicSlots).toBeUndefined(); + expect(strict.diagnostics.map((d) => d.code)).toContain("SC2009"); +}); + test("CLI: --tier and --explain-dynamic", () => { const scriptcCli = join(repoRoot, "packages/cli/src/main.ts"); const outDir = mkdtempSync(join(tmpdir(), "scriptc-tier-")); @@ -530,13 +587,18 @@ test("runtime-optional refined unions report blockers without crashing", () => { }); test("a rejected class reached only through a receiver probe reports its own diagnostic once", () => { - const { coverage } = analyze(join(repoRoot, "tests/diagnostics/deferred-class-receiver/main.ts")); + const program = join(repoRoot, "tests/diagnostics/deferred-class-receiver/main.ts"); + // The class's refusal, as the diagnostics corpus pins it (--tier=static-only). + const { coverage } = analyze(program, { tier: "static-only" }); const messages = coverage.diagnostics.map((d) => d.message); expect( messages.filter((m) => m.startsWith("'MapIterator'-typed class fields")), ).toHaveLength(1); expect(messages.some((m) => m.startsWith("constructing through a class value"))).toBe(true); expect(coverage.unreached?.diagnostics ?? []).toEqual([]); + // Localized failure retries the class where the probe's statement commits + // the flush, with the field policy lifted, and it compiles. + expect(analyze(program).coverage.diagnostics).toEqual([]); }); test("settled generic rest-order fences count each source statement once", () => { diff --git a/tests/harness/diagnostics.test.ts b/tests/harness/diagnostics.test.ts index 77fd5937d..895ed9647 100644 --- a/tests/harness/diagnostics.test.ts +++ b/tests/harness/diagnostics.test.ts @@ -39,7 +39,14 @@ describe(`diagnostics corpus (${files.length} programs)`, () => { if (name.startsWith("checker-panic-")) process.env["SCRIPTC_CHECKER"] = "tsgo"; let result: Awaited>; try { - result = await compile(file, { outPath: join(outDir, "never"), outDir }); + // --tier=static-only: these programs pin today's refusals, which + // localized failure otherwise turns into checked dynamic slots + // (the snapshots double as its proof that static-only keeps them). + result = await compile(file, { + outPath: join(outDir, "never"), + outDir, + tier: "static-only", + }); } finally { if (checker === undefined) delete process.env["SCRIPTC_CHECKER"]; else process.env["SCRIPTC_CHECKER"] = checker; diff --git a/tests/harness/dyncheck.test.ts b/tests/harness/dyncheck.test.ts index e1a869048..97def0cc6 100644 --- a/tests/harness/dyncheck.test.ts +++ b/tests/harness/dyncheck.test.ts @@ -114,6 +114,32 @@ describe(`dynamic-boundary checks (scriptc-only${sanitize ? ", sanitized" : ""}) expect(r.stderr).toBe(""); }); + test("assertions narrowing record members or union arms check the value", async () => { + const r = await compileAndRun( + "narrowing-record-assertions", + ` + interface AlphaOpts { fast?: boolean } + interface Item { id: string; kind: K; opts?: K extends "alpha" ? AlphaOpts : never } + const items: Item<"alpha" | "gamma">[] = [{ id: "g1", kind: "gamma" }, { id: "a1", kind: "alpha", opts: { fast: true } }]; + try { const all = items as Item<"gamma">[]; console.log(all.length); } + catch (error) { console.log(error instanceof TypeError, (error as Error).message); } + try { const one = items[1]! as Item<"gamma">; console.log(one.id); } + catch (error) { console.log(error instanceof TypeError, (error as Error).message); } + interface S { stream: () => string } + function keys(u: S | { a?: S }): string[] { const m = u as Partial>; return Object.keys(m); } + try { console.log(keys({ a: { stream: () => "a" } }).join()); console.log(keys({ stream: () => "s" })); } + catch (error) { console.log(error instanceof TypeError); } + `, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe( + "true expected null | undefined at $[1].opts, got object\n" + + "true expected null | undefined at $.opts, got object\n" + + "a\ntrue\n", + ); + expect(r.stderr).toBe(""); + }); + test("typed Promise exits validate fulfillment and preserve rejection", async () => { const result = await compileAndRun( "promise-exit-payload", @@ -143,6 +169,33 @@ describe(`dynamic-boundary checks (scriptc-only${sanitize ? ", sanitized" : ""}) ).toBe(""); }); + test("a boxed native value cast to another layout is copied (decision D3)", async () => { + // Node's `as` is a no-op, so these writes would show through every + // holder. scriptc copies a boxed native value into a cast's different + // layout for now (the object model's decision D3: copy, throw, or + // share storage); the copy keeps one identity per boxed value. + const r = await compileAndRun( + "native-layout-casts", + `type Point = { x: number; y: number }; +const dict: Record = { x: 1, y: 2 }; +const v: unknown = dict; +const p = v as Point; +p.x = 5; +console.log(dict.x, p.x, (v as Point) === p); +const nums: number[] = [1, 2]; +const u: unknown = nums; +console.log((u as number[]) === nums); +const wide = u as (number | string)[]; +wide.push("s"); +console.log(nums.length, wide.length); +console.log(JSON.stringify({ ...(v as Record) })); +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe(["1 5 true", "true", "2 3", '{"x":1,"y":2}', ""].join("\n")); + expect(r.stderr).toBe(""); + }); + test("wrong-typed field throws with the path", async () => { const r = await compileAndRun( "wrong-type", diff --git a/tests/harness/llvm-differential.test.ts b/tests/harness/llvm-differential.test.ts index 15b7e7c0e..030998b5a 100644 --- a/tests/harness/llvm-differential.test.ts +++ b/tests/harness/llvm-differential.test.ts @@ -1,6 +1,8 @@ /* Both LLVM optimization modes must compile every corpus program and match * Node's stdout, successful stderr, and expected exit status. Any backend - * refusal is a failure. SCRIPTC_SAN=1 instruments emitted code and runtime. */ + * refusal is a failure. SCRIPTC_SAN=1 instruments emitted code and runtime. + * SCRIPTC_CPU_DISPATCH=1 builds with runtime CPU dispatch and runs each + * executable at SCRIPTC_CPU_LEVEL=0 and 3, the baseline and x86-64-v3 copies. */ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { globSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; @@ -30,6 +32,8 @@ const files = corpusSelect( process.env["SCRIPTC_SAN"] === "1", ); const sanitize = process.env["SCRIPTC_SAN"] === "1"; +const cpuDispatch = process.env["SCRIPTC_CPU_DISPATCH"] === "1"; +const cpuLevels: readonly (string | undefined)[] = cpuDispatch ? ["0", "3"] : [undefined]; const requestedMode = process.env["SCRIPTC_LLVM_TEST_MODE"]; if (requestedMode !== undefined && requestedMode !== "release" && requestedMode !== "dev") { throw new Error("SCRIPTC_LLVM_TEST_MODE must be release or dev when set"); @@ -62,14 +66,17 @@ function expectedExitCode(file: string): number { return 0; } -async function runBinary(cmd: string, args: string[]): Promise { +async function runBinary(cmd: string, args: string[], cpuLevel?: string): Promise { // Linux fork/exec race (observed on the sandbox lane): a sibling // worker's fork can inherit the freshly-linked binary's write fd across // its own spawn window, and exec answers ETXTBSY until that fd closes. // The condition is transient by construction — retry briefly, the // npm/cargo stance. for (let attempt = 0; ; attempt++) { - const pending = execFileAsync(cmd, args, { encoding: "buffer" }); + const pending = execFileAsync(cmd, args, { + encoding: "buffer", + ...(cpuLevel === undefined ? {} : { env: { ...process.env, SCRIPTC_CPU_LEVEL: cpuLevel } }), + }); pending.child.stdin?.end(); try { const { stdout, stderr } = await pending; @@ -179,6 +186,7 @@ async function build(file: string, optimization: "release" | "dev") { const key = hash .update(sanitize ? "san" : "plain") .update("llvm-" + optimization) + .update(cpuDispatch ? "cpu-dispatch" : "") .digest("hex") .slice(0, 16); const outDir = join(cacheDir, key); @@ -188,10 +196,11 @@ async function build(file: string, optimization: "release" | "dev") { outDir, sanitize, optimization, + ...(cpuDispatch ? { cpuDispatch } : {}), }); } -describe(`llvm differential corpus (${files.length} programs, ${optimizationModes.join("+")}${sanitize ? ", sanitized" : ""}${shardSuffix()})`, () => { +describe(`llvm differential corpus (${files.length} programs, ${optimizationModes.join("+")}${sanitize ? ", sanitized" : ""}${cpuDispatch ? ", CPU dispatch" : ""}${shardSuffix()})`, () => { test.for(files.map((f) => [f.slice(corpusDir.length + 1), f] as const))( "%s", async ([rel, file]) => { @@ -207,11 +216,14 @@ describe(`llvm differential corpus (${files.length} programs, ${optimizationMode ); expect(result.backend).toBe("llvm"); expect(result.llvmPath.endsWith(".ll")).toBe(true); - const actual = await runBinary(result.binaryPath, []); - expect(actual.stdout, `${optimization} stdout`).toEqual(oracle.stdout); - if (expectedExit === 0) - expect(comparableStderr(actual.stderr), `${optimization} stderr`).toEqual(oracle.stderr); - expect(actual.exitCode, `${optimization} exit status`).toBe(expectedExit); + for (const level of cpuLevels) { + const actual = await runBinary(result.binaryPath, [], level); + const lane = level === undefined ? optimization : `${optimization} level ${level}`; + expect(actual.stdout, `${lane} stdout`).toEqual(oracle.stdout); + if (expectedExit === 0) + expect(comparableStderr(actual.stderr), `${lane} stderr`).toEqual(oracle.stderr); + expect(actual.exitCode, `${lane} exit status`).toBe(expectedExit); + } } }, ); diff --git a/tests/harness/llvm-dist.test.ts b/tests/harness/llvm-dist.test.ts new file mode 100644 index 000000000..25f09d5c4 --- /dev/null +++ b/tests/harness/llvm-dist.test.ts @@ -0,0 +1,74 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { load } from "js-yaml"; +import { afterEach, expect, test } from "vitest"; +import { + LLVM_DIST_TAG, + LLVM_VERSION, + helperLlvmDistArgs, + llvmDistAsset, +} from "../../native/llvm-codegen/llvm-dist.mjs"; + +type Entry = { platform: string; helper?: string; llvm_dist?: string }; +const release = load( + readFileSync(new URL("../../.github/workflows/release.yml", import.meta.url), "utf8"), +) as { jobs: Record }; +const entries = release.jobs["build-native-packages"]!.strategy!.matrix!.include!; + +const saved = { ...process.env }; +afterEach(() => { + process.env = { ...saved }; +}); + +test("every release job that uploads a Linux helper links the same LLVM for it", () => { + // The publish job merges the jobs' artifacts, so two jobs that upload the + // same helper must build it the same way. + const linux = entries.filter((entry) => entry.helper?.startsWith("llvm-linux-")); + expect(linux.map((entry) => entry.platform).sort()).toEqual([ + "linux-arm64", + "linux-arm64-musl", + "linux-x64", + "linux-x64-musl", + "wasm32-wasi", + ]); + for (const entry of linux) + expect(entry.llvm_dist, entry.platform).toBe( + entry.helper!.includes("-arm64-") ? "arm64" : "x64", + ); + for (const entry of entries.filter((entry) => !entry.helper?.startsWith("llvm-linux-"))) + expect(entry.llvm_dist, entry.platform).toBeUndefined(); +}); + +test("the distribution names its archives after the LLVM version", () => { + expect(LLVM_DIST_TAG).toMatch( + new RegExp(`^llvm-${LLVM_VERSION.replaceAll(".", "\\.")}-pgo\\.\\d+$`), + ); + expect(llvmDistAsset("x64")).toBe(`scriptc-llvm-${LLVM_VERSION}-linux-x64.tar.xz`); +}); + +test("helper builds use ThinLTO and the profile only when asked", () => { + const dir = mkdtempSync(join(tmpdir(), "scriptc-llvm-dist-")); + try { + delete process.env["SCRIPTC_LLVM_THINLTO"]; + delete process.env["SCRIPTC_LLVM_PROFILE_GENERATE"]; + expect(helperLlvmDistArgs(dir)).toEqual([ + "-DSCRIPTC_LLVM_THINLTO=OFF", + "-DSCRIPTC_LLVM_PROFILE_USE=", + "-DSCRIPTC_LLVM_PROFILE_GENERATE=OFF", + ]); + process.env["SCRIPTC_LLVM_THINLTO"] = "1"; + const args = helperLlvmDistArgs(dir); + expect(args).toContain("-DCMAKE_CXX_COMPILER=clang++"); + expect(args).toContain("-DSCRIPTC_LLVM_THINLTO=ON"); + const profile = args + .find((arg) => arg.startsWith("-DSCRIPTC_LLVM_PROFILE_USE="))! + .split("=")[1]!; + // An indexed profile starts with the magic `\xfflprofi`. + expect(readFileSync(profile).subarray(0, 8)).toEqual( + Buffer.from([0xff, 0x6c, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x81]), + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/harness/npm-available.test.ts b/tests/harness/npm-available.test.ts new file mode 100644 index 000000000..0970d1f98 --- /dev/null +++ b/tests/harness/npm-available.test.ts @@ -0,0 +1,114 @@ +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { delimiter, join } from "node:path"; +import { expect, test } from "vitest"; + +const repoRoot = join(import.meta.dirname, "../.."); +const script = join(repoRoot, "scripts", "wait-npm-available.mjs"); + +// A stand-in for `npm view version`: each spec resolves on the check +// numbered in registry.json (0 never resolves). Every call is logged. +const fakeNpm = `#!/usr/bin/env node +const { appendFileSync, readFileSync } = require("node:fs"); +const { join } = require("node:path"); +const dir = process.env.FAKE_NPM_DIR; +const args = process.argv.slice(2); +appendFileSync(join(dir, "calls.log"), JSON.stringify(args) + "\\n"); +const spec = args[1]; +const calls = readFileSync(join(dir, "calls.log"), "utf8").trim().split("\\n") + .filter((line) => JSON.parse(line)[1] === spec).length; +const registry = JSON.parse(readFileSync(join(dir, "registry.json"), "utf8")); +const resolvesOn = registry[spec]; +if (resolvesOn === undefined) { + process.stderr.write("npm error code E404\\n"); + process.exit(1); +} +if (resolvesOn !== 0 && calls >= resolvesOn) { + process.stdout.write(spec.slice(spec.lastIndexOf("@") + 1) + "\\n"); +} +`; + +function run(uploads: string, registry: Record, extra: string[]) { + const dir = mkdtempSync(join(tmpdir(), "scriptc-npm-available-")); + try { + writeFileSync(join(dir, "npm"), fakeNpm); + chmodSync(join(dir, "npm"), 0o755); + writeFileSync(join(dir, "registry.json"), JSON.stringify(registry)); + writeFileSync(join(dir, "calls.log"), ""); + writeFileSync(join(dir, "uploads.txt"), uploads); + writeFileSync(join(dir, "summary.md"), ""); + const result = spawnSync(process.execPath, [script, join(dir, "uploads.txt"), ...extra], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${dir}${delimiter}${process.env["PATH"] ?? ""}`, + FAKE_NPM_DIR: dir, + GITHUB_STEP_SUMMARY: join(dir, "summary.md"), + }, + }); + const calls = readFileSync(join(dir, "calls.log"), "utf8") + .trim() + .split("\n") + .filter((line) => line !== "") + .map((line) => JSON.parse(line) as string[]); + return { + status: result.status, + output: result.stdout + result.stderr, + calls, + summary: readFileSync(join(dir, "summary.md"), "utf8"), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +test.skipIf(process.platform === "win32")( + "waits until every package resolves and reports each package's time from upload", + () => { + const accepted = Math.floor(Date.now() / 1000) - 90; + const result = run( + `@scriptc/runtime@1.2.3 ${accepted}\nscriptc@1.2.3\n`, + { "@scriptc/runtime@1.2.3": 1, "scriptc@1.2.3": 3 }, + ["--interval", "0.05", "--timeout", "30"], + ); + expect(result.status, result.output).toBe(0); + expect(result.output).toMatch(/@scriptc\/runtime@1\.2\.3 is available \(1m 3\ds\)/); + expect(result.output).toContain("scriptc@1.2.3 is available (upload time not recorded)"); + expect(result.output).toContain("Every package resolves on npm (2)."); + // A package that resolved is not asked again. + expect(result.calls.filter((args) => args[1] === "@scriptc/runtime@1.2.3")).toHaveLength(1); + expect(result.calls.filter((args) => args[1] === "scriptc@1.2.3")).toHaveLength(3); + for (const args of result.calls) + expect(args).toEqual(["view", args[1], "version", "--prefer-online"]); + expect(result.summary).toContain("| `scriptc@1.2.3` | upload time not recorded |"); + }, +); + +test.skipIf(process.platform === "win32")( + "fails with the names and versions of the packages that do not resolve by the timeout", + () => { + const result = run( + "@scriptc/runtime@1.2.3\n@scriptc/compiler@1.2.3\nscriptc@1.2.3\n", + { "@scriptc/runtime@1.2.3": 1, "@scriptc/compiler@1.2.3": 0 }, + ["--interval", "0.1", "--timeout", "0.5"], + ); + expect(result.status, result.output).toBe(1); + expect(result.output).toContain("2 of 3 packages do not resolve on npm after"); + expect(result.output).toContain("::error::@scriptc/compiler@1.2.3 does not resolve on npm"); + expect(result.output).toContain("::error::scriptc@1.2.3 does not resolve on npm"); + expect(result.output).not.toContain("::error::@scriptc/runtime@1.2.3"); + expect( + result.calls.filter((args) => args[1] === "@scriptc/compiler@1.2.3").length, + ).toBeGreaterThan(1); + expect(result.summary).toContain("| `@scriptc/compiler@1.2.3` | missing |"); + expect(result.summary).toContain("| `scriptc@1.2.3` | missing |"); + }, +); + +test("rejects a malformed uploads file", () => { + const result = run("scriptc 1.2.3\n", {}, []); + expect(result.status).toBe(1); + expect(result.output).toContain("malformed upload line: scriptc 1.2.3"); + expect(result.calls).toHaveLength(0); +}); diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 076733739..368d4257f 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -424,7 +424,7 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { ); test.each(["llvm"] as const)( - "a package probing for optional addons through createRequire compiles with deferred refusals (%s)", + "a package probing for optional addons through createRequire resolves at run time (%s)", async (backend) => { const entry = join(pilotRoot, "addon-probe-cli.ts"); const { coverage } = analyze(entry, { npmStatic: "auto" }); @@ -433,10 +433,18 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { expect( (coverage.runtimeFences ?? []).filter((f) => f.code === "SC2020").map((f) => f.message), ).toEqual([ - expect.stringContaining("'createRequire's require with a computed specifier"), - expect.stringContaining("'createRequire's require.resolve as a value'"), - expect.stringContaining("'createRequire's require.resolve with a computed request'"), - expect.stringContaining("'createRequire's require.resolve with a computed request'"), + expect.stringContaining( + "createRequire's require with a computed specifier resolves at run time", + ), + expect.stringContaining( + "createRequire's require.resolve with a computed specifier resolves at run time", + ), + expect.stringContaining( + "createRequire's require.resolve with a computed specifier resolves at run time", + ), + expect.stringContaining( + "createRequire's require.resolve with a computed specifier resolves at run time", + ), ]); const outDir = join(cacheDir, "addon-probe", sanitize ? "san" : "plain", backend); const result = await compile(entry, { @@ -461,31 +469,59 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { }, ); + // Node would load the installed file, which the compiled program cannot: + // the require ends the program with the refusal instead of throwing, so + // the package's catch clause does not take its fallback. test.each([ [ - "computed specifier", - 'const spec = "node:path"; const path = load(spec);', - "computed specifier", + "addon", + /^scriptc: native addon '.*[\\/]native[\\/]installed\.node' requires Node's addon runtime, which is unavailable in scriptc \[SC2020 at .*index\.js:\d+\]\n$/, ], + [ + "module", + /^scriptc: createRequire's require of '.*[\\/]installed\.js' resolves to '.*[\\/]installed\.js', which is not part of the compiled program: the module graph is fixed at build time \[SC2020 at .*index\.js:\d+\]\n$/, + ], + ] as const)( + "a computed require of an installed %s ends the program with a refusal", + async (kind, stderr) => { + const entry = join(pilotRoot, "addon-probe-installed-cli.ts"); + const outDir = join(cacheDir, "addon-probe-installed", sanitize ? "san" : "plain"); + const result = await compile(entry, { + backend: "llvm", + npmStatic: "auto", + sanitize, + outDir, + outPath: join(outDir, "program"), + }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const nativeRes = await runBinary(result.binaryPath, [kind]); + expect(nativeRes.stdout.toString()).toBe("before\n"); + expect(comparableStderr(nativeRes.stderr).toString()).toMatch(stderr); + expect(nativeRes.exitCode).toBe(1); + }, + ); + + test.each([ ["escaped require", 'const alias = load; const path = alias("node:path");', "escapes"], [ - "computed specifier in a catch clause", - 'try { load("node:path"); } catch { load(String(1)); } const path = load("node:path");', + "computed resolve.paths request in a catch clause", + 'try { load("node:path"); } catch { load.resolve.paths(String(1)); } const path = load("node:path");', "outside a try block", ], [ - "computed specifier in a function inside a try block", - 'try { [1].map((n) => load(String(n))); } catch {} const path = load("node:path");', + "computed resolve.paths request in a function inside a try block", + 'try { [1].map((n) => load.resolve.paths(String(n))); } catch {} const path = load("node:path");', "outside a try block", ], [ - "computed specifier in a class field inside a try block", - 'try { new (class { addon = load(String(1)); })(); } catch {} const path = load("node:path");', + "spread require arguments in a class field inside a try block", + 'try { new (class { addon = load(...[String(1)]); })(); } catch {} const path = load("node:path");', "outside a try block", ], [ - "resolve value outside a try block", - 'const find = load.resolve; const path = load("node:path");', + "resolve.paths value outside a try block", + 'const find = load.resolve.paths; const path = load("node:path");', "outside a try block", ], [ @@ -493,6 +529,16 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { 'try { const alias = load; alias("node:path"); } catch {} const path = load("node:path");', "escapes", ], + [ + "computed resolve.paths request", + 'const spec = "node:path"; load.resolve.paths(spec); const path = load("node:path");', + "outside a try block", + ], + [ + "spread require arguments", + 'const specs = ["node:path"]; load(...specs); const path = load("node:path");', + "outside a try block", + ], [ "reassigned require", 'let load = createLoader(import.meta.url); load = () => ({}); const path = load("node:path");', diff --git a/tests/harness/self-hosting-native-driver.test.ts b/tests/harness/self-hosting-native-driver.test.ts index 1cc920bd2..8cfd16eea 100644 --- a/tests/harness/self-hosting-native-driver.test.ts +++ b/tests/harness/self-hosting-native-driver.test.ts @@ -1,4 +1,4 @@ -import { verifyBootstrapSeed } from "../../scripts/bootstrap-seed.mjs"; +import { seedBuildEnvironment, verifyBootstrapSeed } from "../../scripts/bootstrap-seed.mjs"; import { execFile, spawnSync } from "node:child_process"; import { chmodSync, @@ -89,12 +89,7 @@ test( // seed avoids optimizing the huge compiler module before ASan can // exercise its complete traversal, serialization and LLVM emission. ...options, - env: { - ...process.env, - SCRIPTC_SAN: sanitize ? "1" : "", - SCRIPTC_NATIVE_EMIT_IR: "1", - SCRIPTC_NATIVE_OPTIMIZATION: sanitize ? "dev" : "release", - }, + env: { ...process.env, ...seedBuildEnvironment(sanitize) }, }, ), ); diff --git a/tests/harness/type-patches.test.ts b/tests/harness/type-patches.test.ts index 816418a96..a2996d724 100644 --- a/tests/harness/type-patches.test.ts +++ b/tests/harness/type-patches.test.ts @@ -7,7 +7,10 @@ * apply as written warn (SC8030-SC8033). The `.d.ts`, `@types`, replaced * file and supplied entry cases run in npm.test.ts (type-patch-dts, * type-patch-types-twin, type-patch-files, type-patch-types-entry); here - * their unpatched copies show the patches are what makes them compile. And + * their unpatched copies show the patches are what makes them compile, and + * a failing check names the declaration and the patch (scr_type_origin.c, + * frontend/lowering/decl-origin.ts), its own package's also when two + * packages declare a record of one name and shape. And * commander-calc carries the HelpContext.helpWidth patch. */ import { execFile } from "node:child_process"; import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; @@ -115,7 +118,11 @@ describe("type patches", () => { if (result.ok) { const native = await run(result.binaryPath, []); expect(native.exitCode).not.toBe(0); - expect(native.stderr).toContain("TypeError: expected string at $, got undefined"); + expect(native.stderr).toBe( + "Uncaught TypeError: expected string at $, got undefined\n" + + " profilekit@3.1.0 declares Profile.email as string (index.d.ts:7)\n" + + ` if that declaration is wrong, patch it: scriptc types patch profilekit Profile.email '| undefined'\n`, + ); } } finally { rmSync(copy, { recursive: true, force: true }); @@ -165,6 +172,78 @@ describe("type patches", () => { }, ); + test( + "a failing check names the package declaration and the patch", + { timeout: 600_000 }, + async () => { + const result = await build(join(fixture, "unpatched/main.ts"), "unpatched", ["termctx"]); + const native = await run(result.binaryPath, []); + const patch = (selector: string) => + ` if that declaration is wrong, patch it: scriptc types patch termctx ${selector} '| undefined'`; + expect(native.stdout).toBe( + [ + "expected number at $, got undefined", + " termctx@1.2.0 declares Terminal.columns() as number (index.js:16)", + patch("'Terminal.columns()'"), + "undefined is not representable in the target union (a value narrowed or asserted past it still held it)", + " termctx@1.2.0 declares OutputContext.width as number (index.js:24)", + patch("OutputContext.width"), + "", + ].join("\n"), + ); + }, + ); + + test( + "records of one name and shape from two packages name their own declarations", + { timeout: 600_000 }, + async () => { + const result = await build(join(fixture, "same-name/main.ts"), "same-name", [ + "panectx", + "gridctx", + ]); + const native = await run(result.binaryPath, []); + const failure = (pkg: string, version: string) => [ + "undefined is not representable in the target union (a value narrowed or asserted past it still held it)", + ` ${pkg}@${version} declares OutputContext.width as number (index.js:24)`, + ` if that declaration is wrong, patch it: scriptc types patch ${pkg} OutputContext.width '| undefined'`, + ]; + expect(native.stdout).toBe( + [...failure("panectx", "2.0.0"), ...failure("gridctx", "4.1.0"), ""].join("\n"), + ); + }, + ); + + test( + "a rebuild after the package's version changes names the installed version", + { timeout: 600_000 }, + async () => { + const copy = mkdtempSync(join(tmpdir(), "scriptc-reinstalled-")); + try { + cpSync(fixture, copy, { recursive: true }); + const entry = join(copy, "unpatched/main.ts"); + const declares = async (name: string) => { + const native = await run((await build(entry, name, ["termctx"])).binaryPath, []); + return native.stdout.split("\n").filter((line) => line.includes(" declares ")); + }; + expect(await declares("reinstalled-1.2.0")).toEqual([ + " termctx@1.2.0 declares Terminal.columns() as number (index.js:16)", + " termctx@1.2.0 declares OutputContext.width as number (index.js:24)", + ]); + writeFileSync( + join(copy, "node_modules/termctx/package.json"), + '{ "name": "termctx", "version": "1.3.0", "main": "index.js" }\n', + ); + expect(await declares("reinstalled-1.3.0")).toEqual([ + " termctx@1.3.0 declares Terminal.columns() as number (index.js:16)", + " termctx@1.3.0 declares OutputContext.width as number (index.js:24)", + ]); + } finally { + rmSync(copy, { recursive: true, force: true }); + } + }, + ); + test( "patches that do not apply as written warn and the build goes on", { timeout: 600_000 },