diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e59cf7739..aee1a2d96 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -101,8 +101,8 @@ jobs: if (receiptRun) appendFileSync(process.env.GITHUB_OUTPUT, `receipt_run=${receiptRun}\n`); NODE - # Collects the native compiler's profile and function layout on this commit - # (scripts/refresh-cli-profile.mjs, scripts/refresh-cli-layout.mjs), so every + # Collects the native compiler's function layout and profile on this commit + # (scripts/refresh-cli-layout.mjs, scripts/refresh-cli-profile.mjs), so every # platform's CLI build optimizes and lays out the code it compiles. The # committed ones fall behind main within a day or two. If this job fails, # the CLI builds use the committed profile and layout. @@ -111,7 +111,7 @@ jobs: needs: check-release if: needs.check-release.outputs.should_release == 'true' && needs.check-release.outputs.reuse_artifacts != 'true' runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 75 continue-on-error: true permissions: contents: read @@ -136,14 +136,20 @@ jobs: git clone --quiet --depth 1 https://github.com/vercel-labs/tsc-ts "$RUNNER_TEMP/tsc-ts" echo "tsc-ts $(git -C "$RUNNER_TEMP/tsc-ts" rev-parse HEAD)" npm ci --prefix "$RUNNER_TEMP/tsc-ts" + # The layout comes first: the profile matches the functions of builds + # with the layout its instrumented compiler was built with, so the + # profile is collected with a second instrumented compiler built with + # this layout. + - name: Collect the function layout + run: >- + node scripts/refresh-cli-layout.mjs --jobs 4 + --output "$RUNNER_TEMP/cli-profile/cli.layout.gz" --work "$RUNNER_TEMP/cli-layout-work" - name: Collect the profile + env: + SCRIPTC_NATIVE_LAYOUT: ${{ runner.temp }}/cli-profile/cli.layout.gz run: >- node scripts/refresh-cli-profile.mjs --tsc-ts "$RUNNER_TEMP/tsc-ts" --jobs 4 --output "$RUNNER_TEMP/cli-profile/cli.profdata.gz" --work "$RUNNER_TEMP/cli-profile-work" - - name: Collect the function layout - run: >- - node scripts/refresh-cli-layout.mjs --jobs 4 --instrumented "$RUNNER_TEMP/cli-profile-work/cli" - --output "$RUNNER_TEMP/cli-profile/cli.layout.gz" - uses: actions/upload-artifact@v4 with: name: cli-profile diff --git a/AGENTS.md b/AGENTS.md index 00ce6bfd1..3e631a92f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,8 @@ their supported native-clang contracts locally; other hosts retain those checks in the Sandboxes. Both lanes green is the bar before shipping any change. +`SCRIPTC_SANDBOX_NATIVE_REFERENCE` names a built checkout inside a prepared `SCRIPTC_SANDBOX_SNAPSHOT` or image: each lane copies that checkout's LLVM helper and runtime pack instead of rebuilding them when their source inputs are identical. Sandbox creation is rate limited per team in vCPUs per minute, so the runner retries a rate-limited creation with backoff. Larger Sandboxes (`SCRIPTC_SANDBOX_VCPUS` up to 32, with `SCRIPTC_TEST_WORKERS` at half the vCPUs) shorten the lanes most. + `VERCEL_OIDC_TOKEN` is the preferred credential. `VERCEL_TOKEN` remains supported with explicit `VERCEL_TEAM_ID` and `VERCEL_PROJECT_ID`. The custom `SCRIPTC_SANDBOX_IMAGE` is optional: without it, the gate starts from @@ -58,6 +60,8 @@ A change that must not cost good code anything, such as a compatibility fix that node scripts/zero-cost-gate.mjs --baseline=/vercel/sandbox/prev --candidate=. --allow='\barguments\b' ``` +The whole gate takes 30 to 75 minutes on one 8-vCPU Sandbox, so shard it across Sandboxes. `--shard=i/n` splits the bench and corpus programs; the self-hosted compiler runs in shard 1, or alone with `--set=self`, and its candidate-source build overlaps its comparison build on hosts with 40 GB or more. Each shard writes `--json`, and `node scripts/zero-cost-merge.mjs ...` merges them into one verdict, recomputing the compile-time intervals over every shard. `--record-baseline-cache=` builds only the baseline and records each program's artifact hashes; `--baseline-cache=` then skips the baseline build of every program whose candidate artifacts hash the same, which halves the work for a change that leaves most programs identical. A record applies only to the built baseline checkout, toolchain, and paths it was made with, and is ignored otherwise; a program whose entry, relative imports, or optimization level changed since the record builds its baseline. + Test location follows scope: - Co-locate white-box unit tests with implementation files under diff --git a/Dockerfile.sandbox b/Dockerfile.sandbox index 61f5d8ad9..7a943a85f 100644 --- a/Dockerfile.sandbox +++ b/Dockerfile.sandbox @@ -26,6 +26,7 @@ RUN apt-get update \ libclang-rt-22-dev \ libpolly-22-dev \ libzstd-dev \ + lld-22 \ llvm-22-dev \ ninja-build \ xz-utils \ @@ -33,6 +34,7 @@ RUN apt-get update \ && ln -sf clang-22 /usr/bin/clang \ && ln -sf clang++-22 /usr/bin/clang++ \ && test "$(llvm-config-22 --version)" = 22.1.8 \ + && test -x "$(dirname "$(readlink -f /usr/bin/clang)")/ld.lld" \ && rm -rf /var/lib/apt/lists/* RUN curl --fail --silent --show-error --location \ diff --git a/RELEASING.md b/RELEASING.md index e650554a2..09c0ce3b6 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -16,9 +16,9 @@ CI (`.github/workflows/release.yml`) compares the version in `packages/cli/packa The release job builds each pinned LLVM helper, precompiled runtime pack, and native CLI on its matching host. The native CLI package bundles its TypeScript checker, declarations, and comptime evaluator, and depends on its platform's LLVM helper and runtime pack packages at the same version; the compiler resolves them beside itself. The standalone archives bundle the helper and runtime pack too. npm postinstall installs the matching native executable; Node is needed for npm installation but not native compilation or execution. The user's toolchain performs the final platform link. Sanitized builds also use it to instrument program LLVM and compile the C runtime. Platform packages publish ahead of `@scriptc/compiler` and `scriptc`. The GitHub release attaches standalone compiler archives, their SHA-256 checksums, and the surface manifest after npm publication succeeds. -Release builds of the native CLI optimize the compiler with a profile collected on the release commit. The `cli-profile` job runs `node scripts/refresh-cli-profile.mjs` on Linux x64, which builds an instrumented compiler, runs it on tsc-ts, a slice of the corpus, and the runtime benchmark workloads, and writes the merged sparse profile. Every platform's CLI build uses that profile (`SCRIPTC_NATIVE_PROFILE_USE`), and the GitHub release attaches it as `scriptc--cli.profdata.gz`. Code the profile does not describe, or describes from older compiler sources, is optimized as without a profile. When the profile job fails, the CLI builds use the committed profile `packages/compiler/native/cli.profdata.gz`, which local and CI builds of the native compiler also use. Refresh the committed profile when compiler code has changed substantially since the last refresh: run `node scripts/refresh-cli-profile.mjs --tsc-ts ` on a quiet Linux x64 host and commit the new profile on its own. +Release builds of the native CLI optimize the compiler with a profile collected on the release commit. The `cli-profile` job runs `node scripts/refresh-cli-profile.mjs` on Linux x64, which builds an instrumented compiler, runs it on tsc-ts, a slice of the corpus, and the runtime benchmark workloads, and writes the merged profile. The profile keeps the functions that training never ran, with zero counts, so the CLI builds optimize them as cold code and place them after the code that ran. Every platform's CLI build uses that profile (`SCRIPTC_NATIVE_PROFILE_USE`), and the GitHub release attaches it as `scriptc--cli.profdata.gz`. Code the profile does not describe, or describes from older compiler sources, is optimized as without a profile. When the profile job fails, the CLI builds use the committed profile `packages/compiler/native/cli.profdata.gz`, which local and CI builds of the native compiler also use. Refresh the committed profile when compiler code has changed substantially since the last refresh: run `node scripts/refresh-cli-profile.mjs --tsc-ts ` on a quiet Linux x64 host and commit the new profile on its own. -Builds of the native compiler also order its functions by the committed layout `packages/compiler/native/cli.layout.gz`: the functions that `--version` and rebuilds of unchanged programs run come first, then the ones other builds run, ordered by the share of the layout's training builds that run each, so a run maps fewer pages of the executable. Functions the layout does not list keep their usual order. Function names of closures and generated helpers change with nearby edits, so the `cli-profile` job also collects a layout on the release commit (`scripts/refresh-cli-layout.mjs`, reusing the job's instrumented compiler), and the CLI builds use it (`SCRIPTC_NATIVE_LAYOUT`). Refresh the committed layout the same way as the profile, with `node scripts/refresh-cli-layout.mjs`, and commit it on its own. +Builds of the native compiler also order its functions by the committed layout `packages/compiler/native/cli.layout.gz`: the functions that `--version` and rebuilds of unchanged programs run come first, then the ones other builds run, ordered by the share of the layout's training builds that run each, so a run maps fewer pages of the executable. Functions the layout does not list keep their usual order. Function names of closures and generated helpers change with nearby edits, so the `cli-profile` job also collects a layout on the release commit (`scripts/refresh-cli-layout.mjs`), and the CLI builds use it (`SCRIPTC_NATIVE_LAYOUT`). Instrumented builds use the same layout as optimized builds, because a function laid out in a different partition than in the instrumented build no longer matches its profile record. The job therefore collects the layout first and then builds the instrumented compiler for the profile with that layout. Refresh the committed layout the same way, with `node scripts/refresh-cli-layout.mjs`, before the profile, and commit each on its own. Publishing uses npm trusted publishing (OIDC) — there is no npm token secret. Each published package must have a GitHub Actions trusted publisher for diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 9af4ace00..f3a444ea9 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -28,6 +28,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `numeric-kernels` | cpu | sieve, hashing, `number[][]` matmul, `Float64Array` | | `optional-fields` | cpu | records and class instances with `number \| undefined` and other scalar-union fields | | `published-graphs` | cpu | syntax trees with parent links and a symbol table, published with `@scriptc/threads` and then read | +| `record-untyped-writes` | cpu | typed records written through untyped views: a counter helper taking `unknown` bumps fields of a stats record, and `Object.assign` merges patches into a settings record, with typed reads in between | | `record-boundaries` | cpu | records built from unchecked array reads passed to and returned from annotated functions | | `records-sort` | cpu | comparator sorts, group-by, `reduce`/`filter` | | `regex-logs` | cpu | regex `exec` with groups, `test`, global `replace` | diff --git a/benchmarks/runtime/workloads/record-untyped-writes.json b/benchmarks/runtime/workloads/record-untyped-writes.json new file mode 100644 index 000000000..35f8e53fc --- /dev/null +++ b/benchmarks/runtime/workloads/record-untyped-writes.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/record-untyped-writes.ts", + "args": [ + "4" + ] +} diff --git a/benchmarks/runtime/workloads/record-untyped-writes.ts b/benchmarks/runtime/workloads/record-untyped-writes.ts new file mode 100644 index 000000000..956f7955b --- /dev/null +++ b/benchmarks/runtime/workloads/record-untyped-writes.ts @@ -0,0 +1,49 @@ +// Typed records updated through untyped views: a stats record written by a +// generic counter helper that takes `unknown`, settings merged with +// Object.assign, and typed reads in between. Each write stores into the +// native record; typed code reads the same object. +type Stats = { hits: number; misses: number; bytes: number; name: string; ok: boolean }; +type Settings = { host: string; port: number; debug: boolean; retries: number }; + +const scale = Number(process.argv[2] ?? "1"); + +function bump(target: unknown, key: string, by: number): void { + const record = target as Record; + record[key] = record[key]! + by; +} + +const stats: Stats[] = []; +for (let i = 0; i < 16; i++) stats.push({ hits: 0, misses: 0, bytes: 0, name: "s" + i, ok: true }); + +let seed = 12345; +let checksum = 0; +const rounds = Math.floor(1_000_000 * scale); +for (let i = 0; i < rounds; i++) { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + const s = stats[seed & 15]!; + if ((seed >>> 8) % 5 === 0) bump(s, "misses", 1); + else bump(s, "hits", 1); + bump(s, "bytes", (seed >>> 12) & 1023); + checksum += s.hits - s.misses; +} + +const settings: Settings = { host: "localhost", port: 80, debug: false, retries: 0 }; +const view: object = settings; +const patches: object[] = [ + { port: 8080 }, + { debug: true, retries: 3 }, + { host: "example.test" }, + { port: 443, debug: false }, +]; +for (let i = 0; i < Math.floor(200_000 * scale); i++) { + Object.assign(view, patches[i & 3]!); + checksum += settings.port + settings.retries; +} + +let hits = 0; +let bytes = 0; +for (const s of stats) { + hits += s.hits; + bytes += s.bytes; +} +console.log("hits", hits, "bytes", bytes, "settings", JSON.stringify(settings), "checksum", checksum); diff --git a/docs/content/docs/how-it-works.mdx b/docs/content/docs/how-it-works.mdx index a6520b46e..3747679c4 100644 --- a/docs/content/docs/how-it-works.mdx +++ b/docs/content/docs/how-it-works.mdx @@ -134,8 +134,8 @@ Program objects define `main` and leave selected `scr_*` runtime functions undef ## Native runtime -- **Memory:** Values use reference counting. Acyclic values are freed when their last reference is released, and a cycle collector runs at defined collection points. Some cycles involving opaque native values can remain allocated. When the main thread's script ends, the process exits without freeing what the program still holds, and the operating system reclaims it; worker threads free their heaps when they exit. Builds with `--sanitize`, and processes run with `SCRIPTC_EXIT_FREE=1` (for external leak checkers), free everything at exit. -- **Concurrency:** An `async` function that suspends only at its own `await` expressions compiles to a stackless coroutine whose heap frame holds the values live across those awaits; other suspending `async` functions and async generators use stackful fibers. The native event loop schedules microtasks, timers, and I/O, using kqueue on macOS and epoll on Linux. +- **Memory:** Values use reference counting. Acyclic values are freed when their last reference is released, and a cycle collector runs at defined collection points. When the compiler proves that a group of objects is released together, such as a tree that a recursive function builds and its caller only reads, the group is allocated in an arena and released at once, without freeing each object or collecting its cycles. Some cycles involving opaque native values can remain allocated. When the main thread's script ends, the process exits without freeing what the program still holds, and the operating system reclaims it; worker threads free their heaps when they exit. Builds with `--sanitize`, and processes run with `SCRIPTC_EXIT_FREE=1` (for external leak checkers), free everything at exit. +- **Concurrency:** An `async` function or async generator that suspends only at its own `await` and `yield` expressions compiles to a stackless coroutine whose heap frame holds the values live across them; other suspending `async` functions and async generators use stackful fibers, as do debug builds, libraries, worker programs and Windows executables. The native event loop schedules microtasks, timers, and I/O, using kqueue on macOS and epoll on Linux. - **Networking:** Supported `net`, `http`, `https`, `tls`, `dgram`, and `dns` APIs use native implementations. TLS uses vendored mbedTLS. - **Numbers:** Values use JavaScript's double-precision representation. Number-to-string formatting uses shortest-roundtrip output and is tested against Node.js. - **Regular expressions and Unicode:** The runtime uses QuickJS's regular-expression and Unicode libraries, linked into programs that use them. diff --git a/docs/content/limitations/array-bounds-and-runtime-errors.mdx b/docs/content/limitations/array-bounds-and-runtime-errors.mdx index 7490a1e15..082dd37f3 100644 --- a/docs/content/limitations/array-bounds-and-runtime-errors.mdx +++ b/docs/content/limitations/array-bounds-and-runtime-errors.mdx @@ -11,6 +11,6 @@ Runtime traps cannot be caught. User-thrown values and supported exceptions, including JSON parse, checked-cast, filesystem, regex, and `String.prototype.repeat` count errors, can be caught; hard traps such as out-of-range reads of statically typed ordinary arrays terminate the process. Deep recursion in Linux and macOS executables throws Node's catchable `RangeError: Maximum call stack size exceeded`, on the main thread, in worker threads, and inside async functions and generators. -The depth at which it is thrown differs from Node: the main thread uses the process stack limit (usually 8 MiB), worker threads get an 8 MiB stack, and async functions and generators run on 512 KiB stacks, so native code usually recurses deeper than Node on threads and somewhat less deeply inside async bodies. +The depth at which it is thrown differs from Node: the main thread uses the process stack limit (usually 8 MiB), worker threads get an 8 MiB stack, async functions and generators that keep a fiber run on 512 KiB stacks, and stackless ones run on the stack that resumes them, so native code usually recurses deeper than Node on threads and somewhat less deeply inside fiber bodies. Recursion inside runtime library operations, such as serializing deeply nested data, is not checked. Windows executables, WASI builds and library builds do not check the stack, and exhausting it there terminates the process. diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index c7efef13f..a263ee601 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -212,8 +212,8 @@ struct UnnameStructTypes : PassInfoMixin { // How the --profile-use profile matched the functions it annotated, summed // over partitions: functions that received its counts, functions whose // control flow changed since it was collected (hash mismatch), and functions -// it has no record of (new, or never run in training: the merged profile is -// sparse). A partition reused from the cache without its counts is counted +// it has no record of (new since it was collected, or never run in training +// when the profile was merged with --sparse). A partition reused from the cache without its counts is counted // in Unknown. struct ProfileMatch { std::atomic Annotated{0}; diff --git a/packages/cli/test/target-cpu.test.ts b/packages/cli/test/target-cpu.test.ts index 5bc6717c3..dec7de15b 100644 --- a/packages/cli/test/target-cpu.test.ts +++ b/packages/cli/test/target-cpu.test.ts @@ -54,7 +54,7 @@ test.runIf(process.platform === "linux" && process.arch === "x64")( expect(baseline).not.toContain("sanitize_address"); expect(await build(["--target-cpu=baseline"])).toBe(baseline); const v3 = await build(["--target-cpu=x86-64-v3"]); - expect(v3).toMatch(/^attributes #0 = \{ "target-cpu"="x86-64-v3" \}$/m); + expect(v3).toMatch(/^attributes #0 = \{ uwtable "target-cpu"="x86-64-v3" \}$/m); if (runsV3) expect((await execFileAsync(outPath)).stdout).toBe(expected); // The bootstrap's cached route builds without a level. expect(await build([], true)).toBe(baseline); diff --git a/packages/compiler/native/cli.layout.gz b/packages/compiler/native/cli.layout.gz index 4ede69568..84a2558b5 100644 Binary files a/packages/compiler/native/cli.layout.gz and b/packages/compiler/native/cli.layout.gz differ diff --git a/packages/compiler/native/cli.profdata.gz b/packages/compiler/native/cli.profdata.gz index aa9398b5f..4480e4ecb 100644 Binary files a/packages/compiler/native/cli.profdata.gz and b/packages/compiler/native/cli.profdata.gz differ diff --git a/packages/compiler/src/backend/link-plan-core.ts b/packages/compiler/src/backend/link-plan-core.ts index fc05d2961..f8ad20e6f 100644 --- a/packages/compiler/src/backend/link-plan-core.ts +++ b/packages/compiler/src/backend/link-plan-core.ts @@ -75,7 +75,7 @@ export function executableLinkInputs(options: { * links as needed: the runtime defines the libm functions most programs * reach (scr_glibc_math.c) and Math's transcendentals (scr_ieee754.c), so a * program that calls no other libm function, such as pow, starts without - * loading libm.so. */ + * loading libm.so when it links with lld (see inputsWithoutLld). */ export function systemLibraryLinkerArgs( target: Pick, names: readonly string[], diff --git a/packages/compiler/src/backend/linker.ts b/packages/compiler/src/backend/linker.ts index c4f04133e..a29a7c947 100644 --- a/packages/compiler/src/backend/linker.ts +++ b/packages/compiler/src/backend/linker.ts @@ -21,7 +21,7 @@ import { import { type NativeArtifactDependency } from "./native/contracts.js"; import { RuntimePackError, stageRuntimePackArtifacts } from "./runtime-pack.js"; import type { NativeLinkPlan } from "./link-plan.js"; -import { lldLinkerArgs } from "./lld.js"; +import { inputsWithoutLld, lldLinkerArgs } from "./lld.js"; import { systemLibraryLinkerArgs } from "./link-plan-core.js"; import { createDarwinDebugSymbols, @@ -244,15 +244,19 @@ export async function linkNativeExecutable( stagedRoot = staged.root; // Linux links with lld when the default driver finds it. const lld = explicitLinker === undefined ? lldLinkerArgs(plan.target.platform, linker) : []; - const args = [ - ...(explicitLinker === undefined ? plan.target.defaultLinkerArgs : []), - ...lld, - ...plan.driverFlags, - ...plan.inputs.map((input) => staged.replacements.get(input) ?? input), - ...systemLibraryLinkerArgs(plan.target, plan.systemLibraries), - "-o", - privateOut, - ]; + const linkArgs = (withLld: boolean): string[] => { + const inputs = plan.inputs.map((input) => staged.replacements.get(input) ?? input); + return [ + ...(explicitLinker === undefined ? plan.target.defaultLinkerArgs : []), + ...(withLld ? lld : []), + ...plan.driverFlags, + ...(withLld ? inputs : inputsWithoutLld(plan.target.platform, inputs)), + ...systemLibraryLinkerArgs(plan.target, plan.systemLibraries), + "-o", + privateOut, + ]; + }; + const args = linkArgs(lld.length !== 0); // The pack snapshots bracket both verification passes and private staging; // the program-object snapshot begins before helper emission. Snapshot the // remaining cache-bearing inputs before the linker consumes them, then @@ -285,10 +289,7 @@ export async function linkNativeExecutable( // A failed link with lld runs again with the driver's default linker, // whose error (if any) is the one reported. if (lld.length === 0) throw error; - await execFileAsync( - linker, - args.filter((arg) => !lld.includes(arg)), - ); + await execFileAsync(linker, linkArgs(false)); }); const output = await stat(privateOut); if (!output.isFile() || output.size === 0) throw new Error("linker produced no executable"); diff --git a/packages/compiler/src/backend/lld.test.ts b/packages/compiler/src/backend/lld.test.ts index b0a466cf6..048f2ab94 100644 --- a/packages/compiler/src/backend/lld.test.ts +++ b/packages/compiler/src/backend/lld.test.ts @@ -2,7 +2,7 @@ import { chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test } from "vitest"; -import { driverFindsLld, lldLinkerArgs } from "./lld.js"; +import { driverFindsLld, inputsWithoutLld, lldLinkerArgs } from "./lld.js"; const directories: string[] = []; afterEach(() => { @@ -59,3 +59,10 @@ test.skipIf(process.platform === "win32")( expect(lldLinkerArgs("linux", clang, { ...env, SCRIPTC_LINKER: clang })).toEqual([]); }, ); + +test("Linux links without lld leave out the glibc math unit", () => { + const inputs = ["/pack/scr_lib.o", "/pack/scr_glibc_math.o", "/tmp/program.o"]; + expect(inputsWithoutLld("linux", inputs)).toEqual(["/pack/scr_lib.o", "/tmp/program.o"]); + expect(inputsWithoutLld("darwin", inputs)).toEqual(inputs); + expect(inputsWithoutLld("windows", inputs)).toEqual(inputs); +}); diff --git a/packages/compiler/src/backend/lld.ts b/packages/compiler/src/backend/lld.ts index 790e07062..7a9210407 100644 --- a/packages/compiler/src/backend/lld.ts +++ b/packages/compiler/src/backend/lld.ts @@ -1,5 +1,5 @@ import { realpathSync, statSync } from "node:fs"; -import { delimiter, dirname, isAbsolute, join, resolve } from "node:path"; +import { basename, delimiter, dirname, isAbsolute, join, resolve } from "node:path"; function isFile(path: string): boolean { try { @@ -47,3 +47,21 @@ export function lldLinkerArgs( ? ["-fuse-ld=lld"] : []; } + +/** + * The link inputs for a Linux link without lld. The runtime's + * scr_glibc_math.c defines glibc's floor, fmod, frexp and a few others as + * hidden functions so that a program calling no other libm function starts + * without libm.so. That works with lld, which garbage-collects the unused + * definitions and decides `--as-needed` from the live code. GNU ld decides + * `--as-needed` before section GC, so the runtime's unreachable code keeps + * libm needed, and it keeps every regular definition of a name that a needed + * shared library also defines, hidden or not: the unit would add all of its + * code to every executable and save nothing. Its functions answer what + * glibc's do, so linking glibc's own instead is equivalent. + */ +export function inputsWithoutLld(platform: string, inputs: readonly string[]): string[] { + return platform === "linux" + ? inputs.filter((input) => basename(input) !== "scr_glibc_math.o") + : [...inputs]; +} diff --git a/packages/compiler/src/backend/llvm/alloc.ts b/packages/compiler/src/backend/llvm/alloc.ts index 2efeb96ac..d227ac005 100644 --- a/packages/compiler/src/backend/llvm/alloc.ts +++ b/packages/compiler/src/backend/llvm/alloc.ts @@ -141,7 +141,7 @@ const CYC_GEN = 3; /** ScrCycHdr.buf, an i32 at this header offset. */ export const CYC_BUF = 4; /** The emitted descriptor `{ trace, free_fn, kind, id }` (ScrCycType). */ -const CYC_TYPE = "{ ptr, ptr, i32, i32 }"; +export const CYC_TYPE = "{ ptr, ptr, i32, i32 }"; function declareState(host: AllocHost): void { if (host.threadLocalAlloc !== true) host.declare(`@scr_sa = external global ${SA_TYPE}`); diff --git a/packages/compiler/src/backend/llvm/arenas.ts b/packages/compiler/src/backend/llvm/arenas.ts new file mode 100644 index 000000000..ee5ea8ab0 --- /dev/null +++ b/packages/compiler/src/backend/llvm/arenas.ts @@ -0,0 +1,437 @@ +/* Group arenas (runtime scr_arena.h): objects that region-groups.ts proves + * die together are bump-allocated in an arena and released at once. + * + * - A root (a fresh-result call or `new` whose value its caller keeps in + * one scope) opens an arena before it is evaluated and closes it after. + * The scope that holds the value drops the arena when it exits, on every + * path. A statement-frame entry closes the arena too, so an exception + * that skips the close never leaves it the allocation target. + * - A call region (a call of a leak-free function that returns no + * reference) opens an arena and drops it as soon as the call returns, + * or when its statement frame unwinds. + * + * region-sites.ts keeps only the roots and call regions whose groups grow, + * and picks the allocations that take the arena: those whose value the + * group keeps. Arena-aware functions are the ones reached from the kept + * roots and call regions. There, a kept `new`, record or array literal + * calls the arena-aware constructor (`.r`, `scr_arr_new*_r`), which + * allocates in the current arena when one is open and on the heap + * otherwise, and a call whose value the group does not keep runs with the + * arena suspended. Every other allocation site stays as it is. + * + * Arenas are off in programs that construct weak collections, in programs + * of more than ARENA_FUNCTION_LIMIT functions, in libraries, and with + * lifetime regions. */ +import type { IrClassDef, IrFunction, IrModule, IrType } from "../../ir/ir.js"; +import { + analyzeClosedGroups, + analyzeRegionGroups, + source, + type RegionGroups, +} from "./region-groups.js"; +import { CYC_BUF, CYC_HDR, CYC_TYPE } from "./alloc.js"; +import { ALLOCATIONS, children, forEachNode, isExpr, type Node } from "./region-ir.js"; +import { planRegionSites, reachableFrom, type RegionSites } from "./region-sites.js"; + +/** The count an arena object starts with (scr_arena.h SCR_ARENA_RC). */ +export const ARENA_RC = 2 ** 29; + +/** Group arenas are on unless SCRIPTC_ARENAS=0 (for comparisons). */ +export function arenasRequested(): boolean { + return process.env["SCRIPTC_ARENAS"] !== "0"; +} + +export interface ArenaPlan { + groups: RegionGroups; + sites: RegionSites; + /** Record shapes and classes that get an arena-aware constructor. */ + shapes: Set; + classes: Set; +} + +export interface ArenaEligibility { + shape(shapeId: string): boolean; + cls(def: IrClassDef): boolean; + /** The reference arm of a union stored as a nullable pointer, or null. */ + nullableArm(unionId: string): IrType | null; + /** Whether values of `t` can be on a reference cycle (the collector + * traces them). */ + traced(t: IrType): boolean; +} + +/** Whether `mod` can construct a WeakMap or WeakSet, directly or through + * a stored global constructor. A weak collection's entry for a key goes + * when the key is freed, and an arena object is never freed on its own, so + * such programs use no arenas. */ +function constructsWeakCollections(mod: IrModule): boolean { + let found = false; + const visit = (node: Node): void => { + if (found) return; + if (node.kind === "libCall") { + if (node.fn === "weakMap.new" || node.fn === "weakSet.new") found = true; + else if (node.fn === "dyn.globalConstructor") { + const name = node.args[0]; + if (name?.kind !== "strLit" || name.value === "WeakMap" || name.value === "WeakSet") + found = true; + } + } + const kids: Node[] = []; + children(node, kids); + for (const k of kids) visit(k); + }; + for (const fn of mod.functions) for (const s of fn.body) visit(s); + return found; +} + +/** Programs with more functions than this use no arenas: the analysis + * visits every function, and on programs this large it costs more compile + * time than their arenas save at run time. */ +const ARENA_FUNCTION_LIMIT = 4000; + +/** The arena plan of `mod`, or null when nothing would use an arena. */ +export function planArenas(mod: IrModule, eligible: ArenaEligibility): ArenaPlan | null { + if (mod.functions.length > ARENA_FUNCTION_LIMIT || constructsWeakCollections(mod)) return null; + const groups = analyzeRegionGroups(mod); + const sites = planRegionSites(mod, groups); + if (groups.roots.size === 0 && groups.calls.size === 0) return null; + const shapes = new Set(); + const classes = new Set(); + const defs = new Map((mod.classes ?? []).map((c) => [c.name, c])); + const note = (node: Node): void => { + const kept = sites.arenaSites.has(node) || groups.roots.has(node); + if ( + kept && + node.kind === "recordLit" && + node.type.kind === "record" && + eligible.shape(node.type.shapeId) + ) + shapes.add(node.type.shapeId); + if (kept && node.kind === "new") { + const def = defs.get(node.className); + if (def !== undefined && eligible.cls(def)) classes.add(node.className); + } + const kids: Node[] = []; + children(node, kids); + for (const k of kids) note(k); + }; + for (const fn of mod.functions) + if (groups.reachable.has(fn.name)) for (const s of fn.body) note(s); + for (const r of groups.roots) if (r.kind === "new") note(r); + // A group's heap objects die when its arena drops, unless heap objects + // hold each other in a cycle: the collector frees those later, and by + // then their references into the arena point at recycled memory. So a + // group keeps its arena only if every cycle-capable object it can + // allocate takes the arena. + const takesArena = (node: Node): boolean => + (sites.arenaSites.has(node) || groups.roots.has(node)) && + ((node.kind === "recordLit" && node.type.kind === "record" && shapes.has(node.type.shapeId)) || + (node.kind === "new" && classes.has(node.className)) || + node.kind === "arrayLit"); + const unsafe = cycleUnsafeFunctions(mod, groups.reachable, sites, takesArena, eligible.traced); + for (const r of [...groups.roots, ...groups.calls]) { + const runs = reachableFrom(mod, [r]); + if ([...runs].some((f) => unsafe.has(f)) || (r.kind === "new" && !takesArena(r))) { + groups.roots.delete(r); + groups.calls.delete(r); + groups.drops.delete(r); + } + } + if (groups.roots.size === 0 && groups.calls.size === 0) return null; + groups.reachable = reachableFrom(mod, [...groups.roots, ...groups.calls]); + const arenaType = (t: IrType): boolean => { + if (t.kind === "object") return classes.has(t.className); + if (t.kind === "record") return shapes.has(t.shapeId); + if (t.kind === "union") { + const arm = eligible.nullableArm(t.unionId); + return arm !== null && arenaType(arm); + } + // Arrays of arena objects: kept array literals of them take the arena. + return t.kind === "array" && t.elem.kind !== "union" && arenaType(t.elem); + }; + const keptSite = (e: Node): boolean => sites.arenaSites.has(e) || groups.roots.has(e); + for (const r of analyzeClosedGroups(mod, groups, arenaType, keptSite)) groups.closed.add(r); + return { groups, sites, shapes, classes }; +} + +/** The functions in `reachable` that can allocate a cycle-capable object + * on the heap while an arena is open: one that does not take the arena + * (`takesArena`), or a call that runs with the arena suspended and reaches + * any cycle-capable allocation. */ +function cycleUnsafeFunctions( + mod: IrModule, + reachable: ReadonlySet, + sites: RegionSites, + takesArena: (node: Node) => boolean, + traced: (t: IrType) => boolean, +): Set { + const fns = new Map(mod.functions.map((f) => [f.name, f])); + const allocatesTraced = (node: Node): boolean => + isExpr(node) && + traced(node.type) && + (node.kind === "closure" + ? node.captures.length > 0 + : (ALLOCATIONS.has(node.kind) || source(node) || node.kind === "libCall") && + node.kind !== "call" && + node.kind !== "callValue"); + // Functions with a cycle-capable allocation, and what they call. + const direct = new Set(); + const calls = new Map>(); + const heap = new Set(); + for (const name of reachable) { + const fn = fns.get(name); + if (fn === undefined) continue; + const out = new Set(); + calls.set(name, out); + forEachNode(fn.body, (node) => { + if (allocatesTraced(node)) { + direct.add(name); + if (!takesArena(node)) heap.add(name); + } + if (node.kind === "call") out.add(node.callee); + else if (node.kind === "new") out.add(`%${node.className}.constructor`); + else if (node.kind === "closure") out.add(node.fnName); + }); + } + const tracedSomewhere = new Set(direct); + for (let changed = true; changed;) { + changed = false; + for (const [name, out] of calls) + if (!tracedSomewhere.has(name) && [...out].some((c) => tracedSomewhere.has(c))) { + tracedSomewhere.add(name); + changed = true; + } + } + for (const name of reachable) { + const fn = fns.get(name); + if (fn === undefined || heap.has(name)) continue; + forEachNode(fn.body, (node) => { + if (!sites.heapCalls.has(node)) return; + const target = + node.kind === "call" + ? node.callee + : node.kind === "new" + ? `%${node.className}.constructor` + : null; + // A suspended call of an unknown function (a closure, a virtual call) + // may allocate anything. + if (target === null || tracedSomewhere.has(target)) heap.add(name); + }); + } + return heap; +} + +/** The arena-aware constructor of a record shape or class. */ +export function arenaNewSym(newSym: string): string { + return `${newSym}.r`; +} + +/** The function that releases an arena object's fields at the drop. */ +export function arenaDisposeSym(newSym: string): string { + return `${newSym}.dispose`; +} + +/** Whether `fn` allocates through arena-aware constructors. */ +export function arenaAware(plan: ArenaPlan | null, fn: IrFunction | undefined): boolean { + return plan !== null && fn !== undefined && plan.groups.reachable.has(fn.name); +} + +/** What arenaNewLines needs from its host (a subset of ShapeHost). */ +export interface ArenaHost { + declare(decl: string): void; + threadLocalAlloc?: boolean; + inlineAlloc?: boolean; + /** The target C ABI's `size_t`: also the pointer width. */ + readonly sizeType: "i32" | "i64"; +} + +/** Instructions that set `dest` to the address of the calling thread's + * open arena pointer (scr_arena.c's scr_arena_cur), or null on targets + * without the inline allocator, which call scr_arena_current, + * scr_arena_suspend and scr_arena_resume instead. In worker programs the + * pointer is thread-local. */ +export function arenaCurrentAddress(host: ArenaHost, dest: string): string[] | null { + if (host.inlineAlloc !== true) return null; + if (host.threadLocalAlloc !== true) { + host.declare(`@scr_arena_cur = external global ptr`); + return [`${dest} = getelementptr inbounds i8, ptr @scr_arena_cur, i64 0`]; + } + host.declare(`@scr_arena_cur = external thread_local global ptr`); + host.declare(`declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull)`); + return [`${dest} = call ptr @llvm.threadlocal.address.p0(ptr @scr_arena_cur)`]; +} + +/** + * The arena-aware constructor `.r`: an object of `size` bytes (a + * `size_t` constant expression) in the current arena, behind the collector header + * `cyc` names (or none), which `init` sets up the way the heap constructor + * does (with the count ARENA_RC). When `dispose` names a function and the + * arena is not closed, the object goes on the arena's dispose list, which + * the drop runs. The heap constructor's object when no arena is open or + * the arena is full. The bump, the clear, the header and the dispose entry + * are inline (scr_arena.c's ScrArena ABI); scr_arena_obj takes a new chunk + * and scr_arena_disposal grows the list. + */ +export function arenaNewLines( + host: ArenaHost, + newSym: string, + size: string, + cyc: string | null, + dispose: string | null, + init: string[], + result: string, +): string[] { + const S = host.sizeType; + // ScrArena's fields and dispose entries are pointer-sized (scr_arena.c). + const word = S === "i32" ? 4 : 8; + host.declare(`declare ptr @scr_arena_obj(ptr, ${S}, ptr)`); + host.declare(`declare void @llvm.memset.p0.${S}(ptr, i8, ${S}, i1)`); + const cur = arenaCurrentAddress(host, "%cur"); + const current = + cur === null + ? (host.declare(`declare ptr @scr_arena_current()`), [` %a = call ptr @scr_arena_current()`]) + : [...cur.map((line) => ` ${line}`), ` %a = load ptr, ptr %cur`]; + const lines: string[] = [ + `define internal ptr @${arenaNewSym(newSym)}() nounwind alwaysinline { ; in the open arena`, + `entry:`, + ...current, + ` %on = icmp ne ptr %a, null`, + ` br i1 %on, label %arena, label %heap`, + `arena:`, + ]; + // A traced type's first object registers its descriptor on the slow path. + if (cyc !== null) + lines.push( + ` %tidp = getelementptr inbounds ${CYC_TYPE}, ptr ${cyc}, i32 0, i32 3`, + ` %tid = load atomic i32, ptr %tidp acquire, align 4`, + ` %creg = icmp ne i32 %tid, 0`, + ); + const hdr = cyc === null ? 0 : CYC_HDR; + lines.push( + ` %n0 = add ${S} ${size}, ${hdr + 7}`, + ` %n = and ${S} %n0, -8`, + ` %bp = load ptr, ptr %a`, + ` %limp = getelementptr inbounds i8, ptr %a, ${S} ${word}`, + ` %lim = load ptr, ptr %limp`, + ` %bpi = ptrtoint ptr %bp to ${S}`, + ` %limi = ptrtoint ptr %lim to ${S}`, + ` %room = sub ${S} %limi, %bpi`, + ` %fits0 = icmp ule ${S} %n, %room`, + ); + if (cyc !== null) lines.push(` %fits = and i1 %fits0, %creg`); + lines.push( + ` br i1 ${cyc === null ? "%fits0" : "%fits"}, label %carve, label %slow`, + `carve:`, + ` %end = getelementptr inbounds i8, ptr %bp, ${S} %n`, + ` store ptr %end, ptr %a`, + ` call void @llvm.memset.p0.${S}(ptr align 8 %bp, i8 0, ${S} %n, i1 false)`, + ); + if (cyc !== null) + lines.push( + ` %id = trunc i32 %tid to i16`, + ` store i16 %id, ptr %bp ; ScrCycHdr.id`, + ` %bufp = getelementptr inbounds i8, ptr %bp, i64 ${CYC_BUF}`, + ` store i32 -1, ptr %bufp ; ScrCycHdr.buf = SCR_CYC_ACYCLIC`, + ); + lines.push( + ` %of = getelementptr inbounds i8, ptr %bp, ${S} ${hdr}`, + ` br label %init`, + `slow:`, + ` %os = call ptr @scr_arena_obj(ptr %a, ${S} ${size}, ptr ${cyc ?? "null"})`, + ` %got = icmp ne ptr %os, null`, + ` br i1 %got, label %init, label %heap`, + `init:`, + ` %o = phi ptr [ %of, %carve ], [ %os, %slow ]`, + ...init, + ); + if (dispose !== null) { + host.declare(`declare void @scr_arena_disposal(ptr, ptr, ptr)`); + lines.push( + ` %closedp = getelementptr inbounds i8, ptr %a, ${S} ${4 * word} ; ScrArena.closed`, + ` %closed = load i32, ptr %closedp`, + ` %skip = icmp ne i32 %closed, 0`, + ` br i1 %skip, label %done, label %list`, + `list:`, + ` %dpp = getelementptr inbounds i8, ptr %a, ${S} ${2 * word} ; ScrArena.disposals`, + ` %dp = load ptr, ptr %dpp`, + ` %dep = getelementptr inbounds i8, ptr %a, ${S} ${3 * word} ; ScrArena.disposals_end`, + ` %de = load ptr, ptr %dep`, + ` %full = icmp eq ptr %dp, %de`, + ` br i1 %full, label %grow, label %push`, + `push:`, + ` store ptr %o, ptr %dp ; ScrArenaDisposal.obj`, + ` %dfp = getelementptr inbounds i8, ptr %dp, ${S} ${word}`, + ` store ptr @${dispose}, ptr %dfp ; ScrArenaDisposal.dispose`, + ` %dn = getelementptr inbounds i8, ptr %dp, ${S} ${2 * word}`, + ` store ptr %dn, ptr %dpp`, + ` br label %done`, + `grow:`, + ` call void @scr_arena_disposal(ptr %a, ptr %o, ptr @${dispose})`, + ` br label %done`, + `done:`, + ); + } + lines.push( + ` ret ptr ${result}`, + `heap:`, + ` %h = call ptr @${newSym}()`, + ` ret ptr %h`, + `}`, + ``, + ); + return lines; +} + +/** Dispose lines for field `i` of `%o` at `ptr` (a getelementptr into + * `%o`), released by `release`: a plain reference (no union encoding) is + * released only when it is a heap object, since an arena or immortal + * object's count (the rc word's low half) is at least 2^28 and its release + * would change nothing. */ +export function disposeFieldLines( + i: number, + gep: string, + plain: boolean, + release: string, + comment: string, +): string[] { + const load = [` %f${i} = ${gep}`, ` %v${i} = load ptr, ptr %f${i}`]; + if (!plain) return [...load, ` call void ${release}(ptr %v${i}) ; ${comment}`]; + return [ + ...load, + ` %nn${i} = icmp ne ptr %v${i}, null`, + ` br i1 %nn${i}, label %chk${i}, label %next${i}`, + `chk${i}:`, + ` %rc${i} = load i32, ptr %v${i}`, + ` %heap${i} = icmp ult i32 %rc${i}, ${2 ** 28}`, + ` br i1 %heap${i}, label %rel${i}, label %next${i}`, + `rel${i}:`, + ` call void ${release}(ptr %v${i}) ; ${comment}`, + ` br label %next${i}`, + `next${i}:`, + ]; +} + +/** Whether values of `t` are plain pointers to counted objects. */ +export function plainRefType(t: IrType): boolean { + return ( + t.kind === "object" || + t.kind === "record" || + t.kind === "array" || + t.kind === "map" || + t.kind === "set" || + t.kind === "func" || + t.kind === "string" + ); +} + +/** The dispose function: releases each reference field (`fields`, lines + * that load `%v` and release it) of the object `%o`. */ +export function arenaDisposeLines(newSym: string, fields: string[]): string[] { + return [ + `define internal void @${arenaDisposeSym(newSym)}(ptr %o) nounwind { ; arena drop`, + `entry:`, + ...fields, + ` ret void`, + `}`, + ``, + ]; +} diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts index 742d256b7..76f229215 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts @@ -222,6 +222,55 @@ test("negative factors, rebinding, captures and outer loops are not versioned", expect(match(fn([outer]), inner)).not.toBeNull(); }); +test("loops with nested loops version only for accesses inside them", () => { + // for (let i = 0; i < 64; i++) { + // let t = 0; + // for (let j = 0; j < n; j++) t = t + src[j]; + // out[i] = t; + // } + const sum = (): IrStmt[] => [ + { kind: "varDecl", localId: "t", init: num(0), loc }, + counted( + [ + { + kind: "assign", + localId: "t", + value: bin("+", ref("t"), get(ref("src", f64s), ref("j"))), + loc, + }, + ], + ref("n"), + "j", + ), + ]; + const extra = [local("t", F64, true)]; + // The guard would prove only `out[i]`, which runs once per row. + const rows = counted([...sum(), set(ref("out", f64s), ref("i"), ref("t"))], num(64)); + expect(match(fn([rows], extra), rows)).toBeNull(); + expect(match(fn([rows], extra), rows.body[1] as IrStmt & { kind: "for" })).not.toBeNull(); + // The guard proves `out[i]` inside a nested loop too. + const inside = counted( + [...sum(), counted([set(ref("out", f64s), ref("i"), ref("t"))], num(2), "j")], + num(64), + ); + expect(match(fn([inside], extra), inside)?.bounds.size).toBe(1); + // A loop whose nested loops access no typed array versions for its own + // accesses. + const scalar = counted( + [ + { kind: "varDecl", localId: "t", init: num(0), loc }, + counted( + [{ kind: "assign", localId: "t", value: bin("+", ref("t"), ref("j")), loc }], + ref("n"), + "j", + ), + set(ref("out", f64s), ref("i"), ref("t")), + ], + num(64), + ); + expect(match(fn([scalar], extra), scalar)?.bounds.size).toBe(1); +}); + test("emitted loops version the body and tag typed-array memory", () => { const loop = counted([ set(ref("out", f64s), ref("i"), bin("*", get(ref("src", f64s), ref("i")), num(2))), diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts index ccf217f9f..d0bef7fd6 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts @@ -196,9 +196,10 @@ const INVARIANT_OPS = new Set(["+", "-", "*", "&", "|", "^", "<<", ">>", ">>>"]) * and writes thread-owned views directly. * * Only small synchronous loops are versioned (nested loops included), and - * the emitter bounds how deeply versions nest. The checked loop runs - * whenever the guard fails, and keeps every out-of-bounds read, ignored - * write and NaN-index behavior. */ + * the emitter bounds how deeply versions nest. A loop whose nested loops + * access typed arrays versions only for an access inside them. The checked + * loop runs whenever the guard fails, and keeps every out-of-bounds read, + * ignored write and NaN-index behavior. */ export function matchBytesLoopVersion( loop: IrStmt & { kind: "for" | "while" }, counted: IntegerCountedForLoop | null, @@ -492,7 +493,15 @@ export function matchBytesLoopVersion( const pairedCounters = paired !== null && counted === null ? new Set([paired.low.localId, paired.high.localId]) : null; const pairedNodes = new Map(); + // Accesses inside the loop's nested loops, and inside its nested while + // loops. + const nested = new Set(); + const nestedWhile = new Set(); + let loops = 0, + whiles = 0; const access = (node: Node, array: IrExpr, index: IrExpr, width: number): void => { + if (loops > 0) nested.add(node); + if (whiles > 0) nestedWhile.add(node); const ref = root(array); if (!ref) return; if (facts.workers && ref.type.kind === "bytes" && !facts.owned.has(node)) { @@ -603,7 +612,15 @@ export function matchBytesLoopVersion( access(s, s.arr, s.index, 1); scope(s, s.arr, true); } - return everyStmtChild(s, visitExpr, visitStmt); + const isLoop = + s.kind === "for" || s.kind === "forOf" || s.kind === "while" || s.kind === "doWhile"; + const isWhile = s.kind === "while" || s.kind === "doWhile"; + if (isLoop) loops++; + if (isWhile) whiles++; + const result = everyStmtChild(s, visitExpr, visitStmt); + if (isLoop) loops--; + if (isWhile) whiles--; + return result; }; loop.body.every(visitStmt); if (loop.kind === "while") visitExpr(loop.cond); @@ -714,6 +731,18 @@ export function matchBytesLoopVersion( used.add(receiver); for (const node of nodes) bounds.add(node); } + // A loop whose nested loops access typed arrays versions only when the + // guard proves the bounds of one of those accesses (or, in a worker + // program, the ownership of one inside a nested while loop, which never + // versions for ownership alone). Otherwise the guard covers accesses that + // run once per pass, while each copy of the body repeats every nested + // loop and each nested loop's own version. + if ( + nested.size > 0 && + ![...bounds].some((node) => nested.has(node)) && + ![...ownedNodes].some((node) => nestedWhile.has(node)) + ) + return null; const counterChecks: CounterCheck[] = []; for (const [id, min] of counterMins) { const info = whileAccesses!.counters.get(id)!; diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index 5941f5d5c..695acc273 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -1,3 +1,11 @@ +import { + ARENA_RC, + arenaDisposeLines, + arenaDisposeSym, + arenaNewLines, + disposeFieldLines, + plainRefType, +} from "./arenas.js"; import { InternalCompilerError } from "../../errors.js"; /** Class layouts, reference counting, hierarchy intervals, virtual tables and * constructor values. Opaque LLVM pointers allow virtual slots to store method @@ -1089,9 +1097,15 @@ export function emitClassShapes( // A traced class whose trace visits nothing carries the collector header // only because its hierarchy is traced: it is never on a cycle. Its // descriptor names scr_cyc_no_children and its objects start out - // SCR_CYC_ACYCLIC, so releases never buffer them. + // SCR_CYC_ACYCLIC, so releases never buffer them. An Array subclass's + // trace also visits its elements by their current element ops + // (scr_arr_subclass_trace), so it is never acyclic. const acyclic = - traced && runtimePrefix === null && !isStreamRooted && tracedFields.length === 0; + traced && + runtimePrefix === null && + !isStreamRooted && + tracedFields.length === 0 && + cls.arrayElem === undefined; const bounded = runtimePrefix !== null || isStreamRooted || @@ -1415,6 +1429,8 @@ export function emitClassShapes( storedHeaderBytes(meta), ), ); + // The header and field set-up, shared with the arena-aware constructor. + const initStart = nw.length; if (meta.compact && !meta.hierarchy) nw.push(` store i32 ${compactHeaderWord(meta)}, ptr %o ; no count (compactHeaderWord)`); else if (meta.compact) { @@ -1483,6 +1499,7 @@ export function emitClassShapes( // Young instances: program-owned traced classes. Classes rooted at a // runtime base (errors, emitters, events, streams) carry runtime state // and stay with trial deletion, like every runtime family. + const init = nw.slice(initStart); if ( traced && !lifetime && @@ -1492,6 +1509,52 @@ export function emitClassShapes( nw.push(...youngEntryLines(host)); nw.push(` ret ptr ${meta.hierarchy ? "%o.vt" : "%o"}`, `}`, ``); defs.push(...nw); + if ( + host.arenaClasses?.has(cls.name) === true && + cls.arrayElem === undefined && + !lifetime && + regions === null && + !meta.compact && + runtimePrefix === null && + !isStreamRooted && + !meta.threadSlot + ) { + // The arena count in place of 1, in the rc word or its low half. + const counted = meta.idInRc + ? ` store i64 ${classIdRcWord(meta, false)}, ptr %o.vt` + : ` store ${host.sizeType} 1, ptr %o`; + const arenaCount = meta.idInRc + ? ` store i64 ${(BigInt(meta.pre + 1) * 2n ** 32n + BigInt(ARENA_RC)).toString()}, ptr %o.vt` + : ` store ${host.sizeType} ${ARENA_RC}, ptr %o`; + const newSym = mangleClassNew(cls.name); + const dispose = refFields.length > 0 ? arenaDisposeSym(newSym) : null; + defs.push( + ...arenaNewLines( + host, + newSym, + sizeOf, + traced ? cycType : null, + dispose, + init.map((line) => (line.startsWith(counted) ? arenaCount : line)), + meta.hierarchy ? "%o.vt" : "%o", + ), + ); + if (dispose !== null) + defs.push( + ...arenaDisposeLines( + newSym, + refFields.flatMap((f, t) => + disposeFieldLines( + t, + `getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${f.index}`, + plainRefType(f.type), + releaseSym(host, f.type), + llvmCommentText(f.name), + ), + ), + ), + ); + } if (traced) { // trace: visit exactly the cycle-capable fields; gcFree: release diff --git a/packages/compiler/src/backend/llvm/cpu-dispatch.ts b/packages/compiler/src/backend/llvm/cpu-dispatch.ts index f24fc8674..ced356cca 100644 --- a/packages/compiler/src/backend/llvm/cpu-dispatch.ts +++ b/packages/compiler/src/backend/llvm/cpu-dispatch.ts @@ -18,10 +18,10 @@ const COPY_GROUPS: ReadonlyMap = new Map([ ["2", "4"], ]); -export function cpuDispatchAttributes(asan: string): string[] { +export function cpuDispatchAttributes(common: string): string[] { return [ - `attributes #3 = {${asan} "target-cpu"="x86-64-v3" }`, - `attributes #4 = { noinline${asan} "target-cpu"="x86-64-v3" }`, + `attributes #3 = {${common} "target-cpu"="x86-64-v3" }`, + `attributes #4 = { noinline${common} "target-cpu"="x86-64-v3" }`, ]; } diff --git a/packages/compiler/src/backend/llvm/dense-array-access.ts b/packages/compiler/src/backend/llvm/dense-array-access.ts index e002ecd8a..b1e65a23f 100644 --- a/packages/compiler/src/backend/llvm/dense-array-access.ts +++ b/packages/compiler/src/backend/llvm/dense-array-access.ts @@ -19,6 +19,7 @@ * length, so reads never reach the state check for it; a write past the * bound tests the flag and, without states, appends at the length. */ +import { InternalCompilerError } from "../../errors.js"; import { isRefCounted, type IrExpr, type IrType } from "../../ir/ir.js"; import { f64Lit, @@ -300,6 +301,9 @@ function loadState(host: LlvmEmitterContext, guard: DenseGuard): string { return state; } +/** The ScrElemKind of double arrays (scr_array.h). */ +export const SCR_ELEM_F64 = 0; + /** The ScrElemKind of int32 arrays (scr_array.h). */ export const SCR_ELEM_I32 = 7; @@ -368,7 +372,9 @@ export function emitDenseArrayGet( /** emitDenseArrayGet, with the int32 of an int32 array's element: the * slot itself on the dense path, the runtime getter's double converted - * (exactly: the element is an int32) on the other. */ + * (exactly: the element is an int32) on the other. With `inexact`, the + * getter's double may be something other than an int32 (NaN from + * `scr_arr_get_number`): the read branches there when it is. */ export function emitDenseArrayRead( host: LlvmEmitterContext, array: string, @@ -379,6 +385,7 @@ export function emitDenseArrayRead( getter: string, retain: boolean, proven = false, + inexact?: string, ): { value: string; int32: string | null } { const B = host.B; const ty = valueType(acc); @@ -460,22 +467,68 @@ export function emitDenseArrayRead( : READ_FALLBACK; B.line(`${slowValue} = call ${ty} @${getter}(ptr ${array}, double ${index.name})${attrs}`); let slowInt: string | null = null; - if (fastInt !== null) { + let slowLabel = slow; + if (fastInt !== null && inexact !== undefined) { + // An int32 that is not -0, or the inexact branch. + host.declare("declare i32 @llvm.fptosi.sat.i32.f64(double)"); + slowInt = B.tmp(); + const back = B.tmp(), + exact = B.tmp(), + bits = B.tmp(), + notNegativeZero = B.tmp(), + ok = B.tmp(); + B.line(`${slowInt} = call i32 @llvm.fptosi.sat.i32.f64(double ${slowValue})`); + B.line(`${back} = sitofp i32 ${slowInt} to double`); + B.line(`${exact} = fcmp oeq double ${back}, ${slowValue}`); + B.line(`${bits} = bitcast double ${slowValue} to i64`); + B.line(`${notNegativeZero} = icmp ne i64 ${bits}, -9223372036854775808`); + B.line(`${ok} = and i1 ${exact}, ${notNegativeZero}`); + slowLabel = B.newLabel("arr.get.int32"); + B.condBr(ok, slowLabel, inexact); + B.startBlock(slowLabel); + } else if (fastInt !== null) { slowInt = B.tmp(); B.line(`${slowInt} = fptosi double ${slowValue} to i32`); } B.br(join); B.startBlock(join); const result = B.tmp(); - B.line(`${result} = phi ${ty} [ ${fast}, %${fastLabel} ], [ ${slowValue}, %${slow} ]`); + B.line(`${result} = phi ${ty} [ ${fast}, %${fastLabel} ], [ ${slowValue}, %${slowLabel} ]`); let int32: string | null = null; if (fastInt !== null && slowInt !== null) { int32 = B.tmp(); - B.line(`${int32} = phi i32 [ ${fastInt}, %${fastLabel} ], [ ${slowInt}, %${slow} ]`); + B.line(`${int32} = phi i32 [ ${fastInt}, %${fastLabel} ], [ ${slowInt}, %${slowLabel} ]`); } return { value: result, int32 }; } +/** emitDenseArrayNumber of an int32 array, with the element's int32: a + * read that answers NaN (or another value that is not an int32) branches + * to `inexact`. */ +export function emitDenseArrayNumberInt32( + host: LlvmEmitterContext, + array: string, + index: LlValue, + indexExpr: IrExpr | undefined, + proven: boolean, + inexact: string, +): { value: string; int32: string } { + const read = emitDenseArrayRead( + host, + array, + index, + indexExpr, + "i32", + { kind: "f64" }, + "scr_arr_get_number", + false, + proven, + inexact, + ); + if (read.int32 === null) throw new InternalCompilerError("llvm emitter bug: int32 read"); + return { value: read.value, int32: read.int32 }; +} + /** `scr_arr_get_number`: a numeric element, or NaN for every missing, * hole, or present-undefined slot (answered by the runtime). */ export function emitDenseArrayNumber( diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 417d88006..762bc52f4 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -1,5 +1,5 @@ import { emitPerKeyOrder } from "./key-orders.js"; -import { typedRefConstructor } from "./shapes.js"; +import { rcMembers, typedRefConstructor } from "./shapes.js"; import { bindLiveArrayOps } from "./expr-stream-bridges.js"; import { BYTES_ELEM_NUM, @@ -32,10 +32,9 @@ import { InternalCompilerError } from "../../errors.js"; * are selected from the target C ABI. * v.num double | v.b i8 | v.str/v.bytes ptr at +16; * v.arr { len +16, cap +24, items +32 }; - * v.obj { len +16, cap +24, entries +32 }; + * v.obj { len +16 } (own properties; read them through + * scr_dyn_member_*, which know both object modes); * v.fn { clo +16, thunk +24, sig +32, name +40, arity +48 }. - * ScrDynEntry { key, key_len, value, getter, setter, accessor, writable, - * enumerable, configurable } — 48 bytes on 64-bit, 24 on 32-bit. * ScrDynKind: NULL=0 BOOL=1 NUM=2 STR=3 ARR=4 OBJ=5 UNDEF=6 BYTES=7 * FUNC=8 HANDLE=9. * ScrBytes { rc +0; len +8; elem +16; data +24 }. @@ -46,6 +45,7 @@ import { DYN_HANDLE_KINDS, isDirentShape, KEY_ORDER_FIELD, + shapeHasAccessorSlots, shapeKeyOrders, isDynTypedRefType, isRefCounted, @@ -261,6 +261,8 @@ export class LlDyn { /** Emitted function definitions, in interning order. */ readonly defs: string[] = []; + private readonly prefixSources = new Map(); + constructor(private readonly host: DynHost) {} /** Journal replay (emit-journal.ts): builds or finds the helper a chunk @@ -420,7 +422,7 @@ export class LlDyn { * dyn view: inside a visit session (`wrapped`), a capsule keeps the view * it was first given, so an outer visit can keep reading it. */ private viewFn(wrapped: boolean): string { - const fn = wrapped ? "scr_dyn_visit_view" : "scr_dyn_typed_ref_materialize"; + const fn = wrapped ? "scr_dyn_visit_view" : "scr_dyn_typed_ref_peek"; this.host.declare(`declare ptr @${fn}(ptr)`); return fn; } @@ -849,27 +851,23 @@ export class LlDyn { return e; } - /** Object entry field addresses: entries + i*sizeof(ScrDynEntry), with - * key, key_len, and value at offsets 0, 8, and 16 on 64-bit targets. */ + /** Own property i of object `d` (borrowed key bytes, key length and + * value), read through the runtime's accessors: the object may be in + * layout or dictionary mode (scr_dyn.h). */ private entryAt( B: BlockBuilder, - entries: string, + d: string, i: string, ): { key: string; keyLen: string; value: string } { - const off = B.tmp(); - const base = B.tmp(); - B.line(`${off} = mul ${this.S} ${i}, ${this.abiOffset(48, 24)} ; sizeof(ScrDynEntry)`); - B.line(`${base} = getelementptr inbounds i8, ptr ${entries}, ${this.S} ${off}`); + this.host.declare(`declare ptr @scr_dyn_member_key(ptr, ${this.S})`); + this.host.declare(`declare ${this.S} @scr_dyn_member_key_len(ptr, ${this.S})`); + this.host.declare(`declare ptr @scr_dyn_member_value(ptr, ${this.S})`); const key = B.tmp(); - B.line(`${key} = load ptr, ptr ${base}`); - const klp = B.tmp(); const keyLen = B.tmp(); - B.line(`${klp} = getelementptr inbounds i8, ptr ${base}, i64 ${this.abiOffset(8, 4)}`); - B.line(`${keyLen} = load ${this.S}, ptr ${klp}`); - const vp = B.tmp(); const value = B.tmp(); - B.line(`${vp} = getelementptr inbounds i8, ptr ${base}, i64 ${this.abiOffset(16, 8)}`); - B.line(`${value} = load ptr, ptr ${vp}`); + B.line(`${key} = call ptr @scr_dyn_member_key(ptr ${d}, ${this.S} ${i})`); + B.line(`${keyLen} = call ${this.S} @scr_dyn_member_key_len(ptr ${d}, ${this.S} ${i})`); + B.line(`${value} = call ptr @scr_dyn_member_value(ptr ${d}, ${this.S} ${i})`); return { key, keyLen, value }; } @@ -1155,10 +1153,63 @@ export class LlDyn { * Class layouts share their base prefix, so unboxing retains the same * pointer and virtual dispatch still uses the object's original vtable. */ private typedRefMatches(B: BlockBuilder, t: IrType): string { + return this.typedRefMatchesAny( + B, + isDynTypedRefType(t) + ? this.host.classSubtypes(t.className).map((className) => ({ kind: "object", className })) + : [t], + ); + } + + /** The record shapes whose values a pointer of shape `targetId` may + * reference unchanged: their struct begins with exactly the target's + * fields (same names, types and representations, so the same offsets), + * and everything the target's release, teardown and collector support + * would get wrong on them is absent. The extra fields hold no + * references (the target's release frees the block without visiting + * them; frees are size-class agnostic), neither shape carries a shape + * word, a cycle header, an overflow map, key order variants or accessor + * slots, and both freeze alike. A record that keeps undeclared source + * properties is refused at run time (scr_dyn_typed_ref_prefix_view). */ + private prefixViewSources(targetId: string): string[] { + const memo = this.prefixSources.get(targetId); + if (memo) return memo; + const host = this.host; + const plain = (shape: IrRecordShape): boolean => + !shape.tuple && + !shape.indexValue && + !isDirentShape(shape) && + !shapeHasAccessorSlots(shape) && + shapeKeyOrders(shape).length === 1 && + !shape.fields.some((field) => field.name === KEY_ORDER_FIELD) && + host.nullableUnions.shapeNumber(shape.id) === 0 && + !host.tracedShapes.has(`record:${shape.id}`); + const target = host.recordsById.get(targetId); + const out: string[] = []; + if (target && plain(target) && target.fields.length > 0) { + const base = host.nullableUnions.fieldBase(targetId); + for (const shape of host.recordsById.values()) { + if (shape.id === targetId || shape.fields.length <= target.fields.length) continue; + if (!plain(shape) || shape.freezable !== target.freezable) continue; + if (host.nullableUnions.fieldBase(shape.id) !== base) continue; + const prefix = target.fields.every( + (field, i) => + shape.fields[i]!.name === field.name && + typeKey(shape.fields[i]!.type) === typeKey(field.type), + ); + if (!prefix) continue; + const extra = rcMembers(shape, host.inlineUnions, base).some( + (member) => member.index >= target.fields.length + base && isRefCounted(member.type), + ); + if (!extra) out.push(shape.id); + } + } + this.prefixSources.set(targetId, out); + return out; + } + + private typedRefMatchesAny(B: BlockBuilder, types: readonly IrType[]): string { this.host.declare(`declare zeroext i1 @scr_dyn_typed_ref_is(ptr, ptr, ${this.S})`); - const types: IrType[] = isDynTypedRefType(t) - ? this.host.classSubtypes(t.className).map((className) => ({ kind: "object", className })) - : [t]; let matched = "false"; for (const type of types) { const key = typeKey(type); @@ -1441,9 +1492,8 @@ export class LlDyn { if (shape.indexValue && shape.indexValue.kind !== "dyn") { this.host.declare(`declare i32 @memcmp(ptr, ptr, ${this.S})`); const n = this.lenOf(B, "%d"); - const entries = this.itemsOf(B, "%d"); this.i64Loop(B, "dm.ov", n, (i, brNext) => { - const ent = this.entryAt(B, entries, i); + const ent = this.entryAt(B, "%d", i); // Skip declared keys. for (const f of shape.fields) { const klen = Buffer.byteLength(f.name, "utf8"); @@ -2172,6 +2222,32 @@ export class LlDyn { B.line(`${ref} = call ptr @scr_dyn_typed_ref_unbox(ptr %d)`); B.terminate(`ret ptr ${ref}`); B.startBlock(lNext); + if (t.kind === "record") { + // A record whose storage begins with exactly this type's storage + // is this type's value too: the conversion passes the pointer, so + // both names keep one object (decision D3). + const sources = this.prefixViewSources(t.shapeId); + if (sources.length > 0) { + const viewed = this.typedRefMatchesAny( + B, + sources.map((shapeId) => ({ kind: "record", shapeId })), + ); + const lView = B.newLabel("dc.prefix"); + const lOther = B.newLabel("dc.noprefix"); + B.condBr(viewed, lView, lOther); + B.startBlock(lView); + host.declare(`declare ptr @scr_dyn_typed_ref_prefix_view(ptr)`); + const view = B.tmp(); + B.line(`${view} = call ptr @scr_dyn_typed_ref_prefix_view(ptr %d)`); + const passed = B.tmp(); + B.line(`${passed} = icmp ne ptr ${view}, null`); + const lPass = B.newLabel("dc.prefix.pass"); + B.condBr(passed, lPass, lOther); + B.startBlock(lPass); + B.terminate(`ret ptr ${view}`); + B.startBlock(lOther); + } + } // A constructor-function stream (scr_dyn_builtin_stream.c) carries // its runtime stream. const streamKind = t.kind === "object" ? builtinStreamKind(t.className) : null; @@ -2777,9 +2853,8 @@ export class LlDyn { B.line(`store i8 ${np}, ptr ${flagsp}`); } const n = this.lenOf(B, "%d"); - const entries = this.itemsOf(B, "%d"); this.i64Loop(B, "dcv", n, (i, brNext) => { - const ent = this.entryAt(B, entries, i); + const ent = this.entryAt(B, "%d", i); for (const f of shape.fields) { const klen = Buffer.byteLength(f.name, "utf8"); const lenEq = B.tmp(); diff --git a/packages/compiler/src/backend/llvm/emit-function.ts b/packages/compiler/src/backend/llvm/emit-function.ts index 24e1d06a3..3be3d979f 100644 --- a/packages/compiler/src/backend/llvm/emit-function.ts +++ b/packages/compiler/src/backend/llvm/emit-function.ts @@ -338,6 +338,7 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { } emitter.currentCoro = { kind: fn.generator !== undefined ? "generator" : "async", + awaits: fn.async === true, id, handle, self, @@ -525,6 +526,7 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { } } emitter.scopes.push(fnScope); + emitter.beginArenaFunction(fn); // Stackful native fibers and native coroutine frames retain these frames // while suspended. The active exception context owns the chain, so // concurrent fibers isolate it. @@ -862,10 +864,14 @@ export function emitGenScaffolding(emitter: LlEmitter): string[] { ]; const genNew = coroBody && !emitter.wasi ? "scr_gen_new_stackless" : "scr_gen_new"; if (genNew !== "scr_gen_new") emitter.declare(`declare ptr @${genNew}(ptr, ptr, ptr)`); + const asyncGenNew = + coroBody && !emitter.wasi ? "scr_async_gen_new_stackless" : "scr_async_gen_new"; + if (settleAsync !== null && asyncGenNew !== "scr_async_gen_new") + emitter.declare(`declare ptr @${asyncGenNew}(ptr, ptr, ptr, ptr)`); sp.push( settleAsync === null ? ` %gg = call ptr @${genNew}(ptr @${mangleTrampoline(fn.name)}, ptr %ap, ptr @${mangleGenDrop(fn.name)})` - : ` %gg = call ptr @scr_async_gen_new(ptr @${mangleTrampoline(fn.name)}, ptr %ap, ptr @${mangleGenDrop(fn.name)}, ptr @${settleAsync})`, + : ` %gg = call ptr @${asyncGenNew}(ptr @${mangleTrampoline(fn.name)}, ptr %ap, ptr @${mangleGenDrop(fn.name)}, ptr @${settleAsync})`, ...(fn.generator.tag !== undefined ? [` call void @scr_gen_set_tag(ptr %gg, ptr ${emitter.cstr(fn.generator.tag)})`] : []), diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index 368d10ad3..be01f4d92 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -201,19 +201,22 @@ function readRange(dir: string, range: number): { result: ChunkResult; texts: st /** The attribute groups of the emitted functions: #0 for every function, * #1 for coroutine bodies and #2 for out-of-line record clones. Each carries - * the build's `--target-cpu` level, and `sanitize_address` in sanitized - * builds. A group needs one attribute: without either, #0 holds a string - * attribute that LLVM ignores. Dispatching builds add the x86-64-v3 copies' - * #3 and #4 (cpu-dispatch.ts). */ + * the build's `--target-cpu` level, `sanitize_address` in sanitized builds, + * and `uwtable` on native targets, so every function has an unwind table + * entry, including those LLVM infers `nounwind` (profilers and debuggers + * unwind through them). A group needs one attribute: without any, #0 holds a + * string attribute that LLVM ignores. Dispatching builds add the x86-64-v3 + * copies' #3 and #4 (cpu-dispatch.ts). */ function functionAttributes(emitter: LlEmitter, hasNoInlineRecordClone: boolean): string[] { const cpu = targetCpuAttribute(emitter.targetCpu); const asan = emitter.sanitizeAttributes ? " sanitize_address" : ""; - const base = `${asan}${cpu}` || ` "scriptc-function"`; + const common = `${asan}${emitter.wasi ? "" : " uwtable"}`; + const base = `${common}${cpu}` || ` "scriptc-function"`; return [ `attributes #0 = {${base} }`, - ...(emitter.usesCoroutines ? [`attributes #1 = {${asan} presplitcoroutine${cpu} }`] : []), - ...(hasNoInlineRecordClone ? [`attributes #2 = { noinline${asan}${cpu} }`] : []), - ...(emitter.cpuDispatch ? cpuDispatchAttributes(asan) : []), + ...(emitter.usesCoroutines ? [`attributes #1 = {${common} presplitcoroutine${cpu} }`] : []), + ...(hasNoInlineRecordClone ? [`attributes #2 = { noinline${common}${cpu} }`] : []), + ...(emitter.cpuDispatch ? cpuDispatchAttributes(common) : []), ]; } diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 45d341853..1eeceff4a 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -96,7 +96,7 @@ import { import { emitStringInputs } from "./string-lifetimes.js"; import { LOOPS } from "./loop-state.js"; import { loopUnrollAttachment, periodicGuardPeriod } from "./periodic-guards.js"; -import { loopAccessesTypedArrays } from "../loop-polls.js"; +import { containsLoop, loopAccessesTypedArrays } from "../loop-polls.js"; import { emitConstantChoice, matchConstantChoice } from "./constant-choices.js"; /** A callback loop version copies the loop; keep the copies small. */ @@ -105,8 +105,11 @@ const MAX_CALLBACK_VERSION_COST = 400; export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { const B = emitter.B; emitter.frames.push([]); + if (emitter.arenas !== null) + emitter.statementFrames.add(emitter.frames[emitter.frames.length - 1]!); switch (s.kind) { case "varDecl": { + emitter.ownArenasOf(s.localId); if (s.init && emitter.int32Locals.has(s.localId)) { emitter.storeInt32Local(s.localId, emitter.emitExpr(s.init), s.init); break; @@ -1228,8 +1231,8 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { }; // A counted loop over typed arrays, or a loop to a typed array's // length, polls between blocks of WORKER_POLL_BLOCK iterations instead - // of counting a budget in its body: the inner loop over a block then - // has no exit besides its own bound, which LLVM can vectorize. A typed + // of in its body: the inner loop over a block then has no exit + // besides its own bound, which LLVM can vectorize. A typed // array's length is fixed, so each block can take it as its limit when // the loop does not assign the array's binding. // @@ -1670,7 +1673,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { B.line(`call void @scr_async_gen_return_awaits()`); } if (emitter.currentAsyncGenerator && s.awaited !== true && emitter.finallyStack.length > 0) { - if (emitter.wasi) emitter.emitWasiSuspend(null); + if (emitter.coroAwait) emitter.emitCoroAwait(null); else { emitter.declare(`declare void @scr_await_hop()`); B.line(`call void @scr_await_hop()`); @@ -2760,8 +2763,8 @@ interface StripMinedCounter { * iterations, with the cancellation poll between blocks: the outer header * tests the counter against the loop's limit, each block runs the body up * to the nearer of the limit and WORKER_POLL_BLOCK steps on, and the poll - * follows the block. It polls as often as the budget it replaces, and the - * inner loop's only exit besides `break` and `return` is its bound. + * follows the block. It polls once per block, and the inner loop's only + * exit besides `break` and `return` is its bound. * * With `runSlot` (for a loop that another loop of the function encloses), a * block that ends at the limit leaves the loop without polling, and a run @@ -2895,21 +2898,6 @@ function emitStripMinedCountedLoop( B.br(lc); } -/** Whether `body` holds a loop outside its closures. */ -function containsLoop(body: IrStmt[]): boolean { - const inStmt = (s: IrStmt): boolean => - s.kind === "while" || - s.kind === "doWhile" || - s.kind === "for" || - s.kind === "forOf" || - !everyStmtChild( - s, - () => true, - (child) => !inStmt(child), - ); - return body.some(inStmt); -} - function enterLoopHeader(emitter: LlEmitter, header: string): void { const dispatch = emitter.speculationHeader; emitter.speculationHeader = null; diff --git a/packages/compiler/src/backend/llvm/emitter.test.ts b/packages/compiler/src/backend/llvm/emitter.test.ts index 880685bf2..8e3aae24d 100644 --- a/packages/compiler/src/backend/llvm/emitter.test.ts +++ b/packages/compiler/src/backend/llvm/emitter.test.ts @@ -206,7 +206,7 @@ test("a target CPU level reaches every function attribute group and nothing else const plain = emitLlvmModule(module); expect(plain).not.toContain("target-cpu"); const v3 = emitLlvmModule(module, { targetCpu: "x86-64-v3" }); - expect(v3).toMatch(/^attributes #0 = \{ sanitize_address "target-cpu"="x86-64-v3" \}$/m); + expect(v3).toMatch(/^attributes #0 = \{ sanitize_address uwtable "target-cpu"="x86-64-v3" \}$/m); expect(v3.replaceAll(' "target-cpu"="x86-64-v3"', "")).toBe(plain); expect(emitLlvmModule(module, { targetCpu: "baseline" })).toBe(plain); }); @@ -215,13 +215,21 @@ test("only sanitized builds mark the emitted functions sanitize_address", () => const module = moduleFor([call()]); const sanitized = emitLlvmModule(module); expect(emitLlvmModule(module, { sanitizeAttributes: true })).toBe(sanitized); - expect(sanitized).toMatch(/^attributes #0 = \{ sanitize_address \}$/m); + expect(sanitized).toMatch(/^attributes #0 = \{ sanitize_address uwtable \}$/m); const plain = emitLlvmModule(module, { sanitizeAttributes: false }); expect(plain).not.toContain("sanitize_address"); - // A group needs an attribute: an ignored string attribute stands in. - expect(plain).toMatch(/^attributes #0 = \{ "scriptc-function" \}$/m); + expect(plain).toMatch(/^attributes #0 = \{ uwtable \}$/m); const v3 = emitLlvmModule(module, { sanitizeAttributes: false, targetCpu: "x86-64-v3" }); - expect(v3).toMatch(/^attributes #0 = \{ "target-cpu"="x86-64-v3" \}$/m); + expect(v3).toMatch(/^attributes #0 = \{ uwtable "target-cpu"="x86-64-v3" \}$/m); +}); + +test("native functions get unwind tables and wasm32-wasi functions do not", () => { + const module = moduleFor([call()]); + expect(emitLlvmModule(module)).toMatch(/^attributes #0 = \{ sanitize_address uwtable \}$/m); + const wasi = emitLlvmModule(module, { wasi: true, pointerBits: 32, sanitizeAttributes: false }); + expect(wasi).not.toContain("uwtable"); + // A group needs an attribute: an ignored string attribute stands in. + expect(wasi).toMatch(/^attributes #0 = \{ "scriptc-function" \}$/m); }); test("inline RC inlines every fast path at speed and keeps box releases as calls in release", () => { diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 927ce1d36..031dc27ca 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -27,7 +27,7 @@ import { streamTypedRefEligible, undefinedArmTag, } from "../../ir/analysis.js"; -import { loopTestsEveryIteration } from "../loop-polls.js"; +import { containsLoop, loopTestsEveryIteration } from "../loop-polls.js"; import type { IrBytesElem, IrExpr, @@ -343,6 +343,15 @@ import { type LlClassMeta, } from "./classes.js"; import { LifetimeRegions, REGION_TYPES } from "./regions.js"; +import { + arenaAware, + arenaCurrentAddress, + arenaNewSym, + arenasRequested, + planArenas, + type ArenaPlan, +} from "./arenas.js"; +import { children, type Node } from "./region-ir.js"; import { LlDyn, recordDynKeepsIdentity, type DynHost } from "./dyn.js"; import { VirtualBorrows } from "./virtual-borrows.js"; import { LlvmUnsupportedError } from "./unsupported.js"; @@ -375,6 +384,7 @@ import { retainSym, traceArg, vAdapters, + traceAdapter, } from "./shapes.js"; import type { ExprOf, LibCallExpr, LlStreamTypedRefAdapter, LlValue } from "./expr-context.js"; import { INLINE_LITERAL_BYTES } from "./string-literal-equality.js"; @@ -421,6 +431,7 @@ import { fieldAliasMetadata, isStackUnionSource, } from "./receivers.js"; +import { findLoopCallees } from "./loop-callees.js"; export { LlvmUnsupportedError } from "./unsupported.js"; @@ -465,6 +476,9 @@ export interface LlScopeEntry { slot: string; type: IrType; boxed?: boolean; + /** A group arena list slot (arenas.ts), dropped after the scope's other + * entries. */ + arena?: boolean; /** A lazily created box: the slot stays null on paths that build no * environment, which skip the release call. */ lazy?: boolean; @@ -829,6 +843,11 @@ const NO_BORROWED: ReadonlySet = new Set(); /** The nodes that write a binding by its id (numberGlobalWriters). */ const GLOBAL_WRITE_KINDS: readonly IrNodeKind[] = ["assign", "varDecl", "assignExpr", "incDec"]; + +/** workerLoopBudget's answer for a loop that polls on every iteration + * instead of counting down in a slot. */ +const POLL_EVERY_ITERATION = "%worker.poll.every"; + /** Runtime library calls (readsLoopMetrics). */ const LIB_CALL_KINDS: readonly IrNodeKind[] = ["libCall"]; @@ -843,16 +862,23 @@ function isIdentifierByte(code: number): boolean { } /** Runtime entry points (and await-call entries) that may park the running - * execution context, as they appear in emitted function text. */ + * execution context, as they appear in emitted function text. An async + * generator's next, return and throw only queue a request and run its body + * to its next suspension, which returns to the caller, and the body's own + * bookkeeping calls do not suspend it. */ export const SUSPENDS = - /@(?:scr_(?:await_|module_await\b|promise_await_settled\b|test_sub\b|gen_yield_|async_gen_|fiber_)|sc_a[wd]_)/; + /@(?:scr_(?:await_|module_await\b|promise_await_settled\b|test_sub\b|gen_yield_|async_gen_(?!new\b|next_|return_|throw\b|hop_done\b)|fiber_)|sc_a[wd]_)/; /** The suspensions of a fiber body's own awaits, which its stackless * coroutine form suspends itself at instead. */ -const AWAIT_SUSPENSIONS = /@(?:scr_await_(?:void|f64|bool|str|ref|hop)\b|sc_a[wd]_[A-Za-z0-9_]+)/g; +const AWAIT_SUSPENSIONS = + /@(?:scr_await_(?:void|f64|bool|str|ref|hop|dyn_value)\b|sc_a[wd]_[A-Za-z0-9_]+)/g; +/** The suspensions of a fiber generator body's own yields, which its + * coroutine form also suspends itself at. */ +const YIELD_SUSPENSIONS = /@scr_(?:gen_yield_(?:f64|bool|ref))\b/g; /** SUSPENDS in a stackless coroutine body, whose frame bookkeeping reads its * own fiber and promise. */ const COROUTINE_SUSPENDS = - /@(?:scr_(?:await_|module_await\b|promise_await_settled\b|test_sub\b|gen_yield_|async_gen_|fiber_(?!self\b|promise\b))|sc_a[wd]_)/; + /@(?:scr_(?:await_|module_await\b|promise_await_settled\b|test_sub\b|gen_yield_|async_gen_(?!new\b|next_|return_|throw\b|hop_done\b)|fiber_(?!self\b|promise\b))|sc_a[wd]_)/; /** Captured bindings (boxes a body declares or receives) from which a * function compiles without LLVM optimization (see boxSlotCount). */ export const UNOPTIMIZED_BOX_SLOTS = 512; @@ -1274,11 +1300,13 @@ export class LlEmitter { * generator stacks) and so do debug builds (the debugger reads locals * from the native frame). */ private stacklessGenerators = false; - /** Async bodies that suspend only at their own awaits compile to the same - * switched coroutines on native targets (selectStacklessAsync), under the - * generators' conditions; library and Windows builds keep fibers. */ + /** Async bodies that suspend only at their own awaits (and an async + * generator's yields) compile to the same switched coroutines on native + * targets (selectStacklessAsync), under the generators' conditions; + * library and Windows builds keep fibers. */ private stacklessAsyncAllowed = false; - /** The async functions selectStacklessAsync lowered as coroutines. */ + /** The async functions and async generators selectStacklessAsync lowered + * as coroutines. */ readonly stacklessAsync = new Set(); /** Some function in the module is lowered as a coroutine. */ usesCoroutines = false; @@ -1488,6 +1516,9 @@ export class LlEmitter { /** Functions that check the native stack guard on entry * (stackCheckedFunctions); null where the runtime has no guard. */ readonly stackChecks: ReadonlySet | null; + /** Small leaf functions called only from loops, whose direct calls are + * `alwaysinline` (loop-callees.ts). */ + readonly loopCallees: ReadonlySet; /** Static types whose values can be published (@scriptc/threads): their * stores carry the frozen-object guard. Null without publish calls. */ readonly publishedTypes: PublishedTypes | null; @@ -1674,6 +1705,8 @@ export class LlEmitter { * commit back into their objects. Library builds keep their commits: * their values cross a native boundary. */ readonly liveViewsReadOnly: boolean; + /** IrModule.sloppyDynWrites: untyped writes in sloppy-mode code. */ + readonly sloppyDynWrites: ReadonlySet; /** IrModule.foreignElementKeys: element keys whose arrays can hold * SCR_ARR_FOREIGN elements. */ readonly foreignElementKeys: ReadonlySet; @@ -1902,9 +1935,12 @@ export class LlEmitter { currentUnionResult = false; /** Active only while emitting a body lowered with LLVM's * switched-coroutine intrinsics (wasm32 async bodies and generators, - * native synchronous generators; see coroLowered). */ + * native synchronous generators, and the native async bodies + * selectStacklessAsync chose; see coroLowered). */ currentCoro: { kind: "async" | "generator"; + /** The body awaits: an async function or an async generator. */ + awaits: boolean; id: string; handle: string; self: string; @@ -1982,6 +2018,7 @@ export class LlEmitter { ); this.constantNumericTables = findConstantNumericTables(mod); this.untypedIntegrity = mod.untypedIntegrity === true; + this.sloppyDynWrites = new Set(mod.sloppyDynWrites ?? []); this.liveViewsReadOnly = mod.lib === undefined && !this.untypedIntegrity && @@ -2076,6 +2113,7 @@ export class LlEmitter { : new Set(); this.workerEntryPolls = mt.workerEntryPolls ?? null; this.stackChecks = mt.stackChecks ?? null; + this.loopCallees = findLoopCallees(mod); // A stackless call's eager prefix borrows its caller's stack; the // runtime moves deep prefix nests to a fresh stack using the guard, // which Windows lacks. @@ -2325,6 +2363,39 @@ export class LlEmitter { programLifetime: this.programLifetime, regionAliasTag: (word) => this.memoryAliasAttachment(`region.${word}`), }; + // An instance of a class that extends Array is an array: a slot of its + // element array type can hold it, and the collector visits it there as + // an array, which has a collector header when its element ops trace + // (scr_arr_make_ops_kind). Such a class carries a header too, and so + // does the rest of its hierarchy. A class whose elements are instances + // of another such class follows that class, so repeat until no class + // adds one. + const arrayClasses = (mod.classes ?? []).filter((c) => c.arrayElem !== undefined); + if (arrayClasses.length > 0) { + const baseOf = new Map((mod.classes ?? []).map((c) => [c.name, c.base] as const)); + const rootOf = (name: string): string => { + let root = name; + for ( + let base = baseOf.get(root); + base !== undefined && baseOf.has(base); + base = baseOf.get(root) + ) + root = base; + return root; + }; + for (let changed = true; changed;) { + changed = false; + for (const cls of arrayClasses) { + if (this.tracedShapes.has(`object:${cls.name}`)) continue; + if (traceAdapter(this.shapeHost, { kind: "array", elem: cls.arrayElem! }) === null) + continue; + const root = rootOf(cls.name); + for (const member of mod.classes ?? []) + if (rootOf(member.name) === root) this.tracedShapes.add(`object:${member.name}`); + changed = true; + } + } + } this.walkers = new LlWalkers(this.shapeHost); this.dyn = new LlDyn(this.shapeHost); for (const g of mod.globals ?? []) { @@ -2412,6 +2483,40 @@ export class LlEmitter { mod.lib === undefined && options.lowRegion === true && [...this.classMeta.values()].some((m) => m.compact); + // Group arenas: off with lifetime regions, which own their classes' + // allocation, and in libraries, whose exports any host code calls. + if (arenasRequested() && mod.lifetimeRegions !== true && mod.lib === undefined) { + this.arenas = planArenas(mod, { + shape: (id) => { + const shape = this.recordsById.get(id); + return ( + shape !== undefined && + shape.freezable !== true && + shape.indexValue === undefined && + this.publishedTypes?.recordGuarded(id) !== true + ); + }, + cls: (def) => { + const meta = this.classMeta.get(def.name); + return ( + meta !== undefined && + !meta.compact && + !meta.threadSlot && + // An Array subclass's instance is an array (the runtime's header). + def.arrayElem === undefined && + meta.root.def.arrayElem === undefined && + !this.programLifetime.has(def.name) && + this.publishedTypes?.classGuarded(def.name) !== true + ); + }, + nullableArm: (unionId) => this.nullableUnions.single(unionId)?.arm ?? null, + traced: (t) => traceAdapter(this.shapeHost, t) !== null, + }); + if (this.arenas !== null) { + this.shapeHost.arenaShapes = this.arenas.shapes; + this.shapeHost.arenaClasses = this.arenas.classes; + } + } for (const [name, rec] of RUNTIME_ERROR_CLASSES) { const meta = this.classMeta.get(name); if (!meta) break; // hand-written IR without the builtin defs: no stamps @@ -2713,19 +2818,21 @@ export class LlEmitter { } } - /** Chooses the native async functions whose bodies compile to switched - * coroutines instead of running on a fiber: a suspended call then keeps a - * heap frame of the values live across its awaits rather than a fiber and - * its stack. A body qualifies when its only suspension points are its own - * awaits, which the coroutine form suspends itself at (emitCoroAwait), so - * nothing it runs can park the stack it borrows. It is first emitted as - * usual: that text may suspend only through await runtime calls and - * await-call entries, which the coroutine form replaces, and may call no - * function that parks. The coroutine form is then emitted and must pass - * the same test with no exceptions (its frame bookkeeping excepted); - * otherwise the fiber form stays. Bodies that never suspend stay - * fiberless, and module evaluation bodies and `dyn` results keep fibers - * as they do there. The runtime traps if a stackless body ever parks. */ + /** Chooses the native async functions and async generators whose bodies + * compile to switched coroutines instead of running on a fiber: a + * suspended call then keeps a heap frame of the values live across its + * awaits and yields rather than a fiber and its stack. A body qualifies + * when its only suspension points are its own awaits and yields, which + * the coroutine form suspends itself at (emitCoroAwait, + * emitCoroSuspendPrepared), so nothing it runs can park the stack it + * borrows. It is first emitted as usual: that text may suspend only + * through await and yield runtime calls and await-call entries, which the + * coroutine form replaces, and may call no function that parks. The + * coroutine form is then emitted and must pass the same test with no + * exceptions (its frame bookkeeping excepted); otherwise the fiber form + * stays. Bodies that never suspend stay fiberless, and module evaluation + * bodies and async functions with `dyn` results keep fibers as they do + * there. The runtime traps if a stackless body ever parks. */ selectStacklessAsync(fnDefs: string[]): void { if (!this.stacklessAsyncAllowed) return; const parks = this.parkingFunctions(); @@ -2740,11 +2847,16 @@ export class LlEmitter { return true; }; this.mod.functions.forEach((fn, index) => { - if (fn.async !== true || fn.generator !== undefined) return; + if (fn.async !== true) return; if (fn.asyncCacheGlobal !== undefined || fn.asyncCycleCacheGlobal !== undefined) return; - if (fn.returnType.kind === "dyn" || !parks.has(fn.name)) return; - const fiberText = fnDefs[index]!; - if (!qualifies(fn, fiberText.replace(AWAIT_SUSPENSIONS, ""), SUSPENDS)) return; + // A dyn result may adopt a thenable when it settles the promise; an + // async generator's completion value settles its request's record. + if (fn.generator === undefined && fn.returnType.kind === "dyn") return; + if (!parks.has(fn.name)) return; + // An async generator's body also suspends at its own yields. + let fiberText = fnDefs[index]!.replace(AWAIT_SUSPENSIONS, ""); + if (fn.generator !== undefined) fiberText = fiberText.replace(YIELD_SUSPENSIONS, ""); + if (!qualifies(fn, fiberText, SUSPENDS)) return; this.stacklessAsync.add(fn.name); const coroText = this.emitFunction(fn); if (!qualifies(fn, coroText, COROUTINE_SUSPENDS)) { @@ -3157,7 +3269,7 @@ export class LlEmitter { * native async bodies selectStacklessAsync chose. */ coroLowered(fn: IrFunction): boolean { if (this.wasi) return fn.async === true || fn.generator !== undefined; - if (fn.async === true) return fn.generator === undefined && this.stacklessAsync.has(fn.name); + if (fn.async === true) return this.stacklessAsync.has(fn.name); return this.stacklessGenerators && fn.generator !== undefined; } emitGenScaffolding(): string[] { @@ -3527,6 +3639,7 @@ export class LlEmitter { /** Releases a statement frame's temps and owned slots. */ releaseTemps(frame: LlValue[]): void { for (const v of frame) { + if (v.arena !== undefined) continue; if (v.slot) { const t = this.B.tmp(); this.B.line(`${t} = load ptr, ptr ${v.name}`); @@ -3539,10 +3652,19 @@ export class LlEmitter { this.releaseFrame(v.name, v.type); } } + // Arena lists last: the frame's temps may be arena objects. + for (const v of frame) + if (v.arena === "drop") this.B.line(`call void @scr_arena_drop(ptr ${v.name})`); + else if (v.arena === "close") { + const head = this.B.tmp(); + this.B.line(`${head} = load ptr, ptr ${v.name}`); + this.B.line(`call void @scr_arena_close(ptr ${head})`); + } } releaseScope(scope: LlScopeEntry[]): void { for (const e of scope) { + if (e.arena === true) continue; const t = this.B.tmp(); this.B.line(`${t} = load ptr, ptr ${e.slot}`); if (e.boxed && e.lazy) { @@ -3564,6 +3686,9 @@ export class LlEmitter { this.releaseFrame(t, e.type); // runtime releases are NULL-tolerant } } + // Arena lists last: the scope's bindings may hold arena objects. + for (const e of scope) + if (e.arena === true) this.B.line(`call void @scr_arena_drop(ptr ${e.slot})`); } /** THE release-on-jump path (break/continue/return): pending statement @@ -3888,6 +4013,15 @@ export class LlEmitter { * calls (unwindAbiFunctions excludes their callees). */ emitDirectCall(line: string, callee: string): boolean { const B = this.B; + // A small leaf called only from loops inlines into each of them, past + // LLVM's cost threshold (loop-callees.ts). Debug builds keep the call. + if (this.debug === null && this.loopCallees.has(callee)) { + const comment = line.indexOf(" ; "); + line = + comment < 0 + ? `${line} alwaysinline` + : `${line.slice(0, comment)} alwaysinline${line.slice(comment)}`; + } if (!this.unwindAbi.has(callee) || this.currentCoro !== null || B.isTerminated()) { B.line(line); return false; @@ -4018,10 +4152,22 @@ export class LlEmitter { ); } + /** The countdown slot of `loop`'s cancellation poll (emitWorkerLoopCheck), + * or null when the loop does not poll. An ELF worker executable's poll is + * one ordered load of the thread's own pending word. A loop nest keeps + * one countdown per loop, which its register pressure often leaves in + * memory, so the loops of a nest there poll on every iteration and get + * POLL_EVERY_ITERATION instead of a slot. A loop on its own keeps its + * countdown in a register, which costs less than the load. */ workerLoopBudget( loop: Extract, ): string | null { if (!this.workerLoopPolls(loop)) return null; + if ( + this.executableTls && + (this.jumpTargets.some((target) => target.kind === "loop") || containsLoop(loop.body)) + ) + return POLL_EVERY_ITERATION; const slot = this.B.slot(); this.B.entryAllocas.push(`${slot} = alloca i32 ; worker cancellation budget`); this.B.line(`store i32 0, ptr ${slot}`); @@ -4126,6 +4272,10 @@ export class LlEmitter { * stay in the loop. */ emitWorkerLoopCheck(slot: string | null): void { if (slot === null) return; + if (slot === POLL_EVERY_ITERATION) { + this.emitWorkerPoll(); + return; + } const B = this.B; const budget = B.tmp(), due = B.tmp(), @@ -4145,9 +4295,9 @@ export class LlEmitter { } /** The cancellation poll itself. Calls and throwing operations retain - * their own checks. Loops amortize this extra poll across bounded - * batches of iterations so scalar work does not cross the runtime ABI on - * every iteration. */ + * their own checks. Outside ELF worker executables, loops amortize this + * extra poll across bounded batches of iterations so scalar work does not + * cross the runtime ABI on every iteration. */ emitWorkerPoll(): void { this.emitPendingCheck(undefined, true); } @@ -5212,7 +5362,7 @@ export class LlEmitter { /** Awaits in the current body suspend its coroutine: every wasm32 async * body, and native stackless async bodies (selectStacklessAsync). */ get coroAwait(): boolean { - return this.wasi || this.currentCoro?.kind === "async"; + return this.wasi || this.currentCoro?.awaits === true; } /** Suspends the current coroutine body at an await of `promise` (null: a @@ -5251,7 +5401,7 @@ export class LlEmitter { * coroutine has already suspended until the promise settled, so its * extraction takes no further turn. */ awaitResultFn(kind: "void" | "f64" | "bool" | "str" | "ref"): string { - return !this.wasi && this.currentCoro?.kind === "async" + return !this.wasi && this.currentCoro?.awaits === true ? `scr_coro_await_${kind}` : `scr_await_${kind}`; } @@ -6319,6 +6469,7 @@ export class LlEmitter { type: local.type, slot: true, ...(local.boxed ? { boxed: true } : {}), + ...(local.arena === true ? { arena: "drop" as const } : {}), }); } return result; @@ -7413,9 +7564,196 @@ export class LlEmitter { } emitExpr(e: IrExpr): LlValue { + return this.arenas === null ? emitExpr(this, e) : this.emitArenaExpr(e, this.arenas); + } + + /** emitExpr when group arenas are planned. Kept out of emitExpr, whose + * frame each level of a nested expression holds while it is emitted. */ + private emitArenaExpr(e: IrExpr, arenas: ArenaPlan): LlValue { + if (arenas.groups.calls.has(e)) { + // A call region: everything the call allocates dies when it returns. + // The statement drops the arena after its temps, which an + // arena-aware caller can have allocated in it while evaluating the + // arguments; any other caller drops it as soon as the call returns. + const slot = this.arenaSlot(e); + this.statementFrame().push({ name: slot, type: e.type, slot: true, arena: "drop" }); + this.B.line(`call ptr @scr_arena_open(ptr ${slot}, i32 ${this.arenaClosed(e)})`); + const value = emitExpr(this, e); + if (!arenaAware(arenas, this.fnByName.get(this.currentFunctionName))) + this.B.line(`call void @scr_arena_drop(ptr ${slot})`); + return value; + } + if (arenas.sites.heapCalls.has(e) && this.arenaSite(null)) { + // A call whose value the group does not keep: everything it allocates + // takes the heap. + const saved = this.suspendArena(); + const value = emitExpr(this, e); + this.resumeArena(saved); + return value; + } + if (this.opensArena(e)) { + // A root: its group lives until the scope holding its value exits + // (the statement frame for a value used in place). + const slot = this.arenaSlot(e); + const local = arenas.groups.drops.get(e) ?? null; + this.statementFrame().push({ + name: slot, + type: e.type, + slot: true, + arena: local === null ? "drop" : "close", + }); + const arena = this.B.tmp(); + this.B.line(`${arena} = call ptr @scr_arena_open(ptr ${slot}, i32 ${this.arenaClosed(e)})`); + const value = emitExpr(this, e); + this.B.line(`call void @scr_arena_close(ptr ${arena})`); + return value; + } return emitExpr(this, e); } + /** Group arenas (arenas.ts): the plan, the list slot of each root and + * call region, and which locals' scopes drop which roots' arenas. */ + arenas: ArenaPlan | null = null; + private arenaSlots = new Map(); + /** The frames emitStmtBody opens for whole statements (arenas only). */ + readonly statementFrames = new Set(); + + /** The frame of the statement being emitted: a nested frame (call + * arguments, branches) ends before the values built in it do. */ + statementFrame(): LlValue[] { + for (let i = this.frames.length - 1; i >= 0; i--) + if (this.statementFrames.has(this.frames[i]!)) return this.frames[i]!; + return this.currentFrame(); + } + + /** The arena list slot of root or call region `e`: allocated, cleared and + * (for a root dropped with a local) owned by that local's scope when the + * function starts or the local is declared. */ + arenaSlot(e: Node): string { + let slot = this.arenaSlots.get(e); + if (slot === undefined) { + slot = this.B.tmp(); + this.B.entryAllocas.push(`${slot} = alloca ptr`, `store ptr null, ptr ${slot}`); + this.arenaSlots.set(e, slot); + this.declare(`declare ptr @scr_arena_open(ptr, i32)`); + this.declare(`declare void @scr_arena_close(ptr)`); + this.declare(`declare void @scr_arena_drop(ptr)`); + } + return slot; + } + + /** Whether emitExpr opens an arena for `e`: a root call or `new` of a + * counted value. */ + opensArena(e: Node): e is Extract { + return ( + this.arenas !== null && + this.arenas.groups.roots.has(e) && + (e.kind === "call" || e.kind === "new") && + isRefCounted(e.type) + ); + } + + /** 1 when the group of root or call region `e` is closed: its objects + * hold only each other and immortal values. */ + arenaClosed(e: Node): number { + return this.arenas?.groups.closed.has(e) === true ? 1 : 0; + } + + /** Registers on the innermost scope the arena lists of the roots whose + * arenas `localId`'s scope drops. */ + ownArenasOf(localId: string): void { + const roots = this.arenaDropsByLocal.get(localId); + if (roots === undefined) return; + const scope = this.scopes[this.scopes.length - 1]!; + for (const { root, type } of roots) + scope.push({ slot: this.arenaSlot(root), type, arena: true }); + } + + /** The roots of the function being emitted, by the local that drops + * them; set when a function's emission starts. */ + arenaDropsByLocal = new Map(); + + /** Starts a function's arena bookkeeping: its roots by dropping local, + * with the lists of locals the body never declares (parameters, loop + * and catch bindings) owned by the function's own scope. */ + beginArenaFunction(fn: IrFunction): void { + this.arenaSlots.clear(); + this.statementFrames.clear(); + this.arenaDropsByLocal = new Map(); + const arenas = this.arenas; + if (arenas === null) return; + const declared = new Set(); + const visit = (node: Node): void => { + if (node.kind === "varDecl") declared.add(node.localId); + if (this.opensArena(node)) { + const local = arenas.groups.drops.get(node); + if (local !== undefined && local !== null) { + const list = this.arenaDropsByLocal.get(local) ?? []; + list.push({ root: node, type: node.type }); + this.arenaDropsByLocal.set(local, list); + } + } + const kids: Node[] = []; + children(node, kids); + for (const kid of kids) visit(kid); + }; + for (const s of fn.body) visit(s); + for (const local of [...this.arenaDropsByLocal.keys()]) + if (!declared.has(local)) this.ownArenasOf(local); + } + + /** Whether allocation site `e` (null: any) in the function being + * emitted takes the open arena: the function is arena-aware and the group + * keeps the site's value (region-sites.ts). */ + arenaSite(e: IrExpr | null): boolean { + const arenas = this.arenas; + if (!arenaAware(arenas, this.fnByName.get(this.currentFunctionName))) return false; + return e === null || arenas!.sites.arenaSites.has(e); + } + + /** Stops any arena being the allocation target; returns what restores it. */ + suspendArena(): string { + const saved = this.B.tmp(); + const cur = this.B.tmp(); + const lines = arenaCurrentAddress(this.shapeHost, cur); + if (lines === null) { + this.declare(`declare ptr @scr_arena_suspend()`); + this.B.line(`${saved} = call ptr @scr_arena_suspend()`); + } else { + for (const line of lines) this.B.line(line); + this.B.line(`${saved} = load ptr, ptr ${cur}`); + this.B.line(`store ptr null, ptr ${cur}`); + } + return saved; + } + + /** Makes the arena suspendArena stopped (`saved`) the target again. */ + resumeArena(saved: string): void { + const cur = this.B.tmp(); + const lines = arenaCurrentAddress(this.shapeHost, cur); + if (lines === null) { + this.declare(`declare void @scr_arena_resume(ptr)`); + this.B.line(`call void @scr_arena_resume(ptr ${saved})`); + } else { + for (const line of lines) this.B.line(line); + this.B.line(`store ptr ${saved}, ptr ${cur}`); + } + } + + /** The arena-aware constructor of `newSym` for an allocation of a + * record shape or class in the function being emitted, or `newSym`. */ + allocSym(newSym: string, e: IrExpr): string { + const arenas = this.arenas; + if (arenas === null) return newSym; + const planned = + e.type.kind === "record" + ? arenas.shapes.has(e.type.shapeId) + : e.kind === "new" && arenas.classes.has(e.className); + if (!planned) return newSym; + const aware = this.arenaSite(e) || arenas.groups.roots.has(e); + return aware ? arenaNewSym(newSym) : newSym; + } + /** The helper classification of this module's inline-cache sites. */ get jsIcSites(): JsIcSites { return (this.jsIcSitesMemo ??= new JsIcSites(this.mod)); diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index 88d094800..18181de50 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -1104,7 +1104,7 @@ export function emitCallExpr( `llvm emitter bug: new ${e.className} without a constructor`, ); const o = B.tmp(); - B.line(`${o} = call ptr @${mangleClassNew(e.className)}()`); + B.line(`${o} = call ptr @${host.allocSym(mangleClassNew(e.className), e)}()`); const out = host.own({ name: o, type: e.type }); emitConstruction(host, e, o, ctor); return out; diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 7eec14e4e..279afdcd0 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -62,12 +62,14 @@ import { import { exactInteger, widenInteger } from "./integer-values.js"; import { emitDenseArrayNumber, + emitDenseArrayNumberInt32, emitDenseArrayPop, emitDenseArrayPush, runtimeAccess, type ArrayAccess, } from "./dense-array-access.js"; import { INT32_ARRAY_METHODS } from "../../ir/int32-arrays.js"; +import { INT32_RANGE } from "../../ir/integer-ranges.js"; import { emitUnionWiden } from "./expr-records.js"; import { registerReplay } from "./emit-journal.js"; @@ -694,6 +696,25 @@ export function emitArrIntrinsic( )!; return { name: value, type: e.type }; } + // In a store statement of an int32 kind loop's fast copy, a read that + // is not an int32 (a missing element's NaN) leaves for the + // statement's resume point, so the element is at hand as an i32. + if (!table && elem.kind === "f64" && access === "i32" && host.int32KindResume !== null) { + const read = emitDenseArrayNumberInt32( + host, + r.name, + index, + e.args[0], + host.provenDenseIndex(e.receiver, e.args[0]!), + host.int32KindResume, + ); + return { + name: read.value, + type: e.type, + uint32: read.int32, + integer: { name: read.int32, type: "i32", signed: true, range: INT32_RANGE }, + }; + } if (!table && elem.kind === "f64") return { name: emitDenseArrayNumber( diff --git a/packages/compiler/src/backend/llvm/expr-context.ts b/packages/compiler/src/backend/llvm/expr-context.ts index a5b7f61dd..0df23f10a 100644 --- a/packages/compiler/src/backend/llvm/expr-context.ts +++ b/packages/compiler/src/backend/llvm/expr-context.ts @@ -33,6 +33,9 @@ export interface LlValue { slot?: boolean; /** A sequence local held in a capture box; release the box itself. */ boxed?: boolean; + /** A group arena list slot (arenas.ts): the frame closes the newest + * arena on it, or drops the whole list, after its other temps. */ + arena?: "close" | "drop"; } export interface WideInteger { diff --git a/packages/compiler/src/backend/llvm/expr-control.ts b/packages/compiler/src/backend/llvm/expr-control.ts index 6e0fd4acb..28281478a 100644 --- a/packages/compiler/src/backend/llvm/expr-control.ts +++ b/packages/compiler/src/backend/llvm/expr-control.ts @@ -11,6 +11,10 @@ import { emitRecordKeyGetOr } from "./expr-records.js"; import { NULLABLE_NULL, SHAPE_WORD_INDEX, shapeWordHeader } from "./nullable-unions.js"; import { emitUnionTag } from "./union-repr.js"; +/** Each kind emits in a function of its own: emitting a nested chain of + * these expressions holds one emitControlExpr frame per level, and in an + * unoptimized build of the compiler a frame holds every local of the + * function. */ export function emitControlExpr( host: LlvmEmitterContext, e: ExprOf< @@ -25,446 +29,25 @@ export function emitControlExpr( | "nullish" >, ): LlValue { - const B = host.B; switch (e.kind) { - case "dynDestrCheck": { - // RequireObjectCoercible with V8's destructuring TypeError. dyn - // values check in the runtime helper and pass through unchanged. - const v = host.emitExpr(e.value); - const first = e.firstProp !== undefined ? host.cstr(e.firstProp) : "null"; - - const helper = host.dyn.dynDestrCheckHelper(); - B.line(`call void @${helper}(ptr ${v.name}, ptr ${host.cstr(e.spelling)}, ptr ${first})`); - host.emitPendingCheck(); - return v; - } - case "dynIterN": { - // GetIterator + first-N steps as a fresh array (V8's exact - // not-iterable TypeError on non-iterables) through the native dyn helper. - const v = emitBorrowedInput(host, e.value); - - const helper = host.dyn.dynIterNHelper(); - const t = B.tmp(); - B.line( - `${t} = call ptr @${helper}(ptr ${v.name}, ${host.sizeType} ${e.count}, ptr ${e.notIterableMessage ? host.cstr(e.notIterableMessage) : "null"})`, - ); - const out = host.own({ name: t, type: e.type }); - host.emitPendingCheck(); - return out; - } + case "dynDestrCheck": + return emitDynDestrCheck(host, e); + case "dynIterN": + return emitDynIterN(host, e); case "toBool": return { name: host.truthy(host.emitUnionProjection(e.operand)), type: e.type }; - case "logical": { - const tagSet = e.op === "||" ? unionTagSet(e) : null; - if (tagSet) return { name: emitUnionTagSet(host, tagSet), type: e.type }; - // JS value semantics: the result is the deciding operand itself. - // Left evaluates once, ownership moves into the result slot; when - // the branch takes the right operand the stale left releases first - // and the right runs in its own frame. - const ty = host.llType(e.type); - const l = host.emitExpr(e.left); - host.toFrame(l); - const slot = B.slot(); - B.entryAllocas.push(`${slot} = alloca ${ty}`); - B.line(`store ${ty} ${l.name}, ptr ${slot}`); - const integerSlot = e.type.kind === "f64" && l.uint32 !== undefined ? B.slot() : undefined; - if (integerSlot !== undefined) { - B.entryAllocas.push(`${integerSlot} = alloca i32`); - B.line(`store i32 ${l.uint32!}, ptr ${integerSlot}`); - } - const truthy = host.truthy(l); - const rightLabel = B.newLabel("log.r"); - const joinLabel = B.newLabel("log.j"); - if (e.op === "&&") B.condBr(truthy, rightLabel, joinLabel); - else B.condBr(truthy, joinLabel, rightLabel); - B.startBlock(rightLabel); - if (isRefCounted(e.type)) host.releaseFrame(l.name, e.type); - const rightInteger = host.emitBranchInto(slot, e.right, integerSlot); - B.br(joinLabel); - B.startBlock(joinLabel); - const t = B.tmp(); - B.line(`${t} = load ${ty}, ptr ${slot}`); - if (integerSlot !== undefined && rightInteger) { - const uint32 = B.tmp(); - B.line(`${uint32} = load i32, ptr ${integerSlot}`); - return { name: t, type: e.type, uint32 }; - } - return host.own({ name: t, type: e.type }); - } - case "ternary": { - // Exactly one arm evaluates; each arm runs in its own frame and - // moves the chosen value into the result slot. - const ty = host.llType(e.type); - const c = host.emitExpr(e.cond); - const slot = B.slot(); - B.entryAllocas.push(`${slot} = alloca ${ty}`); - const integerSlot = e.type.kind === "f64" ? B.slot() : undefined; - if (integerSlot !== undefined) B.entryAllocas.push(`${integerSlot} = alloca i32`); - const lt = B.newLabel("tern.t"); - const lf = B.newLabel("tern.f"); - const lj = B.newLabel("tern.j"); - B.condBr(c.name, lt, lf); - B.startBlock(lt); - const thenInteger = host.emitBranchInto(slot, e.then, integerSlot); - B.br(lj); - B.startBlock(lf); - const elseInteger = host.emitBranchInto(slot, e.else_, integerSlot); - B.br(lj); - B.startBlock(lj); - const t = B.tmp(); - B.line(`${t} = load ${ty}, ptr ${slot}`); - // A view is available only when every incoming branch wrote it. - // Incomplete views are unused and disappear during promotion. - if (integerSlot !== undefined && thenInteger && elseInteger) { - const uint32 = B.tmp(); - B.line(`${uint32} = load i32, ptr ${integerSlot}`); - return { name: t, type: e.type, uint32 }; - } - // A choice between interned literals is an interned literal: like a - // strLit operand, nothing owns it (retain and release are no-ops). - if (literalChoice(e)) return host.markImmortal({ name: t, type: e.type }); - return host.own({ name: t, type: e.type }); - } - case "optChain": { - // `a?.b` / `f?.()`: the nullish test inverted. The receiver - // evaluates once into an ordinary frame temp (borrowed); on a unit - // tag the result is the interned undefined arm and the body never - // runs; otherwise the narrowed payload fills the bind slot (+1, - // frame-owned through a SLOT entry — NULL on the unit path, where - // the frame's release is a no-op) and the body reads it through - // chainRecv. - if (e.receiver.type.kind === "dyn") { - // A dyn (dyn) receiver — the `rawName?.match(re)` step: the - // nullish test reads the node's kind tag; the unit path is the - // undefined dyn singleton (dyn results) or nothing (void - // bodies), the body runs over the bound receiver otherwise. - const r = host.emitExpr(e.receiver); - const kd = host.dynKind(r.name); - const isU = B.tmp(); - const isN = B.tmp(); - const isUnit = B.tmp(); - B.line(`${isU} = icmp eq i32 ${kd}, ${DYN_KIND.UNDEF}`); - B.line(`${isN} = icmp eq i32 ${kd}, ${DYN_KIND.NULL}`); - B.line(`${isUnit} = or i1 ${isU}, ${isN}`); - const bind = B.slot(); - B.entryAllocas.push(`${bind} = alloca ptr`); - B.line(`store ptr null, ptr ${bind}`); - host.ownSlot(bind, e.receiver.type); - host.declare(`declare ptr @scr_dyn_retain_v(ptr)`); - if (e.type.kind === "void") { - const lb = B.newLabel("ocd.b"); - const lj = B.newLabel("ocd.j"); - B.condBr(isUnit, lj, lb); - B.startBlock(lb); - const rr = B.tmp(); - B.line(`${rr} = call ptr @scr_dyn_retain_v(ptr ${r.name})`); - B.line(`store ptr ${rr}, ptr ${bind}`); - host.chainSlots.set(e.id, { name: bind, type: e.receiver.type, slot: true }); - host.frames.push([]); - host.emitExpr(e.body); - host.releaseTemps(host.frames.pop()!); - host.chainSlots.delete(e.id); - B.br(lj); - B.startBlock(lj); - return { name: "", type: e.type }; - } - if (e.type.kind !== "dyn") - throw new InternalCompilerError("llvm emitter bug: dyn optChain result kind"); - const slot = B.slot(); - B.entryAllocas.push(`${slot} = alloca ptr`); - const lu = B.newLabel("ocd.u"); - const lb = B.newLabel("ocd.b"); - const lj = B.newLabel("ocd.j"); - B.condBr(isUnit, lu, lb); - B.startBlock(lu); - host.declare(`declare ptr @scr_dyn_undefined()`); - const un = B.tmp(); - const ur = B.tmp(); - B.line(`${un} = call ptr @scr_dyn_undefined()`); - B.line(`${ur} = call ptr @scr_dyn_retain_v(ptr ${un})`); - B.line(`store ptr ${ur}, ptr ${slot}`); - B.br(lj); - B.startBlock(lb); - const rr = B.tmp(); - B.line(`${rr} = call ptr @scr_dyn_retain_v(ptr ${r.name})`); - B.line(`store ptr ${rr}, ptr ${bind}`); - host.chainSlots.set(e.id, { name: bind, type: e.receiver.type, slot: true }); - host.emitBranchInto(slot, e.body); - host.chainSlots.delete(e.id); - B.br(lj); - B.startBlock(lj); - const t = B.tmp(); - B.line(`${t} = load ptr, ptr ${slot}`); - return host.own({ name: t, type: e.type }); - } - - const { isUnit, bindPresent } = emitOptionalChainReceiver(host, e); - if (e.type.kind === "void") { - // Statement form (cb?.()): no result value at all. - const lb = B.newLabel("oc.b"); - const lj = B.newLabel("oc.j"); - B.condBr(isUnit, lj, lb); - B.startBlock(lb); - bindPresent(); - host.frames.push([]); - host.emitExpr(e.body); - host.releaseTemps(host.frames.pop()!); - host.chainSlots.delete(e.id); - B.br(lj); - B.startBlock(lj); - return { name: "", type: e.type }; - } - if (e.type.kind === "dyn") { - // A dyn-typed chain (`pricing?.[key]` over an unknown-valued - // index signature): the unit path is the undefined dyn value — - // dyn represents undefined directly, no union wrapper exists. - const slotD = B.slot(); - B.entryAllocas.push(`${slotD} = alloca ptr`); - const lu = B.newLabel("ocu.u"); - const lb = B.newLabel("ocu.b"); - const lj = B.newLabel("ocu.j"); - B.condBr(isUnit, lu, lb); - B.startBlock(lu); - host.declare(`declare ptr @scr_dyn_undefined()`); - host.declare(`declare ptr @scr_dyn_retain_v(ptr)`); - const un = B.tmp(); - const ur = B.tmp(); - B.line(`${un} = call ptr @scr_dyn_undefined()`); - B.line(`${ur} = call ptr @scr_dyn_retain_v(ptr ${un})`); - B.line(`store ptr ${ur}, ptr ${slotD}`); - B.br(lj); - B.startBlock(lb); - bindPresent(); - host.emitBranchInto(slotD, e.body); - host.chainSlots.delete(e.id); - B.br(lj); - B.startBlock(lj); - const t = B.tmp(); - B.line(`${t} = load ptr, ptr ${slotD}`); - return host.own({ name: t, type: e.type }); - } - if (e.type.kind !== "union") - throw new LlvmUnsupportedError(`optChainResult:${e.type.kind}`, e.loc); - const undefTag = undefinedArmTag(e.type, host.unionsById); - if (undefTag < 0) - throw new InternalCompilerError( - "llvm emitter bug: optChain result lacks its undefined arm", - ); - const ty = host.llType(e.type); - const slot = B.slot(); - B.entryAllocas.push(`${slot} = alloca ${ty}`); - const lu = B.newLabel("oc.u"); - const lb = B.newLabel("oc.b"); - const lj = B.newLabel("oc.j"); - B.condBr(isUnit, lu, lb); - B.startBlock(lu); - B.line(`store ptr ${host.unitInstanceRef(e.type.unionId, undefTag)}, ptr ${slot}`); - B.br(lj); - B.startBlock(lb); - bindPresent(); - host.emitBranchInto(slot, e.body); - host.chainSlots.delete(e.id); - B.br(lj); - B.startBlock(lj); - const t = B.tmp(); - B.line(`${t} = load ${ty}, ptr ${slot}`); - return host.own({ name: t, type: e.type }); - } - case "chainRecv": { - const bound = host.chainSlots.get(e.id); - if (!bound) - throw new InternalCompilerError(`llvm emitter bug: chainRecv "${e.id}" outside its chain`); - const v = B.tmp(); - B.line(`${v} = load ${host.llType(bound.type)}, ptr ${bound.name}`); - if (!isRefCounted(e.type)) return { name: v, type: e.type }; - return host.own({ name: host.retainFrame(v, e.type), type: e.type }); - } - case "orDefault": { - // `u || d` narrowed to the single non-unit arm: nullish's dance - // with the union TRUTHY switch as the test — truthy extracts the - // arm (+1 for ref kinds), falsy releases and runs the default. - if (e.left.type.kind !== "union") - throw new InternalCompilerError("llvm emitter bug: orDefault left is not a union"); - if ( - e.retag === undefined && - !host.nullableUnions.has(e.left.type.unionId) && - host.isStackUnionSource(e.left) - ) { - // A stack box (`arr[i] || d`): test truthiness and extract the - // payload (+1). The box is never retained or released. - const l = host.emitUnionProjection(e.left); - const ty = host.llType(e.type); - const slot = B.slot(); - B.entryAllocas.push(`${slot} = alloca ${ty}`); - const lt = B.newLabel("ord.t"), - lf = B.newLabel("ord.f"), - lj = B.newLabel("ord.j"); - B.condBr(host.truthy(l), lt, lf); - B.startBlock(lt); - B.line( - `store ${ty} ${host.unionExtract(l.name, e.left.type.unionId, e.type)}, ptr ${slot}`, - ); - B.br(lj); - B.startBlock(lf); - host.emitBranchInto(slot, e.right); - B.br(lj); - B.startBlock(lj); - const t = B.tmp(); - B.line(`${t} = load ${ty}, ptr ${slot}`); - return host.own({ name: t, type: e.type }); - } - const l = host.emitExpr(e.left); - host.toFrame(l); - const ty = host.llType(e.type); - const slot = B.slot(); - B.entryAllocas.push(`${slot} = alloca ${ty}`); - const truthy = host.truthy(l); - const lt = B.newLabel("ord.t"); - const lf = B.newLabel("ord.f"); - const lj = B.newLabel("ord.j"); - B.condBr(truthy, lt, lf); - B.startBlock(lt); - if (e.retag !== undefined) { - // Retagged shape: the whole box goes to the union→union helper, - // which CONSUMES it (callees own their params) — no release on - // this side. The store precedes the pending check so an unwind - // leaves only the NULL dummy behind, in a slot nothing reads. - const t = B.tmp(); - B.line( - `${t} = call ${ty} @${host.callTarget(e.retag)}(${host.llType(e.left.type)} ${l.name})`, - ); - B.line(`store ${ty} ${t}, ptr ${slot}`); - if (host.mayThrow.has(e.retag)) host.emitPendingCheck(); - } else if (host.nullableUnions.single(e.left.type.unionId)) { - // The owned nullable pointer IS the owned arm value. - B.line(`store ${ty} ${l.name}, ptr ${slot}`); - } else { - B.line( - `store ${ty} ${host.unionExtract(l.name, e.left.type.unionId, e.type)}, ptr ${slot}`, - ); - host.releaseFrame(l.name, e.left.type); - } - B.br(lj); - B.startBlock(lf); - host.releaseFrame(l.name, e.left.type); - host.emitBranchInto(slot, e.right); - B.br(lj); - B.startBlock(lj); - const t = B.tmp(); - B.line(`${t} = load ${ty}, ptr ${slot}`); - return host.own({ name: t, type: e.type }); - } - case "nullish": { - // `a ?? b`: logical's move/release dance with the left's runtime - // TAG against its unit arms as the test. Pass-through shape: the - // result IS the left box. Narrowed shape: the single non-unit - // arm's payload extracts (+1 for ref kinds) and the box releases. - - if (e.left.type.kind === "dyn") { - // The native checked-value kind decides whether the left is - // nullish; the right runs lazily in its branch. - const l = host.emitExpr(e.left); - host.toFrame(l); - host.declare(`declare zeroext i1 @scr_dyn_is_nullish(ptr)`); - const slot = B.slot(); - B.entryAllocas.push(`${slot} = alloca ptr`); - const isN = B.tmp(); - B.line(`${isN} = call zeroext i1 @scr_dyn_is_nullish(ptr ${l.name})`); - const lu = B.newLabel("nuld.u"); - const lv = B.newLabel("nuld.v"); - const lj = B.newLabel("nuld.j"); - B.condBr(isN, lu, lv); - B.startBlock(lu); - host.releaseFrame(l.name, e.left.type); - host.emitBranchInto(slot, e.right); - B.br(lj); - B.startBlock(lv); - B.line(`store ptr ${l.name}, ptr ${slot}`); - B.br(lj); - B.startBlock(lj); - const t = B.tmp(); - B.line(`${t} = load ptr, ptr ${slot}`); - return host.own({ name: t, type: e.type }); - } - if (e.left.type.kind !== "union") - throw new InternalCompilerError("llvm emitter bug: nullish left is not a union"); - const def = host.unionsById.get(e.left.type.unionId); - if (!def) - throw new InternalCompilerError( - `llvm emitter bug: nullish of unknown union ${e.left.type.unionId}`, - ); - const unitTags = def.arms.flatMap((a, i) => (isUnitType(a) ? [i] : [])); - if (unitTags.length === 0) - throw new InternalCompilerError("llvm emitter bug: nullish union lacks unit arms"); - const fused = emitNullishOptionalChain(host, e); - if (fused) return fused; - if ( - e.left.kind === "recordKeyGet" && - e.left.missNaN !== true && - typeEquals(e.type, e.left.type) && - storesNoUnit(host, e.left) - ) - return emitRecordKeyGetOr(host, e.left, e.right); - if ( - !typeEquals(e.type, e.left.type) && - (e.left.kind === "varRef" || - host.isStackUnionSource(e.left) || - host.nullableFieldGet(e.left) !== null) - ) { - // Narrowed shape over a binding or a stack box (`arr[i] ?? d`, - // `map.get(k) ?? d`): test the tag and extract the payload (+1). - // The box is never retained or released. A binding is borrowed: - // no code runs between its read and the extraction, and the - // default never touches it. Bindings proven projection-only may - // hold stack boxes, which must never reach RC entry points. - const l = host.emitUnionProjection(e.left); - const ty = host.llType(e.type); - const slot = B.slot(); - B.entryAllocas.push(`${slot} = alloca ${ty}`); - const isUnit = host.tagInSet(l.name, def.id, unitTags); - const lu = B.newLabel("nul.u"); - const lv = B.newLabel("nul.v"); - const lj = B.newLabel("nul.j"); - B.condBr(isUnit, lu, lv); - B.startBlock(lu); - host.emitBranchInto(slot, e.right); - B.br(lj); - B.startBlock(lv); - B.line(`store ${ty} ${host.unionExtract(l.name, def.id, e.type)}, ptr ${slot}`); - B.br(lj); - B.startBlock(lj); - const t = B.tmp(); - B.line(`${t} = load ${ty}, ptr ${slot}`); - return host.own({ name: t, type: e.type }); - } - const l = host.emitExpr(e.left); - host.toFrame(l); - const ty = host.llType(e.type); - const slot = B.slot(); - B.entryAllocas.push(`${slot} = alloca ${ty}`); - const isUnit = host.tagInSet(l.name, def.id, unitTags); - const lu = B.newLabel("nul.u"); - const lv = B.newLabel("nul.v"); - const lj = B.newLabel("nul.j"); - B.condBr(isUnit, lu, lv); - B.startBlock(lu); - host.releaseFrame(l.name, e.left.type); - host.emitBranchInto(slot, e.right); - B.br(lj); - B.startBlock(lv); - if (typeEquals(e.type, e.left.type) || host.nullableUnions.single(def.id)) { - // Pass-through, or a nullable pointer that IS its owned arm value. - B.line(`store ${ty} ${l.name}, ptr ${slot}`); - } else { - B.line(`store ${ty} ${host.unionExtract(l.name, def.id, e.type)}, ptr ${slot}`); - host.releaseFrame(l.name, e.left.type); - } - B.br(lj); - B.startBlock(lj); - const t = B.tmp(); - B.line(`${t} = load ${ty}, ptr ${slot}`); - return host.own({ name: t, type: e.type }); - } + case "logical": + return emitLogical(host, e); + case "ternary": + return emitTernary(host, e); + case "optChain": + return emitOptChain(host, e); + case "chainRecv": + return emitChainRecv(host, e); + case "orDefault": + return emitOrDefault(host, e); + case "nullish": + return emitNullish(host, e); default: { const _exhaustive: never = e; void _exhaustive; @@ -473,6 +56,450 @@ export function emitControlExpr( } } +function emitDynDestrCheck(host: LlvmEmitterContext, e: ExprOf<"dynDestrCheck">): LlValue { + const B = host.B; + // RequireObjectCoercible with V8's destructuring TypeError. dyn + // values check in the runtime helper and pass through unchanged. + const v = host.emitExpr(e.value); + const first = e.firstProp !== undefined ? host.cstr(e.firstProp) : "null"; + + const helper = host.dyn.dynDestrCheckHelper(); + B.line(`call void @${helper}(ptr ${v.name}, ptr ${host.cstr(e.spelling)}, ptr ${first})`); + host.emitPendingCheck(); + return v; +} + +function emitDynIterN(host: LlvmEmitterContext, e: ExprOf<"dynIterN">): LlValue { + const B = host.B; + // GetIterator + first-N steps as a fresh array (V8's exact + // not-iterable TypeError on non-iterables) through the native dyn helper. + const v = emitBorrowedInput(host, e.value); + + const helper = host.dyn.dynIterNHelper(); + const t = B.tmp(); + B.line( + `${t} = call ptr @${helper}(ptr ${v.name}, ${host.sizeType} ${e.count}, ptr ${e.notIterableMessage ? host.cstr(e.notIterableMessage) : "null"})`, + ); + const out = host.own({ name: t, type: e.type }); + host.emitPendingCheck(); + return out; +} + +function emitLogical(host: LlvmEmitterContext, e: ExprOf<"logical">): LlValue { + const B = host.B; + const tagSet = e.op === "||" ? unionTagSet(e) : null; + if (tagSet) return { name: emitUnionTagSet(host, tagSet), type: e.type }; + // JS value semantics: the result is the deciding operand itself. + // Left evaluates once, ownership moves into the result slot; when + // the branch takes the right operand the stale left releases first + // and the right runs in its own frame. + const ty = host.llType(e.type); + const l = host.emitExpr(e.left); + host.toFrame(l); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ${ty}`); + B.line(`store ${ty} ${l.name}, ptr ${slot}`); + const integerSlot = e.type.kind === "f64" && l.uint32 !== undefined ? B.slot() : undefined; + if (integerSlot !== undefined) { + B.entryAllocas.push(`${integerSlot} = alloca i32`); + B.line(`store i32 ${l.uint32!}, ptr ${integerSlot}`); + } + const truthy = host.truthy(l); + const rightLabel = B.newLabel("log.r"); + const joinLabel = B.newLabel("log.j"); + if (e.op === "&&") B.condBr(truthy, rightLabel, joinLabel); + else B.condBr(truthy, joinLabel, rightLabel); + B.startBlock(rightLabel); + if (isRefCounted(e.type)) host.releaseFrame(l.name, e.type); + const rightInteger = host.emitBranchInto(slot, e.right, integerSlot); + B.br(joinLabel); + B.startBlock(joinLabel); + const t = B.tmp(); + B.line(`${t} = load ${ty}, ptr ${slot}`); + if (integerSlot !== undefined && rightInteger) { + const uint32 = B.tmp(); + B.line(`${uint32} = load i32, ptr ${integerSlot}`); + return { name: t, type: e.type, uint32 }; + } + return host.own({ name: t, type: e.type }); +} + +function emitTernary(host: LlvmEmitterContext, e: ExprOf<"ternary">): LlValue { + const B = host.B; + // Exactly one arm evaluates; each arm runs in its own frame and + // moves the chosen value into the result slot. + const ty = host.llType(e.type); + const c = host.emitExpr(e.cond); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ${ty}`); + const integerSlot = e.type.kind === "f64" ? B.slot() : undefined; + if (integerSlot !== undefined) B.entryAllocas.push(`${integerSlot} = alloca i32`); + const lt = B.newLabel("tern.t"); + const lf = B.newLabel("tern.f"); + const lj = B.newLabel("tern.j"); + B.condBr(c.name, lt, lf); + B.startBlock(lt); + const thenInteger = host.emitBranchInto(slot, e.then, integerSlot); + B.br(lj); + B.startBlock(lf); + const elseInteger = host.emitBranchInto(slot, e.else_, integerSlot); + B.br(lj); + B.startBlock(lj); + const t = B.tmp(); + B.line(`${t} = load ${ty}, ptr ${slot}`); + // A view is available only when every incoming branch wrote it. + // Incomplete views are unused and disappear during promotion. + if (integerSlot !== undefined && thenInteger && elseInteger) { + const uint32 = B.tmp(); + B.line(`${uint32} = load i32, ptr ${integerSlot}`); + return { name: t, type: e.type, uint32 }; + } + // A choice between interned literals is an interned literal: like a + // strLit operand, nothing owns it (retain and release are no-ops). + if (literalChoice(e)) return host.markImmortal({ name: t, type: e.type }); + return host.own({ name: t, type: e.type }); +} + +function emitOptChain(host: LlvmEmitterContext, e: ExprOf<"optChain">): LlValue { + const B = host.B; + // `a?.b` / `f?.()`: the nullish test inverted. The receiver + // evaluates once into an ordinary frame temp (borrowed); on a unit + // tag the result is the interned undefined arm and the body never + // runs; otherwise the narrowed payload fills the bind slot (+1, + // frame-owned through a SLOT entry — NULL on the unit path, where + // the frame's release is a no-op) and the body reads it through + // chainRecv. + if (e.receiver.type.kind === "dyn") { + // A dyn (dyn) receiver — the `rawName?.match(re)` step: the + // nullish test reads the node's kind tag; the unit path is the + // undefined dyn singleton (dyn results) or nothing (void + // bodies), the body runs over the bound receiver otherwise. + const r = host.emitExpr(e.receiver); + const kd = host.dynKind(r.name); + const isU = B.tmp(); + const isN = B.tmp(); + const isUnit = B.tmp(); + B.line(`${isU} = icmp eq i32 ${kd}, ${DYN_KIND.UNDEF}`); + B.line(`${isN} = icmp eq i32 ${kd}, ${DYN_KIND.NULL}`); + B.line(`${isUnit} = or i1 ${isU}, ${isN}`); + const bind = B.slot(); + B.entryAllocas.push(`${bind} = alloca ptr`); + B.line(`store ptr null, ptr ${bind}`); + host.ownSlot(bind, e.receiver.type); + host.declare(`declare ptr @scr_dyn_retain_v(ptr)`); + if (e.type.kind === "void") { + const lb = B.newLabel("ocd.b"); + const lj = B.newLabel("ocd.j"); + B.condBr(isUnit, lj, lb); + B.startBlock(lb); + const rr = B.tmp(); + B.line(`${rr} = call ptr @scr_dyn_retain_v(ptr ${r.name})`); + B.line(`store ptr ${rr}, ptr ${bind}`); + host.chainSlots.set(e.id, { name: bind, type: e.receiver.type, slot: true }); + host.frames.push([]); + host.emitExpr(e.body); + host.releaseTemps(host.frames.pop()!); + host.chainSlots.delete(e.id); + B.br(lj); + B.startBlock(lj); + return { name: "", type: e.type }; + } + if (e.type.kind !== "dyn") + throw new InternalCompilerError("llvm emitter bug: dyn optChain result kind"); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ptr`); + const lu = B.newLabel("ocd.u"); + const lb = B.newLabel("ocd.b"); + const lj = B.newLabel("ocd.j"); + B.condBr(isUnit, lu, lb); + B.startBlock(lu); + host.declare(`declare ptr @scr_dyn_undefined()`); + const un = B.tmp(); + const ur = B.tmp(); + B.line(`${un} = call ptr @scr_dyn_undefined()`); + B.line(`${ur} = call ptr @scr_dyn_retain_v(ptr ${un})`); + B.line(`store ptr ${ur}, ptr ${slot}`); + B.br(lj); + B.startBlock(lb); + const rr = B.tmp(); + B.line(`${rr} = call ptr @scr_dyn_retain_v(ptr ${r.name})`); + B.line(`store ptr ${rr}, ptr ${bind}`); + host.chainSlots.set(e.id, { name: bind, type: e.receiver.type, slot: true }); + host.emitBranchInto(slot, e.body); + host.chainSlots.delete(e.id); + B.br(lj); + B.startBlock(lj); + const t = B.tmp(); + B.line(`${t} = load ptr, ptr ${slot}`); + return host.own({ name: t, type: e.type }); + } + + const { isUnit, bindPresent } = emitOptionalChainReceiver(host, e); + if (e.type.kind === "void") { + // Statement form (cb?.()): no result value at all. + const lb = B.newLabel("oc.b"); + const lj = B.newLabel("oc.j"); + B.condBr(isUnit, lj, lb); + B.startBlock(lb); + bindPresent(); + host.frames.push([]); + host.emitExpr(e.body); + host.releaseTemps(host.frames.pop()!); + host.chainSlots.delete(e.id); + B.br(lj); + B.startBlock(lj); + return { name: "", type: e.type }; + } + if (e.type.kind === "dyn") { + // A dyn-typed chain (`pricing?.[key]` over an unknown-valued + // index signature): the unit path is the undefined dyn value — + // dyn represents undefined directly, no union wrapper exists. + const slotD = B.slot(); + B.entryAllocas.push(`${slotD} = alloca ptr`); + const lu = B.newLabel("ocu.u"); + const lb = B.newLabel("ocu.b"); + const lj = B.newLabel("ocu.j"); + B.condBr(isUnit, lu, lb); + B.startBlock(lu); + host.declare(`declare ptr @scr_dyn_undefined()`); + host.declare(`declare ptr @scr_dyn_retain_v(ptr)`); + const un = B.tmp(); + const ur = B.tmp(); + B.line(`${un} = call ptr @scr_dyn_undefined()`); + B.line(`${ur} = call ptr @scr_dyn_retain_v(ptr ${un})`); + B.line(`store ptr ${ur}, ptr ${slotD}`); + B.br(lj); + B.startBlock(lb); + bindPresent(); + host.emitBranchInto(slotD, e.body); + host.chainSlots.delete(e.id); + B.br(lj); + B.startBlock(lj); + const t = B.tmp(); + B.line(`${t} = load ptr, ptr ${slotD}`); + return host.own({ name: t, type: e.type }); + } + if (e.type.kind !== "union") + throw new LlvmUnsupportedError(`optChainResult:${e.type.kind}`, e.loc); + const undefTag = undefinedArmTag(e.type, host.unionsById); + if (undefTag < 0) + throw new InternalCompilerError("llvm emitter bug: optChain result lacks its undefined arm"); + const ty = host.llType(e.type); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ${ty}`); + const lu = B.newLabel("oc.u"); + const lb = B.newLabel("oc.b"); + const lj = B.newLabel("oc.j"); + B.condBr(isUnit, lu, lb); + B.startBlock(lu); + B.line(`store ptr ${host.unitInstanceRef(e.type.unionId, undefTag)}, ptr ${slot}`); + B.br(lj); + B.startBlock(lb); + bindPresent(); + host.emitBranchInto(slot, e.body); + host.chainSlots.delete(e.id); + B.br(lj); + B.startBlock(lj); + const t = B.tmp(); + B.line(`${t} = load ${ty}, ptr ${slot}`); + return host.own({ name: t, type: e.type }); +} + +function emitChainRecv(host: LlvmEmitterContext, e: ExprOf<"chainRecv">): LlValue { + const B = host.B; + const bound = host.chainSlots.get(e.id); + if (!bound) + throw new InternalCompilerError(`llvm emitter bug: chainRecv "${e.id}" outside its chain`); + const v = B.tmp(); + B.line(`${v} = load ${host.llType(bound.type)}, ptr ${bound.name}`); + if (!isRefCounted(e.type)) return { name: v, type: e.type }; + return host.own({ name: host.retainFrame(v, e.type), type: e.type }); +} + +function emitOrDefault(host: LlvmEmitterContext, e: ExprOf<"orDefault">): LlValue { + const B = host.B; + // `u || d` narrowed to the single non-unit arm: nullish's dance + // with the union TRUTHY switch as the test — truthy extracts the + // arm (+1 for ref kinds), falsy releases and runs the default. + if (e.left.type.kind !== "union") + throw new InternalCompilerError("llvm emitter bug: orDefault left is not a union"); + if ( + e.retag === undefined && + !host.nullableUnions.has(e.left.type.unionId) && + host.isStackUnionSource(e.left) + ) { + // A stack box (`arr[i] || d`): test truthiness and extract the + // payload (+1). The box is never retained or released. + const l = host.emitUnionProjection(e.left); + const ty = host.llType(e.type); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ${ty}`); + const lt = B.newLabel("ord.t"), + lf = B.newLabel("ord.f"), + lj = B.newLabel("ord.j"); + B.condBr(host.truthy(l), lt, lf); + B.startBlock(lt); + B.line(`store ${ty} ${host.unionExtract(l.name, e.left.type.unionId, e.type)}, ptr ${slot}`); + B.br(lj); + B.startBlock(lf); + host.emitBranchInto(slot, e.right); + B.br(lj); + B.startBlock(lj); + const t = B.tmp(); + B.line(`${t} = load ${ty}, ptr ${slot}`); + return host.own({ name: t, type: e.type }); + } + const l = host.emitExpr(e.left); + host.toFrame(l); + const ty = host.llType(e.type); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ${ty}`); + const truthy = host.truthy(l); + const lt = B.newLabel("ord.t"); + const lf = B.newLabel("ord.f"); + const lj = B.newLabel("ord.j"); + B.condBr(truthy, lt, lf); + B.startBlock(lt); + if (e.retag !== undefined) { + // Retagged shape: the whole box goes to the union→union helper, + // which CONSUMES it (callees own their params) — no release on + // this side. The store precedes the pending check so an unwind + // leaves only the NULL dummy behind, in a slot nothing reads. + const t = B.tmp(); + B.line(`${t} = call ${ty} @${host.callTarget(e.retag)}(${host.llType(e.left.type)} ${l.name})`); + B.line(`store ${ty} ${t}, ptr ${slot}`); + if (host.mayThrow.has(e.retag)) host.emitPendingCheck(); + } else if (host.nullableUnions.single(e.left.type.unionId)) { + // The owned nullable pointer IS the owned arm value. + B.line(`store ${ty} ${l.name}, ptr ${slot}`); + } else { + B.line(`store ${ty} ${host.unionExtract(l.name, e.left.type.unionId, e.type)}, ptr ${slot}`); + host.releaseFrame(l.name, e.left.type); + } + B.br(lj); + B.startBlock(lf); + host.releaseFrame(l.name, e.left.type); + host.emitBranchInto(slot, e.right); + B.br(lj); + B.startBlock(lj); + const t = B.tmp(); + B.line(`${t} = load ${ty}, ptr ${slot}`); + return host.own({ name: t, type: e.type }); +} + +function emitNullish(host: LlvmEmitterContext, e: ExprOf<"nullish">): LlValue { + const B = host.B; + // `a ?? b`: logical's move/release dance with the left's runtime + // TAG against its unit arms as the test. Pass-through shape: the + // result IS the left box. Narrowed shape: the single non-unit + // arm's payload extracts (+1 for ref kinds) and the box releases. + + if (e.left.type.kind === "dyn") { + // The native checked-value kind decides whether the left is + // nullish; the right runs lazily in its branch. + const l = host.emitExpr(e.left); + host.toFrame(l); + host.declare(`declare zeroext i1 @scr_dyn_is_nullish(ptr)`); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ptr`); + const isN = B.tmp(); + B.line(`${isN} = call zeroext i1 @scr_dyn_is_nullish(ptr ${l.name})`); + const lu = B.newLabel("nuld.u"); + const lv = B.newLabel("nuld.v"); + const lj = B.newLabel("nuld.j"); + B.condBr(isN, lu, lv); + B.startBlock(lu); + host.releaseFrame(l.name, e.left.type); + host.emitBranchInto(slot, e.right); + B.br(lj); + B.startBlock(lv); + B.line(`store ptr ${l.name}, ptr ${slot}`); + B.br(lj); + B.startBlock(lj); + const t = B.tmp(); + B.line(`${t} = load ptr, ptr ${slot}`); + return host.own({ name: t, type: e.type }); + } + if (e.left.type.kind !== "union") + throw new InternalCompilerError("llvm emitter bug: nullish left is not a union"); + const def = host.unionsById.get(e.left.type.unionId); + if (!def) + throw new InternalCompilerError( + `llvm emitter bug: nullish of unknown union ${e.left.type.unionId}`, + ); + const unitTags = def.arms.flatMap((a, i) => (isUnitType(a) ? [i] : [])); + if (unitTags.length === 0) + throw new InternalCompilerError("llvm emitter bug: nullish union lacks unit arms"); + const fused = emitNullishOptionalChain(host, e); + if (fused) return fused; + if ( + e.left.kind === "recordKeyGet" && + e.left.missNaN !== true && + typeEquals(e.type, e.left.type) && + storesNoUnit(host, e.left) + ) + return emitRecordKeyGetOr(host, e.left, e.right); + if ( + !typeEquals(e.type, e.left.type) && + (e.left.kind === "varRef" || + host.isStackUnionSource(e.left) || + host.nullableFieldGet(e.left) !== null) + ) { + // Narrowed shape over a binding or a stack box (`arr[i] ?? d`, + // `map.get(k) ?? d`): test the tag and extract the payload (+1). + // The box is never retained or released. A binding is borrowed: + // no code runs between its read and the extraction, and the + // default never touches it. Bindings proven projection-only may + // hold stack boxes, which must never reach RC entry points. + const l = host.emitUnionProjection(e.left); + const ty = host.llType(e.type); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ${ty}`); + const isUnit = host.tagInSet(l.name, def.id, unitTags); + const lu = B.newLabel("nul.u"); + const lv = B.newLabel("nul.v"); + const lj = B.newLabel("nul.j"); + B.condBr(isUnit, lu, lv); + B.startBlock(lu); + host.emitBranchInto(slot, e.right); + B.br(lj); + B.startBlock(lv); + B.line(`store ${ty} ${host.unionExtract(l.name, def.id, e.type)}, ptr ${slot}`); + B.br(lj); + B.startBlock(lj); + const t = B.tmp(); + B.line(`${t} = load ${ty}, ptr ${slot}`); + return host.own({ name: t, type: e.type }); + } + const l = host.emitExpr(e.left); + host.toFrame(l); + const ty = host.llType(e.type); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ${ty}`); + const isUnit = host.tagInSet(l.name, def.id, unitTags); + const lu = B.newLabel("nul.u"); + const lv = B.newLabel("nul.v"); + const lj = B.newLabel("nul.j"); + B.condBr(isUnit, lu, lv); + B.startBlock(lu); + host.releaseFrame(l.name, e.left.type); + host.emitBranchInto(slot, e.right); + B.br(lj); + B.startBlock(lv); + if (typeEquals(e.type, e.left.type) || host.nullableUnions.single(def.id)) { + // Pass-through, or a nullable pointer that IS its owned arm value. + B.line(`store ${ty} ${l.name}, ptr ${slot}`); + } else { + B.line(`store ${ty} ${host.unionExtract(l.name, def.id, e.type)}, ptr ${slot}`); + host.releaseFrame(l.name, e.left.type); + } + B.br(lj); + B.startBlock(lj); + const t = B.tmp(); + B.line(`${t} = load ${ty}, ptr ${slot}`); + return host.own({ name: t, type: e.type }); +} + /** The receiver half of a union-receiver `a?.b`. The receiver evaluates * once; `isUnit` tests its unit arms, and `bindPresent` fills the chain's * bind slot with the narrowed payload for the body to read through diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index 1a4973a3a..5d2a1cfeb 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -1080,16 +1080,21 @@ export function emitContainerExpr( B.line(`${fits} = icmp ule ${host.sizeType} ${capacity}, ${SMALL_ARRAY_SLOTS}`); B.condBr(fits, smallLabel, sizedLabel); B.startBlock(smallLabel); - B.line(`${smallArr} = ${arrNewCall(host.shapeHost, elem, "0", true)}`); + B.line(`${smallArr} = ${arrNewCall(host.shapeHost, elem, "0", true, host.arenaSite(e))}`); B.br(joined); B.startBlock(sizedLabel); - B.line(`${sizedArr} = ${arrNewCall(host.shapeHost, elem, capacity)}`); + B.line( + `${sizedArr} = ${arrNewCall(host.shapeHost, elem, capacity, false, host.arenaSite(e))}`, + ); B.br(joined); B.startBlock(joined); B.line( `${arr} = phi ptr [ ${smallArr}, %${smallLabel} ], [ ${sizedArr}, %${sizedLabel} ]`, ); - } else B.line(`${arr} = ${arrNewCall(host.shapeHost, elem, String(capacity), small)}`); + } else + B.line( + `${arr} = ${arrNewCall(host.shapeHost, elem, String(capacity), small, host.arenaSite(e))}`, + ); } const out = host.own({ name: arr, type: e.type }); const acc: ArrayAccess = int32 ? "i32" : elemAccess(elem); @@ -1362,7 +1367,7 @@ export function emitRecordExpr( throw new InternalCompilerError("llvm emitter bug: recordLit of non-record type"); const shapeId = e.type.shapeId; const rec = B.tmp(); - B.line(`${rec} = call ptr @${mangleRecordNew(shapeId)}()`); + B.line(`${rec} = call ptr @${host.allocSym(mangleRecordNew(shapeId), e)}()`); const out = host.own({ name: rec, type: e.type }); // A null-prototype record: SCR_MAP_NULL_PROTO on the fresh map. if (e.nullProto) B.line(`store i8 1, ptr ${host.recordMapFlagsPtr(rec, shapeId)}`); diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.test.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.test.ts index 244b3afea..4649056e1 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.test.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.test.ts @@ -141,7 +141,7 @@ test("live record capsules bind field hooks in snapshot order", () => { const llvm = emitLlvmModule(module, { pointerBits: 64 }); const tables = [ ...llvm.matchAll( - /^@(\S+)_rops = internal global \{ i64, ptr, ptr, ptr, i8, ptr, ptr \} \{ i64 (\d+), /gm, + /^@(\S+)_rops = internal global \{ i64, ptr, ptr, ptr, i8, ptr, ptr, ptr \} \{ i64 (\d+), /gm, ), ]; // root and leaf get field hooks; the record with an index-like field name diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts index f711795d3..516c08076 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts @@ -1484,6 +1484,10 @@ function liveRecordOps( G.startBlock(bad); G.terminate(`unreachable`); + // Only a program that writes through a dynamic view needs the store + // hook (and its checked conversions), as for the commit adapter. + const setDefs = host.liveViewsReadOnly ? [] : liveRecordSet(host, t, `${snapshot}_set`, fields); + const literals = fields.map((field) => `ptr ${host.internLiteral(field.name)}`); host.resolveThunkDefs.push( `define internal zeroext i1 @${has}(ptr %p, ${host.sizeType} %i) ${FN_ATTRS} { ; live field presence of ${typeKey(t)}`, @@ -1494,13 +1498,90 @@ function liveRecordOps( G.render(), `}`, ``, + ...setDefs, `@${names} = internal constant [${fields.length} x ptr] [${literals.join(", ")}]`, - // Mutable: the runtime fills the trailing key-array slot on first use. - `@${ops} = internal global { ${host.sizeType}, ptr, ptr, ptr, i8, ptr, ptr } { ${host.sizeType} ${fields.length}, ptr @${names}, ptr @${has}, ptr @${get}, i8 ${optional.length === 0 ? 1 : 0}, ptr null, ptr null }`, + // Mutable: the runtime fills the key-array slot on first use. + `@${ops} = internal global { ${host.sizeType}, ptr, ptr, ptr, i8, ptr, ptr, ptr } { ${host.sizeType} ${fields.length}, ptr @${names}, ptr @${has}, ptr @${get}, i8 ${optional.length === 0 ? 1 : 0}, ptr null, ptr null, ptr ${setDefs.length ? `@${snapshot}_set` : "null"} }`, ``, ); } +/** The `set` hook of a live record's ops table: stores a dynamic write of + * present field `index` into the native record in place, so the capsule + * never becomes a snapshot for it. A value the field type does not accept + * answers false with no side effects, leaving the snapshot path to raise + * the commit's diagnostic; an inline union field always does. */ +function liveRecordSet( + host: LlvmEmitterContext, + t: IrType & { kind: "record" }, + sym: string, + fields: readonly IrRecordShape["fields"][number][], +): string[] { + const shape = host.recordsById.get(t.shapeId)!; + const struct = mangleRecordStruct(t.shapeId); + const S = new BlockBuilder(); + const labels = fields.map(() => S.newLabel("live.record.store")); + const refuse = S.newLabel("live.record.refuse"); + S.terminate( + `switch ${host.sizeType} %i, label %${refuse} [ ${fields + .map((_, position) => `${host.sizeType} ${position}, label %${labels[position]}`) + .join(" ")} ]`, + ); + fields.forEach((field, position) => { + S.startBlock(labels[position]!); + const inline = host.inlineUnions.record(field.type); + if (inline) { + S.br(refuse); + return; + } + const fits = S.tmp(); + S.line(`${fits} = call zeroext i1 @${host.dyn.dynMatchHelper(field.type)}(ptr %v)`); + const check = S.newLabel("live.record.check"); + S.condBr(fits, check, refuse); + S.startBlock(check); + const valueTy = host.llType(field.type); + const next = S.tmp(); + S.line( + `${next} = call ${valueTy === "i1" ? "zeroext " : ""}${valueTy} @${host.dyn.dynCheckHelper(field.type)}(ptr %v, ptr null)`, + ); + // A conversion that throws has already raised the write's error. + const pending = S.tmp(); + for (const line of host.pendingTestLines(pending)) S.line(line.trimStart()); + const write = S.newLabel("live.record.write"); + const done = S.newLabel("live.record.done"); + S.condBr(pending, done, write); + S.startBlock(write); + const index = shape.fields.indexOf(field) + host.nullableUnions.fieldBase(t.shapeId); + const fieldPtr = S.tmp(); + S.line(`${fieldPtr} = getelementptr inbounds %${struct}, ptr %p, i64 0, i32 ${index}`); + const fieldTy = llFieldType(field.type, false); + let stored = next; + if (fieldTy === "i8") { + stored = S.tmp(); + S.line(`${stored} = zext i1 ${next} to i8`); + } + if (isRefCounted(field.type)) { + const old = S.tmp(); + S.line(`${old} = load ptr, ptr ${fieldPtr}`); + S.line(`store ptr ${stored}, ptr ${fieldPtr} ; takes ownership`); + S.line(`call void ${releaseSym(host.shapeHost, field.type)}(ptr ${old})`); + } else { + S.line(`store ${fieldTy} ${stored}, ptr ${fieldPtr}`); + } + S.br(done); + S.startBlock(done); + S.terminate(`ret i1 true`); + }); + S.startBlock(refuse); + S.terminate(`ret i1 false`); + return [ + `define internal zeroext i1 @${sym}(ptr %p, ${host.sizeType} %i, ptr %v) ${FN_ATTRS} { ; live field store into ${typeKey(t)}`, + S.render(), + `}`, + ``, + ]; +} + /** `@_rops` for a shape with orderVariants: a record-ops table * whose `select` hook answers the `_o_rops` table (liveRecordOps) of the * record's current key order, which a write of an optional key can change @@ -1524,7 +1605,7 @@ function liveRecordOrderSelect( B.render(), `}`, ``, - `@${snapshot}_rops = internal global { ${host.sizeType}, ptr, ptr, ptr, i8, ptr, ptr } { ${host.sizeType} 0, ptr null, ptr null, ptr null, i8 0, ptr null, ptr @${snapshot}_select }`, + `@${snapshot}_rops = internal global { ${host.sizeType}, ptr, ptr, ptr, i8, ptr, ptr, ptr } { ${host.sizeType} 0, ptr null, ptr null, ptr null, i8 0, ptr null, ptr @${snapshot}_select, ptr null }`, ``, ); } diff --git a/packages/compiler/src/backend/llvm/int32-kind-loops.ts b/packages/compiler/src/backend/llvm/int32-kind-loops.ts index 12e38c1f0..d596a731d 100644 --- a/packages/compiler/src/backend/llvm/int32-kind-loops.ts +++ b/packages/compiler/src/backend/llvm/int32-kind-loops.ts @@ -57,6 +57,8 @@ function isPure(e: IrExpr): boolean { case "arrayGet": case "arrayHas": return e.arr.kind === "varRef" && isPure(e.index); + case "arrIntrinsic": + return e.method === "getNumber" && e.receiver.kind === "varRef" && isPure(e.args[0]!); default: return false; } diff --git a/packages/compiler/src/backend/llvm/integer-clones.ts b/packages/compiler/src/backend/llvm/integer-clones.ts index 314eae5ea..0ea3947b2 100644 --- a/packages/compiler/src/backend/llvm/integer-clones.ts +++ b/packages/compiler/src/backend/llvm/integer-clones.ts @@ -3,6 +3,7 @@ import type { IrExpr, IrFunction, IrLocal, IrModule, IrStmt, IrType } from "../. import { mangleFunction } from "../mangle.js"; import { BlockBuilder } from "./blocks.js"; import { SCR_ARR_FIELD } from "./common.js"; +import { SCR_ELEM_F64, SCR_ELEM_I32 } from "./dense-array-access.js"; /** Speculative integer clones of pure number-valued functions. * @@ -191,6 +192,8 @@ function cloneShape(fn: IrFunction, borrowed?: BorrowedParameter): Set | case "ternary": return expr(e.cond, "bool") && expr(e.then, want) && expr(e.else_, want); case "arrIntrinsic": + if (e.method === "getNumber") + return want === "num" && arrayRef(e.receiver) && expr(e.args[0]!, "num"); return ( want === "num" && e.method === "length" && e.args.length === 0 && arrayRef(e.receiver) ); @@ -268,6 +271,9 @@ function callsMatch(fn: IrFunction, byName: ReadonlyMap): bo case "arrayGet": visit(e.index); return; + case "arrIntrinsic": + if (e.method === "getNumber") visit(e.args[0]!); + return; default: return; } @@ -432,6 +438,8 @@ function callsAnything(e: IrExpr): boolean { return callsAnything(e.cond) || callsAnything(e.then) || callsAnything(e.else_); case "arrayGet": return callsAnything(e.index); + case "arrIntrinsic": + return e.method === "getNumber" && callsAnything(e.args[0]!); default: return false; } @@ -1020,10 +1028,10 @@ class CloneEmitter { * keeps every slot below `filled` a present value) whose element is * exactly an int32. Everything else (holes, indices outside the prefix, * other numbers) bails: the ordinary body reads it. */ - private emitElement(e: Extract): string { + private emitElement(arr: IrExpr, indexExpr: IrExpr): string { const { B } = this; - const array = this.emitArray(e.arr); - const index = this.narrow(this.emitNum(e.index)); + const array = this.emitArray(arr); + const index = this.narrow(this.emitNum(indexExpr)); const filledField = B.tmp(); const filled = B.tmp(); const inside = B.tmp(); @@ -1044,9 +1052,39 @@ class CloneEmitter { ); B.line(`${data} = load ptr, ptr ${dataField}`); B.line(`${offset} = zext i32 ${index} to i64`); + // A number array keeps 8-byte doubles, or 4-byte int32 slots when it is + // proven to hold only int32s (SCR_ELEM_I32); any other storage bails. + const kindField = B.tmp(); + const kind = B.tmp(); + const isInt32 = B.tmp(); + B.line( + `${kindField} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.elem}`, + ); + B.line(`${kind} = load i8, ptr ${kindField}`); + B.line(`${isInt32} = icmp eq i8 ${kind}, ${SCR_ELEM_I32}`); + const int32Slots = B.newLabel("elem.i32"); + const doubleSlots = B.newLabel("elem.f64"); + const join = B.newLabel("elem.join"); + B.condBr(isInt32, int32Slots, doubleSlots); + B.startBlock(int32Slots); + const int32Slot = B.tmp(); + const int32 = B.tmp(); + B.line(`${int32Slot} = getelementptr inbounds i32, ptr ${data}, i64 ${offset}`); + B.line(`${int32} = load i32, ptr ${int32Slot}`); + B.br(join); + B.startBlock(doubleSlots); + const isDouble = B.tmp(); + B.line(`${isDouble} = icmp eq i8 ${kind}, ${SCR_ELEM_F64}`); + this.require(isDouble); B.line(`${slot} = getelementptr inbounds double, ptr ${data}, i64 ${offset}`); B.line(`${number} = load double, ptr ${slot}`); - return this.exactInt32(number); + const exact = this.exactInt32(number); + const doubleEnd = B.currentLabel(); + B.br(join); + B.startBlock(join); + const value = B.tmp(); + B.line(`${value} = phi i32 [ ${int32}, %${int32Slots} ], [ ${exact}, %${doubleEnd} ]`); + return value; } /** The int32 the double `number` equals bit for bit (never -0, NaN, a @@ -1157,9 +1195,13 @@ class CloneEmitter { if (this.inlines(e)) return { name: this.emitInlineCall(e), wide: false }; return { name: this.emitCall(e), wide: true }; case "arrIntrinsic": + // `getNumber` reads an element like `arrayGet`: a missing one bails, + // and the ordinary body reads it as NaN. + if (e.method === "getNumber") + return { name: this.emitElement(e.receiver, e.args[0]!), wide: false }; return { name: this.emitLength(this.emitArray(e.receiver)), wide: false }; case "arrayGet": - return { name: this.emitElement(e), wide: false }; + return { name: this.emitElement(e.arr, e.index), wide: false }; case "unary": { const operand = this.narrow(this.emitNum(e.operand)); const value = B.tmp(); diff --git a/packages/compiler/src/backend/llvm/js-ic.ts b/packages/compiler/src/backend/llvm/js-ic.ts index d51b838bb..c9786c9f2 100644 --- a/packages/compiler/src/backend/llvm/js-ic.ts +++ b/packages/compiler/src/backend/llvm/js-ic.ts @@ -4,13 +4,12 @@ * A property site in a dynamic-tier function owns one zeroed thread-local * ScrJsIc record (`@sc_js_ic_N`). The code at the site is fixed: * - * load ordinary object whose shape is entry 0's own-property shape: - * read entry `slot` inline unless it is an accessor; otherwise - * scr_js_get_probe (polymorphic, prototype hits, absent keys, - * misses) - * store ordinary object whose shape is entry 0's replace shape, outside - * an intact parse region, writable data entry: replace inline; - * otherwise scr_js_store_probe (adds, polymorphic, misses) + * load ordinary object whose layout is entry 0's own-property layout: + * read slot `slot` inline; otherwise scr_js_get_probe + * (polymorphic, prototype hits, absent keys, misses) + * store ordinary object whose layout is entry 0's replace layout, + * outside an intact parse region: replace inline; otherwise + * scr_js_store_probe (adds, polymorphic, misses) * keyed dense array element at an integral number key: inline; * otherwise scr_js_keyed_probe (string keys of ordinary objects) * method dynInvoke's method read: scr_js_load_probe @@ -22,16 +21,19 @@ * defines the operation's semantics; nothing is ever patched. * * Layout facts (64-bit only; 32-bit targets keep the helper calls): - * ScrDyn kind +8; v.obj {len +16, entries +32, region +72, - * shape +80}; v.arr {len +16, items +32, presence +56}; - * v.num +16 - * ScrDynEntry 48 bytes: value +16, accessor bit 0 of +40, writable +41 - * ScrJsIc entries of 40 bytes {shape, target, holder, holder_shape, - * slot}; 168 bytes in all */ -import { BOOL, DYN, F64, STRING, type IrExpr, type IrFunction } from "../../ir/ir.js"; + * ScrDyn kind +8; v.obj {len +16, slots +32, region +72, + * layout +80}; v.arr {len +16, items +32, presence +56}; + * v.num +16. Every property of an object with a layout is + * a writable data property (scr_dyn.h's layout mode); + * dictionary-mode objects have a null layout. + * v.typed_ref {ptr +16, type_key +40}. + * ScrJsIc entries of 40 bytes {layout, target, holder, + * holder_layout, slot}; 168 bytes in all */ +import { BOOL, DYN, F64, STRING, type IrExpr, type IrFunction, type IrStmt } from "../../ir/ir.js"; import { jsIcMethodSites, type JsIcSite } from "../../ir/js-ic.js"; import { mangleBorrowedFunction } from "../mangle.js"; import { emitBorrowedInput } from "./borrowed-inputs.js"; +import { classFieldBase, classFieldIndex } from "./classes.js"; import { DYN_KIND } from "./dyn.js"; import type { ExprOf, LlValue, LlvmEmitterContext } from "./expr-context.js"; import { JS_BITS } from "./jsval-encoding.js"; @@ -42,8 +44,6 @@ type Host = LlvmEmitterContext; /** Words of one ScrJsIc record. */ export const JS_IC_WORDS = 21; -const ENTRY_STRIDE = 48; - /** The cache site a call is, when it is emitted in a dynamic-tier function * of a 64-bit target. */ export function jsIcSiteOf(host: Host, e: ExprOf<"call">): JsIcSite | null { @@ -86,34 +86,22 @@ function kindIs(host: Host, d: string, kind: number, yes: string, no: string): v * inline made polymorphic reads 7-10% faster (the design prototype's W2/W3) * and monomorphic method code no faster, for 65-225 more bytes per site; * the prototype hit made method-heavy code 3-6% faster for 15-90 bytes. - * So a site tests entry 0, own or prototype, and its probe does the rest. - * (0 makes every access probe: with a runtime built with SCR_JS_VERIFY, - * that checks every cached shape against its object.) */ -const INLINE_WAYS: number = 1; + * Two entries cover the common two-layout site (object literals of one + * record built in two key orders): on bench:runtime's dynamic-properties + * they made the run 13% faster than one entry, and 26% faster without its + * class instances; four were no faster than two. So a site tests entries + * 0 and 1, own or prototype, and its probe does the rest. (0 makes every access probe: with a runtime + * built with SCR_JS_VERIFY, that probes every access.) */ +const INLINE_WAYS: number = 2; const INLINE_PROTO = true; -/** The address of entry `slot` of object `d`'s entries. */ -function entryAt(host: Host, d: string, slot: string): string { +/** The address of slot `slot` of layout-mode object `d`. */ +function slotAt(host: Host, d: string, slot: string): string { const B = host.B; - const entries = load(host, "ptr", d, 32); - const offset = B.tmp(); - const entry = B.tmp(); - B.line(`${offset} = mul i64 ${slot}, ${ENTRY_STRIDE}`); - B.line(`${entry} = getelementptr inbounds i8, ptr ${entries}, i64 ${offset}`); - return entry; -} - -/** Branches to `miss` when the entry at `entry` is an accessor. */ -function dataEntry(host: Host, entry: string, miss: string): void { - const B = host.B; - const flags = load(host, "i8", entry, 40); - const accessor = B.tmp(); - const isAccessor = B.tmp(); - B.line(`${accessor} = and i8 ${flags}, 1`); - B.line(`${isAccessor} = icmp ne i8 ${accessor}, 0`); - const data = B.newLabel("ic.data"); - B.condBr(isAccessor, miss, data); - B.startBlock(data); + const slots = load(host, "ptr", d, 32); + const address = B.tmp(); + B.line(`${address} = getelementptr inbounds ptr, ptr ${slots}, i64 ${slot}`); + return address; } /** The inline part of a load site, entered for an ordinary object `d`: @@ -137,7 +125,7 @@ function inlineLoad( B.br(miss); return; } - const shape = load(host, "ptr", d, 80); + const shape = load(host, "ptr", d, 80); // v.obj.layout const present = B.tmp(); B.line(`${present} = icmp ne ptr ${shape}, null`); let next = B.newLabel("ic.w"); @@ -160,9 +148,7 @@ function inlineLoad( B.condBr(own, ownLabel, protoLabel); const hit = (holder: string): void => { const slot = load(host, "i64", ic, base + 32); - const entry = entryAt(host, holder, slot); - dataEntry(host, entry, miss); - const value = load(host, "ptr", entry, 16); + const value = load(host, "ptr", slotAt(host, holder, slot), 0); const out = result(value); incoming.push(`[ ${out}, %${B.currentLabel()} ]`); B.br(join); @@ -194,7 +180,7 @@ function inlineLoad( /** The inline part of a store site that replaces an own data property, * entered for an ordinary object `d` outside an intact parse region: * tests the first `ways` cache entries; a hit calls `store` with the - * address of the entry's value word, which must end its block by + * address of the property's slot, which must end its block by * branching to the site's join. */ function inlineReplace( host: Host, @@ -209,7 +195,7 @@ function inlineReplace( B.br(miss); return; } - const shape = load(host, "ptr", d, 80); + const shape = load(host, "ptr", d, 80); // v.obj.layout const present = B.tmp(); B.line(`${present} = icmp ne ptr ${shape}, null`); let next = B.newLabel("ic.w"); @@ -230,21 +216,7 @@ function inlineReplace( B.condBr(ok, matched, next); B.startBlock(matched); const slot = load(host, "i64", ic, base + 32); - const entry = entryAt(host, d, slot); - const flags = load(host, "i8", entry, 40); - const writable = load(host, "i8", entry, 41); - const accessor = B.tmp(); - const data = B.tmp(); - const canWrite = B.tmp(); - const writes = B.tmp(); - B.line(`${accessor} = and i8 ${flags}, 1`); - B.line(`${data} = icmp eq i8 ${accessor}, 0`); - B.line(`${canWrite} = icmp ne i8 ${writable}, 0`); - B.line(`${writes} = and i1 ${data}, ${canWrite}`); - const hit = B.newLabel("ic.st"); - B.condBr(writes, hit, miss); - B.startBlock(hit); - store(gep(host, entry, 16)); + store(slotAt(host, d, slot)); } } @@ -347,6 +319,222 @@ function keyString(host: Host, key: string, e: IrExpr): string { return host.emitExpr({ kind: "strLit", value: key, type: STRING, loc: e.loc }).name; } +/** A class instance's answer that a read helper gives by loading one + * declared field: the branch `if (dyn.typedRefIs(p, KEY)) { r = + * dynCheck(p); return [js.fromDyn](fieldGet C.f(r)) }`. */ +interface CapsuleRead { + key: string; + className: string; + field: string; +} + +/** The class branches of a read helper that a site can answer inline for + * capsules (class instances boxed as dyn values), or none. + * + * The helper first normalizes its receiver (`%dyn.class.normalize` + * reboxes a capsule keyed with a base class whose instance belongs to a + * subclass), then tests each class branch in turn. The site loads the + * field of a capsule whose key is a branch's key without normalizing, so + * a branch qualifies only when every subclass the normalization can + * rebox its key to has a branch that loads the same field from the same + * slot of a layout that extends the class's. Every other capsule, and + * every helper or normalization of another form, keeps the helper. */ +function capsuleReads(host: Host, helper: IrFunction): CapsuleRead[] { + const receiver = helper.params[0]?.localId; + if (receiver === undefined || helper.params.length !== 1) return []; + const reads = new Map(); + for (const stmt of helper.body) { + if (normalizes(stmt, receiver)) continue; + if (stmt.kind !== "if" || stmt.else_ !== null) break; + if (notCapsule(stmt.cond, receiver)) continue; + const key = brandKey(stmt.cond, receiver); + if (key === null) return []; + if (!reads.has(key)) reads.set(key, capsuleRead(stmt, receiver, key)); + } + const reboxes = normalizationTargets(host); + if (reboxes === null) return []; + const out: CapsuleRead[] = []; + for (const read of reads.values()) { + if (read === null || !plainField(host, read.className, read.field)) continue; + const targets = reboxes.get(read.key) ?? []; + const same = targets.every((target) => { + const other = reads.get(`object:${target}`); + return ( + other != null && + other.field === read.field && + plainField(host, target, read.field) && + extendsLayout(host, target, read.className, read.field) + ); + }); + if (same) out.push(read); + } + return out; +} + +/** `p = %dyn.class.normalize(p)`. */ +function normalizes(stmt: IrStmt, receiver: string): boolean { + return ( + stmt.kind === "assign" && + stmt.localId === receiver && + stmt.value.kind === "call" && + stmt.value.callee === "%dyn.class.normalize" && + stmt.value.args.length === 1 && + isLocal(stmt.value.args[0]!, receiver) + ); +} + +/** `!classCapsule(p)`: false for every capsule of a compiled class, so a + * capsule never enters the branch it guards. */ +function notCapsule(cond: IrExpr, receiver: string): boolean { + return ( + cond.kind === "dynTest" && + cond.test === "classCapsule" && + cond.negated === true && + isLocal(cond.value, receiver) + ); +} + +function isLocal(e: IrExpr, id: string): boolean { + return e.kind === "varRef" && e.localId === id; +} + +/** The capsule key `dyn.typedRefIs(p, "object:C")` tests, or null. */ +function brandKey(cond: IrExpr, receiver: string): string | null { + if (cond.kind !== "libCall" || cond.fn !== "dyn.typedRefIs" || cond.args.length !== 2) + return null; + const [value, key] = cond.args; + if (!isLocal(value!, receiver) || key?.kind !== "strLit" || !key.value.startsWith("object:")) + return null; + return key.value; +} + +/** The field a class branch loads, or null for a branch of another form. */ +function capsuleRead( + branch: IrStmt & { kind: "if" }, + receiver: string, + key: string, +): CapsuleRead | null { + const [decl, ret] = branch.then; + if (branch.then.length !== 2 || decl?.kind !== "varDecl" || ret?.kind !== "return") return null; + if (decl.init?.kind !== "dynCheck" || !isLocal(decl.init.value, receiver)) return null; + let value = ret.value; + if (value?.kind === "libCall" && value.fn === "js.fromDyn" && value.args.length === 1) + value = value.args[0]!; + if (value?.kind !== "fieldGet" || !isLocal(value.obj, decl.localId)) return null; + if (key !== `object:${value.className}`) return null; + return { key, className: value.className, field: value.field }; +} + +/** For each capsule key `%dyn.class.normalize` reboxes, the classes it + * may rebox to; null when the normalization has another form. */ +function normalizationTargets(host: Host): Map | null { + const targets = new Map(); + const normalize = host.fnByName.get("%dyn.class.normalize"); + if (normalize === undefined) return targets; + const value = normalize.params[0]?.localId; + if (value === undefined) return null; + for (const stmt of normalize.body) { + if (stmt.kind === "return") { + if (stmt.value !== null && isLocal(stmt.value, value)) continue; + return null; + } + if (stmt.kind !== "if" || stmt.else_ !== null) return null; + if (notCapsule(stmt.cond, value)) continue; + const key = brandKey(stmt.cond, value); + if (key === null) return null; + const [decl, ...checks] = stmt.then; + if (decl?.kind !== "varDecl" || decl.init?.kind !== "dynCheck") return null; + const into = targets.get(key) ?? []; + for (const check of checks) { + if ( + check.kind !== "if" || + check.else_ !== null || + check.cond.kind !== "instanceOf" || + !isLocal(check.cond.value, decl.localId) + ) + return null; + into.push(check.cond.className); + } + targets.set(key, into); + } + return targets; +} + +/** A declared field of a compiled class that holds a plain dyn cell + * pointer: not compact, compressed, int32 or an inline union. */ +function plainField(host: Host, className: string, field: string): boolean { + const meta = host.classMeta.get(className); + if (meta === undefined || meta.compact) return false; + const decl = meta.def.fields.find((f) => f.name === field); + if (decl === undefined || decl.type.kind !== "dyn") return false; + if (classFieldIndex(meta, field, "compressed").compressed) return false; + return ( + !host.int32Slots.isField(className, field) && !host.inlineUnions.classField(className, field) + ); +} + +/** Whether `sub`'s layout starts with `base`'s through `field`: the same + * header and the same fields in the same slots, so a load through + * `base`'s struct reads `sub`'s field. */ +function extendsLayout(host: Host, sub: string, base: string, field: string): boolean { + const a = host.classMeta.get(base); + const b = host.classMeta.get(sub); + if (a === undefined || b === undefined || b.compact) return false; + if (classFieldBase(a) !== classFieldBase(b)) return false; + const end = a.def.fields.findIndex((f) => f.name === field); + if (end < 0 || end >= b.def.fields.length) return false; + for (let i = 0; i <= end; i++) { + const x = a.def.fields[i]!; + const y = b.def.fields[i]!; + if (x.name !== y.name || x.type.kind !== y.type.kind) return false; + } + return ( + classFieldIndex(a, field, "compressed").index === classFieldIndex(b, field, "compressed").index + ); +} + +/** The inline part of a read site for a capsule `d`: a capsule whose key + * is one of `reads`' keys answers with the (borrowed) field cell, branching + * to `join` and recording its phi input; anything else, including a field + * not yet set, branches to `miss`. */ +function inlineCapsuleLoad( + host: Host, + d: string, + reads: readonly CapsuleRead[], + incoming: string[], + join: string, + miss: string, +): void { + const B = host.B; + const capsule = B.newLabel("icc.cap"); + kindIs(host, d, DYN_KIND.TYPED_REF, capsule, miss); + B.startBlock(capsule); + // Capsules boxed by this program share its key constants. + const key = load(host, "ptr", d, 40); + let next = B.newLabel("icc.k"); + B.br(next); + reads.forEach((read, i) => { + B.startBlock(next); + next = i + 1 < reads.length ? B.newLabel("icc.k") : miss; + const same = B.tmp(); + B.line(`${same} = icmp eq ptr ${key}, ${host.cstr(read.key)}`); + const matched = B.newLabel("icc.m"); + B.condBr(same, matched, next); + B.startBlock(matched); + const instance = load(host, "ptr", d, 16); + const { ptr } = host.classFieldPtr(instance, read.className, read.field); + const cell = B.tmp(); + B.line(`${cell} = load ptr, ptr ${ptr}`); + const set = B.tmp(); + B.line(`${set} = icmp ne ptr ${cell}, null`); + const hit = B.newLabel("icc.hit"); + B.condBr(set, hit, miss); + B.startBlock(hit); + incoming.push(`[ ${cell}, %${B.currentLabel()} ]`); + B.br(join); + }); +} + /** Emits a call to a classified helper as a cache site. */ export function emitJsIcCall( host: Host, @@ -401,9 +589,15 @@ function emitGet(host: Host, e: ExprOf<"call">, callee: IrFunction, key: string) // conversion to the result type. const hit = B.newLabel("icg.hit"); const hits: string[] = []; - kindIs(host, receiver.name, DYN_KIND.OBJ, object, probe); + const reads = capsuleReads(host, callee); + const other = reads.length > 0 ? B.newLabel("icg.other") : probe; + kindIs(host, receiver.name, DYN_KIND.OBJ, object, other); B.startBlock(object); inlineLoad(host, receiver.name, ic, INLINE_WAYS, INLINE_PROTO, (cell) => cell, hits, hit, probe); + if (reads.length > 0) { + B.startBlock(other); + inlineCapsuleLoad(host, receiver.name, reads, hits, hit, probe); + } B.startBlock(probe); // The read's probe also answers keys the receiver's plain chain lacks. // Only `length` and `next` reads can be answered for a native array or diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 7432debb4..eb2acf706 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -218,7 +218,7 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll host.emitCoroAwait(null); return { name: "", type: e.type }; } - if (host.wasi && e.fn === "async.awaitDyn") { + if (host.coroAwait && e.fn === "async.awaitDyn") { const value = host.emitExpr(e.args[0]!); host.declare(`declare ptr @scr_dyn_promise_of(ptr)`); const promise = B.tmp(); @@ -230,15 +230,16 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll B.line(`${isPromise} = icmp ne ptr ${promise}, null`); B.condBr(isPromise, wait, hop); B.startBlock(wait); - host.emitWasiSuspend(promise); + host.emitCoroAwait(promise); B.br(ready); B.startBlock(hop); - host.emitWasiSuspend(null); + host.emitCoroAwait(null); B.br(ready); B.startBlock(ready); - host.declare(`declare ptr @scr_await_dyn_value_settled(ptr)`); + const extract = host.wasi ? "scr_await_dyn_value_settled" : "scr_coro_await_dyn_value"; + host.declare(`declare ptr @${extract}(ptr)`); const result = B.tmp(); - B.line(`${result} = call ptr @scr_await_dyn_value_settled(ptr ${value.name})`); + B.line(`${result} = call ptr @${extract}(ptr ${value.name})`); const output = host.own({ name: result, type: e.type }); host.emitPendingCheck(); return output; @@ -755,7 +756,10 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll // stream behind it (scr_dyn_for_await_stream.c) e.fn === "dyn.asyncIteratorOpen" && host.dynForAwaitStreams ? "scr_stream_dyn_async_iterator_open" - : LIB_FN_SYMS[e.fn]); + : (e.fn === "dyn.keySet" || e.fn === "dyn.keySetComputed") && + host.sloppyDynWrites.has(`${e.loc.file}\0${e.loc.start}\0${e.loc.end}`) + ? `${LIB_FN_SYMS[e.fn]}_sloppy` + : LIB_FN_SYMS[e.fn]); if (sym === undefined) throw new LlvmUnsupportedError(`libCall:${e.fn}`, e.loc); // The runtime's Date functions take and answer time values: a Date // operand passes its time value, read at once from the borrowed object, diff --git a/packages/compiler/src/backend/llvm/loop-callees.test.ts b/packages/compiler/src/backend/llvm/loop-callees.test.ts new file mode 100644 index 000000000..f918af944 --- /dev/null +++ b/packages/compiler/src/backend/llvm/loop-callees.test.ts @@ -0,0 +1,83 @@ +import { expect, test } from "vitest"; +import { + BOOL, + F64, + VOID, + type IrExpr, + type IrFunction, + type IrModule, + type IrStmt, +} from "../../ir/ir.js"; +import { findLoopCallees } from "./loop-callees.js"; + +const loc = { file: "callees.ts", start: 0, end: 0 }; +const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc }); +const ref = (localId: string): IrExpr => ({ kind: "varRef", localId, type: F64, loc }); +const call = (callee: string): IrExpr => ({ kind: "call", callee, args: [num(1)], type: F64, loc }); +const run = (e: IrExpr): IrStmt => ({ kind: "exprStmt", expr: e, loc }); +const ret = (e: IrExpr): IrStmt => ({ kind: "return", value: e, loc }); +const loop = (body: IrStmt[]): IrStmt => ({ + kind: "while", + cond: { kind: "bin", op: "<", left: ref("i"), right: num(10), type: BOOL, loc }, + body, + loc, +}); +const fn = (name: string, body: IrStmt[], extra: Partial = {}): IrFunction => ({ + name, + params: [{ localId: "x", name: "x", type: F64 }], + returnType: F64, + locals: [{ id: "x", name: "x", type: F64, mutable: false }], + body, + loc, + ...extra, +}); +const module = (functions: IrFunction[]): IrModule => ({ + irVersion: 17, + sourceFile: loc.file, + entry: "__main", + functions: [ + { name: "__main", params: [], returnType: VOID, locals: [], body: [], loc }, + ...functions, + ], +}); + +test("a small leaf called only from loops is a loop callee", () => { + const callees = findLoopCallees( + module([ + fn("leaf", [loop([run(ref("x"))]), ret(ref("x"))]), + fn("caller", [loop([run(call("leaf"))]), loop([run(call("leaf"))]), ret(num(0))]), + ]), + ); + expect([...callees]).toEqual(["leaf"]); +}); + +test("a call outside a loop, a third call site, an own call or a large body keeps the call", () => { + const leaf = fn("leaf", [ret(ref("x"))]); + // A call site outside any loop. + expect( + findLoopCallees(module([leaf, fn("caller", [run(call("leaf")), loop([run(call("leaf"))])])])) + .size, + ).toBe(0); + // Three call sites. + const three = fn("caller", [loop([run(call("leaf")), run(call("leaf")), run(call("leaf"))])]); + expect(findLoopCallees(module([leaf, three])).size).toBe(0); + // A callee that calls another function. + const calling = fn("calling", [ret(call("leaf"))]); + const caller = fn("caller", [loop([run(call("calling"))])]); + // (leaf is then called outside a loop, from calling.) + expect(findLoopCallees(module([leaf, calling, caller])).size).toBe(0); + // Two loops, or more than 12 statements. + const twoLoops = fn("leaf", [loop([]), loop([]), ret(ref("x"))]); + expect(findLoopCallees(module([twoLoops, fn("caller", [loop([run(call("leaf"))])])])).size).toBe( + 0, + ); + const long = fn("leaf", [...Array.from({ length: 12 }, () => run(ref("x"))), ret(ref("x"))]); + expect(findLoopCallees(module([long, fn("caller", [loop([run(call("leaf"))])])])).size).toBe(0); + // Async functions and library modules. + const asyncLeaf = fn("leaf", [ret(ref("x"))], { async: true }); + expect(findLoopCallees(module([asyncLeaf, fn("caller", [loop([run(call("leaf"))])])])).size).toBe( + 0, + ); + const lib = { ...module([leaf, fn("caller", [loop([run(call("leaf"))])])]), lib: {} } as IrModule; + expect(findLoopCallees(lib).size).toBe(0); +}); diff --git a/packages/compiler/src/backend/llvm/loop-callees.ts b/packages/compiler/src/backend/llvm/loop-callees.ts new file mode 100644 index 000000000..a3d767dae --- /dev/null +++ b/packages/compiler/src/backend/llvm/loop-callees.ts @@ -0,0 +1,110 @@ +import type { IrExpr, IrFunction, IrModule, IrStmt } from "../../ir/ir.js"; +import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; + +/** Statements a callee may have, counting nested ones. */ +const MAX_STATEMENTS = 12; +/** Direct call sites a callee may have. */ +const MAX_CALL_SITES = 2; + +/** Expressions that run other code: a callee that has none calls nothing, + * so inlining it never inlines a call chain or a recursion. */ +const CALLS = new Set([ + "call", + "callValue", + "virtualCall", + "dynCall", + "dynInvoke", + "new", + "newValue", + "newPromise", + "ffiCall", + "awaitExpr", + "awaitUnionExpr", +]); + +function isLoop(s: IrStmt): boolean { + return s.kind === "for" || s.kind === "while" || s.kind === "doWhile" || s.kind === "forOf"; +} + +/** Small functions that are called only from inside loops: a per-element + * helper such as + * + * function pick(list: Gene[], rand: number): number { + * for (const gene of list) if (rand < gene.c) return gene.sc; + * return list[list.length - 1]!.sc; + * } + * + * called from a loop over the elements. After its loop versions and + * checks, such a function is often past LLVM's inline cost threshold, so + * every element pays a call, the spills around it, and checks that the + * caller's loop could hoist once inlined. + * + * A function qualifies when it is a synchronous, non-generator function + * with no captures and no own call (CALLS), has at most MAX_STATEMENTS + * statements and one loop, and has one to MAX_CALL_SITES direct call sites + * in the module, each inside a loop body of its caller. The emitter marks + * those calls `alwaysinline`. Functions the module exports as a library + * keep the cost model's decision. */ +export function findLoopCallees(mod: IrModule): Set { + const result = new Set(); + if (mod.lib !== undefined) return result; + const candidates = new Map(); + for (const fn of mod.functions) { + if (fn.async || fn.generator || fn.captures !== undefined || fn.classCaptures !== undefined) + continue; + let statements = 0; + let loops = 0; + let calls = false; + const expr = (e: IrExpr): boolean => { + if (CALLS.has(e.kind)) calls = true; + return !calls && everyExprChild(e, expr, stmt); + }; + const stmt = (s: IrStmt): boolean => { + statements++; + if (isLoop(s)) loops++; + return statements <= MAX_STATEMENTS && loops <= 1 && everyStmtChild(s, expr, stmt); + }; + if (!fn.body.every(stmt) || calls) continue; + candidates.set(fn.name, fn); + } + if (candidates.size === 0) return result; + // Call sites of each candidate, and whether every one is inside a loop. + const sites = new Map(); + for (const fn of mod.functions) { + let depth = 0; + const expr = (e: IrExpr): boolean => { + if (e.kind === "call" && candidates.has(e.callee)) { + const site = sites.get(e.callee) ?? { count: 0, inLoops: true }; + site.count++; + if (depth === 0 || e.callee === fn.name) site.inLoops = false; + sites.set(e.callee, site); + } + return everyExprChild(e, expr, stmt); + }; + const stmt = (s: IrStmt): boolean => { + if (!isLoop(s)) return everyStmtChild(s, expr, stmt); + // A loop's header runs per iteration too, except a for's init. + if (s.kind === "for" && s.init) stmt(s.init); + depth++; + if (s.kind === "for") { + if (s.cond) expr(s.cond); + if (s.update) stmt(s.update); + s.body.every(stmt); + } else if (s.kind === "forOf") { + depth--; + expr(s.iterable); + depth++; + s.body.every(stmt); + } else if (s.kind === "while" || s.kind === "doWhile") { + expr(s.cond); + s.body.every(stmt); + } + depth--; + return true; + }; + fn.body.every(stmt); + } + for (const [name, site] of sites) + if (site.inLoops && site.count >= 1 && site.count <= MAX_CALL_SITES) result.add(name); + return result; +} diff --git a/packages/compiler/src/backend/llvm/region-groups.test.ts b/packages/compiler/src/backend/llvm/region-groups.test.ts new file mode 100644 index 000000000..1e4e8dd0c --- /dev/null +++ b/packages/compiler/src/backend/llvm/region-groups.test.ts @@ -0,0 +1,205 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../../index.js"; +import type { IrExpr, IrModule, IrType } from "../../ir/ir.js"; +import { analyzeClosedGroups, analyzeRegionGroups } from "./region-groups.js"; +import { children, type Node } from "./region-ir.js"; + +async function moduleFor(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-region-groups-")); + try { + const entry = join(dir, "main.ts"), + output = join(dir, "main.ir.json"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath: output, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((item) => `${item.code}: ${item.message}`).join("\n")); + return deserializeModule(await readFile(output, "utf8")); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +/** The calls of `callee` in the module, in order. */ +function callsOf(mod: IrModule, callee: string): IrExpr[] { + const out: IrExpr[] = []; + const visit = (node: Node): void => { + if (node.kind === "call" && node.callee === callee) out.push(node); + const kids: Node[] = []; + children(node, kids); + for (const k of kids) visit(k); + }; + for (const fn of mod.functions) for (const s of fn.body) visit(s); + return out; +} + +// Every object and record is an arena type here, and every union a nullable +// pointer to one (the emitter decides both from layouts). +const everyObject = (t: IrType): boolean => + t.kind === "object" || t.kind === "record" || t.kind === "union"; + +const TREE = ` +class Node { + left: Node | null; + right: Node | null; + constructor(left: Node | null, right: Node | null) { + this.left = left; + this.right = right; + } + count(): number { + return 1 + (this.left === null ? 0 : this.left.count()) + (this.right === null ? 0 : this.right.count()); + } +} +function tree(depth: number): Node { + return depth === 0 ? new Node(null, null) : new Node(tree(depth - 1), tree(depth - 1)); +} +`; + +test("a builder whose result its caller only reads is a closed root", async () => { + const mod = await moduleFor(`${TREE} +let total = 0; +for (let d = 0; d < 4; d++) total += tree(d).count(); +console.log(total); +`); + const groups = analyzeRegionGroups(mod); + expect(groups.fresh.has("tree")).toBe(true); + const [root] = callsOf(mod, "tree").filter((c) => groups.roots.has(c)); + expect(root).toBeDefined(); + expect(groups.drops.get(root!)).toBeNull(); + expect(groups.reachable.has("tree")).toBe(true); + expect(analyzeClosedGroups(mod, groups, everyObject).has(root!)).toBe(true); +}); + +test("a result kept past its caller's frame is no root", async () => { + const mod = await moduleFor(`${TREE} +const kept: Node[] = []; +for (let d = 0; d < 4; d++) kept.push(tree(d)); +console.log(kept.length); +`); + const groups = analyzeRegionGroups(mod); + expect(callsOf(mod, "tree").some((c) => groups.roots.has(c))).toBe(false); +}); + +test("a function that stores what it allocates in a global is not leak-free", async () => { + const mod = await moduleFor(` +interface P { x: number } +const seen: P[] = []; +function note(x: number): void { + seen.push({ x }); +} +function sum(n: number): number { + let s = 0; + for (let i = 0; i < n; i++) { + const p = { x: i }; + s += p.x; + } + return s; +} +note(1); +console.log(sum(3), seen.length); +`); + const groups = analyzeRegionGroups(mod); + expect(groups.leakFree.has("note")).toBe(false); + expect(callsOf(mod, "note").some((c) => groups.calls.has(c))).toBe(false); + expect(groups.leakFree.has("sum")).toBe(true); +}); + +test("a leak-free call that returns no reference is a call region", async () => { + const mod = await moduleFor(` +interface V { x: number; y: number } +function add(a: V, b: V): V { + return { x: a.x + b.x, y: a.y + b.y }; +} +function walk(n: number): number { + let at: V = { x: 0, y: 0 }; + for (let i = 0; i < n; i++) at = add(at, { x: i, y: 1 }); + return at.x + at.y; +} +console.log(walk(10)); +`); + const groups = analyzeRegionGroups(mod); + const [region] = callsOf(mod, "walk"); + expect(groups.calls.has(region!)).toBe(true); +}); + +test("a graph that holds strings or outside objects is not closed", async () => { + const mod = await moduleFor(` +class Tag { + name: string; + constructor(name: string) { + this.name = name; + } +} +class Box { + tag: Tag; + owner: Tag | null; + constructor(tag: Tag, owner: Tag | null) { + this.tag = tag; + this.owner = owner; + } +} +const shared = new Tag("shared"); +function labelled(n: number): Box { + let box = new Box(new Tag("t" + n), null); + for (let i = 0; i < n; i++) box = new Box(new Tag("t" + i), null); + return box; +} +function owned(n: number): Box { + let box = new Box(new Tag("x"), shared); + for (let i = 0; i < n; i++) box = new Box(new Tag("x"), shared); + return box; +} +let n = 0; +for (let i = 0; i < 3; i++) n += labelled(i).tag.name.length + owned(i).tag.name.length; +console.log(n); +`); + const groups = analyzeRegionGroups(mod); + const closed = analyzeClosedGroups(mod, groups, everyObject); + const labelled = callsOf(mod, "labelled").filter((c) => groups.roots.has(c)); + const owned = callsOf(mod, "owned").filter((c) => groups.roots.has(c)); + expect(labelled.length).toBe(1); + expect(owned.length).toBe(1); + expect(closed.has(labelled[0]!)).toBe(false); + expect(closed.has(owned[0]!)).toBe(false); +}); + +test("a root bound in a loop body is dropped with that body's local", async () => { + const mod = await moduleFor(`${TREE} +function run(n: number): number { + let total = 0; + for (let i = 0; i < n; i++) { + const t = tree(i % 4); + total += t.count(); + } + return total; +} +console.log(run(10)); +`); + const groups = analyzeRegionGroups(mod); + const [root] = callsOf(mod, "tree").filter((c) => groups.roots.has(c)); + const run = mod.functions.find((f) => f.name === "run")!; + const local = groups.drops.get(root!); + expect(run.locals.find((l) => l.id === local)?.name).toBe("t"); +}); + +test("a root whose value a callee writes into is not closed", async () => { + const mod = await moduleFor(`${TREE} +function graft(n: Node, extra: Node): void { + n.left = extra; +} +const spare = new Node(null, null); +let total = 0; +for (let d = 0; d < 3; d++) { + const t = tree(d); + graft(t, spare); + total += t.count(); +} +console.log(total); +`); + const groups = analyzeRegionGroups(mod); + const closed = analyzeClosedGroups(mod, groups, everyObject); + for (const root of callsOf(mod, "tree").filter((c) => groups.roots.has(c))) + expect(closed.has(root)).toBe(false); +}); diff --git a/packages/compiler/src/backend/llvm/region-groups.ts b/packages/compiler/src/backend/llvm/region-groups.ts new file mode 100644 index 000000000..ce8354c92 --- /dev/null +++ b/packages/compiler/src/backend/llvm/region-groups.ts @@ -0,0 +1,1697 @@ +/* Region groups: call sites whose result is a fresh object graph the + * caller only borrows and then drops, so everything the call allocates + * can live in an arena that is dropped at once (arenas.ts). + * + * The group of a call is every object allocated while it runs. It can be + * dropped with its result when nothing outside reaches any member once the + * call returns except through the result. A function is leak-free when + * every value it allocates (an allocation, a fresh-producing builtin, or + * the result of a call) dies in its frame, is returned, or is stored into + * the graph of a value it returns, and every function it calls is + * leak-free too. Members may refer out of the group (to longer-lived + * objects, which an arena would release when it drops); nothing outside + * may refer in. A function is fresh when it is leak-free and returns only + * values it allocated. + * + * A root is a call of a fresh function (or a leak-free construction) whose + * value, and every reference read out of it, the caller keeps in its frame: + * deep parameter summaries, where reading a reference out of a parameter + * makes a value the callee must keep too. Everything is solved + * optimistically over the call graph and refined to a fixpoint. */ +import { + isRefCounted, + type IrExpr, + type IrFunction, + type IrModule, + type IrStmt, + type IrType, +} from "../../ir/ir.js"; +import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; +import { + ALLOCATIONS, + FRESH_ARRAY_METHODS, + argsOf, + children, + forEachNode, + indexOfNode, + isExpr, + keepsInFrame, + same, + refType, + transparent, + type Node, +} from "./region-ir.js"; +import { VirtualTargets } from "./reference-effects.js"; + +/** The module functions a virtual call can run, or null when unknown (or + * when `node` is no virtual call). */ +function virtualTargetsOf(vt: VirtualTargets, node: Node): readonly string[] | null { + return node.kind === "virtualCall" ? vt.targets(node.className, node.method) : null; +} + +export interface RegionGroups { + /** Fresh call and `new` expressions whose value the caller only borrows. */ + roots: Set; + /** Calls of leak-free functions that return no reference: everything + * they allocate is dead when they return. */ + calls: Set; + /** Where each root's arena is dropped: the local whose scope exit drops + * it, or null for the end of the statement. */ + drops: Map; + /** Functions whose allocations go to the open arena. */ + reachable: Set; + /** Roots and call regions whose groups are closed: every reference a + * member holds is to a member or an immortal value (analyzeClosedGroups + * fills it in). */ + closed: Set; + /** The locals that share a root's value in its function (empty for a + * value used in place), and the function of each root and call region. */ + rootLocals: Map>; + rootFunction: Map; + /** Functions whose results are fresh. */ + fresh: Set; + /** Functions that keep nothing they allocate except through their result. */ + leakFree: Set; + /** Functions that allocate, directly or through what they call. */ + allocates: Set; +} + +/** Projections whose result is a reference read out of the operand. + * Strings are leaves, not group members: reading one out of a member and + * keeping it anywhere keeps no member. */ +function projection(child: Node, parent: Node | undefined): boolean { + if (parent === undefined || !isExpr(parent) || !refType(parent.type)) return false; + switch (parent.kind) { + case "fieldGet": + case "recordGet": + case "recordKeyGet": + return same(parent.obj, child); + case "arrayGet": + return same(parent.arr, child); + case "unionKeyGet": + return same(parent.value, child); + case "mapIntrinsic": + case "setIntrinsic": + return ( + same(parent.receiver, child) && + (parent.method === "get" || parent.method === "iterKey" || parent.method === "iterValue") + ); + default: + return false; + } +} + +const LOOP_KINDS = new Set(["while", "doWhile", "for", "forOf"]); + +/** Collection methods that add their arguments to the receiver. */ +const ADDING_METHODS = new Set(["push", "unshift", "set", "add"]); + +export const throws = (e: IrExpr): boolean => e.kind === "libCall" && e.fn === "error.nodeThrow"; + +/** Receiver methods that read a collection or remove from it, and so add + * nothing to its graph. */ +const READ_RECEIVER = new Set([ + "get", + "has", + "delete", + "iterKey", + "iterValue", + "size", + "clear", + "length", + "at", + "includes", + "indexOf", + "lastIndexOf", + "indexEq", + "getNumber", + "join", + "nextPresent", + "slice", + "concatSpread", + "pop", + "shift", +]); + +/** The local a reference is read out of, through casts, checked reads and + * projections (`l.items[0]` → l), or the local itself. */ +function projectionBase(e: IrExpr, through?: (call: IrExpr) => IrExpr | null): string | null { + let at: IrExpr = e; + for (let i = 0; i < 64; i++) { + if (at.kind === "varRef") return at.localId; + if (at.kind === "call" && through !== undefined) { + const next = through(at); + if (next === null) return null; + at = next; + continue; + } + if (at.kind === "fieldGet" || at.kind === "recordGet" || at.kind === "recordKeyGet") + at = at.obj; + else if (at.kind === "arrayGet") at = at.arr; + else if (at.kind === "unionKeyGet") at = at.value; + else if ( + (at.kind === "mapIntrinsic" || at.kind === "setIntrinsic") && + (at.method === "get" || at.method === "iterKey" || at.method === "iterValue") + ) + at = at.receiver; + else if ( + at.kind === "unionNarrow" || + at.kind === "upcast" || + at.kind === "downcast" || + at.kind === "unionWrap" || + at.kind === "unionRetype" + ) + at = at.value; + else if (at.kind === "ternary" && throws(at.then)) at = at.else_; + else if (at.kind === "ternary" && throws(at.else_)) at = at.then; + else return null; + } + return null; +} + +/** The local whose graph a store into `target` writes (`l.f.items` → l). */ +export function storeTarget(target: IrExpr | undefined): string | null { + let at: IrExpr | undefined = target; + for (let i = 0; at !== undefined && i < 64; i++) { + if (at.kind === "varRef") return at.localId; + if (at.kind === "fieldGet" || at.kind === "recordGet" || at.kind === "recordKeyGet") + at = at.obj; + else if (at.kind === "arrayGet") at = at.arr; + else if (at.kind === "unionNarrow" || at.kind === "upcast" || at.kind === "downcast") + at = at.value; + // A checked read (`entry.skus` on `entry | undefined`) throws on one side. + else if (at.kind === "ternary" && throws(at.then)) at = at.else_; + else if (at.kind === "ternary" && throws(at.else_)) at = at.then; + else return null; + } + return null; +} + +/** An expression that produces a value it allocates (a group member). */ +export function source(e: Node): boolean { + if (!isExpr(e)) return false; + if (ALLOCATIONS.has(e.kind)) return true; + if (e.kind === "arrIntrinsic") return FRESH_ARRAY_METHODS.has(e.method); + if (e.kind === "strIntrinsic") return e.method === "split"; + if (e.kind === "libCall") return e.fn === "json.parse"; + if (e.kind === "dynCheck") return e.value.kind === "libCall" && e.value.fn === "json.parse"; + if (e.kind === "call" || e.kind === "callValue") return refType(e.type); + return false; +} + +/** Store-like uses: the value goes into the graph of `target`'s value. */ +export function storeOf(value: Node, parent: Node): IrExpr | undefined { + switch (parent.kind) { + case "fieldSet": + case "recordSet": + case "recordKeySet": + return same(parent.value, value) ? parent.obj : undefined; + case "arraySet": + return same(parent.value, value) ? parent.arr : undefined; + case "arrIntrinsic": + return (parent.method === "push" || parent.method === "unshift") && + indexOfNode(parent.args, value) >= 0 + ? parent.receiver + : undefined; + case "mapIntrinsic": + return parent.method === "set" && indexOfNode(parent.args, value) >= 0 + ? parent.receiver + : undefined; + case "setIntrinsic": + return parent.method === "add" && indexOfNode(parent.args, value) >= 0 + ? parent.receiver + : undefined; + default: + return undefined; + } +} + +/** Content positions: the value becomes part of the new value `parent`. */ +export function contentOf(value: Node, parent: Node): boolean { + switch (parent.kind) { + case "arrayLit": + case "recordLit": + case "new": + case "mapNew": + case "setNew": + return !same(parent, value); + case "recordClone": + return !same(parent.source, value); + default: + return false; + } +} + +/** Whether `parent` is a fresh array holding the elements of `child`, an + * array of references (`xs.slice()`, `a.concat(b)`): the elements share the + * fate of the copy. Strings are leaves, not members. */ +function copiesElements(child: Node, parent: Node | undefined): boolean { + if (parent === undefined || parent.kind !== "arrIntrinsic" || !isExpr(child)) return false; + if (!FRESH_ARRAY_METHODS.has(parent.method) && !SPREAD_METHODS.has(parent.method)) return false; + if ( + !same(parent.receiver, child) && + !(parent.method === "concatSpread" && indexOfNode(parent.args, child) >= 0) + ) + return false; + const t = child.type; + return t.kind === "array" && refType(t.elem); +} + +/** Array intrinsics that copy their array arguments' elements. */ +const SPREAD_METHODS = new Set(["pushSpread", "unshiftSpread", "concatSpread"]); + +/** Array intrinsics that call their function-typed arguments natively. */ +const CALLBACK_INTRINSICS = new Set(["sortDense", "sortValues"]); +const callbackArg = (child: Node, parent: Node | undefined): boolean => + parent !== undefined && + parent.kind === "arrIntrinsic" && + CALLBACK_INTRINSICS.has(parent.method) && + indexOfNode(parent.args, child) >= 0 && + isExpr(child) && + child.type.kind === "func"; + +type Use = "bound" | "kept" | "returned" | "stored" | "content" | "internal" | "thrown" | "escapes"; +interface UseInfo { + bound?: string; + /** For "internal": the index of the call in `parents`. */ + at?: number; + target?: string | null; +} + +export function analyzeRegionGroups(mod: IrModule): RegionGroups { + const fns = new Map(); + for (const fn of mod.functions) fns.set(fn.name, fn); + const vt = new VirtualTargets(mod.classes ?? []); + const suspends = (fn: IrFunction): boolean => fn.async === true || fn.generator !== undefined; + // Who refers to each function: by calls (direct, virtual, constructions), + // and by closures too. A function's summaries depend only on those of + // the functions it refers to, so the fixpoints below revisit just the + // referrers of a function whose summary changed. + const callers = new Map>(); + const referrers = new Map>(); + const directAlloc = new Set(); + for (const fn of mod.functions) { + const note = (map: Map>, name: string): void => { + let set = map.get(name); + if (set === undefined) map.set(name, (set = new Set())); + set.add(fn.name); + }; + forEachNode(fn.body, (node) => { + if (node.kind === "call") { + note(callers, node.callee); + note(referrers, node.callee); + } else if (node.kind === "virtualCall") + for (const t of virtualTargetsOf(vt, node) ?? []) { + note(callers, t); + note(referrers, t); + } + else if (node.kind === "new") { + const ctor = `%${node.className}.constructor`; + note(callers, ctor); + note(referrers, ctor); + } else if (node.kind === "closure") note(referrers, node.fnName); + if (source(node) && node.kind !== "call") directAlloc.add(fn.name); + }); + } + /** Runs `step` on the functions named, and again on the referrers of each + * whose summary it reports changed, until nothing changes. */ + const solve = ( + names: Iterable, + by: Map>, + step: (fn: IrFunction) => boolean, + ): void => { + const queue = [...names]; + const queued = new Set(queue); + while (queue.length > 0) { + const name = queue.pop()!; + queued.delete(name); + const fn = fns.get(name); + if (fn === undefined || !step(fn)) continue; + for (const r of by.get(name) ?? new Set()) + if (!queued.has(r)) { + queued.add(r); + queue.push(r); + } + } + }; + + // Deep summaries. + const deep = new Map>(); + // Internal parameters: ones a leak-free function only borrows, returns, or + // stores into the graphs of values it allocates (so an argument a caller + // allocated stays inside the caller's group). Optimistic, refined below. + const internal = new Map>(); + // Callback parameters (see below), and the function being analyzed. + const callbacks = new Map>(); + let current: IrFunction | null = null; + // While finding roots: a value passed to the function's own callback + // parameter leaves the frame (its callers' callbacks are unknown here). + let framePass = false; + const keeps = (child: Node, parent: Node | undefined): boolean => { + if (parent?.kind === "virtualCall") { + const index = indexOfNode(parent.args, child); + const targets = virtualTargetsOf(vt, parent); + return ( + index >= 0 && + targets !== null && + targets.length > 0 && + targets.every((t) => deep.get(t)?.has(index) === true) + ); + } + // A spread copies the array's elements into another array: they + // escape with it. + if ( + parent?.kind === "arrIntrinsic" && + SPREAD_METHODS.has(parent.method) && + !same(parent.receiver, child) && + isExpr(child) + ) { + const t = child.type; + return t.kind !== "array" || !refType(t.elem); + } + if (callbackArg(child, parent)) return true; + if (parent?.kind === "jsonStringify") return true; + // Comparisons only read their operands. + if (parent?.kind === "strCmp" || parent?.kind === "strEq") return true; + if (parent?.kind === "bin" && !refType(parent.type)) return true; + return keepsInFrame(child, parent, deep); + }; + + /** How the value of `node` (`parents` its ancestors) is used, after the + * expressions that pass it on (and, with `projections`, the reads of a + * reference out of it). */ + const useOf = (node: Node, parents: readonly Node[], projections: boolean, out: UseInfo): Use => { + let value: Node = node; + let at = parents.length - 1; + while ( + at >= 0 && + (transparent(value, parents[at]) || + copiesElements(value, parents[at]) || + (projections && projection(value, parents[at]))) + ) + value = parents[at--]!; + const p = at >= 0 ? parents[at] : undefined; + if (p === undefined) return "escapes"; + if ( + (p.kind === "varDecl" && p.init !== null && same(p.init, value)) || + (p.kind === "assign" && same(p.value, value)) + ) { + out.bound = p.localId; + return "bound"; + } + if (p.kind === "return") return "returned"; + if (p.kind === "throw") return "thrown"; + if (p.kind === "exprStmt") return "kept"; + const target = storeOf(value, p); + if (target !== undefined) { + out.target = storeTarget(target); + return "stored"; + } + if (contentOf(value, p)) return "content"; + if (keeps(value, p)) return "kept"; + if (p.kind === "callValue" && current !== null && !framePass) { + const i = indexOfNode(p.args, value) >= 0 ? paramIndex(current, p.callee) : -1; + if (i >= 0 && callbacks.get(current.name)!.has(i)) return "kept"; + } + if (p.kind === "call") { + const c = p; + const i = indexOfNode(c.args, value); + // The callee may hand the value to the callbacks it is passed, which + // must keep it too. + if ( + i >= 0 && + internal.get(c.callee)?.has(i) === true && + [...(callbacks.get(c.callee) ?? new Set())].every((j) => callbackClean(c.args[j])) + ) { + out.at = at; + return "internal"; + } + } + return "escapes"; + }; + + /** How the result of the call at `parents[at]` is used, through calls + * that keep it in their own results (`report(select(load()))`). */ + const resultUse = (at: number, parents: readonly Node[], out: UseInfo): Use => { + let index = at; + for (let i = 0; i < 64; i++) { + const info: UseInfo = {}; + const use = useOf(parents[index]!, parents.slice(0, index), false, info); + if (use !== "internal") { + // Field by field: an Object.assign onto a record makes the + // self-hosted compiler read class and record properties dynamically. + if (info.bound !== undefined) out.bound = info.bound; + if (info.at !== undefined) out.at = info.at; + if (info.target !== undefined) out.target = info.target; + return use; + } + index = info.at!; + } + return "escapes"; + }; + /** `fn`'s body in preorder, with each node's parent (-1 for statements + * of the body), the uses of locals that can hold references, and the + * sources: walked once, for escapingLocals' many passes. */ + interface Flat { + nodes: Node[]; + parent: number[]; + refs: number[]; + sources: number[]; + } + const flats = new Map(); + const flatOf = (fn: IrFunction): Flat => { + let flat = flats.get(fn.name); + if (flat !== undefined) return flat; + flat = { nodes: [], parent: [], refs: [], sources: [] }; + const f = flat; + // Parents come before their children, so each node's parent is indexed. + const index = new Map(); + forEachNode(fn.body, (node, parent) => { + const i = f.nodes.length; + index.set(node, i); + f.nodes.push(node); + f.parent.push(parent === undefined ? -1 : index.get(parent)!); + if (source(node)) f.sources.push(i); + if (node.kind === "varRef" && (refType(node.type) || node.type.kind === "string")) + f.refs.push(i); + }); + flats.set(fn.name, flat); + return flat; + }; + /** The ancestors of node `i` of `flat`, outermost first, into `out`. */ + const ancestors = (flat: Flat, i: number, out: Node[]): Node[] => { + out.length = 0; + for (let p = flat.parent[i]!; p >= 0; p = flat.parent[p]!) out.push(flat.nodes[p]!); + out.reverse(); + return out; + }; + /** Locals of `fn` whose value (or a reference read out of it) escapes. + * With `members` (the locals bound to group members), a value may also be + * returned, be content of a new value, or be stored into a member's + * graph. */ + const escapingLocals = ( + fn: IrFunction, + members: Set | null, + classes?: Map, + ): Set => { + const out = new Set(); + const add = (id: string): void => { + out.add(id); + }; + const own = new Set(fn.locals.map((l) => l.id)); + const group = new Map(); + const find = (id: string): string => { + let r = id; + while (group.has(r) && group.get(r) !== r) r = group.get(r)!; + return r; + }; + const unite = (a: string, b: string): void => { + const ra = find(a), + rb = find(b); + if (ra !== rb) group.set(ra, rb); + }; + // Copies: locals bound to a local or to a fresh copy of one (`slice`). + // A store of a value read out of a copy into the graph of another copy + // adds no reference into the group, unless a member went into a copy. + const copy = new Map(); + const findCopy = (id: string): string => { + let r = id; + while (copy.has(r) && copy.get(r) !== r) r = copy.get(r)!; + return r; + }; + const uniteCopy = (a: string, b: string): void => { + const ra = findCopy(a), + rb = findCopy(b); + if (ra !== rb) copy.set(ra, rb); + }; + const filled = new Set(); + const selfStores: [string, string][] = []; + const flat = flatOf(fn); + const parents: Node[] = []; + if (members !== null) + for (const i of flat.sources) { + const node = flat.nodes[i]!; + ancestors(flat, i, parents); + // A copy class holds something new once a value it did not have + // goes into it: stored into its graph, or bound to one of its locals + // (`visited = new Map(); this.visited = visited`). A `slice` of a + // local in the class holds only what the class held. + const info: UseInfo = {}; + const use = useOf(node, parents, false, info); + if (use === "stored" && info.target) filled.add(info.target); + if (use === "bound" && !(node.kind === "arrIntrinsic" && node.method === "slice")) + filled.add(info.bound!); + } + // Uses of locals that can hold references (a value without references + // cannot keep a member). + for (const i of flat.refs) { + const node = flat.nodes[i]!; + if (node.kind !== "varRef") continue; + ancestors(flat, i, parents); + const id = node.localId; + const info: UseInfo = {}; + const use = useOf(node, parents, true, info); + if (use === "bound") { + if (own.has(info.bound!)) { + unite(id, info.bound!); + uniteCopy(id, info.bound!); + } else add(id); + } else if (use === "internal" && members === null) { + // The callee keeps the value only in its result: the value stays + // in this frame while the result does (a module global needs no + // frame to hold it). + const result: UseInfo = {}; + const r = resultUse(info.at!, parents, result); + if (r === "bound" && own.has(result.bound!)) { + if (own.has(id)) unite(id, result.bound!); + } else if (r !== "kept") add(id); + } else if (use === "thrown") { + // A thrown value leaves the frame and outlives the group. + add(id); + } else if (use === "returned" || use === "content" || use === "internal") { + if (members === null) add(id); + } else if (use === "stored") { + if (members?.has(id) && info.target) filled.add(info.target); + if ( + members !== null && + info.target !== null && + info.target !== undefined && + !members.has(info.target) && + own.has(info.target) + ) + selfStores.push([id, info.target]); + else if ( + members === null || + info.target === null || + info.target === undefined || + !members.has(info.target) + ) + add(id); + } else if (use === "escapes") add(id); + // `const copy = id.slice()`. + // An optional chain on a Node, or a field read in the same condition + // as its kind test, tests every Node kind in turn: the kind is read + // after an undefined check, and the fields in a statement of their own. + const p = parents[parents.length - 1]; + if (members !== null && p !== undefined && p.kind === "arrIntrinsic") { + if (same(p.receiver, node) && p.method === "slice") { + const copied: UseInfo = {}; + if (useOf(p, parents.slice(0, -1), false, copied) === "bound" && own.has(copied.bound!)) + uniteCopy(id, copied.bound!); + } + } + } + const filledCopies = new Set([...filled].map(findCopy)); + for (const [id, target] of selfStores) + if (findCopy(id) !== findCopy(target) || filledCopies.has(findCopy(id))) add(id); + for (const local of fn.locals) if (local.boxed === true) out.add(local.id); + const roots = new Set(); + for (const id of out) roots.add(find(id)); + for (const local of fn.locals) if (roots.has(find(local.id))) out.add(local.id); + if (classes !== undefined) for (const local of fn.locals) classes.set(local.id, find(local.id)); + return out; + }; + + const solveDeep = (): void => { + for (const fn of mod.functions) { + const safe = new Set(); + if (!suspends(fn)) + fn.params.forEach((param, i) => { + const local = fn.locals.find((l) => l.id === param.localId); + if ((refType(param.type) || param.type.kind === "string") && local?.boxed !== true) + safe.add(i); + }); + deep.set(fn.name, safe); + } + solve(fns.keys(), referrers, (fn) => { + const safe = deep.get(fn.name)!; + if (safe.size === 0) return false; + const esc = escapingLocals(fn, null); + let changed = false; + fn.params.forEach((param, i) => { + if (safe.has(i) && esc.has(param.localId)) { + safe.delete(i); + changed = true; + } + }); + return changed; + }); + }; + solveDeep(); + + // Leak-free and fresh functions, optimistic. + const leakFree = new Set(); + const fresh = new Set(); + for (const fn of mod.functions) { + if (suspends(fn)) continue; + leakFree.add(fn.name); + if (refType(fn.returnType)) fresh.add(fn.name); + const inner = new Set(); + fn.params.forEach((param, i) => { + const local = fn.locals.find((l) => l.id === param.localId); + if ((refType(param.type) || param.type.kind === "string") && local?.boxed !== true) + inner.add(i); + }); + internal.set(fn.name, inner); + } + let internalChanged = false; + const unwrap = (e: Node): Node => { + let at: Node = e; + for (let i = 0; i < 16; i++) { + if ( + at.kind === "unionWrap" || + at.kind === "upcast" || + at.kind === "downcast" || + at.kind === "unionRetype" + ) + at = at.value; + else break; + } + return at; + }; + // Callback parameters: parameters a function calls (or passes on to a + // callback parameter). Each argument for one must be a leak-free closure, + // or a callback parameter of the caller. + const reassignedMemo = new Map>(); + const reassignedIn = (fn: IrFunction): Set => { + let out = reassignedMemo.get(fn.name); + if (out !== undefined) return out; + out = new Set(); + for (const node of flatOf(fn).nodes) if (node.kind === "assign") out.add(node.localId); + for (const l of fn.locals) if (l.boxed === true) out.add(l.id); + reassignedMemo.set(fn.name, out); + return out; + }; + const paramIndex = (fn: IrFunction, e: Node): number => { + const u = unwrap(e); + if (u.kind !== "varRef") return -1; + const id = u.localId; + if (reassignedIn(fn).has(id)) return -1; + return fn.params.findIndex((p) => p.localId === id); + }; + for (const fn of mod.functions) callbacks.set(fn.name, new Set()); + solve(fns.keys(), callers, (fn) => { + const set = callbacks.get(fn.name)!; + const before = set.size; + for (const node of flatOf(fn).nodes) { + if (node.kind === "callValue") { + const i = paramIndex(fn, node.callee); + if (i >= 0) set.add(i); + } else if (node.kind === "arrIntrinsic" && CALLBACK_INTRINSICS.has(node.method)) { + for (const a of node.args) + if (a.type.kind === "func") { + const i = paramIndex(fn, a); + if (i >= 0) set.add(i); + } + } else if (node.kind === "call") { + for (const j of callbacks.get(node.callee) ?? new Set()) { + const arg = node.args[j]; + const i = arg === undefined ? -1 : paramIndex(fn, arg); + if (i >= 0) set.add(i); + } + } + } + return set.size !== before; + }); + const callbackClean = (arg: IrExpr | undefined): boolean => { + if (arg === undefined) return true; + const u = unwrap(arg); + if (u.kind === "closure") { + // The callee passes the callback values it borrows (or that came from + // its caller): its parameters must stay internal. + const name = u.fnName; + const target = fns.get(name); + if (target === undefined || !leakFree.has(name)) return false; + const inner = internal.get(name)!; + return target.params.every((p, i) => !refType(p.type) || inner.has(i)); + } + if (current === null || framePass) return false; + const i = paramIndex(current, u); + return i >= 0 && callbacks.get(current.name)!.has(i); + }; + const calleesClean = (node: Node): boolean => { + if (node.kind === "call") { + if (!leakFree.has(node.callee)) return false; + for (const j of callbacks.get(node.callee) ?? new Set()) + if (!callbackClean(node.args[j])) return false; + return true; + } + if (node.kind === "callValue" && current !== null) { + const i = paramIndex(current, node.callee); + return i >= 0 && callbacks.get(current.name)!.has(i); + } + if (node.kind === "arrIntrinsic" && CALLBACK_INTRINSICS.has(node.method)) + return node.args.every((a) => a.type.kind !== "func" || callbackClean(a)); + if (node.kind === "virtualCall") { + const t = virtualTargetsOf(vt, node); + return t !== null && t.length > 0 && t.every((x) => leakFree.has(x)); + } + if ( + node.kind === "callValue" || + node.kind === "newValue" || + node.kind === "dynCall" || + node.kind === "dynInvoke" + ) + return false; + if (node.kind === "new") return leakFree.has(`%${node.className}.constructor`); + return true; + }; + /** The values `e` can produce, through casts and conditionals. */ + const leaves = (e: Node, out: Node[] = []): Node[] => { + const u = unwrap(e); + if (u.kind === "ternary") { + // A side that throws produces no value. + if (!throws(u.then)) leaves(u.then, out); + if (!throws(u.else_)) leaves(u.else_, out); + } else out.push(u); + return out; + }; + /** A source whose value is a new group member (a call result only when + * the callee is fresh). */ + const memberSource = (u: Node): boolean => + source(u) && u.kind !== "callValue" && (u.kind !== "call" || fresh.has(u.callee)); + /** What analyze reads of `fn` that no summary changes: the values bound + * to each local, the stores into locals' graphs, and the locals that are + * not closed (see analyze). */ + const bodyFacts = new Map< + string, + { bindings: Map; stores: [string, Node][]; open: Set } + >(); + const factsOf = ( + fn: IrFunction, + ): { bindings: Map; stores: [string, Node][]; open: Set } => { + const memo = bodyFacts.get(fn.name); + if (memo !== undefined) return memo; + const bindings = new Map(); + const scan = (node: Node): void => { + if (node.kind === "varDecl" || node.kind === "assign") { + const v = node.kind === "varDecl" ? node.init : node.value; + if (v !== null) bindings.set(node.localId, [...(bindings.get(node.localId) ?? []), v]); + } + const kids: Node[] = []; + children(node, kids); + for (const k of kids) scan(k); + }; + for (const s of fn.body) scan(s); + const stores: [string, Node][] = []; + const open = new Set(); + { + const parents: Node[] = []; + const walk = (node: Node): void => { + const parent = parents[parents.length - 1]; + if (parent !== undefined) { + const t = storeOf(node, parent); + if (t !== undefined) { + const target = storeTarget(t); + if (target !== null) stores.push([target, node]); + } + } + if (node.kind === "varRef") { + const id = node.localId; + let value: Node = node; + let at = parents.length - 1; + while (at >= 0 && (transparent(value, parents[at]) || projection(value, parents[at]))) + value = parents[at--]!; + const p = parents[at]; + if (p !== undefined && isExpr(p)) { + const x = p; + if ( + x.kind === "call" || + x.kind === "virtualCall" || + x.kind === "callValue" || + x.kind === "new" || + x.kind === "newValue" || + x.kind === "dynCall" || + x.kind === "dynInvoke" || + x.kind === "libCall" + ) + open.add(id); + else if ( + (x.kind === "arrIntrinsic" || + x.kind === "mapIntrinsic" || + x.kind === "setIntrinsic") && + same(x.receiver, value) && + !READ_RECEIVER.has(x.method) && + !x.method.startsWith("iter") && + storeOf(x.args[0] ?? x, x) === undefined + ) + open.add(id); + } + } + const kids: Node[] = []; + children(node, kids); + parents.push(node); + for (const kid of kids) walk(kid); + parents.pop(); + }; + for (const st of fn.body) walk(st); + } + const facts = { bindings, stores, open }; + bodyFacts.set(fn.name, facts); + return facts; + }; + const analyze = (fn: IrFunction): { leakFree: boolean; fresh: boolean } => { + current = fn; + // Member locals: bound only to sources, other members, or non-references. + const members = new Set(fn.locals.filter((l) => l.boxed !== true).map((l) => l.id)); + for (const p of fn.params) members.delete(p.localId); + // A constructor's `this` is the object its `new` allocated. + const ctor = fn.name.endsWith(".constructor") && fn.params.length > 0; + const thisId = ctor ? fn.params[0]!.localId : null; + // Closed members: members whose graph holds only members and leaves + // (values without a graph: numbers, strings). A reference read out of a + // closed member is a member too. A local is not closed when it is passed + // to a call (which may store into its graph) or used as the receiver of + // a method that adds what this analysis does not see. + const { bindings, stores, open } = factsOf(fn); + const closed = new Set([...members].filter((id) => !open.has(id) && id !== thisId)); + const leaf = (u: Node): boolean => isExpr(u) && !refType(u.type); + const closedSource = (u: Node): boolean => { + if (u.kind === "libCall") return u.fn === "json.parse"; + if (u.kind === "dynCheck") return closedSource(u.value); + if (u.kind === "strIntrinsic") return u.method === "split"; + if ( + u.kind !== "arrayLit" && + u.kind !== "recordLit" && + u.kind !== "mapNew" && + u.kind !== "setNew" + ) + return false; + const kids: Node[] = []; + children(u, kids); + return kids.every((k) => !isExpr(k) || closedValue(k)); + }; + const closedValue = (v: Node): boolean => + leaves(v).every((u) => { + if (leaf(u) || closedSource(u)) return true; + const base = isExpr(u) ? projectionBase(u) : null; + return base !== null && closed.has(base); + }); + for (let again = true; again;) { + again = false; + for (const id of [...members]) { + const list = bindings.get(id) ?? []; + if (id === thisId) continue; + const ok = + list.length > 0 && + list.every((v) => + leaves(v).every((u) => { + if (memberSource(u) || leaf(u)) return true; + if (u.kind === "varRef") return members.has(u.localId); + const base = isExpr(u) ? projectionBase(u) : null; + return base !== null && closed.has(base); + }), + ); + if (!ok) { + members.delete(id); + closed.delete(id); + again = true; + } + } + for (const id of [...closed]) { + const ok = + members.has(id) && + (bindings.get(id) ?? []).every(closedValue) && + stores.every(([target, value]) => target !== id || closedValue(value)); + if (!ok) { + closed.delete(id); + again = true; + } + } + // A local bound to a reference read out of another shares its graph. + for (const [id, list] of bindings) + if (!closed.has(id)) + for (const v of list) + for (const u of leaves(v)) { + const base = isExpr(u) ? projectionBase(u) : null; + if (base !== null && base !== id && closed.delete(base)) again = true; + } + } + if (thisId !== null) members.add(thisId); + // Members that escape (other than by return or into a member's graph) + // are no longer members; repeat, since a store into a former member's + // graph now escapes too. + let esc = escapingLocals(fn, members); + for (let again = true; again;) { + again = false; + for (const id of esc) if (members.delete(id)) again = true; + if (again) esc = escapingLocals(fn, members); + } + // Every source must die, be returned, or go into a member's graph; a + // constructor's `this` must stay a member. + const inner = internal.get(fn.name)!; + fn.params.forEach((param, i) => { + if (inner.has(i) && esc.has(param.localId)) { + inner.delete(i); + internalChanged = true; + } + }); + let clean = thisId === null || members.has(thisId); + let returnsFresh = true; + const parents: Node[] = []; + const walk = (node: Node): void => { + if (!calleesClean(node)) clean = false; + if (source(node)) { + const info: UseInfo = {}; + const use = useOf(node, parents, false, info); + if (use === "bound") { + if (esc.has(info.bound!) || !fn.locals.some((l) => l.id === info.bound)) { + clean = false; + } + } else if (use === "stored") { + if (info.target === null || info.target === undefined || !members.has(info.target)) { + clean = false; + } + } else if (use === "escapes") { + clean = false; + } else if (use === "thrown") { + // A thrown value outlives the group, so it may hold no member and + // nothing allocated with it (a message string is fine). + const kids: Node[] = []; + children(node, kids); + for (const k of kids) + if ( + isExpr(k) && + refType(k.type) && + leaves(k).some((u) => { + if (source(u)) return true; + const base = isExpr(u) ? projectionBase(u) : null; + return base !== null && members.has(base); + }) + ) + clean = false; + } + } + if (node.kind === "return") { + const v = node.value; + if (v !== null && refType(v.type)) { + for (const u of leaves(v)) { + if (!(memberSource(u) || (u.kind === "varRef" && members.has(u.localId)))) + returnsFresh = false; + if (u.kind === "call" && !fresh.has(u.callee)) returnsFresh = false; + } + } + } + const kids: Node[] = []; + children(node, kids); + parents.push(node); + for (const kid of kids) walk(kid); + parents.pop(); + }; + for (const s of fn.body) walk(s); + current = null; + return { leakFree: clean, fresh: clean && returnsFresh }; + }; + solve(leakFree, referrers, (fn) => { + if (!leakFree.has(fn.name)) return false; + internalChanged = false; + const r = analyze(fn); + if (!r.leakFree) { + leakFree.delete(fn.name); + fresh.delete(fn.name); + internal.get(fn.name)!.clear(); + return true; + } + return (!r.fresh && fresh.delete(fn.name)) || internalChanged; + }); + + // With internal parameters known, a value passed to one stays in the + // frame while the call's result does: solve the deep summaries again. + solveDeep(); + // Functions that allocate, directly or through what they call. + const allocates = new Set(); + solve(directAlloc, callers, (fn) => { + if (allocates.has(fn.name)) return false; + allocates.add(fn.name); + return true; + }); + // Roots, and call regions: calls of leak-free functions that allocate and + // return no reference, so all they allocate is dead when they return. + const roots = new Set(); + const calls = new Set(); + /** Whether the operands of `node` (evaluated inside its group) are clean: + * every call in them is leak-free, no library call returns a reference + * (the runtime may keep it: `process.argv`'s cached array), and every value they + * allocate goes to `node` as an internal or borrowed argument, or into the + * value of another operand. */ + const operandsClean = (node: Node, parents: readonly Node[]): boolean => { + const stack: Node[] = [...parents, node]; + let ok = true; + const visit = (n: Node): void => { + if (!ok) return; + if (!same(n, node)) { + if (!calleesClean(n)) ok = false; + if (n.kind === "libCall" && n.fn !== "json.parse" && refType(n.type)) ok = false; + // `JSON.parse(text) as T`: the parse is the checked value itself. + const checked = node.kind === "dynCheck" && same(node.value, n); + if (source(n) && !checked) { + const info: UseInfo = {}; + const use = useOf(n, stack, false, info); + const at = info.at ?? -1; + if ( + !(use === "kept" || use === "content" || (use === "internal" && at >= parents.length)) + ) + ok = false; + } + } + const kids: Node[] = []; + children(n, kids); + const inner = !same(n, node); + if (inner) stack.push(n); + for (const k of kids) visit(k); + if (inner) stack.pop(); + }; + visit(node); + return ok; + }; + const drops = new Map(); + const rootLocals = new Map>(); + const rootFunction = new Map(); + framePass = true; + for (const fn of mod.functions) { + if (suspends(fn)) continue; + current = fn; + const classes = new Map(); + const esc = escapingLocals(fn, null, classes); + const own = new Set(fn.locals.map((l) => l.id)); + const paths = declarationPaths(fn); + /** The local of `bound`'s class whose scope holds every other one's, + * or undefined when no one does. */ + const classOf = (bound: string): string[] => { + const cls = classes.get(bound); + return fn.locals.filter((l) => classes.get(l.id) === cls).map((l) => l.id); + }; + const dropLocal = (bound: string): string | undefined => { + const members = classOf(bound); + const path = (id: string): readonly number[] => paths.get(id) ?? []; + return members.find((m) => + members.every((o) => { + const a = path(m); + const b = path(o); + return a.length <= b.length && a.every((x, i) => b[i] === x); + }), + ); + }; + const parents: Node[] = []; + const walk = (node: Node): void => { + const isRoot = + isExpr(node) && + ((node.kind === "call" && fresh.has(node.callee)) || + (node.kind === "new" && leakFree.has(`%${node.className}.constructor`)) || + ((node.kind === "libCall" || node.kind === "dynCheck" || node.kind === "strIntrinsic") && + source(node)) || + (node.kind === "arrIntrinsic" && source(node))); + const callbacksClean = + node.kind !== "call" || + [...(callbacks.get(node.callee) ?? new Set())].every((j) => + callbackClean(node.args[j]), + ); + const clean = + (isRoot || node.kind === "call") && callbacksClean && operandsClean(node, parents); + if ( + node.kind === "call" && + clean && + !refType(node.type) && + // A constructor (a `super(...)` call) stores into `this`, which its + // caller holds: it is leak-free only as part of a `new`. + !node.callee.endsWith(".constructor") && + leakFree.has(node.callee) && + allocates.has(node.callee) + ) { + calls.add(node); + rootFunction.set(node, fn); + } + if (isRoot && clean) { + const info: UseInfo = {}; + let use = useOf(node, parents, false, info); + if (use === "internal") use = resultUse(info.at!, parents, info); + rootFunction.set(node, fn); + if (use === "kept") { + roots.add(node); + drops.set(node, null); + rootLocals.set(node, new Set()); + } else if (use === "bound" && own.has(info.bound!) && !esc.has(info.bound!)) { + const local = dropLocal(info.bound!); + // An assignment in a loop would add a group per iteration to a + // list that is dropped only when the local's scope exits. + let binding = parents.length - 1; + while ( + binding >= 0 && + parents[binding]!.kind !== "varDecl" && + parents[binding]!.kind !== "assign" + ) + binding--; + const reassignedInLoop = + binding >= 0 && + parents[binding]!.kind === "assign" && + parents.slice(0, binding).some((p) => LOOP_KINDS.has(p.kind)); + if (local !== undefined && !reassignedInLoop) { + roots.add(node); + drops.set(node, local); + rootLocals.set(node, new Set(classOf(info.bound!))); + } + } + } + const kids: Node[] = []; + children(node, kids); + parents.push(node); + for (const kid of kids) walk(kid); + parents.pop(); + }; + for (const s of fn.body) walk(s); + } + current = null; + // Functions that run inside an arena: what roots and call regions call, + // and what those call, with the closures they pass. + const reachable = new Set(); + const queue: string[] = []; + const reach = (name: string): void => { + if (fns.has(name) && !reachable.has(name)) { + reachable.add(name); + queue.push(name); + } + }; + const callees = (node: Node): void => { + if (node.kind === "call") reach(node.callee); + else if (node.kind === "virtualCall") + for (const t of virtualTargetsOf(vt, node) ?? []) reach(t); + else if (node.kind === "new") reach(`%${node.className}.constructor`); + else if (node.kind === "closure") reach(node.fnName); + }; + const scanAll = (node: Node, visit: (n: Node) => void): void => { + visit(node); + const kids: Node[] = []; + children(node, kids); + for (const k of kids) scanAll(k, visit); + }; + for (const r of [...roots, ...calls]) scanAll(r, callees); + while (queue.length > 0) { + const fn = fns.get(queue.pop()!)!; + for (const st of fn.body) scanAll(st, callees); + } + const closed = new Set(); + return { + roots, + calls, + drops, + rootLocals, + rootFunction, + reachable, + closed, + fresh, + leakFree, + allocates, + }; +} + +/** For each local declared in `fn`, the statement lists that enclose its + * declaration, outermost first (the body included); a declaration inside an + * expression (a sequence's statements) is nested in the statement that + * holds it. Parameters and locals without a declaration have none: the + * function's own scope. A `for…of` variable counts as declared in the list + * holding the loop. */ +function declarationPaths(fn: IrFunction): Map { + const out = new Map(); + // Each statement list (and each expression holding one) gets an id. + let next = 0; + const walk = (list: readonly IrStmt[], path: readonly number[]): void => { + if (list.length === 0) return; + const here = [...path, next++]; + for (const stmt of list) { + if (stmt.kind === "varDecl" || stmt.kind === "forOf") out.set(stmt.localId, here); + for (const inner of stmtLists(stmt)) walk(inner, here); + // Statements inside the statement's expressions, one level deeper. + const visit = (e: IrExpr): boolean => { + if (e.kind === "seqExpr") walk(e.stmts, [...here, next++]); + everyExprChild(e, visit, () => true); + return true; + }; + everyStmtChild(stmt, visit, () => true); + } + }; + walk(fn.body, []); + return out; +} + +/** The statement lists directly inside `stmt`. */ +function stmtLists(stmt: IrStmt): readonly (readonly IrStmt[])[] { + switch (stmt.kind) { + case "if": + return stmt.else_ === null ? [stmt.then] : [stmt.then, stmt.else_]; + case "while": + case "doWhile": + case "for": + case "forOf": + case "block": + return [stmt.body]; + case "switch": + return stmt.cases.map((c) => c.body); + case "tryCatch": + return [stmt.tryBody, stmt.catchBody ?? [], stmt.finallyBody ?? []]; + default: + return []; + } +} + +/** + * The roots and call regions whose groups are closed: every reference an + * arena object of the group holds is to an arena object of the group or an + * immortal value, so dropping the arena releases nothing. + * + * `arenaType(t)` says whether a value of type `t` is a plain reference to + * an object the arena-aware constructors allocate (no box around it). + * + * A function is closed when every reference it stores (into a field, an + * element, a collection, or the content of a new object) is closed: + * - a value without references, an immortal literal, or null/undefined; + * - an object of an arena type it allocates, or the result of a call of a + * closed function whose results are closed; + * - a local bound only to closed values, or a reference of an arena type + * read out of one; + * - a constructor's `this`; + * - a parameter, or a reference read out of one: its callers must pass a + * closed value (the function's stored parameters). + * Every function it calls must be closed and get closed values for their + * stored parameters. A root's group is then closed when its callee is, + * its arguments for stored parameters are closed in its caller, and the + * caller stores only closed values into the root's graph afterwards (there, + * only values without references, literals, and references read out of the + * root's own locals are closed: the caller's own allocations are on the + * heap). Solved optimistically to a fixpoint. + */ +export function analyzeClosedGroups( + mod: IrModule, + groups: RegionGroups, + arenaType: (t: IrType) => boolean, + /** Whether an allocation or call site's objects take the arena. */ + kept: (e: Node) => boolean = () => true, +): Set { + const fns = new Map(); + for (const fn of mod.functions) fns.set(fn.name, fn); + const vt = new VirtualTargets(mod.classes ?? []); + const closedFn = new Set([...groups.reachable].filter((f) => groups.leakFree.has(f))); + const closedFresh = new Set([...closedFn].filter((f) => groups.fresh.has(f))); + const stored = new Map>(); + const storedOf = (name: string): Set => stored.get(name) ?? new Set(); + const leavesOf = (e: IrExpr, out: IrExpr[] = []): IrExpr[] => { + let at = e; + for (let i = 0; i < 16; i++) + if ( + at.kind === "unionWrap" || + at.kind === "upcast" || + at.kind === "downcast" || + at.kind === "unionRetype" + ) + at = at.value; + else break; + if (at.kind === "ternary") { + if (!throws(at.then)) leavesOf(at.then, out); + if (!throws(at.else_)) leavesOf(at.else_, out); + } else out.push(at); + return out; + }; + // Functions that return only references read out of one parameter (or + // null/undefined), such as the checked element read `%arr.idxOr`: a call + // of one reads out of that argument. + const projects = new Map(); + for (const fn of mod.functions) { + const assigned = new Set(); + let index: number | null = null; + let ok = true; + const visit = (node: Node): void => { + if (!ok) return; + if (node.kind === "assign") assigned.add(node.localId); + if (node.kind === "return") { + const v = node.value; + if (v === null) ok = false; + else + for (const u of leavesOf(v)) { + if (u.kind === "unitLit") continue; + const base = projectionBase(u); + const i = base === null ? -1 : fn.params.findIndex((p) => p.localId === base); + if (i < 0 || (index !== null && index !== i)) ok = false; + else index = i; + } + } + const kids: Node[] = []; + children(node, kids); + for (const k of kids) visit(k); + }; + if (!refType(fn.returnType)) continue; + for (const st of fn.body) visit(st); + if (ok && index !== null && !assigned.has(fn.params[index]!.localId)) + projects.set(fn.name, index); + } + const through = (call: IrExpr): IrExpr | null => { + if (call.kind !== "call") return null; + const i = projects.get(call.callee); + return i === undefined ? null : (call.args[i] ?? null); + }; + /** Whether `e` (in `fn`, `own` its closed locals, `inArena` whether its + * allocations are arena objects) is a closed value; parameters it reads + * go into `params`. */ + const closedValue = ( + e: Node, + fn: IrFunction, + own: ReadonlySet, + inArena: boolean, + params: Set, + ): boolean => { + if (!isExpr(e)) return false; + const t = e.type; + if (t.kind === "string") return e.kind === "strLit"; + // Any other counted value (a typed array, a promise) is a heap object. + if (!refType(t) && !isRefCounted(t)) return true; + if (!arenaType(t)) return false; + return leavesOf(e).every((u) => { + if (u.kind === "unitLit") return true; + if (u.kind === "recordLit" || u.kind === "new" || u.kind === "arrayLit") + return inArena && arenaType(u.type) && kept(u); + if (u.kind === "call" && !projects.has(u.callee)) + return inArena && closedFresh.has(u.callee) && arenaType(u.type) && kept(u); + const base = projectionBase(u, through); + if (base === null) return false; + const i = fn.params.findIndex((p) => p.localId === base); + if (i >= 0) { + if (i === 0 && fn.name.endsWith(".constructor") && u.kind === "varRef") return true; + params.add(i); + return arenaType(u.type); + } + return own.has(base) && arenaType(u.type); + }); + }; + /** Every reference store and content position in `fn`'s body, and every + * call with the arguments its callee stores. */ + const siteMemo = new Map(); + const sites = (fn: IrFunction): { value: Node; target: string | null }[] => { + const memo = siteMemo.get(fn.name); + if (memo !== undefined) return memo; + const out: { value: Node; target: string | null }[] = []; + siteMemo.set(fn.name, out); + const walk = (node: Node, parent: Node | undefined): void => { + if (parent !== undefined && isExpr(node)) { + const target = storeOf(node, parent); + if (target !== undefined) out.push({ value: node, target: storeTarget(target) }); + else if (contentOf(node, parent) && parent.kind !== "new") + out.push({ value: node, target: null }); + } + const kids: Node[] = []; + children(node, kids); + for (const k of kids) walk(k, node); + }; + for (const st of fn.body) walk(st, undefined); + return out; + }; + const bindingMemo = new Map>(); + const bindingsOf = (fn: IrFunction): Map => { + const memo = bindingMemo.get(fn.name); + if (memo !== undefined) return memo; + const out = new Map(); + bindingMemo.set(fn.name, out); + const scan = (node: Node): void => { + if (node.kind === "varDecl" || node.kind === "assign") { + const v = node.kind === "varDecl" ? node.init : node.value; + if (v !== null) out.set(node.localId, [...(out.get(node.localId) ?? []), v]); + } + // A loop variable holds elements read out of the iterable. + if (node.kind === "forOf") + out.set(node.localId, [...(out.get(node.localId) ?? []), node.iterable]); + const kids: Node[] = []; + children(node, kids); + for (const k of kids) scan(k); + }; + for (const st of fn.body) scan(st); + return out; + }; + // Written parameters: those whose graph a function may store into, itself + // or through what it calls (any function, not only arena-aware ones). + const writes = new Map>(); + /** The parameter whose graph `local` reaches into, through bindings to + * references read out of it, or -1. */ + const paramOf = (fn: IrFunction, bindings: Map, local: string): number => { + let at = local; + for (let i = 0; i < 16; i++) { + const p = fn.params.findIndex((q) => q.localId === at); + if (p >= 0) return p; + const list = bindings.get(at); + if (list === undefined || list.length !== 1) return -1; + const base = projectionBase(list[0]!); + if (base === null || base === at) return -1; + at = base; + } + return -1; + }; + // Each function's own writes, and the parameters it passes to calls: a + // parameter passed where the callee writes is written too. + const passed = new Map< + string, + { param: number; callees: readonly string[] | null; index: number }[] + >(); + const writers = new Map>(); + // Only what the functions holding roots call, transitively, is asked. + const asked = new Set(); + { + const queue = [ + ...new Set( + [...groups.roots, ...groups.calls].flatMap((r) => groups.rootFunction.get(r)?.name ?? []), + ), + ]; + for (const name of queue) asked.add(name); + while (queue.length > 0) { + const fn = fns.get(queue.pop()!); + if (fn === undefined) continue; + const visit = (n: Node): void => { + const targets = + n.kind === "call" + ? [n.callee] + : n.kind === "virtualCall" + ? (virtualTargetsOf(vt, n) ?? []) + : []; + for (const t of targets) + if (fns.has(t) && !asked.has(t)) { + asked.add(t); + queue.push(t); + } + const kids: Node[] = []; + children(n, kids); + for (const k of kids) visit(k); + }; + for (const st of fn.body) visit(st); + } + } + for (const fn of mod.functions) { + if (!asked.has(fn.name)) continue; + const bindings = bindingsOf(fn); + const set = new Set(); + for (const { target } of sites(fn)) + if (target !== null) { + const i = paramOf(fn, bindings, target); + if (i >= 0) set.add(i); + } + const list: { param: number; callees: readonly string[] | null; index: number }[] = []; + const visit = (n: Node): void => { + const args = argsOf(n); + if (args !== null) { + const callees = + n.kind === "call" + ? [n.callee] + : n.kind === "virtualCall" + ? (virtualTargetsOf(vt, n) ?? []) + : null; + args.forEach((a, j) => { + const base = projectionBase(a); + const i = base === null ? -1 : paramOf(fn, bindings, base); + if (i < 0) return; + if (callees === null || callees.length === 0 || callees.some((c) => !fns.has(c))) + set.add(i); + else { + list.push({ param: i, callees, index: j }); + for (const c of callees) { + let w = writers.get(c); + if (w === undefined) writers.set(c, (w = new Set())); + w.add(fn.name); + } + } + }); + } + const kids: Node[] = []; + children(n, kids); + for (const k of kids) visit(k); + }; + for (const st of fn.body) visit(st); + writes.set(fn.name, set); + passed.set(fn.name, list); + } + { + const queue = [...asked]; + const queued = new Set(queue); + while (queue.length > 0) { + const name = queue.pop()!; + queued.delete(name); + const set = writes.get(name)!; + const before = set.size; + for (const { param, callees, index } of passed.get(name)!) + if (callees!.some((c) => writes.get(c)?.has(index) === true)) set.add(param); + if (set.size !== before) + for (const r of writers.get(name) ?? new Set()) + if (!queued.has(r)) { + queued.add(r); + queue.push(r); + } + } + } + /** The locals of `fn` bound only to closed values. */ + const closedLocals = ( + fn: IrFunction, + inArena: boolean, + seed?: ReadonlySet, + ): Set => { + const bindings = bindingsOf(fn); + const own = new Set( + fn.locals + .filter((l) => l.boxed !== true && !fn.params.some((p) => p.localId === l.id)) + .map((l) => l.id), + ); + for (const id of seed ?? new Set()) own.add(id); + for (let again = true; again;) { + again = false; + for (const id of [...own]) { + if (seed?.has(id) === true) continue; + const list = bindings.get(id) ?? []; + if (!list.every((v) => closedValue(v, fn, own, inArena, new Set()))) { + own.delete(id); + again = true; + } + } + } + return own; + }; + /** Whether the calls in `node`'s subtree are of closed functions with + * closed arguments for their stored parameters. */ + const callsClosed = ( + node: Node, + fn: IrFunction, + own: ReadonlySet, + inArena: boolean, + params: Set, + ): boolean => { + let ok = true; + const visit = (n: Node): void => { + if (!ok) return; + if (n.kind === "call" || n.kind === "new" || n.kind === "virtualCall") { + const targets = + n.kind === "call" + ? [n.callee] + : n.kind === "new" + ? [`%${n.className}.constructor`] + : (virtualTargetsOf(vt, n) ?? []); + if (targets.length === 0) ok = false; + const shift = n.kind === "new" ? 1 : 0; + for (const t of targets) { + if (!closedFn.has(t)) ok = false; + for (const j of storedOf(t)) { + const arg = n.args[j - shift]; + if (arg !== undefined && !closedValue(arg, fn, own, inArena, params)) ok = false; + } + } + } else if (n.kind === "callValue" || n.kind === "dynCall" || n.kind === "dynInvoke") + ok = false; + else if ( + (n.kind === "arrIntrinsic" || n.kind === "mapIntrinsic" || n.kind === "setIntrinsic") && + !READ_RECEIVER.has(n.method) && + !ADDING_METHODS.has(n.method) && + n.args.some((a) => refType(a.type) || isRefCounted(a.type)) + ) + ok = false; + const kids: Node[] = []; + children(n, kids); + for (const k of kids) visit(k); + }; + visit(node); + return ok; + }; + for (let changed = true; changed;) { + changed = false; + for (const name of [...closedFn]) { + const fn = fns.get(name)!; + const params = new Set(); + const own = closedLocals(fn, true); + let ok = sites(fn).every(({ value }) => closedValue(value, fn, own, true, params)); + for (const st of fn.body) if (ok) ok = callsClosed(st, fn, own, true, params); + let returnsClosed = true; + const scanReturns = (node: Node): void => { + if (node.kind === "return") { + const v = node.value; + if (v !== null && !closedValue(v, fn, own, true, params)) returnsClosed = false; + } + const kids: Node[] = []; + children(node, kids); + for (const k of kids) scanReturns(k); + }; + for (const st of fn.body) scanReturns(st); + const before = storedOf(name); + if ([...params].some((i) => !before.has(i))) { + stored.set(name, new Set([...before, ...params])); + changed = true; + } + if (!ok) { + closedFn.delete(name); + closedFresh.delete(name); + changed = true; + } else if (!returnsClosed && closedFresh.delete(name)) changed = true; + } + } + const out = new Set(); + for (const root of [...groups.roots, ...groups.calls]) { + const fn = groups.rootFunction.get(root); + if (fn === undefined) continue; + const locals = groups.rootLocals.get(root) ?? new Set(); + const bindings = bindingsOf(fn); + /** Whether `local` reaches into the root's graph: one of its locals, + * or bound (a loop variable included) to a reference read out of one. */ + const intoRoot = (local: string, depth = 0): boolean => + locals.has(local) || + (depth < 16 && + (bindings.get(local) ?? []).some((v) => { + const base = projectionBase(v); + return base !== null && base !== local && intoRoot(base, depth + 1); + })); + // In the caller only the root's own locals and what is read out of them + // are closed references. + const params = new Set(); + let ok = callsClosed(root, fn, locals, false, params) && params.size === 0; + // After the arena closes, a call that is passed the root's value (or a + // reference read out of it) must not write into it. + const passes = (n: Node, parent: Node | undefined): void => { + if (!ok) return; + if (parent !== undefined && isExpr(n)) { + const args = argsOf(parent); + const j = args === null ? -1 : indexOfNode(args, n); + const base = projectionBase(n); + if (j >= 0 && refType(n.type) && (same(n, root) || (base !== null && intoRoot(base)))) { + const callees = + parent.kind === "call" + ? [parent.callee] + : parent.kind === "virtualCall" + ? (virtualTargetsOf(vt, parent) ?? []) + : []; + if (callees.length === 0 || callees.some((c) => writes.get(c)?.has(j) !== false)) + ok = false; + } + } + const kids: Node[] = []; + children(n, kids); + for (const k of kids) passes(k, n); + }; + if (ok && !groups.calls.has(root)) for (const st of fn.body) passes(st, undefined); + if (ok && locals.size > 0) + ok = sites(fn).every( + ({ value, target }) => + target === null || + !intoRoot(target) || + (closedValue(value, fn, locals, false, params) && params.size === 0), + ); + if (ok) out.add(root); + } + return out; +} diff --git a/packages/compiler/src/backend/llvm/region-ir.ts b/packages/compiler/src/backend/llvm/region-ir.ts new file mode 100644 index 000000000..405558f06 --- /dev/null +++ b/packages/compiler/src/backend/llvm/region-ir.ts @@ -0,0 +1,271 @@ +/* IR helpers shared by the region analyses (region-groups.ts): node + * traversal, allocation kinds, and how a use of a value passes it on or + * keeps it in the frame. */ +import type { IrExpr, IrStmt, IrType } from "../../ir/ir.js"; +import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; + +export type Node = IrExpr | IrStmt; + +/** The kinds of heap allocation sites counted. */ +export const ALLOCATIONS = new Set([ + "arrayLit", + "arrayNewLen", + "mapNew", + "setNew", + "new", + "recordLit", + "recordClone", + "closure", +]); + +/** Array methods that return a fresh array without keeping the receiver. */ +export const FRESH_ARRAY_METHODS = new Set([ + "slice", + "concatSpread", + "toReversed", + "toSpliced", + "with", + "withUndefined", + "flatCopy", + "flatOne", + "toSortedPrimitive", + "splice", +]); +/** Array methods that return the receiver itself. */ +export const RECEIVER_RESULT = new Set([ + "reverse", + "fill", + "fillUndefined", + "copyWithin", + "sortPrimitive", +]); +/** Array methods whose arguments are only read (compared, copied from). */ +export const READ_ARGS = new Set([ + "includes", + "indexOf", + "indexEq", + "getNumber", + "join", + "slice", + "concatSpread", + "pushSpread", + "unshiftSpread", + "nextPresent", + "length", +]); + +export function refType(t: IrType): boolean { + return ( + t.kind === "array" || + t.kind === "map" || + t.kind === "set" || + t.kind === "object" || + t.kind === "record" || + t.kind === "func" || + t.kind === "union" + ); +} + +export function children(node: Node, out: Node[]): void { + const push = (child: Node): boolean => { + out.push(child); + return true; + }; + if ("type" in node && (node as IrExpr).type !== undefined && isExpr(node)) + everyExprChild(node as IrExpr, push, push); + else everyStmtChild(node as IrStmt, push, push); +} + +const STMT_KINDS = new Set([ + "varDecl", + "assign", + "exprStmt", + "if", + "while", + "doWhile", + "switch", + "for", + "arraySet", + "arraySetLength", + "arraySetUndefined", + "arrayDelete", + "bytesSet", + "forOf", + "return", + "fieldSet", + "recordSet", + "recordKeySet", + "recordKeyDelete", + "break", + "continue", + "block", + "throw", + "runtimeFence", + "rethrow", + "tryCatch", +]); +export function isExpr(node: Node): node is IrExpr { + return !STMT_KINDS.has(node.kind); +} + +/** Whether `a` and `b` are the same node. */ +export function same(a: Node | undefined, b: Node): boolean { + return a !== undefined && a === b; +} + +/** Whether `parent` passes `child`'s value on as its own value (a branch of + * a conditional, a union wrap or narrow, a class up- or downcast, the + * result of a sequence), so the value's fate is the parent's. */ +export function transparent(child: Node, parent: Node | undefined): boolean { + if (parent === undefined) return false; + switch (parent.kind) { + case "ternary": + return same(parent.then, child) || same(parent.else_, child); + case "nullish": + case "logical": + return true; + case "seqExpr": + return same(parent.result, child); + case "unionWrap": + case "unionRetype": + case "unionNarrow": + case "upcast": + case "downcast": + return same(parent.value, child); + default: + return false; + } +} + +/** Whether a use of a local's value as `child` of `parent` keeps it in the + * frame (given the current parameter summaries). */ +export function keepsInFrame( + child: Node, + parent: Node | undefined, + summaries: ReadonlyMap>, +): boolean { + if (parent === undefined) return false; + switch (parent.kind) { + case "exprStmt": + return true; + case "arrayGet": + case "arrayHas": + case "arrayState": + case "arraySetUndefined": + case "arrayDelete": + case "arraySetLength": + case "arraySet": + return same(parent.arr, child); + case "fieldGet": + case "recordGet": + case "recordKeyGet": + case "recordHas": + case "recordOvfHas": + case "recordOvfKeys": + case "fieldSet": + case "recordSet": + case "recordKeySet": + case "recordKeyDelete": + return same(parent.obj, child); + case "forOf": + return same(parent.iterable, child); + case "arrIntrinsic": + if (same(parent.receiver, child)) return !RECEIVER_RESULT.has(parent.method); + return READ_ARGS.has(parent.method); + case "mapIntrinsic": + if (same(parent.receiver, child)) return true; + return parent.method === "get" || parent.method === "has" || parent.method === "delete"; + case "setIntrinsic": + if (same(parent.receiver, child)) return true; + return parent.method === "has" || parent.method === "delete"; + case "bin": + return parent.op === "===" || parent.op === "!=="; + case "toBool": + case "unary": + case "unionIsTag": + case "unionEq": + return true; + case "unionDisc": + case "unionKeyGet": + return same(parent.value, child); + case "callValue": + return same(parent.callee, child); + case "call": { + const index = indexOfNode(parent.args, child); + return index >= 0 && summaries.get(parent.callee)?.has(index) === true; + } + default: + return false; + } +} + +/** The index of `node` in `list`, or -1. */ +export function indexOfNode(list: readonly IrExpr[], node: Node): number { + for (let i = 0; i < list.length; i++) if (same(list[i], node)) return i; + return -1; +} + +/** The arguments of a node that takes them (calls, constructions, + * intrinsics), or null. */ +export function argsOf(node: Node): readonly IrExpr[] | null { + switch (node.kind) { + case "call": + case "callValue": + case "virtualCall": + case "new": + case "newValue": + case "ffiCall": + case "dynCall": + case "dynInvoke": + case "intrinsic": + case "libCall": + case "arrIntrinsic": + case "bytesIntrinsic": + case "mapIntrinsic": + case "setIntrinsic": + case "strIntrinsic": + case "regexIntrinsic": + return node.args; + default: + return null; + } +} + +const LOOP_STMTS = new Set(["while", "doWhile", "for", "forOf"]); + +/** Calls `visit(node, parent, loops)` for every node of `body`, parents + * before their children (siblings in no set order), where `loops` counts + * the loops around `node`. Allocates nothing per node. */ +export function forEachNode( + body: readonly IrStmt[], + visit: (node: Node, parent: Node | undefined, loops: number) => void, +): void { + const nodes: Node[] = []; + const parents: (Node | undefined)[] = []; + const depths: number[] = []; + let parent: Node | undefined = undefined; + let depth = 0; + const pushExpr = (child: IrExpr): boolean => { + nodes.push(child); + parents.push(parent); + depths.push(depth); + return true; + }; + const pushStmt = (child: IrStmt): boolean => { + nodes.push(child); + parents.push(parent); + depths.push(depth); + return true; + }; + for (const st of body) pushStmt(st); + while (nodes.length > 0) { + const node = nodes.pop()!; + const above = parents.pop(); + const loops = depths.pop()!; + visit(node, above, loops); + parent = node; + depth = loops + (LOOP_STMTS.has(node.kind) ? 1 : 0); + if (isExpr(node)) everyExprChild(node, pushExpr, pushStmt); + else everyStmtChild(node, pushExpr, pushStmt); + } +} diff --git a/packages/compiler/src/backend/llvm/region-sites.test.ts b/packages/compiler/src/backend/llvm/region-sites.test.ts new file mode 100644 index 000000000..97fa94577 --- /dev/null +++ b/packages/compiler/src/backend/llvm/region-sites.test.ts @@ -0,0 +1,223 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../../index.js"; +import type { IrExpr, IrModule } from "../../ir/ir.js"; +import { analyzeRegionGroups } from "./region-groups.js"; +import { children, type Node } from "./region-ir.js"; +import { planRegionSites } from "./region-sites.js"; + +async function moduleFor(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-region-sites-")); + try { + const entry = join(dir, "main.ts"), + output = join(dir, "main.ir.json"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath: output, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((item) => `${item.code}: ${item.message}`).join("\n")); + return deserializeModule(await readFile(output, "utf8")); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +/** The nodes in function `fn` matching `pick`, in order. */ +function nodesIn(mod: IrModule, fn: string, pick: (e: IrExpr) => boolean): IrExpr[] { + const out: IrExpr[] = []; + const visit = (node: Node): void => { + if (pick(node as IrExpr)) out.push(node as IrExpr); + const kids: Node[] = []; + children(node, kids); + for (const k of kids) visit(k); + }; + for (const s of mod.functions.find((f) => f.name === fn)!.body) visit(s); + return out; +} + +const calls = (callee: string) => (e: IrExpr) => e.kind === "call" && e.callee === callee; +const records = (e: IrExpr) => e.kind === "recordLit"; + +function plan(mod: IrModule) { + const groups = analyzeRegionGroups(mod); + const sites = planRegionSites(mod, groups); + return { groups, sites }; +} + +const TREE = ` +class Node { + left: Node | null; + right: Node | null; + constructor(left: Node | null, right: Node | null) { + this.left = left; + this.right = right; + } + count(): number { + return 1 + (this.left === null ? 0 : this.left.count()) + (this.right === null ? 0 : this.right.count()); + } +} +function tree(depth: number): Node { + return depth === 0 ? new Node(null, null) : new Node(tree(depth - 1), tree(depth - 1)); +} +`; + +test("a recursive builder's root takes an arena for every node", async () => { + const mod = await moduleFor(`${TREE} +let total = 0; +for (let d = 0; d < 4; d++) total += tree(d).count(); +console.log(total); +`); + const { groups, sites } = plan(mod); + const roots = nodesIn(mod, "%init.0", calls("tree")).filter((c) => groups.roots.has(c)); + expect(roots.length).toBe(1); + for (const n of nodesIn(mod, "tree", (e) => e.kind === "new")) + expect(sites.arenaSites.has(n)).toBe(true); + for (const c of nodesIn(mod, "tree", calls("tree"))) expect(sites.arenaSites.has(c)).toBe(true); +}); + +test("a builder of a bounded number of objects is no root", async () => { + const mod = await moduleFor(` +interface V { x: number; y: number } +function add(a: V, b: V): V { + return { x: a.x + b.x, y: a.y + b.y }; +} +let at: V = { x: 0, y: 0 }; +let sum = 0; +for (let i = 0; i < 10; i++) { + const step = add(at, { x: i, y: 1 }); + sum += step.x; +} +console.log(sum, at.y); +`); + const { groups } = plan(mod); + expect(groups.roots.size).toBe(0); + expect(groups.calls.size).toBe(0); + expect(groups.reachable.size).toBe(0); +}); + +test("a list a builder appends to in a loop takes the arena", async () => { + const mod = await moduleFor(` +interface Row { id: number; score: number } +function rows(n: number): Row[] { + const out: Row[] = []; + for (let i = 0; i < n; i++) { + const row: Row = { id: i, score: i * 2 }; + out.push(row); + } + return out; +} +let total = 0; +for (let k = 0; k < 3; k++) total += rows(10 + k).length; +console.log(total); +`); + const { groups, sites } = plan(mod); + expect(nodesIn(mod, "%init.0", calls("rows")).some((c) => groups.roots.has(c))).toBe(true); + const [row] = nodesIn(mod, "rows", records); + expect(sites.arenaSites.has(row!)).toBe(true); + const [out] = nodesIn(mod, "rows", (e) => e.kind === "arrayLit"); + expect(sites.arenaSites.has(out!)).toBe(true); +}); + +test("values that replace elements or die in the loop take the heap", async () => { + const mod = await moduleFor(` +interface V { x: number; y: number } +function vec(x: number, y: number): V { + return { x, y }; +} +function add(a: V, b: V): V { + return { x: a.x + b.x, y: a.y + b.y }; +} +function simulate(n: number, steps: number): number { + const points: V[] = []; + for (let i = 0; i < n; i++) points.push(vec(i, i)); + for (let s = 0; s < steps; s++) + for (let i = 0; i < n; i++) points[i] = add(points[i]!, vec(0, 1)); + let sum = 0; + for (const p of points) sum += p.x + p.y; + return sum; +} +console.log(simulate(100, 10)); +`); + const { groups, sites } = plan(mod); + // The list's elements are replaced, so it keeps nothing: no group grows. + expect(groups.calls.size).toBe(0); + expect(groups.roots.size).toBe(0); + expect(sites.arenaSites.size).toBe(0); +}); + +test("a value reassigned in a loop and returned takes the heap", async () => { + const mod = await moduleFor(` +class C { + re: number; + im: number; + constructor(re: number, im: number) { + this.re = re; + this.im = im; + } + times(o: C): C { + return new C(this.re * o.re - this.im * o.im, this.re * o.im + this.im * o.re); + } +} +function power(c: C, n: number): C { + let z = new C(1, 0); + for (let i = 0; i < n; i++) z = z.times(c); + return z; +} +let s = 0; +for (let k = 0; k < 5; k++) s += power(new C(0.5, 0.5), k).re; +console.log(s); +`); + const { groups } = plan(mod); + expect(nodesIn(mod, "%init.0", calls("power")).some((c) => groups.roots.has(c))).toBe(false); +}); + +test("calls in an arena-aware function whose values die take the heap", async () => { + const mod = await moduleFor(` +interface P { x: number; next: P | null } +function weight(p: P): number { + return p.x * 2; +} +function probe(x: number): P { + return { x, next: null }; +} +function chain(n: number): P { + let head: P = { x: 0, next: null }; + const first = head; + for (let i = 1; i < n; i++) { + const node: P = { x: weight(probe(i)), next: null }; + head.next = node; + head = node; + } + return first; +} +let total = 0; +for (let k = 0; k < 3; k++) total += chain(5).x; +console.log(total); +`); + const { groups, sites } = plan(mod); + expect(nodesIn(mod, "%init.0", calls("chain")).some((c) => groups.roots.has(c))).toBe(true); + const [probe] = nodesIn(mod, "chain", calls("probe")); + expect(sites.arenaSites.has(probe!)).toBe(false); + expect(sites.heapCalls.has(probe!)).toBe(true); + for (const node of nodesIn(mod, "chain", records)) expect(sites.arenaSites.has(node)).toBe(true); +}); + +test("a root assigned in a loop to a local declared outside it is no root", async () => { + const mod = await moduleFor(`${TREE} +function run(n: number): number { + let t = tree(1); + let total = t.count(); + for (let d = 0; d < n; d++) { + t = tree(d); + total += t.count(); + } + return total; +} +console.log(run(6)); +`); + const { groups } = plan(mod); + const [first, inLoop] = nodesIn(mod, "run", calls("tree")); + expect(groups.roots.has(first!)).toBe(true); + expect(groups.roots.has(inLoop!)).toBe(false); +}); diff --git a/packages/compiler/src/backend/llvm/region-sites.ts b/packages/compiler/src/backend/llvm/region-sites.ts new file mode 100644 index 000000000..95167524b --- /dev/null +++ b/packages/compiler/src/backend/llvm/region-sites.ts @@ -0,0 +1,349 @@ +/* Which allocations of a region group take its arena (arenas.ts), and + * which roots and call regions are worth one. + * + * An arena pays off for objects that live until the group is dropped: it + * saves their frees and teardown, but holds their memory until the drop. + * A temporary that dies early is cheaper on the heap, which reuses its + * memory at once. So in an arena-aware function an allocation takes the + * arena only when its value is kept by the group (its fate is the arena): + * - it is returned; + * - it is content of a new value that is kept; + * - it is stored, other than by an element write, into the graph of a + * local that is kept (`out.push(row)`, `last.next = member`); + * - it is bound to a local that is kept. + * A local is kept when its value is returned or stored or put into + * something kept; a constructor's `this` is kept (its object follows its + * `new`); and in the callee of a call region, a local declared at the top + * of the body lives until the region drops. An element write (`a[i] = v`) + * usually replaces a value, so its value takes the heap. + * + * Every other allocation takes the heap, and so does everything a call + * allocates when the call's value is not kept: the emitter suspends the + * arena around such a call (heapCalls). Heap objects are always safe in a + * group; only arena objects need the group's proof. + * + * A root or call region is worth an arena when its group grows: a kept + * allocation runs in a loop, or a kept value comes from a recursive call + * or from a call of a function whose group grows. Other groups are small, + * and opening and dropping their arenas costs more than their frees. */ +import type { IrFunction, IrModule } from "../../ir/ir.js"; +import { contentOf, source, storeOf, storeTarget, type RegionGroups } from "./region-groups.js"; +import { + argsOf, + children, + indexOfNode, + isExpr, + refType, + same, + transparent, + type Node, +} from "./region-ir.js"; +import { VirtualTargets } from "./reference-effects.js"; + +export interface RegionSites { + /** Allocation sites (and calls) whose value the group keeps. */ + arenaSites: Set; + /** Calls, constructions and closure calls in arena-aware functions whose + * value the group does not keep and whose callee may allocate: the + * arena is suspended while they run. */ + heapCalls: Set; +} + +const LOOPS = new Set(["while", "doWhile", "for", "forOf"]); + +/** Collection methods that remove what their receiver holds. */ +const REMOVALS = new Set(["pop", "shift", "splice", "delete", "clear"]); + +/** The local whose graph `node` drops values from (an element write, a + * length write, a delete, a removing method), or null. */ +function removalTarget(node: Node): string | null { + switch (node.kind) { + case "arraySet": + case "arraySetLength": + case "arraySetUndefined": + case "arrayDelete": + return storeTarget(node.arr); + case "recordKeyDelete": + return storeTarget(node.obj); + case "arrIntrinsic": + case "mapIntrinsic": + case "setIntrinsic": + return REMOVALS.has(node.method) ? storeTarget(node.receiver) : null; + default: + return null; + } +} + +/** The functions a call-like node may run, or null when unknown. */ +function targetsOf(vt: VirtualTargets, node: Node): readonly string[] | null { + if (node.kind === "call") return [node.callee]; + if (node.kind === "new") return [`%${node.className}.constructor`]; + if (node.kind === "virtualCall") { + const t = vt.targets(node.className, node.method); + return t !== null && t.length > 0 ? t : null; + } + return null; +} + +const callLike = (node: Node): boolean => + node.kind === "call" || + node.kind === "new" || + node.kind === "virtualCall" || + node.kind === "callValue" || + node.kind === "newValue"; + +/** The functions that run inside the arenas of `roots`: what they call, + * transitively, with the closures they pass. */ +export function reachableFrom(mod: IrModule, roots: Iterable): Set { + const fns = new Map(mod.functions.map((f) => [f.name, f])); + const vt = new VirtualTargets(mod.classes ?? []); + const reachable = new Set(); + const queue: string[] = []; + const visit = (node: Node): void => { + const reach = (name: string): void => { + if (fns.has(name) && !reachable.has(name)) { + reachable.add(name); + queue.push(name); + } + }; + if (node.kind === "closure") reach(node.fnName); + else for (const t of targetsOf(vt, node) ?? []) reach(t); + const kids: Node[] = []; + children(node, kids); + for (const k of kids) visit(k); + }; + for (const r of roots) visit(r); + while (queue.length > 0) for (const st of fns.get(queue.pop()!)!.body) visit(st); + return reachable; +} + +interface FnSites { + /** Kept call-like nodes and allocations, with whether each is in a loop. */ + kept: Map; + /** Call-like nodes whose value is not kept. */ + dropped: Node[]; +} + +/** Classifies the allocations and calls of `fn`. */ +function classify(fn: IrFunction, topLocalsKept: boolean): FnSites { + const locals = new Set(fn.locals.map((l) => l.id)); + // Alias classes: a local bound to another local shares its fate. + const parent = new Map(); + const find = (id: string): string => { + let r = id; + while (parent.has(r)) r = parent.get(r)!; + return r; + }; + const unite = (a: string, b: string): void => { + const ra = find(a), + rb = find(b); + if (ra !== rb) parent.set(ra, rb); + }; + const kept = new Set(); + const keep = (id: string): boolean => { + const r = find(id); + if (kept.has(r)) return false; + kept.add(r); + return true; + }; + const isKept = (id: string): boolean => kept.has(find(id)); + // Kept from the start (applied once the alias classes are known). + const seeds: string[] = []; + if (fn.name.endsWith(".constructor") && fn.params.length > 0) seeds.push(fn.params[0]!.localId); + if (topLocalsKept) { + const types = new Map(fn.locals.map((l) => [l.id, l.type])); + for (const st of fn.body) + if (st.kind === "varDecl" && refType(types.get(st.localId) ?? { kind: "void" })) + seeds.push(st.localId); + } + + // Uses of locals and of value-producing nodes, recorded once. + type Fate = + | { kind: "kept" } + | { kind: "local"; id: string } + | { kind: "target"; id: string } + | { kind: "node"; node: Node } + | { kind: "heap" }; + const fateOf = (node: Node, parents: readonly Node[]): Fate => { + let value: Node = node; + let at = parents.length - 1; + while (at >= 0 && transparent(value, parents[at])) value = parents[at--]!; + const p = at >= 0 ? parents[at] : undefined; + if (p === undefined) return { kind: "heap" }; + if (p.kind === "return") return { kind: "kept" }; + if ( + (p.kind === "varDecl" && p.init !== null && same(p.init, value)) || + (p.kind === "assign" && same(p.value, value)) + ) { + const id = p.localId; + // A local assigned in a loop usually drops its old value. + if (!locals.has(id) || (p.kind === "assign" && depth > 0)) return { kind: "heap" }; + return { kind: "local", id }; + } + const target = storeOf(value, p); + if (target !== undefined) { + if (p.kind === "arraySet") return { kind: "heap" }; + const id = storeTarget(target); + return id !== null && locals.has(id) ? { kind: "target", id } : { kind: "heap" }; + } + if (isExpr(p) && contentOf(value, p)) return { kind: "node", node: p }; + // An argument of a call that returns a reference follows the call's + // value: the callee may put it into its result. + const args = argsOf(p); + if ( + callLike(p) && + isExpr(p) && + refType(p.type) && + args !== null && + indexOfNode(args, value) >= 0 + ) + return { kind: "node", node: p }; + return { kind: "heap" }; + }; + + // Lossy locals: their graphs drop values (element writes, removals), so + // what is stored into them is not kept. + const lossy = new Set(); + const fates = new Map(); + const loops = new Map(); + const localUses: [string, Fate][] = []; + const parents: Node[] = []; + let depth = 0; + const walk = (node: Node): void => { + const removal = removalTarget(node); + if (removal !== null) lossy.add(removal); + if (source(node) || callLike(node)) { + fates.set(node, fateOf(node, parents)); + loops.set(node, depth > 0); + } else if (node.kind === "varRef" && locals.has(node.localId) && refType(node.type)) { + const f = fateOf(node, parents); + if (f.kind !== "heap") localUses.push([node.localId, f]); + } + const loop = LOOPS.has(node.kind); + if (loop) depth++; + const kids: Node[] = []; + children(node, kids); + parents.push(node); + for (const k of kids) walk(k); + parents.pop(); + if (loop) depth--; + }; + for (const st of fn.body) walk(st); + + // A local bound to another local joins its class. + for (const [id, f] of localUses) if (f.kind === "local") unite(id, f.id); + for (const id of [...lossy]) lossy.add(find(id)); + for (const id of seeds) keep(id); + const keptFate = (f: Fate, seen = 0): boolean => { + if (seen > 64) return false; + switch (f.kind) { + case "kept": + return true; + case "local": + return isKept(f.id); + case "target": + return isKept(f.id) && !lossy.has(find(f.id)); + case "node": { + const g = fates.get(f.node); + return g !== undefined && keptFate(g, seen + 1); + } + case "heap": + return false; + } + }; + for (let changed = true; changed;) { + changed = false; + for (const [id, f] of localUses) + if (f.kind !== "local" && keptFate(f) && keep(id)) changed = true; + } + const out: FnSites = { kept: new Map(), dropped: [] }; + for (const [e, f] of fates) + if (keptFate(f)) out.kept.set(e, loops.get(e)!); + else if (callLike(e)) out.dropped.push(e); + return out; +} + +/** + * Narrows `groups` to the roots and call regions worth an arena (with + * `reachable` recomputed for them) and classifies the allocations of the + * functions that run in their arenas. + */ +export function planRegionSites(mod: IrModule, groups: RegionGroups): RegionSites { + const fns = new Map(mod.functions.map((f) => [f.name, f])); + const vt = new VirtualTargets(mod.classes ?? []); + const regionCallees = new Set(); + for (const c of groups.calls) if (c.kind === "call") regionCallees.add(c.callee); + const sites = new Map(); + for (const name of groups.reachable) { + const fn = fns.get(name); + if (fn !== undefined) sites.set(name, classify(fn, regionCallees.has(name))); + } + // Which functions reach which (for recursion), over the reachable ones. + const calleesOf = (name: string): Set => { + const out = new Set(); + const s = sites.get(name); + if (s === undefined) return out; + for (const e of [...s.kept.keys(), ...s.dropped]) + for (const t of targetsOf(vt, e) ?? []) if (sites.has(t)) out.add(t); + return out; + }; + const edges = new Map([...sites.keys()].map((n) => [n, calleesOf(n)])); + const reaches = (from: string, to: string): boolean => { + const seen = new Set(); + const stack = [from]; + while (stack.length > 0) { + const n = stack.pop()!; + if (n === to) return true; + if (seen.has(n)) continue; + seen.add(n); + for (const m of edges.get(n) ?? new Set()) stack.push(m); + } + return false; + }; + const grows = new Set(); + for (let changed = true; changed;) { + changed = false; + for (const [name, s] of sites) { + if (grows.has(name)) continue; + let g = false; + for (const [e, inLoop] of s.kept) { + if (inLoop) g = true; + for (const t of targetsOf(vt, e) ?? []) if (grows.has(t) || reaches(t, name)) g = true; + if (g) break; + } + if (g) { + grows.add(name); + changed = true; + } + } + } + const worth = (r: Node): boolean => { + // Module code runs once, and its groups hold the program's own state. + if (r.kind === "call") return grows.has(r.callee) && !r.callee.startsWith("%init."); + if (r.kind !== "new") return false; + if (grows.has(`%${r.className}.constructor`)) return true; + return r.args.some((a) => a.kind === "call" && grows.has(a.callee)); + }; + for (const r of [...groups.roots]) + if (!worth(r)) { + groups.roots.delete(r); + groups.drops.delete(r); + } + for (const c of [...groups.calls]) if (!worth(c)) groups.calls.delete(c); + groups.reachable = reachableFrom(mod, [...groups.roots, ...groups.calls]); + + const arenaSites = new Set(); + const heapCalls = new Set(); + const mayAllocate = (e: Node): boolean => { + const t = targetsOf(vt, e); + return t === null || t.some((n) => groups.allocates.has(n) || !fns.has(n)); + }; + for (const name of groups.reachable) { + const s = sites.get(name); + if (s === undefined) continue; + for (const e of s.kept.keys()) arenaSites.add(e); + for (const e of s.dropped) + if (!groups.roots.has(e) && !groups.calls.has(e) && mayAllocate(e)) heapCalls.add(e); + } + return { arenaSites, heapCalls }; +} diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index ea429d71a..b5b1e735a 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -60,6 +60,14 @@ import { type InlineUnionFields, } from "./inline-unions.js"; import { JS_BITS, JS_TAGS } from "./jsval-encoding.js"; +import { + ARENA_RC, + arenaDisposeLines, + arenaDisposeSym, + arenaNewLines, + disposeFieldLines, + plainRefType, +} from "./arenas.js"; import { recordMayKeepSource } from "./record-sources.js"; import { LlvmUnsupportedError } from "./unsupported.js"; @@ -137,6 +145,10 @@ export interface ShapeHost { * allocate in regions and count references in their region chunk instead * of being empty (llvm/regions.ts). */ regions?: RegionHooks | null; + /** Record shapes and classes with an arena-aware constructor + * (arenas.ts). */ + arenaShapes?: ReadonlySet; + arenaClasses?: ReadonlySet; /** The target C ABI's `size_t` / refcount integer type. */ readonly sizeType: "i32" | "i64"; /** Worker executables: the inline allocator state is thread-local. */ @@ -1198,22 +1210,25 @@ export function arrNewCall( /** An empty array expected to receive a few elements (its first slots * live in its own block): capText must be "0". */ small = false, + /** In the current group arena when one is open (arenas.ts). */ + arena = false, ): string { + const r = arena ? "_r" : ""; const useRef = arrUsesRef(host, elem); if (!useRef) { if (small) { - host.declare(`declare ptr @scr_arr_new_small(i32)`); - return `call ptr @scr_arr_new_small(i32 ${elemKindNum(elem)})`; + host.declare(`declare ptr @scr_arr_new_small${r}(i32)`); + return `call ptr @scr_arr_new_small${r}(i32 ${elemKindNum(elem)})`; } - host.declare(`declare ptr @scr_arr_new(i32, ${host.sizeType})`); - return `call ptr @scr_arr_new(i32 ${elemKindNum(elem)}, ${host.sizeType} ${capText})`; + host.declare(`declare ptr @scr_arr_new${r}(i32, ${host.sizeType})`); + return `call ptr @scr_arr_new${r}(i32 ${elemKindNum(elem)}, ${host.sizeType} ${capText})`; } if (small) { - host.declare(`declare ptr @scr_arr_new_ops_small(ptr)`); - return `call ptr @scr_arr_new_ops_small(ptr ${arrOps(host, elem)})`; + host.declare(`declare ptr @scr_arr_new_ops_small${r}(ptr)`); + return `call ptr @scr_arr_new_ops_small${r}(ptr ${arrOps(host, elem)})`; } - host.declare(`declare ptr @scr_arr_new_ops(ptr, ${host.sizeType})`); - return `call ptr @scr_arr_new_ops(ptr ${arrOps(host, elem)}, ${host.sizeType} ${capText})`; + host.declare(`declare ptr @scr_arr_new_ops${r}(ptr, ${host.sizeType})`); + return `call ptr @scr_arr_new_ops${r}(ptr ${arrOps(host, elem)}, ${host.sizeType} ${capText})`; } /** The ScrElemKind and ops descriptor (or `null`) an array of `elem` @@ -2327,6 +2342,34 @@ export function emitRecordShapes( if (traced) nw.push(...youngEntryLines(host)); nw.push(` ret ptr %o`, `}`, ``); defs.push(...nw); + if (host.arenaShapes?.has(shape.id) === true && !shape.indexValue) { + const newSym = mangleRecordNew(shape.id); + const dispose = refMembers.length > 0 ? arenaDisposeSym(newSym) : null; + const init = [` store ${host.sizeType} ${ARENA_RC}, ptr %o`]; + if (shapeNumber !== 0) + init.push( + ` %shp = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${SHAPE_WORD_INDEX}`, + ` store i32 ${shapeNumber}, ptr %shp ; shape number`, + ); + defs.push( + ...arenaNewLines(host, newSym, sizeOf, traced ? cycType : null, dispose, init, "%o"), + ); + if (dispose !== null) + defs.push( + ...arenaDisposeLines( + newSym, + refMembers.flatMap((m, i) => + disposeFieldLines( + i, + `getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${m.index}`, + plainRefType(m.type), + releaseSym(host, m.type), + llvmCommentText(m.name), + ), + ), + ), + ); + } if (host.recordCloneShapes.has(shape.id)) { const attrs = shape.fields.length >= 16 ? "#2" : FN_ATTRS; diff --git a/packages/compiler/src/backend/llvm/stack-objects.ts b/packages/compiler/src/backend/llvm/stack-objects.ts index ae2b904db..36793598b 100644 --- a/packages/compiler/src/backend/llvm/stack-objects.ts +++ b/packages/compiler/src/backend/llvm/stack-objects.ts @@ -57,8 +57,9 @@ export function canStackObject(host: LlvmEmitterContext, e: IrExpr): e is StackO if (e.type.kind === "record" && host.recordsById.get(e.type.shapeId)?.freezable === true) return false; // A producer with a result-slot entry builds its fresh result in the - // caller's storage. - if (e.kind === "call") return host.fnByName.has(resultEntryName(e.callee)); + // caller's storage, unless the call roots a group: then the result goes + // in the group's arena with the objects it holds (arenas.ts). + if (e.kind === "call") return host.fnByName.has(resultEntryName(e.callee)) && !host.opensArena(e); if (e.kind === "recordLit") { if (e.type.kind !== "record") return false; const shape = host.recordsById.get(e.type.shapeId); diff --git a/packages/compiler/src/backend/loop-polls.ts b/packages/compiler/src/backend/loop-polls.ts index 866711fb7..b49026fa2 100644 --- a/packages/compiler/src/backend/loop-polls.ts +++ b/packages/compiler/src/backend/loop-polls.ts @@ -137,3 +137,18 @@ export function loopAccessesTypedArrays(loop: Loop): boolean { }; return loop.body.some(inStmt); } + +/** Whether `body` holds a loop outside its closures. */ +export function containsLoop(body: readonly IrStmt[]): boolean { + const inStmt = (s: IrStmt): boolean => + s.kind === "while" || + s.kind === "doWhile" || + s.kind === "for" || + s.kind === "forOf" || + !everyStmtChild( + s, + () => true, + (child) => !inStmt(child), + ); + return body.some(inStmt); +} diff --git a/packages/compiler/src/backend/may-throw.test.ts b/packages/compiler/src/backend/may-throw.test.ts index 0a950c2cc..1bbbdb299 100644 --- a/packages/compiler/src/backend/may-throw.test.ts +++ b/packages/compiler/src/backend/may-throw.test.ts @@ -406,7 +406,7 @@ test("worker programs poll for termination only where execution can be unbounded expect(computeMayThrow({ ...mod, workers: false }).workerEntryPolls).toBeUndefined(); }); -test("stack checks cover recursive components and recursion through reentrant operations", () => { +test("stack checks cover every recursive cycle, including recursion through reentrant operations", () => { const mod = moduleWith( fn("caller", [call("self"), call("left"), call("leaf"), call("walker")], []), fn("self", [call("self"), call("leaf")], []), @@ -420,7 +420,12 @@ test("stack checks cover recursive components and recursion through reentrant op fn("visitor", [callClosure("leaf")], []), ); const answer = computeMayThrow(mod, { stackChecks: true }); - expect([...answer.stackChecks!].sort()).toEqual(["left", "right", "self", "step", "walker"]); + // One check per cycle: self, one of left and right, and one of walker and + // step (the earlier function on ties). + expect([...answer.stackChecks!].sort()).toEqual(["left", "self", "walker"]); + expect( + [...computeMayThrow(mod, { stackChecks: "everyCycleMember" }).stackChecks!].sort(), + ).toEqual(["left", "right", "self", "step", "walker"]); // Checked functions throw the overflow RangeError, so their callers poll; // a throwing closure target makes every indirect call poll as well. expect(answer.indirect).toBe(true); @@ -437,7 +442,7 @@ test("stack checks cover recursive components and recursion through reentrant op expect(computeMayThrow(mod).fns.size).toBe(0); }); -test("stack-checked synchronous functions skip the worker entry poll", () => { +test("synchronous functions on a checked cycle skip the worker entry poll", () => { const asyncSelf: IrFunction = { ...fn("asyncSelf", [call("asyncSelf")], []), async: true }; const mod: IrModule = { ...moduleWith( @@ -452,10 +457,11 @@ test("stack-checked synchronous functions skip the worker entry poll", () => { workers: true, }; const answer = computeMayThrow(mod, { stackChecks: true }); - expect([...answer.stackChecks!].sort()).toEqual(["asyncSelf", "caller", "left", "right", "self"]); - // Callers test after every call of a stack-checked function, and its - // overflow unwinds through those tests, so only the module entry, async - // bodies and unbounded functions without a stack check poll on entry. + expect([...answer.stackChecks!].sort()).toEqual(["asyncSelf", "caller", "left", "self"]); + // Callers test after every call of a function on a checked cycle (it may + // throw), and an overflow unwinds through those tests, so only the module + // entry, async bodies and unbounded functions off any cycle poll on entry; + // right needs no poll although left holds the cycle's check. expect([...answer.workerEntryPolls!].sort()).toEqual(["asyncSelf", "caller", "walker"]); expect(answer.fns.has("self") && answer.fns.has("left") && answer.fns.has("right")).toBe(true); // Without stack checks (WASI, Windows, libraries) every cycle keeps its poll. @@ -469,6 +475,48 @@ test("stack-checked synchronous functions skip the worker entry poll", () => { ]); }); +test("stack checks meet every cycle once and bound the runs between checks", () => { + // A ring of 80 functions, with a chord from fn10 back to fn0, and a + // separate ring of three whose functions also call each other backwards. + const ring = Array.from({ length: 80 }, (_, i) => + fn(`fn${i}`, i === 10 ? [call("fn11"), call("fn0")] : [call(`fn${(i + 1) % 80}`)], []), + ); + const mod = moduleWith( + ...ring, + fn("a", [call("b"), call("c")], []), + fn("b", [call("c"), call("a")], []), + fn("c", [call("a"), call("b")], []), + ); + const checks = computeMayThrow(mod, { stackChecks: true }).stackChecks!; + // Removing the checked functions leaves no cycle, and no run of more than + // 32 unchecked calls. + const callees = new Map( + mod.functions.map((f) => [ + f.name, + f.body.flatMap((s) => + s.kind === "exprStmt" && s.expr.kind === "call" ? [s.expr.callee] : [], + ), + ]), + ); + const longest = new Map(); + const visiting = new Set(); + const run = (name: string): number => { + if (checks.has(name)) return 0; + expect(visiting.has(name)).toBe(false); + const known = longest.get(name); + if (known !== undefined) return known; + visiting.add(name); + const depth = 1 + Math.max(0, ...callees.get(name)!.map(run)); + visiting.delete(name); + longest.set(name, depth); + return depth; + }; + for (const f of mod.functions) expect(run(f.name)).toBeLessThanOrEqual(32); + // fn0 breaks both ring cycles, fn33 and fn66 bound the run along the + // ring, and two of a, b and c break their four cycles. + expect([...checks].sort()).toEqual(["a", "b", "fn0", "fn33", "fn66"]); +}); + test("class members are reachable from reentrant operations", () => { const mod: IrModule = { ...moduleWith( diff --git a/packages/compiler/src/backend/may-throw.ts b/packages/compiler/src/backend/may-throw.ts index 693d20b3b..53b0e56b4 100644 --- a/packages/compiler/src/backend/may-throw.ts +++ b/packages/compiler/src/backend/may-throw.ts @@ -37,7 +37,11 @@ export type MayThrowReason = export function computeMayThrow( mod: IrModule, options: { - stackChecks?: boolean; + /** Native stack checks: true for the functions that check on entry (one + * per call cycle, see stackCheckedFunctions); "everyCycleMember" for a + * model in which every function on a cycle checks, which answers a + * superset that does not depend on that choice. */ + stackChecks?: boolean | "everyCycleMember"; explain?: boolean; /** computePublishedTypes(mod), when the caller already has it. */ published?: ReturnType; @@ -56,8 +60,16 @@ export function computeMayThrow( stackChecks?: Set; } { const terminations = mod.workers === true ? workerTerminationPolls(mod, options.nodes) : null; + const stackGraph = + options.stackChecks === undefined || options.stackChecks === false + ? null + : stackCheckGraph(mod, options.nodes); const stackChecks = - options.stackChecks === true ? stackCheckedFunctions(mod, options.nodes) : null; + stackGraph === null + ? null + : options.stackChecks === "everyCycleMember" + ? stackGraph.cyclic + : cycleGuards(stackGraph); // Stores that can reach a published object throw the frozen TypeError. const published = options.published !== undefined ? options.published : computePublishedTypes(mod); @@ -488,7 +500,9 @@ export function computeMayThrow( fns: may, ...(why !== null ? { why } : {}), indirect, - ...(terminations ? { workerEntryPolls: entryPolls(mod, terminations.entry, stackChecks) } : {}), + ...(terminations + ? { workerEntryPolls: entryPolls(mod, terminations.entry, stackGraph?.cyclic ?? null, may) } + : {}), ...(stackChecks ? { stackChecks } : {}), }; } @@ -720,28 +734,32 @@ function describeThrowingNode(rec: IrExpr | IrStmt): string { } /** The termination polls a worker program places on function entry. A - * stack-checked synchronous function needs none: it may throw (the stack - * guard seeds the fixpoint), so every caller tests the pending state after - * the call, and a descent that never returns ends at the stack guard, whose - * RangeError unwinds through those tests. An outstanding terminate request - * keeps the alert word nonzero, so the first test that runs afterwards (a - * post-call test, a loop poll or the unwinding) stops the thread; dropping - * the entry poll only delays the stop by bounded work, which Node's - * asynchronous terminate allows. The entry poll's slow path rejoins the - * body, so every polled function kept its arguments live across a call. */ + * synchronous function on a call cycle that may throw needs none: every + * caller tests the pending state after the call, and a descent that never + * returns passes a stack-checked function of each cycle again and again + * (stackCheckedFunctions) and ends at its guard, whose RangeError unwinds + * through those tests. An outstanding terminate request keeps the alert word + * nonzero, so the first test that runs afterwards (a post-call test, a loop + * poll or the unwinding) stops the thread; dropping the entry poll only + * delays the stop by bounded work, which Node's asynchronous terminate + * allows. The entry poll's slow path rejoins the body, so every polled + * function kept its arguments live across a call. */ function entryPolls( mod: IrModule, entry: Set, - stackChecks: Set | null, + cyclic: Set | null, + may: Set, ): Set { - if (stackChecks === null) return entry; + if (cyclic === null) return entry; // Calling an async or generator function never tests the caller's pending // state, and the module entry has no caller: those keep their polls. const keep = new Set( mod.functions.filter((fn) => fn.async || fn.generator !== undefined).map((fn) => fn.name), ); keep.add(mod.entry); - return new Set([...entry].filter((name) => keep.has(name) || !stackChecks.has(name))); + return new Set( + [...entry].filter((name) => keep.has(name) || !cyclic.has(name) || !may.has(name)), + ); } /** Statement and expression kinds that cannot reenter scriptc code except @@ -971,16 +989,31 @@ export function workerTerminationPolls( return { entry, polls }; } -/** The functions that can take part in unbounded recursion, which check the - * native stack guard on entry: every function on a cycle of the call graph. - * Direct edges are calls, constructions and virtual dispatch. A function - * containing any operation that can reenter scriptc code other than through - * those edges (an indirect call, a callback-taking or dynamic operation, a - * library call; see boundedNode) also reaches every function that such an - * operation can invoke: closure targets and class members. Non-recursive - * functions, including leaf callbacks, stay check-free. */ +/** The call graph that bounds recursion, and the functions on its cycles: + * the ones that can take part in unbounded recursion. Direct edges are + * calls, constructions and virtual dispatch. A function containing any + * operation that can reenter scriptc code other than through those edges + * (an indirect call, a callback-taking or dynamic operation, a library call; + * see boundedNode) also reaches every function that such an operation can + * invoke: closure targets and class members, behind one shared `reentry` + * node. */ +interface StackCheckGraph { + edges: Map; + cyclic: Set; +} + +const REENTRY = "\0reentry"; + +/** The functions that check the native stack guard on entry: one function + * of every cycle of the call graph (stackCheckGraph), which bounds every + * unbounded recursion, since its frames repeat the cycle's guarded function + * again and again. Non-recursive functions, including leaf callbacks, stay + * check-free. See cycleGuards for the choice. */ export function stackCheckedFunctions(mod: IrModule, nodes?: IrNodeIndex): Set { - const reentry = "\0reentry"; + return cycleGuards(stackCheckGraph(mod, nodes)); +} + +function stackCheckGraph(mod: IrModule, nodes?: IrNodeIndex): StackCheckGraph { const methodImpls = new Map(); const memberPrefixes = new Set(); for (const cls of mod.classes ?? []) { @@ -1019,16 +1052,157 @@ export function stackCheckedFunctions(mod: IrModule, nodes?: IrNodeIndex): Set names.has(callee)); - if (reenters) callees.push(reentry); + if (reenters) callees.push(REENTRY); edges.set(fn.name, callees); } edges.set( - reentry, + REENTRY, [...targets].filter((name) => names.has(name)), ); const cyclic = cyclicFunctions(edges); - cyclic.delete(reentry); - return cyclic; + cyclic.delete(REENTRY); + return { edges, cyclic }; +} + +/** The longest run of calls between unchecked functions of a cycle that + * cycleGuards allows; deeper runs get a check. Acyclic call chains below a + * check are as deep as the program makes them already, so this keeps the + * stack used between two checks of the same order. */ +const MAX_UNCHECKED_CYCLE_RUN = 32; + +/** A small set of functions that meets every cycle of the stack-check graph + * (a feedback vertex set), chosen greedily: functions that cannot lie on a + * remaining cycle (no caller or no callee left among the candidates) drop + * out, a function that calls itself is chosen, and otherwise the function + * with the most caller-callee pairs, the one that breaks the most cycles, + * is chosen and removed; ties go to the earlier function. The reentry node + * is not a function and is never chosen. A last pass in topological order + * of the unchosen cycle functions adds a check where a run of them would + * exceed MAX_UNCHECKED_CYCLE_RUN calls. */ +function cycleGuards(graph: StackCheckGraph): Set { + const { edges, cyclic } = graph; + const chosen = new Set(); + if (cyclic.size === 0) return chosen; + const order = new Map(); + const callees = new Map>(); + const callers = new Map>(); + const candidates = [...[...edges.keys()].filter((name) => cyclic.has(name)), REENTRY]; + for (const name of candidates) { + order.set(name, order.size); + callees.set(name, new Set()); + callers.set(name, new Set()); + } + for (const name of candidates) + for (const callee of edges.get(name) ?? []) + if (callees.has(callee)) { + callees.get(name)!.add(callee); + callers.get(callee)!.add(name); + } + // Max-heap of [score, order, name]; an entry is stale once its function + // is gone or its score changed, and is skipped when popped. + const heap: [number, number, string][] = []; + const score = (name: string): number => + name === REENTRY ? -1 : callers.get(name)!.size * callees.get(name)!.size; + const above = (x: [number, number, string], y: [number, number, string]): boolean => + x[0] > y[0] || (x[0] === y[0] && x[1] < y[1]); + const push = (name: string): void => { + heap.push([score(name), order.get(name)!, name]); + for (let i = heap.length - 1; i > 0;) { + const parent = (i - 1) >> 1; + if (!above(heap[i]!, heap[parent]!)) break; + [heap[i], heap[parent]] = [heap[parent]!, heap[i]!]; + i = parent; + } + }; + const pop = (): [number, number, string] => { + const top = heap[0]!; + const last = heap.pop()!; + if (heap.length > 0) { + heap[0] = last; + for (let i = 0; ;) { + let best = i; + for (const child of [2 * i + 1, 2 * i + 2]) + if (child < heap.length && above(heap[child]!, heap[best]!)) best = child; + if (best === i) break; + [heap[i], heap[best]] = [heap[best]!, heap[i]!]; + i = best; + } + } + return top; + }; + const doomed: string[] = []; + const remove = (name: string): void => { + const out = callees.get(name)!; + const into = callers.get(name)!; + callees.delete(name); + callers.delete(name); + for (const callee of out) + if (callee !== name) { + const set = callers.get(callee)!; + set.delete(name); + if (set.size === 0) doomed.push(callee); + else push(callee); + } + for (const caller of into) + if (caller !== name) { + const set = callees.get(caller)!; + set.delete(name); + if (set.size === 0) doomed.push(caller); + else push(caller); + } + }; + // Removes the functions that can no longer lie on a cycle, transitively. + const settle = (): void => { + while (doomed.length > 0) { + const name = doomed.pop()!; + if (callees.has(name)) remove(name); + } + }; + for (const name of candidates) { + if (callees.get(name)!.size === 0 || callers.get(name)!.size === 0) doomed.push(name); + else push(name); + } + settle(); + for (const name of candidates) + if (callees.get(name)?.has(name) === true) { + chosen.add(name); + remove(name); + settle(); + } + while (heap.length > 0) { + const [entryScore, , name] = pop(); + if (!callees.has(name) || entryScore !== score(name) || name === REENTRY) continue; + chosen.add(name); + remove(name); + settle(); + } + // Bound the runs of unchecked cycle functions (acyclic once the chosen + // ones are removed; the reentry node takes no frame): the longest run + // ending at each function, in topological order, and a check where it + // would grow too long. + const unchecked = candidates.filter((name) => !chosen.has(name)); + const pending = new Map(unchecked.map((name) => [name, 0])); + for (const name of unchecked) + for (const callee of new Set(edges.get(name))) + if (pending.has(callee)) pending.set(callee, pending.get(callee)! + 1); + const run = new Map(); + const ready = unchecked.filter((name) => pending.get(name) === 0); + for (let i = 0; i < ready.length; i++) { + const name = ready[i]!; + let depth = (run.get(name) ?? 0) + (name === REENTRY ? 0 : 1); + if (depth > MAX_UNCHECKED_CYCLE_RUN && name !== REENTRY) { + chosen.add(name); + depth = 0; + } + for (const callee of new Set(edges.get(name))) { + if (!pending.has(callee)) continue; + run.set(callee, Math.max(run.get(callee) ?? 0, depth)); + const left = pending.get(callee)! - 1; + pending.set(callee, left); + if (left === 0) ready.push(callee); + } + } + return chosen; } /** Functions on a cycle of the call graph (Tarjan's strongly connected diff --git a/packages/compiler/src/backend/native-toolchain.test.ts b/packages/compiler/src/backend/native-toolchain.test.ts index 0ac21e853..699847a2a 100644 --- a/packages/compiler/src/backend/native-toolchain.test.ts +++ b/packages/compiler/src/backend/native-toolchain.test.ts @@ -39,6 +39,8 @@ import { warmNativeCaches, } from "./native-toolchain.js"; import { splitLlvmProgram } from "./llvm/split.js"; +import { implicitToolchainFingerprints } from "./native/compiler-fingerprint.js"; +import { fingerprintDependencyPaths } from "./native/dependency-files.js"; const scratch: string[] = []; const TEST_CACHE_IDENTITY = "cc-cache-tests-v1"; @@ -326,9 +328,9 @@ test.skipIf( let vendorEntries = await readdir(vendorRoot); expect(vendorEntries.filter((name) => name.startsWith("mbedtls-")).length).toBe(2); - // Host clang uses system zlib, so only the Zig build materializes a zlib - // object family in the shared vendor root. - expect(vendorEntries.filter((name) => name.startsWith("zlib-")).length).toBe(1); + // Host clang and Zig each materialize their own vendored zlib object + // family in the shared vendor root. + expect(vendorEntries.filter((name) => name.startsWith("zlib-")).length).toBe(2); // Invalidate the host archive. A targetless Zig rebuild must continue to // use the separately keyed Zig archive instead of repairing or consuming // the host-clang entry. @@ -630,20 +632,27 @@ test.skipIf(clangExecutable === undefined)( async () => { // clang ships its x86 intrinsic headers (, ...) for every // target, so __has_include finds them on arm64, where including them is - // an #error, and the same holds for the ARM headers on x86. The probe - // must guard them by target. Only the compiler's own freestanding - // headers are visible here, so the result does not depend on the host's - // C library. + // an #error, and the same holds for the ARM headers on x86 and for + // (zlib's RVV code) outside RISC-V with the vector + // extension. The probe must guard them by target. Only the compiler's own + // freestanding headers are visible here, so the result does not depend on + // the host's C library. const includes = (await implicitDependencyProbeIncludes(runtimeSrcDir())).filter( (include) => - include !== "" && /intrin\.h>$|^$|^$/.test(include), + include !== "" && + /intrin\.h>$|^$|^$|^$/.test(include), ); expect(includes).toContain(""); + expect(includes).toContain(""); const resourceDir = execFileSync(clangExecutable!, ["-print-resource-dir"], { encoding: "utf8", }).trim(); const source = includes.map(implicitDependencyIncludeDirective).join(""); - for (const target of ["aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]) { + for (const target of [ + "aarch64-unknown-linux-gnu", + "riscv64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + ]) { const result = spawnSync( clangExecutable!, [ @@ -665,6 +674,29 @@ test.skipIf(clangExecutable === undefined)( }, ); +test.skipIf(clangExecutable === undefined)( + "the implicit toolchain probe leaves out a header the compiler rejects", + async () => { + // A system header that the runtime spells, shadowed by one that is an + // #error, as clang 18's is outside RISC-V. The probe + // still covers the other headers. + const dir = await mkdtemp(join(tmpdir(), "scriptc-probe-header-")); + scratch.push(dir); + await mkdir(join(dir, "sys")); + const rejected = join(dir, "sys", "auxv.h"); + await writeFile(rejected, '#error "not for this target"\n'); + const includes = await implicitDependencyProbeIncludes(runtimeSrcDir()); + expect(includes).toContain(""); + const fingerprints = await implicitToolchainFingerprints( + { argv: [clangExecutable!, "-isystem", dir], targetArgs: [], target: null }, + "probe-rejected-header", + ); + const dependencies = fingerprintDependencyPaths(fingerprints.complete); + expect(dependencies).not.toContain(rejected); + expect(dependencies.some((path) => path.endsWith("/stdatomic.h"))).toBe(true); + }, +); + test.skipIf( process.platform === "win32" || clangExecutable === undefined || diff --git a/packages/compiler/src/backend/native/compiler-fingerprint.ts b/packages/compiler/src/backend/native/compiler-fingerprint.ts index 6f92707f6..3df6cb449 100644 --- a/packages/compiler/src/backend/native/compiler-fingerprint.ts +++ b/packages/compiler/src/backend/native/compiler-fingerprint.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { type CcDriver } from "./driver.js"; import { resolvedToolIdentity, resolvedTool } from "./tool-identity.js"; import { execFileAsync } from "./process.js"; @@ -168,7 +168,7 @@ export function effectiveCompilerInvocationFingerprint( export async function implicitDependencyProbeIncludes(rtDir: string): Promise { const vendor = join(rtDir, "..", "vendor"); const quickjsSources = new Set(LRE_SOURCES); - const zlibSources = new Set(ZLIB_SOURCES); + const zlibSources = new Set(ZLIB_SOURCES.map((source) => basename(source))); const fileGroups = await Promise.all([ nativeSourceFiles(rtDir, false), nativeSourceFiles(join(vendor, "ryu"), false), @@ -181,7 +181,7 @@ export async function implicitDependencyProbeIncludes(rtDir: string): Promise name.endsWith(".h") || zlibSources.has(name), ), ]); @@ -199,13 +199,16 @@ export async function implicitDependencyProbeIncludes(rtDir: string): Promise, * , ...), , and . */ +/** clang's x86 intrinsic headers: every <...intrin.h> but the Windows + * (, , ...), and . */ function x86OnlyHeader(include: string): boolean { - return include === "" || /^<(x86|x86gpr|[a-z]*mm)intrin\.h>$/.test(include); + return include === "" || /^<[a-z0-9_]+intrin\.h>$/.test(include); } /** One guarded line of the implicit-toolchain probe source: the header is * included only when the compiler can find it and it belongs to the target - * being probed. */ + * being probed. clang ships for every target, and through + * clang 18 including it without the RISC-V vector extension is an #error. */ export function implicitDependencyIncludeDirective(include: string): string { const targetCondition = x86OnlyHeader(include) ? " && (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64))" @@ -213,7 +216,9 @@ export function implicitDependencyIncludeDirective(include: string): string { ? " && defined(_WIN32)" : include === "" || include === "" || include === "" ? " && (defined(__arm__) || defined(__aarch64__) || defined(_M_ARM) || defined(_M_ARM64))" - : ""; + : include === "" + ? " && defined(__riscv_vector)" + : ""; return `#if __has_include(${include})${targetCondition}\n#include ${include}\n#endif\n`; } @@ -394,11 +399,8 @@ async function implicitToolchainFingerprintsFresh( ]; const [dependencyResults, linker, assembler, compilerInvocation] = await Promise.all([ Promise.all( - sources.map((source) => - execFileAsync(compiler, [...probeArgs, source], { - cwd: probeDir, - maxBuffer: 16 * 1024 * 1024, - }), + sources.map((source, index) => + probeIncludeDependencies(compiler, probeArgs, probeDir, source, sourceGroups[index]!), ), ), execFileAsync(compiler, [...prefix, "-print-prog-name=ld"], { cwd: probeDir }), @@ -418,14 +420,7 @@ async function implicitToolchainFingerprintsFresh( const toolSpellings = [linker.stdout.trim(), assemblerSpelling].filter( (value, index, all) => value !== "" && all.indexOf(value) === index, ); - const sourceSet = new Set(sources); - const dependencyPaths = [ - ...new Set( - dependencyResults - .flatMap((dependencies) => parseMakeDependencies(dependencies.stdout)) - .filter((path) => !sourceSet.has(path)), - ), - ].sort(); + const dependencyPaths = [...new Set(dependencyResults.flat())].sort(); probe = { compilerIdentity: compilerIdentity ?? `\0${compiler}`, compilerInvocation: normalizedProbeInvocation(compilerInvocation, probeDir), @@ -508,6 +503,51 @@ async function implicitToolchainFingerprintsFresh( }; } +/** Headers the dependency probe preprocesses at a time when it probes them + * one by one. */ +const SINGLE_HEADER_PROBES = 8; + +/** The files the compiler reads to preprocess `source`, a probe source that + * includes `includes`, apart from the probe's own sources. When the compiler + * rejects `source`, for example for a header it finds that is an #error + * outside its target ( through clang 18 without the RISC-V + * vector extension), each header is preprocessed on its own, and a header + * that the compiler rejects on its own is left out, as an absent one is. + * Throws the compiler's error when it rejects every header. */ +async function probeIncludeDependencies( + compiler: string, + probeArgs: readonly string[], + probeDir: string, + source: string, + includes: readonly string[], +): Promise { + const dependencies = async (path: string): Promise => { + const result = await execFileAsync(compiler, [...probeArgs, path], { + cwd: probeDir, + maxBuffer: 16 * 1024 * 1024, + }); + return parseMakeDependencies(result.stdout).filter((dependency) => dependency !== path); + }; + try { + return await dependencies(source); + } catch (error) { + const found: string[][] = []; + for (let start = 0; start < includes.length; start += SINGLE_HEADER_PROBES) { + const batch = includes.slice(start, start + SINGLE_HEADER_PROBES); + const results = await Promise.all( + batch.map(async (include, offset) => { + const single = `${source.slice(0, -2)}-${start + offset}.c`; + await writeFile(single, implicitDependencyIncludeDirective(include)); + return dependencies(single).catch(() => null); + }), + ); + for (const result of results) if (result !== null) found.push(result); + } + if (found.length === 0) throw error; + return found.flat(); + } +} + const stableImplicitToolchainMemos = new Map>(); export function implicitToolchainFingerprints( driver: Pick, diff --git a/packages/compiler/src/backend/native/contracts.ts b/packages/compiler/src/backend/native/contracts.ts index 252518ffb..b4dd18459 100644 --- a/packages/compiler/src/backend/native/contracts.ts +++ b/packages/compiler/src/backend/native/contracts.ts @@ -76,14 +76,9 @@ export interface CcOptions { fetch?: boolean; /** The program uses zlib (index.ts detects zlib.* libCalls on the IR): * compiles scr_zlib.c — the regex gating precedent, so zlib-free - * binaries keep their exact link line. The default host-clang build links - * the SYSTEM libz (macOS ships it), byte-identical to the historical line; - * every Zig build compiles the vendored zlib with the selected driver instead - * (ensureZlibObjects — zig has no libz in its sysroots). Compressed bytes may - * differ between the - * system and vendored libraries, which is why the corpus only ever - * compares round-trips and fixed-blob inflation, never raw deflate - * output. */ + * binaries keep their exact link line. Every driver compiles the vendored + * zlib, the copy Node bundles (ensureZlibObjects), so compressed bytes match + * Node's. */ zlib?: boolean; /** The program uses node:zlib's Brotli forms (with zlib): compiles * scr_brotli.c and links the vendored brotli archive. */ diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index b5072bf25..8b5c8bd8e 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -217,18 +217,13 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ // Blob and File (every blob.* call selects dynAsync, whose unit boxes // their reads as dyn promises). unit((u) => u.opts.dynAsync, "scr_blob.c"), - // The zlib UNIT (scr_zlib.c) gates on zlib.* IR use; the LINK (system - // libz on the default host-clang build, vendored objects on Zig builds) - // also serves the native fetch's gzip decoder — spread exactly once. + // The zlib UNIT (scr_zlib.c) gates on zlib.* IR use; the vendored zlib + // objects also serve the native fetch's gzip decoder — spread exactly once. unitArgs((u) => u.opts.zlib - ? isZigDriver(u.driver) - ? ["-I", vendorZlibDir(), u.rt(join(u.rtDir, "scr_zlib.c")), ...u.zlibObjects] - : [u.rt(join(u.rtDir, "scr_zlib.c"))] + ? ["-I", vendorZlibDir(), u.rt(join(u.rtDir, "scr_zlib.c")), ...u.zlibObjects] : u.nativeFetch - ? isZigDriver(u.driver) - ? ["-I", vendorZlibDir(), ...u.zlibObjects] - : [] + ? ["-I", vendorZlibDir(), ...u.zlibObjects] : [], ), // Brotli (scr_brotli.c) plugs into the zlib engine and links the @@ -390,7 +385,7 @@ const RUNTIME_DEFINES: readonly { when: (opts: CcOptions) => boolean | undefined; }[] = [ { define: "SCR_TEXT_DECODER_LEGACY", when: (opts) => opts.textDecoderLegacy }, - // Checked objects carry the dynamic tier's shapes (scr_json.c's SCR_JS_TIER build); only + // scr_json.c's SCR_JS_TIER build exports the dynamic tier's cache primitives; only // programs with dynamic-tier code select it. { define: "SCR_JS_TIER", when: (opts) => opts.dynTier === true }, { define: "SCR_WARNING_REPLACE", when: (opts) => opts.warningReplace }, @@ -676,11 +671,11 @@ export async function compileCInternal( let lreObjects = regex ? lreObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot) : []; - // Vendored zlib is the Zig story — default host-clang builds keep the exact - // historical `-lz` system link (see CcOptions.zlib). The native fetch's - // gzip decoder rides the same objects/link. + // Every driver links the vendored zlib (the copy Node bundles), so + // compressed bytes match Node's. The native fetch's gzip decoder rides the + // same objects. let zlibObjects = - ((opts.zlib ?? false) || nativeFetch) && isZigDriver(driver) + (opts.zlib ?? false) || nativeFetch ? zlibObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot) : []; const materializeVendorPrerequisites = async ( @@ -829,11 +824,6 @@ export async function compileCInternal( ...(opts.linkInputs ?? []), ...(opts.systemLibraries ?? []).map((name) => `-l${name}`), ...(opts.frameworks ?? []).flatMap((name) => ["-framework", name]), - // GNU ld resolves libraries from left to right and commonly enables - // --as-needed: host-clang libz must follow scr_zlib.c/scr_fetch.c and every - // generated/native input that references inflate symbols. Cross - // and targetless Zig builds use vendored zlib objects in the input section above. - ...(((opts.zlib ?? false) || nativeFetch) && !isZigDriver(driver) ? ["-lz"] : []), ...unwinderLinkArgs(driver), // glibc keeps libm separate from libc. This must trail the generated // program and every native FFI input because GNU ld resolves archives @@ -1087,7 +1077,6 @@ export async function compileCInternal( ...(targetPlatform(driver) === "win32" ? ["-ladvapi32", "-liphlpapi", "-lws2_32"] : []), ...(mbedtls && targetPlatform(driver) === "win32" ? ["-lbcrypt"] : []), ...(tlsCa && targetPlatform(driver) === "win32" ? ["-lcrypt32"] : []), - ...(((opts.zlib ?? false) || nativeFetch) && !isZigDriver(driver) ? ["-lz"] : []), ...unwinderLinkArgs(driver), ...driver.linkArgs, ...executableLinkFlags, @@ -1100,7 +1089,6 @@ export async function compileCInternal( ...(targetPlatform(driver) === "win32" ? ["-ladvapi32", "-liphlpapi", "-lws2_32"] : []), ...(mbedtls && targetPlatform(driver) === "win32" ? ["-lbcrypt"] : []), ...(tlsCa && targetPlatform(driver) === "win32" ? ["-lcrypt32"] : []), - ...(((opts.zlib ?? false) || nativeFetch) && !isZigDriver(driver) ? ["-lz"] : []), ...unwinderLinkArgs(driver), ...driver.linkArgs, ...executableLinkFlags, diff --git a/packages/compiler/src/backend/native/library.ts b/packages/compiler/src/backend/native/library.ts index 06b4ce1df..3cf4afa53 100644 --- a/packages/compiler/src/backend/native/library.ts +++ b/packages/compiler/src/backend/native/library.ts @@ -78,9 +78,8 @@ import { CacheInputsChangedError } from "./session.js"; * External-symbol contract: undefined references only to the target's C/math * runtime and system APIs. Windows embedders additionally link advapi32, * iphlpapi, and ws2_32; the platform driver supplies its ordinary CRT and - * kernel imports. Zlib rides the VENDORED per-flavor objects here even on - * hosts because the executable lane's system `-lz` cannot ride inside an - * archive. */ + * kernel imports. Zlib rides the vendored per-flavor objects, as in the + * executable lane. */ export async function compileLibArchive(opts: LibArchiveOptions): Promise { clearCcCaches(); diff --git a/packages/compiler/src/backend/native/runtime-inputs.ts b/packages/compiler/src/backend/native/runtime-inputs.ts index 891fa9312..60f1a8960 100644 --- a/packages/compiler/src/backend/native/runtime-inputs.ts +++ b/packages/compiler/src/backend/native/runtime-inputs.ts @@ -70,6 +70,7 @@ export const EXECUTABLE_RUNTIME_SOURCES = [ "scr_cycle.c", "scr_publish.c", "scr_region.c", + "scr_arena.c", "scr_itable.c", "scr_slots.c", "scr_alloc.c", diff --git a/packages/compiler/src/backend/vendor-archives.ts b/packages/compiler/src/backend/vendor-archives.ts index ef63aa4e4..363d14b08 100644 --- a/packages/compiler/src/backend/vendor-archives.ts +++ b/packages/compiler/src/backend/vendor-archives.ts @@ -18,6 +18,12 @@ const execFileAsync = promisify(execFile); * runtime pack's executable_section_elimination flags). */ const LRE_SECTION_FLAGS = ["-ffunction-sections", "-fdata-sections"]; +/** A vendored source's object file name: sources in subdirectories + * (zlib's contrib/optimizations/inflate.c) flatten to one directory. */ +function vendorObjectName(source: string): string { + return source.replaceAll("/", "-").replace(/\.c$/, ".o"); +} + import { QJS_COMMIT, MBEDTLS_VERSION, @@ -25,6 +31,7 @@ import { BROTLI_VERSION, LRE_SOURCES, ZLIB_SOURCES, + zlibInputs, } from "./vendor-inputs.js"; export { QJS_COMMIT, @@ -243,11 +250,18 @@ export function createVendorArchives(context: VendorArchiveContext) { return join(runtimeSrcDir(), "..", "vendor", "zlib"); } - /** The vendored zlib TUs behind Zig zlib support: every root *.c - * except the gzFile file-I/O units (gz*.c — nothing in scr_zlib.c - * references the gzFile API, and those TUs alone want unistd/io headers). - * The default host-clang build links the system libz; Zig builds use this - * list so targetless and explicit-target Zig inputs stay on one toolchain. */ + /** The vendored zlib TUs every C-lane build links (zlibInputs picks them + * per target; the gzFile file-I/O units are never compiled, nothing in + * scr_zlib.c references the gzFile API). */ + + /** The triple zlibInputs reads for a driver: its explicit target, else + * the host. */ + function zlibTriple(driver: Pick): string { + if (driver.target !== null) return driver.target; + const cpu = + process.arch === "x64" ? "x86_64" : process.arch === "arm64" ? "aarch64" : process.arch; + return `${cpu}-${process.platform === "win32" ? "windows" : process.platform}`; + } function zlibObjectPaths( sanitize: boolean, @@ -260,15 +274,15 @@ export function createVendorArchives(context: VendorArchiveContext) { (driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") + `-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`; const cacheDir = join(cacheRoot, `zlib-${ZLIB_VERSION}-${flavor}`); - return ZLIB_SOURCES.map((f) => join(cacheDir, f.replace(/\.c$/, ".o"))); + return zlibInputs(zlibTriple(driver)).sources.map((f) => join(cacheDir, vendorObjectName(f))); } /** The zlib objects for one flavor, compiled lazily on the first zlib-using - * Zig build (~1s) and cached like the lre objects — + * C-lane build (~1s) and cached like the lre objects — * /vendor/zlib----/*.o — with the same atomic-rename * publish (parallel first builds race safely; losers discard their work). - * Plain is -Os, asan matches the final link so the sanitized lane - * instruments the codec too. The flavor keys the driver and target exactly + * Plain is -O3, as Node builds zlib; asan matches the final link so the + * sanitized lane instruments the codec too. The flavor keys the driver and target exactly * like the lre flavor: the keying * must never hand a zig-built object set to a clang link off a shared * directory. */ @@ -282,6 +296,7 @@ export function createVendorArchives(context: VendorArchiveContext) { (sanitize ? "asan" : "plain") + (driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") + `-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`; + const inputs = zlibInputs(zlibTriple(driver)); return ensureVendorObjects({ sanitize, driver, @@ -289,8 +304,9 @@ export function createVendorArchives(context: VendorArchiveContext) { flavor, name: "zlib", vendor: vendorZlibDir(), - sources: ZLIB_SOURCES, + sources: inputs.sources, objects: zlibObjectPaths(sanitize, driver, buildIdentity, cacheRoot), + flags: [...(sanitize ? [] : ["-O3"]), ...inputs.defines.map((define) => `-D${define}`)], }); } @@ -327,7 +343,7 @@ export function createVendorArchives(context: VendorArchiveContext) { "-c", join(vendor, f), "-o", - join(buildDir, f.replace(/\.c$/, ".o")), + join(buildDir, vendorObjectName(f)), ], { cwd: buildDir }, ); diff --git a/packages/compiler/src/backend/vendor-inputs.test.ts b/packages/compiler/src/backend/vendor-inputs.test.ts new file mode 100644 index 000000000..53e9b8c22 --- /dev/null +++ b/packages/compiler/src/backend/vendor-inputs.test.ts @@ -0,0 +1,80 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { expect, test } from "vitest"; +import { ZLIB_SOURCES, zlibInputs } from "./vendor-inputs.js"; + +const repoRoot = fileURLToPath(new URL("../../../..", import.meta.url)); +const zlibDir = join(repoRoot, "packages/runtime/vendor/zlib"); + +const TRIPLES = [ + // The runtime packs' targets. + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "aarch64-unknown-linux-android26", + "arm64-apple-macosx14.0.0", + "x86_64-apple-macosx14.0.0", + "arm64-apple-ios15.0", + "arm64-apple-ios15.0-simulator", + "x86_64-pc-windows-msvc", + "wasm32-unknown-wasi", + // Zig cross targets and host spellings. + "x86_64-linux-gnu", + "aarch64-linux-musl", + "aarch64-macos", + "x86_64-windows-gnu", + "aarch64-windows-gnu", + "wasm32-wasi", + "x86_64-linux", + "aarch64-darwin", +]; + +test("the compiler's zlib inputs match the runtime packs' for every target", async () => { + const packInputs = (await import( + pathToFileURL(join(repoRoot, "packages/runtime-pack-common/scripts/zlib-inputs.mjs")).href + )) as { zlibInputs(triple: string): { sources: string[]; defines: string[] } }; + for (const triple of TRIPLES) { + expect(zlibInputs(triple), triple).toEqual(packInputs.zlibInputs(triple)); + } +}); + +test("every zlib input exists, and ZLIB_SOURCES lists every source a target compiles", () => { + const listed = new Set(ZLIB_SOURCES); + for (const triple of TRIPLES) { + for (const source of zlibInputs(triple).sources) { + expect(listed.has(source), `${triple}: ${source}`).toBe(true); + expect(existsSync(join(zlibDir, source)), source).toBe(true); + } + } +}); + +test("zlib inputs follow Node's deps/zlib/zlib.gyp", () => { + const x64 = zlibInputs("x86_64-unknown-linux-gnu"); + expect(x64.sources).toContain("adler32_simd.c"); + expect(x64.sources).toContain("contrib/optimizations/inflate.c"); + expect(x64.sources).not.toContain("inflate.c"); + expect(x64.defines).toEqual([ + "HAVE_HIDDEN", + "ADLER32_SIMD_SSSE3", + "X86_NOT_WINDOWS", + "INFLATE_CHUNK_SIMD_SSE2", + "INFLATE_CHUNK_READ_64LE", + "DEFLATE_SLIDE_HASH_SSE2", + ]); + expect(zlibInputs("x86_64-pc-windows-msvc").defines).toContain("X86_WINDOWS"); + const mac = zlibInputs("arm64-apple-macosx14.0.0"); + expect(mac.sources).toContain("crc32_simd.c"); + expect(mac.defines).toContain("ARMV8_OS_MACOS"); + expect(mac.defines).toContain("CRC32_ARMV8_CRC32"); + const ios = zlibInputs("arm64-apple-ios15.0"); + expect(ios.sources).not.toContain("crc32_simd.c"); + expect(ios.defines).not.toContain("CRC32_ARMV8_CRC32"); + expect(ios.defines).toContain("INFLATE_CHUNK_SIMD_NEON"); + expect(zlibInputs("aarch64-unknown-linux-android26").defines).toContain("ARMV8_OS_LINUX"); + const wasm = zlibInputs("wasm32-unknown-wasi"); + expect(wasm.sources).toContain("inflate.c"); + expect(wasm.sources).not.toContain("contrib/optimizations/inflate.c"); + expect(wasm.defines).toEqual(["HAVE_HIDDEN", "CPU_NO_SIMD"]); +}); diff --git a/packages/compiler/src/backend/vendor-inputs.ts b/packages/compiler/src/backend/vendor-inputs.ts index a9d13e2f0..270815b00 100644 --- a/packages/compiler/src/backend/vendor-inputs.ts +++ b/packages/compiler/src/backend/vendor-inputs.ts @@ -1,22 +1,98 @@ /** The pinned QuickJS, mbedTLS, zlib, and brotli inputs used by native recipes. */ export const QJS_COMMIT = "3c8f3d68953955950074c41c6e4d999562ae82a7"; export const MBEDTLS_VERSION = "3.6.7"; -export const ZLIB_VERSION = "1.3.1"; +/** The zlib Node v24.22.0 bundles (`process.versions.zlib` there), the + * Chromium fork in Node's deps/zlib (packages/runtime/vendor/zlib). */ +export const ZLIB_VERSION = "1.3.2.1-motley-285e94b"; /** The brotli release Node v24.22.0 bundles (packages/runtime/vendor/brotli). */ export const BROTLI_VERSION = "1.2.0"; /** Exact translation-unit membership shared by cache builds and external * source-pack recipes. */ export const LRE_SOURCES = ["libregexp.c", "libunicode.c"] as const; -export const ZLIB_SOURCES = [ +const ZLIB_COMMON_SOURCES = [ "adler32.c", "compress.c", + "cpu_features.c", "crc32.c", "deflate.c", "infback.c", "inffast.c", - "inflate.c", "inftrees.c", "trees.c", "uncompr.c", "zutil.c", ] as const; + +/** Every zlib translation unit some target compiles (zlibInputs). */ +export const ZLIB_SOURCES = [ + ...ZLIB_COMMON_SOURCES, + "adler32_simd.c", + "crc32_simd.c", + "inflate.c", + "contrib/optimizations/inffast_chunk.c", + "contrib/optimizations/inflate.c", +] as const; + +/** The zlib translation units and defines for a target, chosen the way + * Node's deps/zlib/zlib.gyp chooses them. The SIMD units carry their own + * target attributes and check the CPU at run time, so every unit builds with + * the same flags, and compressed output does not depend on the choice. + * packages/runtime-pack-common/scripts/zlib-inputs.mjs keeps the same table + * for the runtime packs. `triple` is an LLVM or Zig target triple. */ +export function zlibInputs(triple: string): { sources: string[]; defines: string[] } { + const [cpu = "", ...rest] = triple.split("-"); + const system = rest.join("-"); + const architecture = + cpu === "x86_64" || cpu === "amd64" + ? "x64" + : cpu === "aarch64" || cpu === "arm64" + ? "arm64" + : cpu; + const ios = system.includes("ios"); + const windows = + system.includes("windows") || system.includes("win32") || system.includes("mingw"); + // Android reads the CPU features through getauxval, as Linux does: Node's + // ARMV8_OS_ANDROID path needs the NDK's cpu-features library. + const linux = system.includes("linux") || system.includes("android"); + const mac = !ios && (system.includes("darwin") || system.includes("macos")); + const sources: string[] = [...ZLIB_COMMON_SOURCES]; + const defines = windows ? [] : ["HAVE_HIDDEN"]; + if (architecture === "x64" && !ios) { + sources.push( + "adler32_simd.c", + "contrib/optimizations/inffast_chunk.c", + "contrib/optimizations/inflate.c", + ); + defines.push( + "ADLER32_SIMD_SSSE3", + windows ? "X86_WINDOWS" : "X86_NOT_WINDOWS", + "INFLATE_CHUNK_SIMD_SSE2", + "INFLATE_CHUNK_READ_64LE", + "DEFLATE_SLIDE_HASH_SSE2", + ); + } else if (architecture === "arm64") { + sources.push("contrib/optimizations/inffast_chunk.c", "contrib/optimizations/inflate.c"); + defines.push( + "INFLATE_CHUNK_SIMD_NEON", + "INFLATE_CHUNK_READ_64LE", + "__ARM_NEON__", + "DEFLATE_SLIDE_HASH_NEON", + ); + if (windows) { + defines.push("ARMV8_OS_WINDOWS"); + } else if (linux || mac) { + // Node builds the Adler-32 and CRC-32 units for every arm64 target but + // iOS and Windows. + sources.push("adler32_simd.c", "crc32_simd.c"); + defines.push( + "ADLER32_SIMD_NEON", + "CRC32_ARMV8_CRC32", + linux ? "ARMV8_OS_LINUX" : "ARMV8_OS_MACOS", + ); + } + } else { + sources.push("inflate.c"); + defines.push("CPU_NO_SIMD"); + } + return { sources, defines }; +} diff --git a/packages/compiler/src/contracts/check.ts b/packages/compiler/src/contracts/check.ts index 972d9ea12..6d8a19956 100644 --- a/packages/compiler/src/contracts/check.ts +++ b/packages/compiler/src/contracts/check.ts @@ -519,8 +519,10 @@ class ContractChecker { for (const fn of this.fnByLoc.get(key) ?? []) targets.push({ fn, note }); if (targets.length === 0) return; // Target-independent: the native stack guard (Linux, macOS) counts, so a - // contract holds on every platform. - const mt = computeMayThrow(this.mod, { stackChecks: true, explain: true }); + // contract holds on every platform. Every function on a call cycle counts + // as checking it, a superset of the functions the backend picks, so the + // answer does not depend on that choice. + const mt = computeMayThrow(this.mod, { stackChecks: "everyCycleMember", explain: true }); const proven: string[] = []; for (const { fn, note } of targets) { if (fn.async === true || fn.generator !== undefined) { @@ -1582,6 +1584,15 @@ class ImmutableContracts { for (let steps = 0; steps < 16; steps++) { if (root.kind === "fieldGet" || root.kind === "recordGet") root = root.obj; else if (root.kind === "arrayGet") root = root.arr; + else if (root.kind === "seqExpr") root = root.result; + // A receiver that is the element when it is present and otherwise + // throws (`xs[i]!.push(v)`) is that element read. + else if ( + root.kind === "ternary" && + root.else_.kind === "libCall" && + root.else_.fn === "error.nodeThrow" + ) + root = root.then; else if ( root.kind === "upcast" || root.kind === "downcast" || diff --git a/packages/compiler/src/frontend/contract-types.ts b/packages/compiler/src/frontend/contract-types.ts index 5905a12b6..71bbeb376 100644 --- a/packages/compiler/src/frontend/contract-types.ts +++ b/packages/compiler/src/frontend/contract-types.ts @@ -12,6 +12,7 @@ import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; import type { SrcLoc } from "../ir/ir.js"; import { locOf } from "./program.js"; import * as ts from "./ts7/adapter.js"; +import { type HeritageElement, isHeritageElement } from "./heritage-elements.js"; export const CONTRACT_TYPES_MODULE = "@scriptc/types"; @@ -87,11 +88,11 @@ function markerOf(checker: ts.TypeChecker, ref: ts.TypeReferenceNode): ContractM } /** The marker a heritage clause names (`implements Acyclic`, - * `implements sc.ProgramLifetime`). */ -function markerOfHeritage( - checker: ts.TypeChecker, - node: ts.ExpressionWithTypeArguments, -): ContractMarker | null { + * `implements sc.ProgramLifetime`), as an ExpressionWithTypeArguments or, + * in TypeScript 7.1's interface `extends` and class `implements`, a + * TypeReference. */ +function markerOfHeritage(checker: ts.TypeChecker, node: HeritageElement): ContractMarker | null { + if (ts.isTypeReferenceNode(node)) return markerOf(checker, node); const e = node.expression; if (ts.isIdentifier(e)) return markerNamed(checker, e, e.text, false); if (ts.isPropertyAccessExpression(e) && ts.isIdentifier(e.expression) && ts.isIdentifier(e.name)) @@ -169,7 +170,7 @@ export function collectContractAnnotations( ts.walkPreorder( sf, (node) => { - if (ts.isExpressionWithTypeArguments(node)) { + if (isHeritageElement(node)) { const marker = markerOfHeritage(checker, node); if (marker === null) return; out.count++; @@ -232,7 +233,7 @@ function unsupported(out: ContractAnnotations, node: ts.Node, marker: ContractMa function placeHeritage( out: ContractAnnotations, checker: ts.TypeChecker, - node: ts.ExpressionWithTypeArguments, + node: HeritageElement, marker: ContractMarker, resolveUnit: ContractUnitResolver, ): void { diff --git a/packages/compiler/src/frontend/heritage-elements.test.ts b/packages/compiler/src/frontend/heritage-elements.test.ts new file mode 100644 index 000000000..4ceb3c983 --- /dev/null +++ b/packages/compiler/src/frontend/heritage-elements.test.ts @@ -0,0 +1,82 @@ +import { fileURLToPath } from "node:url"; +import { dirname } from "node:path"; +import { expect, test } from "vitest"; +import { collectContractAnnotations } from "./contract-types.js"; +import { isHeritageElement, heritageElementTarget } from "./heritage-elements.js"; +import * as ts from "./ts7/adapter.js"; +import { Ts7Api } from "./ts7/api.js"; +import { Ts7Host } from "./ts7/program-host.js"; +import { Ts7RpcClient } from "./ts7/rpc-client.js"; +import { TS7_FILE_SYSTEM_CALLBACKS } from "./ts7/rpc-filesystem.js"; +import { spawnTs7Wire } from "./ts7/rpc-process.js"; +import { ts7TestServers, type Ts7TestServer } from "./ts7/test-servers.js"; + +const corpus = fileURLToPath(new URL("../../../../tests/corpus/", import.meta.url)); + +/** Corpus programs whose classes implement contract markers and interfaces. */ +const programs = [ + "4636-contract-program-lifetime-references.ts", + "5010-lifetime-region-generations.ts", + "7401-program-lifetime-array-teardown.ts", +]; + +function host(server: Ts7TestServer): Ts7Host { + return new Ts7Host( + (options) => + new Ts7Api( + new Ts7RpcClient( + spawnTs7Wire(server.executable, [ + "--api", + "--cwd", + options.cwd, + `--callbacks=${TS7_FILE_SYSTEM_CALLBACKS}`, + ]), + ), + options.fs, + options.collectTiming, + ), + { cwd: dirname(corpus) }, + ); +} + +/** Each heritage element's clause token, kind and the text it refers to, + * and the program's contract annotations. */ +function facts(server: Ts7TestServer, file: string) { + const programHost = host(server); + try { + const program = programHost.createProgram([corpus + file], { + strict: true, + types: [], + target: ts.ScriptTarget.ESNext, + }); + const source = program.getSourceFile(corpus + file)!; + const elements: string[] = []; + ts.walkPreorder(source, (node) => { + if (!isHeritageElement(node)) return undefined; + const clause = node.parent as ts.HeritageClause; + elements.push( + `${clause.token === ts.SyntaxKind.ImplementsKeyword ? "implements" : "extends"} ${ts.getTextOfNode(heritageElementTarget(node))}`, + ); + return undefined; + }); + const contracts = collectContractAnnotations(program.getTypeChecker(), [source]); + return { + elements, + classes: [...contracts.classes.values()].map((note) => `${note.marker} ${note.name}`), + acyclic: contracts.acyclic.map((entry) => entry.note.name), + diagnostics: contracts.diagnostics.map((diagnostic) => diagnostic.code), + }; + } finally { + programHost.close(); + } +} + +test("heritage elements and the contracts they name read the same from tsc 7.0 and tsc 7.1", () => { + const [old, next] = ts7TestServers(); + for (const file of programs) { + const expected = facts(old!, file); + expect(expected.classes.length, file).toBeGreaterThan(0); + expect(expected.diagnostics, file).toEqual([]); + expect(facts(next!, file), file).toEqual(expected); + } +}); diff --git a/packages/compiler/src/frontend/heritage-elements.ts b/packages/compiler/src/frontend/heritage-elements.ts new file mode 100644 index 000000000..b937c073a --- /dev/null +++ b/packages/compiler/src/frontend/heritage-elements.ts @@ -0,0 +1,28 @@ +import * as ts from "./ts7/adapter.js"; + +/** An element of a heritage clause. TypeScript 7.0 parses every element as + * an ExpressionWithTypeArguments. TypeScript 7.1 parses the elements of an + * interface's `extends` and a class's `implements` as TypeReference nodes, + * whose names are entity names (an Identifier or a QualifiedName); a class's + * `extends` element stays an ExpressionWithTypeArguments. */ +export type HeritageElement = ts.ExpressionWithTypeArguments | ts.TypeReferenceNode; + +/** Whether `node` is a TypeReference that is a heritage clause element. */ +export function isHeritageTypeReference(node: ts.Node): node is ts.TypeReferenceNode { + return ( + ts.isTypeReferenceNode(node) && node.parent !== undefined && ts.isHeritageClause(node.parent) + ); +} + +/** Whether `node` is a heritage clause element in either form. */ +export function isHeritageElement(node: ts.Node): node is HeritageElement { + return ts.isExpressionWithTypeArguments(node) || isHeritageTypeReference(node); +} + +/** The name or expression a heritage clause element refers to: its + * expression (an ExpressionWithTypeArguments) or its type name (a + * TypeReference). */ +export function heritageElementTarget(element: ts.Node): ts.Expression | ts.EntityName { + if (ts.isTypeReferenceNode(element)) return element.typeName; + return (element as ts.ExpressionWithTypeArguments).expression; +} diff --git a/packages/compiler/src/frontend/lowering/builtins/array-classes.ts b/packages/compiler/src/frontend/lowering/builtins/array-classes.ts index 4ed92faa7..1922be0af 100644 --- a/packages/compiler/src/frontend/lowering/builtins/array-classes.ts +++ b/packages/compiler/src/frontend/lowering/builtins/array-classes.ts @@ -10,6 +10,7 @@ import * as ts from "../../ts7/adapter.js"; import { type Lowerer } from "../lowerer.js"; import { isJsSourceFile, locOf } from "../../program.js"; import { stripTypeWrappers } from "../typed-arguments.js"; +import { keyedHelperName } from "../stable-names.js"; import { dynUndefinedExpr, nodeThrowExpr } from "../../../ir/build.js"; import { lowerArrayFromCall, lowerArrayOfCall } from "../containers/array-construction.js"; import { boundedArrayHofHelper } from "../containers/array-callbacks.js"; @@ -305,6 +306,7 @@ export function lowerArrayClassInstanceOf( const value = lowerer.lowerExpr(expr.left); if (value.type.kind === "array") return { kind: "instanceOf", value, className: info.def.name, type: BOOL, loc }; + if (value.type.kind === "union") return arrayClassUnionInstanceOf(lowerer, value, info, loc); if (value.type.kind === "dyn") return { kind: "libCall", @@ -324,6 +326,62 @@ export function lowerArrayClassInstanceOf( return null; } +/** `x instanceof X` for a class X declared `extends Array` on a union (an + * element read of an array of X instances is `X | undefined`): an array arm + * tests the array's class at run time, a checked-dynamic arm asks the + * runtime, and every other arm is no instance. */ +function arrayClassUnionInstanceOf( + lowerer: Lowerer, + value: IrExpr, + info: ClassInfo, + loc: SrcLoc, +): IrExpr | null { + if (value.type.kind !== "union") return null; + const unionId = value.type.unionId; + const arms = lowerer.unions.get(unionId)?.arms; + if (arms === undefined) return null; + const className = info.def.name; + const key = `instanceof.arrayClass:${unionId}:${className}`; + let helper = lowerer.valueHelpers.get(key); + if (!helper) { + helper = keyedHelperName(lowerer, "%instanceof.arrayClass", key); + lowerer.valueHelpers.set(key, helper); + const param: IrExpr = { kind: "varRef", localId: "value.0", type: value.type, loc }; + const body: IrStmt[] = []; + arms.forEach((arm, tag) => { + if (arm.kind !== "array" && arm.kind !== "dyn") return; + const narrowed: IrExpr = { kind: "unionNarrow", unionId, tag, value: param, type: arm, loc }; + const test: IrExpr = + arm.kind === "array" + ? { kind: "instanceOf", value: narrowed, className, type: BOOL, loc } + : { + kind: "libCall", + fn: "array.dynIsClass", + args: [narrowed, { kind: "strLit", value: className, type: STRING, loc }], + type: BOOL, + loc, + }; + body.push({ + kind: "if", + cond: { kind: "unionIsTag", unionId, tag, negated: false, value: param, type: BOOL, loc }, + then: [{ kind: "return", value: test, loc }], + else_: null, + loc, + }); + }); + body.push({ kind: "return", value: { kind: "boolLit", value: false, type: BOOL, loc }, loc }); + lowerer.liftedFns.push({ + name: helper, + params: [{ localId: "value.0", name: "value", type: value.type }], + returnType: BOOL, + locals: [{ id: "value.0", name: "value", type: value.type, mutable: false }], + body, + loc, + }); + } + return { kind: "call", callee: helper, args: [value], type: BOOL, loc }; +} + /** The class declared `extends Array` that a value of `type` is an instance * of, or null. */ export function arrayClassOfType(lowerer: Lowerer, type: ts.Type): ClassInfo | null { diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts index bbdf271c0..3e7b7783e 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts @@ -41,6 +41,7 @@ function context() { protocolDispatches: new Map(), classMethodValueSelections: new Map(), isSubclassOf: () => false, + nodeAtLoc: () => null, } as unknown as Lowerer; const boxed = fn("box", [ statement({ diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index 1c6df3d5c..c19763e8d 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -38,6 +38,7 @@ import { import { streamTypedRefEligible } from "../../ir/analysis.js"; import { everyStmtList, transformExpr, transformStmtList } from "../../ir/traverse.js"; import { PoisonError, type Lowerer } from "./lowerer.js"; +import { isSloppyAt } from "./frozen-records.js"; import { erasedGenericInstance, implicitDefaultInstance } from "./generic-functions.js"; import { type ParamShape } from "./call-signatures.js"; import { @@ -2418,6 +2419,11 @@ export class ClassDynamicDispatch { computedRead.key.operand.type.kind === "f64" ? computedRead.key : null; + // A sloppy-mode write ignores a refused assignment + // (sloppyDynamicWrites), so its sites share their own helper, + // whose write takes the first such site's source range. + const sloppyWrite = + !!computedWrite && !reflect && isSloppyAt(lowerer, computedWrite.loc); const key = JSON.stringify([ !!computedWrite, computedRead?.optional ?? false, @@ -2426,6 +2432,7 @@ export class ClassDynamicDispatch { numericKey, reflect, !!indexKey, + ...(sloppyWrite ? ["sloppy"] : []), ]); let dispatch = this.computed.get(key); if (!dispatch) { diff --git a/packages/compiler/src/frontend/lowering/expressions/binary.ts b/packages/compiler/src/frontend/lowering/expressions/binary.ts index 0d24561de..5c326ab1e 100644 --- a/packages/compiler/src/frontend/lowering/expressions/binary.ts +++ b/packages/compiler/src/frontend/lowering/expressions/binary.ts @@ -37,6 +37,7 @@ import { dynCompoundValue, lowerAbsenceProbe, lowerDynMemberAssignment, + lowerDereferencedElementAssignment, lowerOptionalNumber, lowerPresenceOperand, nullishBranches, @@ -334,6 +335,25 @@ function isMemberReceiver(node: ts.Expression): boolean { ); } +/** A literal default (`0`, `-1`, `""`, `true`): it always has its own type. + * Other defaults, such as an element read that can be undefined, keep the + * dynamic path. */ +function literalDefault(node: ts.Expression): boolean { + while (ts.isParenthesizedExpression(node)) node = node.expression; + if ( + ts.isPrefixUnaryExpression(node) && + (node.operator === ts.SyntaxKind.MinusToken || node.operator === ts.SyntaxKind.PlusToken) + ) + node = node.operand; + return ( + ts.isNumericLiteral(node) || + ts.isStringLiteral(node) || + ts.isNoSubstitutionTemplateLiteral(node) || + node.kind === ts.SyntaxKind.TrueKeyword || + node.kind === ts.SyntaxKind.FalseKeyword + ); +} + export function lowerNullishPair( lowerer: Lowerer, expr: ts.BinaryExpression, @@ -388,6 +408,26 @@ export function lowerNullishPair( lowerer.runtimeOptionalArithmeticTypes.set(expr, type); } if (type.kind === "dyn" && lowerer.dynConvertible(left.type)) { + // `counts.get(k) ?? 0` in JavaScript: a scalar optional left and a + // default of its value's type give a scalar, so the test and the default + // run on the typed path and only the result converts. Converting the + // optional left itself would allocate a union for every read. + if ( + rest.length === 1 && + primitive(rest[0]!) && + rightType && + typeEquals(rightType, rest[0]!) && + literalDefault(expr.right) + ) { + const typed: IrExpr = { + kind: "nullish", + left, + right: lowerer.lowerExprExpecting(expr.right, rest[0]!), + type: rest[0]!, + loc, + }; + return lowerer.coerceToExpected(typed, DYN); + } return { kind: "nullish", left: lowerer.coerceToExpected(left, DYN), @@ -1029,6 +1069,8 @@ export function lowerBinary(lowerer: Lowerer, expr: ts.BinaryExpression): IrExpr if (prototypeWrite) return prototypeWrite; const callableWrite = lowerNativeFunctionAssignment(lowerer, expr); if (callableWrite) return callableWrite; + const dereferenced = lowerDereferencedElementAssignment(lowerer, expr); + if (dereferenced) return dereferenced; } if ( op === ts.SyntaxKind.QuestionQuestionEqualsToken || diff --git a/packages/compiler/src/frontend/lowering/expressions/relational.ts b/packages/compiler/src/frontend/lowering/expressions/relational.ts index 51d48b127..d29226685 100644 --- a/packages/compiler/src/frontend/lowering/expressions/relational.ts +++ b/packages/compiler/src/frontend/lowering/expressions/relational.ts @@ -840,9 +840,10 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc if (left.type.kind === "union") { const unionId = left.type.unionId; const arms = lowerer.unions.get(unionId)?.arms; - // Ordinary records, arrays and scalar values do not carry a class - // prototype (no program class extends Array). Dynamic/opaque runtime - // objects keep their existing instanceof fences. + // Records, arrays and scalar values are no instances of a class that + // does not extend Array (lowerArrayClassInstanceOf tests the classes + // that do). Dynamic/opaque runtime objects keep their existing + // instanceof fences. if ( arms?.every( (arm) => diff --git a/packages/compiler/src/frontend/lowering/frozen-records.ts b/packages/compiler/src/frontend/lowering/frozen-records.ts index fc770a0cb..3700b57ff 100644 --- a/packages/compiler/src/frontend/lowering/frozen-records.ts +++ b/packages/compiler/src/frontend/lowering/frozen-records.ts @@ -362,3 +362,37 @@ export function freezableRecordType(lowerer: Lowerer, type: IrType): type is Rec const shape = lowerer.shapes.get(type.shapeId); return shape !== undefined && !shapeHasAccessorSlots(shape); } + +/** The source ranges (writeKey) of untyped property writes in sloppy-mode + * code (IrModule.sloppyDynWrites): an assignment there that strict mode + * refuses with a TypeError (a frozen or non-extensible object, a read-only + * property) leaves the object unchanged instead. Only programs with such + * writes list any; the backend calls the sloppy entry points for them. */ +export function sloppyDynamicWrites(lowerer: Lowerer, functions: readonly IrFunction[]): string[] { + const keys = new Set(); + const strictness = new Map(); + for (const fn of functions) { + everyStmtList(fn.body, { + stmt: () => true, + expr: (e) => { + if (e.kind !== "libCall" || (e.fn !== "dyn.keySet" && e.fn !== "dyn.keySetComputed")) + return true; + const key = writeKey(e.loc); + let sloppy = strictness.get(key); + if (sloppy === undefined) { + sloppy = isSloppyAt(lowerer, e.loc); + strictness.set(key, sloppy); + } + if (sloppy) keys.add(key); + return true; + }, + }); + } + return [...keys].sort(); +} + +/** Whether the source at `loc` is sloppy-mode code (isStrictCode). */ +export function isSloppyAt(lowerer: Lowerer, loc: SrcLoc): boolean { + const node = lowerer.nodeAtLoc(loc); + return node !== null && !isStrictCode(node); +} diff --git a/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts index db542412f..459af8751 100644 --- a/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts @@ -1,7 +1,8 @@ -import { dynUndefinedExpr, varRef } from "../../ir/build.js"; +import { dynUndefinedExpr, numLit, varRef } from "../../ir/build.js"; import { BOOL, DYN, + F64, STRING, canConvertToDyn, funcOf, @@ -16,7 +17,7 @@ import { import { transformStmtList } from "../../ir/traverse.js"; import { genResultRecord } from "../type-mapper.js"; import type { DispatchSites } from "./dispatch-sites.js"; -import type { Lowerer } from "./lowerer.js"; +import type { Lowerer, UntypedLoopSite } from "./lowerer.js"; type GeneratorType = Extract; type Invoke = Extract; @@ -401,6 +402,310 @@ export class GeneratorDynamicDispatch { }); changed = true; } + return this.stepUntypedLoops(lowerer, functions) || changed; + } + + /** The generator types a loop over an untyped iterable can step natively: + * sync generators for `for … of`, and both kinds for `for await` (a sync + * generator through the async-from-sync rules). Insertion order, so a + * type keeps its index as later waves add types. */ + private loopTypes(lowerer: Lowerer, site: UntypedLoopSite): [string, GeneratorType][] { + const out: [string, GeneratorType][] = []; + for (const [key, type] of this.generators) { + if (type.async === true && !site.async) continue; + const result = genResultRecord(type.yieldT, type.retT, lowerer.shapes, lowerer.unions); + if (result === null) continue; + // next() with no argument, as the protocol calls it. + const channel = type.nextT; + if (channel.kind !== "undefinedT" && channel.kind !== "void" && channel.kind !== "dyn") + continue; + if ( + !canConvertToDyn( + result, + (id) => lowerer.shapes.get(id), + (id) => lowerer.unions.get(id), + ) + ) + continue; + out.push([key, type]); + } + return out; + } + + /** Loops over untyped iterables step the program's own generators on the + * typed path. Once the loop has its iterator and next method, a kind local + * records which generator type the iterator is, when its next method is + * the generator's own (`%dyn.generator.method.next`); each step then + * resumes the generator directly and boxes only the value, where the + * protocol would box the result record and its promise and read them + * back. Every other iterator, and every early exit (return() through the + * protocol), keeps the dynamic path. The awaits are the protocol's: a + * `for await` over an async generator awaits next()'s promise, and over a + * sync generator awaits once for the result and, reached through + * Symbol.iterator, once for the value. */ + private stepUntypedLoops(lowerer: Lowerer, functions: readonly IrFunction[]): boolean { + if (this.generators.size === 0 || lowerer.untypedLoops.size === 0) return false; + // The loop reads next through the getter only once it exists. + if (!this.properties.has("next")) return false; + let changed = false; + for (const fn of functions) { + if (this.generated.has(fn)) continue; + const byKind = new Map(); + const byValue = new Map(); + for (const local of fn.locals) { + const site = lowerer.untypedLoops.get(local); + if (site === undefined) continue; + byKind.set(site.kind, site); + byValue.set(site.value, site); + } + if (byKind.size === 0) continue; + fn.body = transformStmtList(fn.body, { + expr: (expr) => expr, + stmt: (stmt) => { + if (stmt.kind !== "varDecl" || stmt.init === null) return stmt; + const opened = byKind.get(stmt.localId); + if (opened !== undefined) { + const types = this.loopTypes(lowerer, opened); + const init = this.loopKind(opened, types, stmt.loc); + if (this.loopKindCount(stmt.init) === types.length) return stmt; + changed = true; + return { ...stmt, init }; + } + const site = byValue.get(stmt.localId); + if (site === undefined) return stmt; + const types = this.loopTypes(lowerer, site); + // The protocol step is the innermost else of the type chain. + let fallback = stmt.init; + let present = 0; + while ( + fallback.kind === "ternary" && + fallback.cond.kind === "bin" && + fallback.cond.left.kind === "varRef" && + fallback.cond.left.localId === site.kind + ) { + fallback = fallback.else_; + present++; + } + if (present === types.length) return stmt; + let init = fallback; + for (let i = types.length - 1; i >= 0; i--) { + const [, type] = types[i]!; + const record = `${site.kind}.r${i + 1}`; + init = { + kind: "ternary", + cond: { + kind: "bin", + op: "===", + left: varRef(site.kind, F64, stmt.loc), + right: numLit(i + 1, stmt.loc), + type: BOOL, + loc: stmt.loc, + }, + then: this.loopStep(lowerer, fn, site, type, record, stmt.loc), + else_: init, + type: DYN, + loc: stmt.loc, + }; + } + changed = true; + return { ...stmt, init }; + }, + }); + } return changed; } + + /** How many generator types a loop's kind initializer tests: the chain + * inside its own-method test (loopKind). */ + private loopKindCount(init: IrExpr): number { + if (init.kind !== "ternary") return 0; + let count = 0; + for (let chain = init.then; chain.kind === "ternary"; chain = chain.else_) count++; + return count; + } + + /** The loop's kind: the 1-based index of the generator type its iterator + * is, when the next method it read is that generator's own, else 0. */ + private loopKind( + site: UntypedLoopSite, + types: readonly [string, GeneratorType][], + loc: IrExpr["loc"], + ): IrExpr { + const own: IrExpr = { + kind: "libCall", + fn: "dyn.sameValue", + args: [ + varRef(site.next, DYN, loc), + { + kind: "dynFrom", + value: { + kind: "closure", + fnName: "%dyn.generator.method.next", + captures: [], + type: funcOf([DYN], DYN), + loc, + }, + type: DYN, + loc, + }, + ], + type: BOOL, + loc, + }; + let kind: IrExpr = numLit(0, loc); + for (let i = types.length - 1; i >= 0; i--) + kind = { + kind: "ternary", + cond: { + kind: "libCall", + fn: "dyn.typedRefIs", + args: [ + varRef(site.iterator, DYN, loc), + { kind: "strLit", value: types[i]![0], type: STRING, loc }, + ], + type: BOOL, + loc, + }, + then: numLit(i + 1, loc), + else_: kind, + type: F64, + loc, + }; + return types.length === 0 + ? kind + : { kind: "ternary", cond: own, then: kind, else_: numLit(0, loc), type: F64, loc }; + } + + /** One step of a loop over an untyped iterable that is a `type` + * generator: sets the loop's done flag and answers the value, boxed. */ + private loopStep( + lowerer: Lowerer, + fn: IrFunction, + site: UntypedLoopSite, + type: GeneratorType, + recordId: string, + loc: IrExpr["loc"], + ): IrExpr { + const gen: IrExpr = { kind: "dynCheck", value: varRef(site.iterator, DYN, loc), type, loc }; + const setDone = (value: IrExpr): IrStmt => ({ kind: "assign", localId: site.done, value, loc }); + const doneRef = varRef(site.done, BOOL, loc); + const result = genResultRecord(type.yieldT, type.retT, lowerer.shapes, lowerer.unions)!; + if (!fn.locals.some((local) => local.id === recordId)) + fn.locals.push({ id: recordId, name: recordId, type: result, mutable: false }); + const field = (name: "value" | "done", fieldType: IrType): IrExpr => ({ + kind: "recordGet", + obj: varRef(recordId, result, loc), + shapeId: result.shapeId, + field: name, + type: fieldType, + loc, + }); + const valueType = lowerer.shapes + .get(result.shapeId)! + .fields.find((f) => f.name === "value")!.type; + const boxed: IrExpr = { kind: "dynFrom", value: field("value", valueType), type: DYN, loc }; + const awaitDyn = (value: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "async.awaitDyn", + args: [value], + type: DYN, + loc, + }); + const resumed: IrExpr = { + kind: "genResume", + mode: "next", + gen, + arg: null, + type: type.async === true ? { kind: "promise", inner: result } : result, + loc, + }; + if (!site.async) + return { + kind: "seqExpr", + stmts: [ + { kind: "varDecl", localId: recordId, init: resumed, loc }, + setDone(field("done", BOOL)), + ], + result: { + kind: "ternary", + cond: doneRef, + then: dynUndefinedExpr(loc), + else_: boxed, + type: DYN, + loc, + }, + type: DYN, + loc, + }; + if (type.async === true) + return { + kind: "seqExpr", + stmts: [ + { + kind: "varDecl", + localId: recordId, + init: { kind: "awaitExpr", value: resumed, type: result, loc }, + loc, + }, + setDone(field("done", BOOL)), + ], + result: { + kind: "ternary", + cond: doneRef, + then: dynUndefinedExpr(loc), + else_: boxed, + type: DYN, + loc, + }, + type: DYN, + loc, + }; + // for await over a sync generator: the result awaits once. When the + // loop reached it through Symbol.iterator, the async-from-sync iterator + // also awaits the value (the final one too); a sync generator that an + // object's Symbol.asyncIterator returns passes the value as it is, as + // the protocol does. + const fromSync: IrExpr = + site.sync === null + ? { kind: "boolLit", value: false, type: BOOL, loc } + : varRef(site.sync, BOOL, loc); + return { + kind: "seqExpr", + stmts: [ + { kind: "varDecl", localId: recordId, init: resumed, loc }, + setDone(field("done", BOOL)), + { kind: "exprStmt", expr: awaitDyn(dynUndefinedExpr(loc)), loc }, + ], + result: { + kind: "ternary", + cond: doneRef, + then: { + kind: "ternary", + cond: fromSync, + then: { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: awaitDyn(boxed), loc }], + result: dynUndefinedExpr(loc), + type: DYN, + loc, + }, + else_: dynUndefinedExpr(loc), + type: DYN, + loc, + }, + else_: { + kind: "ternary", + cond: fromSync, + then: awaitDyn(boxed), + else_: boxed, + type: DYN, + loc, + }, + type: DYN, + loc, + }, + type: DYN, + loc, + }; + } } diff --git a/packages/compiler/src/frontend/lowering/generic-functions.ts b/packages/compiler/src/frontend/lowering/generic-functions.ts index dcd2efc08..046b34008 100644 --- a/packages/compiler/src/frontend/lowering/generic-functions.ts +++ b/packages/compiler/src/frontend/lowering/generic-functions.ts @@ -4,6 +4,7 @@ import { staticNameBindings } from "./static-function-names.js"; import { positionTag } from "./structural-positions.js"; import { keyedHelperName } from "./stable-names.js"; import * as ts from "../ts7/adapter.js"; +import { heritageElementTarget, isHeritageTypeReference } from "../heritage-elements.js"; import { bodyReadsArguments } from "../arguments-usage.js"; import type { Lowerer } from "./lowerer.js"; import { @@ -655,8 +656,9 @@ function extendsParameter( } for (const clause of decl.heritageClauses?.nodes ?? []) for (const base of clause.types.nodes) { - const symbol = ts.isIdentifier(base.expression) - ? lowerer.checker.getSymbolAtLocation(base.expression) + const target = heritageElementTarget(base); + const symbol = ts.isIdentifier(target) + ? lowerer.checker.getSymbolAtLocation(target) : undefined; if (symbol !== undefined && params.has(symbol)) return true; } @@ -3036,7 +3038,12 @@ function bindingReferences( resolved = lowerer.checker.getShorthandAssignmentValueSymbol(parent) ?? resolved; if (resolved !== symbol) return false; for (let p: ts.Node | undefined = parent; p !== undefined; p = p.parent) { - if (ts.isTypeNode(p) && p.kind !== ts.SyntaxKind.ExpressionWithTypeArguments) return false; + if ( + ts.isTypeNode(p) && + p.kind !== ts.SyntaxKind.ExpressionWithTypeArguments && + !isHeritageTypeReference(p) + ) + return false; if (ts.isStatement(p) || ts.isFunctionLike(p)) break; } return true; diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 380c34615..9ef02cc9b 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -8732,7 +8732,13 @@ function lowerObjectStaticCall( return { kind: "libCall", fn: "dyn.freeze", args: [value], type: DYN, loc: locOf(call) }; } if (ts.isObjectLiteralExpression(inner) || ts.isArrayLiteralExpression(inner)) { - return value; // fresh — freeze is identity here, honestly + // A fresh literal can still reach untyped code, which observes and + // must honor its frozen state: records and arrays keep the bit. + if (freezableRecordType(lowerer, value.type)) + return lowerNativeFreeze(lowerer, "record.freeze", value, locOf(call)); + if (value.type.kind === "array") + return lowerNativeFreeze(lowerer, "array.freeze", value, locOf(call)); + return value; } if ( value.type.kind === "string" || diff --git a/packages/compiler/src/frontend/lowering/lower-exprs.ts b/packages/compiler/src/frontend/lowering/lower-exprs.ts index 55db76117..9eacfa2b2 100644 --- a/packages/compiler/src/frontend/lowering/lower-exprs.ts +++ b/packages/compiler/src/frontend/lowering/lower-exprs.ts @@ -83,6 +83,8 @@ import { unitOnlyUnion } from "../type-mapper.js"; import { lowerYield } from "./lower-generators.js"; import { unionWideningTags } from "../../ir/analysis.js"; import { isSafeToDiscard, isSafeToRepeat } from "./expressions/evaluation-safety.js"; +import { everyExprChild } from "../../ir/traverse.js"; +import { declareContextLocal } from "./function-context.js"; import { hasOptionalChainGuard, isOptionalChainTail } from "./expressions/optional-chains.js"; import { classToStringConversion, classUnionToString } from "./class-to-primitive.js"; import { lowerCjsTopLevelThis, lowerRequireCacheElement } from "./lower-node-module.js"; @@ -413,6 +415,10 @@ function lowerExprInner(lowerer: Lowerer, expr: ts.Expression): IrExpr { const probe = lowerAbsenceProbe(lowerer, expr.expression); if (probe?.type.kind === "union") return probe; } + const dereferenced = dereferencedElementReceiver(lowerer, expr, loc); + if (dereferenced) return dereferenced; + const element = assertedElementOperand(lowerer, expr, loc); + if (element) return element; const inner = lowerer.lowerExpr(expr.expression); if (inner.type.kind === "union") { // `!` performs no runtime check: a missing array element stays @@ -1075,18 +1081,10 @@ function assertionOperand( loc: SrcLoc, ): IrExpr | null { if (inner.type.kind !== "union" || target === null || target.kind === "union") return null; - const arms = lowerer.unions.get(inner.type.unionId)?.arms; - if ( - !arms?.some(isUnitType) || - !arms.every( - (arm) => - arm.kind === "f64" || arm.kind === "string" || arm.kind === "bool" || isUnitType(arm), - ) - ) - return null; - // Array element reads keep their own handling of a missing element. A - // typed-array read is optional here only where its use tells undefined - // from NaN (assertedReadObservesUndefined). + if (!scalarUnion(lowerer, inner.type.unionId)) return null; + // Array element reads take assertedElementOperand or keep their own + // handling of a missing element. A typed-array read is optional here only + // where its use tells undefined from NaN (assertedReadObservesUndefined). let operand: ts.Expression = expr.expression; while (ts.isParenthesizedExpression(operand)) operand = operand.expression; if (ts.isElementAccessExpression(operand)) { @@ -1095,19 +1093,489 @@ function assertionOperand( } const use = convertingUseOf(lowerer, expr); if (use === null) return null; - if (use === "value") return inner; + if (use === "value" || use === "string") return inner; return lowerPrimitiveUnionNumber(lowerer, inner, expr, loc); } -/** How the converting use of `x!` (see assertionOperand) reads it: - * "number" where the use applies ToNumber, "value" where the consumer - * lowers the union itself (strings, equality, truthiness), null for other +/** `xs[i]!` on an array as the operand of a converting use (see + * assertionOperand). The read answers undefined for a missing element, as + * in Node, instead of trapping: a numeric use reads NaN for it, and other + * uses get the optional element value. Returns null for other reads and * uses. */ -function convertingUseOf(lowerer: Lowerer, expr: ts.NonNullExpression): "number" | "value" | null { +function assertedElementOperand( + lowerer: Lowerer, + expr: ts.NonNullExpression, + loc: SrcLoc, +): IrExpr | null { + let read: ts.Expression = expr.expression; + while (ts.isParenthesizedExpression(read)) read = read.expression; + if (!ts.isElementAccessExpression(read) || read.questionDotToken) return null; + // Plain arrays only: regex match results and other array subtypes keep + // their own element reads. + const receiverType = lowerer.typeOf(read.expression); + if ( + !lowerer.checker.isArrayType(receiverType) || + lowerer.mapTypeOf(receiverType)?.kind !== "array" + ) + return null; + // A counter below the length reads an existing index: the read keeps + // its plain element access. + if (lowerer.nanCodedReads?.counterIndexInBounds(read)) return null; + const target = lowerer.mapTypeOf(lowerer.typeOf(expr)); + if (!target || !isScalarArm(target)) return null; + const use = convertingUseOf(lowerer, expr); + if (use === null) return null; + if (use === "string" && target.kind === "string") + return stringElementOrUndefined(lowerer, read, loc); + const probe = lowerAbsenceProbe(lowerer, read); + if (probe?.type.kind !== "union" || !scalarUnion(lowerer, probe.type.unionId)) return null; + if (use === "value" || use === "string") return probe; + return lowerer.stripUndefinedArm(probe.type).kind === "f64" + ? lowerOptionalNumber(lowerer, probe, loc) + : lowerPrimitiveUnionNumber(lowerer, probe, expr, loc); +} + +/** `xs[i]!` on an array as the receiver of a member read + * (`xs[i]!.name`, `xs[i]!.size()`, `xs[i]![0]`). `!` performs no runtime + * check, so a missing element is undefined there and the member access + * throws Node's TypeError ("Cannot read properties of undefined (reading + * 'name')") instead of the bounds trap. The receiver is the element when it + * is present. A write (`xs[i]!.name = v`) is lowered as a statement by + * lowerDereferencedElementWrite, which evaluates the value before the + * receiver throws; elsewhere it keeps the bounds trap. Returns null for + * other reads and uses. */ +function dereferencedElementReceiver( + lowerer: Lowerer, + expr: ts.NonNullExpression, + loc: SrcLoc, +): IrExpr | null { + const use = runtimeOptionalUseOf(expr); + if (!use || isAssignmentTarget(use.access)) return null; + const parts = dereferencedElementParts(lowerer, expr, loc, false); + if (parts?.message == null) return null; + const message = parts.message; + const value: IrExpr = { + kind: "ternary", + cond: parts.present, + then: parts.element, + else_: nodeTypeError(message, parts.element.type, loc), + type: parts.element.type, + loc, + }; + return parts.stmts.length === 0 + ? value + : { kind: "seqExpr", stmts: parts.stmts, result: value, type: value.type, loc }; +} + +/** The parts of a member access through `xs[i]!` (see + * dereferencedElementReceiver): statements that evaluate the array and the + * index, whether the element is present, the element read, and Node's + * TypeError message for a missing one. A `write` holds its key, so its + * message may be left for dereferencedElementWrite to build from the key. */ +export function dereferencedElementParts( + lowerer: Lowerer, + expr: ts.NonNullExpression, + loc: SrcLoc, + write: boolean, +): { stmts: IrStmt[]; present: IrExpr; element: IrExpr; message: IrExpr | null } | null { + let read: ts.Expression = expr.expression; + while (ts.isParenthesizedExpression(read)) read = read.expression; + if (!ts.isElementAccessExpression(read) || read.questionDotToken) return null; + const receiverType = lowerer.typeOf(read.expression); + if ( + !lowerer.checker.isArrayType(receiverType) || + lowerer.mapTypeOf(receiverType)?.kind !== "array" + ) + return null; + if (lowerer.nanCodedReads?.counterIndexInBounds(read)) return null; + const use = runtimeOptionalUseOf(expr); + if (!use || use.optional || use.complex || use.comma || use.kind === "call") return null; + // A write holds its key, so the message can name any number or string + // key (dereferencedElementWrite builds it). + const message = + memberAccessMessage(lowerer, use.access) ?? + (write && ts.isElementAccessExpression(use.access) && namedKey(lowerer, use.access) + ? null + : undefined); + if (message === undefined) return null; + if (lowerer.mapTypeOf(lowerer.typeOf(read.argumentExpression))?.kind !== "f64") return null; + // The element must already have the asserted type: an element union the + // assertion or a guard narrows keeps its checked narrow. + const target = lowerer.mapTypeOf(lowerer.typeOf(expr)); + const declared = lowerer.mapTypeOf(receiverType); + if ( + !target || + declared?.kind !== "array" || + declared.elem.kind === "void" || + !typeEquals(declared.elem, target) + ) + return null; + const arr = lowerer.lowerExpr(read.expression); + if (arr.type.kind !== "array" || !typeEquals(arr.type.elem, target)) return null; + const elem = arr.type.elem; + const index = lowerer.lowerExprExpecting(read.argumentExpression, F64); + // Plain reads (a local, a field of one) evaluate once more instead of + // through a hidden local: the receiver then stays a borrowed element + // read, like the plain `xs[i]` it replaces. An index that can assign + // can replace the array binding or field (`xs[(xs = other, 0)]!`), so the + // array is held before it, as JavaScript reads it first. + const stmts: IrStmt[] = []; + const plain = (value: IrExpr): boolean => isSafeToRepeat(value) || plainProjection(value); + const held = (value: IrExpr, name: string): IrExpr => { + const tmp = lowerer.declareHiddenLocal(name, value.type); + stmts.push({ kind: "varDecl", localId: tmp.id, init: value, loc }); + return varRef(tmp.id, value.type, loc); + }; + const a = !mayAssign(index) && plain(arr) ? arr : held(arr, "%elementArray"); + const i = plain(index) ? index : held(index, "%elementIndex"); + return { + stmts, + present: { + kind: "bin", + op: "===", + left: { kind: "arrayState", arr: a, index: i, type: F64, loc }, + right: numLit(1, loc), + type: BOOL, + loc, + }, + element: { kind: "arrayGet", arr: a, index: i, type: elem, loc }, + message, + }; +} + +/** A write through `xs[i]!` (`xs[i]!.name = v`, `xs[i]![k] = v`): as in + * Node, the array, the index, the key and the value evaluate in order (the + * statements), then a missing element makes the write throw Node's + * TypeError and a present one takes the write, which reads the element, + * the key and the value through `overrides`. Null for other targets. */ +export function dereferencedElementWrite( + lowerer: Lowerer, + expr: ts.BinaryExpression, +): { + stmts: IrStmt[]; + present: IrExpr; + message: IrExpr; + overrides: { node: ts.Expression; value: IrExpr }[]; +} | null { + if (expr.operatorToken.kind !== ts.SyntaxKind.EqualsToken) return null; + const target = expr.left; + if ( + !(ts.isPropertyAccessExpression(target) || ts.isElementAccessExpression(target)) || + target.questionDotToken || + lowerer.chainRecvByNode.has(target.expression) + ) + return null; + let receiver: ts.Expression = target.expression; + while (ts.isParenthesizedExpression(receiver)) receiver = receiver.expression; + if (!ts.isNonNullExpression(receiver)) return null; + const loc = locOf(expr); + const keyValue = ts.isElementAccessExpression(target) + ? lowerer.lowerExpr(target.argumentExpression) + : null; + const rightValue = lowerer.lowerExpr(expr.right); + const parts = dereferencedElementParts(lowerer, receiver, loc, true); + if (parts === null) return null; + const stmts = [...parts.stmts]; + let present = parts.present; + let element = parts.element; + // JavaScript reads the element before the key and the value. When either + // has an effect, which can replace, remove or add the element, its + // presence and value are read first; otherwise the write tests and reads + // it after them. + if (!unobservable(rightValue) || (keyValue !== null && !unobservable(keyValue))) { + const presentLocal = lowerer.declareHiddenLocal("%elementPresent", BOOL); + // Assigned only when the element is present. + const elementLocal = declareContextLocal( + lowerer.ctx, + "%elementValue", + element.type, + true, + undefined, + undefined, + ); + stmts.push( + { kind: "varDecl", localId: presentLocal.id, init: present, loc }, + { kind: "varDecl", localId: elementLocal.id, init: null, loc }, + ); + present = varRef(presentLocal.id, BOOL, loc); + stmts.push({ + kind: "if", + cond: present, + then: [{ kind: "assign", localId: elementLocal.id, value: element, loc }], + else_: null, + loc, + }); + element = varRef(elementLocal.id, element.type, loc); + } + const overrides: { node: ts.Expression; value: IrExpr }[] = []; + // The write lowers the node with or without its parentheses. + const override = (node: ts.Expression, value: IrExpr): void => { + for (let n: ts.Expression = node; ; n = n.expression) { + overrides.push({ node: n, value }); + if (!ts.isParenthesizedExpression(n)) break; + } + }; + const held = (value: IrExpr, name: string, inline: boolean): IrExpr => { + if (inline) return value; + // A call without a result evaluates to undefined. + if (value.type.kind === "void") { + stmts.push({ kind: "exprStmt", expr: value, loc }); + return { kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc }; + } + const tmp = lowerer.declareHiddenLocal(name, value.type); + stmts.push({ kind: "varDecl", localId: tmp.id, init: value, loc }); + return varRef(tmp.id, value.type, loc); + }; + override(target.expression, element); + let message = parts.message; + if (ts.isElementAccessExpression(target) && keyValue !== null) { + // The key evaluates before the value, which can assign the binding or + // field it reads; a literal, or an unobservable key the value cannot + // change, may evaluate in the write. + const key = held( + keyValue, + "%elementWriteKey", + keyValue.kind === "numLit" || + keyValue.kind === "strLit" || + (unobservable(keyValue) && !mayAssign(rightValue)), + ); + override(target.argumentExpression, key); + // The message names the key's value as the write evaluated it. + if (key !== keyValue || message === null) + message = keyMessage(messageHead(target), key, loc) ?? message; + } + // A value whose evaluation is unobservable may evaluate in the write + // itself, after the element is tested. + override( + expr.right, + held(rightValue, "%elementWriteValue", isSafeToRepeat(rightValue) || unobservable(rightValue)), + ); + if (message === null) return null; + return { stmts, present, message, overrides }; +} + +/** dereferencedElementWrite in value position (`f(xs[i]!.name = v)`). */ +export function lowerDereferencedElementAssignment( + lowerer: Lowerer, + expr: ts.BinaryExpression, +): IrExpr | null { + const write = dereferencedElementWrite(lowerer, expr); + if (write === null) return null; + const loc = locOf(expr); + const value = lowerer.withExpressionOverrides(write.overrides, () => lowerer.lowerExpr(expr)); + return { + kind: "seqExpr", + stmts: write.stmts, + result: { + kind: "ternary", + cond: write.present, + then: value, + else_: nodeTypeError(write.message, value.type, loc), + type: value.type, + loc, + }, + type: value.type, + loc, + }; +} + +/** Whether evaluating `e` has no effect and cannot throw: a discardable + * expression, or the compiler's element-or-undefined read of one. */ +function unobservable(e: IrExpr): boolean { + if (e.kind === "call" && e.callee.startsWith("%arr.idxOr.")) return e.args.every(unobservable); + return isSafeToDiscard(e); +} + +/** Whether evaluating `e` can assign a binding or a field: it contains an + * assignment, an increment, statements, a suspension, or a call that can + * run program code (a library call only through a function it is given). */ +function mayAssign(e: IrExpr): boolean { + switch (e.kind) { + case "call": + // The compiler's element-or-undefined read and union narrows run no + // program code. + if (!e.callee.startsWith("%arr.idxOr.") && !e.callee.startsWith("%union.narrow.")) + return true; + break; + case "assignExpr": + case "incDec": + case "seqExpr": + case "callValue": + case "virtualCall": + case "dynCall": + case "dynInvoke": + case "ffiCall": + case "new": + case "newValue": + case "awaitExpr": + case "awaitUnionExpr": + case "yieldExpr": + return true; + case "closure": + return false; + case "libCall": + if (e.args.some((arg) => arg.type.kind === "func")) return true; + break; + } + return !everyExprChild( + e, + (child) => !mayAssign(child), + () => false, + ); +} + +/** Node's TypeError, thrown in place of a value of `type`. */ +export function nodeTypeError(message: IrExpr, type: IrType, loc: SrcLoc): IrExpr { + return { + kind: "libCall", + fn: "error.nodeThrow", + args: [numLit(1, loc), strLit("", loc), message], + type, + loc, + }; +} + +/** Whether `access` is the target of `=`. */ +function isAssignmentTarget(access: ts.Expression): boolean { + let outer: ts.Expression = access; + while (ts.isParenthesizedExpression(outer.parent)) outer = outer.parent; + const parent = outer.parent; + return ( + ts.isBinaryExpression(parent) && + parent.left === outer && + parent.operatorToken.kind === ts.SyntaxKind.EqualsToken + ); +} + +/** A local, a number literal, or a field or record member read through a + * local: reading it again has no effect and yields the same value. */ +function plainProjection(e: IrExpr): boolean { + if (e.kind === "varRef" || e.kind === "numLit") return true; + return (e.kind === "fieldGet" || e.kind === "recordGet") && plainProjection(e.obj); +} + +/** Node's TypeError message for a member access on undefined: reading the + * member, or setting it when the access is the target of `=`. Null when + * the key is not a name, a literal, or a plain read of a number or string + * (which the message reads a second time). */ +function memberAccessMessage( + lowerer: Lowerer, + access: ts.PropertyAccessExpression | ts.ElementAccessExpression | ts.CallExpression, +): IrExpr | null { + const head = messageHead(access); + const loc = locOf(access); + if (ts.isPropertyAccessExpression(access)) { + if (ts.isPrivateIdentifier(access.name)) return null; + return strLit(`${head}${access.name.text}')`, loc); + } + if (!ts.isElementAccessExpression(access)) return null; + const key = runtimeOptionalElementKey(access.argumentExpression); + if (key !== null) return strLit(`${head}${key}')`, loc); + let keyNode = access.argumentExpression; + while (ts.isParenthesizedExpression(keyNode)) keyNode = keyNode.expression; + if (!ts.isIdentifier(keyNode)) return null; + const keyType = lowerer.mapTypeOf(lowerer.typeOf(keyNode))?.kind; + if (keyType !== "f64" && keyType !== "string") return null; + const keyValue = lowerer.lowerExpr(keyNode); + if (!isSafeToRepeat(keyValue) || keyValue.type.kind !== keyType) return null; + return keyMessage(head, keyValue, loc); +} + +/** The start of Node's TypeError message for a member access on undefined: + * setting the member when the access is the target of `=`, else reading + * it. */ +function messageHead(access: ts.Expression): string { + return isAssignmentTarget(access) + ? "Cannot set properties of undefined (setting '" + : "Cannot read properties of undefined (reading '"; +} + +/** Whether `access`'s key is typed a number or a string, which the message + * can name from its value. */ +function namedKey(lowerer: Lowerer, access: ts.ElementAccessExpression): boolean { + const kind = lowerer.mapTypeOf(lowerer.typeOf(access.argumentExpression))?.kind; + return kind === "f64" || kind === "string"; +} + +/** The message naming a key from its value (a number or a string), or null + * for another type. */ +function keyMessage(head: string, key: IrExpr, loc: SrcLoc): IrExpr | null { + if (key.type.kind !== "f64" && key.type.kind !== "string") return null; + const text: IrExpr = + key.type.kind === "string" ? key : { kind: "toString", operand: key, type: STRING, loc }; + const concat = (left: IrExpr, right: IrExpr): IrExpr => ({ + kind: "strConcat", + left, + right, + type: STRING, + loc, + }); + return concat(concat(strLit(head, loc), text), strLit("')", loc)); +} + +/** A string array's element converted to a string: the element, or + * "undefined" for a missing index or a hole. */ +function stringElementOrUndefined( + lowerer: Lowerer, + read: ts.ElementAccessExpression, + loc: SrcLoc, +): IrExpr | null { + if (lowerer.mapTypeOf(lowerer.typeOf(read.argumentExpression))?.kind !== "f64") return null; + const arr = lowerer.lowerExpr(read.expression); + if (arr.type.kind !== "array" || arr.type.elem.kind !== "string") return null; + const index = lowerer.lowerExprExpecting(read.argumentExpression, F64); + const stmts: IrStmt[] = []; + const held = (value: IrExpr, name: string): IrExpr => { + const tmp = lowerer.declareHiddenLocal(name, value.type); + stmts.push({ kind: "varDecl", localId: tmp.id, init: value, loc }); + return varRef(tmp.id, value.type, loc); + }; + // An index that can assign can replace the array binding or field, so + // the array is held before it, as JavaScript reads it first. + const a = !mayAssign(index) && isSafeToRepeat(arr) ? arr : held(arr, "%elementArray"); + const i = isSafeToRepeat(index) ? index : held(index, "%elementIndex"); + const value: IrExpr = { + kind: "ternary", + cond: { + kind: "bin", + op: "===", + left: { kind: "arrayState", arr: a, index: i, type: F64, loc }, + right: numLit(1, loc), + type: BOOL, + loc, + }, + then: { kind: "arrayGet", arr: a, index: i, type: STRING, loc }, + else_: strLit("undefined", loc), + type: STRING, + loc, + }; + return stmts.length === 0 ? value : { kind: "seqExpr", stmts, result: value, type: STRING, loc }; +} + +function isScalarArm(arm: IrType): boolean { + return arm.kind === "f64" || arm.kind === "string" || arm.kind === "bool" || isUnitType(arm); +} + +/** A union of numbers, strings, booleans, null and undefined with at least + * one unit arm. */ +function scalarUnion(lowerer: Lowerer, unionId: string): boolean { + const arms = lowerer.unions.get(unionId)?.arms; + return !!arms?.some(isUnitType) && arms.every(isScalarArm); +} + +/** How the converting use of `x!` (see assertionOperand) reads it: + * "number" where the use applies ToNumber, "string" where it converts the + * value to a string (a template span, `+` with a string), "value" where the + * consumer lowers the union itself (equality, truthiness, a comparison with + * a string), null for other uses. */ +function convertingUseOf( + lowerer: Lowerer, + expr: ts.NonNullExpression, +): "number" | "string" | "value" | null { let outer: ts.Expression = expr; while (ts.isParenthesizedExpression(outer.parent)) outer = outer.parent; const parent = outer.parent; - if (ts.isTemplateSpan(parent)) return parent.expression === outer ? "value" : null; + if (ts.isTemplateSpan(parent)) return parent.expression === outer ? "string" : null; if (ts.isTypeOfExpression(parent)) return "value"; if (ts.isPrefixUnaryExpression(parent)) { if (parent.operator === ts.SyntaxKind.ExclamationToken) return "value"; @@ -1140,7 +1608,7 @@ function convertingUseOf(lowerer: Lowerer, expr: ts.NonNullExpression): "number" if (op === ts.SyntaxKind.PlusEqualsToken && parent.right !== outer) return null; let partner = other; while (ts.isParenthesizedExpression(partner)) partner = partner.expression; - if (isString(other)) return ts.isNonNullExpression(partner) ? null : "value"; + if (isString(other)) return ts.isNonNullExpression(partner) ? null : "string"; if (!numericPartner(lowerer.typeOf(other))) return "value"; return isString(expr) ? null : "number"; } @@ -1226,7 +1694,8 @@ export function assertedReadObservesUndefined( lowerer: Lowerer, node: ts.NonNullExpression, ): boolean { - if (convertingUseOf(lowerer, node) === "value") return true; + const use = convertingUseOf(lowerer, node); + if (use === "value" || use === "string") return true; let outer: ts.Expression = node; while (ts.isParenthesizedExpression(outer.parent)) outer = outer.parent; const parent = outer.parent; diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 8c49ae4d9..d43aaabf8 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -242,10 +242,12 @@ import { globalSymbolKey } from "./expressions/global-symbols.js"; import { identifiersNamed } from "./identifier-uses.js"; import { lowerShortCircuitAssignment } from "./expressions/nullish-assignment.js"; import { + dereferencedElementWrite, jsHoldsDynObject, lowerEnvironmentKey, lowerNativeFunctionAssignment, lowerStrictEquality, + nodeTypeError, } from "./lower-exprs.js"; import { keptIdentityAssertion } from "./assertion-identity.js"; import { isNativeProxyInitializer } from "./expressions/native-proxy.js"; @@ -7992,6 +7994,29 @@ function lowerCjsSingleValueExport( * assignment, and `({ a, b } = e);` (destructuring ASSIGNMENT, which JS * requires parenthesized in statement position) lands in the * object-literal-target branch below. */ +/** dereferencedElementWrite in statement position. */ +function lowerDereferencedElementWrite(lowerer: Lowerer, expr: ts.BinaryExpression): IrStmt | null { + const write = dereferencedElementWrite(lowerer, expr); + if (write === null) return null; + const loc = locOf(expr); + return { + kind: "block", + body: [ + ...write.stmts, + { + kind: "if", + cond: write.present, + then: [ + lowerer.withExpressionOverrides(write.overrides, () => lowerExprStatement(lowerer, expr)), + ], + else_: [{ kind: "exprStmt", expr: nodeTypeError(write.message, VOID, loc), loc }], + loc, + }, + ], + loc, + }; +} + export function lowerExprStatement(lowerer: Lowerer, expr: ts.Expression): IrStmt { const stmtNode = expr.parent; while (ts.isParenthesizedExpression(expr)) expr = expr.expression; @@ -8254,6 +8279,10 @@ export function lowerExprStatement(lowerer: Lowerer, expr: ts.Expression): IrStm const ex = lowerExpandoAssignStmt(lowerer, expr); if (ex) return ex; } + { + const dereferenced = lowerDereferencedElementWrite(lowerer, expr); + if (dereferenced) return dereferenced; + } if (ts.isElementAccessExpression(expr.left)) return lowerer.lowerElementWrite(expr); if (ts.isPropertyAccessExpression(expr.left)) { fenceNodeModuleMutation(lowerer, expr.left, "assignment"); @@ -8951,13 +8980,17 @@ export function lowerExprStatement(lowerer: Lowerer, expr: ts.Expression): IrStm return { kind: "block", body: [], loc: locOf(expr) }; } if ( - stmtNode !== undefined && - ts.isExpressionStatement(stmtNode) && ts.isBinaryExpression(expr) && (expr.operatorToken.kind === ts.SyntaxKind.AmpersandAmpersandToken || expr.operatorToken.kind === ts.SyntaxKind.BarBarToken) ) { - return lowerDiscardedLogical(lowerer, expr); + // A right operand without a value (`skip || (flag ? warnA() : warnB())`, + // where both arms are void calls) leaves the logical nothing to join: + // it is the branch `if (!skip) ...` wherever its result is discarded, + // a for loop's initializer and update included. + if (needsNoValue(lowerer, expr.right)) return lowerLogicalBranch(lowerer, expr); + if (stmtNode !== undefined && ts.isExpressionStatement(stmtNode)) + return lowerDiscardedLogical(lowerer, expr); } const value = lowerer.lowerExpr(expr); // A bare unit-literal statement (`undefined;`): a pure no-op in JS, @@ -9000,6 +9033,32 @@ function lowerDiscardedLogical(lowerer: Lowerer, expr: ts.BinaryExpression): IrS if (!local) throw e; lowerer.diags.length = diagsBefore; } + return lowerLogicalBranch(lowerer, expr); +} + +/** True when the discarded `expr` needs no value: its type maps to void, + * or it is a logical or comma expression whose last operand needs none. A + * discarded logical whose right operand needs no value lowers as a branch + * that runs the operand as a statement. */ +function needsNoValue(lowerer: Lowerer, expr: ts.Expression): boolean { + while (ts.isParenthesizedExpression(expr)) expr = expr.expression; + if ( + ts.isBinaryExpression(expr) && + (expr.operatorToken.kind === ts.SyntaxKind.AmpersandAmpersandToken || + expr.operatorToken.kind === ts.SyntaxKind.BarBarToken || + expr.operatorToken.kind === ts.SyntaxKind.CommaToken) + ) + return needsNoValue(lowerer, expr.right); + const type = lowerer.typeOf(expr); + return ( + (type.flags & (ts.TypeFlags.Void | ts.TypeFlags.TypeParameter)) !== 0 && + lowerer.mapTypeOf(type)?.kind === "void" + ); +} + +/** `a && b` as `if (a) b;` and `a || b` as `if (!a) b;`, with `b` lowered + * as a statement under the narrowing `a` gives it. */ +function lowerLogicalBranch(lowerer: Lowerer, expr: ts.BinaryExpression): IrStmt { const and = expr.operatorToken.kind === ts.SyntaxKind.AmpersandAmpersandToken; const cond = lowerer.lowerCondition(expr.left); if (cond.kind === "boolLit") { @@ -11794,6 +11853,7 @@ function lowerForOfDyn( const iterator = lowerer.declareHiddenLocal("%dofIterator", DYN); const next = lowerer.declareHiddenLocal("%dofNext", DYN); const fast = lowerer.declareHiddenLocal("%dofNative", BOOL); + const generator = lowerer.declareHiddenLocal("%dofGenerator", F64); const done = lowerer.declareHiddenLocal("%dofDone", BOOL); done.mutable = true; const value = lowerer.declareHiddenLocal("%dofValue", DYN); @@ -11803,6 +11863,15 @@ function lowerForOfDyn( needsClose.mutable = true; const error = lowerer.declareHiddenLocal("%dofError", CAUGHT); const syncIterator = asyncLoop ? lowerer.declareHiddenLocal("%dofSync", BOOL) : null; + lowerer.untypedLoops.set(generator, { + kind: generator.id, + value: value.id, + iterator: iterator.id, + next: next.id, + done: done.id, + async: asyncLoop, + sync: syncIterator?.id ?? null, + }); const awaitDyn = (value: IrExpr): IrExpr => ({ kind: "libCall", fn: "async.awaitDyn", @@ -12077,6 +12146,14 @@ function lowerForOfDyn( : iteratorCanStep(varRef(iterator.id, DYN, loc), varRef(next.id, DYN, loc), loc), loc, }, + // Which of the program's generator types the iterator is, once + // generator-dynamic-dispatch.ts knows them; 0 keeps the protocol. + { + kind: "varDecl", + localId: generator.id, + init: { kind: "numLit", value: 0, type: F64, loc }, + loc, + }, { kind: "varDecl", localId: done.id, diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 7138070e2..4a42b2d93 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -21,7 +21,7 @@ import { import { isDeclBoundaryFile } from "../decl-projection.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; import { moduleNamespaceObject, moduleNamespaceValue } from "./module-namespace-elements.js"; -import { keyedHelperName } from "./stable-names.js"; +import { keyedHelperName, moduleTagNumber, portableRelativePath } from "./stable-names.js"; import { positionTag, StructuralPositions } from "./structural-positions.js"; import { bindingSymbolOfRead, onlyForwarded, stripTypeWrappers } from "./typed-arguments.js"; import { @@ -181,6 +181,7 @@ import { } from "./class-dynamic-dispatch.js"; import { addUntypedIntegrityWords, guardClassAttributeWrites } from "./class-attributes.js"; import { inferredCollectionType } from "./js-collection-types.js"; +import { sloppyDynamicWrites } from "./frozen-records.js"; import { guardFrozenWrites, markSloppyRecordWrites } from "./frozen-records.js"; import { applyHandleProperties } from "./handle-properties.js"; import { installBufferValue } from "./containers/bytes.js"; @@ -217,7 +218,7 @@ export { newFnCtx, type FnCtx } from "./function-context.js"; * - Lexical scoping is resolved here: locals get function-unique ids * ("x.0", "x.1" for shadowing); the IR is scope-flat. */ -import { dirname, relative, resolve } from "node:path"; +import { dirname, resolve } from "node:path"; import { bindingSource } from "../binding-source.js"; import { compilerOwnedFileName, tsgoPath } from "../dts-paths.js"; import * as ts from "../ts7/adapter.js"; @@ -2069,6 +2070,21 @@ function byName(fns: readonly IrFunction[]): IrFunction[] { ); } +/** A loop over an untyped iterable (lowerForOfDyn). `kind` is a local the + * loop sets once it has opened the iterator: 0 for the dynamic protocol, + * or the 1-based index of the program's generator type it steps natively. + * `value` is the per-step local whose initializer takes one step. */ +export interface UntypedLoopSite { + kind: string; + value: string; + iterator: string; + next: string; + done: string; + /** `for await`; `sync` names its async-from-sync flag. */ + async: boolean; + sync: string | null; +} + /** The program module a createRequire binding requires. */ type CreateRequireTarget = NonNullable>; @@ -3605,6 +3621,11 @@ export class Lowerer { * values other than fresh literals (noteUntypedIntegrity). */ untypedIntegrity = false; readonly liftedFns: IrFunction[] = []; + /** `for … of` and `for await` loops over untyped iterables, by their + * generator-kind local (generator-dynamic-dispatch.ts steps the program's + * own generators there on the typed path). Local ids repeat across + * functions; the local itself is in its function's locals. */ + readonly untypedLoops = new Map(); /** global-values.ts's answer per JavaScript source: the receiver of its * first read of the global object by a computed name, or null. */ readonly computedGlobalReads = new Map(); @@ -3630,12 +3651,17 @@ export class Lowerer { * names do not depend on where the program sits, default library files * by their base name (two checkers install them in different * directories), and the files scriptc ships by their name inside scriptc, - * so names do not depend on where scriptc is installed either. */ - readonly positions = new StructuralPositions((sf) => - this.program.isSourceFileDefaultLibrary(sf) + * so names do not depend on where scriptc is installed either. Relative + * paths use `/` on every platform. */ + readonly positions = new StructuralPositions((sf) => this.stableFileName(sf)); + + /** The name of `sf` in positions and module tags. */ + stableFileName(sf: ts.SourceFile): string { + return this.program.isSourceFileDefaultLibrary(sf) ? `lib:${sf.fileName.slice(sf.fileName.lastIndexOf("/") + 1)}` - : (compilerOwnedFileName(sf.fileName) ?? relative(dirname(this.entry.fileName), sf.fileName)), - ); + : (compilerOwnedFileName(sf.fileName) ?? + portableRelativePath(dirname(this.entry.fileName), sf.fileName)); + } /** The outermost node whose token span is exactly `loc`, or null. */ nodeAtLoc(loc: SrcLoc): ts.Node | null { @@ -4334,9 +4360,20 @@ export class Lowerer { // moduleOrder BEFORE any pass constructed — lowerToIr runs // appendDynamicImportModules once on the shared array (a per-pass run // here would repeatedly extend the graph and duplicate cycle reports). - this.moduleOrder.forEach((sf, i) => { - this.fileTag.set(sf, sf === entry ? "" : `%m${i}.`); - }); + // A module's tag hashes its stable file name rather than numbering its + // place in moduleOrder, so adding or reordering modules leaves the other + // modules' function names, and the closure names keyed by them, + // unchanged. The native compiler's profile matches functions by name. + const tagNumbers = new Set(); + for (const sf of this.moduleOrder) { + if (sf === entry) { + this.fileTag.set(sf, ""); + continue; + } + const n = moduleTagNumber(this.stableFileName(sf), tagNumbers); + tagNumbers.add(n); + this.fileTag.set(sf, `%m${n}.`); + } if (this.moduleOrder.length === 0) this.fileTag.set(entry, ""); const jsDynamic = jsDynamicDeclarations(this.moduleOrder, this.checker); if (jsDynamic.size > 0) this.typeCtx.jsDynamicDeclaration = (decl) => jsDynamic.has(decl); @@ -8826,6 +8863,7 @@ export class Lowerer { checkRequireBindingAccesses(this, functions); refuseX509DynValues(this, functions); markInferredFactChecks(functions, this.program); + const sloppyDynWrites = sloppyDynamicWrites(this, functions); const module: IrModule | null = this.diags.length > 0 ? null @@ -8860,6 +8898,7 @@ export class Lowerer { ? { freezableArrays: [...this.freezableArrays].sort() } : {}), ...(this.untypedIntegrity ? { untypedIntegrity: true as const } : {}), + ...(sloppyDynWrites.length > 0 ? { sloppyDynWrites } : {}), ...(isNodeEsmFile(this.entry, this.program) ? {} : { commonJsEntry: true }), ...(this.ffiImports.length > 0 ? { ffiImports: [...this.ffiImports] } : {}), }; diff --git a/packages/compiler/src/frontend/lowering/nan-coded-reads.ts b/packages/compiler/src/frontend/lowering/nan-coded-reads.ts index 24c981bd3..9456cfa18 100644 --- a/packages/compiler/src/frontend/lowering/nan-coded-reads.ts +++ b/packages/compiler/src/frontend/lowering/nan-coded-reads.ts @@ -451,6 +451,19 @@ export class NanCodedReads { // ---- present reads ------------------------------------------------------ + /** `xs[i]` whose index is a non-negative integer counter of the reading + * function and that a dominating `i < xs.length` guard bounds by the + * length: the read names an existing index. */ + counterIndexInBounds(read: ts.ElementAccessExpression): boolean { + const index = peelTransparent(read.argumentExpression); + if (!ts.isIdentifier(index)) return false; + const symbol = this.symbolOf(index); + if (!symbol || !this.nonNegativeCounter(symbol)) return false; + const decl = this.host.declarationOf(symbol); + if (!decl || this.containingFunction(decl) !== this.containingFunction(read)) return false; + return indexReadInBounds(read); + } + /** `t[i]` names an existing element: `i` is a non-negative integer * counter and a dominating guard bounds it by `t.length` (or by the * length of a typed array that always has the same length). diff --git a/packages/compiler/src/frontend/lowering/narrowing-stability.ts b/packages/compiler/src/frontend/lowering/narrowing-stability.ts index d0c9b04a6..9fae65a89 100644 --- a/packages/compiler/src/frontend/lowering/narrowing-stability.ts +++ b/packages/compiler/src/frontend/lowering/narrowing-stability.ts @@ -1,4 +1,5 @@ import * as ts from "../ts7/adapter.js"; +import { heritageElementTarget } from "../heritage-elements.js"; import { InternalCompilerError } from "../../errors.js"; import { typeEquals } from "../../ir/ir.js"; import type { IrType } from "../../ir/ir.js"; @@ -674,8 +675,9 @@ export class NarrowingStability { if (node.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.Decorator)) return true; for (const clause of node.heritageClauses?.nodes ?? []) { for (const heritage of clause.types.nodes) { - const e = heritage.expression; - if (!ts.isIdentifier(e) && !ts.isPropertyAccessExpression(e)) return true; + const e = heritageElementTarget(heritage); + if (!ts.isIdentifier(e) && !ts.isPropertyAccessExpression(e) && !ts.isQualifiedName(e)) + return true; } } return node.members.nodes.some( diff --git a/packages/compiler/src/frontend/lowering/stable-names.test.ts b/packages/compiler/src/frontend/lowering/stable-names.test.ts new file mode 100644 index 000000000..dbef9b29e --- /dev/null +++ b/packages/compiler/src/frontend/lowering/stable-names.test.ts @@ -0,0 +1,21 @@ +import { expect, test } from "vitest"; +import { moduleTagNumber, portableRelativePath } from "./stable-names.js"; + +test("module tag numbers depend only on the file name and stay distinct", () => { + const first = moduleTagNumber("lib/a.ts", new Set()); + expect(moduleTagNumber("lib/a.ts", new Set([7, 8]))).toBe(first); + expect(moduleTagNumber("lib/b.ts", new Set())).not.toBe(first); + expect(moduleTagNumber("lib/a.ts", new Set([first]))).toBe((first + 1) >>> 0); + expect(moduleTagNumber("lib/a.ts", new Set([first, (first + 1) >>> 0]))).toBe((first + 2) >>> 0); +}); + +test("relative module paths use / on every platform", () => { + expect(portableRelativePath("/repo/src", "/repo/src/lib/a.ts", "linux")).toBe("lib/a.ts"); + expect(portableRelativePath("C:/repo/src", "C:/repo/src/lib/a.ts", "win32")).toBe("lib/a.ts"); + expect(portableRelativePath("C:\\repo\\src", "C:/repo/node_modules/x/index.d.ts", "win32")).toBe( + "../node_modules/x/index.d.ts", + ); + expect( + moduleTagNumber(portableRelativePath("C:/repo", "C:/repo/lib/a.ts", "win32"), new Set()), + ).toBe(moduleTagNumber(portableRelativePath("/repo", "/repo/lib/a.ts", "linux"), new Set())); +}); diff --git a/packages/compiler/src/frontend/lowering/stable-names.ts b/packages/compiler/src/frontend/lowering/stable-names.ts index 8f7d0967e..0d57bd2c9 100644 --- a/packages/compiler/src/frontend/lowering/stable-names.ts +++ b/packages/compiler/src/frontend/lowering/stable-names.ts @@ -1,4 +1,7 @@ +import * as posix from "node:path/posix"; +import * as win32 from "node:path/win32"; import { InternalCompilerError } from "../../errors.js"; +import { tsgoPath } from "../ts7/session-path.js"; /** The owner of the names keyedHelperName hands out. */ export interface KeyedNameOwner { @@ -46,3 +49,26 @@ export function keyedHelperName( throw new InternalCompilerError(`helper name ${name} is shared by two helper keys`); return name; } + +/** The number in the `%m.` tag of the module named `fileName`: a + * 32-bit FNV-1a hash of the name, so a module keeps its tag when other + * modules are added, removed or reordered. A number in `taken` moves to the + * next free one, which keeps tags distinct. */ +export function moduleTagNumber(fileName: string, taken: ReadonlySet): number { + let h = 0x811c9dc5; + for (let i = 0; i < fileName.length; i++) h = Math.imul(h ^ fileName.charCodeAt(i), 0x01000193); + let n = h >>> 0; + while (taken.has(n)) n = (n + 1) >>> 0; + return n; +} + +/** The path of `file` relative to the directory `from`, with `/` separators + * on every platform, so names built from it match across hosts. */ +export function portableRelativePath( + from: string, + file: string, + platform: NodeJS.Platform = process.platform, +): string { + const path = platform === "win32" ? win32.relative(from, file) : posix.relative(from, file); + return tsgoPath(path, platform); +} diff --git a/packages/compiler/src/ir/fields.generated.ts b/packages/compiler/src/ir/fields.generated.ts index e9f5efaa3..86c6a15e5 100644 --- a/packages/compiler/src/ir/fields.generated.ts +++ b/packages/compiler/src/ir/fields.generated.ts @@ -2069,3 +2069,252 @@ export function walkTypeFields(node: IrType, walker: IrFieldWalker, skipType: bo } } } + +/** A field ir.ts declares that this file does not read. */ +type FieldsNotRead = Exclude; +/** The string or object values a field read as a scalar can hold. */ +type ValuesNotRead = Extract, string | object>; +/** Fails type checking unless `T` is never: run `pnpm generate:ir-fields`. */ +type Unread = T; + +/** Every walked object type's fields as ir.ts declared them when this file + * was generated. */ +export type IrFieldsCurrent = [ + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc">>, + Unread, "kind" | "arr" | "index" | "type" | "loc">>, + Unread, "kind" | "arr" | "index" | "type" | "loc">>, + Unread, "kind" | "elems" | "spreads" | "capacity" | "type" | "loc">>, + Unread["capacity"]>>, + Unread, "kind" | "length" | "type" | "loc">>, + Unread, "kind" | "arr" | "index" | "type" | "loc">>, + Unread, "kind" | "localId" | "value" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "adopt" | "loc">>, + Unread["adopt"]>>, + Unread, "kind" | "value" | "promiseTag" | "type" | "loc">>, + Unread["promiseTag"]>>, + Unread, "kind" | "op" | "left" | "right" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread["value"]>>, + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc" | "invalidNaN">>, + Unread["invalidNaN"]>>, + Unread, "kind" | "source" | "from" | "type" | "loc">>, + Unread["from"]>>, + Unread, "kind" | "callee" | "args" | "type" | "loc">>, + Unread, "kind" | "callee" | "receiver" | "args" | "type" | "loc">>, + Unread, "kind" | "value" | "className" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "value" | "test" | "className" | "negated" | "type" | "loc">>, + Unread["negated"]>>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "id" | "type" | "loc">>, + Unread, "kind" | "className" | "captures" | "type" | "loc">>, + Unread, "kind" | "fnName" | "captures" | "adapts" | "type" | "loc">>, + Unread["adapts"]>>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "elems" | "type" | "loc">>, + Unread["spreads"]>, readonly unknown[]>[number]>, object>, "arg" | "what">>, + Unread["spreads"]>, readonly unknown[]>[number]>, object>["arg"]>>, + Unread, "kind" | "callee" | "receiver" | "calleeName" | "calleeNameValue" | "args" | "spreads" | "type" | "loc">>, + Unread, "kind" | "value" | "preserveRefs" | "adopt" | "exact" | "inferredSite" | "origin" | "type" | "loc">>, + Unread["preserveRefs"]>>, + Unread["adopt"]>>, + Unread["exact"]>>, + Unread["origin"]>>, + Unread, "kind" | "value" | "spelling" | "firstProp" | "type" | "loc">>, + Unread, "kind" | "value" | "fnName" | "fnLength" | "liveRef" | "type" | "loc">>, + Unread["fnLength"]>>, + Unread["liveRef"]>>, + Unread, "kind" | "key" | "negated" | "value" | "type" | "loc">>, + Unread["negated"]>>, + Unread, "kind" | "recv" | "method" | "calleeName" | "calleeNameValue" | "args" | "type" | "loc">>, + Unread, "kind" | "value" | "count" | "notIterableMessage" | "type" | "loc">>, + Unread["count"]>>, + Unread, "kind" | "key" | "optional" | "value" | "type" | "loc">>, + Unread["optional"]>>, + Unread["fields"]>, readonly unknown[]>[number]>, object>, "key" | "value">>, + Unread, "kind" | "fields" | "type" | "loc">>, + Unread, "kind" | "left" | "right" | "negated" | "type" | "loc">>, + Unread["negated"]>>, + Unread, "kind" | "test" | "bytesElem" | "negated" | "value" | "type" | "loc">>, + Unread["negated"]>>, + Unread, "kind" | "import" | "args" | "type" | "loc">>, + Unread, "kind" | "unionId" | "type" | "loc">>, + Unread, "kind" | "obj" | "className" | "field" | "type" | "loc">>, + Unread, "kind" | "op" | "prefix" | "obj" | "className" | "field" | "fieldDyn" | "type" | "loc">>, + Unread["prefix"]>>, + Unread["fieldDyn"]>>, + Unread, "kind" | "mode" | "gen" | "arg" | "type" | "loc">>, + Unread, "kind" | "op" | "prefix" | "localId" | "type" | "loc">>, + Unread["prefix"]>>, + Unread, "kind" | "value" | "className" | "type" | "loc">>, + Unread, "kind" | "value" | "classValue" | "type" | "loc">>, + Unread, "kind" | "name" | "args" | "type" | "loc">>, + Unread, "kind" | "value" | "indent" | "circular" | "type" | "loc">>, + Unread["circular"]>>, + Unread["assertOptions"]>, object>, "diff" | "skipPrototype">>, + Unread["assertOptions"]>, object>["skipPrototype"]>>, + Unread, "kind" | "fn" | "args" | "type" | "loc" | "prototypeIdentityOnly" | "realTimer" | "assertOptions">>, + Unread["prototypeIdentityOnly"]>>, + Unread["realTimer"]>>, + Unread, "kind" | "op" | "left" | "right" | "type" | "loc">>, + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc">>, + Unread["seed"]>, readonly unknown[]>[number]>, object>, "key" | "value">>, + Unread, "kind" | "seed" | "type" | "loc">>, + Unread, "kind" | "moduleId" | "type" | "loc">>, + Unread, "kind" | "className" | "args" | "type" | "loc">>, + Unread, "kind" | "executor" | "type" | "loc">>, + Unread, "kind" | "callee" | "args" | "type" | "loc">>, + Unread, "kind" | "left" | "right" | "type" | "loc">>, + Unread, "kind" | "value" | "spelling" | "type" | "loc">>, + Unread["value"]>>, + Unread, "kind" | "id" | "receiver" | "body" | "type" | "loc">>, + Unread, "kind" | "left" | "right" | "retag" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "type" | "loc">>, + Unread["overrides"]>, readonly unknown[]>[number]>, object>, "name" | "value">>, + Unread, "kind" | "source" | "overrides" | "type" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "field" | "type" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "field" | "type" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "key" | "overflowOnly" | "missNaN" | "type" | "loc">>, + Unread["overflowOnly"]>>, + Unread["missNaN"]>>, + Unread["fields"]>, readonly unknown[]>[number]>, object>, "name" | "value" | "overflow">>, + Unread["fields"]>, readonly unknown[]>[number]>, object>["overflow"]>>, + Unread, "kind" | "fields" | "nullProto" | "type" | "loc">>, + Unread["nullProto"]>>, + Unread, "kind" | "obj" | "shapeId" | "type" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "key" | "type" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "type" | "loc">>, + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc">>, + Unread, "kind" | "pattern" | "flags" | "type" | "loc">>, + Unread, "kind" | "type" | "loc">>, + Unread, "kind" | "stmts" | "result" | "generatorDelegate" | "diverges" | "type" | "loc">>, + Unread["generatorDelegate"]>>, + Unread["diverges"]>>, + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc">>, + Unread, "kind" | "seed" | "type" | "loc">>, + Unread, "kind" | "op" | "left" | "right" | "type" | "loc">>, + Unread, "kind" | "left" | "right" | "type" | "loc">>, + Unread, "kind" | "negated" | "left" | "right" | "type" | "loc">>, + Unread["negated"]>>, + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "key" | "cooked" | "type" | "loc">>, + Unread, "kind" | "cond" | "then" | "else_" | "type" | "loc">>, + Unread, "kind" | "operand" | "type" | "loc">>, + Unread, "kind" | "operand" | "type" | "loc">>, + Unread, "kind" | "op" | "operand" | "type" | "loc">>, + Unread, "kind" | "unionId" | "field" | "value" | "type" | "loc">>, + Unread, "kind" | "unionId" | "negated" | "sameValue" | "left" | "right" | "type" | "loc">>, + Unread["negated"]>>, + Unread["sameValue"]>>, + Unread, "kind" | "unionId" | "tag" | "union" | "func" | "negated" | "type" | "loc">>, + Unread["tag"]>>, + Unread["negated"]>>, + Unread, "kind" | "unionId" | "tag" | "negated" | "value" | "type" | "loc">>, + Unread["tag"]>>, + Unread["negated"]>>, + Unread, "kind" | "unionId" | "key" | "value" | "type" | "loc">>, + Unread, "kind" | "unionId" | "tag" | "value" | "type" | "loc">>, + Unread["tag"]>>, + Unread, "kind" | "fromId" | "unionId" | "tags" | "value" | "type" | "loc">>, + Unread, "kind" | "unionId" | "tag" | "value" | "type" | "loc">>, + Unread["tag"]>>, + Unread, "kind" | "unit" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "localId" | "type" | "loc">>, + Unread, "kind" | "className" | "method" | "args" | "type" | "loc" | "targets">>, + Unread["captureCompletion"]>, object>, "returnType">>, + Unread, "kind" | "value" | "awaited" | "delegated" | "captureCompletion" | "type" | "loc">>, + Unread["awaited"]>>, + Unread["delegated"]>>, + Unread, "kind" | "arr" | "index" | "loc">>, + Unread, "kind" | "arr" | "index" | "value" | "loc">>, + Unread, "kind" | "arr" | "length" | "loc" | "pop" | "reserve">>, + Unread["pop"]>>, + Unread["reserve"]>>, + Unread, "kind" | "arr" | "index" | "loc">>, + Unread, "kind" | "localId" | "value" | "initializes" | "loc">>, + Unread["initializes"]>>, + Unread, "kind" | "body" | "labels" | "loc">>, + Unread, "kind" | "label" | "loc">>, + Unread, "kind" | "arr" | "index" | "value" | "loc">>, + Unread, "kind" | "label" | "loc">>, + Unread, "kind" | "body" | "cond" | "labels" | "loc">>, + Unread, "kind" | "expr" | "loc">>, + Unread, "kind" | "obj" | "className" | "field" | "value" | "sloppy" | "define" | "attributesChecked" | "loc">>, + Unread["sloppy"]>>, + Unread["define"]>>, + Unread["attributesChecked"]>>, + Unread, "kind" | "init" | "cond" | "update" | "body" | "labels" | "loc">>, + Unread, "kind" | "localId" | "iterable" | "body" | "labels" | "loc">>, + Unread, "kind" | "cond" | "then" | "else_" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "key" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "key" | "value" | "overflowOnly" | "loc">>, + Unread["overflowOnly"]>>, + Unread, "kind" | "obj" | "shapeId" | "field" | "value" | "loc">>, + Unread, "kind" | "localId" | "loc">>, + Unread, "kind" | "value" | "awaited" | "loc">>, + Unread["awaited"]>>, + Unread, "kind" | "code" | "message" | "loc">>, + Unread["cases"]>, readonly unknown[]>[number]>, object>, "test" | "body">>, + Unread, "kind" | "disc" | "cases" | "labels" | "loc">>, + Unread, "kind" | "value" | "loc">>, + Unread, "kind" | "tryBody" | "catchBody" | "catchLocalId" | "finallyBody" | "suppressFinallyErrors" | "dropFinallyErrors" | "loc">>, + Unread["suppressFinallyErrors"]>>, + Unread["dropFinallyErrors"]>>, + Unread, "kind" | "localId" | "init" | "loc">>, + Unread, "kind" | "cond" | "body" | "labels" | "loc">>, + Unread, "kind" | "elem" | "optionalElements">>, + Unread["optionalElements"]>>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "elem">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "className">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "params" | "ret" | "rest" | "restAbi" | "argumentsAll">>, + Unread["rest"]>>, + Unread["argumentsAll"]>>, + Unread, "kind" | "async" | "yieldT" | "retT" | "nextT">>, + Unread["async"]>>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "key" | "value">>, + Unread, "kind" | "moduleId">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "className">>, + Unread, "kind">>, + Unread, "kind" | "inner">>, + Unread, "kind" | "shapeId">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "elem">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "unionId">>, + Unread, "kind">>, + Unread, "kind">>, +]; diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index 48fa8a841..25ad47137 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -234,6 +234,7 @@ const MODULE_FIELDS: Record>, Role> = { "nodeIsland.modules.files[].sha256": "other", "nodeIsland.modules.links[].path": "other", "nodeIsland.modules.links[].target": "other", + "sloppyDynWrites[]": "other", "inferredFactFiles[].file": "other", "inferredFactFiles[].pkg": "other", "declOrigins[]": "other", diff --git a/packages/compiler/src/ir/int32-slots.ts b/packages/compiler/src/ir/int32-slots.ts index 1334b9b99..be5f29768 100644 --- a/packages/compiler/src/ir/int32-slots.ts +++ b/packages/compiler/src/ir/int32-slots.ts @@ -1484,6 +1484,39 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): continue; } const int32 = (e: IrExpr): boolean => withinInt32(ranges.get(e)); + // The range of an exact integer that may also be NaN: an element read + // of a candidate through `getNumber` (`a[k]!` in arithmetic, NaN for a + // missing element), and `+`, `-` and `*` over such values and integers. + // A checked store widens the array at a NaN like at any value that is + // not an int32. + const integerOrNaN = (e: IrExpr): IntegerRange | null => { + const range = ranges.get(e); + if (range) return range; + if (e.kind === "numLit") + return Number.isInteger(e.value) ? { min: e.value, max: e.value } : null; + if (e.kind === "arrIntrinsic" && e.method === "getNumber" && e.receiver.kind === "varRef") + return slots.hasInt32ElementSlot(int32ArraySlot(int32ArrayKey(fn.name, e.receiver.localId))) + ? INT32_RANGE + : null; + if (e.kind !== "bin" || (e.op !== "+" && e.op !== "-" && e.op !== "*")) return null; + const left = integerOrNaN(e.left); + const right = left && integerOrNaN(e.right); + if (!left || !right) return null; + const bounds = + e.op === "+" + ? [left.min + right.min, left.max + right.max] + : e.op === "-" + ? [left.min - right.max, left.max - right.min] + : [ + left.min * right.min, + left.min * right.max, + left.max * right.min, + left.max * right.max, + ]; + const min = Math.min(...bounds); + const max = Math.max(...bounds); + return Math.abs(min) < 2 ** 53 && Math.abs(max) < 2 ** 53 ? { min, max } : null; + }; // Every element stored into an int32 array candidate must be an int32 // under the facts that assume the candidates' own elements are; a // failing store drops its array's element facts, and the function is @@ -1513,6 +1546,10 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): for (const [id, values] of int32ArrayStores(fn, arrays)) { const key = int32ArrayKey(fn.name, id); if (!slots.hasInt32ElementSlot(int32ArraySlot(key)) || values.every(int32)) continue; + // Checked stores serve a lone local: its allocations are the only + // arrays it holds. Its stores are judged under the element facts + // the ranges were computed with. + const checkable = values.every((value) => int32(value) || integerOrNaN(value) !== null); // The array's group drops together: every function reading a // member is analyzed again without its facts. const group = arrayGroups.group.get(key) ?? [key]; @@ -1523,17 +1560,11 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): if (names) for (const name of names) requeue(name); } else requeue(member.slice(0, member.indexOf("\0"))); } - // Checked stores serve a lone local: its allocations are the only - // arrays it holds. - const checkable = (value: IrExpr): boolean => { - const range = ranges.get(value); - return int32(value) || range != null; - }; if ( group.length === 1 && !key.startsWith("\0") && !params.has(id) && - values.every(checkable) && + checkable && checkableInt32Allocations(fn, id) ) slots.checkedInt32Arrays.add(key); diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 4663e2736..cc04205bd 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1101,6 +1101,11 @@ export interface IrModule { * (frontend frozen-records.ts): their writes test the array's frozen bit * (SCR_ARR_FROZEN) and throw Node's TypeError when it is set. */ freezableArrays?: string[]; + /** Source ranges ("file\0start\0end") of untyped property writes in + * sloppy-mode code (frontend frozen-records.ts sloppyDynamicWrites): + * their dyn.keySet/keySetComputed calls ignore an assignment that strict + * mode refuses with a TypeError. */ + sloppyDynWrites?: string[]; /** The program freezes, seals or prevents extensions of untyped values * (frontend frozen-records.ts noteUntypedIntegrity). Class hierarchies * that can be held there carry an integrity word, which the runtime sets diff --git a/packages/compiler/src/ir/parameter-versions.ts b/packages/compiler/src/ir/parameter-versions.ts index 5156d4bff..ca2958585 100644 --- a/packages/compiler/src/ir/parameter-versions.ts +++ b/packages/compiler/src/ir/parameter-versions.ts @@ -122,7 +122,7 @@ function numericLoop(loop: IrStmt, functions: ReadonlyMap): const target = functions.get(e.callee); allowed = target !== undefined && simple(target); } else if (e.kind === "libCall") allowed = e.fn.startsWith("math."); - else if (e.kind === "arrIntrinsic") allowed = e.method === "length"; + else if (e.kind === "arrIntrinsic") allowed = e.method === "length" || e.method === "getNumber"; return allowed && everyExprChild(e, expr, stmt); }; const stmt = (s: IrStmt): boolean => diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index a1d3ea990..b394f5da7 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -1373,7 +1373,7 @@ function scanRuntimeFeatures( for (const t of nodes.types) type(t); } else if (keepGoing()) everyModuleNode(mod, { expr, stmt: keepGoing, type }); for (const { name, scan } of scans) if (scan.result()) features[name] = true; - // Inline-cache sites of dynamic-tier functions use the tier's shapes + // Inline-cache sites of dynamic-tier functions use the tier's caches // (scr_js_object.c, with the SCR_JS_TIER build of scr_json.c). if (!features.dynTier && mod.functions.some((fn) => fn.tier === "dynamic")) { const sites = new JsIcSites(mod); diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index ea3772db3..942565327 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -70,7 +70,7 @@ import { import { splitLlvmLibraryProgram } from "../backend/llvm/split.js"; import { prepareNativeExecutable, prepareNativeLibrary } from "./prepare.js"; import { buildSanitizedRuntime, sanitizerDriver, sanitizerFlags } from "./sanitizer.js"; -import { lldLinkerArgs } from "../backend/lld.js"; +import { inputsWithoutLld, lldLinkerArgs } from "../backend/lld.js"; import { openNativeExecutableCache } from "./executable-cache.js"; import { LinkerJobs } from "./linker-job.js"; import { nativeFileIdentity } from "./file-identity.js"; @@ -569,15 +569,16 @@ export class NativeCompiler { }); // Linux links with lld when the driver finds it. const lld = options.sanitize ? [] : lldLinkerArgs(target.platform, toolchain.linker); - const linkArgs = [ + const linkArgsWith = (withLld: boolean): string[] => [ ...(options.sanitize ? ["-fsanitize=address"] : toolchain.linkerArgs), - ...lld, + ...(withLld ? lld : []), ...plan.driverFlags, - ...plan.inputs, + ...(withLld ? plan.inputs : inputsWithoutLld(target.platform, plan.inputs)), ...systemLibraryLinkerArgs(target, plan.systemLibraries), "-o", stagedOutput, ]; + const linkArgs = linkArgsWith(lld.length !== 0); // Host Linux links with lld run the linker job the driver would. const jobs = LinkerJobs.open(this.cache, toolchain, lld, options.sanitize ?? false); const job = jobs?.link([...plan.inputs, stagedOutput], stage); @@ -593,10 +594,7 @@ export class NativeCompiler { // A failed link with lld runs again with the driver's default // linker, whose error (if any) is the one reported. if (lld.length === 0) throw error; - runNativeTool( - driver, - linkArgs.filter((arg) => !lld.includes(arg)), - ); + runNativeTool(driver, linkArgsWith(false)); } } timing("native-link"); diff --git a/packages/compiler/src/native/sanitizer.ts b/packages/compiler/src/native/sanitizer.ts index abc15d043..81667b28b 100644 --- a/packages/compiler/src/native/sanitizer.ts +++ b/packages/compiler/src/native/sanitizer.ts @@ -14,7 +14,7 @@ import { selectedRuntimeUnits, type RuntimePackMode, } from "../backend/runtime-pack-core.js"; -import { LRE_SOURCES, ZLIB_SOURCES } from "../backend/vendor-inputs.js"; +import { LRE_SOURCES, zlibInputs } from "../backend/vendor-inputs.js"; import { contentDigest, type NativeCache } from "./cache.js"; import type { NativeToolchain } from "./toolchain.js"; @@ -171,9 +171,10 @@ export function buildSanitizedRuntime( directory = quickjs; args = ["-std=c11", "-I", quickjs]; } else if (name === "zlib") { - sources = ZLIB_SOURCES; + const inputs = zlibInputs(toolchain.target.llvmTriple); + sources = inputs.sources; directory = zlib; - args = ["-std=c11", "-I", zlib]; + args = ["-std=c11", ...inputs.defines.map((define) => "-D" + define), "-I", zlib]; } else if (name === "mbedtls") { directory = join(mbedtls, "library"); sources = readdirSync(directory) diff --git a/packages/compiler/test/await-call-emission.test.ts b/packages/compiler/test/await-call-emission.test.ts index 5a967cc5f..cb575e97c 100644 --- a/packages/compiler/test/await-call-emission.test.ts +++ b/packages/compiler/test/await-call-emission.test.ts @@ -9,13 +9,19 @@ afterEach(async () => { await Promise.all(directories.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); }); -async function emit(source: string): Promise { +async function emit(source: string, optimization?: "dev"): Promise { const outDir = await mkdtemp(join(tmpdir(), "scriptc-await-call-")); directories.push(outDir); const entry = join(outDir, "main.ts"); await writeFile(entry, source); const outPath = join(outDir, "program.ll"); - const result = await compile(entry, { entry, outDir, outPath, outputKind: "llvm" }); + const result = await compile(entry, { + entry, + outDir, + outPath, + outputKind: "llvm", + ...(optimization !== undefined ? { optimization } : {}), + }); expect(result.ok, JSON.stringify(result)).toBe(true); return readFile(outPath, "utf8"); } @@ -28,8 +34,9 @@ function definition(llvm: string, symbol: string): string { return match![0]; } -// Consuming an async generator keeps a body on its fiber: the await-call -// entries serve fiber bodies (stackless bodies suspend at their awaits). +// The await-call entries serve fiber bodies (stackless bodies suspend at +// their awaits). Debug builds keep every async body on its fiber, so the +// sources below that test them build with optimization "dev". const tick = ` async function* tick(): AsyncGenerator { yield 0; @@ -58,7 +65,7 @@ void main(); `; test("awaiting a direct call to a body that never suspends skips the promise", async () => { - const llvm = await emit(source); + const llvm = await emit(source, "dev"); expect(definition(llvm, "sc_f_middle")).toContain("call double @sc_aw_leaf(double"); const entry = definition(llvm, "sc_aw_leaf"); expect(entry).toContain("call double @sc_f_leaf(double %a0)"); @@ -74,7 +81,7 @@ test("awaiting a direct call to a body that never suspends skips the promise", a }); test("awaiting a direct call to a suspending body spawns and awaits in one call", async () => { - const llvm = await emit(source); + const llvm = await emit(source, "dev"); const main = definition(llvm, "sc_f_main"); expect(main).toContain("call double @sc_aw_middle(double"); // A call whose promise is kept keeps the promise-returning entry. @@ -82,7 +89,7 @@ test("awaiting a direct call to a suspending body spawns and awaits in one call" expect(definition(llvm, "sc_aw_middle")).toContain( "call double @scr_async_spawn_await_f64(ptr @sc_tr_middle, ptr %ap)", ); - // middle consumes an async generator, so it keeps its fiber. + // The debug build keeps middle on its fiber. expect(definition(llvm, "sc_as_middle")).toContain("@scr_async_spawn("); }); @@ -119,7 +126,7 @@ void main(); `; test("returning a direct call adopts its promise one microtask turn later", async () => { - const llvm = await emit(adoptSource); + const llvm = await emit(adoptSource, "dev"); expect(definition(llvm, "sc_f_relay")).toContain("call double @sc_ad_leaf(double"); // A body that never suspends: the adoption's then-job turn, then the // settled promise's own turn, still without materializing the promise. @@ -141,7 +148,7 @@ test("returning a direct call adopts its promise one microtask turn later", asyn }); test("a returned promise is adopted after finally regions, outside catch handlers", async () => { - const llvm = await emit(adoptSource); + const llvm = await emit(adoptSource, "dev"); const guarded = definition(llvm, "sc_f_guarded"); // The call produces the promise; the finally body runs; only then the // adoption turn and the await. @@ -167,7 +174,7 @@ async function middle(x: number): Promise { const z = await leaf(y); return y * 2 + z; } -async function fibered(): Promise { +async function fibered(): Promise { let s = 0; for await (const t of tick()) s += t; return s + (await middle(1)); @@ -206,8 +213,9 @@ test("bodies that suspend only at their own awaits compile to stackless coroutin expect(middle).not.toContain("@scr_await_f64("); expect(definition(llvm, "sc_as_middle")).toContain("@scr_async_spawn_stackless("); expect(definition(llvm, "sc_f_main")).toContain("@scr_coro_await_prepare("); - // A body that keeps its fiber still awaits a stackless callee in one - // call: spawn, then the ordinary await of its promise. + // A body with a dynamic result keeps its fiber, and still awaits a + // stackless callee in one call: spawn, then the ordinary await of its + // promise. expect(definition(llvm, "sc_as_fibered")).toContain("@scr_async_spawn("); expect(definition(llvm, "sc_f_fibered")).toContain("call double @sc_aw_middle(double"); const entry = definition(llvm, "sc_aw_middle"); diff --git a/packages/compiler/test/bytes-element-emission.test.ts b/packages/compiler/test/bytes-element-emission.test.ts index c0a78a898..1998ff0af 100644 --- a/packages/compiler/test/bytes-element-emission.test.ts +++ b/packages/compiler/test/bytes-element-emission.test.ts @@ -404,7 +404,7 @@ test("large record clones stay outlined while small clones remain inlineable", ( expect(small).toContain("define internal ptr @sc_rclone_small(ptr %src) #0"); expect(small).not.toContain("attributes #2"); expect(large).toContain("define internal ptr @sc_rclone_large(ptr %src) #2"); - expect(large).toContain("attributes #2 = { noinline sanitize_address }"); + expect(large).toContain("attributes #2 = { noinline sanitize_address uwtable }"); }); test("Uint8ClampedArray stores clamp and round without a runtime call", () => { diff --git a/packages/compiler/test/counted-loop-emission.test.ts b/packages/compiler/test/counted-loop-emission.test.ts index 66d31e14b..914dc81d8 100644 --- a/packages/compiler/test/counted-loop-emission.test.ts +++ b/packages/compiler/test/counted-loop-emission.test.ts @@ -137,6 +137,59 @@ test("worker programs poll counted loops over typed arrays between blocks of ite } }); +test("ELF worker executables poll the loops of a nest on every iteration", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-nest-poll-")); + try { + const entry = join(dir, "nest.ts"); + await writeFile( + entry, + [ + 'import { isMainThread, Worker } from "node:worker_threads";', + "export function steps(limit: number): number {", + " let total = 0;", + " for (let start = 1; start < limit; start++) {", + " let x = start;", + " while (x !== 1) {", + " x = x % 2 === 0 ? x / 2 : 3 * x + 1;", + " total++;", + " }", + " }", + " return total;", + "}", + "export function mix(n: number): number {", + " let t = 0;", + " for (let i = 0; i < n; i++) t = (t * 31 + i) % 1000003;", + " return t;", + "}", + "console.log(steps(8), mix(8));", + "if (!isMainThread) new Worker(new URL(import.meta.url));", + ].join("\n"), + ); + const outPath = join(dir, "nest.ir.json"); + const result = await compile(entry, { outPath, outDir: dir, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + const body = (ll: string, name: string): string => + ll.match( + new RegExp( + `define internal [^\\n]+ @sc_b?f_${name}(?:\\.i32)?\\([^\\n]*\\n([\\s\\S]*?)\\n\\}`, + ), + )![1]!; + // Both loops of the nest test the thread's pending word on every + // iteration and keep no countdown; a loop on its own keeps one. + const elf = emitLlvmModule(mod, { targetTriple: "x86_64-unknown-linux-gnu" }); + const steps = body(elf, "steps"); + expect(steps).not.toContain("worker cancellation budget"); + expect(steps.match(/= load atomic i32, ptr @scr_exc_flag monotonic/g)?.length).toBe(2); + expect(body(elf, "mix")).toContain("worker cancellation budget"); + // Elsewhere the poll reads the process-wide alert word: the nest counts. + const darwin = emitLlvmModule(mod, { targetTriple: "arm64-apple-macosx14.0.0" }); + expect(body(darwin, "steps").match(/worker cancellation budget/g)?.length).toBe(2); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test("worker programs poll no loop whose proven ranges bound its runs inside another loop", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-bounded-loop-polls-")); try { diff --git a/packages/compiler/test/cpu-dispatch-emission.test.ts b/packages/compiler/test/cpu-dispatch-emission.test.ts index 170c28ede..43d80b335 100644 --- a/packages/compiler/test/cpu-dispatch-emission.test.ts +++ b/packages/compiler/test/cpu-dispatch-emission.test.ts @@ -119,10 +119,10 @@ test("a dispatching x86-64 build gives each program function a baseline and an x for (const [name] of coroutines) expect(defs.has(`${name}.v3`), name).toBe(false); // Functions used as values are referenced by their dispatching name. expect(ll).not.toMatch(/ptr @[\w.$-]+\.(v3|base)(?!\()/); - expect(ll).toMatch(/^attributes #3 = \{ sanitize_address "target-cpu"="x86-64-v3" \}$/m); + expect(ll).toMatch(/^attributes #3 = \{ sanitize_address uwtable "target-cpu"="x86-64-v3" \}$/m); // Unsanitized builds leave sanitize_address out of the copies' groups too. expect(emit({ ...x86, cpuDispatch: true, sanitizeAttributes: false })).toMatch( - /^attributes #3 = \{ "target-cpu"="x86-64-v3" \}$/m, + /^attributes #3 = \{ uwtable "target-cpu"="x86-64-v3" \}$/m, ); // main sets the level before anything else runs. expect(defs.get("main")).toMatch( diff --git a/packages/compiler/test/forin-keys.test.ts b/packages/compiler/test/forin-keys.test.ts index b35ff2154..4a8d24c7e 100644 --- a/packages/compiler/test/forin-keys.test.ts +++ b/packages/compiler/test/forin-keys.test.ts @@ -36,7 +36,7 @@ console.log(count(row), count(JSON.parse('{"b":1}'))); expect(llvm).not.toContain("@scr_dyn_for_in_keys("); // Fully required record shapes are marked fixed for the stable walk. expect(llvm).toMatch( - /_rops = internal global \{ i64, ptr, ptr, ptr, i8, ptr, ptr \} \{[^}]*i8 1, ptr null, ptr null \}/, + /_rops = internal global \{ i64, ptr, ptr, ptr, i8, ptr, ptr, ptr \} \{[^}]*i8 1, ptr null, ptr null, ptr null \}/, ); }); diff --git a/packages/compiler/test/generator-emission.test.ts b/packages/compiler/test/generator-emission.test.ts index 47e5f16c5..2442c5e09 100644 --- a/packages/compiler/test/generator-emission.test.ts +++ b/packages/compiler/test/generator-emission.test.ts @@ -9,13 +9,19 @@ afterEach(async () => { await Promise.all(directories.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); }); -async function emit(source: string): Promise { +async function emit(source: string, optimization?: "dev"): Promise { const outDir = await mkdtemp(join(tmpdir(), "scriptc-generator-")); directories.push(outDir); const entry = join(outDir, "main.ts"); await writeFile(entry, source); const outPath = join(outDir, "program.ll"); - const result = await compile(entry, { entry, outDir, outPath, outputKind: "llvm" }); + const result = await compile(entry, { + entry, + outDir, + outPath, + outputKind: "llvm", + ...(optimization !== undefined ? { optimization } : {}), + }); expect(result.ok, JSON.stringify(result)).toBe(true); return readFile(outPath, "utf8"); } @@ -50,7 +56,7 @@ console.log(sum); expect(definition(llvm, "sc_f_range")).not.toContain("@scr_coro_yield_f64("); expect(llvm).not.toContain("@scr_gen_take_out_f64("); expect(llvm).toContain("%ScrGenHead = type { i64, ptr, ptr, i32, %ScrGenSlot }"); - expect(llvm).toContain("attributes #1 = { presplitcoroutine }"); + expect(llvm).toContain("attributes #1 = { uwtable presplitcoroutine }"); expect(definition(llvm, "sc_gs_range")).toContain("call ptr @scr_gen_new_stackless("); expect(llvm).not.toContain("@scr_gen_yield_f64("); }); @@ -117,7 +123,7 @@ console.log(sum, r.value); expect(llvm).toMatch(/define internal ptr @sc_gres_\d+\(ptr %g\)[^\n]*\(dyn channel\)/); }); -test("async generators keep their fibers", async () => { +test("async generators that suspend only at their own awaits and yields compile to stackless coroutines", async () => { const llvm = await emit(` async function* ticks(n: number): AsyncGenerator { for (let i = 0; i < n; i++) { @@ -130,6 +136,36 @@ async function main(): Promise { } void main(); `); + const body = definition(llvm, "sc_f_ticks"); + expect(body).toMatch(/^define internal void @sc_f_ticks(?:\.i32)?\(double [^)]*\) #1 /); + // The await suspends the coroutine (one turn for a non-promise operand), + // and the yield stores OUT before its own suspension. + expect(body).toContain("call zeroext i1 @scr_coro_hop_prepare(ptr"); + expect(body).toContain("call void @scr_coro_yield_f64(ptr"); + expect(body).toContain("call void @scr_gen_coro_finish(ptr"); + expect(body).not.toContain("@scr_await_hop("); + expect(body).not.toContain("@scr_gen_yield_f64("); + expect(definition(llvm, "sc_gs_ticks")).toContain("call ptr @scr_async_gen_new_stackless("); + // Consuming the generator does not park its consumer. + expect(definition(llvm, "sc_as_main")).toContain("@scr_async_spawn_stackless("); +}); + +test("debug builds keep async generators on their fibers", async () => { + const llvm = await emit( + ` +async function* ticks(n: number): AsyncGenerator { + for (let i = 0; i < n; i++) { + await null; + yield i; + } +} +async function main(): Promise { + for await (const t of ticks(3)) console.log(t); +} +void main(); +`, + "dev", + ); expect(definition(llvm, "sc_gs_ticks")).toContain("call ptr @scr_async_gen_new("); expect(llvm).not.toContain("presplitcoroutine"); }); diff --git a/packages/compiler/test/js-ic-emission.test.ts b/packages/compiler/test/js-ic-emission.test.ts index b350c743d..aa4d83959 100644 --- a/packages/compiler/test/js-ic-emission.test.ts +++ b/packages/compiler/test/js-ic-emission.test.ts @@ -50,6 +50,10 @@ console.log(read(o), at(dyn([5, 6]), 1), call(o), o.n); ); expect(body(llvm, "read")).toContain("@scr_js_get_probe("); expect(body(llvm, "write")).toContain("@scr_js_store_probe("); + // Reads and writes test the first two entries inline: the second + // starts 40 bytes into the record. + for (const site of ["read", "write"]) + expect(body(llvm, site)).toMatch(/getelementptr inbounds i8, ptr @sc_js_ic_\d+, i64 40$/m); expect(body(llvm, "at")).toContain("@scr_js_keyed_probe("); expect(body(llvm, "call")).toContain("@scr_js_load_probe("); // A compound assignment stores its number result from the jsval: an @@ -139,6 +143,50 @@ console.log(read(o), read(p), read(t), at(dyn([5, 6]), 1), at(p, "x"), at(t, "ta } }); +test("property reads answer class instances' declared fields inline", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-capsule-")); + try { + const entry = join(dir, "main.js"); + const outPath = join(dir, "main.ir.json"); + await writeFile( + entry, + ` +class Point { + constructor(x) { this.x = x; } +} +class Tagged extends Point { + constructor(x, tag) { super(x); this.tag = tag; } +} +class Gauge { + get x() { return 9; } +} +function dyn(v) { return v; } +function read(o) { return o.x; } +function tag(o) { return o.tag; } +var p = dyn(new Point(4)); +var t = dyn(new Tagged(6, "t")); +console.log(read(p), read(t), read(dyn(new Gauge())), read({ x: 1 }), tag(t)); +`, + ); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + expect(validateModule(mod)).toEqual([]); + const llvm = emitLlvmModule(mod); + const read = body(llvm, "read"); + // Point and Tagged instances load the field after one key compare + // each; Gauge's getter, and every other receiver, keeps the probe and + // the helper. + expect(read.match(/^icc\.m\d+:/gm)).toHaveLength(2); + expect(read).toContain("@scr_js_get_probe("); + expect(read).toMatch(/@sc_b?f__x25_dyn_class_property_\d+\(ptr %t\d+\) noinline cold/); + expect(body(llvm, "tag").match(/^icc\.m\d+:/gm)).toHaveLength(1); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test("typed programs have no cache sites and keep the ordinary runtime", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-typed-")); try { diff --git a/packages/compiler/test/stable-names.test.ts b/packages/compiler/test/stable-names.test.ts index c82db039d..13ed000df 100644 --- a/packages/compiler/test/stable-names.test.ts +++ b/packages/compiler/test/stable-names.test.ts @@ -117,3 +117,44 @@ test("names do not depend on the directory the program sits in", async () => { const [second] = await lower(recursive); expect(names(second!)).toEqual(names(first!)); }); + +/** Lowers a program of several files; `main.ts` is the entry. */ +async function lowerFiles(files: Record): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-stable-names-")); + try { + for (const [name, source] of Object.entries(files)) await writeFile(join(dir, name), source); + const outPath = join(dir, "main.ir.json"); + const result = await compile(join(dir, "main.ts"), { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + return deserializeModule(await readFile(outPath, "utf8")); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +test("a module's names do not depend on the modules loaded before it", async () => { + const shapes = ` +export class Box { + constructor(private value: number) {} + scaled(by: number): number { return this.value * by; } +} +export function boxes(count: number): number[] { + return Array.from({ length: count }, (_, i) => new Box(i).scaled(2)); +} +`; + const before = await lowerFiles({ + "shapes.ts": shapes, + "main.ts": `import { boxes } from "./shapes";\nconsole.log(boxes(3).join(","));\n`, + }); + const after = await lowerFiles({ + "offsets.ts": `export function offset(x: number): number { return x + 1; }\n`, + "shapes.ts": shapes, + "main.ts": `import { offset } from "./offsets";\nimport { boxes } from "./shapes";\nconsole.log(offset(1), boxes(3).join(","));\n`, + }); + // The class, its methods, the function and the closure it lifts. + const ofShapes = (mod: IrModule) => + names(mod).filter((name) => /Box|boxes/.test(name) || name.startsWith("%fn.")); + expect(ofShapes(before).length).toBeGreaterThanOrEqual(4); + expect(ofShapes(after)).toEqual(ofShapes(before)); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7393-non-null-element-operands.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7393-non-null-element-operands.ts.json new file mode 100644 index 000000000..e50ca92f4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7393-non-null-element-operands.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7393-non-null-element-operands.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7395-non-null-element-dereference.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7395-non-null-element-dereference.ts.json new file mode 100644 index 000000000..17551e626 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7395-non-null-element-dereference.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7395-non-null-element-dereference.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8370-stack-overflow-cycles/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8370-stack-overflow-cycles/main.ts.json new file mode 100644 index 000000000..7e5125dde --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8370-stack-overflow-cycles/main.ts.json @@ -0,0 +1,7 @@ +{ + "order": [ + "/tests/corpus/8370-stack-overflow-cycles/shapes.ts", + "/tests/corpus/8370-stack-overflow-cycles/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8380-untyped-generator-loops.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8380-untyped-generator-loops.mjs.json new file mode 100644 index 000000000..91ef13397 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8380-untyped-generator-loops.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8380-untyped-generator-loops.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8381-untyped-generator-stages.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8381-untyped-generator-stages.mjs.json new file mode 100644 index 000000000..f272e7d88 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8381-untyped-generator-stages.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8381-untyped-generator-stages.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8382-stackless-async-generator-steps.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8382-stackless-async-generator-steps.mjs.json new file mode 100644 index 000000000..aa47502ae --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8382-stackless-async-generator-steps.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8382-stackless-async-generator-steps.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8383-stackless-async-generator-steps.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8383-stackless-async-generator-steps.ts.json new file mode 100644 index 000000000..2c9f5537a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8383-stackless-async-generator-steps.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8383-stackless-async-generator-steps.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8405-zlib-compressed-bytes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8405-zlib-compressed-bytes.ts.json new file mode 100644 index 000000000..509b30d34 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8405-zlib-compressed-bytes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8405-zlib-compressed-bytes.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8406-zlib-inflate-chunks.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8406-zlib-inflate-chunks.cjs.json new file mode 100644 index 000000000..772d91511 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8406-zlib-inflate-chunks.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8406-zlib-inflate-chunks.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8407-string-index-of-from-ascii.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8407-string-index-of-from-ascii.ts.json new file mode 100644 index 000000000..15651f491 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8407-string-index-of-from-ascii.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8407-string-index-of-from-ascii.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8410-dynamic-class-instance-reads.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8410-dynamic-class-instance-reads.js.json new file mode 100644 index 000000000..cec5ee64b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8410-dynamic-class-instance-reads.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8410-dynamic-class-instance-reads.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8411-dynamic-base-keyed-instances/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8411-dynamic-base-keyed-instances/main.ts.json new file mode 100644 index 000000000..5d15c0479 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8411-dynamic-base-keyed-instances/main.ts.json @@ -0,0 +1,7 @@ +{ + "order": [ + "/tests/corpus/8411-dynamic-base-keyed-instances/reader.cjs", + "/tests/corpus/8411-dynamic-base-keyed-instances/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-array-subclass.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-array-subclass.ts.json new file mode 100644 index 000000000..84bff57e0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-array-subclass.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-array-subclass.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-arrays.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-arrays.ts.json new file mode 100644 index 000000000..eafbe937c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-arrays.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-arrays.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-cap.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-cap.ts.json new file mode 100644 index 000000000..70593bc13 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-cap.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-cap.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-escapes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-escapes.ts.json new file mode 100644 index 000000000..402b73088 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-escapes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-escapes.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-mixed.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-mixed.ts.json new file mode 100644 index 000000000..17269a5b1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-mixed.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-mixed.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-outside.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-outside.ts.json new file mode 100644 index 000000000..59a1a597c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-outside.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-outside.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-region-throws.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-region-throws.ts.json new file mode 100644 index 000000000..392abe549 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-region-throws.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-region-throws.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-regions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-regions.ts.json new file mode 100644 index 000000000..7f2425ffc --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-regions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-regions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-thrown-member.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-thrown-member.ts.json new file mode 100644 index 000000000..8657adc5c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-thrown-member.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-thrown-member.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-throws.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-throws.ts.json new file mode 100644 index 000000000..c21f13a02 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-throws.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-throws.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-trees.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-trees.ts.json new file mode 100644 index 000000000..9d9b75df2 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-trees.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-trees.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-workers.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-workers.ts.json new file mode 100644 index 000000000..97915bb85 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/arena-workers.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/arena-workers.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-subclass-element-cycles.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-subclass-element-cycles.ts.json new file mode 100644 index 000000000..136c92da1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-subclass-element-cycles.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/array-subclass-element-cycles.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-subclass-element-instanceof.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-subclass-element-instanceof.ts.json new file mode 100644 index 000000000..0a542048a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-subclass-element-instanceof.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/array-subclass-element-instanceof.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/discarded-logical-void-conditionals.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/discarded-logical-void-conditionals.mjs.json new file mode 100644 index 000000000..837cbad85 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/discarded-logical-void-conditionals.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/discarded-logical-void-conditionals.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/discarded-logical-void-conditionals.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/discarded-logical-void-conditionals.ts.json new file mode 100644 index 000000000..79c4868ee --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/discarded-logical-void-conditionals.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/discarded-logical-void-conditionals.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/frozen-record-untyped-writes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/frozen-record-untyped-writes.ts.json new file mode 100644 index 000000000..800f25e9e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/frozen-record-untyped-writes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/frozen-record-untyped-writes.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/frozen-untyped-writes-sloppy.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/frozen-untyped-writes-sloppy.cjs.json new file mode 100644 index 000000000..e8af9b305 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/frozen-untyped-writes-sloppy.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/frozen-untyped-writes-sloppy.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/integer-clone-int32-slots.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/integer-clone-int32-slots.ts.json new file mode 100644 index 000000000..e26a5a727 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/integer-clone-int32-slots.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/integer-clone-int32-slots.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/native-worker-terminate-short-inner-loops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/native-worker-terminate-short-inner-loops.ts.json new file mode 100644 index 000000000..8332ee20d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/native-worker-terminate-short-inner-loops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/native-worker-terminate-short-inner-loops.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-dynamic-writes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-dynamic-writes.ts.json new file mode 100644 index 000000000..8de4b7d03 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-dynamic-writes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/record-dynamic-writes.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-prefix-identity.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-prefix-identity.ts.json new file mode 100644 index 000000000..273bff2c1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-prefix-identity.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/record-prefix-identity.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/typed-array-nested-loop-versions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/typed-array-nested-loop-versions.ts.json new file mode 100644 index 000000000..66c967b58 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/typed-array-nested-loop-versions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/typed-array-nested-loop-versions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/untyped-nullish-scalar-defaults.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/untyped-nullish-scalar-defaults.mjs.json new file mode 100644 index 000000000..2bce0e918 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/untyped-nullish-scalar-defaults.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/untyped-nullish-scalar-defaults.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/untyped-nullish-scalar.test.ts b/packages/compiler/test/untyped-nullish-scalar.test.ts new file mode 100644 index 000000000..c9e496b88 --- /dev/null +++ b/packages/compiler/test/untyped-nullish-scalar.test.ts @@ -0,0 +1,46 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule, validateModule } from "../src/index.js"; + +test("`??` in JavaScript tests a scalar optional left on the typed path when the default has its type", async () => { + const dir = mkdtempSync(join(tmpdir(), "scriptc-untyped-nullish-")); + try { + const entry = join(dir, "main.mjs"); + writeFileSync( + entry, + [ + "const counts = new Map();", + 'for (const w of ["a", "b", "a"]) counts.set(w, (counts.get(w) ?? 0) + 1);', + 'console.log(counts.get("a") ?? 0, counts.get("z") ?? "none");', + ].join("\n"), + ); + const output = join(dir, "main.ir.json"); + const lowered = await compile(entry, { outputKind: "ir", outDir: dir, outPath: output }); + if (!lowered.ok) throw new Error(JSON.stringify(lowered.diagnostics)); + const module = deserializeModule(readFileSync(output, "utf8")); + expect(validateModule(module)).toEqual([]); + const nullish: { type: string; left: string }[] = []; + const visit = (node: unknown, parent: string | null): void => { + if (node === null || typeof node !== "object") return; + if (Array.isArray(node)) return node.forEach((child) => visit(child, parent)); + const n = node as { kind?: string; type?: { kind: string }; left?: { kind: string } }; + if (n.kind === "nullish") + nullish.push({ type: `${parent}>${n.type!.kind}`, left: n.left!.kind }); + for (const [key, value] of Object.entries(node)) + if (key !== "loc") visit(value, n.kind ?? parent); + }; + visit(module.functions, null); + // The loop's `?? 0`, whose result is an untyped value, reads the Map's + // number on the typed path and converts only the result. (The first + // console.log argument is a number already.) `?? "none"` takes another + // type, so its left converts first. + expect( + nullish.filter((n) => n.type === "dynFrom>f64" && n.left === "mapIntrinsic"), + ).toHaveLength(1); + expect(nullish.filter((n) => n.type.endsWith(">dyn") && n.left === "dynFrom")).toHaveLength(1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index dc6c24a47..95fe91ab3 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -16,6 +16,7 @@ import { import { createDeterministicArchive } from "./archive.mjs"; import { installRuntimePack, withBuildLock } from "./build-state.mjs"; import { shareIdenticalArtifacts } from "../../runtime-pack-common/scripts/share-artifacts.mjs"; +import { zlibInputs } from "../../runtime-pack-common/scripts/zlib-inputs.mjs"; const run = promisify(execFile); const packageRoot = dirname(dirname(fileURLToPath(import.meta.url))); @@ -76,19 +77,7 @@ async function build() { const brotli = join(vendorRoot, "brotli", "c"); const QJS_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"]; const LRE_SOURCES = ["libregexp.c", "libunicode.c"]; - const ZLIB_SOURCES = [ - "adler32.c", - "compress.c", - "crc32.c", - "deflate.c", - "infback.c", - "inffast.c", - "inflate.c", - "inftrees.c", - "trees.c", - "uncompr.c", - "zutil.c", - ]; + const ZLIB_BUILD = zlibInputs(RUNTIME_PACK_MATRIX.target.llvm_triple); // Speed-flavor executable units are emitted through the host LLVM helper // so their object and import bitcode come from one promoted module. The @@ -309,11 +298,12 @@ async function build() { "-I", quickjs, ]), - await archive("zlib", ZLIB_SOURCES, zlib, [ + await archive("zlib", ZLIB_BUILD.sources, zlib, [ "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=c11", - "-Os", + "-O3", + ...ZLIB_BUILD.defines.map((define) => `-D${define}`), ...codeAlignment, "-I", zlib, @@ -356,6 +346,7 @@ async function build() { ], [join(vendorRoot, "unicode", "LICENSE"), "artifacts/licenses/unicode.txt", "Unicode-3.0"], [join(zlib, "LICENSE"), "artifacts/licenses/zlib.txt", "Zlib"], + [join(zlib, "LICENSE.chromium"), "artifacts/licenses/zlib-chromium.txt", "BSD-3-Clause"], [join(vendorRoot, "brotli", "LICENSE"), "artifacts/licenses/brotli.txt", "MIT"], [join(mbedtls, "LICENSE"), "artifacts/licenses/mbedtls.txt", "Apache-2.0"], ]; diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 86f947010..a148543d9 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -47,6 +47,7 @@ export const BASE_RUNTIME_SOURCES = [ "scr_cycle.c", "scr_publish.c", "scr_region.c", + "scr_arena.c", "scr_itable.c", "scr_slots.c", "scr_alloc.c", @@ -190,9 +191,9 @@ export const OPTIONAL_RUNTIME_SOURCES = [ export function variantsFor(source) { const variants = [{ id: "default", when: {}, defines: [] }]; - // Programs with dynamic-tier code keep the tier's hidden classes current - // in checked objects (two-tier design S4); every other program links the - // ordinary object, byte for byte. + // Programs with dynamic-tier code link the build that exports the + // primitives the tier's inline caches use (two-tier design S4); every + // other program links the ordinary object, byte for byte. if (source === "scr_json.c") { variants.push({ id: "dyn-tier", when: { dynTier: true }, defines: ["SCR_JS_TIER"] }); } diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 2e4efd10e..c89390a70 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -17,6 +17,7 @@ import { buildImportBitcode, buildRuntimeUnit, resolveRuntimeUnitHelper } from " import { installRuntimePack, withBuildLock } from "./build-state.mjs"; import { shareIdenticalArtifacts } from "./share-artifacts.mjs"; import { parallelMap } from "./parallel-map.mjs"; +import { zlibInputs } from "./zlib-inputs.mjs"; const run = promisify(execFile); const packageRoot = process.env.SCRIPTC_RUNTIME_PACK_ROOT; @@ -89,19 +90,7 @@ async function build() { const brotli = join(vendorRoot, "brotli", "c"); const qjsSources = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"]; const lreSources = ["libregexp.c", "libunicode.c"]; - const zlibSources = [ - "adler32.c", - "compress.c", - "crc32.c", - "deflate.c", - "infback.c", - "inffast.c", - "inflate.c", - "inftrees.c", - "trees.c", - "uncompr.c", - "zutil.c", - ]; + const zlibBuild = zlibInputs(config.target.llvm_triple); const sha256 = async (path) => createHash("sha256") .update(await readFile(path)) @@ -400,10 +389,11 @@ async function build() { : []), ...(requestedArchives.has("zlib") ? [ - await archive("zlib", zlibSources, zlib, [ + await archive("zlib", zlibBuild.sources, zlib, [ ...vendorTarget, "-std=c11", - "-Os", + "-O3", + ...zlibBuild.defines.map((define) => `-D${define}`), "-I", zlib, ]), @@ -454,6 +444,7 @@ async function build() { "BSD-3-Clause", ], [join(zlib, "LICENSE"), "artifacts/licenses/zlib.txt", "Zlib"], + [join(zlib, "LICENSE.chromium"), "artifacts/licenses/zlib-chromium.txt", "BSD-3-Clause"], [join(vendorRoot, "brotli", "LICENSE"), "artifacts/licenses/brotli.txt", "MIT"], [join(mbedtls, "LICENSE"), "artifacts/licenses/mbedtls.txt", "Apache-2.0"], ]; diff --git a/packages/runtime-pack-common/scripts/zlib-inputs.mjs b/packages/runtime-pack-common/scripts/zlib-inputs.mjs new file mode 100644 index 000000000..efb433ff8 --- /dev/null +++ b/packages/runtime-pack-common/scripts/zlib-inputs.mjs @@ -0,0 +1,83 @@ +// The translation units and defines of the vendored zlib (packages/runtime/vendor/zlib, +// the copy Node v24.22.0 bundles in deps/zlib), chosen per target the way Node's +// deps/zlib/zlib.gyp chooses them. The SIMD units carry their own target attributes +// and check the CPU at run time, so every unit builds with the same flags. Compressed +// output does not depend on these choices. packages/compiler/src/backend/vendor-inputs.ts +// keeps the same table for cross builds; a unit test checks that the two agree. + +const COMMON_SOURCES = [ + "adler32.c", + "compress.c", + "cpu_features.c", + "crc32.c", + "deflate.c", + "infback.c", + "inffast.c", + "inftrees.c", + "trees.c", + "uncompr.c", + "zutil.c", +]; + +/** + * @param {string} triple an LLVM or Zig target triple, such as + * "x86_64-unknown-linux-gnu", "arm64-apple-ios15.0" or "aarch64-linux-musl". + * @returns {{ sources: string[], defines: string[] }} + */ +export function zlibInputs(triple) { + const [cpu = "", ...rest] = triple.split("-"); + const system = rest.join("-"); + const architecture = + cpu === "x86_64" || cpu === "amd64" + ? "x64" + : cpu === "aarch64" || cpu === "arm64" + ? "arm64" + : cpu; + const ios = system.includes("ios"); + const windows = + system.includes("windows") || system.includes("win32") || system.includes("mingw"); + // Android reads the CPU features through getauxval, as Linux does: Node's + // ARMV8_OS_ANDROID path needs the NDK's cpu-features library. + const linux = system.includes("linux") || system.includes("android"); + const mac = !ios && (system.includes("darwin") || system.includes("macos")); + const sources = [...COMMON_SOURCES]; + const defines = windows ? [] : ["HAVE_HIDDEN"]; + if (architecture === "x64" && !ios) { + sources.push( + "adler32_simd.c", + "contrib/optimizations/inffast_chunk.c", + "contrib/optimizations/inflate.c", + ); + defines.push( + "ADLER32_SIMD_SSSE3", + windows ? "X86_WINDOWS" : "X86_NOT_WINDOWS", + "INFLATE_CHUNK_SIMD_SSE2", + "INFLATE_CHUNK_READ_64LE", + "DEFLATE_SLIDE_HASH_SSE2", + ); + } else if (architecture === "arm64") { + sources.push("contrib/optimizations/inffast_chunk.c", "contrib/optimizations/inflate.c"); + defines.push( + "INFLATE_CHUNK_SIMD_NEON", + "INFLATE_CHUNK_READ_64LE", + "__ARM_NEON__", + "DEFLATE_SLIDE_HASH_NEON", + ); + if (windows) { + defines.push("ARMV8_OS_WINDOWS"); + } else if (linux || mac) { + // Node builds the Adler-32 and CRC-32 units for every arm64 target but + // iOS and Windows. + sources.push("adler32_simd.c", "crc32_simd.c"); + defines.push( + "ADLER32_SIMD_NEON", + "CRC32_ARMV8_CRC32", + linux ? "ARMV8_OS_LINUX" : "ARMV8_OS_MACOS", + ); + } + } else { + sources.push("inflate.c"); + defines.push("CPU_NO_SIMD"); + } + return { sources, defines }; +} diff --git a/packages/runtime/src/scr_arena.c b/packages/runtime/src/scr_arena.c new file mode 100644 index 000000000..e04a6ed35 --- /dev/null +++ b/packages/runtime/src/scr_arena.c @@ -0,0 +1,263 @@ +/* Group arenas: see scr_arena.h. */ +#include "scr_runtime.h" + +#include +#include +#include + +/* The first chunk lives inside the arena, so a small group (one result + * record, a short builder) touches no other memory. */ +#define SCR_ARENA_INLINE 512 +#define SCR_ARENA_CHUNK_MIN ((size_t)4096) +#define SCR_ARENA_CHUNK_MAX ((size_t)1 << 20) +/* Bytes an arena that is not closed takes before its call allocates on the + * heap again: a group whose objects die before it is dropped holds at most + * this much dead memory. A closed arena has no cap: once on the heap, its + * later objects could come to be held by earlier ones, which are not on + * its dispose list. Neither do objects with a collector header: on the + * heap, they could form cycles that outlive the drop (llvm/arenas.ts). */ +#define SCR_ARENA_CAP ((size_t)32 << 20) +/* What a thread keeps for reuse: arenas, chunk bytes, and dispose list + * entries per pooled arena. */ +#define SCR_ARENA_POOL 64 +#define SCR_ARENA_POOL_BYTES ((size_t)8 << 20) +#define SCR_ARENA_POOL_DISPOSALS ((size_t)4096) + +/* A chunk's objects run from scr_arena_chunk_start to its end. */ +typedef struct ScrArenaChunk { + struct ScrArenaChunk *next; + size_t size; /* bytes, this header included */ +} ScrArenaChunk; + +/* The leading fields are an ABI: emitted arena-aware constructors bump + * `bump` up to `limit`, test `closed`, and append to the dispose list at + * `disposals` up to `disposals_end` inline (llvm/arenas.ts). Without the + * small-object allocator `bump` and `limit` stay NULL, so every object + * takes scr_arena_obj and is a block of its own. */ +struct ScrArena { + char *bump, *limit; + ScrArenaDisposal *disposals, *disposals_end; + uint32_t closed; + uint32_t open; /* still the allocation target, or under one that is */ + ScrArena *outer; /* the allocation target before it opened */ + ScrArena *next; /* its scope's list, or the pool */ + ScrArenaDisposal *disposal_base; + ScrArenaChunk *chunks; /* newest first */ + size_t bytes; +#if !SCR_SMALL_ALLOC + char **blocks; + size_t nblocks, cap; +#endif + _Alignas(16) char inline_chunk[SCR_ARENA_INLINE]; +}; +/* llvm/arenas.ts addresses these fields in pointer-sized units. */ +#define SCR_ARENA_PTR sizeof(void *) +_Static_assert(offsetof(ScrArena, bump) == 0 && offsetof(ScrArena, limit) == SCR_ARENA_PTR, + "llvm/arenas.ts bumps the arena inline"); +_Static_assert(offsetof(ScrArena, disposals) == 2 * SCR_ARENA_PTR && + offsetof(ScrArena, disposals_end) == 3 * SCR_ARENA_PTR && + offsetof(ScrArena, closed) == 4 * SCR_ARENA_PTR, + "llvm/arenas.ts appends to the dispose list inline"); +_Static_assert(sizeof(ScrArenaDisposal) == 2 * SCR_ARENA_PTR, + "llvm/arenas.ts stores dispose entries of two pointers"); +_Static_assert(offsetof(ScrArenaHead, disposals) == offsetof(ScrArena, disposals) && + offsetof(ScrArenaHead, closed) == offsetof(ScrArena, closed), + "ScrArenaHead is ScrArena's prefix"); + +SCR_TL ScrArena *scr_arena_cur = NULL; +static SCR_TL ScrArena *scr_arena_pool; +static SCR_TL unsigned scr_arena_pooled; +static SCR_TL ScrArenaChunk *scr_arena_chunk_pool; +static SCR_TL size_t scr_arena_chunk_bytes; + +void *scr_arena_current(void) { return scr_arena_cur; } + +void *scr_arena_suspend(void) { + ScrArena *a = scr_arena_cur; + scr_arena_cur = NULL; + return a; +} + +void scr_arena_resume(void *arena) { scr_arena_cur = (ScrArena *)arena; } + +void *scr_arena_open(void **list, int32_t closed) { + ScrArena *a = scr_arena_pool; + if (a) { + scr_arena_pool = a->next; + scr_arena_pooled--; + } else { + a = malloc(sizeof *a); + if (!a) scr_trap("scriptc: out of memory\n"); + a->disposal_base = NULL; + a->disposals_end = NULL; +#if !SCR_SMALL_ALLOC + a->blocks = NULL; + a->cap = 0; +#endif + } + a->closed = closed != 0; + a->chunks = NULL; + a->bytes = 0; + a->disposals = a->disposal_base; +#if SCR_SMALL_ALLOC + a->bump = a->inline_chunk; + a->limit = a->inline_chunk + SCR_ARENA_INLINE; +#else + a->bump = a->limit = NULL; + a->nblocks = 0; +#endif + a->open = 1; + a->outer = scr_arena_cur; + scr_arena_cur = a; + a->next = (ScrArena *)*list; + *list = a; + return a; +} + +/* Makes `a`'s outer target current again, if `a` is still the target or + * under it: an exception can leave the arenas a call opened unclosed until + * their scopes drop them. */ +static void scr_arena_unlink(ScrArena *a) { + if (!a->open) return; + a->open = 0; + for (ScrArena *t = scr_arena_cur; t; t = t->outer) + if (t == a) { + scr_arena_cur = a->outer; + return; + } +} + +void scr_arena_close(void *arena) { + if (arena) scr_arena_unlink((ScrArena *)arena); +} + +void scr_arena_disposal(void *arena, void *obj, ScrArenaDispose dispose) { + ScrArena *a = (ScrArena *)arena; + if (a->disposals == a->disposals_end) { + size_t n = (size_t)(a->disposals - a->disposal_base); + size_t cap = n ? n * 2 : 64; + ScrArenaDisposal *grown = realloc(a->disposal_base, cap * sizeof *grown); + if (!grown) scr_trap("scriptc: out of memory\n"); + a->disposal_base = grown; + a->disposals = grown + n; + a->disposals_end = grown + cap; + } + a->disposals->obj = obj; + a->disposals->dispose = dispose; + a->disposals++; +} + +#if SCR_SMALL_ALLOC +static ScrArenaChunk *scr_arena_chunk(size_t want) { + for (ScrArenaChunk **p = &scr_arena_chunk_pool; *p; p = &(*p)->next) + if ((*p)->size >= want) { + ScrArenaChunk *c = *p; + *p = c->next; + scr_arena_chunk_bytes -= c->size; + return c; + } + ScrArenaChunk *c = malloc(want); + if (!c) scr_trap("scriptc: out of memory\n"); + c->size = want; + return c; +} + +static char *scr_arena_chunk_start(ScrArenaChunk *c) { + return (char *)(((uintptr_t)(c + 1) + 15) & ~(uintptr_t)15); +} + +/* A new current chunk with room for `n` bytes, or false past the cap + * when `capped`. */ +static bool scr_arena_refill(ScrArena *a, size_t n, bool capped) { + size_t want = a->chunks ? a->chunks->size * 2 : SCR_ARENA_CHUNK_MIN; + if (want > SCR_ARENA_CHUNK_MAX) want = SCR_ARENA_CHUNK_MAX; + size_t need = n + sizeof(ScrArenaChunk) + 15; + if (want < need) want = need; + if (capped && a->bytes + want > SCR_ARENA_CAP) return false; + ScrArenaChunk *c = scr_arena_chunk(want); + a->bytes += c->size; + c->next = a->chunks; + a->chunks = c; + a->bump = scr_arena_chunk_start(c); + a->limit = (char *)c + c->size; + return true; +} +#endif + +void *scr_arena_obj(void *arena, size_t size, void *cyc) { + ScrArena *a = (ScrArena *)arena; + size_t hdr = cyc ? sizeof(ScrCycHdr) : 0; + bool capped = !a->closed && !cyc; + if (capped && size > SCR_ARENA_CAP) return NULL; + size_t n = (hdr + size + 7) & ~(size_t)7; +#if SCR_SMALL_ALLOC + if ((size_t)(a->limit - a->bump) < n && !scr_arena_refill(a, n, capped)) return NULL; + char *block = a->bump; + a->bump += n; + memset(block, 0, n); +#else + if (capped && a->bytes + n > SCR_ARENA_CAP) return NULL; + a->bytes += n; + if (a->nblocks == a->cap) { + size_t cap = a->cap ? a->cap * 2 : 16; + char **grown = realloc(a->blocks, cap * sizeof *grown); + if (!grown) scr_trap("scriptc: out of memory\n"); + a->blocks = grown; + a->cap = cap; + } + char *block = calloc(1, n); + if (!block) scr_trap("scriptc: out of memory\n"); + a->blocks[a->nblocks++] = block; +#endif + if (!cyc) return block; + ScrCycHdr *ch = (ScrCycHdr *)block; + ch->id = (uint16_t)scr_cyc_type_id((ScrCycType *)cyc); + ch->buf = SCR_CYC_ACYCLIC; + return block + hdr; +} + +static void scr_arena_free(ScrArena *a) { + /* Every entry before any memory: a release can reach another object of + * the arena. Newest first, the reverse of construction. */ + for (ScrArenaDisposal *d = a->disposals; d != a->disposal_base;) { + d--; + d->dispose(d->obj); + } + if ((size_t)(a->disposals_end - a->disposal_base) > SCR_ARENA_POOL_DISPOSALS) { + free(a->disposal_base); + a->disposal_base = a->disposals_end = NULL; + } +#if SCR_SMALL_ALLOC + for (ScrArenaChunk *c = a->chunks, *next; c; c = next) { + next = c->next; + if (scr_arena_chunk_bytes + c->size <= SCR_ARENA_POOL_BYTES) { + c->next = scr_arena_chunk_pool; + scr_arena_chunk_pool = c; + scr_arena_chunk_bytes += c->size; + } else + free(c); + } + if (scr_arena_pooled < SCR_ARENA_POOL) { + a->next = scr_arena_pool; + scr_arena_pool = a; + scr_arena_pooled++; + return; + } +#else + for (size_t i = 0; i < a->nblocks; i++) free(a->blocks[i]); + free(a->blocks); +#endif + free(a->disposal_base); + free(a); +} + +void scr_arena_drop(void **list) { + ScrArena *a = (ScrArena *)*list; + *list = NULL; + while (a) { + ScrArena *next = a->next; + scr_arena_unlink(a); + scr_arena_free(a); + a = next; + } +} diff --git a/packages/runtime/src/scr_arena.h b/packages/runtime/src/scr_arena.h new file mode 100644 index 000000000..cdfe5a476 --- /dev/null +++ b/packages/runtime/src/scr_arena.h @@ -0,0 +1,115 @@ +#ifndef SCR_ARENA_H +#define SCR_ARENA_H +/* Group arenas (llvm/region-groups.ts, llvm/arenas.ts): the objects a call + * allocates while the compiler has proven they die together, bump-allocated + * and released at once. + * + * Two kinds of call open one: + * - a fresh-result call (a root) whose value its caller keeps in one scope: + * the arena stays the allocation target while the call runs, and is + * dropped when that scope exits, on every path; + * - a call region, a call of a function that keeps nothing it allocates + * and returns no reference: the arena is dropped when the call returns. + * + * Only allocation sites in functions the compiler reached from such a call + * consult the target (scr_arena_cur), and only those whose value the group + * keeps (llvm/region-sites.ts), so runtime-internal allocations and + * temporaries never land in an arena. + * + * An arena object is an ordinary object with an ordinary count, starting + * at SCR_ARENA_RC instead of 1, so no release reaches zero while it lives: + * no teardown, no free, and (with `buf` SCR_CYC_ACYCLIC) never a collector + * candidate. The count is far below SCR_RC_PUBLISHED, so the arena object + * stays mutable and thread-local like any other. Dropping the arena runs + * its dispose list, whose entries release their objects' fields (and + * storage) without freeing the objects, and then recycles the memory. A + * closed arena's objects hold only each other and immortal values, so only + * objects with storage of their own (arrays) go on its list. + * + * Without the small-object allocator (sanitized and audit builds), every + * arena object is a block of its own that the drop frees, so a use after + * the drop is a use after free that the sanitizer reports. */ +#include "scr_core.h" + +#include +#include + +/* The starting count of an arena object (a count, in the low 32 bits of + * the rc word, so it also suits classes that keep their id in the high + * half). */ +#define SCR_ARENA_RC ((uint32_t)1 << 29) + +typedef struct ScrArena ScrArena; +/* Releases what an arena object holds (its fields, an array's storage) + * without freeing the object. */ +typedef void (*ScrArenaDispose)(void *obj); + +/* An entry of an arena's dispose list. The emitted arena-aware + * constructors append one inline for an object whose type has fields to + * release, unless the arena is closed (llvm/arenas.ts). */ +typedef struct ScrArenaDisposal { + void *obj; + ScrArenaDispose dispose; +} ScrArenaDisposal; + +/* The arena that allocation sites in arena-aware functions use, or NULL. + * Emitted code reads scr_arena_cur directly outside worker programs. */ +extern SCR_TL ScrArena *scr_arena_cur; +void *scr_arena_current(void); +/* Stops any arena being the allocation target while a call whose value + * its group does not keep runs; returns the target, which + * scr_arena_resume restores. */ +void *scr_arena_suspend(void); +void scr_arena_resume(void *arena); +/* Opens an arena as the allocation target and links it onto `*list`, the + * list its scope drops. Returns it. A `closed` arena's objects hold only + * each other and immortal values (the compiler proved it), so dropping it + * releases nothing. */ +void *scr_arena_open(void **list, int32_t closed); +/* Stops `arena` (may be NULL) being the allocation target; its objects + * live on until its list is dropped. */ +void scr_arena_close(void *arena); +/* Drops every arena on `*list`, newest first, and clears the list. */ +void scr_arena_drop(void **list); +/* A zeroed object of `size` bytes in `arena` (behind a collector header + * when `cyc` is not NULL, its `buf` SCR_CYC_ACYCLIC), for when the emitted + * bump finds no room: a new chunk. Returns NULL when an arena that is not + * closed is past its byte cap and `cyc` is NULL, and the caller allocates + * on the heap. The caller stores the count (SCR_ARENA_RC). */ +void *scr_arena_obj(void *arena, size_t size, void *cyc); +/* Appends `obj` to `arena`'s dispose list, which the drop runs. */ +void scr_arena_disposal(void *arena, void *obj, ScrArenaDispose dispose); + +/* The leading fields of an arena (scr_arena.c's ScrArena), for the + * runtime's own inline allocation. */ +typedef struct ScrArenaHead { + char *bump, *limit; + ScrArenaDisposal *disposals, *disposals_end; + uint32_t closed; +} ScrArenaHead; + +/* `n` bytes (a multiple of 8) from `arena`'s current chunk, not cleared, + * or NULL when the chunk has no room (scr_arena_obj takes a new one). */ +static inline void *scr_arena_bump(void *arena, size_t n) { + ScrArenaHead *h = (ScrArenaHead *)arena; + if ((size_t)(h->limit - h->bump) < n) return NULL; + char *p = h->bump; + h->bump = p + n; + return p; +} + +/* scr_arena_disposal with the append inline. */ +static inline void scr_arena_dispose_later(void *arena, void *obj, ScrArenaDispose dispose) { + ScrArenaHead *h = (ScrArenaHead *)arena; + if (h->disposals == h->disposals_end) { + scr_arena_disposal(arena, obj, dispose); + return; + } + h->disposals->obj = obj; + h->disposals->dispose = dispose; + h->disposals++; +} + +static inline int scr_arena_is_closed(void *arena) { return ((ScrArenaHead *)arena)->closed != 0; } + +#endif diff --git a/packages/runtime/src/scr_arena.test.c b/packages/runtime/src/scr_arena.test.c new file mode 100644 index 000000000..8f369d4e3 --- /dev/null +++ b/packages/runtime/src/scr_arena.test.c @@ -0,0 +1,108 @@ +/* Group arenas (scr_arena.h): allocation, nesting, closing and dropping, + * and the drop's dispose list. Prints one line per check. */ +#include "scr_runtime.h" + +#include +#include + +typedef struct { + int64_t rc; + int64_t id; +} TestObj; + +static int disposed; +static int64_t disposed_sum; +static void test_dispose(void *obj) { + disposed++; + disposed_sum += ((TestObj *)obj)->id; +} +/* What an emitted arena-aware constructor does: the object, and its + * dispose entry unless the arena is closed. */ +static TestObj *obj(void *arena, int64_t id, int closed) { + TestObj *o = scr_arena_obj(arena, sizeof *o, NULL); + if (!o) return NULL; + o->rc = SCR_ARENA_RC; + o->id = id; + if (!closed) scr_arena_disposal(arena, o, test_dispose); + return o; +} + +int main(void) { + /* An open arena is the allocation target until it closes. */ + void *list = NULL; + void *a = scr_arena_open(&list, 0); + printf("current while open: %s\n", scr_arena_current() == a ? "yes" : "no"); + TestObj *first = obj(a, 1, 0); + int zero = 1, adjacent = 1; + TestObj *prev = first; + for (int i = 0; i < 16; i++) { + TestObj *o = scr_arena_obj(a, sizeof *o, NULL); + if (o->rc != 0 || o->id != 0) zero = 0; + if ((char *)o != (char *)(prev + 1)) adjacent = 0; + prev = o; + } + printf("objects start zeroed: %s\n", zero ? "yes" : "no"); +#if !SCR_SMALL_ALLOC + adjacent = 1; /* a block per object */ +#endif + printf("objects without headers: %s\n", adjacent ? "yes" : "no"); + /* A nested arena makes the outer one current again when it closes. */ + void *inner_list = NULL; + void *b = scr_arena_open(&inner_list, 0); + for (int i = 0; i < 10; i++) obj(b, 100, 0); + scr_arena_close(b); + scr_arena_close(b); + printf("outer current after inner close: %s\n", scr_arena_current() == a ? "yes" : "no"); + scr_arena_drop(&inner_list); + printf("inner drop disposed %d (sum %lld)\n", disposed, (long long)disposed_sum); + /* Many objects span chunks; each is disposed once. */ + disposed = 0; + disposed_sum = 0; + for (int i = 0; i < 100000; i++) obj(a, 1, 0); + scr_arena_close(a); + printf("current after close: %s\n", scr_arena_current() == NULL ? "none" : "some"); + scr_arena_drop(&list); + printf("drop disposed %d (sum %lld), list %s\n", disposed, (long long)disposed_sum, + list == NULL ? "cleared" : "kept"); + + /* A closed arena's objects take no dispose entries, and it has no cap. */ + disposed = 0; + void *closed = scr_arena_open(&list, 1); + int held = 0; + for (int i = 0; i < 3000000; i++) + if (obj(closed, 1, 1)) held++; + scr_arena_drop(&list); + printf("closed arena held %d, disposed %d, current %s\n", held, disposed, + scr_arena_current() == NULL ? "none" : "some"); + + /* Past the byte cap an arena that is not closed refuses, and its drop + * disposes everything it holds. */ + disposed = 0; + void *capped = scr_arena_open(&list, 0); + int refused = 0; + held = 0; + for (int i = 0; i < 3000000 && !refused; i++) { + if (obj(capped, 1, 0)) + held++; + else + refused = 1; + } + scr_arena_drop(&list); + printf("cap refused: %s, all disposed: %s\n", refused ? "yes" : "no", disposed == held ? "yes" : "no"); + + /* A drop of an arena that is still open (an exception skipped its close) + * restores the target it replaced. */ + void *outer_list = NULL, *skipped_list = NULL; + void *outer = scr_arena_open(&outer_list, 0); + scr_arena_open(&skipped_list, 0); + scr_arena_drop(&skipped_list); + printf("target after dropping an open arena: %s\n", scr_arena_current() == outer ? "outer" : "other"); + /* A suspended arena is no target until it resumes. */ + void *saved = scr_arena_suspend(); + printf("target while suspended: %s\n", scr_arena_current() == NULL ? "none" : "some"); + scr_arena_resume(saved); + printf("target after resume: %s\n", scr_arena_current() == outer ? "outer" : "other"); + scr_arena_close(outer); + scr_arena_drop(&outer_list); + return 0; +} diff --git a/packages/runtime/src/scr_arena.test.ts b/packages/runtime/src/scr_arena.test.ts new file mode 100644 index 000000000..0e48c8ea8 --- /dev/null +++ b/packages/runtime/src/scr_arena.test.ts @@ -0,0 +1,67 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, test } from "vitest"; +import { EXECUTABLE_RUNTIME_SOURCES } from "../../compiler/src/backend/native-toolchain.js"; + +const exec = promisify(execFile); + +const EXPECTED = [ + "current while open: yes", + "objects start zeroed: yes", + "objects without headers: yes", + "outer current after inner close: yes", + "inner drop disposed 10 (sum 1000)", + "current after close: none", + "drop disposed 100001 (sum 100001), list cleared", + "closed arena held 3000000, disposed 0, current none", + "cap refused: yes, all disposed: yes", + "target after dropping an open arena: outer", + "target while suspended: none", + "target after resume: outer", +]; + +// The size-class allocator build (chunks, the inline bump's fields), and +// the sanitized lane's build (a block per object, freed by the drop). +describe.skipIf(process.platform === "win32")("group arenas", () => { + test.each([ + ["size-class", ["-fsanitize=undefined"]], + ["sanitized", ["-fsanitize=address,undefined", "-DSCR_RC_AUDIT"]], + ])( + "open, allocate, close and drop (%s)", + async (_lane, flags) => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-arena-")); + try { + const bin = join(dir, "arena"); + await exec("clang", [ + "-std=c11", + "-O1", + "-pthread", + "-Wno-deprecated-declarations", + ...(process.platform === "linux" ? ["-D_GNU_SOURCE"] : []), + ...flags, + ...["scr_arena.test.c", ...EXECUTABLE_RUNTIME_SOURCES].map((source) => + join(import.meta.dirname, source), + ), + ...(process.platform === "linux" ? ["-lm"] : []), + "-o", + bin, + ]); + const { stdout } = await exec(bin, [], { + timeout: 60000, + env: { + ...process.env, + ASAN_OPTIONS: "halt_on_error=1", + UBSAN_OPTIONS: "halt_on_error=1", + }, + }); + expect(stdout.trim().split("\n")).toEqual(EXPECTED); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, + 300000, + ); +}); diff --git a/packages/runtime/src/scr_array.c b/packages/runtime/src/scr_array.c index b9e0efa74..0e4513c6f 100644 --- a/packages/runtime/src/scr_array.c +++ b/packages/runtime/src/scr_array.c @@ -1426,15 +1426,53 @@ static void scr_arr_put_slots(ScrArr *a, size_t to, size_t count, const uint64_t /* Inlined into each constructor, so `small` and the slot unit fold to * constants there. */ +static void scr_arr_arena_dispose(void *object); +static void scr_arr_arena_dispose_closed(void *object); +static ScrCycType scr_arr_cyc_type; + +/* An array block of `size` bytes in the current group arena, or NULL. + * Its storage beyond the block is the heap's, so it goes on the arena's + * dispose list even when the arena is closed; there, its elements are + * arena objects and immortal values, which need no release. */ +static inline ScrArr *scr_arr_arena_block(size_t size, bool cyc) { + void *arena = scr_arena_cur; + if (!arena) return NULL; + size_t hdr = cyc ? sizeof(ScrCycHdr) : 0; + size_t n = (hdr + size + 7) & ~(size_t)7; + uint32_t id = cyc ? scr_cyc_type_id(&scr_arr_cyc_type) : 0; + char *block = scr_arena_bump(arena, n); + ScrArr *a; + if (block) { + /* The constructor sets every header field, as on the heap; only the + * collector header needs setting here. */ + if (cyc) *(ScrCycHdr *)block = (ScrCycHdr){.id = (uint16_t)id, .buf = SCR_CYC_ACYCLIC}; + a = (ScrArr *)(block + hdr); + } else { + a = scr_arena_obj(arena, size, cyc ? &scr_arr_cyc_type : NULL); + if (!a) return NULL; + } + scr_arena_dispose_later(arena, a, + scr_arena_is_closed(arena) ? scr_arr_arena_dispose_closed + : scr_arr_arena_dispose); + return a; +} + static inline __attribute__((always_inline)) ScrArr *scr_arr_make(ScrElemKind elem, size_t initial_cap, - bool small) { + bool small, + bool arena) { size_t unit = scr_elem_slot_width(elem); size_t inline_cap = small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET, unit) : scr_arr_known_cap(SCR_ARR_INLINE_OFFSET, initial_cap, unit); - ScrArr *a = scr_mem_alloc(inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * unit : sizeof(ScrArr)); - if (!a) scr_arr_oom(); - a->rc = 1; + size_t size = inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * unit : sizeof(ScrArr); + ScrArr *a = arena ? scr_arr_arena_block(size, false) : NULL; + if (a) + a->rc = SCR_ARENA_RC; + else { + a = scr_mem_alloc(size); + if (!a) scr_arr_oom(); + a->rc = 1; + } a->len = 0; a->cap = 0; a->elem = (uint8_t)elem; @@ -1451,10 +1489,16 @@ static inline __attribute__((always_inline)) ScrArr *scr_arr_make(ScrElemKind el } ScrArr *scr_arr_new(ScrElemKind elem, size_t initial_cap) { - return scr_arr_make(elem, initial_cap, false); + return scr_arr_make(elem, initial_cap, false, false); } -ScrArr *scr_arr_new_small(ScrElemKind elem) { return scr_arr_make(elem, 0, true); } +ScrArr *scr_arr_new_small(ScrElemKind elem) { return scr_arr_make(elem, 0, true, false); } + +ScrArr *scr_arr_new_r(ScrElemKind elem, size_t initial_cap) { + return scr_arr_make(elem, initial_cap, false, true); +} + +ScrArr *scr_arr_new_small_r(ScrElemKind elem) { return scr_arr_make(elem, 0, true, true); } /* A non-cycle array whose final length the runtime knows when it builds * it (regex match results): its `cap` slots share the header's block, so @@ -1463,7 +1507,7 @@ ScrArr *scr_arr_new_small(ScrElemKind elem) { return scr_arr_make(elem, 0, true) * `cap` moves the slots out like any inline array. */ ScrArr *scr_arr_new_inline(ScrElemKind elem, size_t cap, bool states) { if (cap == 0) { - ScrArr *a = scr_arr_make(elem, 0, false); + ScrArr *a = scr_arr_make(elem, 0, false, false); if (states) a->spare |= SCR_ARR_STATES; return a; } @@ -1572,13 +1616,16 @@ static void scr_arr_gc_free(void *a0) { scr_cyc_free_inline(a); } +static ScrCycType scr_arr_cyc_type = {&scr_arr_trace_v, &scr_arr_gc_free, SCR_CYC_KIND_ARRAY}; + /* Inlined into each constructor like scr_arr_make: with `small`, the count * and `elem` constant, scr_arr_new_ops_small's size class folds to a table * index and the slot sizes to constants. */ static inline __attribute__((always_inline)) ScrArr *scr_arr_make_ops_kind(const ScrArrOps *ops, size_t initial_cap, bool small, - uint8_t elem) { + uint8_t elem, + bool arena) { ScrArr *a; bool cyc = ops->trace != NULL; /* The array starts without states: slots of the value width alone. */ @@ -1587,14 +1634,18 @@ static inline __attribute__((always_inline)) ScrArr *scr_arr_make_ops_kind(const size_t inline_cap = small ? scr_arr_small_cap(prefix, unit) : scr_arr_known_cap(prefix, initial_cap, unit); size_t size = inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * unit : sizeof(ScrArr); - if (cyc) { - static ScrCycType cyc_type = { &scr_arr_trace_v, &scr_arr_gc_free, SCR_CYC_KIND_ARRAY }; - a = scr_cyc_alloc_inline(size, &cyc_type); - } else { - a = scr_mem_alloc(size); - if (!a) scr_arr_oom(); + a = arena ? scr_arr_arena_block(size, cyc) : NULL; + if (a) + a->rc = SCR_ARENA_RC; + else { + if (cyc) + a = scr_cyc_alloc_inline(size, &scr_arr_cyc_type); + else { + a = scr_mem_alloc(size); + if (!a) scr_arr_oom(); + } + a->rc = 1; } - a->rc = 1; a->len = 0; a->cap = 0; a->elem = elem; @@ -1622,17 +1673,26 @@ static inline __attribute__((always_inline)) ScrArr *scr_arr_make_ops_kind(const * branch inlines scr_arr_make_ops_kind with a constant kind. */ static inline __attribute__((always_inline)) ScrArr *scr_arr_make_ops(const ScrArrOps *ops, size_t initial_cap, - bool small) { + bool small, + bool arena) { if (__builtin_expect(scr_arr_ops_compact(ops), 0)) - return scr_arr_make_ops_kind(ops, initial_cap, small, SCR_ELEM_CREF); - return scr_arr_make_ops_kind(ops, initial_cap, small, SCR_ELEM_REF); + return scr_arr_make_ops_kind(ops, initial_cap, small, SCR_ELEM_CREF, arena); + return scr_arr_make_ops_kind(ops, initial_cap, small, SCR_ELEM_REF, arena); } ScrArr *scr_arr_new_ops(const ScrArrOps *ops, size_t initial_cap) { - return scr_arr_make_ops(ops, initial_cap, false); + return scr_arr_make_ops(ops, initial_cap, false, false); } -ScrArr *scr_arr_new_ops_small(const ScrArrOps *ops) { return scr_arr_make_ops(ops, 0, true); } +ScrArr *scr_arr_new_ops_small(const ScrArrOps *ops) { return scr_arr_make_ops(ops, 0, true, false); } + +/* The constructors arena-aware functions call (llvm/arenas.ts): in the + * current group arena when one is open. */ +ScrArr *scr_arr_new_ops_r(const ScrArrOps *ops, size_t initial_cap) { + return scr_arr_make_ops(ops, initial_cap, false, true); +} + +ScrArr *scr_arr_new_ops_small_r(const ScrArrOps *ops) { return scr_arr_make_ops(ops, 0, true, true); } /* Frees an acyclic array whose elements are already released. */ static inline __attribute__((always_inline)) void scr_arr_free(ScrArr *a) { @@ -1746,6 +1806,26 @@ static void scr_arr_destroy(void *object) { else scr_arr_free(a); } +/* An arena array at its arena's drop: its contents and the storage beyond + * its block go; the block goes with the arena. */ +static void scr_arr_arena_dispose(void *object) { + ScrArr *a = object; + scr_arr_release_contents(a); + scr_arr_free_parts(a); +#ifdef SCR_RC_AUDIT + scr_live_arrays--; +#endif +} + +/* The same in a closed arena, whose objects hold only each other and + * immortal values: only the storage beyond the block goes. */ +static void scr_arr_arena_dispose_closed(void *object) { + scr_arr_free_parts((ScrArr *)object); +#ifdef SCR_RC_AUDIT + scr_live_arrays--; +#endif +} + void scr_arr_subclass_init(ScrArr *a, uint8_t elem, const ScrArrOps *ops, const ScrArrClass *cls, bool cyc) { a->len = 0; @@ -5982,8 +6062,8 @@ void scr_arr_copy_metadata(ScrArr *a, ScrDyn *target) { target->v.arr.properties->v.obj.class_name = cls->name; ScrDyn *fields = scr_arr_class_fields(a); if (fields) { - for (size_t i = 0; i < fields->v.obj.len; i++) - scr_dyn_copy_entry(target, fields, &fields->v.obj.entries[i]); + for (size_t i = 0; i < scr_dyn_obj_count(fields); i++) + scr_dyn_copy_prop_at(target, fields, i); scr_dyn_release(fields); } return; @@ -5994,7 +6074,7 @@ void scr_arr_copy_metadata(ScrArr *a, ScrDyn *target) { return; metadata = SCR_AX(a)->metadata; if (!metadata) return; - for (size_t i = 0; i < metadata->v.obj.len; i++) scr_dyn_copy_entry(target, metadata, &metadata->v.obj.entries[i]); + for (size_t i = 0; i < scr_dyn_obj_count(metadata); i++) scr_dyn_copy_prop_at(target, metadata, i); } ScrArr *scr_arr_named_keys(ScrArr *a) { @@ -6008,9 +6088,9 @@ ScrArr *scr_arr_named_keys(ScrArr *a) { scr_arr_push_ref(keys, scr_str_new(x->props[i].key, x->props[i].key_len)); ScrDyn *fields = scr_arr_class_fields(a); if (fields) { - for (size_t i = 0; i < fields->v.obj.len; i++) { - const ScrDynEntry *entry = &fields->v.obj.entries[i]; - if (entry->enumerable) scr_arr_push_ref(keys, scr_str_new(entry->key, entry->key_len)); + for (size_t i = 0; i < scr_dyn_obj_count(fields); i++) { + ScrDynProp entry = scr_dyn_obj_prop_at(fields, i); + if (entry.enumerable) scr_arr_push_ref(keys, scr_str_new(entry.key, entry.key_len)); } scr_dyn_release(fields); return keys; @@ -6022,10 +6102,10 @@ ScrArr *scr_arr_named_keys(ScrArr *a) { return keys; } if (!metadata || metadata->kind != SCR_DYN_OBJ) return keys; - for (size_t i = 0; i < metadata->v.obj.len; i++) { - const ScrDynEntry *entry = &metadata->v.obj.entries[i]; - if (!entry->enumerable || scr_arr_prop_index(a, entry->key, entry->key_len) != SIZE_MAX) continue; - scr_arr_push_ref(keys, scr_str_new(entry->key, entry->key_len)); + for (size_t i = 0; i < scr_dyn_obj_count(metadata); i++) { + ScrDynProp entry = scr_dyn_obj_prop_at(metadata, i); + if (!entry.enumerable || scr_arr_prop_index(a, entry.key, entry.key_len) != SIZE_MAX) continue; + scr_arr_push_ref(keys, scr_str_new(entry.key, entry.key_len)); } return keys; } @@ -6038,9 +6118,9 @@ bool scr_arr_has_named_key(ScrArr *a, ScrStr *key) { if (metadata && metadata->kind == SCR_DYN_HANDLE && metadata->v.handle.tag == SCR_DYNH_MATCH_SOURCE) return true; /* index, input and groups, the only keys a pending result lists */ if (!metadata || metadata->kind != SCR_DYN_OBJ) return false; - for (size_t i = 0; i < metadata->v.obj.len; i++) { - const ScrDynEntry *entry = &metadata->v.obj.entries[i]; - if (entry->key_len == key->len && memcmp(entry->key, key->data, key->len) == 0) return true; + for (size_t i = 0; i < scr_dyn_obj_count(metadata); i++) { + ScrDynProp entry = scr_dyn_obj_prop_at(metadata, i); + if (entry.key_len == key->len && memcmp(entry.key, key->data, key->len) == 0) return true; } return false; } @@ -6067,20 +6147,22 @@ void scr_arr_adopt_view_properties(ScrArr *a, ScrDyn *view, ScrArr *target) { from->prop_cap = cap; } ScrDyn *named = view && view->kind == SCR_DYN_ARR ? view->v.arr.properties : NULL; - if (!named || named->v.obj.len == 0) return; + if (!named || scr_dyn_obj_count(named) == 0) return; /* The view's numeric properties are copies of the native ones (every * write to them goes to the native array), so only the others move. */ bool numeric = false; - for (size_t i = 0; i < named->v.obj.len && !numeric; i++) { + for (size_t i = 0; i < scr_dyn_obj_count(named) && !numeric; i++) { double number; - numeric = scr_arr_numeric_key(named->v.obj.entries[i].key, named->v.obj.entries[i].key_len, &number); + size_t key_len; + const char *key = scr_dyn_obj_key_at(named, i, &key_len); + numeric = scr_arr_numeric_key(key, key_len, &number); } if (numeric) { ScrDyn *rest = scr_dyn_new_obj(); - for (size_t i = 0; i < named->v.obj.len; i++) { - const ScrDynEntry *entry = &named->v.obj.entries[i]; + for (size_t i = 0; i < scr_dyn_obj_count(named); i++) { + ScrDynProp entry = scr_dyn_obj_prop_at(named, i); double number; - if (!scr_arr_numeric_key(entry->key, entry->key_len, &number)) scr_dyn_copy_entry(rest, named, entry); + if (!scr_arr_numeric_key(entry.key, entry.key_len, &number)) scr_dyn_copy_prop_at(rest, named, i); } named = rest; } else { @@ -6088,7 +6170,7 @@ void scr_arr_adopt_view_properties(ScrArr *a, ScrDyn *view, ScrArr *target) { } ScrArrExt *x = scr_arr_ext(a); ScrDyn *old = x->metadata; - x->metadata = named->v.obj.len ? named : NULL; + x->metadata = scr_dyn_obj_count(named) ? named : NULL; if (!x->metadata) scr_dyn_release(named); scr_dyn_release(old); } diff --git a/packages/runtime/src/scr_array.h b/packages/runtime/src/scr_array.h index 0b162ff5e..86ef8d999 100644 --- a/packages/runtime/src/scr_array.h +++ b/packages/runtime/src/scr_array.h @@ -490,6 +490,12 @@ ScrArr *scr_arr_new_small(ScrElemKind elem); * SCR_ELEM_F64); no-op for any other kind. */ void scr_arr_widen_i32(ScrArr *a); ScrArr *scr_arr_new_ops_small(const ScrArrOps *ops); +/* The same constructors in the current group arena, when one is open + * (scr_arena.h): what arena-aware functions call. */ +ScrArr *scr_arr_new_r(ScrElemKind elem, size_t initial_cap); +ScrArr *scr_arr_new_small_r(ScrElemKind elem); +ScrArr *scr_arr_new_ops_r(const ScrArrOps *ops, size_t initial_cap); +ScrArr *scr_arr_new_ops_small_r(const ScrArrOps *ops); /* scr_arr_new(elem, cap) for a known final length: the `cap` slots share * the array's own block (one allocation). With `states`, the array starts * with states (for holes or present undefined). Returns +1. */ diff --git a/packages/runtime/src/scr_assert.c b/packages/runtime/src/scr_assert.c index bf5d1c875..6e9384f15 100644 --- a/packages/runtime/src/scr_assert.c +++ b/packages/runtime/src/scr_assert.c @@ -718,7 +718,7 @@ static bool scr_assert_dyn_deep_eq(const ScrDyn *a, const ScrDyn *b) { if (a->v.arr.properties || b->v.arr.properties) { if (!a->v.arr.properties || !b->v.arr.properties) { const ScrDyn *table = a->v.arr.properties ? a->v.arr.properties : b->v.arr.properties; - if (table->v.obj.len != 0) return false; + if (scr_dyn_obj_count(table) != 0) return false; } else if (!scr_assert_dyn_deep_eq(a->v.arr.properties, b->v.arr.properties)) return false; } return true; @@ -738,11 +738,11 @@ static bool scr_assert_dyn_deep_eq(const ScrDyn *a, const ScrDyn *b) { } else if (a->null_proto != b->null_proto || a->prototype != b->prototype) { return false; } - if (a->v.obj.len != b->v.obj.len) return false; - for (size_t i = 0; i < a->v.obj.len; i++) { - const ScrDynEntry *ent = &a->v.obj.entries[i]; - ScrDyn *bv = scr_dyn_obj_get(b, ent->key, ent->key_len); - if (!bv || !scr_assert_dyn_deep_eq(ent->value, bv)) return false; + if (scr_dyn_obj_count(a) != scr_dyn_obj_count(b)) return false; + for (size_t i = 0; i < scr_dyn_obj_count(a); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(a, i); + ScrDyn *bv = scr_dyn_obj_get(b, ent.key, ent.key_len); + if (!bv || !scr_assert_dyn_deep_eq(ent.value, bv)) return false; } return true; } @@ -782,30 +782,29 @@ static bool scr_assert_loose_nan(const ScrDyn *d) { return d->kind == SCR_DYN_NUM && d->v.num != d->v.num; } -/* An enumerable own entry, or NULL. Borrowed. */ -static const ScrDynEntry *scr_assert_loose_entry(const ScrDyn *d, const char *key, size_t len) { - for (size_t i = 0; i < d->v.obj.len; i++) { - const ScrDynEntry *ent = &d->v.obj.entries[i]; - if (ent->key_len == len && memcmp(ent->key, key, len) == 0) return ent->enumerable ? ent : NULL; - } - return NULL; +/* The value of an enumerable own property, or NULL. Borrowed. */ +static const ScrDyn *scr_assert_loose_entry(const ScrDyn *d, const char *key, size_t len) { + size_t i = scr_dyn_obj_index_of(d, key, len); + if (i == SIZE_MAX) return NULL; + ScrDynProp ent = scr_dyn_obj_prop_at(d, i); + return ent.enumerable ? ent.value : NULL; } -static bool scr_assert_loose_counted(const ScrDynEntry *ent) { - return ent->enumerable && !(ent->key_len == 6 && memcmp(ent->key, "%error", 6) == 0); +static bool scr_assert_loose_counted(ScrDynProp ent) { + return ent.enumerable && !(ent.key_len == 6 && memcmp(ent.key, "%error", 6) == 0); } /* keyCheck and objEquiv over two objects' enumerable own keys. */ static bool scr_assert_loose_keys(const ScrDyn *a, const ScrDyn *b) { size_t na = 0, nb = 0; - for (size_t i = 0; i < a->v.obj.len; i++) na += scr_assert_loose_counted(&a->v.obj.entries[i]); - for (size_t i = 0; i < b->v.obj.len; i++) nb += scr_assert_loose_counted(&b->v.obj.entries[i]); + for (size_t i = 0; i < scr_dyn_obj_count(a); i++) na += scr_assert_loose_counted(scr_dyn_obj_prop_at(a, i)); + for (size_t i = 0; i < scr_dyn_obj_count(b); i++) nb += scr_assert_loose_counted(scr_dyn_obj_prop_at(b, i)); if (na != nb) return false; - for (size_t i = 0; i < b->v.obj.len; i++) { - const ScrDynEntry *ent = &b->v.obj.entries[i]; + for (size_t i = 0; i < scr_dyn_obj_count(b); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(b, i); if (!scr_assert_loose_counted(ent)) continue; - const ScrDynEntry *other = scr_assert_loose_entry(a, ent->key, ent->key_len); - if (!other || !scr_assert_loose_eq(other->value, ent->value)) return false; + const ScrDyn *other = scr_assert_loose_entry(a, ent.key, ent.key_len); + if (!other || !scr_assert_loose_eq(other, ent.value)) return false; } return true; } @@ -1180,8 +1179,8 @@ static bool scr_assert_loose_array(const ScrDyn *a, const ScrDyn *b) { if (pa || pb) { if (!pa || !pb) { const ScrDyn *table = pa ? pa : pb; - for (size_t i = 0; i < table->v.obj.len; i++) - if (scr_assert_loose_counted(&table->v.obj.entries[i])) return false; + for (size_t i = 0; i < scr_dyn_obj_count(table); i++) + if (scr_assert_loose_counted(scr_dyn_obj_prop_at(table, i))) return false; } else if (!scr_assert_loose_keys(pa, pb)) { return false; } @@ -1295,7 +1294,7 @@ static void scr_assert_cf_value(ScrAssertBuf *b, const ScrDyn *d, size_t indent, * or `{`, then its entries. */ static void scr_assert_cf_object(ScrAssertBuf *b, const ScrDyn *d, size_t indent, size_t depth, const char *cls) { - if (d->v.obj.len == 0) { + if (scr_dyn_obj_count(d) == 0) { if (cls) { ab_cstr(b, cls); ab_cstr(b, " {}"); @@ -1320,25 +1319,25 @@ static void scr_assert_cf_object(ScrAssertBuf *b, const ScrDyn *d, size_t indent } else if (d->null_proto) ab_cstr(b, "[Object: null prototype] "); /* Render every entry, then sort the RENDERED texts (Node's * sorted:true sorts formatted entries, quotes included). */ - ScrCfEntry *ents = malloc(d->v.obj.len * sizeof *ents); + ScrCfEntry *ents = malloc(scr_dyn_obj_count(d) * sizeof *ents); if (!ents) scr_assert_oom(); - for (size_t i = 0; i < d->v.obj.len; i++) { - const ScrDynEntry *ent = &d->v.obj.entries[i]; + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(d, i); ScrAssertBuf eb = {0}; - scr_assert_cf_key(&eb, ent->key, ent->key_len); + scr_assert_cf_key(&eb, ent.key, ent.key_len); ab_cstr(&eb, ": "); - scr_assert_cf_value(&eb, ent->value, indent + 2, depth - 1); + scr_assert_cf_value(&eb, ent.value, indent + 2, depth - 1); ents[i].text = eb.data ? eb.data : malloc(1); if (!ents[i].text) scr_assert_oom(); ents[i].len = eb.len; } - qsort(ents, d->v.obj.len, sizeof *ents, scr_assert_cf_entry_cmp); + qsort(ents, scr_dyn_obj_count(d), sizeof *ents, scr_assert_cf_entry_cmp); ab_char(b, '{'); - for (size_t i = 0; i < d->v.obj.len; i++) { + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { ab_char(b, '\n'); scr_assert_cf_pad(b, indent + 2); ab_bytes(b, ents[i].text, ents[i].len); - if (i + 1 < d->v.obj.len) ab_char(b, ','); + if (i + 1 < scr_dyn_obj_count(d)) ab_char(b, ','); free(ents[i].text); } free(ents); @@ -1364,11 +1363,11 @@ static const char *scr_assert_cf_bytes_class(ScrBytesElem elem) { } -static bool scr_assert_cf_error_extra(const ScrDynEntry *ent) { - if (!ent->enumerable || (ent->key_len > 0 && ent->key[0] == '%')) return false; - return !((ent->key_len == 4 && memcmp(ent->key, "name", 4) == 0) || - (ent->key_len == 7 && memcmp(ent->key, "message", 7) == 0) || - (ent->key_len == 5 && memcmp(ent->key, "stack", 5) == 0)); +static bool scr_assert_cf_error_extra(ScrDynProp ent) { + if (!ent.enumerable || (ent.key_len > 0 && ent.key[0] == '%')) return false; + return !((ent.key_len == 4 && memcmp(ent.key, "name", 4) == 0) || + (ent.key_len == 7 && memcmp(ent.key, "message", 7) == 0) || + (ent.key_len == 5 && memcmp(ent.key, "stack", 5) == 0)); } /* An encoded Error: Node's stackless `[Name: message]` (assertion_error.js @@ -1386,8 +1385,8 @@ static void scr_assert_cf_error(ScrAssertBuf *b, const ScrDyn *d, size_t indent, } ab_char(b, ']'); size_t extra = 0; - for (size_t i = 0; i < d->v.obj.len; i++) { - const ScrDynEntry *ent = &d->v.obj.entries[i]; + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(d, i); if (scr_assert_cf_error_extra(ent)) extra++; } if (extra == 0) return; @@ -1395,13 +1394,13 @@ static void scr_assert_cf_error(ScrAssertBuf *b, const ScrDyn *d, size_t indent, ScrCfEntry *ents = malloc(extra * sizeof *ents); if (!ents) scr_assert_oom(); size_t n = 0; - for (size_t i = 0; i < d->v.obj.len; i++) { - const ScrDynEntry *ent = &d->v.obj.entries[i]; + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(d, i); if (!scr_assert_cf_error_extra(ent)) continue; ScrAssertBuf eb = {0}; - scr_assert_cf_key(&eb, ent->key, ent->key_len); + scr_assert_cf_key(&eb, ent.key, ent.key_len); ab_cstr(&eb, ": "); - scr_assert_cf_value(&eb, ent->value, indent + 2, depth - 1); + scr_assert_cf_value(&eb, ent.value, indent + 2, depth - 1); ents[n].text = eb.data ? eb.data : malloc(1); if (!ents[n].text) scr_assert_oom(); ents[n++].len = eb.len; @@ -1989,11 +1988,11 @@ static bool scr_assert_partial_eq(const ScrDyn *a, const ScrDyn *b); /* objEquiv over the expected side's enumerable own keys. */ static bool scr_assert_partial_keys(const ScrDyn *a, const ScrDyn *b) { - for (size_t i = 0; i < b->v.obj.len; i++) { - const ScrDynEntry *ent = &b->v.obj.entries[i]; + for (size_t i = 0; i < scr_dyn_obj_count(b); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(b, i); if (!scr_assert_loose_counted(ent)) continue; - const ScrDynEntry *other = scr_assert_loose_entry(a, ent->key, ent->key_len); - if (!other || !scr_assert_partial_eq(other->value, ent->value)) return false; + const ScrDyn *other = scr_assert_loose_entry(a, ent.key, ent.key_len); + if (!other || !scr_assert_partial_eq(other, ent.value)) return false; } return true; } @@ -2043,11 +2042,11 @@ static bool scr_assert_partial_array(const ScrDyn *a, const ScrDyn *b) { const ScrDyn *pb = b->v.arr.properties; if (pb) { const ScrDyn *pa = a->v.arr.properties; - for (size_t i = 0; i < pb->v.obj.len; i++) { - const ScrDynEntry *ent = &pb->v.obj.entries[i]; + for (size_t i = 0; i < scr_dyn_obj_count(pb); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(pb, i); if (!scr_assert_loose_counted(ent)) continue; - const ScrDynEntry *other = pa ? scr_assert_loose_entry(pa, ent->key, ent->key_len) : NULL; - if (!other || !scr_assert_partial_eq(other->value, ent->value)) return false; + const ScrDyn *other = pa ? scr_assert_loose_entry(pa, ent.key, ent.key_len) : NULL; + if (!other || !scr_assert_partial_eq(other, ent.value)) return false; } } size_t pos = 0; @@ -2434,11 +2433,11 @@ void scr_assert_loose_deep_result(bool equal, bool negated, ScrStr *msg, bool ha void scr_assert_expects_err_dyn(ScrDyn *actual, ScrDyn *expected, ScrStr *msg, bool has_msg) { if (actual->kind == SCR_DYN_OBJ && expected->kind == SCR_DYN_OBJ) { bool ok = true; - for (size_t i = 0; i < expected->v.obj.len && ok; i++) { - const ScrDynEntry *e = &expected->v.obj.entries[i]; - if (e->key_len == 6 && memcmp(e->key, "%error", 6) == 0) continue; - ScrDyn *av = scr_dyn_obj_get(actual, e->key, e->key_len); - if (av == NULL || !scr_assert_dyn_deep_eq(av, e->value)) ok = false; + for (size_t i = 0; i < scr_dyn_obj_count(expected) && ok; i++) { + ScrDynProp e = scr_dyn_obj_prop_at(expected, i); + if (e.key_len == 6 && memcmp(e.key, "%error", 6) == 0) continue; + ScrDyn *av = scr_dyn_obj_get(actual, e.key, e.key_len); + if (av == NULL || !scr_assert_dyn_deep_eq(av, e.value)) ok = false; } if (ok) return; } diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index 9d4719d1f..a1ebb203c 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -1518,7 +1518,11 @@ static void scr_switch(ScrCtx *from, ScrCtx *to, ScrFiber *to_fiber) { (void)to; scr_trap("scriptc: internal error: native stack switch reached on WASI\n"); #elif defined(SCR_ASAN_FIBERS) - void **save = from_fiber ? &from_fiber->fake_stack : &scr_main_fake_stack; + /* A finished fiber leaves for the last time: a NULL slot has + * AddressSanitizer free its fake stack, which otherwise stays mapped + * until exit. */ + void **save = from_fiber ? (from_fiber->done ? NULL : &from_fiber->fake_stack) + : &scr_main_fake_stack; const void *bottom = to_fiber ? to_fiber->stack : NULL; size_t size = to_fiber ? SCR_FIBER_STACK : 0; __sanitizer_start_switch_fiber(save, bottom, size); @@ -1527,10 +1531,13 @@ static void scr_switch(ScrCtx *from, ScrCtx *to, ScrFiber *to_fiber) { #else swapcontext(from, to); #endif + /* Back in this frame: the context that saved into `save` above runs + * again, whichever fiber `scr_current` names (a finished fiber returns + * to its resumer with no fiber to switch to). */ const void *old_bottom; size_t old_size; - __sanitizer_finish_switch_fiber( - scr_current ? scr_current->fake_stack : scr_main_fake_stack, &old_bottom, &old_size); + __sanitizer_finish_switch_fiber(from_fiber ? from_fiber->fake_stack : scr_main_fake_stack, + &old_bottom, &old_size); #elif defined(SCR_FAST_CTX) scr_ctx_swap(from, to); #else @@ -2228,6 +2235,7 @@ static void scr_prefix_trampoline(void) { scr_als_active = &f->als; entry(f, argpack); scr_current = carrier; + carrier->done = true; /* its stack is freed once the spawner is back */ scr_switch(&carrier->ctx, carrier->return_to, NULL); /* unreachable */ } @@ -5456,6 +5464,11 @@ static ScrGen *scr_gen_new_common(void (*entry)(ScrFiber *, void *), void *argpa bool stackless) { ScrGen *g = scr_mem_calloc(sizeof *g); if (!g) scr_oom(); + /* Install the tag reader with each generator rather than in a constructor: + * a constructor is a link root, so it would keep scr_gen_tag and this + * unit's string literals in every executable, generators or not. Threads + * store the same value, relaxed. */ + __atomic_store_n(&scr_gen_tag_reader, scr_gen_tag, __ATOMIC_RELAXED); g->rc = 1; g->receiver = scr_dyn_this_get(); g->state = SCR_GEN_UNSTARTED; @@ -5506,6 +5519,14 @@ ScrGen *scr_async_gen_new(void (*entry)(ScrFiber *, void *), void *argpack, return scr_gen_new_common(entry, argpack, drop_args, settle, false); } +#ifndef __wasi__ +ScrGen *scr_async_gen_new_stackless(void (*entry)(ScrFiber *, void *), void *argpack, + void (*drop_args)(void *), + void (*settle)(ScrGen *, ScrPromise *)) { + return scr_gen_new_common(entry, argpack, drop_args, settle, true); +} +#endif + ScrGen *scr_gen_retain(ScrGen *g) { if (g) g->rc++; return g; @@ -5854,7 +5875,7 @@ void scr_gen_ret_to_out(ScrGen *g) { * settle the generator's state, moving a finished body's exception to the * resumer and tearing the finished fiber down. `release` drops the hop's * own reference (see scr_gen_switch_in); `sync` marks a body known to be a - * synchronous generator (every stackless one is). */ + * synchronous generator. */ __attribute__((always_inline)) static inline void scr_gen_switched_back(ScrGen *g, ScrFiber *f, bool release, bool sync) { if (__builtin_expect(f->done, 0)) { @@ -5891,10 +5912,12 @@ static __attribute__((noinline)) void scr_async_gen_step_end(ScrFiber *f) { scr_ #ifndef __wasi__ /* The stackless form of the hop: the coroutine frame runs on this stack - * until its next yield or its final suspension returns here. `retain` - * holds the generator across the hop; a resumer whose own reference - * provably outlives the hop passes false. */ -__attribute__((always_inline)) static inline void scr_gen_coro_hop(ScrGen *g, bool retain) { + * until its next yield, an async generator's await, or its final + * suspension returns here. `retain` holds the generator across the hop; a + * resumer whose own reference provably outlives the hop passes false. + * `sync` marks a body known to be a synchronous generator. */ +__attribute__((always_inline)) static inline void scr_gen_coro_hop(ScrGen *g, bool retain, + bool sync) { ScrFiber *f = g->fiber; if (retain) scr_gen_retain(g); /* A synchronous generator has no AsyncLocalStorage context of its own: @@ -5919,10 +5942,29 @@ __attribute__((always_inline)) static inline void scr_gen_coro_hop(ScrGen *g, bo scr_exc_swap_cell(me != NULL ? &me->exc : NULL); scr_als_active = resumer_slot; if (g->is_async) scr_async_gen_step_end(f); - scr_gen_switched_back(g, f, retain, true); + scr_gen_switched_back(g, f, retain, sync); } -static void scr_gen_coro_in(ScrGen *g) { scr_gen_coro_hop(g, true); } +/* An async generator's step can resume the next one down a chain of them + * before it suspends, all on one stack, through runtime frames that check + * no guard: a step that would start below the stack guard throws the + * RangeError compiled code throws there, which rejects its request. */ +static __attribute__((noinline)) void scr_async_gen_coro_in(ScrGen *g) { + uintptr_t guard = scr_stack_guard; + if (guard != 0 && (uintptr_t)__builtin_frame_address(0) < guard) { + /* scr_stack_overflow's error, thrown here: this unit links without + * scr_context.c in the runtime's own tests. */ + static const char text[] = "Maximum call stack size exceeded"; + scr_throw_error_msg(SCR_ERR_RANGE, text, sizeof text - 1); + return; + } + scr_gen_coro_hop(g, true, false); +} + +static void scr_gen_coro_in(ScrGen *g) { + if (g->is_async) scr_async_gen_coro_in(g); + else scr_gen_coro_hop(g, true, true); +} #endif /* The consumer→fiber hop shared by every resume mode: switches in, and on @@ -6001,10 +6043,6 @@ const char *scr_gen_tag(const ScrGen *g) { return g->tag ? g->tag : g->is_async ? "AsyncGenerator" : "Generator"; } -__attribute__((constructor)) static void scr_gen_tag_reader_init(void) { - scr_gen_tag_reader = scr_gen_tag; -} - void scr_gen_close(ScrGen *g) { scr_gen_slot_reset(&g->ret); scr_gen_resume_return(g); @@ -6018,8 +6056,9 @@ void scr_gen_close(ScrGen *g) { bool scr_gen_step(ScrGen *g) { if (g->in.kind != SCR_EXC_NONE) scr_gen_slot_reset(&g->in); #ifndef __wasi__ - if ((g->state == SCR_GEN_SUSPENDED || g->state == SCR_GEN_UNSTARTED) && g->fiber->stackless) { - scr_gen_coro_hop(g, false); + if ((g->state == SCR_GEN_SUSPENDED || g->state == SCR_GEN_UNSTARTED) && g->fiber->stackless && + !g->is_async) { + scr_gen_coro_hop(g, false, true); return g->state == SCR_GEN_DONE; } #endif diff --git a/packages/runtime/src/scr_async.h b/packages/runtime/src/scr_async.h index e31ba5b1d..d2e16be43 100644 --- a/packages/runtime/src/scr_async.h +++ b/packages/runtime/src/scr_async.h @@ -143,6 +143,9 @@ void scr_promise_rethrow_top_level(ScrPromise *p); * value), or NULL with the rejection re-thrown into the awaiter. */ ScrDyn *scr_await_dyn(ScrPromise *p); ScrDyn *scr_await_dyn_value_settled(ScrDyn *value); +/* The native coroutine form's extraction: the body has already suspended + * until the value's promise settled, or taken the non-promise hop. */ +ScrDyn *scr_coro_await_dyn_value(ScrDyn *value); void scr_promise_resolve_dyn(ScrPromise *destination, ScrDyn *value); /* `await v` over a checked-dynamic VALUE: dyn promises adopt, everything * else takes JS's one-hop non-thenable await and answers itself (+1). */ diff --git a/packages/runtime/src/scr_async_dyn.c b/packages/runtime/src/scr_async_dyn.c index 046aebb38..6e5d9856e 100644 --- a/packages/runtime/src/scr_async_dyn.c +++ b/packages/runtime/src/scr_async_dyn.c @@ -827,6 +827,19 @@ ScrDyn *scr_await_dyn_value_settled(ScrDyn *value) { return value->kind == SCR_DYN_PROMISE ? scr_await_dyn(value->v.promise) : scr_dyn_retain(value); } +static ScrDyn *scr_dyn_promise_payload(ScrPromise *p); + +/* The same for a native stackless body: the suspension has taken the turn + * scr_await_dyn would take, so the extraction takes none. */ +ScrDyn *scr_coro_await_dyn_value(ScrDyn *value) { + if (value->kind != SCR_DYN_PROMISE) return scr_dyn_retain(value); + ScrPromise *p = value->v.promise; + if (!scr_promise_settled_now(p)) + scr_trap("scriptc: internal error: resumed before awaited promise settled\n"); + if (scr_promise_is_rejected(p)) return NULL; + return scr_dyn_promise_payload(p); +} + /* ── process warnings (emitWarning + the 'warning' event) ───────────── * Gated with the rest of this TU (a deprecation-emitting unit's gate * must imply the dynAsync link). Listeners are dyn functions; the diff --git a/packages/runtime/src/scr_builtin_values.c b/packages/runtime/src/scr_builtin_values.c index f8dd2b79b..084a1d50d 100644 --- a/packages/runtime/src/scr_builtin_values.c +++ b/packages/runtime/src/scr_builtin_values.c @@ -441,8 +441,7 @@ ScrDyn *scr_buffer_constructor_value(void) { ScrDyn *table = scr_dyn_function_properties(scr_bv_buffer); ScrDyn *getter = scr_dyn_new_func(scr_closure_new(NULL, 0), scr_bv_prototype, 0, "native:function", "get prototype"); scr_dyn_obj_set_accessor(table, "prototype", 9, getter, NULL); - ScrDynEntry *entry = &table->v.obj.entries[table->v.obj.len - 1]; - entry->enumerable = entry->configurable = false; + scr_dyn_obj_set_attrs_at(table, scr_dyn_obj_count(table) - 1, false, false, false); scr_dyn_obj_set(table, "poolSize", 8, scr_dyn_new_num(pool_size)); ScrDyn *is_encoding = NULL; for (size_t i = 0; i < sizeof statics / sizeof statics[0]; i++) { diff --git a/packages/runtime/src/scr_bytes_io.c b/packages/runtime/src/scr_bytes_io.c index daeb47c60..0c4bee4ee 100644 --- a/packages/runtime/src/scr_bytes_io.c +++ b/packages/runtime/src/scr_bytes_io.c @@ -1757,11 +1757,11 @@ ScrStr *scr_fs_mkdtemp_sync_chk(const ScrDyn *prefix, const ScrDyn *opts, const if (!scr_fs_encoding_chk(opts)) return NULL; bool utf8 = true; if (opts->kind == SCR_DYN_OBJ) { - for (size_t i = 0; i < opts->v.obj.len; i++) { - const ScrDynEntry *e = &opts->v.obj.entries[i]; - if (scr_fs_dyn_absent(e->value)) continue; - if (strcmp(e->key, "encoding") == 0) { - if (!scr_fs_utf8_encoding(e->value)) utf8 = false; + for (size_t i = 0; i < scr_dyn_obj_count(opts); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(opts, i); + if (scr_fs_dyn_absent(e.value)) continue; + if (strcmp(e.key, "encoding") == 0) { + if (!scr_fs_utf8_encoding(e.value)) utf8 = false; continue; } utf8 = false; /* an unmodeled effectful option */ diff --git a/packages/runtime/src/scr_console_native.c b/packages/runtime/src/scr_console_native.c index 492d57d82..e11ffd3af 100644 --- a/packages/runtime/src/scr_console_native.c +++ b/packages/runtime/src/scr_console_native.c @@ -507,8 +507,8 @@ static bool scr_console_holds_buffer(const ScrDyn *d) { if (scr_console_holds_buffer(d->v.arr.items[i])) return true; return false; case SCR_DYN_OBJ: - for (size_t i = 0; i < d->v.obj.len; i++) { - const ScrDyn *value = d->v.obj.entries[i].value; + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { + const ScrDyn *value = scr_dyn_obj_value_at(d, i); if (value && scr_console_holds_buffer(value)) return true; } return false; @@ -626,7 +626,7 @@ static ScrStr **scr_table_own_keys(const ScrDyn *item, size_t *count) { return keys; } if (item->kind != SCR_DYN_OBJ) return NULL; - size_t len = item->v.obj.len; + size_t len = scr_dyn_obj_count(item); const char **names = calloc(len ? len : 1, sizeof *names); size_t *lens = calloc(len ? len : 1, sizeof *lens); size_t *order = calloc(len ? len : 1, sizeof *order); @@ -634,10 +634,10 @@ static ScrStr **scr_table_own_keys(const ScrDyn *item, size_t *count) { if (!names || !lens || !order || !keys) scr_trap("scriptc: out of memory\n"); size_t n = 0; for (size_t i = 0; i < len; i++) { - const ScrDynEntry *ent = &item->v.obj.entries[i]; - if (!ent->enumerable) continue; - names[n] = ent->key; - lens[n] = ent->key_len; + ScrDynProp prop = scr_dyn_obj_prop_at(item, i); + if (!prop.enumerable) continue; + names[n] = prop.key; + lens[n] = prop.key_len; n++; } scr_table_key_order(names, lens, n, order); @@ -652,16 +652,14 @@ static ScrStr **scr_table_own_keys(const ScrDyn *item, size_t *count) { /* item[key] when it is an own property; NULL otherwise. +1. */ static ScrDyn *scr_table_own_value(const ScrDyn *item, const ScrStr *key) { if (item->kind == SCR_DYN_OBJ) { - for (size_t i = 0; i < item->v.obj.len; i++) { - const ScrDynEntry *ent = &item->v.obj.entries[i]; - if (ent->key_len != key->len || memcmp(ent->key, key->data, key->len) != 0) continue; - if (ent->accessor) { - scr_table_fence("console.table rows with accessor properties have no lowering"); - return NULL; - } - return scr_dyn_retain(ent->value); + size_t i = scr_dyn_obj_index_of(item, key->data, key->len); + if (i == SIZE_MAX) return NULL; + ScrDynProp prop = scr_dyn_obj_prop_at(item, i); + if (prop.accessor) { + scr_table_fence("console.table rows with accessor properties have no lowering"); + return NULL; } - return NULL; + return scr_dyn_retain(prop.value); } if (item->kind == SCR_DYN_ARR || item->kind == SCR_DYN_BYTES) { size_t len = item->kind == SCR_DYN_ARR ? item->v.arr.len : item->v.bytes->len; diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index d421e8e45..a7157f9af 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -199,6 +199,8 @@ typedef struct { * depth budget covers every participating destructor. Queued zero-count * objects still own their children until their callback runs. */ #define SCR_DESTROY_DEPTH 64 +/* How deep the collector's walks recurse (see "the walks" below). */ +#define SCR_CYC_WALK_DEPTH 64 #ifdef SCR_WORKERS /* Worker executables keep the collector's per-thread state in one * thread-local block: on Darwin every distinct thread-local costs a call @@ -218,7 +220,7 @@ typedef struct { ScrVec white; ScrVec xgen; ScrVec xgen_edges; - void *gray_holder; + void *gray_holders[SCR_CYC_WALK_DEPTH]; ScrVec pub_holes[SCR_CYC_NGENS]; unsigned gen_limit; size_t live_after[SCR_CYC_NGENS]; @@ -260,7 +262,7 @@ static _Thread_local ScrCycThread scr_cyc_thread = {.gen_limit = SCR_CYC_OLD}; #define scr_white (scr_cyc_thread.white) #define scr_xgen (scr_cyc_thread.xgen) #define scr_xgen_edges (scr_cyc_thread.xgen_edges) -#define scr_gray_holder (scr_cyc_thread.gray_holder) +#define scr_gray_holders (scr_cyc_thread.gray_holders) #define scr_pub_holes (scr_cyc_thread.pub_holes) #define scr_gen_limit (scr_cyc_thread.gen_limit) #define scr_cyc_live_after (scr_cyc_thread.live_after) @@ -475,11 +477,13 @@ static SCR_TL ScrVec scr_xgen; /* Edges from a gray object into a generation the pass does not walk, as * (holder, target) pairs, recorded by markGray (each gray object is traced - * there exactly once, so each edge appears once), and the object markGray is - * tracing. */ + * there exactly once, so each edge appears once), and the objects markGray is + * tracing, by walk depth: an edge's holder is the entry one level above the + * depth its visit sees. Nothing is restored after an object's children are + * traced, so markGray ends in a tail call to the trace. */ #ifndef SCR_WORKERS static SCR_TL ScrVec scr_xgen_edges; -static SCR_TL void *scr_gray_holder; +static SCR_TL void *scr_gray_holders[SCR_CYC_WALK_DEPTH]; #endif /* Objects above this generation are invisible to the pass in flight. */ @@ -1338,12 +1342,12 @@ void scr_cyc_on_release(void *obj) { ((child) == NULL || SCR_RC(child) == SCR_RC_IMMORTAL \ || SCR_CYC_WALK_GEN(scr_cyc_hdr(child)) > scr_gen_limit) -/* The walks are depth-first and recursive up to SCR_CYC_WALK_DEPTH, where - * an object is queued instead (scr_pending, or scr_restore_pending for the - * restoration, which can run inside scan) and traced from the queue. Trace - * callbacks forward that integer depth through their opaque context; it is - * never dereferenced. */ -#define SCR_CYC_WALK_DEPTH 64 +/* The walks are depth-first and recursive up to SCR_CYC_WALK_DEPTH (defined + * with the per-thread state, which it sizes), where an object is queued + * instead (scr_pending, or scr_restore_pending for the restoration, which + * can run inside scan) and traced from the queue. Trace callbacks forward + * that integer depth through their opaque context; it is never + * dereferenced. */ /* A walk visits a dense cycle-capable array's children itself: the present * dense elements, which are all scr_arr_trace_v visits when the array has @@ -1382,9 +1386,10 @@ SCR_CYC_ARRAY_WALK(scr_sb_array, scr_sb_visit) SCR_CYC_ARRAY_WALK(scr_scan_array, scr_scan_visit) static inline __attribute__((always_inline)) void scr_mark_gray(void *obj, uintptr_t depth); -/* An edge into a generation this pass does not walk, for settle. */ -static __attribute__((noinline)) void scr_mg_skip_edge(void *child) { - scr_cyc_push_work(&scr_xgen_edges, scr_gray_holder); +/* An edge into a generation this pass does not walk, for settle, seen at + * `depth`. */ +static __attribute__((noinline)) void scr_mg_skip_edge(void *child, uintptr_t depth) { + scr_cyc_push_work(&scr_xgen_edges, scr_gray_holders[depth - 1]); scr_cyc_push_work(&scr_xgen_edges, child); } /* The first visit of an object: gray it and walk its children. */ @@ -1398,7 +1403,7 @@ static void scr_mg_visit(void *child, void *ctx) { if (child == NULL || SCR_RC(child) == SCR_RC_IMMORTAL) return; ScrCycHdr *h = scr_cyc_hdr(child); if (SCR_CYC_WALK_GEN(h) > scr_gen_limit) { - scr_mg_skip_edge(child); + scr_mg_skip_edge(child, (uintptr_t)ctx); return; } --SCR_RC(child); @@ -1438,11 +1443,9 @@ static inline __attribute__((always_inline)) void scr_mark_gray(void *obj, uintp scr_cyc_push_work(&scr_pending, obj); return; } - void *holder = scr_gray_holder; - scr_gray_holder = obj; + scr_gray_holders[depth] = obj; const ScrCycType *type = scr_cyc_type(h); SCR_CYC_CHILDREN(obj, type, scr_mg_array, scr_mg_visit, depth + 1); - scr_gray_holder = holder; } /* markGray from one (black) candidate, through the queue. */ static void scr_mark_gray_from(void *root) { @@ -1450,7 +1453,7 @@ static void scr_mark_gray_from(void *root) { while (scr_pending.n) { void *obj = scr_pending.v[--scr_pending.n]; const ScrCycType *type = scr_cyc_type(scr_cyc_hdr(obj)); - scr_gray_holder = obj; + scr_gray_holders[0] = obj; SCR_CYC_CHILDREN(obj, type, scr_mg_array, scr_mg_visit, 1); } } @@ -1953,7 +1956,9 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit, bool doomed) { /* The list holds the dead set: doomed components, and what scan whitened * (black once a later outside root restored it), or, when the pass found * nothing externally referenced, every gray object. `doomed` says that - * every entry is a doomed component's already. */ + * every entry is a doomed component's: every candidate and every holder + * of a recorded edge is about to be freed, and the entries can still be + * gray (see "Components" in scr_cyc_pass). */ for (size_t i = 0; i < scr_white.n && !doomed; i++) { void *obj = scr_white.v[i]; ScrCycHdr *h = scr_cyc_hdr(obj); @@ -1974,7 +1979,7 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit, bool doomed) { * what changes them). A full pass skips no edge, so it pins nothing. */ scr_xgen.n = 0; for (size_t i = 0; i < scr_xgen_edges.n; i += 2) { - if (scr_cyc_color(scr_cyc_hdr(scr_xgen_edges.v[i])) != SCR_CYC_DOOMED) continue; + if (!doomed && scr_cyc_color(scr_cyc_hdr(scr_xgen_edges.v[i])) != SCR_CYC_DOOMED) continue; void *target = scr_xgen_edges.v[i + 1]; SCR_RC(target) += 1; /* pin across the teardowns */ scr_cyc_push_work(&scr_xgen, target); @@ -2005,7 +2010,7 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit, bool doomed) { * genuine outside reference and Bacon-Rajan's own reasoning retires the * candidate. Re-buffering costs a walk, never correctness: an object that * is truly live survives the next pass at its level and is dropped then. */ - if (gen_limit < SCR_CYC_OLD) { + if (gen_limit < SCR_CYC_OLD && !doomed) { for (size_t i = 0; i < scr_cands.n; i++) { void *obj = scr_cands.v[i]; ScrCycHdr *h = scr_cyc_hdr(obj); @@ -2113,6 +2118,17 @@ static size_t scr_cyc_pass_done(size_t freed) { return freed; } +/* Color the doomed components' entries of the gray list: every entry + * outside the referenced components' ranges. */ +static SCR_CYC_COMPACT void scr_cyc_doom_components(void) { + size_t from = 0; + for (size_t k = 0; k <= scr_live_ranges.n; k += 2) { + size_t end = k < scr_live_ranges.n ? (uintptr_t)scr_live_ranges.v[k] : scr_white.n; + for (size_t j = from; j < end; j++) scr_cyc_set_color(scr_cyc_hdr(scr_white.v[j]), SCR_CYC_DOOMED); + if (k < scr_live_ranges.n) from = (uintptr_t)scr_live_ranges.v[k + 1]; + } +} + static size_t scr_cyc_pass(unsigned gen_limit) { if (scr_collecting) return 0; scr_cyc_route_bypassed(); @@ -2193,9 +2209,15 @@ static size_t scr_cyc_pass(unsigned gen_limit) { * cannot add one: its objects would be referencing this component, and * those references are still counted. scan would whiten exactly this * set, so it is doomed at once and scan runs only over the components - * that are externally referenced, if any. A limited pass keeps the single - * whole-pass decision below (its untaken candidates cannot carry the - * component number). */ + * that are externally referenced, if any. A doomed component's entries + * stay gray, outside the referenced components' ranges, until something + * reads their color: settle colors them while it filters the list, and + * needs no colors when every component is doomed; a scan from the + * candidates colors them first (scr_cyc_doom_components); and with a + * dispose observer installed they are colored here, because a weak map's + * teardown tells a collected owner by its color. A limited pass keeps + * the single whole-pass decision below (its untaken candidates cannot + * carry the component number). */ bool split = scr_cyc_take == NULL; uint32_t comp = 0; for (size_t i = 0; i < scr_cands.n; i++) { @@ -2208,8 +2230,11 @@ static size_t scr_cyc_pass(unsigned gen_limit) { if (!split) continue; size_t end = scr_white.n; if (scr_comp_referenced == 0 && end < SCR_CYC_GRAY_LIST_MAX) { - for (size_t j = start; j < end; j++) scr_cyc_set_color(scr_cyc_hdr(scr_white.v[j]), SCR_CYC_DOOMED); - } else if (scr_live_ranges.n && (uintptr_t)scr_live_ranges.v[scr_live_ranges.n - 1] == start) { + if (scr_weak_dispose_hook) + for (size_t j = start; j < end; j++) scr_cyc_set_color(scr_cyc_hdr(scr_white.v[j]), SCR_CYC_DOOMED); + continue; + } + if (scr_live_ranges.n && (uintptr_t)scr_live_ranges.v[scr_live_ranges.n - 1] == start) { scr_live_ranges.v[scr_live_ranges.n - 1] = (void *)(uintptr_t)end; } else { scr_cyc_push_work(&scr_live_ranges, (void *)(uintptr_t)start); @@ -2255,7 +2280,9 @@ static size_t scr_cyc_pass(unsigned gen_limit) { } /* Otherwise scan walks from the candidates. Keep only the doomed - * components' entries; scan appends what it whitens. */ + * components' entries, colored first so that scan passes over them; scan + * appends what it whitens. */ + if (split) scr_cyc_doom_components(); if (!split) { scr_white.n = 0; } else if (scr_live_ranges.n) { diff --git a/packages/runtime/src/scr_decl_adopt.c b/packages/runtime/src/scr_decl_adopt.c index fc1c4456c..ffac1f802 100644 --- a/packages/runtime/src/scr_decl_adopt.c +++ b/packages/runtime/src/scr_decl_adopt.c @@ -63,10 +63,10 @@ static bool scr_decl_adopt_plain(const ScrDyn *n, const ScrDynPath *path, bool l else if (n->v.obj.source_access && !n->v.obj.source_identity) want = "a plain object"; else { - for (size_t i = 0; i < n->v.obj.len && !want; i++) { - const ScrDynEntry *e = &n->v.obj.entries[i]; - if (e->accessor) want = "a plain object (this value has accessor properties)"; - else if (!e->writable || !e->enumerable || !e->configurable) + for (size_t i = 0; i < scr_dyn_obj_count(n) && !want; i++) { + ScrDynProp e = scr_dyn_obj_prop_at(n, i); + if (e.accessor) want = "a plain object (this value has accessor properties)"; + else if (!e.writable || !e.enumerable || !e.configurable) want = "a plain object (this value has non-default property attributes)"; } } diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 98d4aef0b..3a1e83308 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -428,6 +428,13 @@ typedef struct ScrDynTypedRecordOps { * optional key can change after the record is boxed. NULL in every other * table. */ const struct ScrDynTypedRecordOps *(*select)(void *rec); + /* Stores `value` into present field `index` in place when it fits the + * field's type and answers true (also when the conversion itself threw, + * which has then raised the write's error); answers false with no side + * effects otherwise, leaving the snapshot path to raise its diagnostic. + * NULL when the program never writes through a dynamic view, and for a + * field whose storage it cannot replace (an inline union). */ + bool (*set)(void *rec, size_t index, const ScrDyn *value); } ScrDynTypedRecordOps; /* A record capsule's ops while it answers from the native record (no view * created yet), or NULL. */ @@ -514,9 +521,16 @@ struct ScrDyn { struct { ScrSym *value; void (*release)(ScrSym *); ScrStr *(*render)(ScrSym *); ScrStr *(*description)(ScrSym *); } symbol; struct { size_t len; size_t cap; ScrDyn **items; struct ScrDyn *properties; bool sealed; bool frozen; unsigned char *presence; struct ScrDynRegion *region; } arr; /* owned; NULL presence means dense; region: see ScrDynRegion */ struct { - size_t len; - size_t cap; - ScrDynEntry *entries; + size_t len; /* own string-keyed properties, in both modes */ + size_t cap; /* capacity of entries (dictionary mode) or slots (layout mode) */ + /* Layout mode (`layout` non-NULL): slots[i] holds the value of the + * property layout->keys[i] names, a writable, enumerable, + * configurable data property. Dictionary mode (`layout` NULL): + * entries hold keys, values and attributes. See ScrLayout. */ + union { + ScrDynEntry *entries; + struct ScrDyn **slots; + }; /* Optional identity of the typed record that produced this ordinary * snapshot. Empty records and callback interfaces can retain their * source across repeated static-to-dynamic crossings. */ @@ -524,7 +538,7 @@ struct ScrDyn { * interfaces may request a fresh snapshot with the true arm. */ void *source_identity; ScrDyn *(*source_access)(void *, bool materialize); - struct ScrDynObjectIndex *index; /* optional wide-object lookup; owns no values */ + struct ScrDynObjectIndex *index; /* dictionary mode: optional wide-object lookup; owns no values */ /* On a runtime-built class prototype (e.g. url.Url.prototype) and * on a compiled class's prototype object: the constructor name * (static or immortal text) util.inspect prints for objects @@ -533,19 +547,13 @@ struct ScrDyn { * exits, and those clear it before release; reuse of a freed object * resets it (scr_dyn_alloc). */ const char *class_name; - union { - struct ScrDynRegion *region; /* see ScrDynRegion */ - /* While the node is pooled: the key bytes it keeps for reuse, - * counted in the pool's key budget (scr_dyn_obj_drop_keys). */ - size_t cached_key_bytes; - }; - /* The dynamic tier's hidden class (scr_js_object.c): the transition - * tree node naming this object's key sequence, or NULL while it has - * none. Only programs with dynamic-tier code read or maintain it: - * their runtime is built with SCR_JS_TIER, where every allocation and - * every change to the key set resets it. Other programs never look - * at it, so it may hold anything there. */ - struct ScrJsShape *shape; + struct ScrDynRegion *region; /* see ScrDynRegion */ + /* The object's layout in layout mode, NULL in dictionary mode. The + * dynamic tier's inline caches key on it. */ + const struct ScrLayout *layout; + /* Layout mode: bit i is the collector's snapshot of whether slot i's + * value is traced (ScrDynEntry.value_traced in dictionary mode). */ + size_t slots_traced; } obj; /* owned */ /* SCR_DYN_FUNC: the boxed closure (owned) + its call descriptor. `sig` * and `name` are static compiler-emitted literals (never freed); name @@ -624,10 +632,155 @@ _Static_assert(offsetof(ScrDyn, v.typed_ref.casts) == 16 + 8 * sizeof(void *), _Static_assert(sizeof(((ScrDyn *)0)->v.arr) <= sizeof(((ScrDyn *)0)->v.typed_ref) && sizeof(((ScrDyn *)0)->v.obj) <= sizeof(((ScrDyn *)0)->v.typed_ref), "dyn region links must not grow the node"); -/* Dynamic-tier inline caches read the shape word in place (64-bit). */ -_Static_assert(sizeof(void *) != 8 || offsetof(ScrDyn, v.obj.shape) == 80, "v.obj.shape offset"); +/* Dynamic-tier inline caches read the layout word and slots in place (64-bit). */ +_Static_assert(sizeof(void *) != 8 || offsetof(ScrDyn, v.obj.layout) == 80, "v.obj.layout offset"); +_Static_assert(sizeof(void *) != 8 || offsetof(ScrDyn, v.obj.slots) == 32, "v.obj.slots offset"); _Static_assert(sizeof(void *) != 8 || offsetof(ScrDyn, v.obj.region) == 72, "v.obj.region offset"); +/* ── layouts ─────────────────────────────────────────────────────────── + * An ordinary object (SCR_DYN_OBJ) is in one of two modes. + * + * Layout mode: v.obj.layout names the object's key sequence, a node of the + * thread's interned transition tree, and v.obj.slots holds one value per + * key (8 bytes per property). Every property of a layout-mode object is a + * writable, enumerable, configurable data property; objects of one key + * sequence share one layout, so a layout pointer proves which key sits at + * each slot and which keys the object lacks (the dynamic tier's inline + * caches key on it). New objects start in layout mode at the thread's + * empty layout. + * + * Dictionary mode: v.obj.layout is NULL and v.obj.entries holds keys, + * values and attributes (ScrDynEntry). An object moves there, once and for + * good, when it gets an accessor, a non-default attribute, a delete, more + * than SCR_LAYOUT_MAX_KEYS keys, when the thread's layout budgets are + * spent, when it is published to other threads, and whenever code that + * reads entries directly touches it (scr_dyn_obj_dict_entries): the conversion + * is unobservable, so code that has not been taught layouts stays correct. + * + * Layouts belong to one thread and live until its runtime state is torn + * down; they are never owned by a parse region. Each holds one reference + * to its key's bytes (scr_json.c's shared key storage). */ +#define SCR_LAYOUT_MAX_KEYS (8 * sizeof(size_t)) + +typedef struct ScrLayoutKey { + char *key; /* shared key bytes; one reference belongs to the layout that added it */ + uint32_t len; + uint32_t hash; /* low bits of scr_key_hash */ +} ScrLayoutKey; + +typedef struct ScrLayout { + const ScrLayoutKey *keys; /* keys[0..count); shared along a chain of layouts */ + uint32_t count; + uint32_t lookup_mask; /* capacity - 1 of `lookup`; 0 while it has none */ + uint8_t *lookup; /* lazily built hash index: slot + 1, 0 = empty */ + const struct ScrLayout *parent; + struct ScrLayout *last_child; /* the most recent transition from here */ + struct ScrLayoutKeys *store; /* the key array `keys` lives in */ + struct ScrLayout *all_next; + bool owns_store; +} ScrLayout; + +/* The index of `key` in layout `layout`, or UINT32_MAX. */ +uint32_t scr_layout_find(const ScrLayout *layout, const char *key, size_t key_len); +/* The layout that appends `key` (borrowed bytes) to `layout`, created on + * first use; NULL past SCR_LAYOUT_MAX_KEYS or the thread's budgets. */ +const ScrLayout *scr_layout_add(const ScrLayout *layout, const char *key, size_t key_len); +/* The thread's empty layout. */ +const ScrLayout *scr_layout_root(void); +/* Moves a layout-mode object to dictionary mode (no-op otherwise). */ +void scr_dyn_to_dictionary(ScrDyn *object); + +/* v.obj.entries of an object in dictionary mode, converting a layout-mode + * object first: code that reads or writes entries directly goes through + * this. */ +static inline __attribute__((always_inline)) ScrDynEntry *scr_dyn_obj_dict_entries(const ScrDyn *object) { + if (__builtin_expect(object->v.obj.layout != NULL, 0)) scr_dyn_to_dictionary((ScrDyn *)object); + return object->v.obj.entries; +} + +/* scr_dyn_obj_key_at and scr_dyn_obj_value_at out of line, for emitted + * code (borrowed results). */ +const char *scr_dyn_member_key(const ScrDyn *object, size_t i); +size_t scr_dyn_member_key_len(const ScrDyn *object, size_t i); +ScrDyn *scr_dyn_member_value(const ScrDyn *object, size_t i); + +/* ── own properties of ordinary objects ────────────────────────────── + * The accessors are always inlined, so a binary that never sees an + * ordinary object links no out-of-line copies. Code outside the object core (scr_json.c and scr_js_object.c) reads and + * adjusts an SCR_DYN_OBJ's own string-keyed properties through these + * accessors, never through v.obj.entries or v.obj.slots, so they work in + * both modes. Index i runs over the properties in creation order, + * 0 <= i < scr_dyn_obj_count. Keys and values are borrowed: they stay + * valid until the object changes. */ +typedef struct { + const char *key; + size_t key_len; + ScrDyn *value; /* data property; the undefined value for an accessor */ + ScrDyn *getter; /* accessor property; NULL when absent */ + ScrDyn *setter; /* accessor property; NULL when absent */ + bool accessor; + bool writable; + bool enumerable; + bool configurable; +} ScrDynProp; + +static inline __attribute__((always_inline)) size_t scr_dyn_obj_count(const ScrDyn *o) { + return o->v.obj.len; +} + +static inline __attribute__((always_inline)) const char *scr_dyn_obj_key_at(const ScrDyn *o, size_t i, size_t *key_len) { + const ScrLayout *layout = o->v.obj.layout; + if (layout) { + *key_len = layout->keys[i].len; + return layout->keys[i].key; + } + *key_len = o->v.obj.entries[i].key_len; + return o->v.obj.entries[i].key; +} + +/* The value of data property i (borrowed); the undefined value for an + * accessor property, whose functions scr_dyn_obj_prop_at answers. */ +static inline __attribute__((always_inline)) ScrDyn *scr_dyn_obj_value_at(const ScrDyn *o, size_t i) { + return o->v.obj.layout ? o->v.obj.slots[i] : o->v.obj.entries[i].value; +} + +static inline __attribute__((always_inline)) ScrDynProp scr_dyn_obj_prop_at(const ScrDyn *o, size_t i) { + const ScrLayout *layout = o->v.obj.layout; + if (layout) + return (ScrDynProp){layout->keys[i].key, layout->keys[i].len, o->v.obj.slots[i], NULL, NULL, + false, true, true, true}; + const ScrDynEntry *e = &o->v.obj.entries[i]; + return (ScrDynProp){e->key, e->key_len, e->value, e->getter, e->setter, + e->accessor, e->writable, e->enumerable, e->configurable}; +} + +/* The index of own property `key`, or SIZE_MAX. A linear scan in + * dictionary mode: for code that looks up options and headers, not for hot + * property reads. */ +static inline size_t scr_dyn_obj_index_of(const ScrDyn *o, const char *key, size_t key_len) { + if (o->v.obj.layout) { + uint32_t i = scr_layout_find(o->v.obj.layout, key, key_len); + return i == UINT32_MAX ? SIZE_MAX : i; + } + for (size_t i = 0; i < o->v.obj.len; i++) { + const ScrDynEntry *e = &o->v.obj.entries[i]; + if (e->key_len == key_len && memcmp(e->key, key, key_len) == 0) return i; + } + return SIZE_MAX; +} + +/* Sets the attributes of property i (`writable` is ignored for an + * accessor property): builtin objects the runtime builds restrict their + * members this way right after adding them. */ +static inline void scr_dyn_obj_set_attrs_at(ScrDyn *o, size_t i, bool writable, bool enumerable, + bool configurable) { + if (o->v.obj.layout && writable && enumerable && configurable) return; + ScrDynEntry *e = &scr_dyn_obj_dict_entries(o)[i]; + if (!e->accessor) e->writable = writable; + e->enumerable = enumerable; + e->configurable = configurable; +} + /* ── intact parse regions ───────────────────────────────────────────── * Every heap dyn node carries a cycle header, so an ordinary release that * leaves an array or object alive buffers it as a possible cycle root, and @@ -792,6 +945,27 @@ ScrDyn *scr_dyn_get_own_property_symbols(const ScrDyn *value); /* The own property stored under a symbol key on an object (borrowed), or * NULL when it has none. */ ScrDynEntry *scr_dyn_own_symbol_entry(const ScrDyn *object, const ScrDyn *key); +/* The own symbol-keyed property `key` of `object` as a property view + * (scr_dyn_obj_prop_at; `key` and `key_len` name its private storage + * key); false when there is none. */ +static inline bool scr_dyn_own_symbol_prop(const ScrDyn *object, const ScrDyn *key, ScrDynProp *out) { + const ScrDynEntry *e = scr_dyn_own_symbol_entry(object, key); + if (!e) return false; + *out = (ScrDynProp){e->key, e->key_len, e->value, e->getter, e->setter, + e->accessor, e->writable, e->enumerable, e->configurable}; + return true; +} +/* Copies own property i of `source` to `target` (scr_dyn_copy_entry). */ +static inline void scr_dyn_copy_prop_at(ScrDyn *target, const ScrDyn *source, size_t i) { + if (source->v.obj.layout) { + ScrDynProp prop = scr_dyn_obj_prop_at(source, i); + ScrDynEntry entry = {(char *)prop.key, prop.key_len, prop.value, NULL, NULL, + false, false, false, true, true, true}; + scr_dyn_copy_entry(target, source, &entry); + return; + } + scr_dyn_copy_entry(target, source, &source->v.obj.entries[i]); +} /* The private storage name of a symbol key (+1). With it, * scr_dyn_symbol_init_own gives an object that has no symbol properties * yet its first one, a writable, enumerable, configurable data property @@ -1141,8 +1315,10 @@ bool scr_dyn_typed_ref_is( const ScrDyn *d, const char *type_key, size_t type_key_len); bool scr_dyn_typed_ref_is_key(const ScrDyn *d, const ScrStr *type_key); void *scr_dyn_typed_ref_unbox(const ScrDyn *d); /* +1 */ +void *scr_dyn_typed_ref_prefix_view(const ScrDyn *d); /* +1 or NULL */ ScrDyn *scr_dyn_class_view_unavailable(void *ptr); ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d); /* +1 */ +ScrDyn *scr_dyn_typed_ref_peek(const ScrDyn *d); /* +1 */ /* What string conversion renders for a capsule (+1): its materialized * view, or, for an EventTarget or Event subclass without its own * toString, the "[object EventTarget]" / "[object Event]" string of its @@ -1330,6 +1506,10 @@ bool scr_dyn_is_array_prototype(const ScrDyn *d); * non-object kinds throw Node's catchable TypeErrors (strict-mode * wording). All three operands BORROWED (the value is retained in). */ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value); +/* The same in sloppy-mode code: an assignment strict mode refuses with a + * TypeError is ignored. */ +void scr_dyn_key_set_sloppy(ScrDyn *recv, ScrStr *key, ScrDyn *value); +void scr_dyn_key_set_computed_sloppy(ScrDyn *recv, ScrDyn *key, ScrDyn *value); /* A named (non-index) property of an extensible, unfrozen ARR, stored as * scr_dyn_key_set stores it. Takes ownership of value. */ void scr_dyn_arr_named_put(ScrDyn *arr, const char *key, size_t key_len, ScrDyn *value); @@ -2027,7 +2207,7 @@ void scr_dyn_trace_v(void *d, ScrTraceVisit visit, void *ctx); long scr_dyn_live_count(void); #endif -/* Checked-object primitives the dynamic tier's shapes and caches build on +/* Checked-object primitives the dynamic tier's inline caches build on * (scr_js_object.c). They exist only in the SCR_JS_TIER build of * scr_json.c, which programs with dynamic-tier code link. */ /* The builtin Object.prototype node, or NULL before its first use. */ @@ -2035,12 +2215,9 @@ const ScrDyn *scr_dyn_tier_object_prototype(void); /* A read site's answer for a native array's length or a native iterator's * next (scr_js_get_probe_native): +1, or NULL. */ ScrDyn *scr_dyn_tier_unshaped_get(const ScrDyn *receiver, const ScrStr *key); -/* Shared key bytes for `key` (+1, NUL-terminated), and their release. */ -char *scr_dyn_tier_key_copy(const char *key, size_t key_len); -void scr_dyn_tier_key_release(char *key, size_t key_len); -/* Appends an own data property (writable, enumerable, configurable) that - * `obj` does not have, sharing `key`'s bytes and taking ownership of - * `value`, exactly as an ordinary insertion would; leaves the shape reset. */ -void scr_dyn_tier_append(ScrDyn *obj, char *key, size_t key_len, ScrDyn *value); +/* Appends to a layout-mode object the property that `next`, a child of + * its layout, adds, taking ownership of `value`, exactly as an ordinary + * insertion would. */ +void scr_dyn_tier_append(ScrDyn *obj, const ScrLayout *next, ScrDyn *value); #endif /* SCR_DYN_H */ diff --git a/packages/runtime/src/scr_dyn_handle_props.c b/packages/runtime/src/scr_dyn_handle_props.c index c7de963f0..88677836f 100644 --- a/packages/runtime/src/scr_dyn_handle_props.c +++ b/packages/runtime/src/scr_dyn_handle_props.c @@ -60,17 +60,17 @@ static ScrDyn *hp_props_of(void *child) { return props; } -static ScrDynEntry *hp_entry(const ScrDyn *props, const char *key, size_t key_len) { - if (!props) return NULL; - for (size_t i = 0; i < props->v.obj.len; i++) { - ScrDynEntry *e = &props->v.obj.entries[i]; - if (e->key_len == key_len && !memcmp(e->key, key, key_len)) return e; - } - return NULL; +/* Own property `key` of the table object, in *out; false when absent. */ +static bool hp_entry(const ScrDyn *props, const char *key, size_t key_len, ScrDynProp *out) { + if (!props) return false; + size_t i = scr_dyn_obj_index_of(props, key, key_len); + if (i == SIZE_MAX) return false; + *out = scr_dyn_obj_prop_at(props, i); + return true; } static bool hp_has(const ScrDyn *props, const char *key, size_t key_len) { - return hp_entry(props, key, key_len) != NULL; + return props && scr_dyn_obj_index_of(props, key, key_len) != SIZE_MAX; } /* A property the program added, read with the child as `this`: an @@ -78,10 +78,10 @@ static bool hp_has(const ScrDyn *props, const char *key, size_t key_len) { * it (`Object.defineProperty(child, "x", { get() { return this.pid; } })`). * +1, or NULL with the exception pending. */ static ScrDyn *hp_read(ScrDyn *props, ScrDyn *self, const char *key, size_t key_len) { - ScrDynEntry *e = hp_entry(props, key, key_len); - if (!e || !e->accessor) return scr_dyn_obj_read(props, key, key_len); - if (!e->getter) return scr_dyn_retain(scr_dyn_undefined()); - ScrDyn *getter = scr_dyn_retain(e->getter); + ScrDynProp e; + if (!hp_entry(props, key, key_len, &e) || !e.accessor) return scr_dyn_obj_read(props, key, key_len); + if (!e.getter) return scr_dyn_retain(scr_dyn_undefined()); + ScrDyn *getter = scr_dyn_retain(e.getter); scr_dyn_this_push_dyn(self); ScrDyn *result = scr_dyn_call(getter, NULL, 0, "getter"); scr_dyn_this_pop(); @@ -132,7 +132,8 @@ static ScrDyn *hp_get(void *h, const char *key, size_t key_len) { static bool hp_set(void *h, const char *key, size_t key_len, const ScrDyn *value) { ScrDyn *props = hp_find(h); - ScrDynEntry *entry = hp_entry(props, key, key_len); + ScrDynProp found; + const ScrDynProp *entry = hp_entry(props, key, key_len, &found) ? &found : NULL; if (entry && entry->accessor) { /* An accessor's setter runs on the child. Without one the write throws * the strict-mode TypeError, as the other handle getters do. */ diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index d28707d5d..1b30c451d 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -4821,14 +4821,14 @@ static ScrArr *sf_headers_convert_init(const ScrDyn *headers, bool *canonical_ou return converted; } if (headers->kind == SCR_DYN_OBJ) { - for (size_t i = 0; i < headers->v.obj.len; i++) { - const ScrDynEntry *e = &headers->v.obj.entries[i]; - ScrStr *raw_name = scr_str_new(e->key, e->key_len); + for (size_t i = 0; i < scr_dyn_obj_count(headers); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(headers, i); + ScrStr *raw_name = scr_str_new(e.key, e.key_len); ScrStr *name_bytes = sf_header_value_bytestring(raw_name); ScrStr *name = name_bytes ? sf_latin1_to_utf8(name_bytes) : NULL; scr_str_release(name_bytes); scr_str_release(raw_name); - ScrDyn *source = scr_dyn_retain(e->value); + ScrDyn *source = scr_dyn_retain(e.value); ScrStr *raw_value = name ? scr_dyn_string_coerce_js(source) : NULL; scr_dyn_release(source); ScrStr *value_bytes = raw_value @@ -6088,11 +6088,11 @@ static int8_t sf_light_header_form(const ScrDyn *headers, bool json_body, } int8_t form = 0; if (headers->kind == SCR_DYN_OBJ) { - ScrArr *out = scr_arr_new(SCR_ELEM_STR, 2 * headers->v.obj.len); + ScrArr *out = scr_arr_new(SCR_ELEM_STR, 2 * scr_dyn_obj_count(headers)); form = 3; - for (size_t i = 0; i < headers->v.obj.len && form == 3; i++) { - const ScrDynEntry *e = &headers->v.obj.entries[i]; - const ScrDyn *value = e->value; + for (size_t i = 0; i < scr_dyn_obj_count(headers) && form == 3; i++) { + ScrDynProp e = scr_dyn_obj_prop_at(headers, i); + const ScrDyn *value = e.value; size_t count = value->kind == SCR_DYN_ARR ? value->v.arr.len : 1; for (size_t j = 0; j < count; j++) { const ScrDyn *item = value->kind == SCR_DYN_ARR ? value->v.arr.items[j] : value; @@ -6106,7 +6106,7 @@ static int8_t sf_light_header_form(const ScrDyn *headers, bool json_body, form = 0; break; } - scr_arr_push_ref(out, scr_str_new(e->key, e->key_len)); + scr_arr_push_ref(out, scr_str_new(e.key, e.key_len)); scr_arr_push_ref(out, text); } } @@ -6666,27 +6666,27 @@ ScrPromise *scr_fetch_static(ScrStr *url, ScrDyn *init) { return sf_reject_now(promise, "fetch failed"); } if (init && init->kind == SCR_DYN_OBJ) { - for (size_t i = 0; i < init->v.obj.len; i++) { - const ScrDynEntry *entry = &init->v.obj.entries[i]; - bool supported = sf_name(entry->key, entry->key_len, "method") || - sf_name(entry->key, entry->key_len, "headers") || - sf_name(entry->key, entry->key_len, "body") || - sf_name(entry->key, entry->key_len, "duplex") || - sf_name(entry->key, entry->key_len, "redirect") || - sf_name(entry->key, entry->key_len, "signal"); + for (size_t i = 0; i < scr_dyn_obj_count(init); i++) { + ScrDynProp entry = scr_dyn_obj_prop_at(init, i); + bool supported = sf_name(entry.key, entry.key_len, "method") || + sf_name(entry.key, entry.key_len, "headers") || + sf_name(entry.key, entry.key_len, "body") || + sf_name(entry.key, entry.key_len, "duplex") || + sf_name(entry.key, entry.key_len, "redirect") || + sf_name(entry.key, entry.key_len, "signal"); bool recognized_unsupported = - sf_name(entry->key, entry->key_len, "cache") || - sf_name(entry->key, entry->key_len, "credentials") || - sf_name(entry->key, entry->key_len, "dispatcher") || - sf_name(entry->key, entry->key_len, "integrity") || - sf_name(entry->key, entry->key_len, "keepalive") || - sf_name(entry->key, entry->key_len, "mode") || - sf_name(entry->key, entry->key_len, "priority") || - sf_name(entry->key, entry->key_len, "referrer") || - sf_name(entry->key, entry->key_len, "referrerPolicy") || - sf_name(entry->key, entry->key_len, "window"); - if (!supported && recognized_unsupported && entry->value && - entry->value->kind != SCR_DYN_UNDEF) { + sf_name(entry.key, entry.key_len, "cache") || + sf_name(entry.key, entry.key_len, "credentials") || + sf_name(entry.key, entry.key_len, "dispatcher") || + sf_name(entry.key, entry.key_len, "integrity") || + sf_name(entry.key, entry.key_len, "keepalive") || + sf_name(entry.key, entry.key_len, "mode") || + sf_name(entry.key, entry.key_len, "priority") || + sf_name(entry.key, entry.key_len, "referrer") || + sf_name(entry.key, entry.key_len, "referrerPolicy") || + sf_name(entry.key, entry.key_len, "window"); + if (!supported && recognized_unsupported && entry.value && + entry.value->kind != SCR_DYN_UNDEF) { return sf_reject_now(promise, "fetch failed"); } } diff --git a/packages/runtime/src/scr_generator.h b/packages/runtime/src/scr_generator.h index c73ceb900..4f43892b4 100644 --- a/packages/runtime/src/scr_generator.h +++ b/packages/runtime/src/scr_generator.h @@ -60,6 +60,14 @@ void scr_gen_coro_finish(ScrFiber *self); ScrGen *scr_async_gen_new(void (*entry)(ScrFiber *, void *), void *argpack, void (*drop_args)(void *), void (*settle)(ScrGen *, ScrPromise *)); +/* An async generator whose body is a switched coroutine, as + * scr_gen_new_stackless's: its steps run on the resumer's stack (the loop's + * after an await), its awaits suspend through scr_coro_await_prepare and + * scr_coro_hop_prepare, and its yields store OUT through scr_coro_yield_*. + * Native targets only. */ +ScrGen *scr_async_gen_new_stackless(void (*entry)(ScrFiber *, void *), void *argpack, + void (*drop_args)(void *), + void (*settle)(ScrGen *, ScrPromise *)); /* Runs when return() reaches the generator before its first resume, with * the packed arguments still alive: an iterator helper closes its source. */ void scr_gen_set_unstarted_return(ScrGen *g, void (*hook)(void *)); @@ -74,8 +82,9 @@ void scr_gen_set_tag(ScrGen *g, const char *tag); void scr_gen_copy_tag(ScrGen *g, const ScrGen *from); const char *scr_gen_tag(const ScrGen *g); /* scr_gen_tag for units that also link without the generator runtime (a - * library links none): the generator runtime sets it at startup, and only - * then can a generator capsule exist. */ + * library links none): the generator runtime sets it when it creates a + * generator, so it is set whenever a generator capsule exists. Read it with + * a relaxed atomic load. */ extern const char *(*scr_gen_tag_reader)(const ScrGen *g); ScrGen *scr_gen_retain(ScrGen *g); void scr_gen_release(ScrGen *g); /* NULL-tolerant */ diff --git a/packages/runtime/src/scr_http.c b/packages/runtime/src/scr_http.c index 3fb0fdcba..0284c31c4 100644 --- a/packages/runtime/src/scr_http.c +++ b/packages/runtime/src/scr_http.c @@ -2047,10 +2047,10 @@ void scr_http_res_write_head_dyn(ScrHttpRes *r, double status, const ScrDyn *hea scr_dyn_arg_type_fail("headers", "an instance of Object", headers); return; } - for (size_t i = 0; i < headers->v.obj.len; i++) { - const ScrDynEntry *e = &headers->v.obj.entries[i]; - const ScrDyn *v = e->value; - ScrStr *name = scr_str_new(e->key, e->key_len); + for (size_t i = 0; i < scr_dyn_obj_count(headers); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(headers, i); + const ScrDyn *v = e.value; + ScrStr *name = scr_str_new(e.key, e.key_len); if (v->kind == SCR_DYN_STR) { scr_http_res_set_header(r, name, v->v.str); } else if (v->kind == SCR_DYN_NUM) { @@ -6200,10 +6200,10 @@ static bool scr_http_dynh_req_set(void *h, const char *key, size_t key_len, cons * consecutive same-name pairs). Numbers format through String(n). NULL = * a value this surface cannot honestly carry (exception pending). */ static ScrArr *scr_http_dynh_pairs(const ScrDyn *headers) { - ScrArr *pairs = scr_arr_new(SCR_ELEM_STR, headers->v.obj.len * 2); - for (size_t i = 0; i < headers->v.obj.len; i++) { - const ScrDynEntry *e = &headers->v.obj.entries[i]; - const ScrDyn *v = e->value; + ScrArr *pairs = scr_arr_new(SCR_ELEM_STR, scr_dyn_obj_count(headers) * 2); + for (size_t i = 0; i < scr_dyn_obj_count(headers); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(headers, i); + const ScrDyn *v = e.value; size_t reps = v->kind == SCR_DYN_ARR ? v->v.arr.len : 1; for (size_t k = 0; k < reps; k++) { const ScrDyn *one = v->kind == SCR_DYN_ARR ? v->v.arr.items[k] : v; @@ -6218,7 +6218,7 @@ static ScrArr *scr_http_dynh_pairs(const ScrDyn *headers) { scr_jb_puts(&b, "Invalid value \""); scr_jb_puts(&b, one->kind == SCR_DYN_UNDEF ? "undefined" : "object"); scr_jb_puts(&b, "\" for header \""); - for (size_t c = 0; c < e->key_len; c++) scr_jb_putc(&b, e->key[c]); + for (size_t c = 0; c < e.key_len; c++) scr_jb_putc(&b, e.key[c]); scr_jb_putc(&b, '"'); ScrStr *msg = scr_jb_finish(&b); scr_throw_error_msg_code(SCR_ERR_TYPE, msg->data, msg->len, "ERR_HTTP_INVALID_HEADER_VALUE"); @@ -6226,7 +6226,7 @@ static ScrArr *scr_http_dynh_pairs(const ScrDyn *headers) { scr_arr_release(pairs); return NULL; } - scr_arr_push_ref(pairs, scr_str_new(e->key, e->key_len)); + scr_arr_push_ref(pairs, scr_str_new(e.key, e.key_len)); scr_arr_push_ref(pairs, vs); } } @@ -6945,8 +6945,8 @@ static ScrDyn *scr_http_dynh_agent_get(void *h, const char *key, size_t key_len) scr_dyn_arr_push(arr, item); } /* the list is most recent first: reverse each array into Node's order */ - for (size_t i = 0; i < obj->v.obj.len; i++) { - ScrDyn *arr = obj->v.obj.entries[i].value; + for (size_t i = 0; i < scr_dyn_obj_count(obj); i++) { + ScrDyn *arr = scr_dyn_obj_value_at(obj, i); for (size_t l = 0, r = arr->v.arr.len ? arr->v.arr.len - 1 : 0; l < r; l++, r--) { ScrDyn *t = arr->v.arr.items[l]; arr->v.arr.items[l] = arr->v.arr.items[r]; @@ -7449,13 +7449,13 @@ bool scr_http_req_opts_read(const ScrDyn *opts, bool secure, const char *api, Sc headers = headers_view; } if (headers != NULL && headers->kind == SCR_DYN_OBJ) { - for (size_t i = 0; i < headers->v.obj.len; i++) { - const ScrDynEntry *e = &headers->v.obj.entries[i]; - ScrStr *name = scr_str_new(e->key, e->key_len); + for (size_t i = 0; i < scr_dyn_obj_count(headers); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(headers, i); + ScrStr *name = scr_str_new(e.key, e.key_len); ScrStr *label = scr_str_new("", 0); scr_http_validate_header_name(name, label); scr_str_release(label); - const ScrDyn *v = e->value; + const ScrDyn *v = e.value; if (!scr_exc_pending()) { if (v->kind == SCR_DYN_ARR) { for (size_t k = 0; k < v->v.arr.len && !scr_exc_pending(); k++) diff --git a/packages/runtime/src/scr_http2.c b/packages/runtime/src/scr_http2.c index 489946a12..30fddf8a4 100644 --- a/packages/runtime/src/scr_http2.c +++ b/packages/runtime/src/scr_http2.c @@ -2559,9 +2559,9 @@ ScrNetServer *scr_http2_create_secure_server_dyn(const ScrDyn *opts /*borrowed*/ ScrHttpReqFn fn) { bool allow_http1 = false; if (opts != NULL && opts->kind == SCR_DYN_OBJ) { - for (size_t i = 0; i < opts->v.obj.len; i++) { - if (strcmp(opts->v.obj.entries[i].key, "allowHTTP1") == 0) { - allow_http1 = scr_dyn_truthy(opts->v.obj.entries[i].value); + for (size_t i = 0; i < scr_dyn_obj_count(opts); i++) { + if (strcmp(scr_dyn_obj_prop_at(opts, i).key, "allowHTTP1") == 0) { + allow_http1 = scr_dyn_truthy(scr_dyn_obj_value_at(opts, i)); } } } @@ -3105,15 +3105,15 @@ static void scr_h2_dyn_fire_goaway(ScrClosure *cb, double code, double last) { /* Flatten a dyn headers object into the flat pairs array (numbers via * String(n); the :status/:method/... pseudo-headers pass through). */ static ScrArr *scr_h2_dom_to_pairs(const ScrDyn *headers) { - ScrArr *pairs = scr_arr_new(SCR_ELEM_STR, headers->v.obj.len * 2); - for (size_t i = 0; i < headers->v.obj.len; i++) { - const ScrDynEntry *e = &headers->v.obj.entries[i]; - const ScrDyn *v = e->value; + ScrArr *pairs = scr_arr_new(SCR_ELEM_STR, scr_dyn_obj_count(headers) * 2); + for (size_t i = 0; i < scr_dyn_obj_count(headers); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(headers, i); + const ScrDyn *v = e.value; ScrStr *vs; if (v->kind == SCR_DYN_STR) vs = scr_str_retain(v->v.str); else if (v->kind == SCR_DYN_NUM) vs = scr_f64_to_scrstr(v->v.num); else continue; /* undefined/null header: dropped, Node's stance */ - scr_arr_push_ref(pairs, scr_str_new(e->key, e->key_len)); + scr_arr_push_ref(pairs, scr_str_new(e.key, e.key_len)); scr_arr_push_ref(pairs, vs); } return pairs; diff --git a/packages/runtime/src/scr_inspect.c b/packages/runtime/src/scr_inspect.c index 1de583af5..e6df0f4cf 100644 --- a/packages/runtime/src/scr_inspect.c +++ b/packages/runtime/src/scr_inspect.c @@ -973,21 +973,23 @@ ScrStr *scr_insp_regex(ScrRegex *re) { * the k-th symbol key. */ static size_t insp_view_property_slots(const ScrDyn *props) { if (!props) return 0; - return props->v.obj.len + (props->symbol_keys ? props->symbol_keys->v.arr.len : 0); + return scr_dyn_obj_count(props) + (props->symbol_keys ? props->symbol_keys->v.arr.len : 0); } -static ScrStr *insp_accessor_tag(const ScrDynEntry *ent); +static ScrStr *insp_accessor_tag(const ScrDynProp *ent); static void insp_key_into(InspBuf *b, const char *key, size_t len); /* Slot k's `key: value` entry when it is enumerable, else NULL. */ static ScrStr *insp_view_property(ScrDyn *props, size_t k, double recurse, double depth) { - const ScrDynEntry *ent; + ScrDynProp found; + const ScrDynProp *ent = NULL; ScrStr *key = NULL; - if (k < props->v.obj.len) { - ent = &props->v.obj.entries[k]; + if (k < scr_dyn_obj_count(props)) { + found = scr_dyn_obj_prop_at(props, k); + ent = &found; } else { - ScrDyn *symbol = props->symbol_keys->v.arr.items[k - props->v.obj.len]; - ent = scr_dyn_own_symbol_entry(props, symbol); + ScrDyn *symbol = props->symbol_keys->v.arr.items[k - scr_dyn_obj_count(props)]; + if (scr_dyn_own_symbol_prop(props, symbol, &found)) ent = &found; if (ent && ent->enumerable) key = scr_insp_dyn(symbol, recurse, depth); } if (!ent || !ent->enumerable) return NULL; @@ -1005,10 +1007,11 @@ static ScrStr *insp_view_property(ScrDyn *props, size_t k, double recurse, doubl static size_t insp_view_property_count(ScrDyn *props) { size_t count = 0; for (size_t k = 0; k < insp_view_property_slots(props); k++) { - if (k < props->v.obj.len) count += props->v.obj.entries[k].enumerable; + if (k < scr_dyn_obj_count(props)) count += scr_dyn_obj_prop_at(props, k).enumerable; else { - const ScrDynEntry *ent = scr_dyn_own_symbol_entry(props, props->symbol_keys->v.arr.items[k - props->v.obj.len]); - count += ent && ent->enumerable; + ScrDynProp ent; + count += scr_dyn_own_symbol_prop(props, props->symbol_keys->v.arr.items[k - scr_dyn_obj_count(props)], &ent) && + ent.enumerable; } } return count; @@ -1157,11 +1160,11 @@ static ScrDyn *insp_error_cause(ScrError *e, bool *enumerable, ScrDyn **view) { ScrDyn *node = scr_errdyn_view_of(e); if (node != NULL && node->kind == SCR_DYN_TYPED_REF) node = *view = scr_dyn_typed_ref_materialize(node); if (node != NULL && node->kind == SCR_DYN_OBJ) { - for (size_t i = 0; i < node->v.obj.len; i++) { - ScrDynEntry *ent = &node->v.obj.entries[i]; - if (ent->key_len == 5 && memcmp(ent->key, "cause", 5) == 0) { - *enumerable = ent->enumerable; - return ent->value; + for (size_t i = 0; i < scr_dyn_obj_count(node); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(node, i); + if (ent.key_len == 5 && memcmp(ent.key, "cause", 5) == 0) { + *enumerable = ent.enumerable; + return ent.value; } } return NULL; @@ -1176,9 +1179,9 @@ static ScrDyn *insp_error_cause(ScrError *e, bool *enumerable, ScrDyn **view) { * other non-enumerable property of an error. */ bool scr_insp_error_bag_has_properties(ScrDyn *bag) { if (bag == NULL || bag->kind != SCR_DYN_OBJ) return false; - for (size_t i = 0; i < bag->v.obj.len; i++) { - const ScrDynEntry *ent = &bag->v.obj.entries[i]; - if (ent->enumerable && !(ent->key_len == 5 && memcmp(ent->key, "cause", 5) == 0)) return true; + for (size_t i = 0; i < scr_dyn_obj_count(bag); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(bag, i); + if (ent.enumerable && !(ent.key_len == 5 && memcmp(ent.key, "cause", 5) == 0)) return true; } return false; } @@ -1282,7 +1285,7 @@ ScrStr *scr_insp_error(ScrError *e, double recurse, double depth) { * (/^[a-zA-Z_][a-zA-Z_0-9]*$/, '__proto__' excepted), quoted otherwise. */ /* util.inspect's text for an accessor property: [Getter], [Setter] or * [Getter/Setter], by which of the two are functions. +1. */ -static ScrStr *insp_accessor_tag(const ScrDynEntry *ent) { +static ScrStr *insp_accessor_tag(const ScrDynProp *ent) { bool get = ent->getter && ent->getter->kind == SCR_DYN_FUNC; bool set = ent->setter && ent->setter->kind == SCR_DYN_FUNC; const char *text = get && set ? "[Getter/Setter]" : get ? "[Getter]" : set ? "[Setter]" : "undefined"; @@ -1371,12 +1374,12 @@ static ScrDyn *insp_assertion_ellipsis(ScrDyn *value) { * Node's `{ ... }` tail. */ /* An own non-enumerable entry of the error encoding, NULL when absent or * enumerable (an enumerable one prints with the other own keys). */ -static const ScrDynEntry *insp_error_hidden_entry(const ScrDyn *d, const char *key, size_t len) { - for (size_t i = 0; i < d->v.obj.len; i++) { - const ScrDynEntry *ent = &d->v.obj.entries[i]; - if (ent->key_len == len && memcmp(ent->key, key, len) == 0) return ent->enumerable ? NULL : ent; - } - return NULL; +static const ScrDynProp *insp_error_hidden_entry(const ScrDyn *d, const char *key, size_t len, + ScrDynProp *out) { + size_t i = scr_dyn_obj_index_of(d, key, len); + if (i == SIZE_MAX) return NULL; + *out = scr_dyn_obj_prop_at(d, i); + return out->enumerable ? NULL : out; } static ScrStr *scr_insp_dyn_error(ScrDyn *d, double recurse, double depth) { @@ -1390,19 +1393,20 @@ static ScrStr *scr_insp_dyn_error(ScrDyn *d, double recurse, double depth) { memcmp(name->v.str->data, "AssertionError", 14) == 0; double value_depth = assertion ? recurse : depth; size_t extra = 0; - for (size_t i = 0; i < d->v.obj.len; i++) { - ScrDynEntry *ent = &d->v.obj.entries[i]; - if (!ent->enumerable) continue; - if ((ent->key_len == 4 && memcmp(ent->key, "name", 4) == 0) || - (ent->key_len == 7 && memcmp(ent->key, "message", 7) == 0) || - (ent->key_len == 5 && memcmp(ent->key, "stack", 5) == 0)) + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(d, i); + if (!ent.enumerable) continue; + if ((ent.key_len == 4 && memcmp(ent.key, "name", 4) == 0) || + (ent.key_len == 7 && memcmp(ent.key, "message", 7) == 0) || + (ent.key_len == 5 && memcmp(ent.key, "stack", 5) == 0)) continue; extra++; } /* formatError shows a non-enumerable `cause`, and an AggregateError's * `errors` array, after the enumerable keys: `[cause]`, `[errors]`. */ - const ScrDynEntry *cause = insp_error_hidden_entry(d, "cause", 5); - const ScrDynEntry *errors = insp_error_hidden_entry(d, "errors", 6); + ScrDynProp cause_prop, errors_prop; + const ScrDynProp *cause = insp_error_hidden_entry(d, "cause", 5, &cause_prop); + const ScrDynProp *errors = insp_error_hidden_entry(d, "errors", 6, &errors_prop); if (errors && errors->value->kind != SCR_DYN_ARR) errors = NULL; extra += (cause != NULL) + (errors != NULL); InspBuf base = {0}; @@ -1426,20 +1430,20 @@ static ScrStr *scr_insp_dyn_error(ScrDyn *d, double recurse, double depth) { if (extra == 0) return base_str; scr_insp_begin(recurse + 1); scr_insp_seen_push(d); - for (size_t i = 0; i < d->v.obj.len; i++) { - ScrDynEntry *ent = &d->v.obj.entries[i]; - if (!ent->enumerable) continue; - if ((ent->key_len == 4 && memcmp(ent->key, "name", 4) == 0) || - (ent->key_len == 7 && memcmp(ent->key, "message", 7) == 0) || - (ent->key_len == 5 && memcmp(ent->key, "stack", 5) == 0)) + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(d, i); + if (!ent.enumerable) continue; + if ((ent.key_len == 4 && memcmp(ent.key, "name", 4) == 0) || + (ent.key_len == 7 && memcmp(ent.key, "message", 7) == 0) || + (ent.key_len == 5 && memcmp(ent.key, "stack", 5) == 0)) continue; - bool shortened = assertion && ((ent->key_len == 6 && memcmp(ent->key, "actual", 6) == 0) || - (ent->key_len == 8 && memcmp(ent->key, "expected", 8) == 0)); - ScrDyn *shown = shortened ? insp_assertion_ellipsis(ent->value) : scr_dyn_retain(ent->value); + bool shortened = assertion && ((ent.key_len == 6 && memcmp(ent.key, "actual", 6) == 0) || + (ent.key_len == 8 && memcmp(ent.key, "expected", 8) == 0)); + ScrDyn *shown = shortened ? insp_assertion_ellipsis(ent.value) : scr_dyn_retain(ent.value); ScrStr *val = scr_insp_dyn(shown, recurse + 1, value_depth); scr_dyn_release(shown); InspBuf eb = {0}; - insp_key_into(&eb, ent->key, ent->key_len); + insp_key_into(&eb, ent.key, ent.key_len); ib_cstr(&eb, ": "); ib_bytes(&eb, val->data, val->len); scr_str_release(val); @@ -1447,7 +1451,7 @@ static ScrStr *scr_insp_dyn_error(ScrDyn *d, double recurse, double depth) { scr_insp_entry(entry, false); scr_str_release(entry); } - const ScrDynEntry *hidden[] = {cause, errors}; + const ScrDynProp *hidden[] = {cause, errors}; for (size_t i = 0; i < 2; i++) { if (!hidden[i]) continue; ScrStr *val = scr_insp_dyn(hidden[i]->value, recurse + 1, value_depth); @@ -1609,7 +1613,7 @@ static bool scr_insp_is_generator(const ScrDyn *d) { } static ScrStr *scr_insp_generator(const ScrDyn *d) { - const char *tag = scr_gen_tag_reader(d->v.typed_ref.ptr); + const char *tag = __atomic_load_n(&scr_gen_tag_reader, __ATOMIC_RELAXED)(d->v.typed_ref.ptr); if (tag[0] == '!') { static const char message[] = "util.inspect of this iterator object is not supported yet"; scr_throw_error_msg(SCR_ERR_ERROR, message, sizeof message - 1); @@ -1690,9 +1694,9 @@ static ScrStr *scr_insp_dyn_collection(ScrDyn *d, double recurse, double depth, /* An own symbol-keyed property of `d` (borrowed) this render shows, or * NULL: enumerable ones, and under showHidden the non-enumerable ones too. */ -static ScrDynEntry *scr_insp_symbol_entry(ScrDyn *d, size_t i) { - ScrDynEntry *ent = scr_dyn_own_symbol_entry(d, d->symbol_keys->v.arr.items[i]); - return ent && (ent->enumerable || g_show_hidden) ? ent : NULL; +static bool scr_insp_symbol_entry(ScrDyn *d, size_t i, ScrDynProp *out) { + return scr_dyn_own_symbol_prop(d, d->symbol_keys->v.arr.items[i], out) && + (out->enumerable || g_show_hidden); } /* An ordinary object: `Name { key: value }` with its constructor name @@ -1700,15 +1704,21 @@ static ScrDynEntry *scr_insp_symbol_entry(ScrDyn *d, size_t i) { * enumerable string keys, then the own enumerable symbol keys. */ /* An own entry this render shows: enumerable ones, and under showHidden * the non-enumerable ones too (never the encoding's `%` markers). */ -static bool insp_obj_shows(const ScrDynEntry *ent) { +static bool insp_obj_shows(const ScrDynProp *ent) { return ent->enumerable || (g_show_hidden && !(ent->key_len && ent->key[0] == '%')); } static ScrStr *scr_insp_obj(ScrDyn *d, double recurse, double depth, const char *cls) { size_t shown = 0; - for (size_t i = 0; i < d->v.obj.len; i++) shown += insp_obj_shows(&d->v.obj.entries[i]); + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(d, i); + shown += insp_obj_shows(&ent); + } size_t nsymbols = d->symbol_keys ? d->symbol_keys->v.arr.len : 0; - for (size_t i = 0; i < nsymbols; i++) shown += scr_insp_symbol_entry(d, i) != NULL; + for (size_t i = 0; i < nsymbols; i++) { + ScrDynProp ent; + shown += scr_insp_symbol_entry(d, i, &ent); + } if (shown == 0) { if (cls) return scr_insp_class_tag(cls, " {}"); return d->null_proto ? scr_str_new("[Object: null prototype] {}", 27) @@ -1720,17 +1730,17 @@ static ScrStr *scr_insp_obj(ScrDyn *d, double recurse, double depth, const char : insp_styled_lit("[Object]", 8, INSP_SPECIAL); } scr_insp_begin(recurse + 1); - for (size_t i = 0; i < d->v.obj.len; i++) { - ScrDynEntry *ent = &d->v.obj.entries[i]; - if (!insp_obj_shows(ent)) continue; - ScrStr *val = ent->accessor ? insp_accessor_tag(ent) : scr_insp_dyn(ent->value, recurse + 1, depth); + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(d, i); + if (!insp_obj_shows(&ent)) continue; + ScrStr *val = ent.accessor ? insp_accessor_tag(&ent) : scr_insp_dyn(ent.value, recurse + 1, depth); InspBuf eb = {0}; - if (ent->enumerable) { - insp_key_into(&eb, ent->key, ent->key_len); + if (ent.enumerable) { + insp_key_into(&eb, ent.key, ent.key_len); } else { /* formatProperty's hidden key: the key's rendering in brackets. */ ib_char(&eb, '['); - insp_key_into(&eb, ent->key, ent->key_len); + insp_key_into(&eb, ent.key, ent.key_len); ib_char(&eb, ']'); } ib_cstr(&eb, ": "); @@ -1742,8 +1752,9 @@ static ScrStr *scr_insp_obj(ScrDyn *d, double recurse, double depth, const char if (scr_exc_pending()) break; } for (size_t i = 0; i < nsymbols; i++) { - ScrDynEntry *ent = scr_insp_symbol_entry(d, i); - if (!ent) continue; + ScrDynProp found; + if (!scr_insp_symbol_entry(d, i, &found)) continue; + const ScrDynProp *ent = &found; ScrStr *key = scr_insp_dyn(d->symbol_keys->v.arr.items[i], recurse + 1, depth); ScrStr *val = ent->accessor ? insp_accessor_tag(ent) : scr_insp_dyn(ent->value, recurse + 1, depth); InspBuf eb = {0}; @@ -2063,7 +2074,7 @@ static ScrDyn *insp_inspect_value(void) { ScrDyn *set = scr_dyn_new_func(scr_closure_new((void *)members[i], 0), insp_inspect_member_call, 1, "native:function", members[i]); scr_dyn_obj_set_accessor(table, members[i], strlen(members[i]), get, set); - table->v.obj.entries[table->v.obj.len - 1].enumerable = false; + scr_dyn_obj_set_attrs_at(table, scr_dyn_obj_count(table) - 1, true, false, true); } scr_dyn_release(table); } @@ -2100,15 +2111,15 @@ static ScrDyn *insp_user_options(double depth) { insp_opt_set(options, "getters", scr_dyn_new_bool(false)); insp_opt_set(options, "numericSeparator", scr_dyn_new_bool(false)); if (g_user_options && g_user_options->kind == SCR_DYN_OBJ) { - for (size_t i = 0; i < g_user_options->v.obj.len; i++) { - const ScrDynEntry *ent = &g_user_options->v.obj.entries[i]; - if (!ent->enumerable) continue; - ScrDyn *value = scr_dyn_obj_read((ScrDyn *)g_user_options, ent->key, ent->key_len); + for (size_t i = 0; i < scr_dyn_obj_count(g_user_options); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(g_user_options, i); + if (!ent.enumerable) continue; + ScrDyn *value = scr_dyn_obj_read((ScrDyn *)g_user_options, ent.key, ent.key_len); if (!value) { scr_dyn_release(options); return NULL; } - scr_dyn_obj_set(options, ent->key, ent->key_len, value); + scr_dyn_obj_set(options, ent.key, ent.key_len, value); } } return options; @@ -2343,9 +2354,12 @@ static ScrStr *scr_insp_dyn_func(ScrDyn *d, double recurse, double depth) { } size_t shown = 0; if (table && table->kind == SCR_DYN_OBJ) { - for (size_t i = 0; i < table->v.obj.len; i++) shown += table->v.obj.entries[i].enumerable; + for (size_t i = 0; i < scr_dyn_obj_count(table); i++) shown += scr_dyn_obj_prop_at(table, i).enumerable; size_t nsymbols = table->symbol_keys ? table->symbol_keys->v.arr.len : 0; - for (size_t i = 0; i < nsymbols; i++) shown += scr_insp_symbol_entry(table, i) != NULL; + for (size_t i = 0; i < nsymbols; i++) { + ScrDynProp ent; + shown += scr_insp_symbol_entry(table, i, &ent); + } } if (shown == 0) { scr_dyn_release(table); @@ -2498,7 +2512,10 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { * own `length` comes first, and non-enumerable properties show too. */ const ScrDyn *props = d->v.arr.properties; size_t nprops = 0; - for (size_t i = 0; props && i < props->v.obj.len; i++) nprops += insp_obj_shows(&props->v.obj.entries[i]); + for (size_t i = 0; props && i < scr_dyn_obj_count(props); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(props, i); + nprops += insp_obj_shows(&ent); + } /* An instance of a class that extends Array: Node's `Name(length) ` * prefix (scr_arr_copy_metadata names the class). */ const char *cls = props ? props->v.obj.class_name : NULL; @@ -2562,15 +2579,15 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { scr_str_release(s); entries++; } - for (size_t k = 0; props && k < props->v.obj.len; k++) { - const ScrDynEntry *ent = &props->v.obj.entries[k]; - if (!insp_obj_shows(ent)) continue; - ScrStr *val = ent->accessor ? insp_accessor_tag(ent) : scr_insp_dyn(ent->value, recurse + 1, depth); + for (size_t k = 0; props && k < scr_dyn_obj_count(props); k++) { + ScrDynProp ent = scr_dyn_obj_prop_at(props, k); + if (!insp_obj_shows(&ent)) continue; + ScrStr *val = ent.accessor ? insp_accessor_tag(&ent) : scr_insp_dyn(ent.value, recurse + 1, depth); InspBuf eb = {0}; /* formatProperty's hidden key: the key's rendering in brackets. */ - if (!ent->enumerable) ib_char(&eb, '['); - insp_key_into(&eb, ent->key, ent->key_len); - if (!ent->enumerable) ib_char(&eb, ']'); + if (!ent.enumerable) ib_char(&eb, '['); + insp_key_into(&eb, ent.key, ent.key_len); + if (!ent.enumerable) ib_char(&eb, ']'); ib_cstr(&eb, ": "); ib_bytes(&eb, val->data, val->len); scr_str_release(val); @@ -2753,8 +2770,8 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { scr_str_release(iso); ScrStr *prefix = ib_take(&head); bool fields = false; - for (size_t i = 0; i < materialized->v.obj.len && !fields; i++) - fields = materialized->v.obj.entries[i].enumerable; + for (size_t i = 0; i < scr_dyn_obj_count(materialized) && !fields; i++) + fields = scr_dyn_obj_prop_at(materialized, i).enumerable; if (!fields || recurse > depth) { scr_dyn_release(materialized); return prefix; @@ -3323,12 +3340,12 @@ bool scr_insp_format_read_options(const ScrDyn *options, ScrInspFormatOptions *o "maxArrayLength", "maxStringLength", "breakLength", "compact", "sorted", "getters", "numericSeparator", "stylize", }; - for (size_t i = 0; i < options->v.obj.len && !o->user_options; i++) { - const ScrDynEntry *ent = &options->v.obj.entries[i]; - if (!ent->enumerable) continue; + for (size_t i = 0; i < scr_dyn_obj_count(options) && !o->user_options; i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(options, i); + if (!ent.enumerable) continue; bool read = false; for (size_t k = 0; k < sizeof known / sizeof *known && !read; k++) - read = ent->key_len == strlen(known[k]) && memcmp(ent->key, known[k], ent->key_len) == 0; + read = ent.key_len == strlen(known[k]) && memcmp(ent.key, known[k], ent.key_len) == 0; if (!read) o->user_options = options; } } @@ -3436,10 +3453,10 @@ static ScrStr *insp_table_cell(ScrDyn *v) { } const char *base = v->null_proto ? "[Object: null prototype]" : "Object"; size_t n = 0; - for (size_t i = 0; i < v->v.obj.len; i++) { - ScrDynEntry *ent = &v->v.obj.entries[i]; - if (!ent->enumerable) continue; - if (ent->accessor) { + for (size_t i = 0; i < scr_dyn_obj_count(v); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(v, i); + if (!ent.enumerable) continue; + if (ent.accessor) { static const char message[] = "console.table cells holding accessor properties have no lowering"; scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); return scr_str_new("", 0); @@ -3463,12 +3480,12 @@ static ScrStr *insp_table_cell(ScrDyn *v) { } ScrStr *items[2]; size_t count = 0; - for (size_t i = 0; i < v->v.obj.len; i++) { - ScrDynEntry *ent = &v->v.obj.entries[i]; - if (!ent->enumerable) continue; - ScrStr *val = scr_insp_dyn(ent->value, 1, 0); + for (size_t i = 0; i < scr_dyn_obj_count(v); i++) { + ScrDynProp ent = scr_dyn_obj_prop_at(v, i); + if (!ent.enumerable) continue; + ScrStr *val = scr_insp_dyn(ent.value, 1, 0); InspBuf eb = {0}; - insp_key_into(&eb, ent->key, ent->key_len); + insp_key_into(&eb, ent.key, ent.key_len); ib_cstr(&eb, ": "); ib_str(&eb, val); scr_str_release(val); diff --git a/packages/runtime/src/scr_js.h b/packages/runtime/src/scr_js.h index 5abab2053..e3c37560d 100644 --- a/packages/runtime/src/scr_js.h +++ b/packages/runtime/src/scr_js.h @@ -180,45 +180,37 @@ ScrJsVal scr_js_increment(ScrJsVal value, bool increment); double scr_js_to_number(ScrJsVal value); bool scr_js_truthy(ScrJsVal value); -/* ── shapes and inline caches (scr_js_object.c) ───────────────────────── - * A checked object's hidden class (ScrDyn.v.obj.shape) names its key - * sequence: a node of the thread's transition tree, whose depth is the - * entry count and whose last key sits at entry index depth - 1. Shapes are - * per thread and live until the thread's runtime state is torn down; a - * thread's shape count and the key bytes they hold are bounded, and an - * object whose key sequence would need a shape past either bound stays - * unshaped, as do immortal objects, which threads share. The tier - * computes an object's shape on a cache miss; the SCR_JS_TIER build of - * scr_json.c resets it on every change to the key set. - * Attributes stay in the entries and the prototype in the node: cache hits - * test both live, so a shape only ever proves which key sits at which entry - * index and which keys an object lacks. +/* ── inline caches (scr_js_object.c) ─────────────────────────────────── + * Caches key on object layouts (scr_dyn.h): a layout-mode object's layout + * names its key sequence, and all its properties are writable data + * properties, so a layout proves which key sits at which slot and which + * keys the object lacks. Dictionary-mode objects (layout NULL), including + * immortal objects, which threads share, never hit. The prototype stays in + * the node: cache hits test it live. * * Each property site of compiled dynamic-tier code owns one zeroed, * thread-local ScrJsIc with static storage: the reset that frees a - * thread's shapes (at thread exit, and when a library session ends) clears + * thread's layouts (at thread exit, and when a library session ends) clears * every record that holds entries. The code at the site is fixed: it tests * entry 0 inline and calls a probe for everything else. Probes never run * user code: they answer from plain data properties or return a miss * (NULL / false) with nothing done, and the site then runs its generic * path. * - * load entry: shape = receiver shape; target = NULL for an own property - * (slot indexes the receiver's entries), else the shape of - * `holder`, the receiver's prototype, whose entry `slot` holds it. - * store entry: shape = receiver shape before the store; target == shape - * replaces entry `slot`; any other target adds the key as entry - * `slot` and moves the receiver to `target`. An add requires the - * receiver's prototype to be `holder` (NULL or an object whose - * shape is holder_shape and that lacks the key). - * keyed entry: slot only; a hit compares the key bytes stored there. */ -typedef struct ScrJsShape ScrJsShape; - + * load entry: layout = receiver layout; target = NULL for an own + * property (slot indexes the receiver's slots), else the layout of + * `holder`, the receiver's prototype, whose slot `slot` holds it. + * store entry: layout = receiver layout before the store; target == + * layout replaces slot `slot`; any other target adds the key as + * slot `slot` and moves the receiver to `target`. An add requires + * the receiver's prototype to be `holder` (NULL or an object whose + * layout is holder_layout and that lacks the key). + * keyed entry: slot only; a hit compares the key stored there. */ typedef struct ScrJsIcEntry { - const ScrJsShape *shape; - const ScrJsShape *target; + const ScrLayout *layout; + const ScrLayout *target; const ScrDyn *holder; - const ScrJsShape *holder_shape; + const ScrLayout *holder_layout; size_t slot; } ScrJsIcEntry; diff --git a/packages/runtime/src/scr_js_object.c b/packages/runtime/src/scr_js_object.c index 4cee9e7e8..4d94a4e7e 100644 --- a/packages/runtime/src/scr_js_object.c +++ b/packages/runtime/src/scr_js_object.c @@ -1,19 +1,15 @@ -/* scriptc runtime — the dynamic tier's hidden classes and inline caches - * (scr_js.h; two-tier design stage S4). +/* scriptc runtime — the dynamic tier's inline caches (scr_js.h; two-tier + * design stage S4). * - * Shapes are attached to the existing checked objects (ScrDyn OBJ nodes) - * rather than replacing their entry storage: static code and the shared - * runtime read those entries in place, so a second object representation - * would have to be understood or converted at every boundary. A shape is a - * node of the thread's transition tree over key sequences; an object's - * shape proves which key sits at each entry index and which keys the - * object lacks. Everything else a cache hit depends on is tested live: the - * node kind, the accessor and writable bits of the entry it uses, and the - * prototype pointer. + * Caches key on object layouts (scr_dyn.h): a layout-mode object's layout + * proves which key sits at each slot and which keys the object lacks, and + * every property of such an object is a writable data property, so a hit + * tests the layout word and, for prototype hits, the prototype pointer. + * Dictionary-mode objects (layout NULL) never hit and take the generic + * paths. * * Linked only into programs with dynamic-tier code (the dynTier runtime - * feature), together with the SCR_JS_TIER build of scr_json.c, which - * resets an object's shape on every change to its key set. */ + * feature). */ #include "scr_js.h" #include "scr_key.h" @@ -24,61 +20,26 @@ * backend/llvm/js-ic.ts). */ #if SCR_JS_IMMEDIATES _Static_assert(offsetof(ScrDyn, kind) == 8, "kind offset"); -_Static_assert(offsetof(ScrDyn, v.obj.len) == 16 && offsetof(ScrDyn, v.obj.entries) == 32, - "object entry offsets"); +_Static_assert(offsetof(ScrDyn, v.obj.len) == 16 && offsetof(ScrDyn, v.obj.slots) == 32 && + offsetof(ScrDyn, v.obj.layout) == 80, "object slot offsets"); _Static_assert(offsetof(ScrDyn, v.arr.len) == 16 && offsetof(ScrDyn, v.arr.items) == 32 && offsetof(ScrDyn, v.arr.presence) == 56, "array element offsets"); _Static_assert(offsetof(ScrDyn, prototype) == 96, "prototype offset"); -_Static_assert(sizeof(ScrDynEntry) == 48 && offsetof(ScrDynEntry, value) == 16 && - offsetof(ScrDynEntry, writable) == 41, "entry layout"); _Static_assert(sizeof(ScrJsIcEntry) == 40 && sizeof(ScrJsIc) == 168, "cache layout"); #endif -/* Objects with more entries stay unshaped (dictionary mode): their reads - * take the generic path and its hash index. */ -#define SCR_JS_SHAPE_MAX 64 - -/* Shapes live until the thread exits or a library session ends, so a thread - * creates at most this many, holding at most this many key bytes. Once - * either budget is spent, objects whose key sequence has no shape yet stay - * unshaped; shapes that exist keep serving. */ -#define SCR_JS_SHAPE_BUDGET ((size_t)1 << 16) -#define SCR_JS_SHAPE_KEY_BYTES_BUDGET ((size_t)4 << 20) - -struct ScrJsShape { - const ScrJsShape *parent; - char *key; /* one reference to shared key bytes (scr_json.c) */ - size_t key_len; - uint64_t hash; - uint32_t count; /* entries: the depth in the tree */ - ScrJsShape *all_next; -}; - -/* The root is the empty key sequence. Transitions live in one open - * addressing table per thread keyed by (parent, key). */ -static SCR_TL ScrJsShape scr_js_root; -static SCR_TL ScrJsShape *scr_js_all_shapes; -static SCR_TL ScrJsShape **scr_js_transitions; -static SCR_TL size_t scr_js_transition_mask; /* capacity - 1; 0 when absent */ -static SCR_TL size_t scr_js_transition_count; -static SCR_TL size_t scr_js_shape_key_bytes; -static SCR_TL bool scr_js_shapes_registered; - -/* The cache records that hold entries. Entries name shapes and prototype - * objects, so the reset that frees the shapes also clears these records: a - * library's next session runs the same sites with the same records. */ +/* The cache records that hold entries. Entries name layouts and prototype + * objects, so the reset of a thread's runtime state (which frees its + * layouts) also clears these records: a library's next session runs the + * same sites with the same records. */ static SCR_TL ScrJsIc **scr_js_filled_ics; static SCR_TL size_t scr_js_filled_count; static SCR_TL size_t scr_js_filled_capacity; - -/* The shape word of an object whose key sequence has no shape within the - * budgets. No cache entry names it, so its accesses take the probes, - * which answer with a miss until the key set changes. */ -static ScrJsShape scr_js_unshaped; +static SCR_TL bool scr_js_ics_registered; /* The second-level cache of load sites whose own record is full: one entry - * per (receiver shape, key bytes) slot, filled by the probe and checked like - * a site entry. It names shapes, so the shape reset clears it. */ + * per (receiver layout, key bytes) slot, filled by the probe and checked + * like a site entry. It names layouts, so the reset clears it. */ #define SCR_JS_SHARED_LOADS 2048 typedef struct ScrJsSharedLoad { const char *key; @@ -86,7 +47,7 @@ typedef struct ScrJsSharedLoad { } ScrJsSharedLoad; static SCR_TL ScrJsSharedLoad *scr_js_shared_loads; -static void scr_js_shapes_reset(void) { +static void scr_js_ics_reset(void) { for (size_t i = 0; i < scr_js_filled_count; i++) memset(scr_js_filled_ics[i], 0, sizeof *scr_js_filled_ics[i]); free(scr_js_shared_loads); @@ -95,130 +56,33 @@ static void scr_js_shapes_reset(void) { scr_js_filled_ics = NULL; scr_js_filled_count = 0; scr_js_filled_capacity = 0; - while (scr_js_all_shapes) { - ScrJsShape *shape = scr_js_all_shapes; - scr_js_all_shapes = shape->all_next; - scr_dyn_tier_key_release(shape->key, shape->key_len); - free(shape); - } - free(scr_js_transitions); - scr_js_transitions = NULL; - scr_js_transition_mask = 0; - scr_js_transition_count = 0; - scr_js_shape_key_bytes = 0; - scr_js_shapes_registered = false; + scr_js_ics_registered = false; } -static size_t scr_js_transition_slot(const ScrJsShape *parent, uint64_t hash) { - return (size_t)scr_key_mix(hash ^ (uint64_t)(uintptr_t)parent); -} - -static void scr_js_transition_insert(ScrJsShape *child) { - size_t i = scr_js_transition_slot(child->parent, child->hash) & scr_js_transition_mask; - while (scr_js_transitions[i]) i = (i + 1) & scr_js_transition_mask; - scr_js_transitions[i] = child; -} - -static void scr_js_transitions_grow(void) { - size_t old_cap = scr_js_transition_mask ? scr_js_transition_mask + 1 : 0; - size_t cap = old_cap ? old_cap * 2 : 256; - ScrJsShape **old = scr_js_transitions; - scr_js_transitions = calloc(cap, sizeof *scr_js_transitions); - if (!scr_js_transitions) scr_trap("scriptc: out of memory\n"); - scr_js_transition_mask = cap - 1; - for (size_t i = 0; i < old_cap; i++) - if (old[i]) scr_js_transition_insert(old[i]); - free(old); -} - -/* The child of `parent` for appending `key` (borrowed bytes), created on - * first use; NULL when creating it would exceed a budget. */ -static const ScrJsShape *scr_js_transition(const ScrJsShape *parent, const char *key, size_t key_len) { - uint64_t hash = scr_key_hash(key, key_len); - if (scr_js_transition_mask) { - size_t i = scr_js_transition_slot(parent, hash) & scr_js_transition_mask; - for (ScrJsShape *child; (child = scr_js_transitions[i]); i = (i + 1) & scr_js_transition_mask) { - if (child->parent == parent && child->hash == hash && child->key_len == key_len && - scr_key_equal(child->key, key, key_len)) - return child; - } - } - if (scr_js_transition_count >= SCR_JS_SHAPE_BUDGET || - key_len > SCR_JS_SHAPE_KEY_BYTES_BUDGET - scr_js_shape_key_bytes) - return NULL; - if ((scr_js_transition_count + 1) * 2 > (scr_js_transition_mask ? scr_js_transition_mask + 1 : 0)) - scr_js_transitions_grow(); - ScrJsShape *child = malloc(sizeof *child); - if (!child) scr_trap("scriptc: out of memory\n"); - *child = (ScrJsShape){parent, scr_dyn_tier_key_copy(key, key_len), key_len, hash, parent->count + 1, - scr_js_all_shapes}; - scr_js_all_shapes = child; - if (!scr_js_shapes_registered) { - scr_js_shapes_registered = true; - scr_atexit(scr_js_shapes_reset); - } - scr_js_transition_insert(child); - scr_js_transition_count++; - scr_js_shape_key_bytes += key_len; - return child; -} - -/* An object's shape, computed from its entries when it has none; NULL for - * objects that stay unshaped (other kinds, immortal objects, the builtin - * Object.prototype, dictionary-sized objects, key sequences past the - * budgets). Immortal objects (rc == SIZE_MAX: published graphs, which - * threads share by pointer) are never shaped: shapes belong to one thread - * and die with it, and no thread writes an immortal object's header. */ -#ifdef SCR_JS_VERIFY -/* Debugging aid (build the runtime with -DSCR_JS_VERIFY and compile with - * SCRIPTC_JS_IC_WAYS=0 so every access probes): traps when an object's - * shape no longer names its key sequence, i.e. when some mutation of the - * key set missed its SCR_JS_SHAPE_RESET. */ -static void scr_js_verify(const ScrDyn *object, const ScrJsShape *shape) { - size_t len = object->v.obj.len; - if (shape->count != len) scr_trap("scriptc: stale shape (entry count)\n"); - for (const ScrJsShape *s = shape; s->parent; s = s->parent) { - const ScrDynEntry *entry = &object->v.obj.entries[s->count - 1]; - if (entry->key_len != s->key_len || !scr_key_equal(entry->key, s->key, s->key_len)) - scr_trap("scriptc: stale shape (key)\n"); - } -} -#endif - -static const ScrJsShape *scr_js_shape_of(ScrDyn *object) { +/* An object's layout, or NULL for objects caches leave alone: other kinds, + * dictionary-mode objects (immortal objects, which threads share, are + * always in dictionary mode), and the builtin Object.prototype. */ +static inline const ScrLayout *scr_js_layout_of(const ScrDyn *object) { if (object->kind != SCR_DYN_OBJ || object->rc == SIZE_MAX) return NULL; - const ScrJsShape *shape = object->v.obj.shape; - if (shape == &scr_js_unshaped) return NULL; -#ifdef SCR_JS_VERIFY - if (shape) scr_js_verify(object, shape); -#endif - if (shape) return shape; - size_t len = object->v.obj.len; - if (len > SCR_JS_SHAPE_MAX || object == scr_dyn_tier_object_prototype()) return NULL; - shape = &scr_js_root; - for (size_t i = 0; i < len; i++) { - ScrDynEntry *entry = &object->v.obj.entries[i]; - shape = scr_js_transition(shape, entry->key, entry->key_len); - if (!shape) { - object->v.obj.shape = &scr_js_unshaped; - return NULL; - } - } - object->v.obj.shape = (ScrJsShape *)shape; - return shape; + const ScrLayout *layout = object->v.obj.layout; + if (!layout || object == scr_dyn_tier_object_prototype()) return NULL; + return layout; } -/* The entry index of `key` in an object of shape `shape`, or SIZE_MAX. */ -static size_t scr_js_shape_find(const ScrJsShape *shape, const char *key, size_t key_len) { - for (; shape->parent; shape = shape->parent) - if (shape->key_len == key_len && scr_key_equal(shape->key, key, key_len)) return shape->count - 1; - return SIZE_MAX; +/* The slot of `key` in an object of layout `layout`, or SIZE_MAX. */ +static size_t scr_js_layout_find(const ScrLayout *layout, const char *key, size_t key_len) { + uint32_t slot = scr_layout_find(layout, key, key_len); + return slot == UINT32_MAX ? SIZE_MAX : slot; } /* Fills a cache entry: `stale` (a matching entry whose hit failed) when * there is one, else a free or replaceable one. */ static void scr_js_ic_add(ScrJsIc *ic, uint32_t stale, ScrJsIcEntry entry) { if (ic->count == 0) { + if (!scr_js_ics_registered) { + scr_js_ics_registered = true; + scr_atexit(scr_js_ics_reset); + } if (scr_js_filled_count == scr_js_filled_capacity) { size_t capacity = scr_js_filled_capacity ? scr_js_filled_capacity * 2 : 64; ScrJsIc **grown = realloc(scr_js_filled_ics, capacity * sizeof *grown); @@ -243,26 +107,25 @@ static void scr_js_ic_add(ScrJsIc *ic, uint32_t stale, ScrJsIcEntry entry) { /* ── loads ────────────────────────────────────────────────────────────── */ -/* The cached answer of an entry whose receiver shape matched, or NULL. */ +/* The cached answer of an entry whose receiver layout matched, or NULL. */ static ScrDyn *scr_js_load_hit(ScrDyn *receiver, const ScrJsIcEntry *entry) { const ScrDyn *holder = receiver; if (entry->target) { holder = entry->holder; if (receiver->prototype != holder || holder->kind != SCR_DYN_OBJ || - holder->v.obj.shape != entry->target) + holder->v.obj.layout != entry->target) return NULL; } - const ScrDynEntry *found = &holder->v.obj.entries[entry->slot]; - return found->accessor ? NULL : scr_dyn_retain(found->value); + return scr_dyn_retain(holder->v.obj.slots[entry->slot]); } /* An absent entry: its target is this tag and its holder is a placeholder * that is never a prototype, so the inline test of entry 0 in compiled code * (a prototype hit needs the receiver's prototype to be the holder) always - * sends such a receiver to the probe. Its holder_shape and slot name the one - * prototype the read walked past (shape and object), or NULL and 0 when the - * receiver had no prototype. */ -static const ScrJsShape scr_js_absent; + * sends such a receiver to the probe. Its holder_layout and slot name the + * one prototype the read walked past (layout and object), or NULL and 0 when + * the receiver had no prototype. */ +static const ScrLayout scr_js_absent; static const ScrDyn scr_js_absent_holder; /* Keys whose read on a plain object is not just its [[Get]]: the members of @@ -288,7 +151,7 @@ static bool scr_js_absent_cacheable(const ScrStr *key) { return true; } -static ScrJsSharedLoad *scr_js_shared_load(const ScrJsShape *shape, const ScrStr *key) { +static ScrJsSharedLoad *scr_js_shared_load(const ScrLayout *shape, const ScrStr *key) { if (!scr_js_shared_loads) { scr_js_shared_loads = calloc(SCR_JS_SHARED_LOADS, sizeof *scr_js_shared_loads); if (!scr_js_shared_loads) scr_trap("scriptc: out of memory\n"); @@ -305,7 +168,7 @@ static bool scr_js_absent_hit(const ScrDyn *receiver, const ScrJsIcEntry *entry) const ScrDyn *walked = (const ScrDyn *)entry->slot; if (!walked) return prototype == NULL; return prototype == walked && walked->kind == SCR_DYN_OBJ && - walked->v.obj.shape == entry->holder_shape && walked->prototype == NULL; + walked->v.obj.layout == entry->holder_layout && walked->prototype == NULL; } /* The answer of a matching entry for `receiver`, or NULL when the entry no @@ -344,11 +207,11 @@ static void scr_js_load_fill(ScrJsIc *ic, uint32_t stale, ScrJsSharedLoad *share * methods to the slow path, which names the missing function. */ static __attribute__((noinline)) ScrDyn *scr_js_probe_load(ScrDyn *receiver, const ScrStr *key, ScrJsIc *ic, bool absent) { - const ScrJsShape *shape = scr_js_shape_of(receiver); + const ScrLayout *shape = scr_js_layout_of(receiver); if (!shape) return NULL; uint32_t stale = UINT32_MAX; for (uint32_t i = 0; i < ic->count; i++) { - if (ic->entry[i].shape != shape) continue; + if (ic->entry[i].layout != shape) continue; ScrDyn *value = scr_js_entry_answer(receiver, &ic->entry[i]); if (value) return value; if (stale == UINT32_MAX) stale = i; @@ -356,18 +219,16 @@ static __attribute__((noinline)) ScrDyn *scr_js_probe_load(ScrDyn *receiver, con ScrJsSharedLoad *shared = NULL; if (ic->count == SCR_JS_IC_WAYS) { shared = scr_js_shared_load(shape, key); - if (shared->key == key->data && shared->entry.shape == shape && + if (shared->key == key->data && shared->entry.layout == shape && (absent || shared->entry.target != &scr_js_absent)) { ScrDyn *value = scr_js_entry_answer(receiver, &shared->entry); if (value) return value; } } - size_t slot = scr_js_shape_find(shape, key->data, key->len); + size_t slot = scr_js_layout_find(shape, key->data, key->len); if (slot != SIZE_MAX) { - const ScrDynEntry *found = &receiver->v.obj.entries[slot]; - if (found->accessor) return NULL; scr_js_load_fill(ic, stale, shared, key, (ScrJsIcEntry){shape, NULL, NULL, NULL, slot}); - return scr_dyn_retain(found->value); + return scr_dyn_retain(receiver->v.obj.slots[slot]); } absent = absent && scr_js_absent_cacheable(key); /* One prototype link: an ordinary object other than the builtin @@ -379,9 +240,9 @@ static __attribute__((noinline)) ScrDyn *scr_js_probe_load(ScrDyn *receiver, con (ScrJsIcEntry){shape, &scr_js_absent, &scr_js_absent_holder, NULL, 0}); return scr_dyn_retain(scr_dyn_undefined()); } - const ScrJsShape *holder_shape = scr_js_shape_of(holder); + const ScrLayout *holder_shape = scr_js_layout_of(holder); if (!holder_shape) return NULL; - slot = scr_js_shape_find(holder_shape, key->data, key->len); + slot = scr_js_layout_find(holder_shape, key->data, key->len); if (slot == SIZE_MAX) { if (!absent || holder->prototype != NULL) return NULL; scr_js_load_fill(ic, stale, shared, key, @@ -389,13 +250,11 @@ static __attribute__((noinline)) ScrDyn *scr_js_probe_load(ScrDyn *receiver, con (size_t)holder}); return scr_dyn_retain(scr_dyn_undefined()); } - const ScrDynEntry *found = &holder->v.obj.entries[slot]; - if (found->accessor) return NULL; scr_js_load_fill(ic, stale, shared, key, (ScrJsIcEntry){shape, holder_shape, holder, NULL, slot}); - return scr_dyn_retain(found->value); + return scr_dyn_retain(holder->v.obj.slots[slot]); } -/* Receivers other than ordinary objects have no shape: they miss before +/* Receivers other than ordinary objects have no layout: they miss before * the probe sets up its frame. */ ScrDyn *scr_js_load_probe(ScrDyn *receiver, const ScrStr *key, ScrJsIc *ic) { if (receiver->kind != SCR_DYN_OBJ) return NULL; @@ -427,62 +286,56 @@ static bool scr_js_error_synced_key(const ScrStr *key) { } static bool scr_js_replace(ScrDyn *receiver, size_t slot, ScrDyn *value) { - ScrDynEntry *found = &receiver->v.obj.entries[slot]; - if (found->accessor || !found->writable || found->fenced) return false; scr_dyn_region_store(receiver, value); - ScrDyn *old = found->value; - found->value = scr_dyn_retain(value); + ScrDyn **found = &receiver->v.obj.slots[slot]; + ScrDyn *old = *found; + *found = scr_dyn_retain(value); scr_dyn_release(old); return true; } /* Whether an add may skip the generic path's prototype-chain walk: the * chain is empty, or one ordinary object that lacks the key and ends it. */ -static bool scr_js_add_chain(const ScrDyn *prototype, const ScrJsShape *prototype_shape, +static bool scr_js_add_chain(const ScrDyn *prototype, const ScrLayout *prototype_shape, const ScrStr *key) { if (!prototype) return true; return prototype_shape && !prototype->prototype && - scr_js_shape_find(prototype_shape, key->data, key->len) == SIZE_MAX; + scr_js_layout_find(prototype_shape, key->data, key->len) == SIZE_MAX; } -static bool scr_js_append_shaped(ScrDyn *receiver, const ScrJsShape *target, ScrDyn *value) { +static bool scr_js_append_shaped(ScrDyn *receiver, const ScrLayout *target, ScrDyn *value) { if (receiver->non_extensible) return false; - scr_dyn_tier_append(receiver, target->key, target->key_len, scr_dyn_retain(value)); - receiver->v.obj.shape = (ScrJsShape *)target; + scr_dyn_tier_append(receiver, target, scr_dyn_retain(value)); return true; } bool scr_js_store_probe(ScrDyn *receiver, const ScrStr *key, ScrDyn *value, ScrJsIc *ic) { - const ScrJsShape *shape = scr_js_shape_of(receiver); + const ScrLayout *shape = scr_js_layout_of(receiver); if (!shape || scr_js_error_synced_key(key)) return false; uint32_t stale = UINT32_MAX; for (uint32_t i = 0; i < ic->count; i++) { const ScrJsIcEntry *entry = &ic->entry[i]; - if (entry->shape != shape) continue; + if (entry->layout != shape) continue; if (entry->target == shape) { if (scr_js_replace(receiver, entry->slot, value)) return true; } else if (receiver->prototype == entry->holder && (!entry->holder || (entry->holder->kind == SCR_DYN_OBJ && - entry->holder->v.obj.shape == entry->holder_shape && + entry->holder->v.obj.layout == entry->holder_layout && !entry->holder->prototype))) { if (scr_js_append_shaped(receiver, entry->target, value)) return true; } stale = i; break; } - size_t slot = scr_js_shape_find(shape, key->data, key->len); + size_t slot = scr_js_layout_find(shape, key->data, key->len); if (slot != SIZE_MAX) { - const ScrDynEntry *found = &receiver->v.obj.entries[slot]; - if (found->accessor || !found->writable || found->fenced) return false; scr_js_ic_add(ic, stale, (ScrJsIcEntry){shape, shape, NULL, NULL, slot}); return scr_js_replace(receiver, slot, value); } ScrDyn *prototype = receiver->prototype; - const ScrJsShape *prototype_shape = prototype ? scr_js_shape_of(prototype) : NULL; - if (!scr_js_add_chain(prototype, prototype_shape, key) || receiver->non_extensible || - shape->count >= SCR_JS_SHAPE_MAX) - return false; - const ScrJsShape *target = scr_js_transition(shape, key->data, key->len); + const ScrLayout *prototype_shape = prototype ? scr_js_layout_of(prototype) : NULL; + if (!scr_js_add_chain(prototype, prototype_shape, key) || receiver->non_extensible) return false; + const ScrLayout *target = scr_layout_add(shape, key->data, key->len); if (!target) return false; scr_js_ic_add(ic, stale, (ScrJsIcEntry){shape, target, prototype, prototype_shape, target->count - 1}); return scr_js_append_shaped(receiver, target, value); @@ -505,20 +358,18 @@ ScrDyn *scr_js_keyed_probe(ScrDyn *receiver, const ScrDyn *key, ScrJsIc *ic) { if (key->kind == SCR_DYN_NUM) return scr_js_index_probe(receiver, key->v.num); if (key->kind != SCR_DYN_STR || receiver->kind != SCR_DYN_OBJ) return NULL; const ScrStr *name = key->v.str; + const ScrLayout *shape = scr_js_layout_of(receiver); + if (!shape) return NULL; size_t len = receiver->v.obj.len; for (uint32_t i = 0; i < ic->count; i++) { size_t slot = ic->entry[i].slot; if (slot >= len) continue; - const ScrDynEntry *found = &receiver->v.obj.entries[slot]; - if (found->key_len == name->len && scr_key_equal(found->key, name->data, name->len)) - return found->accessor ? NULL : scr_dyn_retain(found->value); + const ScrLayoutKey *found = &shape->keys[slot]; + if (found->len == name->len && scr_key_equal(found->key, name->data, name->len)) + return scr_dyn_retain(receiver->v.obj.slots[slot]); } - const ScrJsShape *shape = scr_js_shape_of(receiver); - if (!shape) return NULL; - size_t slot = scr_js_shape_find(shape, name->data, name->len); + size_t slot = scr_js_layout_find(shape, name->data, name->len); if (slot == SIZE_MAX) return NULL; - const ScrDynEntry *found = &receiver->v.obj.entries[slot]; - if (found->accessor) return NULL; scr_js_ic_add(ic, UINT32_MAX, (ScrJsIcEntry){NULL, NULL, NULL, NULL, slot}); - return scr_dyn_retain(found->value); + return scr_dyn_retain(receiver->v.obj.slots[slot]); } diff --git a/packages/runtime/src/scr_js_object.test.c b/packages/runtime/src/scr_js_object.test.c index 06b804759..02368ca74 100644 --- a/packages/runtime/src/scr_js_object.test.c +++ b/packages/runtime/src/scr_js_object.test.c @@ -5,7 +5,7 @@ #include #include -/* Mirrors scr_js_object.c's budgets. */ +/* Mirrors scr_json.c's layout budgets. */ #define SHAPE_BUDGET ((size_t)1 << 16) #define KEY_BYTES_BUDGET ((size_t)4 << 20) @@ -63,7 +63,8 @@ static void budget_bounds_shapes(size_t key_len, size_t budget) { scr_str_release(key); scr_dyn_release(object); } - assert(shaped == budget); + /* Objects the earlier checks built hold a few layouts already. */ + assert(shaped <= budget && shaped + 64 > budget); /* A key sequence shaped before the budget ran out. */ ScrStr *first = key_of(bytes, key_len, 0); @@ -73,7 +74,7 @@ static void budget_bounds_shapes(size_t key_len, size_t budget) { /* A store probe that would add a new shape leaves the add to the generic * store; one that replaces through an existing shape still runs. */ memset(&store_ic, 0, sizeof store_ic); - ScrStr *added = scr_str_new("added", 5); + ScrStr *added = key_of(bytes, key_len, budget + 1000); ScrDyn *value = scr_dyn_new_num(8); assert(!scr_js_store_probe(object, added, value, &store_ic)); assert(object->v.obj.len == 1); @@ -94,11 +95,11 @@ static void immortal_objects_stay_unshaped(void) { ScrDyn *object = object_with(key, 3); object->rc = SIZE_MAX; assert(!probe_reads(object, key, 3)); - assert(object->v.obj.shape == NULL); + assert(object->v.obj.len == 1); memset(&store_ic, 0, sizeof store_ic); ScrDyn *value = scr_dyn_new_num(4); assert(!scr_js_store_probe(object, key, value, &store_ic)); - assert(object->v.obj.shape == NULL); + assert(object->v.obj.len == 1); scr_dyn_release(value); scr_str_release(key); } diff --git a/packages/runtime/src/scr_js_object.test.ts b/packages/runtime/src/scr_js_object.test.ts index 1fb90cdaa..505bd4062 100644 --- a/packages/runtime/src/scr_js_object.test.ts +++ b/packages/runtime/src/scr_js_object.test.ts @@ -7,7 +7,7 @@ import { expect, test } from "vitest"; const exec = promisify(execFile); -test("dynamic-tier shapes stay within the shape and key-byte budgets, and absent reads are cached", async () => { +test("layouts stay within the layout and key-byte budgets, and dynamic-tier absent reads are cached", async () => { const scratch = await mkdtemp(join(tmpdir(), "scriptc-js-object-")); const src = import.meta.dirname; const binary = join(scratch, "js-object"); @@ -29,6 +29,7 @@ test("dynamic-tier shapes stay within the shape and key-byte budgets, and absent "scr_json.c", "scr_string.c", "scr_array.c", + "scr_arena.c", "scr_map.c", "scr_exception.c", "scr_error.c", diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index d0526f538..eb40100cf 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -24,16 +24,6 @@ #include "scr_runtime.h" #include "scr_key.h" -/* Programs with dynamic-tier code link the SCR_JS_TIER build of this unit: - * its object nodes carry the tier's hidden class (ScrDyn.v.obj.shape), and - * every allocation and change to an object's key set resets it so the - * tier recomputes it on its next cache miss. Other programs link the - * ordinary build, where this expands to nothing. */ -#ifdef SCR_JS_TIER -#define SCR_JS_SHAPE_RESET(d) ((d)->v.obj.shape = NULL) -#else -#define SCR_JS_SHAPE_RESET(d) ((void)0) -#endif static SCR_TL ScrDyn *scr_builtin_object_prototype; @@ -842,7 +832,7 @@ static size_t scr_dyn_key_slot(const char *key, size_t length) { return hash % SCR_DYN_KEY_SHARE_SLOTS; } -static char *scr_dyn_key_copy(const char *key, size_t length) { +static inline __attribute__((always_inline)) char *scr_dyn_key_copy(const char *key, size_t length) { ScrDynKeySlot *slot = length <= SCR_DYN_KEY_SHARE_LENGTH ? &scr_dyn_keys[scr_dyn_key_slot(key, length)] : NULL; if (slot && slot->key && slot->length == length && @@ -877,6 +867,246 @@ static void scr_dyn_key_release(char *key, size_t length) { free(owned); } +static inline void scr_dyn_key_retain(char *key) { + ((ScrDynKey *)(key - offsetof(ScrDynKey, bytes)))->rc++; +} + +/* ── layouts (scr_dyn.h) ─────────────────────────────────────────────── + * The thread's transition tree: the root is the empty key sequence, and a + * child appends one key to its parent's sequence. Transitions live in one + * open addressing table keyed by (parent, key); each parent also remembers + * its most recent child, which is all a loop building records of one + * shape ever needs. A chain of layouts shares one key array: a child + * appends its key to its parent's array when nothing was appended past the + * parent yet, and copies the parent's keys into a new array otherwise. + * + * A thread creates at most SCR_LAYOUT_BUDGET layouts holding at most + * SCR_LAYOUT_KEY_BYTES_BUDGET key bytes (and key arrays of at most + * SCR_LAYOUT_ARRAY_BYTES_BUDGET); past either, objects whose key sequence + * has no layout yet move to dictionary mode, and layouts that exist keep + * serving. Everything is freed with the thread's runtime state (at thread + * exit and when a library session ends). */ +#define SCR_LAYOUT_BUDGET ((size_t)1 << 16) +#define SCR_LAYOUT_KEY_BYTES_BUDGET ((size_t)4 << 20) +#define SCR_LAYOUT_ARRAY_BYTES_BUDGET ((size_t)16 << 20) +/* Layouts with more keys than this find keys through a hash index. */ +#define SCR_LAYOUT_LINEAR_MAX 8 + +typedef struct ScrLayoutKeys { + uint32_t used; + uint32_t cap; + ScrLayoutKey items[]; +} ScrLayoutKeys; + +static SCR_TL ScrLayout scr_layout_empty; +static SCR_TL ScrLayout *scr_layout_all; +static SCR_TL ScrLayout **scr_layout_transitions; +static SCR_TL size_t scr_layout_transition_mask; /* capacity - 1; 0 when absent */ +static SCR_TL size_t scr_layout_count; +static SCR_TL size_t scr_layout_key_bytes; +static SCR_TL size_t scr_layout_array_bytes; +static SCR_TL bool scr_layout_registered; + +const ScrLayout *scr_layout_root(void) { return &scr_layout_empty; } + +static void scr_layout_reset(void) { + while (scr_layout_all) { + ScrLayout *layout = scr_layout_all; + scr_layout_all = layout->all_next; + const ScrLayoutKey *key = &layout->keys[layout->count - 1]; + scr_dyn_key_release(key->key, key->len); + /* Layouts sharing a key array are newer than the one that owns it. */ + if (layout->owns_store) free(layout->store); + free(layout->lookup); + free(layout); + } + free(scr_layout_empty.lookup); + scr_layout_empty = (ScrLayout){0}; + free(scr_layout_transitions); + scr_layout_transitions = NULL; + scr_layout_transition_mask = 0; + scr_layout_count = 0; + scr_layout_key_bytes = 0; + scr_layout_array_bytes = 0; + scr_layout_registered = false; +} + +#ifdef SCR_RC_AUDIT +/* For the runtime's unit tests: frees the layouts while no object uses one. */ +void scr_layout_reset_idle(void) { scr_layout_reset(); } +#endif + +static size_t scr_layout_slot(const ScrLayout *parent, uint32_t hash) { + return (size_t)scr_key_mix((uint64_t)hash ^ (uint64_t)(uintptr_t)parent); +} + +static void scr_layout_insert(ScrLayout *child) { + size_t i = scr_layout_slot(child->parent, child->keys[child->count - 1].hash) & scr_layout_transition_mask; + while (scr_layout_transitions[i]) i = (i + 1) & scr_layout_transition_mask; + scr_layout_transitions[i] = child; +} + +static void scr_layout_transitions_grow(void) { + size_t old_cap = scr_layout_transition_mask ? scr_layout_transition_mask + 1 : 0; + size_t cap = old_cap ? old_cap * 2 : 256; + ScrLayout **old = scr_layout_transitions; + scr_layout_transitions = calloc(cap, sizeof *scr_layout_transitions); + if (!scr_layout_transitions) scr_json_oom(); + scr_layout_transition_mask = cap - 1; + for (size_t i = 0; i < old_cap; i++) + if (old[i]) scr_layout_insert(old[i]); + free(old); +} + +static inline bool scr_layout_key_is(const ScrLayoutKey *k, const char *key, size_t key_len) { + return k->len == key_len && (k->key == key || scr_key_equal(k->key, key, key_len)); +} + +static __attribute__((noinline)) void scr_layout_index(ScrLayout *layout) { + size_t cap = 16; + while (cap < 2 * (size_t)layout->count) cap *= 2; + uint8_t *table = calloc(cap, 1); + if (!table) scr_json_oom(); + for (uint32_t i = 0; i < layout->count; i++) { + size_t slot = layout->keys[i].hash & (cap - 1); + while (table[slot]) slot = (slot + 1) & (cap - 1); + table[slot] = (uint8_t)(i + 1); + } + layout->lookup = table; + layout->lookup_mask = (uint32_t)(cap - 1); +} + +uint32_t scr_layout_find(const ScrLayout *layout, const char *key, size_t key_len) { + uint32_t count = layout->count; + if (count <= SCR_LAYOUT_LINEAR_MAX) { + for (uint32_t i = 0; i < count; i++) + if (scr_layout_key_is(&layout->keys[i], key, key_len)) return i; + return UINT32_MAX; + } + if (!layout->lookup) scr_layout_index((ScrLayout *)layout); + uint32_t hash = (uint32_t)scr_key_hash(key, key_len); + size_t mask = layout->lookup_mask; + for (size_t slot = hash & mask;; slot = (slot + 1) & mask) { + uint8_t found = layout->lookup[slot]; + if (!found) return UINT32_MAX; + const ScrLayoutKey *k = &layout->keys[found - 1]; + if (k->hash == hash && scr_layout_key_is(k, key, key_len)) return found - 1u; + } +} + +const ScrLayout *scr_layout_add(const ScrLayout *layout, const char *key, size_t key_len) { + ScrLayout *parent = (ScrLayout *)layout; + ScrLayout *last = parent->last_child; + if (last && scr_layout_key_is(&last->keys[last->count - 1], key, key_len)) return last; + if (parent->count >= SCR_LAYOUT_MAX_KEYS || key_len > UINT32_MAX) return NULL; + uint32_t hash = (uint32_t)scr_key_hash(key, key_len); + if (scr_layout_transition_mask) { + size_t i = scr_layout_slot(parent, hash) & scr_layout_transition_mask; + for (ScrLayout *child; (child = scr_layout_transitions[i]); i = (i + 1) & scr_layout_transition_mask) { + const ScrLayoutKey *k = &child->keys[child->count - 1]; + if (child->parent == parent && k->hash == hash && scr_layout_key_is(k, key, key_len)) { + parent->last_child = child; + return child; + } + } + } + if (scr_layout_count >= SCR_LAYOUT_BUDGET || + key_len > SCR_LAYOUT_KEY_BYTES_BUDGET - scr_layout_key_bytes) + return NULL; + uint32_t count = parent->count + 1; + ScrLayoutKeys *store = parent->store; + bool owns = !store || store->used != parent->count || store->used == store->cap; + if (owns) { + uint32_t cap = count < 4 ? 4 : count * 2; + if (cap > SCR_LAYOUT_MAX_KEYS) cap = SCR_LAYOUT_MAX_KEYS; + size_t bytes = sizeof *store + cap * sizeof(ScrLayoutKey); + if (bytes > SCR_LAYOUT_ARRAY_BYTES_BUDGET - scr_layout_array_bytes) return NULL; + store = malloc(bytes); + if (!store) scr_json_oom(); + if (parent->count) memcpy(store->items, parent->keys, parent->count * sizeof(ScrLayoutKey)); + store->used = parent->count; + store->cap = cap; + scr_layout_array_bytes += bytes; + } + if ((scr_layout_count + 1) * 2 > (scr_layout_transition_mask ? scr_layout_transition_mask + 1 : 0)) + scr_layout_transitions_grow(); + ScrLayout *child = malloc(sizeof *child); + if (!child) scr_json_oom(); + store->items[store->used++] = (ScrLayoutKey){scr_dyn_key_copy(key, key_len), (uint32_t)key_len, hash}; + *child = (ScrLayout){store->items, count, 0, NULL, parent, NULL, store, scr_layout_all, owns}; + scr_layout_all = child; + if (!scr_layout_registered) { + scr_layout_registered = true; + scr_atexit(scr_layout_reset); + } + scr_layout_insert(child); + scr_layout_count++; + scr_layout_key_bytes += key_len; + parent->last_child = child; + return child; +} + +#define SCR_DYN_OBJECT_INDEX_MIN 32 + +/* Layout mode → dictionary mode: the same properties in the same order, + * each a writable, enumerable, configurable data property. */ +void scr_dyn_to_dictionary(ScrDyn *object) { + const ScrLayout *layout = object->v.obj.layout; + if (!layout) return; + size_t len = object->v.obj.len; + ScrDyn **slots = object->v.obj.slots; + size_t cap = len < 4 ? 4 : len; + ScrDynEntry *entries = scr_mem_realloc(NULL, cap * sizeof *entries); + if (!entries) scr_json_oom(); + memset(entries, 0, cap * sizeof *entries); + size_t traced = object->v.obj.slots_traced; + for (size_t i = 0; i < len; i++) { + const ScrLayoutKey *k = &layout->keys[i]; + scr_dyn_key_retain(k->key); + ScrDynEntry *e = &entries[i]; + e->key = k->key; + e->key_len = k->len; + e->value = slots[i]; + e->value_traced = (traced >> i) & 1; + e->writable = e->enumerable = e->configurable = true; + } + scr_mem_free(slots); + object->v.obj.entries = entries; + object->v.obj.cap = cap; + object->v.obj.layout = NULL; + object->v.obj.slots_traced = 0; + /* A wide object gets its hash index with its next insertion + * (scr_dyn_obj_put); until then lookups scan its entries. */ + object->v.obj.index = NULL; +} + +/* Appends a property `key` does not name yet to a layout-mode object, + * taking ownership of `value`; false (nothing done) when no layout fits, + * and the caller then takes the dictionary path. */ +/* Appends `value` (owned) to a layout-mode object as the slot of the last + * key of `next`, a child of the object's layout. */ +static inline void scr_dyn_obj_layout_push(ScrDyn *obj, const ScrLayout *next, ScrDyn *value) { + size_t len = obj->v.obj.len; + if (len == obj->v.obj.cap) { + size_t cap = obj->v.obj.cap ? obj->v.obj.cap * 2 : 4; + if (cap > SCR_LAYOUT_MAX_KEYS) cap = SCR_LAYOUT_MAX_KEYS; + ScrDyn **slots = scr_mem_realloc(obj->v.obj.slots, cap * sizeof *slots); + if (!slots) scr_json_oom(); + obj->v.obj.slots = slots; + obj->v.obj.cap = cap; + } + obj->v.obj.slots[len] = value; + obj->v.obj.len = len + 1; + obj->v.obj.layout = next; +} + +static inline bool scr_dyn_obj_layout_append(ScrDyn *obj, const char *key, size_t key_len, ScrDyn *value) { + const ScrLayout *next = scr_layout_add(obj->v.obj.layout, key, key_len); + if (!next) return false; + scr_dyn_obj_layout_push(obj, next, value); + return true; +} + #ifndef SCR_RC_AUDIT static SCR_TL ScrDyn *scr_dyn_free_arr, *scr_dyn_free_obj, *scr_dyn_free_misc; static SCR_TL size_t scr_dyn_free_count; @@ -887,39 +1117,25 @@ static SCR_TL size_t scr_dyn_free_count; * arena never returns them to the system either), at the cost of reserving * them, with their item/entry buffers, for dyn nodes. */ #define SCR_DYN_FREE_MAX ((size_t)1 << 20) -/* Recycled small objects may retain key bytes, but never values or an - * unbounded document's keys. This budget applies across the whole pool. */ -static SCR_TL size_t scr_dyn_cached_key_bytes; -#define SCR_DYN_KEY_CACHE_MAX ((size_t)64 * 1024) -#define SCR_DYN_KEY_CACHE_SLOTS 16 -#define SCR_DYN_KEY_CACHE_LENGTH 64 #endif -/* Returns the key bytes it kept for reuse, which it added to the budget; - * the pooled node records them (v.obj.cached_key_bytes) so its reuse takes - * them back without walking its slots. */ -static size_t scr_dyn_obj_drop_keys(ScrDyn *d, bool cache) { - size_t kept = 0; - for (size_t i = 0; i < d->v.obj.cap; i++) { - ScrDynEntry *entry = &d->v.obj.entries[i]; - if (!entry->key) continue; -#ifndef SCR_RC_AUDIT - size_t bytes = sizeof(ScrDynKey) + entry->key_len + 1; - if (cache && d->v.obj.cap <= SCR_DYN_KEY_CACHE_SLOTS && - entry->key_len <= SCR_DYN_KEY_CACHE_LENGTH && - bytes <= SCR_DYN_KEY_CACHE_MAX - scr_dyn_cached_key_bytes) { - scr_dyn_cached_key_bytes += bytes; - kept += bytes; - continue; +/* Drops a disposed object's keys and storage: a dictionary's entries with the keys + * they hold (recycled entries past len may hold keys too), or a layout-mode + * object's slots unless `keep_slots` (a pooled node keeps them for its next + * life). Leaves the node empty, in layout mode's representation. */ +static void scr_dyn_obj_drop_keys(ScrDyn *d, bool keep_slots) { + if (d->v.obj.layout) { + if (keep_slots) return; + scr_mem_free(d->v.obj.slots); + } else { + for (size_t i = 0; i < d->v.obj.cap; i++) { + ScrDynEntry *entry = &d->v.obj.entries[i]; + if (entry->key) scr_dyn_key_release(entry->key, entry->key_len); } -#else - (void)cache; -#endif - scr_dyn_key_release(entry->key, entry->key_len); - entry->key = NULL; - entry->key_len = 0; + scr_mem_free(d->v.obj.entries); } - return kept; + d->v.obj.slots = NULL; + d->v.obj.cap = 0; } /* Primitive kinds cannot acquire properties or change kind while live, so @@ -948,6 +1164,17 @@ void scr_dyn_trace_v(void *ptr, ScrTraceVisit visit, void *ctx) { if (d->v.arr.properties) visit(d->v.arr.properties, ctx); break; case SCR_DYN_OBJ: + if (d->v.obj.layout) { + size_t traced = 0; + for (size_t i = 0; i < d->v.obj.len; i++) { + ScrDyn *value = d->v.obj.slots[i]; + if (!scr_dyn_member_is_traced(value)) continue; + traced |= (size_t)1 << i; + visit(value, ctx); + } + d->v.obj.slots_traced = traced; + break; + } for (size_t i = 0; i < d->v.obj.len; i++) { ScrDynEntry *entry = &d->v.obj.entries[i]; entry->value_traced = scr_dyn_member_is_traced(entry->value); @@ -1120,12 +1347,11 @@ static ScrDyn *scr_dyn_alloc(ScrDynKind kind) { d->v.arr.presence = NULL; d->v.arr.region = NULL; } else if (kind == SCR_DYN_OBJ) { - d->v.obj.len = 0; /* cap/entries preserved */ - /* The kept keys leave the pool's budget with the node. */ - scr_dyn_cached_key_bytes -= d->v.obj.cached_key_bytes; + d->v.obj.len = 0; /* cap/slots preserved */ d->v.obj.region = NULL; d->v.obj.class_name = NULL; /* a released class prototype's name */ - SCR_JS_SHAPE_RESET(d); + d->v.obj.layout = &scr_layout_empty; + d->v.obj.slots_traced = 0; d->v.obj.source_identity = NULL; d->v.obj.source_access = NULL; } else if (kind != SCR_DYN_NUM && kind != SCR_DYN_STR) { @@ -1144,9 +1370,10 @@ static ScrDyn *scr_dyn_alloc(ScrDynKind kind) { fresh->prototype = NULL; fresh->symbol_properties = NULL; fresh->symbol_keys = NULL; -#ifdef SCR_JS_TIER - if (kind == SCR_DYN_OBJ) SCR_JS_SHAPE_RESET(fresh); -#endif + if (kind == SCR_DYN_OBJ) { + memset(&fresh->v, 0, sizeof fresh->v); + fresh->v.obj.layout = &scr_layout_empty; + } #ifdef SCR_RC_AUDIT scr_live_dyns++; #endif @@ -1225,7 +1452,7 @@ static void scr_dyn_dispose(ScrDyn *d, bool collected) { return; } #endif - if (d->kind == SCR_DYN_OBJ) { + if (d->kind == SCR_DYN_OBJ && !d->v.obj.layout) { free(d->v.obj.index); d->v.obj.index = NULL; } @@ -1259,7 +1486,12 @@ static void scr_dyn_dispose(ScrDyn *d, bool collected) { } break; case SCR_DYN_OBJ: - for (size_t i = 0; i < d->v.obj.len; i++) { + if (d->v.obj.layout) { + size_t traced = collected ? d->v.obj.slots_traced : 0; + for (size_t i = 0; i < d->v.obj.len; i++) + if (!((traced >> i) & 1)) scr_dyn_release_child(d->v.obj.slots[i]); + d->v.obj.len = 0; + } else for (size_t i = 0; i < d->v.obj.len; i++) { ScrDyn *value = d->v.obj.entries[i].value; if (!collected || !d->v.obj.entries[i].value_traced) scr_dyn_release_child(value); if (!collected) { @@ -1313,7 +1545,7 @@ static void scr_dyn_dispose(ScrDyn *d, bool collected) { #endif #ifndef SCR_RC_AUDIT if (!collected && scr_dyn_free_count < SCR_DYN_FREE_MAX) { - if (d->kind == SCR_DYN_OBJ) d->v.obj.cached_key_bytes = scr_dyn_obj_drop_keys(d, true); + if (d->kind == SCR_DYN_OBJ) scr_dyn_obj_drop_keys(d, true); /* Pooled blocks retain their headers and count toward the bounded * allocated heap, but never remain in a candidate buffer. */ scr_dyn_weak_dispose(d); @@ -1327,10 +1559,7 @@ static void scr_dyn_dispose(ScrDyn *d, bool collected) { } #endif if (d->kind == SCR_DYN_ARR) scr_mem_free(d->v.arr.items); - else if (d->kind == SCR_DYN_OBJ) { - scr_dyn_obj_drop_keys(d, false); - scr_mem_free(d->v.obj.entries); - } + else if (d->kind == SCR_DYN_OBJ) scr_dyn_obj_drop_keys(d, false); scr_cyc_free_inline(d); } @@ -1412,7 +1641,6 @@ void scr_dyn_release(ScrDyn *d) { /* Small objects retain their compact linear representation. A wide object * owns a bucket index of entry positions; entries remain insertion ordered. * No key/value ownership lives here, and callbacks never retain its slots. */ -#define SCR_DYN_OBJECT_INDEX_MIN 32 typedef struct ScrDynObjectIndex { size_t buckets; size_t slots[]; /* zero is empty; occupied slots store entry + 1 */ @@ -1472,8 +1700,12 @@ static __attribute__((noinline)) ScrDynEntry *scr_dyn_find_indexed_entry( return NULL; } +/* The entry of own property `key`. Its callers read or write entries, so + * a layout-mode object moves to dictionary mode first; paths that know + * layouts look up slots with scr_dyn_layout_slot instead. */ static ScrDynEntry *scr_dyn_find_entry(const ScrDyn *object, const char *key, size_t length) { if (object->kind != SCR_DYN_OBJ) return NULL; + if (object->v.obj.layout) scr_dyn_to_dictionary((ScrDyn *)object); if (object->v.obj.index) return scr_dyn_find_indexed_entry(object, key, length); size_t slot = (length + (length ? (unsigned char)key[0] * 7u : 0)) % SCR_DYN_PROPERTY_HINTS; size_t hint = scr_dyn_property_hints[slot]; @@ -1486,16 +1718,49 @@ static ScrDynEntry *scr_dyn_find_entry(const ScrDyn *object, const char *key, si return entry; } +/* Out-of-line property accessors for emitted object walks (the + * index-signature checks of backend/llvm/dyn.ts). */ +const char *scr_dyn_member_key(const ScrDyn *object, size_t i) { + size_t key_len; + return scr_dyn_obj_key_at(object, i, &key_len); +} + +size_t scr_dyn_member_key_len(const ScrDyn *object, size_t i) { + size_t key_len; + scr_dyn_obj_key_at(object, i, &key_len); + return key_len; +} + +ScrDyn *scr_dyn_member_value(const ScrDyn *object, size_t i) { + return scr_dyn_obj_value_at(object, i); +} + +/* The slot of own property `key` of a layout-mode object, or NULL. */ +static inline ScrDyn **scr_dyn_layout_slot(const ScrDyn *object, const char *key, size_t length) { + uint32_t i = scr_layout_find(object->v.obj.layout, key, length); + return i == UINT32_MAX ? NULL : &object->v.obj.slots[i]; +} + +/* Whether `object` is an ordinary object in layout mode. */ +static inline bool scr_dyn_is_layout(const ScrDyn *object) { + return object->kind == SCR_DYN_OBJ && object->v.obj.layout; +} + ScrDyn *scr_dyn_obj_get(const ScrDyn *d, const char *key, size_t key_len) { if (d->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("raw object inspection"); return NULL; } + if (scr_dyn_is_layout(d)) { + ScrDyn **slot = scr_dyn_layout_slot(d, key, key_len); + return slot ? *slot : NULL; + } ScrDynEntry *entry = scr_dyn_find_entry(d, key, key_len); return entry ? entry->value : NULL; } bool scr_dyn_obj_enumerable(const ScrDyn *d, const char *key, size_t key_len) { + if (scr_dyn_is_layout(d)) return scr_dyn_layout_slot(d, key, key_len) != NULL; ScrDynEntry *entry = scr_dyn_find_entry(d, key, key_len); return entry && entry->enumerable; } @@ -1539,8 +1804,18 @@ int32_t scr_dyn_discriminant_case(const ScrDyn *d, const ScrStr *key, ScrStr *co if (!d) return -1; if (d->kind == SCR_DYN_TYPED_REF) return -2; if (d->kind != SCR_DYN_OBJ) return -1; + if (d->v.obj.layout) { + ScrDyn **slot = scr_dyn_layout_slot(d, key->data, key->len); + if (!slot) return -2; + const ScrDyn *member = *slot; + if (!member || member->kind != SCR_DYN_STR) return -1; + for (size_t k = 0; k < count; k++) + if (scr_str_eq(member->v.str, values[k])) return (int32_t)k; + return -1; + } for (size_t i = 0; i < d->v.obj.len; i++) { - const ScrDynEntry *entry = &d->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(d, i); + const ScrDynProp *entry = &entry_prop; if (entry->key_len != key->len || memcmp(entry->key, key->data, key->len) != 0) continue; if (entry->accessor) return -2; const ScrDyn *member = entry->value; @@ -1868,6 +2143,17 @@ ScrDyn *scr_dyn_arr_named_get(const ScrDyn *d, const ScrStr *key) { * A recycled entry can reuse equal key bytes without a malloc/free pair. */ static void scr_dyn_obj_put(ScrDyn *obj, const char *key, size_t key_len, ScrDyn *value) { scr_dyn_region_store(obj, value); + if (obj->v.obj.layout) { + ScrDyn **slot = scr_dyn_layout_slot(obj, key, key_len); + if (slot) { + ScrDyn *old = *slot; + *slot = value; + scr_dyn_release(old); + return; + } + if (!obj->non_extensible && scr_dyn_obj_layout_append(obj, key, key_len, value)) return; + scr_dyn_to_dictionary(obj); + } /* Construction usually appends an absent name. Avoid a read hint that * repeats comparisons on compact objects; wide objects use their index. */ ScrDynEntry *existing = obj->v.obj.index ? scr_dyn_find_entry(obj, key, key_len) @@ -1901,14 +2187,13 @@ static void scr_dyn_obj_put(ScrDyn *obj, const char *key, size_t key_len, ScrDyn } if (obj->v.obj.len == obj->v.obj.cap) { size_t cap = obj->v.obj.cap ? obj->v.obj.cap * 2 : 4; - ScrDynEntry *entries = scr_mem_realloc(obj->v.obj.entries, cap * sizeof *entries); + ScrDynEntry *entries = scr_mem_realloc(scr_dyn_obj_dict_entries(obj), cap * sizeof *entries); if (!entries) scr_json_oom(); memset(entries + obj->v.obj.cap, 0, (cap - obj->v.obj.cap) * sizeof *entries); obj->v.obj.entries = entries; obj->v.obj.cap = cap; } - SCR_JS_SHAPE_RESET(obj); - ScrDynEntry *e = &obj->v.obj.entries[obj->v.obj.len++]; + ScrDynEntry *e = &scr_dyn_obj_dict_entries(obj)[obj->v.obj.len++]; if (!e->key || e->key_len != key_len || !scr_key_equal(e->key, key, key_len)) { char *owned = scr_dyn_key_copy(key, key_len); scr_dyn_key_release(e->key, e->key_len); @@ -2550,7 +2835,7 @@ static void scr_dyn_record_view_freeze(ScrDyn *view) { view->non_extensible = true; if (view->kind == SCR_DYN_OBJ) { for (size_t i = 0; i < view->v.obj.len; i++) { - ScrDynEntry *entry = &view->v.obj.entries[i]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(view)[i]; entry->configurable = false; if (!entry->accessor) entry->writable = false; } @@ -2614,7 +2899,8 @@ double scr_dyn_class_view_integrity(const ScrDyn *view, double bits, ScrDyn *bag for (const ScrDyn *table = view; table; table = table == view ? view->symbol_properties : NULL) { if (table->kind != SCR_DYN_OBJ) continue; for (size_t i = 0; i < table->v.obj.len; i++) { - const ScrDynEntry *entry = &table->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(table, i); + const ScrDynProp *entry = &entry_prop; if (entry->configurable) sealed = false; if (!entry->accessor && entry->writable) frozen = false; } @@ -2795,7 +3081,7 @@ ScrDyn *scr_dyn_seal(ScrDyn *object) { } if (object->kind == SCR_DYN_OBJ) { object->non_extensible = true; - for (size_t i = 0; i < object->v.obj.len; i++) object->v.obj.entries[i].configurable = false; + for (size_t i = 0; i < object->v.obj.len; i++) scr_dyn_obj_dict_entries(object)[i].configurable = false; } if (object->kind == SCR_DYN_ARR) { object->non_extensible = true; @@ -2826,7 +3112,7 @@ bool scr_dyn_is_sealed(const ScrDyn *object) { if (object->kind == SCR_DYN_BYTES) return object->v.bytes->properties && scr_dyn_is_sealed(object->v.bytes->properties); if (object->kind != SCR_DYN_OBJ) return true; if (!object->non_extensible) return false; - for (size_t i = 0; i < object->v.obj.len; i++) if (object->v.obj.entries[i].configurable) return false; + for (size_t i = 0; i < object->v.obj.len; i++) if (scr_dyn_obj_prop_at(object, i).configurable) return false; return true; } @@ -2855,6 +3141,10 @@ static const ScrDyn *scr_dyn_property_owner(const ScrDyn *object, const char *ke return owner; } if (current->kind != SCR_DYN_OBJ) continue; + if (current->v.obj.layout) { + if (scr_layout_find(current->v.obj.layout, key, length) != UINT32_MAX) return current; + continue; + } for (size_t i = 0; i < current->v.obj.len; i++) { const ScrDynEntry *entry = ¤t->v.obj.entries[i]; if (entry->key_len == length && memcmp(entry->key, key, length) == 0) return current; @@ -2897,7 +3187,7 @@ ScrDyn *scr_dyn_proxy_new(const ScrDyn *target, const ScrDyn *handler) { ScrDyn *view = scr_dyn_typed_ref_materialize(target); bool plain = view && view->kind == SCR_DYN_OBJ; if (plain) for (size_t i = 0; i < view->v.obj.len; i++) { - ScrDynEntry *entry = &view->v.obj.entries[i]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(view)[i]; if (entry->accessor || !entry->configurable || !entry->writable) plain = false; } scr_dyn_release(view); @@ -3267,6 +3557,16 @@ ScrDyn *scr_dyn_own_descriptor(const ScrDyn *value, const ScrStr *key) { if (value->kind == SCR_DYN_BOOL || value->kind == SCR_DYN_NUM) { return scr_dyn_retain(scr_dyn_undefined()); } + if (value->kind == SCR_DYN_OBJ && value->v.obj.layout) { + ScrDyn **slot = scr_dyn_layout_slot(value, key->data, key->len); + if (!slot) return scr_dyn_retain(scr_dyn_undefined()); + ScrDyn *out = scr_dyn_new_obj(); + scr_dyn_obj_set(out, "value", 5, scr_dyn_retain(*slot)); + scr_dyn_obj_set(out, "writable", 8, scr_dyn_new_bool(true)); + scr_dyn_obj_set(out, "enumerable", 10, scr_dyn_new_bool(true)); + scr_dyn_obj_set(out, "configurable", 12, scr_dyn_new_bool(true)); + return out; + } if (value->kind == SCR_DYN_OBJ) { ScrDynEntry *entry = scr_dyn_entry((ScrDyn *)value, key); if (!entry) return scr_dyn_retain(scr_dyn_undefined()); @@ -3445,7 +3745,24 @@ typedef struct { const char *names; const char *lens; size_t name_stride, len_stride, count; + /* A record that a narrower record type also references (a prefix view, + * scr_dyn_typed_ref_prefix_view) has an entry with no view and this, its + * own capsule's parameters: every capsule of the record then answers + * for its whole storage (scr_dyn_typed_ref_new), and reflective surfaces + * of the narrower type go through that capsule (scr_record_source_has). + * Owned; NULL for an ordinary source. */ + struct ScrRecordPrefix *prefix; } ScrRecordSource; +typedef struct ScrRecordPrefix { + void *(*retain)(void *); + void (*release)(void *); + const char *type_key; + size_t type_key_len; + ScrDyn *(*materialize)(void *); + void (*commit)(void *, const ScrDyn *); + const ScrDynTypedArrayOps *ops; /* the record's ScrDynTypedRecordOps */ + bool traced, record; +} ScrRecordPrefix; static SCR_TL ScrRecordSource *scr_record_sources; static SCR_TL size_t scr_record_source_capacity, scr_record_source_count; /* Deleted slots keep this marker so probe runs stay intact without moving @@ -3476,7 +3793,7 @@ static size_t scr_record_source_bucket(const void *rec) { return (size_t)((run << 6) | (key & 63)) & (scr_record_source_capacity - 1); } -static ScrRecordSource *scr_record_source_find(const void *rec) { +static ScrRecordSource *scr_record_source_slot(const void *rec) { if (!scr_record_source_count) return NULL; size_t mask = scr_record_source_capacity - 1; for (size_t slot = scr_record_source_bucket(rec); scr_record_sources[slot].rec; @@ -3486,6 +3803,17 @@ static ScrRecordSource *scr_record_source_find(const void *rec) { return NULL; } +/* The record's kept source, or NULL (also for a prefix-view entry). */ +static ScrRecordSource *scr_record_source_find(const void *rec) { + ScrRecordSource *entry = scr_record_source_slot(rec); + return entry && !entry->prefix ? entry : NULL; +} + +static const ScrRecordPrefix *scr_record_prefix_find(const void *rec) { + ScrRecordSource *entry = scr_record_source_slot(rec); + return entry ? entry->prefix : NULL; +} + /* The retained view of a record with undeclared keys (borrowed), or NULL. */ ScrDyn *scr_record_source_view(const void *rec) { ScrRecordSource *entry = scr_record_source_find(rec); @@ -3493,7 +3821,7 @@ ScrDyn *scr_record_source_view(const void *rec) { } bool scr_record_source_has(const void *rec) { - return scr_record_source_count && scr_record_source_find(rec) != NULL; + return scr_record_source_count && scr_record_source_slot(rec) != NULL; } /* Drops the entry at `found` from the table, releasing its view. The table @@ -3501,6 +3829,7 @@ bool scr_record_source_has(const void *rec) { * the dispose hook sees a consistent table. */ static void scr_record_source_drop(ScrRecordSource *found) { ScrDyn *view = found->view; + free(found->prefix); *found = (ScrRecordSource){ .rec = SCR_RECORD_SOURCE_DELETED }; scr_record_source_count--; scr_record_source_deleted++; @@ -3510,7 +3839,7 @@ static void scr_record_source_drop(ScrRecordSource *found) { static void scr_record_source_disposed(void *rec) { size_t bit = scr_record_source_filter_bit(rec); if (!(scr_record_source_filter[bit >> 6] & (UINT64_C(1) << (bit & 63)))) return; - ScrRecordSource *found = scr_record_source_find(rec); + ScrRecordSource *found = scr_record_source_slot(rec); if (found) scr_record_source_drop(found); } @@ -3583,7 +3912,10 @@ static void scr_record_source_teardown(void) { free(scr_record_source_filter); scr_record_source_filter = NULL; for (size_t i = 0; i < capacity; i++) { - if (table[i].rec && table[i].rec != SCR_RECORD_SOURCE_DELETED) scr_dyn_release(table[i].view); + if (table[i].rec && table[i].rec != SCR_RECORD_SOURCE_DELETED) { + free(table[i].prefix); + scr_dyn_release(table[i].view); + } } free(table); } @@ -3626,7 +3958,8 @@ void scr_record_source_note(void *rec, const ScrDyn *src, const char *const *nam bool keep = false; size_t next = 0; for (size_t i = 0; i < src->v.obj.len && !keep; i++) { - const ScrDynEntry *entry = &src->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(src, i); + const ScrDynProp *entry = &entry_prop; size_t index = scr_record_source_index(&probe, entry->key, entry->key_len); keep = index == count || index < next; next = index + 1; @@ -3634,7 +3967,8 @@ void scr_record_source_note(void *rec, const ScrDyn *src, const char *const *nam if (!keep) return; ScrDyn *view = scr_dyn_new_obj(); for (size_t i = 0; i < src->v.obj.len; i++) { - const ScrDynEntry *entry = &src->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(src, i); + const ScrDynProp *entry = &entry_prop; if (scr_record_source_declared(&probe, entry->key, entry->key_len)) { scr_dyn_obj_set(view, entry->key, entry->key_len, scr_dyn_retain(scr_dyn_undefined())); } else if (!entry->accessor && entry->writable && entry->enumerable && entry->configurable) { @@ -3667,7 +4001,8 @@ void scr_record_source_spread(void *dst, const void *src, const char *const *nam bool keep = false; size_t next = 0; for (size_t i = 0; i < from->v.obj.len && !keep; i++) { - const ScrDynEntry *entry = &from->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(from, i); + const ScrDynProp *entry = &entry_prop; size_t index = scr_record_source_index(&probe, entry->key, entry->key_len); if (index == count) continue; keep = index != next; @@ -3676,7 +4011,8 @@ void scr_record_source_spread(void *dst, const void *src, const char *const *nam if (!keep) return; ScrDyn *view = scr_dyn_new_obj(); for (size_t i = 0; i < from->v.obj.len; i++) { - const ScrDynEntry *entry = &from->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(from, i); + const ScrDynProp *entry = &entry_prop; if (scr_record_source_declared(&probe, entry->key, entry->key_len)) scr_dyn_obj_set(view, entry->key, entry->key_len, scr_dyn_retain(scr_dyn_undefined())); } @@ -3687,7 +4023,7 @@ void scr_record_source_spread(void *dst, const void *src, const char *const *nam if (scr_dyn_obj_get(view, names[i], lens[i])) continue; scr_dyn_obj_set(view, names[i], lens[i], scr_dyn_retain(scr_dyn_undefined())); } - ScrRecordSource *old = scr_record_source_find(dst); + ScrRecordSource *old = scr_record_source_slot(dst); if (old) scr_record_source_drop(old); probe.view = view; scr_record_source_put(&probe); @@ -3695,9 +4031,15 @@ void scr_record_source_spread(void *dst, const void *src, const char *const *nam void scr_record_source_move(void *from, void *to) { if (!scr_record_source_count) return; - ScrRecordSource *old = scr_record_source_find(to); - if (old) scr_record_source_drop(old); + ScrRecordSource *old = scr_record_source_slot(to); ScrRecordSource *found = scr_record_source_find(from); + if (old && old->prefix) { + /* A prefix-viewed record keeps its entry: its capsule's view holds the + * undeclared properties, and narrower names read them through it. */ + if (found) scr_record_source_drop(found); + return; + } + if (old) scr_record_source_drop(old); if (!found) return; ScrRecordSource moved = *found; *found = (ScrRecordSource){ .rec = SCR_RECORD_SOURCE_DELETED }; @@ -3715,7 +4057,8 @@ static ScrDyn *scr_record_source_merge(const ScrRecordSource *source, const ScrD const ScrDyn *fresh) { ScrDyn *out = scr_dyn_new_obj(); for (size_t i = 0; i < cached->v.obj.len; i++) { - const ScrDynEntry *entry = &cached->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(cached, i); + const ScrDynProp *entry = &entry_prop; ScrDyn *value = scr_dyn_obj_get(fresh, entry->key, entry->key_len); if (value) { scr_dyn_obj_set(out, entry->key, entry->key_len, scr_dyn_retain(value)); @@ -3736,7 +4079,8 @@ static ScrDyn *scr_record_source_merge(const ScrRecordSource *source, const ScrD scr_str_release(name); } for (size_t i = 0; i < fresh->v.obj.len; i++) { - const ScrDynEntry *entry = &fresh->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(fresh, i); + const ScrDynProp *entry = &entry_prop; if (scr_dyn_obj_get(out, entry->key, entry->key_len)) continue; scr_dyn_obj_set(out, entry->key, entry->key_len, scr_dyn_retain(entry->value)); } @@ -3755,6 +4099,7 @@ ScrDyn *scr_record_source_apply(ScrDyn *snapshot, const void *rec) { } static ScrDyn *scr_json_member(const ScrDyn *holder, const ScrStr *key); +static void scr_jb_put_record_prefix(ScrJsonBuf *b, void *rec); static bool scr_jb_try_pure_member(ScrJsonBuf *b, const char *key, size_t key_len, const ScrStr *key_str, const ScrDyn *value, bool *first); static bool scr_jb_put_member(ScrJsonBuf *b, const ScrStr *key, ScrDyn *value, bool *first, @@ -3812,7 +4157,8 @@ static void scr_record_source_key_order(const ScrDyn *view, size_t *out) { if (!keys) scr_json_oom(); size_t integers = 0, next = 0; for (size_t i = 0; i < len; i++) { - const ScrDynEntry *entry = &view->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(view, i); + const ScrDynProp *entry = &entry_prop; double index; if (scr_dyn_key_is_index(entry->key, entry->key_len, &index)) keys[integers++] = (ScrRecordSourceIntegerKey){ index, i }; @@ -3821,7 +4167,8 @@ static void scr_record_source_key_order(const ScrDyn *view, size_t *out) { qsort(keys, integers, sizeof *keys, scr_record_source_integer_cmp); for (size_t i = 0; i < integers; i++) out[next++] = keys[i].entry; for (size_t i = 0; i < len; i++) { - const ScrDynEntry *entry = &view->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(view, i); + const ScrDynProp *entry = &entry_prop; double index; if (!scr_dyn_key_is_index(entry->key, entry->key_len, &index)) out[next++] = i; } @@ -3830,6 +4177,10 @@ static void scr_record_source_key_order(const ScrDyn *view, size_t *out) { bool scr_jb_try_record_source(ScrJsonBuf *b, void *rec, bool (*field)(ScrJsonBuf *, void *, size_t, bool)) { + if (scr_record_prefix_find(rec)) { + scr_jb_put_record_prefix(b, rec); + return true; + } ScrRecordSource *found = scr_record_source_find(rec); if (!found) return false; /* Program code run by a member can add record sources and move the @@ -3854,7 +4205,7 @@ bool scr_jb_try_record_source(ScrJsonBuf *b, void *rec, size_t *order = NULL; for (size_t k = 0; k < view->v.obj.len && !order; k++) { double index; - if (!scr_dyn_key_is_index(view->v.obj.entries[k].key, view->v.obj.entries[k].key_len, &index)) continue; + if (!scr_dyn_key_is_index(scr_dyn_obj_dict_entries(view)[k].key, scr_dyn_obj_dict_entries(view)[k].key_len, &index)) continue; order = view->v.obj.len <= 64 ? order_small : malloc(view->v.obj.len * sizeof *order); if (!order) scr_json_oom(); scr_record_source_key_order(view, order); @@ -3865,7 +4216,8 @@ bool scr_jb_try_record_source(ScrJsonBuf *b, void *rec, * SerializeJSONObject takes them before reading any value. */ size_t i = 0; for (; i < view->v.obj.len && !scr_exc_pending(); i++) { - const ScrDynEntry *entry = &view->v.obj.entries[order ? order[i] : i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(view, order ? order[i] : i); + const ScrDynProp *entry = &entry_prop; size_t index = scr_record_source_index(&source, entry->key, entry->key_len); if (index < count) { if (!seen[index] && scr_jb_put_source_field(b, rec, &source, index, first, on_stack, field)) @@ -3885,7 +4237,8 @@ bool scr_jb_try_record_source(ScrJsonBuf *b, void *rec, if (i < view->v.obj.len && !scr_exc_pending()) { ScrDyn *rest = scr_dyn_new_arr(); for (size_t j = i; j < view->v.obj.len; j++) { - const ScrDynEntry *entry = &view->v.obj.entries[order ? order[j] : j]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(view, order ? order[j] : j); + const ScrDynProp *entry = &entry_prop; if (!entry->enumerable && !scr_record_source_declared(&source, entry->key, entry->key_len)) continue; ScrStr *name = scr_str_new(entry->key, entry->key_len); scr_dyn_arr_push(rest, scr_dyn_new_str(name)); @@ -3925,6 +4278,20 @@ static ScrDyn *scr_dyn_typed_ref_new( const char *type_key, size_t type_key_len, ScrDyn *(*materialize)(void *), void (*commit)(void *, const ScrDyn *), bool traced, bool observed) { + const ScrRecordPrefix *prefix = + scr_record_source_count && type_key_len >= 7 && memcmp(type_key, "record:", 7) == 0 + ? scr_record_prefix_find(ptr) + : NULL; + if (prefix) { + /* A record seen through a narrower type: its capsule is its own. */ + retain = prefix->retain; + release = prefix->release; + type_key = prefix->type_key; + type_key_len = prefix->type_key_len; + materialize = prefix->materialize; + commit = prefix->commit; + traced = traced || prefix->traced; + } scr_typed_ref_reserve(); size_t slot = scr_typed_ref_bucket(ptr); for (ScrDyn *value; (value = scr_typed_ref_slots[slot]); slot = (slot + 1) & (scr_typed_ref_capacity - 1)) { @@ -3960,6 +4327,11 @@ static ScrDyn *scr_dyn_typed_ref_new( d->v.typed_ref.record = false; d->v.typed_ref.klass = false; d->v.typed_ref.bag_class = false; + if (prefix) { + /* Bound here: the narrower name's bind keeps them (bind_record). */ + d->v.typed_ref.array = prefix->ops; + d->v.typed_ref.record = prefix->record; + } scr_typed_ref_slots[slot] = d; /* the empty slot that ended the probe */ scr_typed_ref_count++; return d; @@ -4006,6 +4378,32 @@ void *scr_dyn_typed_ref_unbox(const ScrDyn *d) { return d->v.typed_ref.retain(d->v.typed_ref.ptr); } +/* A record capsule's native record for a conversion into a type whose + * storage is a prefix of the record's (the compiler's prefix-view sources): + * the pointer itself (+1), so both names keep one object. The record + * remembers its own capsule parameters (ScrRecordPrefix), so every later + * capsule of it, whichever name boxes it, answers for its whole storage. + * NULL while the record keeps undeclared source properties, whose table + * is keyed by the record's own declaration; the conversion then builds + * its copy as before. */ +void *scr_dyn_typed_ref_prefix_view(const ScrDyn *d) { + void *record = d->v.typed_ref.ptr; + if (!scr_record_prefix_find(record)) { + if (scr_record_source_find(record) || !d->v.typed_ref.record) return NULL; + ScrRecordPrefix *prefix = malloc(sizeof *prefix); + if (!prefix) scr_json_oom(); + *prefix = (ScrRecordPrefix){ + d->v.typed_ref.retain, d->v.typed_ref.release, d->v.typed_ref.type_key, + d->v.typed_ref.type_key_len, d->v.typed_ref.materialize, d->v.typed_ref.commit, + d->v.typed_ref.array, d->v.typed_ref.traced, d->v.typed_ref.record, + }; + ScrRecordSource entry = { .rec = record, .prefix = prefix }; + scr_record_source_put(&entry); + } + return d->v.typed_ref.retain(record); +} + + static void scr_dyn_typed_ref_preserve_child( ScrDyn **fresh_slot, ScrDyn *cached_child) { ScrDyn *fresh_child = *fresh_slot; @@ -4038,7 +4436,7 @@ static void scr_dyn_typed_ref_preserve_children( } if (cached->kind == SCR_DYN_OBJ) { for (size_t i = 0; i < fresh->v.obj.len; i++) { - ScrDynEntry *entry = &fresh->v.obj.entries[i]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(fresh)[i]; ScrDyn *cached_child = scr_dyn_obj_get( cached, entry->key, entry->key_len); scr_dyn_typed_ref_preserve_child(&entry->value, cached_child); @@ -4086,7 +4484,7 @@ static void scr_typed_ref_view_numeric_properties(const ScrDyn *capsule, ScrDyn if (value) scr_dyn_arr_named_put(view, x->props[i].key, x->props[i].key_len, value); } if (!named) return; - for (size_t i = 0; i < named->v.obj.len; i++) scr_dyn_copy_entry(view, named, &named->v.obj.entries[i]); + for (size_t i = 0; i < named->v.obj.len; i++) scr_dyn_copy_entry(view, named, &scr_dyn_obj_dict_entries(named)[i]); scr_dyn_release(named); } @@ -4121,7 +4519,7 @@ ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d) { fresh = ordered; } else if (record_view) { for (size_t i = 0; i < cached->v.obj.len; i++) { - ScrDynEntry *entry = &cached->v.obj.entries[i]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(cached)[i]; if (scr_dyn_obj_get(fresh, entry->key, entry->key_len)) continue; ScrStr *name = scr_str_new(entry->key, entry->key_len); ScrDyn *key = scr_dyn_new_str(name); @@ -4155,6 +4553,23 @@ ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d) { return scr_dyn_retain(capsule->v.typed_ref.materialized); } +static const ScrDynTypedRecordOps *scr_typed_ref_record_ops(const ScrDyn *d); + +/* A capsule's dyn view for a checked conversion or match, which only reads + * it (+1): a live record capsule without a view gets a fresh snapshot that + * is not kept, so the capsule keeps answering from the record; any other + * capsule its cached view (scr_dyn_typed_ref_materialize). */ +ScrDyn *scr_dyn_typed_ref_peek(const ScrDyn *d) { + if (!scr_typed_ref_record_ops(d)) return scr_dyn_typed_ref_materialize(d); + ScrDyn *fresh = d->v.typed_ref.materialize(d->v.typed_ref.ptr); + if (scr_exc_pending()) { + scr_dyn_release(fresh); + return scr_dyn_retain(scr_dyn_undefined()); + } + if (scr_dyn_typed_ref_frozen(d)) scr_dyn_record_view_freeze(fresh); + return fresh; +} + void scr_dyn_typed_ref_commit(ScrDyn *d) { /* A frozen record's or array's view cannot have changed its contents. */ if (d && d->kind == SCR_DYN_TYPED_REF && d->v.typed_ref.commit && @@ -4192,7 +4607,9 @@ void scr_dyn_typed_ref_bind_array(ScrDyn *d, const ScrDynTypedArrayOps *ops) { * hold state the record cannot (added properties, a prototype, changed * attributes), and every operation goes through it as before. */ void scr_dyn_typed_ref_bind_record(ScrDyn *d, const ScrDynTypedRecordOps *ops) { - if (d && d->kind == SCR_DYN_TYPED_REF) { + /* A capsule binds once; a prefix-viewed record's capsule keeps its own + * table when a narrower name boxes it (scr_dyn_typed_ref_new). */ + if (d && d->kind == SCR_DYN_TYPED_REF && !d->v.typed_ref.record) { d->v.typed_ref.array = (const ScrDynTypedArrayOps *)(const void *)ops; d->v.typed_ref.record = true; d->v.typed_ref.klass = false; @@ -4335,6 +4752,18 @@ static bool scr_typed_ref_record_field(const ScrDynTypedRecordOps *ops, const Sc return false; } +static bool scr_typed_ref_record_field_bytes(const ScrDynTypedRecordOps *ops, const char *key, + size_t key_len, size_t *out) { + for (size_t i = 0; i < ops->count; i++) { + const ScrStr *name = ops->names[i]; + if (name->len == key_len && memcmp(name->data, key, key_len) == 0) { + *out = i; + return true; + } + } + return false; +} + /* +1 own field value, or NULL when `key` is not a present field. */ static ScrDyn *scr_typed_ref_record_get(const ScrDyn *d, const ScrStr *key) { const ScrDynTypedRecordOps *ops = scr_typed_ref_record_ops(d); @@ -4482,6 +4911,51 @@ static bool scr_typed_ref_array_has_own(const ScrDyn *d, const ScrStr *key) { /* Writes an element in place or appends at `length`. False (nothing done) * when the value does not fit the element type or the write could grow the * array by more than one element: the snapshot path owns those outcomes. */ +/* A write of a present declared field of a live record stores into the + * native record itself, so the capsule keeps answering from the record + * instead of turning into a permanent snapshot. A present field keeps its + * own-key position, so order observers see what the snapshot would show. + * Absent optional fields, frozen records, and values the field type does + * not accept take the snapshot path, which reports the mismatch. */ +static bool scr_typed_ref_record_store(ScrDyn *d, const ScrDynTypedRecordOps *ops, size_t index, + const ScrDyn *value) { + void *record = d->v.typed_ref.ptr; + if (!ops->set || scr_rc_frozen(record)) return false; + if (!ops->fixed && !ops->has(record, index)) return false; + return ops->set(record, index, value); +} + +static bool scr_typed_ref_record_set(ScrDyn *d, const ScrStr *key, const ScrDyn *value) { + const ScrDynTypedRecordOps *ops = scr_typed_ref_record_ops(d); + size_t index; + if (!ops || !scr_typed_ref_record_field(ops, key, &index)) return false; + return scr_typed_ref_record_store(d, ops, index, value); +} + +/* Object.assign onto a live record from a plain object whose own + * properties are all data properties: each enumerable one that names a + * present declared field stores in place. Answers false at the first key + * it cannot store, leaving every key to the snapshot path, which stores + * the already stored values again unchanged. */ +static bool scr_typed_ref_record_assign(ScrDyn *d, const ScrDyn *src) { + const ScrDynTypedRecordOps *ops = scr_typed_ref_record_ops(d); + if (!ops || !ops->set || src->kind != SCR_DYN_OBJ || src->symbol_properties || src->symbol_keys) + return false; + size_t count = scr_dyn_obj_count(src); + for (size_t i = 0; i < count; i++) { + if (scr_dyn_obj_prop_at(src, i).accessor) return false; + } + for (size_t i = 0; i < count; i++) { + ScrDynProp prop = scr_dyn_obj_prop_at(src, i); + if (!prop.enumerable) continue; + size_t index; + if (!scr_typed_ref_record_field_bytes(ops, prop.key, prop.key_len, &index)) return false; + if (!scr_typed_ref_record_store(d, ops, index, prop.value)) return false; + if (scr_exc_pending()) return true; + } + return true; +} + static bool scr_typed_ref_array_set(ScrDyn *d, const ScrStr *key, const ScrDyn *value) { const ScrDynTypedArrayOps *ops = scr_dyn_typed_ref_array_ops(d); if (!ops || !ops->set) return false; @@ -4855,16 +5329,16 @@ static ScrDyn *scr_bytes_constructor_for(ScrBytesElem elem) { scr_bytes_ctor_call, 3, "typed-array-constructor", scr_bytes_elem_name(elem)); ScrDyn *properties = scr_dyn_fn_properties(scr_bytes_ctors[elem]); scr_dyn_obj_set(properties, "BYTES_PER_ELEMENT", 17, scr_dyn_new_num((double)scr_bytes_elem_size(elem))); - ScrDynEntry *entry = &properties->v.obj.entries[properties->v.obj.len - 1]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(properties)[properties->v.obj.len - 1]; entry->writable = entry->enumerable = entry->configurable = false; /* from and of live on %TypedArray%, which is not modeled: each * constructor carries them as non-enumerable properties. */ scr_dyn_obj_set(properties, "from", 4, scr_dyn_new_func(scr_closure_new(NULL, 0), scr_bytes_from_call, 1, "native:TypedArray", "from")); - properties->v.obj.entries[properties->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(properties)[properties->v.obj.len - 1].enumerable = false; scr_dyn_obj_set(properties, "of", 2, scr_dyn_new_func(scr_closure_new(NULL, 0), scr_bytes_of_call, 0, "native:TypedArray", "of")); - properties->v.obj.entries[properties->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(properties)[properties->v.obj.len - 1].enumerable = false; scr_dyn_release(properties); } return scr_dyn_retain(scr_bytes_ctors[elem]); @@ -5937,7 +6411,7 @@ ScrDyn *scr_dyn_class_prototype(ScrDyn *value, ScrDyn *base) { * `constructor`: the class value. The cycle it closes through the class * object is one the collector traces. */ scr_dyn_obj_set(data, "constructor", 11, scr_dyn_retain(value)); - data->v.obj.entries[data->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(data)[data->v.obj.len - 1].enumerable = false; cls->prototype_data = scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); scr_box_set_ref(cls->prototype_data, data); } @@ -6069,7 +6543,7 @@ const char *scr_dyn_specific_type(const ScrDyn *cb, char *detail, size_t cap) { if (cb->null_proto) { /* No constructor name: Node renders inspect(value, { depth: -1 }). */ bool keys = false; - for (size_t i = 0; i < cb->v.obj.len && !keys; i++) keys = cb->v.obj.entries[i].enumerable; + for (size_t i = 0; i < cb->v.obj.len && !keys; i++) keys = scr_dyn_obj_dict_entries(cb)[i].enumerable; d = keys ? "[Object: null prototype]" : "[Object: null prototype] {}"; } else { d = "an instance of Object"; @@ -6717,6 +7191,21 @@ void scr_dyn_obj_set_accessor(ScrDyn *obj, const char *key, size_t key_len, ScrD * attributes copied into it (a frozen instance's view is built the same way * as any other). Consumes value. */ void scr_dyn_obj_set_field(ScrDyn *obj, const char *key, size_t key_len, ScrDyn *value) { + if (obj->v.obj.layout) { + ScrDyn **slot = scr_dyn_layout_slot(obj, key, key_len); + if (slot) { + scr_dyn_region_store(obj, value); + ScrDyn *old = *slot; + *slot = value; + scr_dyn_release(old); + return; + } + bool non_extensible = obj->non_extensible; + obj->non_extensible = false; + scr_dyn_obj_put(obj, key, key_len, value); + obj->non_extensible = non_extensible; + return; + } ScrDynEntry *existing = obj->v.obj.index ? scr_dyn_find_entry(obj, key, key_len) : scr_dyn_find_linear_entry(obj, key, key_len); if (existing && existing->fenced) { @@ -6822,8 +7311,8 @@ static ScrDyn *scr_dyn_fn_properties(const ScrDyn *function) { scr_dyn_obj_set(table, "name", 4, scr_dyn_new_str(name_string)); scr_str_release(name_string); for (size_t i = 0; i < table->v.obj.len; i++) { - table->v.obj.entries[i].writable = false; - table->v.obj.entries[i].enumerable = false; + scr_dyn_obj_dict_entries(table)[i].writable = false; + scr_dyn_obj_dict_entries(table)[i].enumerable = false; } ScrBox *box = scr_box_new_obj(&scr_dyn_retain_v, &scr_dyn_release_v, &scr_dyn_trace_v); scr_box_set_ref(box, table); @@ -6832,9 +7321,9 @@ static ScrDyn *scr_dyn_fn_properties(const ScrDyn *function) { if (owner->function_kind & SCR_CLOSURE_KIND_OWN_PROTOTYPE) { ScrDyn *prototype = scr_dyn_new_obj(); scr_dyn_obj_set(prototype, "constructor", 11, scr_dyn_retain((ScrDyn *)function)); - prototype->v.obj.entries[0].enumerable = false; + scr_dyn_obj_dict_entries(prototype)[0].enumerable = false; scr_dyn_obj_set(table, "prototype", 9, prototype); - ScrDynEntry *entry = &table->v.obj.entries[table->v.obj.len - 1]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(table)[table->v.obj.len - 1]; entry->enumerable = false; entry->configurable = false; } @@ -7524,7 +8013,7 @@ static ScrDyn *scr_native_iterator_new(ScrDyn *source) { } #ifdef SCR_JS_TIER -/* A read site's answer for a receiver without a JS shape whose property is +/* A read site's answer for a receiver without a layout whose property is * plain data the runtime holds directly: a native array's length (the * generic read would build its checked view first) and a native iterator's * `next` (the builtin the generic read returns). +1, or NULL for the slow @@ -9308,7 +9797,7 @@ ScrStr *scr_dyn_object_tag(const ScrDyn *d) { if (d->kind == SCR_DYN_TYPED_REF) { if (scr_dyn_ref_is_error(d)) return scr_str_new("[object Error]", 14); if (scr_dyn_generator(d)) { - const char *gen_tag = scr_gen_tag_reader(d->v.typed_ref.ptr); + const char *gen_tag = __atomic_load_n(&scr_gen_tag_reader, __ATOMIC_RELAXED)(d->v.typed_ref.ptr); if (gen_tag[0] == '!') { static const char message[] = "Object.prototype.toString of this iterator is not supported yet"; scr_throw_error_msg(SCR_ERR_ERROR, message, sizeof message - 1); @@ -9636,7 +10125,7 @@ static void scr_errdyn_teardown(void); static size_t scr_errdyn_self_refs(const ScrDyn *d) { if (d->kind != SCR_DYN_OBJ) return 0; size_t n = 0; - for (size_t i = 0; i < d->v.obj.len; i++) n += d->v.obj.entries[i].value == d; + for (size_t i = 0; i < d->v.obj.len; i++) n += scr_dyn_obj_dict_entries(d)[i].value == d; return n; } @@ -9644,7 +10133,7 @@ static size_t scr_errdyn_self_refs(const ScrDyn *d) { * first so the release frees it. */ static void scr_errdyn_release_node(ScrDyn *d) { for (size_t i = 0; d->kind == SCR_DYN_OBJ && i < d->v.obj.len; i++) { - ScrDynEntry *entry = &d->v.obj.entries[i]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(d)[i]; if (entry->value != d) continue; entry->value = scr_dyn_retain(scr_dyn_undefined()); scr_dyn_release(d); @@ -9819,9 +10308,9 @@ static ScrDyn *scr_error_stack_get_call(ScrClosure *closure, ScrDyn *const *args static ScrDyn *scr_error_stack_set_call(ScrClosure *closure, ScrDyn *const *args, size_t argc); /* Whether an own property is an error's `stack` accessor. */ -static bool scr_dyn_entry_is_error_stack(const ScrDynEntry *entry) { - return entry->accessor && entry->setter && entry->setter->kind == SCR_DYN_FUNC && - entry->setter->v.fn.thunk == scr_error_stack_set_call; +static bool scr_dyn_setter_is_error_stack(bool accessor, const ScrDyn *setter) { + return accessor && setter && setter->kind == SCR_DYN_FUNC && + setter->v.fn.thunk == scr_error_stack_set_call; } static ScrDyn *scr_error_stack_set_call(ScrClosure *closure, ScrDyn *const *args, size_t argc) { @@ -9838,7 +10327,7 @@ static ScrDyn *scr_error_stack_set_call(ScrClosure *closure, ScrDyn *const *args ScrStr *key = scr_str_new("stack", 5); ScrDynEntry *entry = scr_dyn_entry(self, key); scr_str_release(key); - if (entry && scr_dyn_entry_is_error_stack(entry)) { + if (entry && scr_dyn_setter_is_error_stack(entry->accessor, entry->setter)) { ScrDyn *old_value = entry->value, *old_getter = entry->getter, *old_setter = entry->setter; entry->value = scr_dyn_retain(value); entry->getter = entry->setter = NULL; @@ -9866,7 +10355,7 @@ static void scr_error_dyn_add_stack(ScrDyn *d) { scr_dyn_new_func(scr_closure_new(NULL, 0), scr_error_stack_set_call, 1, "native:error.stack", NULL); } scr_dyn_obj_set(d, "stack", 5, scr_dyn_retain(scr_dyn_undefined())); - ScrDynEntry *entry = &d->v.obj.entries[d->v.obj.len - 1]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(d)[d->v.obj.len - 1]; entry->accessor = true; entry->getter = scr_dyn_retain(scr_error_stack_accessors[0]); entry->setter = scr_dyn_retain(scr_error_stack_accessors[1]); @@ -9889,25 +10378,25 @@ static bool scr_dyn_error_view_on_error_prototype(const ScrDyn *object) { ScrDyn *scr_error_dyn_fields(const ScrError *e) { ScrDyn *d = scr_dyn_new_obj(); scr_dyn_obj_set(d, "%error", 6, scr_dyn_new_bool(true)); /* the checked-dynamic tree's error marker */ - d->v.obj.entries[d->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(d)[d->v.obj.len - 1].enumerable = false; /* Node lists `stack` first among an error's own properties. */ scr_error_dyn_add_stack(d); if (!e->name_present || !e->message_present) { ScrDyn *prototype = scr_dyn_new_obj(); if (!e->name_present) { scr_dyn_obj_set(prototype, "name", 4, scr_dyn_new_str(e->name)); - prototype->v.obj.entries[prototype->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(prototype)[prototype->v.obj.len - 1].enumerable = false; } if (!e->message_present) { scr_dyn_obj_set(prototype, "message", 7, scr_dyn_new_str(e->message)); - prototype->v.obj.entries[prototype->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(prototype)[prototype->v.obj.len - 1].enumerable = false; } scr_dyn_release(scr_dyn_set_prototype(d, prototype)); scr_dyn_release(prototype); } if (e->message_present) { scr_dyn_obj_set(d, "message", 7, scr_dyn_new_str(e->message)); - d->v.obj.entries[d->v.obj.len - 1].enumerable = e->message_enumerable; + scr_dyn_obj_dict_entries(d)[d->v.obj.len - 1].enumerable = e->message_enumerable; } if (e->system_message) { /* SystemError: code, the read-only `info` context ({ message, path, @@ -9921,7 +10410,7 @@ ScrDyn *scr_error_dyn_fields(const ScrError *e) { scr_dyn_obj_set(info, "errno", 5, scr_dyn_new_num(e->system_errno)); if (e->system_code) scr_dyn_obj_set(info, "code", 4, scr_dyn_new_str(e->system_code)); scr_dyn_obj_set(d, "info", 4, info); - d->v.obj.entries[d->v.obj.len - 1].writable = false; + scr_dyn_obj_dict_entries(d)[d->v.obj.len - 1].writable = false; scr_dyn_obj_set(d, "errno", 5, scr_dyn_new_num(e->system_errno)); if (e->system_call) scr_dyn_obj_set(d, "syscall", 7, scr_dyn_new_str(e->system_call)); } else if (e->errno_order) { @@ -9947,7 +10436,7 @@ ScrDyn *scr_error_dyn_fields(const ScrError *e) { if (e->system_dest) scr_dyn_obj_set(d, "dest", 4, scr_dyn_new_str(e->system_dest)); if (e->error_cause) { scr_dyn_obj_set(d, "cause", 5, scr_dyn_retain(e->error_cause)); - d->v.obj.entries[d->v.obj.len - 1].enumerable = e->cause_enumerable; + scr_dyn_obj_dict_entries(d)[d->v.obj.len - 1].enumerable = e->cause_enumerable; } /* DOMException: `code` is the WebIDL legacy NUMBER (never the errno * string slot), and the options form's cause crosses as itself. */ @@ -9963,20 +10452,20 @@ ScrDyn *scr_error_dyn_fields(const ScrError *e) { if (e->vt == &scr_error_vts[SCR_ERR_SUPPRESSED] && scr_exc_suppressed_values(e, &suppressed_error, &suppressed_value)) { scr_dyn_obj_set(d, "error", 5, scr_caught_to_dyn(suppressed_error)); - d->v.obj.entries[d->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(d)[d->v.obj.len - 1].enumerable = false; scr_dyn_obj_set(d, "suppressed", 10, scr_caught_to_dyn(suppressed_value)); - d->v.obj.entries[d->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(d)[d->v.obj.len - 1].enumerable = false; } /* AggregateError's own non-enumerable `errors` list, shared with the * typed read so both spellings answer the same array. */ if (e->vt == &scr_error_vts[SCR_ERR_AGGREGATE]) { scr_dyn_obj_set(d, "errors", 6, scr_aggregate_errors((ScrError *)e)); - d->v.obj.entries[d->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(d)[d->v.obj.len - 1].enumerable = false; } /* An own `name` is assigned after construction defines the others. */ if (e->name_present) { scr_dyn_obj_set(d, "name", 4, scr_dyn_new_str(e->name)); - d->v.obj.entries[d->v.obj.len - 1].enumerable = e->name_enumerable; + scr_dyn_obj_dict_entries(d)[d->v.obj.len - 1].enumerable = e->name_enumerable; } return d; } @@ -9987,7 +10476,7 @@ ScrDyn *scr_error_dyn_fields(const ScrError *e) { * has no own `name` or no `after` property. */ void scr_error_dyn_place_name(ScrDyn *view, const char *after, size_t after_len) { if (view->kind != SCR_DYN_OBJ) return; - ScrDynEntry *entries = view->v.obj.entries; + ScrDynEntry *entries = scr_dyn_obj_dict_entries(view); size_t n = view->v.obj.len, from = n, to = n; for (size_t i = 0; i < n; i++) { if (entries[i].key_len == 4 && memcmp(entries[i].key, "name", 4) == 0) from = i; @@ -10000,14 +10489,14 @@ void scr_error_dyn_place_name(ScrDyn *view, const char *after, size_t after_len) memmove(&entries[from], &entries[from + 1], (n - from - 1) * sizeof *entries); memmove(&entries[at + 1], &entries[at], (n - 1 - at) * sizeof *entries); entries[at] = name; - SCR_JS_SHAPE_RESET(view); free(view->v.obj.index); view->v.obj.index = NULL; } static size_t scr_dyn_entry_index(const ScrDyn *object, const char *key, size_t key_len) { for (size_t i = 0; i < object->v.obj.len; i++) { - const ScrDynEntry *entry = &object->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(object, i); + const ScrDynProp *entry = &entry_prop; if (entry->key_len == key_len && memcmp(entry->key, key, key_len) == 0) return i; } return SIZE_MAX; @@ -10018,7 +10507,8 @@ static size_t scr_dyn_entry_index(const ScrDyn *object, const char *key, size_t * starting with '%'), and a `message` that directly follows them. */ static bool scr_error_bag_constructed(const ScrDyn *bag, size_t b) { for (size_t i = 0; i <= b; i++) { - const ScrDynEntry *entry = &bag->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(bag, i); + const ScrDynProp *entry = &entry_prop; bool constructed = (entry->key_len == 5 && memcmp(entry->key, "stack", 5) == 0) || (entry->key_len > 0 && entry->key[0] == '%') || (entry->key_len == 7 && memcmp(entry->key, "message", 7) == 0); @@ -10040,7 +10530,8 @@ static size_t scr_error_bag_place(const ScrDyn *view, const ScrDyn *bag, size_t bool name_placed, size_t from) { bool late = name || !scr_error_bag_constructed(bag, b); for (size_t p = b; p-- > 0;) { - const ScrDynEntry *anchor = &bag->v.obj.entries[p]; + const ScrDynProp anchor_prop = scr_dyn_obj_prop_at(bag, p); + const ScrDynProp *anchor = &anchor_prop; if (late && scr_error_bag_constructed(bag, p)) break; if (late && name_placed && anchor->key_len == 4 && memcmp(anchor->key, "name", 4) == 0) continue; size_t after = scr_dyn_entry_index(view, anchor->key, anchor->key_len); @@ -10048,7 +10539,7 @@ static size_t scr_error_bag_place(const ScrDyn *view, const ScrDyn *bag, size_t } if (!late) return 0; for (size_t p = b + 1; p < bag->v.obj.len; p++) { - size_t before = scr_dyn_entry_index(view, bag->v.obj.entries[p].key, bag->v.obj.entries[p].key_len); + size_t before = scr_dyn_entry_index(view, scr_dyn_obj_dict_entries(bag)[p].key, scr_dyn_obj_dict_entries(bag)[p].key_len); if (before != SIZE_MAX) return before > from ? before - 1 : before; } return from; @@ -10068,16 +10559,17 @@ void scr_error_dyn_follow_bag(ScrDyn *view, const ScrDyn *bag, const ScrError *e bool name = s == 0; if (!(name ? e->name_present : e->message_present)) continue; size_t at = scr_dyn_entry_index(view, slots[s].key, slots[s].len); - if (at == SIZE_MAX || view->v.obj.entries[at].accessor) continue; + if (at == SIZE_MAX || scr_dyn_obj_dict_entries(view)[at].accessor) continue; ScrDyn *value = scr_dyn_new_str(name ? e->name : e->message); scr_dyn_region_store(view, value); - ScrDyn *old = view->v.obj.entries[at].value; - view->v.obj.entries[at].value = value; + ScrDyn *old = scr_dyn_obj_dict_entries(view)[at].value; + scr_dyn_obj_dict_entries(view)[at].value = value; scr_dyn_release(old); } bool moved = false; for (size_t b = 0; b < bag->v.obj.len; b++) { - const ScrDynEntry *bag_entry = &bag->v.obj.entries[b]; + const ScrDynProp bag_entry_prop = scr_dyn_obj_prop_at(bag, b); + const ScrDynProp *bag_entry = &bag_entry_prop; bool slot = (bag_entry->key_len == 4 && memcmp(bag_entry->key, "name", 4) == 0) || (bag_entry->key_len == 7 && memcmp(bag_entry->key, "message", 7) == 0); bool name = bag_entry->key_len == 4; @@ -10087,7 +10579,7 @@ void scr_error_dyn_follow_bag(ScrDyn *view, const ScrDyn *bag, const ScrError *e if (from == SIZE_MAX) continue; size_t to = scr_error_bag_place(view, bag, b, name, name_placed, from); if (to == from) continue; - ScrDynEntry *entries = view->v.obj.entries; + ScrDynEntry *entries = scr_dyn_obj_dict_entries(view); ScrDynEntry entry = entries[from]; if (to < from) memmove(&entries[to + 1], &entries[to], (from - to) * sizeof *entries); else memmove(&entries[from], &entries[from + 1], (to - from) * sizeof *entries); @@ -10095,7 +10587,6 @@ void scr_error_dyn_follow_bag(ScrDyn *view, const ScrDyn *bag, const ScrError *e moved = true; } if (moved) { - SCR_JS_SHAPE_RESET(view); free(view->v.obj.index); view->v.obj.index = NULL; } @@ -10194,7 +10685,8 @@ void scr_error_commit_dyn(ScrError *e, const ScrDyn *view) { e->error_cause = replacement; e->cause_enumerable = false; for (size_t i = 0; cause && i < view->v.obj.len; i++) { - const ScrDynEntry *entry = &view->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(view, i); + const ScrDynProp *entry = &entry_prop; if (entry->key_len == 5 && memcmp(entry->key, "cause", 5) == 0) e->cause_enumerable = entry->enumerable; } } @@ -10228,7 +10720,8 @@ ScrError *scr_error_from_dyn(const ScrDyn *d) { scr_error_install_cause_drop(&scr_error_cause_drop_impl); e->error_cause = scr_dyn_retain(cause); for (size_t i = 0; i < d->v.obj.len; i++) { - const ScrDynEntry *entry = &d->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(d, i); + const ScrDynProp *entry = &entry_prop; if (entry->key_len == 5 && memcmp(entry->key, "cause", 5) == 0) e->cause_enumerable = entry->enumerable; } } @@ -11337,6 +11830,25 @@ bool scr_dyn_in(const ScrDyn *v, const ScrStr *key) { /* Node's TypeErrors for assigning a read-only property and deleting a * non-configurable one of a checked object: the object reads as * #, or [object Object] without a prototype. */ +/* Failed assignments. Sloppy-mode code ignores an assignment that strict + * mode refuses with a TypeError (a read-only or getter-only property, a + * non-extensible object, a frozen array element). The write paths raise + * the strict TypeError and note its payload here; the sloppy entry points + * (scr_dyn_key_set_sloppy) drop exactly that exception when it reaches + * them, while any other exception (a setter's) keeps propagating. */ +static SCR_TL void *scr_dyn_assign_failure; + +static void scr_dyn_note_assign_failure(void) { + if (scr_exc_pending()) scr_dyn_assign_failure = scr_exc_current_cell()->payload; +} + +static void scr_dyn_sloppy_settle(void) { + if (scr_dyn_assign_failure && scr_exc_pending() && + scr_exc_current_cell()->payload == scr_dyn_assign_failure) + scr_exc_clear(); + scr_dyn_assign_failure = NULL; +} + static void scr_dyn_throw_property(const ScrDyn *object, const char *key, size_t key_len, bool deleting) { const char *name = scr_dyn_class_name(object); ScrJsonBuf buffer; @@ -11358,13 +11870,25 @@ static void scr_dyn_throw_property(const ScrDyn *object, const char *key, size_t } if (!deleting) scr_jb_puts(&buffer, "'"); scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&buffer)); + if (!deleting) scr_dyn_note_assign_failure(); } /* One lookup of the own property decides the assignment: an own property * is written (or runs its setter, or refuses the write); an absent one * consults the prototype chain and is then added. */ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, const ScrDyn *receiver) { - ScrDynEntry *entry = scr_dyn_entry(recv, key); + if (recv->v.obj.layout) { + ScrDyn **slot = scr_dyn_layout_slot(recv, key->data, key->len); + if (slot) { + scr_dyn_region_store(recv, value); + ScrDyn *old = *slot; + *slot = scr_dyn_retain(value); + scr_dyn_release(old); + scr_error_sync_cause(recv, key); + return; + } + } + ScrDynEntry *entry = recv->v.obj.layout ? NULL : scr_dyn_entry(recv, key); if (entry) { if (entry->accessor && entry->setter) { ScrDyn *args[] = {value}; @@ -11396,6 +11920,13 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con ScrDyn *table = current->kind == SCR_DYN_FUNC ? scr_dyn_fn_properties(current) : NULL; const ScrDyn *object = table ? table : current; if (object->kind != SCR_DYN_OBJ) { scr_dyn_release(table); break; } + if (object->v.obj.layout) { + /* A layout-mode prototype's properties are writable data. */ + bool found = scr_layout_find(object->v.obj.layout, key->data, key->len) != UINT32_MAX; + current = found ? NULL : scr_dyn_proto_next(object); + scr_dyn_release(table); + continue; + } ScrDynEntry *inherited = scr_dyn_entry((ScrDyn *)object, key); if (!inherited) { current = scr_dyn_proto_next(object); @@ -11429,6 +11960,7 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con scr_jb_write(&buffer, key->data, key->len); scr_jb_puts(&buffer, ", object is not extensible"); scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&buffer)); + scr_dyn_note_assign_failure(); return; } scr_dyn_obj_set(recv, key->data, key->len, scr_dyn_retain(value)); @@ -11456,6 +11988,7 @@ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value) { if (recv->kind == SCR_DYN_PROXY) { scr_dyn_proxy_set(recv, key, value); return; } if (recv->kind == SCR_DYN_TYPED_REF) { if (scr_typed_ref_array_set(recv, key, value)) return; + if (scr_typed_ref_record_set(recv, key, value)) return; if (scr_event_ref_hooks && scr_event_ref_hooks->set(recv, key, value)) return; ScrDyn *materialized = scr_dyn_typed_ref_materialize(recv); scr_dyn_key_set(materialized, key, value); @@ -11494,6 +12027,7 @@ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value) { if (key->len == 6 && memcmp(key->data, "length", 6) == 0) { if (recv->v.arr.frozen) { scr_array_frozen_error(3, recv->v.arr.len, 0, 0, 0); + scr_dyn_note_assign_failure(); return; } double number; @@ -11513,6 +12047,7 @@ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value) { while (recv->v.arr.len > length) { if (recv->v.arr.sealed && scr_dyn_arr_has_index(recv, recv->v.arr.len - 1)) { scr_array_frozen_error(5, recv->v.arr.len, (double)(recv->v.arr.len - 1), 0, 0); + scr_dyn_note_assign_failure(); return; } scr_dyn_release(recv->v.arr.items[--recv->v.arr.len]); @@ -11532,11 +12067,13 @@ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value) { if (scr_dyn_array_prototype_element_fence(recv)) return; if (recv->v.arr.frozen && idx < recv->v.arr.len && scr_dyn_arr_has_index(recv, idx)) { scr_array_frozen_error(0, recv->v.arr.len, (double)idx, 0, 0); + scr_dyn_note_assign_failure(); return; } if (recv->non_extensible && !scr_dyn_arr_has_index(recv, idx)) { /* An absent index, a hole included, is a property to add. */ scr_array_frozen_error(0, 0, (double)idx, 0, 0); + scr_dyn_note_assign_failure(); return; } if (idx > recv->v.arr.len + 1000000) { @@ -11633,6 +12170,19 @@ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value) { scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&b)); } +/* Sloppy-mode assignments (the compiler's IrModule.sloppyDynWrites). */ +void scr_dyn_key_set_sloppy(ScrDyn *recv, ScrStr *key, ScrDyn *value) { + scr_dyn_assign_failure = NULL; + scr_dyn_key_set(recv, key, value); + scr_dyn_sloppy_settle(); +} + +void scr_dyn_key_set_computed_sloppy(ScrDyn *recv, ScrDyn *key, ScrDyn *value) { + scr_dyn_assign_failure = NULL; + scr_dyn_key_set_computed(recv, key, value); + scr_dyn_sloppy_settle(); +} + void scr_dyn_key_set_computed(ScrDyn *recv, ScrDyn *key, ScrDyn *value) { if (key->kind == SCR_DYN_SYMBOL) { scr_dyn_symbol_key_set(recv, key, value); @@ -12553,6 +13103,13 @@ static void scr_json_fit_array(ScrDyn *arr) { static void scr_json_fit_object(ScrDyn *obj) { size_t len = obj->v.obj.len, cap = obj->v.obj.cap; if (len == 0 || len == cap) return; + if (obj->v.obj.layout) { + ScrDyn **slots = scr_json_fit_block(obj->v.obj.slots, len * sizeof *slots); + if (!slots) return; /* keep the larger block */ + obj->v.obj.slots = slots; + obj->v.obj.cap = len; + return; + } for (size_t i = len; i < cap; i++) { ScrDynEntry *entry = &obj->v.obj.entries[i]; scr_dyn_key_release(entry->key, entry->key_len); @@ -12616,6 +13173,7 @@ static ScrDyn *scr_json_array(ScrJsonP *p) { typedef struct ScrJsonShape { char *key; size_t key_len; + const ScrLayout *layout; /* the object's layout after this transition */ uint32_t child; /* first transition; 0 = none (the root is never a child) */ uint32_t sibling; /* next transition from the same layout; 0 = none */ } ScrJsonShape; @@ -12649,13 +13207,14 @@ static inline uint32_t scr_json_shape_follow(ScrJsonP *p, uint32_t shape) { return 0; } -/* Record `entry`'s key as a transition from `shape`; 0 when the budget is - * spent (the object then finishes on the ordinary path). */ -static uint32_t scr_json_shape_add(ScrJsonP *p, uint32_t shape, const ScrDynEntry *entry) { +/* Record the last key of `layout`, an object's layout after a put appended + * it, as a transition from `shape`; 0 when the budget is spent (the object + * then finishes on the ordinary path). */ +static uint32_t scr_json_shape_add(ScrJsonP *p, uint32_t shape, const ScrLayout *layout) { if (!p->shapes) { p->shapes = malloc(16 * sizeof *p->shapes); if (!p->shapes) scr_json_oom(); - p->shapes[0] = (ScrJsonShape){0}; + p->shapes[0] = (ScrJsonShape){NULL, 0, &scr_layout_empty, 0, 0}; p->nshapes = 1; p->shape_cap = 16; } @@ -12667,41 +13226,18 @@ static uint32_t scr_json_shape_add(ScrJsonP *p, uint32_t shape, const ScrDynEntr p->shape_cap *= 2; } uint32_t added = p->nshapes++; - ((ScrDynKey *)(entry->key - offsetof(ScrDynKey, bytes)))->rc++; - p->shapes[added] = (ScrJsonShape){entry->key, entry->key_len, 0, p->shapes[shape].child}; + const ScrLayoutKey *key = &layout->keys[layout->count - 1]; + scr_dyn_key_retain(key->key); + p->shapes[added] = (ScrJsonShape){key->key, key->len, layout, 0, p->shapes[shape].child}; p->shapes[shape].child = added; return added; } -/* scr_dyn_obj_put for a fresh parser object and a key it does not hold yet, - * sharing the transition's key bytes. */ -static void scr_dyn_obj_append_shared(ScrDyn *obj, char *key, size_t key_len, ScrDyn *value) { +/* scr_dyn_obj_put for a fresh parser object in layout mode and a key it + * does not hold yet: the transition's layout extends the object's. */ +static void scr_dyn_obj_append_shared(ScrDyn *obj, const ScrLayout *layout, ScrDyn *value) { scr_dyn_region_store(obj, value); - if (obj->v.obj.len == obj->v.obj.cap) { - size_t cap = obj->v.obj.cap ? obj->v.obj.cap * 2 : 4; - ScrDynEntry *entries = scr_mem_realloc(obj->v.obj.entries, cap * sizeof *entries); - if (!entries) scr_json_oom(); - memset(entries + obj->v.obj.cap, 0, (cap - obj->v.obj.cap) * sizeof *entries); - obj->v.obj.entries = entries; - obj->v.obj.cap = cap; - } - SCR_JS_SHAPE_RESET(obj); - ScrDynEntry *e = &obj->v.obj.entries[obj->v.obj.len++]; - if (e->key != key) { - ((ScrDynKey *)(key - offsetof(ScrDynKey, bytes)))->rc++; - scr_dyn_key_release(e->key, e->key_len); - e->key = key; - } - e->key_len = key_len; - e->value = value; - e->getter = NULL; - e->setter = NULL; - e->accessor = false; - e->fenced = false; - e->writable = true; - e->enumerable = true; - e->configurable = true; - if (obj->v.obj.len >= SCR_DYN_OBJECT_INDEX_MIN) scr_dyn_index_append(obj); + scr_dyn_obj_layout_push(obj, layout, value); } static ScrDyn *scr_json_object(ScrJsonP *p) { @@ -12776,14 +13312,14 @@ static ScrDyn *scr_json_object(ScrJsonP *p) { return NULL; } if (shared_key) { - scr_dyn_obj_append_shared(obj, shared_key, key_len, value); + scr_dyn_obj_append_shared(obj, p->shapes[follow].layout, value); shape = follow; } else { size_t before = obj->v.obj.len; scr_dyn_obj_put(obj, key, key_len, value); /* later duplicate keys win */ if (shaped) { - uint32_t added = !decoded_key && obj->v.obj.len == before + 1 - ? scr_json_shape_add(p, shape, &obj->v.obj.entries[before]) : 0; + uint32_t added = !decoded_key && obj->v.obj.len == before + 1 && obj->v.obj.layout + ? scr_json_shape_add(p, shape, obj->v.obj.layout) : 0; if (added) shape = added; else shaped = false; } @@ -13594,7 +14130,7 @@ static ScrDyn *scr_json_member(const ScrDyn *holder, const ScrStr *key) { static void scr_json_delete_member(ScrDyn *object, const ScrStr *key) { for (size_t i = 0; i < object->v.obj.len; i++) { - ScrDynEntry *entry = &object->v.obj.entries[i]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(object)[i]; if (entry->key_len != key->len || memcmp(entry->key, key->data, key->len) != 0) continue; if (entry->fenced) { scr_dyn_namespace_member_fence(); @@ -13603,12 +14139,11 @@ static void scr_json_delete_member(ScrDyn *object, const ScrStr *key) { ScrDynEntry removed = *entry; memmove(entry, entry + 1, (object->v.obj.len - i - 1) * sizeof *entry); object->v.obj.len--; - SCR_JS_SHAPE_RESET(object); free(object->v.obj.index); object->v.obj.index = NULL; /* The moved tail aliases a live entry; spare slots own only their own * retained key bytes, never a second pointer to a surviving property. */ - memset(&object->v.obj.entries[object->v.obj.len], 0, sizeof *entry); + memset(&scr_dyn_obj_dict_entries(object)[object->v.obj.len], 0, sizeof *entry); scr_dyn_key_release(removed.key, removed.key_len); scr_dyn_release(removed.value); scr_dyn_release(removed.getter); @@ -13983,7 +14518,8 @@ static bool scr_json_plain_container(const ScrDyn *value) { } if (value->kind != SCR_DYN_OBJ) return false; for (size_t i = 0; i < value->v.obj.len; i++) { - const ScrDynEntry *entry = &value->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(value, i); + const ScrDynProp *entry = &entry_prop; if (!entry->enumerable) continue; double index; if (entry->accessor || (!intact && !scr_json_plain_value(entry->value)) || @@ -14006,7 +14542,8 @@ static __attribute__((noinline)) bool scr_json_plain_write(ScrJsonBuf *buffer, c scr_jb_putc(buffer, array ? '[' : '{'); bool first = true; for (size_t i = 0; i < length; i++) { - const ScrDynEntry *entry = array ? NULL : &value->v.obj.entries[i]; + const ScrDynProp entry_prop = array ? (ScrDynProp){0} : scr_dyn_obj_prop_at(value, i); + const ScrDynProp *entry = array ? NULL : &entry_prop; if (entry && !entry->enumerable) continue; const ScrDyn *child = array ? value->v.arr.items[i] : entry->value; bool omitted = scr_json_omitted(child); @@ -14308,6 +14845,13 @@ ScrStr *scr_util_format_j_text(ScrStr *text) { /* The gap of a compact walk; the walks read only its length. */ static const ScrStr scr_json_compact_gap = { .rc = SIZE_MAX, .len = 0 }; +/* JSON for a prefix-viewed record: its own capsule's. */ +static void scr_jb_put_record_prefix(ScrJsonBuf *b, void *rec) { + ScrDyn *capsule = scr_dyn_new_typed_ref(rec, NULL, NULL, "record:", 7, NULL, NULL); + scr_json_replaced_write(b, capsule, scr_dyn_undefined(), NULL, &scr_json_compact_gap, b->seen_len); + scr_dyn_release(capsule); +} + #define SCR_JSON_PURE_DEPTH 32 /* Writes `value` when writing it runs no program code and meets no cycle: @@ -14342,7 +14886,8 @@ static bool scr_jb_put_pure(ScrJsonBuf *b, const ScrDyn *value, size_t depth) { scr_jb_putc(b, '{'); bool first = true; for (size_t i = 0; i < value->v.obj.len; i++) { - const ScrDynEntry *entry = &value->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(value, i); + const ScrDynProp *entry = &entry_prop; if (!entry->enumerable) continue; double index; if (entry->accessor || (entry->key_len == 6 && memcmp(entry->key, "toJSON", 6) == 0) || @@ -14873,14 +15418,15 @@ static ScrDyn *scr_sc_clone(const ScrDyn *v, const ScrScParent *up, ScrDynRegion } ScrDyn *out = scr_dyn_new_obj(); for (size_t i = 0; i < v->v.obj.len; i++) { - const ScrDynEntry *e = &v->v.obj.entries[i]; - if (scr_dyn_entry_is_error_stack(e)) { + const ScrDynProp e_prop = scr_dyn_obj_prop_at(v, i); + const ScrDynProp *e = &e_prop; + if (scr_dyn_setter_is_error_stack(e->accessor, e->setter)) { /* A stack that was already formatted copies as a non-enumerable * data property; an unformatted one has nothing to copy. */ ScrError *error = scr_errdyn_err_of(v); if (error && error->stack) { scr_dyn_obj_set(out, "stack", 5, scr_dyn_new_str(error->stack)); - out->v.obj.entries[out->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(out)[out->v.obj.len - 1].enumerable = false; } scr_error_release(error); continue; @@ -14993,6 +15539,11 @@ static ScrDyn *scr_dyn_obj_read_receiver(const ScrDyn *d, const char *key, size_ continue; } if (current->kind != SCR_DYN_OBJ) continue; + if (current->v.obj.layout) { + ScrDyn **slot = scr_dyn_layout_slot(current, key, key_len); + if (slot) return scr_dyn_retain(*slot); + continue; + } ScrDynEntry *entry = scr_dyn_find_entry(current, key, key_len); if (entry) { if (!entry->accessor) return scr_dyn_retain(entry->value); @@ -15118,9 +15669,38 @@ static void scr_dyn_read_ics_reset(void) { scr_dyn_read_ics_registered = false; } +#ifndef SCR_RC_AUDIT +/* Points the site at `key` (shared key bytes). */ +static void scr_dyn_read_ic_key(ScrDynReadIc *ic, char *key, size_t key_len) { + if (ic->key == key) return; + scr_dyn_key_retain(key); + if (ic->key) { + scr_dyn_key_release(ic->key, ic->key_len); + } else { + if (!scr_dyn_read_ics_registered) { + scr_dyn_read_ics_registered = true; + scr_atexit(scr_dyn_read_ics_reset); + } + ic->next = scr_dyn_read_ics; + scr_dyn_read_ics = ic; + } + ic->key = key; + ic->key_len = key_len; +} +#endif + static __attribute__((noinline)) ScrDyn *scr_dyn_obj_read_ic_miss(const ScrDyn *d, const ScrStr *key, ScrDynReadIc *ic) { #ifndef SCR_RC_AUDIT + if (d->kind == SCR_DYN_OBJ && d->v.obj.layout) { + uint32_t i = d != scr_builtin_object_prototype + ? scr_layout_find(d->v.obj.layout, key->data, key->len) : UINT32_MAX; + if (i == UINT32_MAX) return scr_dyn_obj_read_receiver(d, key->data, key->len, d); + const ScrLayoutKey *k = &d->v.obj.layout->keys[i]; + scr_dyn_read_ic_key(ic, k->key, k->len); + ic->index = i; + return scr_dyn_retain(d->v.obj.slots[i]); + } ScrDynEntry *entry = d != scr_builtin_object_prototype ? scr_dyn_find_entry(d, key->data, key->len) : NULL; if (entry && !entry->accessor) { if (ic->key != entry->key) { @@ -15151,6 +15731,16 @@ static __attribute__((noinline)) ScrDyn *scr_dyn_obj_read_ic_miss(const ScrDyn * * do not, and compare their bytes at the remembered position instead. */ ScrDyn *scr_dyn_obj_read_ic(const ScrDyn *d, const ScrStr *key, ScrDynReadIc *ic) { size_t index = ic->index; + const ScrLayout *layout = d->kind == SCR_DYN_OBJ ? d->v.obj.layout : NULL; + if (layout) { + if (index < d->v.obj.len) { + const ScrLayoutKey *k = &layout->keys[index]; + if ((k->key == ic->key || (k->len == key->len && scr_key_equal(k->key, key->data, key->len))) && + d != scr_builtin_object_prototype) + return scr_dyn_retain(d->v.obj.slots[index]); + } + return scr_dyn_obj_read_ic_miss(d, key, ic); + } if (index < d->v.obj.len) { const ScrDynEntry *entry = &d->v.obj.entries[index]; if ((entry->key == ic->key || @@ -15347,7 +15937,8 @@ static size_t *scr_dyn_obj_key_order(const ScrDyn *v) { ScrKeyIndex *indices = NULL; size_t count = 0; for (size_t i = 0; i < n; i++) { - const ScrDynEntry *entry = &v->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(v, i); + const ScrDynProp *entry = &entry_prop; uint32_t index; if (!scr_key_array_index(entry->key, entry->key_len, &index)) continue; if (!indices) { @@ -15364,7 +15955,8 @@ static size_t *scr_dyn_obj_key_order(const ScrDyn *v) { for (size_t i = 0; i < count; i++) order[i] = indices[i].entry; free(indices); for (size_t i = 0; i < n; i++) { - const ScrDynEntry *entry = &v->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(v, i); + const ScrDynProp *entry = &entry_prop; uint32_t index; if (!scr_key_array_index(entry->key, entry->key_len, &index)) order[count++] = i; } @@ -15375,8 +15967,9 @@ static size_t *scr_dyn_obj_key_order(const ScrDyn *v) { * walk over its values runs no user code and the object cannot change * under it. */ static bool scr_dyn_obj_data_only(const ScrDyn *v) { + if (v->v.obj.layout) return true; for (size_t i = 0; i < v->v.obj.len; i++) - if (v->v.obj.entries[i].accessor) return false; + if (scr_dyn_obj_dict_entries(v)[i].accessor) return false; return true; } @@ -15385,7 +15978,8 @@ ScrDyn *scr_dyn_obj_own_keys(const ScrDyn *v) { ScrDyn *keys = scr_dyn_new_arr(); scr_dyn_arr_reserve(keys, v->v.obj.len); for (size_t i = 0; i < v->v.obj.len; i++) { - const ScrDynEntry *entry = &v->v.obj.entries[order ? order[i] : i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(v, order ? order[i] : i); + const ScrDynProp *entry = &entry_prop; scr_dyn_arr_push(keys, scr_dyn_objwalk_key(entry->key, entry->key_len)); } free(order); @@ -15598,7 +16192,8 @@ static ScrDyn *scr_dyn_obj_data_walk(const ScrDyn *v, ScrObjWalk mode) { ScrDyn *out = scr_dyn_new_arr(); scr_dyn_arr_reserve(out, v->v.obj.len); for (size_t i = 0; i < v->v.obj.len; i++) { - const ScrDynEntry *entry = &v->v.obj.entries[order ? order[i] : i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(v, order ? order[i] : i); + const ScrDynProp *entry = &entry_prop; if (entry->enumerable) scr_dyn_objwalk_push(out, mode, entry->key, entry->key_len, entry->value); } @@ -15623,7 +16218,8 @@ static ScrDyn *scr_dyn_objwalk(const ScrDyn *v, ScrObjWalk mode) { * the shared numeric/insertion ordering over a names-only table. */ ScrDyn *names = scr_dyn_new_obj(); for (size_t i = 0; i < v->v.proxy.target->v.obj.len; i++) { - const ScrDynEntry *entry = &v->v.proxy.target->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(v->v.proxy.target, i); + const ScrDynProp *entry = &entry_prop; scr_dyn_obj_set(names, entry->key, entry->key_len, scr_dyn_retain(scr_dyn_undefined())); } raw = scr_dyn_obj_keys(names); @@ -15665,7 +16261,8 @@ static ScrDyn *scr_dyn_objwalk(const ScrDyn *v, ScrObjWalk mode) { scr_dyn_release(raw); const ScrDyn *target = v->v.proxy.target; for (size_t i = 0; target->kind == SCR_DYN_OBJ && i < target->v.obj.len; i++) { - const ScrDynEntry *entry = &target->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(target, i); + const ScrDynProp *entry = &entry_prop; if (entry->configurable) continue; bool found = false; for (size_t j = 0; j < keys->v.arr.len; j++) { @@ -15934,7 +16531,8 @@ static ScrArr *scr_dyn_own_key_strs(const ScrDyn *v, bool shared) { size_t *order = scr_dyn_obj_key_order(v); ScrArr *out = scr_arr_new(SCR_ELEM_STR, v->v.obj.len); for (size_t i = 0; i < v->v.obj.len; i++) { - const ScrDynEntry *entry = &v->v.obj.entries[order ? order[i] : i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(v, order ? order[i] : i); + const ScrDynProp *entry = &entry_prop; if (entry->enumerable) scr_arr_push_ref(out, scr_str_new(entry->key, entry->key_len)); } free(order); @@ -16004,7 +16602,8 @@ ScrDynEntryPairs *scr_dyn_obj_entry_pairs(const ScrDyn *v) { size_t *order = scr_dyn_obj_key_order(v); ScrDynEntryPairs *out = scr_dyn_entry_pairs_new(v->v.obj.len); for (size_t i = 0; i < v->v.obj.len; i++) { - const ScrDynEntry *entry = &v->v.obj.entries[order ? order[i] : i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(v, order ? order[i] : i); + const ScrDynProp *entry = &entry_prop; if (!entry->enumerable) continue; out->pairs[out->count++] = (ScrDynEntryPair){scr_str_new(entry->key, entry->key_len), scr_dyn_retain(entry->value)}; @@ -16067,7 +16666,8 @@ ScrArr *scr_dyn_obj_value_list(const ScrDyn *v) { uint64_t *slots = scr_dyn_value_slots(v->v.obj.len, stack, 16); size_t count = 0; for (size_t i = 0; i < v->v.obj.len; i++) { - const ScrDynEntry *entry = &v->v.obj.entries[order ? order[i] : i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(v, order ? order[i] : i); + const ScrDynProp *entry = &entry_prop; if (entry->enumerable) slots[count++] = (uint64_t)(uintptr_t)scr_dyn_retain(entry->value); } free(order); @@ -16211,6 +16811,8 @@ ScrDyn *scr_dyn_assign(ScrDyn *target, const ScrDyn *src) { return NULL; } + if (target->kind == SCR_DYN_TYPED_REF && scr_typed_ref_record_assign(target, src)) + return scr_exc_pending() ? NULL : scr_dyn_retain(target); if (target->kind == SCR_DYN_TYPED_REF) { ScrDyn *materialized = scr_dyn_typed_ref_materialize(target); ScrDyn *assigned = scr_dyn_assign(materialized, src); @@ -16318,7 +16920,7 @@ static void scr_dyn_copy_property_descriptor(ScrDyn *target, const ScrDyn *src, ScrDyn *setter = accessor && from->setter ? scr_dyn_retain(from->setter) : NULL; ScrStr *name = slot ? scr_str_new(from->key, from->key_len) : NULL; scr_dyn_obj_set(target, from->key, from->key_len, value); - ScrDynEntry *entry = &target->v.obj.entries[target->v.obj.len - 1]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(target)[target->v.obj.len - 1]; entry->accessor = accessor; entry->getter = getter; entry->setter = setter; @@ -16341,7 +16943,7 @@ ScrDyn *scr_dyn_copy_property_descriptors(ScrDyn *target, const ScrDyn *src) { if (src->v.obj.len) scr_dyn_region_store(target, NULL); size_t *order = scr_dyn_obj_key_order(src); for (size_t i = 0; i < src->v.obj.len && !scr_exc_pending(); i++) - scr_dyn_copy_property_descriptor(target, src, &src->v.obj.entries[order ? order[i] : i]); + scr_dyn_copy_property_descriptor(target, src, &scr_dyn_obj_dict_entries(src)[order ? order[i] : i]); free(order); } else { ScrDyn *keys = scr_dyn_obj_own_keys(src); @@ -16673,9 +17275,9 @@ ScrError *scr_error_new_options(int kind, const ScrDyn *message, const ScrDyn *o static void scr_suppressed_attach(ScrError *e, ScrDyn *error, ScrDyn *suppressed) { ScrDyn *view = scr_dyn_from_error(e); scr_dyn_obj_set(view, "error", 5, error); - view->v.obj.entries[view->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(view)[view->v.obj.len - 1].enumerable = false; scr_dyn_obj_set(view, "suppressed", 10, suppressed); - view->v.obj.entries[view->v.obj.len - 1].enumerable = false; + scr_dyn_obj_dict_entries(view)[view->v.obj.len - 1].enumerable = false; scr_dyn_release(view); } @@ -17274,7 +17876,8 @@ static ScrDyn *scr_ffi_open(ScrDyn *catalog, const ScrDyn *path, const ScrDyn *d if (scr_ffi_text_equal(scr_dyn_obj_get(catalog->v.arr.items[i], "library", 7), path)) found = true; if (!found) goto invalid; for (size_t i = 0; i < definitions->v.obj.len; i++) { - const ScrDynEntry *requested = &definitions->v.obj.entries[i]; + const ScrDynProp requested_prop = scr_dyn_obj_prop_at(definitions, i); + const ScrDynProp *requested = &requested_prop; if (!requested->enumerable) continue; if (requested->accessor || requested->value->kind != SCR_DYN_OBJ) goto invalid; ScrDyn *entry = NULL; @@ -18484,7 +19087,7 @@ ScrDyn *scr_dyn_freeze_data(ScrDyn *value) { if (value->kind == SCR_DYN_OBJ && !value->copied_from_native) { value->non_extensible = true; for (size_t i = 0; i < value->v.obj.len; i++) { - ScrDynEntry *entry = &value->v.obj.entries[i]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(value)[i]; entry->configurable = false; if (!entry->accessor) entry->writable = false; } @@ -18520,7 +19123,8 @@ bool scr_dyn_is_frozen(const ScrDyn *value) { if (value->kind == SCR_DYN_OBJ && !value->copied_from_native) { if (!value->non_extensible) return false; for (size_t i = 0; i < value->v.obj.len; i++) { - const ScrDynEntry *entry = &value->v.obj.entries[i]; + const ScrDynProp entry_prop = scr_dyn_obj_prop_at(value, i); + const ScrDynProp *entry = &entry_prop; if (entry->configurable || (!entry->accessor && entry->writable)) return false; } return true; @@ -18582,7 +19186,7 @@ void scr_dyn_class_field_attributes(ScrDyn *view, ScrStr *name, double bits) { static void scr_dyn_class_table_attributes(ScrDyn *table, unsigned b) { table->non_extensible = true; for (size_t i = 0; i < table->v.obj.len; i++) { - ScrDynEntry *entry = &table->v.obj.entries[i]; + ScrDynEntry *entry = &scr_dyn_obj_dict_entries(table)[i]; if (b & (SCR_CLASS_SEALED | SCR_CLASS_FROZEN)) entry->configurable = false; if ((b & SCR_CLASS_FROZEN) && !entry->accessor) entry->writable = false; } @@ -18803,12 +19407,8 @@ bool scr_dyn_iterates_by_protocol(const ScrDyn *value) { const ScrDyn *scr_dyn_tier_object_prototype(void) { return scr_builtin_object_prototype; } -char *scr_dyn_tier_key_copy(const char *key, size_t key_len) { return scr_dyn_key_copy(key, key_len); } - -void scr_dyn_tier_key_release(char *key, size_t key_len) { scr_dyn_key_release(key, key_len); } - -void scr_dyn_tier_append(ScrDyn *obj, char *key, size_t key_len, ScrDyn *value) { - scr_dyn_obj_append_shared(obj, key, key_len, value); +void scr_dyn_tier_append(ScrDyn *obj, const ScrLayout *next, ScrDyn *value) { + scr_dyn_obj_append_shared(obj, next, value); } #endif diff --git a/packages/runtime/src/scr_json.test.c b/packages/runtime/src/scr_json.test.c index 29ba4bfe3..4ecae73b5 100644 --- a/packages/runtime/src/scr_json.test.c +++ b/packages/runtime/src/scr_json.test.c @@ -10,6 +10,7 @@ #ifdef SCR_RC_AUDIT long scr_str_live_count(void); long scr_dyn_key_live_count(void); +void scr_layout_reset_idle(void); #endif static void put(ScrDyn *map, ScrDyn *key, ScrDyn *value) { @@ -167,7 +168,7 @@ static void checked_storage(void) { char key[16]; int length = snprintf(key, sizeof key, "key-%d", i); scr_dyn_obj_set(object, key, (size_t)length, scr_dyn_new_num(i)); - assert((object->v.obj.index != NULL) == (i >= 31)); + assert((object->v.obj.index != NULL) == (i >= (int)SCR_LAYOUT_MAX_KEYS)); } for (int i = 0; i < 129; i++) { char key[16]; @@ -229,6 +230,8 @@ static void checked_storage(void) { static void shared_property_keys(void) { #ifdef SCR_RC_AUDIT + /* Layouts hold key references until the thread's state is torn down. */ + scr_layout_reset_idle(); long before = scr_dyn_key_live_count(); #endif /* Distinct objects share bytes, not values or property attributes. Key @@ -238,7 +241,8 @@ static void shared_property_keys(void) { char key[] = "shared\0name"; scr_dyn_obj_set(first, key, sizeof key - 1, scr_dyn_new_num(1)); scr_dyn_obj_set(second, key, sizeof key - 1, scr_dyn_new_num(2)); - assert(first->v.obj.entries[0].key == second->v.obj.entries[0].key); + size_t first_len, second_len; + assert(scr_dyn_obj_key_at(first, 0, &first_len) == scr_dyn_obj_key_at(second, 0, &second_len)); key[0] = 'x'; ScrStr *name = scr_str_new("shared\0name", sizeof key - 1); scr_dyn_key_delete(first, name, true); @@ -269,6 +273,7 @@ static void shared_property_keys(void) { } } #ifdef SCR_RC_AUDIT + scr_layout_reset_idle(); assert(scr_dyn_key_live_count() == before); #endif } @@ -651,8 +656,10 @@ static void json_schema_parse(void) { static const char *const order[] = {"tags", "ok", "extra", "qty", "sku"}; assert(view->kind == SCR_DYN_OBJ && view->v.obj.len == 5); for (size_t i = 0; i < 5; i++) { - assert(view->v.obj.entries[i].key_len == strlen(order[i])); - assert(memcmp(view->v.obj.entries[i].key, order[i], strlen(order[i])) == 0); + size_t key_len; + const char *key = scr_dyn_obj_key_at(view, i, &key_len); + assert(key_len == strlen(order[i])); + assert(memcmp(key, order[i], strlen(order[i])) == 0); } assert(scr_dyn_obj_get(view, "extra", 5)->kind == SCR_DYN_OBJ); test_item_release(second); diff --git a/packages/runtime/src/scr_json.test.ts b/packages/runtime/src/scr_json.test.ts index e181674ce..88a89c321 100644 --- a/packages/runtime/src/scr_json.test.ts +++ b/packages/runtime/src/scr_json.test.ts @@ -30,6 +30,7 @@ test.each([false, true])( "scr_json.c", "scr_string.c", "scr_array.c", + "scr_arena.c", "scr_map.c", "scr_exception.c", "scr_error.c", diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 928400da3..0b780603b 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -716,8 +716,9 @@ ScrDyn *scr_process_versions(void) { scr_dyn_obj_set(scr_versions_object, "openssl", 7, scr_dyn_new_str(openssl)); scr_str_release(node); scr_str_release(openssl); - for (size_t i = 0; i < scr_versions_object->v.obj.len; i++) { - scr_versions_object->v.obj.entries[i].writable = false; + for (size_t i = 0; i < scr_dyn_obj_count(scr_versions_object); i++) { + ScrDynProp prop = scr_dyn_obj_prop_at(scr_versions_object, i); + scr_dyn_obj_set_attrs_at(scr_versions_object, i, false, prop.enumerable, prop.configurable); } scr_atexit(&scr_versions_object_cleanup); } @@ -914,8 +915,8 @@ static void scr_env_store(ScrDyn *out, const char *text, size_t ks, size_t ke, } static int scr_env_entry_order(const void *left, const void *right) { - const ScrDynEntry *a = *(const ScrDynEntry *const *)left; - const ScrDynEntry *b = *(const ScrDynEntry *const *)right; + const ScrDynProp *a = left; + const ScrDynProp *b = right; size_t n = a->key_len < b->key_len ? a->key_len : b->key_len; int cmp = memcmp(a->key, b->key, n); return cmp ? cmp : (a->key_len > b->key_len) - (a->key_len < b->key_len); @@ -991,14 +992,14 @@ static ScrDyn *scr_env_parse_content(const ScrStr *input, bool ordinary) { /* Node's std::map orders UTF-8 keys before installing object properties. * Ordinary objects ignore string-valued __proto__ assignments; loading * the native environment retains that name as an ordinary variable. */ - size_t count = entries->v.obj.len; - const ScrDynEntry **sorted = count ? malloc(count * sizeof(*sorted)) : NULL; + size_t count = scr_dyn_obj_count(entries); + ScrDynProp *sorted = count ? malloc(count * sizeof(*sorted)) : NULL; if (count && !sorted) scr_trap("scriptc: out of memory\n"); - for (size_t i = 0; i < count; i++) sorted[i] = &entries->v.obj.entries[i]; + for (size_t i = 0; i < count; i++) sorted[i] = scr_dyn_obj_prop_at(entries, i); if (count > 1) qsort(sorted, count, sizeof(*sorted), scr_env_entry_order); ScrDyn *result = ordinary ? scr_dyn_new_obj() : scr_dyn_new_obj_null_proto(); for (size_t i = 0; i < count; i++) { - const ScrDynEntry *entry = sorted[i]; + const ScrDynProp *entry = &sorted[i]; if (ordinary && entry->key_len == 9 && memcmp(entry->key, "__proto__", 9) == 0) continue; if (!ordinary) { scr_dyn_obj_set(result, entry->key, entry->key_len, scr_dyn_retain(entry->value)); @@ -1103,15 +1104,15 @@ void scr_process_load_env_file(const ScrDyn *value) { if (!content) return; ScrDyn *entries = scr_env_parse_content(content, false); scr_str_release(content); - for (size_t i = 0; i < entries->v.obj.len; i++) { - const ScrDynEntry *entry = &entries->v.obj.entries[i]; + for (size_t i = 0; i < scr_dyn_obj_count(entries); i++) { + ScrDynProp entry = scr_dyn_obj_prop_at(entries, i); /* Node normalizes each raw sorted entry immediately before checking * the environment. Distinct invalid byte keys can normalize alike; * the first absent name wins, rather than the last parsed entry. */ - ScrStr *key = scr_str_from_utf8_lossy((const uint8_t *)entry->key, entry->key_len); + ScrStr *key = scr_str_from_utf8_lossy((const uint8_t *)entry.key, entry.key_len); ScrStr *existing = scr_env_get(key); if (!existing) { - const ScrStr *raw = entry->value->v.str; + const ScrStr *raw = entry.value->v.str; ScrStr *value = scr_str_from_utf8_lossy((const uint8_t *)raw->data, raw->len); scr_env_set(key, value); scr_str_release(value); @@ -2810,10 +2811,9 @@ ScrDyn *scr_process_release(void) { #endif #undef SCR_RELEASE_FILE #undef SCR_RELEASE_PREFIX - for (size_t i = 0; i < scr_release_object->v.obj.len; i++) { - scr_release_object->v.obj.entries[i].writable = false; - scr_release_object->v.obj.entries[i].configurable = false; - } + for (size_t i = 0; i < scr_dyn_obj_count(scr_release_object); i++) + scr_dyn_obj_set_attrs_at(scr_release_object, i, false, + scr_dyn_obj_prop_at(scr_release_object, i).enumerable, false); scr_atexit(&scr_release_object_cleanup); } return scr_dyn_retain(scr_release_object); diff --git a/packages/runtime/src/scr_map.test.ts b/packages/runtime/src/scr_map.test.ts index 88aef9c63..b567884fc 100644 --- a/packages/runtime/src/scr_map.test.ts +++ b/packages/runtime/src/scr_map.test.ts @@ -27,6 +27,7 @@ test("number-key maps keep Map semantics across linear, direct and hashed indexe "scr_json.c", "scr_string.c", "scr_array.c", + "scr_arena.c", "scr_map.c", "scr_exception.c", "scr_error.c", diff --git a/packages/runtime/src/scr_message.c b/packages/runtime/src/scr_message.c index d4eb263cf..382907219 100644 --- a/packages/runtime/src/scr_message.c +++ b/packages/runtime/src/scr_message.c @@ -636,9 +636,9 @@ static bool scr_publish_container_ok(const ScrDyn *d) { if (!scr_dyn_is_frozen(d)) return false; if (d->kind == SCR_DYN_ARR) return d->v.arr.properties == NULL; if (d->null_proto || d->v.obj.source_identity || d->v.obj.source_access) return false; - for (size_t i = 0; i < d->v.obj.len; i++) { - const ScrDynEntry *e = &d->v.obj.entries[i]; - if (e->accessor || !e->enumerable) return false; + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(d, i); + if (e.accessor || !e.enumerable) return false; } return true; } @@ -666,8 +666,8 @@ static bool scr_publish_collect(ScrPublish *p, ScrDyn *root) { break; case SCR_DYN_OBJ: if (!scr_publish_container_ok(d)) return false; - for (size_t i = 0; i < d->v.obj.len; i++) { - ScrDyn *child = d->v.obj.entries[i].value; + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { + ScrDyn *child = scr_dyn_obj_value_at(d, i); if (child && !scr_publish_seen(p, child)) scr_publish_push(p, child); } break; @@ -702,9 +702,10 @@ static ScrMessage *scr_message_try_publish(const ScrDyn *value) { } else if (d->rc != SIZE_MAX) { scr_cyc_on_dead(d); /* leave the candidate and tenured buffers */ d->rc = SIZE_MAX; - /* A shape names a node of this thread's transition tree (scr_js.h), - * which other threads cannot use and which ends with this thread. */ - if (d->kind == SCR_DYN_OBJ) d->v.obj.shape = NULL; + /* A layout names a node of this thread's transition tree + * (scr_dyn.h), which other threads cannot use and which ends with + * this thread: the object moves to dictionary mode. */ + if (d->kind == SCR_DYN_OBJ) scr_dyn_to_dictionary(d); #ifdef SCR_RC_AUDIT scr_dyn_live_forget(); #endif @@ -1033,14 +1034,15 @@ ScrDyn *scr_message_decode(const ScrMessage *message) { } else scr_dyn_obj_set(values[i], property->key, property->length, value); } if (node->kind == SCR_MESSAGE_ERROR) { - for (size_t j = 0; j < values[i]->v.obj.len; j++) { - ScrDynEntry *entry = &values[i]->v.obj.entries[j]; - entry->enumerable = false; + for (size_t j = 0; j < scr_dyn_obj_count(values[i]); j++) { + ScrDynProp entry = scr_dyn_obj_prop_at(values[i], j); + bool enumerable = false; for (size_t k = 0; k < node->count; k++) { const ScrMessageProperty *property = &node->properties[k]; - if (!property->hidden && property->length == entry->key_len && !memcmp(property->key, entry->key, entry->key_len)) - entry->enumerable = true; + if (!property->hidden && property->length == entry.key_len && !memcmp(property->key, entry.key, entry.key_len)) + enumerable = true; } + scr_dyn_obj_set_attrs_at(values[i], j, entry.writable, enumerable, entry.configurable); } ScrError *error = scr_errdyn_err_of(values[i]); scr_error_commit_dyn(error, values[i]); diff --git a/packages/runtime/src/scr_message.test.c b/packages/runtime/src/scr_message.test.c index b6adf34c5..10459255b 100644 --- a/packages/runtime/src/scr_message.test.c +++ b/packages/runtime/src/scr_message.test.c @@ -225,8 +225,11 @@ static void check_large(void) { ScrDyn *decoded = scr_message_decode(message); assert(decoded && !scr_exc_pending()); assert(decoded->v.obj.len == 42); - assert(decoded->v.obj.entries[0].key_len == 5 && !memcmp(decoded->v.obj.entries[0].key, "item0", 5)); - assert(decoded->v.obj.entries[41].key_len == 6 && !memcmp(decoded->v.obj.entries[41].key, "nested", 6)); + size_t member_len; + const char *member = scr_dyn_obj_key_at(decoded, 0, &member_len); + assert(member_len == 5 && !memcmp(member, "item0", 5)); + member = scr_dyn_obj_key_at(decoded, 41, &member_len); + assert(member_len == 6 && !memcmp(member, "nested", 6)); assert(scr_dyn_obj_get(decoded, "nested", 6)->v.num == 42); ScrDyn *first = scr_dyn_obj_get(decoded, "item0", 5); for (int i = 0; i < 40; i++) { diff --git a/packages/runtime/src/scr_os.c b/packages/runtime/src/scr_os.c index 8bc476045..637bb7e61 100644 --- a/packages/runtime/src/scr_os.c +++ b/packages/runtime/src/scr_os.c @@ -745,9 +745,8 @@ double scr_os_cpus_field(double i, double field) { static void scr_os_constant(ScrDyn *o, const char *name, double value) { scr_dyn_obj_set(o, name, strlen(name), scr_dyn_new_num(value)); - ScrDynEntry *entry = &o->v.obj.entries[o->v.obj.len - 1]; - entry->writable = false; - entry->configurable = false; + size_t last = scr_dyn_obj_count(o) - 1; + scr_dyn_obj_set_attrs_at(o, last, false, scr_dyn_obj_prop_at(o, last).enumerable, false); } #define SCR_OS_CONSTANT(o, name) scr_os_constant((o), #name, (double)(name)) @@ -1514,10 +1513,10 @@ ScrDyn *scr_node_constants(const ScrDyn *crypto_table) { scr_os_signal_constants(o); scr_fs_constants(o); if (crypto && crypto->kind == SCR_DYN_OBJ) { - for (size_t i = 0; i < crypto->v.obj.len; i++) { - const ScrDynEntry *entry = &crypto->v.obj.entries[i]; - if (entry->accessor || !entry->value) continue; - scr_dyn_obj_set(o, entry->key, entry->key_len, scr_dyn_retain(entry->value)); + for (size_t i = 0; i < scr_dyn_obj_count(crypto); i++) { + ScrDynProp entry = scr_dyn_obj_prop_at(crypto, i); + if (entry.accessor || !entry.value) continue; + scr_dyn_obj_set(o, entry.key, entry.key_len, scr_dyn_retain(entry.value)); } } scr_dyn_release(crypto); @@ -1718,10 +1717,12 @@ double scr_util_signal_exit_code(const ScrDyn *code) { ScrJsonBuf reason; scr_jb_init(&reason); scr_jb_puts(&reason, "must be one of: "); - for (size_t i = 0; i < signals->v.obj.len; i++) { + for (size_t i = 0; i < scr_dyn_obj_count(signals); i++) { if (i > 0) scr_jb_puts(&reason, ", "); scr_jb_putc(&reason, '\''); - scr_jb_write(&reason, signals->v.obj.entries[i].key, signals->v.obj.entries[i].key_len); + size_t key_len; + const char *key = scr_dyn_obj_key_at(signals, i, &key_len); + scr_jb_write(&reason, key, key_len); scr_jb_putc(&reason, '\''); } ScrStr *text = scr_jb_finish(&reason); diff --git a/packages/runtime/src/scr_publish.c b/packages/runtime/src/scr_publish.c index 8d11d3806..1b29a63aa 100644 --- a/packages/runtime/src/scr_publish.c +++ b/packages/runtime/src/scr_publish.c @@ -278,8 +278,8 @@ static bool scr_pub_dyn_ok(const ScrDyn *d, const char **why) { *why = "Cannot publish an Object object"; return false; } - for (size_t i = 0; i < d->v.obj.len; i++) - if (d->v.obj.entries[i].accessor) { + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) + if (scr_dyn_obj_prop_at(d, i).accessor) { *why = "Cannot publish an accessor property"; return false; } @@ -325,9 +325,9 @@ void scr_pub_dyn_fn(void *v, ScrPub *ctx) { case SCR_DYN_ARR: case SCR_DYN_OBJ: { if (!scr_pub_dyn_ok(d, &why)) break; - size_t n = d->kind == SCR_DYN_ARR ? d->v.arr.len : d->v.obj.len; + size_t n = d->kind == SCR_DYN_ARR ? d->v.arr.len : scr_dyn_obj_count(d); for (size_t i = 0; i < n; i++) { - ScrDyn *child = d->kind == SCR_DYN_ARR ? d->v.arr.items[i] : d->v.obj.entries[i].value; + ScrDyn *child = d->kind == SCR_DYN_ARR ? d->v.arr.items[i] : scr_dyn_obj_value_at(d, i); if (child && child->rc != SIZE_MAX) SCR_PUB_ADD((uintptr_t)child); } break; @@ -481,9 +481,9 @@ void scr_publish(void *root, ScrPubFn fn) { void *obj = scr_pub_log_obj(&ctx.log[i]); if (ctx.log[i].obj & SCR_PUB_DYN) { ScrDyn *d = obj; - /* Other threads share the object; its shape belongs to this one - * (scr_js.h). */ - if (d->kind == SCR_DYN_OBJ) d->v.obj.shape = NULL; + /* Other threads share the object; layouts belong to this thread + * (scr_dyn.h), so it moves to dictionary mode. */ + if (d->kind == SCR_DYN_OBJ) scr_dyn_to_dictionary(d); scr_dyn_release(scr_dyn_freeze_data(d)); } #ifdef SCR_RC_AUDIT diff --git a/packages/runtime/src/scr_region.c b/packages/runtime/src/scr_region.c index 25bbcf7ae..dacd9f163 100644 --- a/packages/runtime/src/scr_region.c +++ b/packages/runtime/src/scr_region.c @@ -858,12 +858,12 @@ static void scr_rgn_walk_dyn(ScrRgnWalk *w, ScrDyn *d) { break; case SCR_DYN_OBJ: /* source_identity is owned through its access callback: not walked. */ - for (size_t i = 0; i < d->v.obj.len; i++) { - const ScrDynEntry *e = &d->v.obj.entries[i]; - if (e->accessor) { - scr_rgn_enter(w, e->getter, SCR_RGN_W_DYN, NULL); - scr_rgn_enter(w, e->setter, SCR_RGN_W_DYN, NULL); - } else scr_rgn_enter(w, e->value, SCR_RGN_W_DYN, NULL); + for (size_t i = 0; i < scr_dyn_obj_count(d); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(d, i); + if (e.accessor) { + scr_rgn_enter(w, e.getter, SCR_RGN_W_DYN, NULL); + scr_rgn_enter(w, e.setter, SCR_RGN_W_DYN, NULL); + } else scr_rgn_enter(w, e.value, SCR_RGN_W_DYN, NULL); } break; case SCR_DYN_FUNC: scr_rgn_enter(w, d->v.fn.clo, SCR_RGN_W_CLOSURE, NULL); break; diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 93ecaa468..f828f31f6 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -56,6 +56,7 @@ #include "scr_blob.h" #include "scr_object.h" #include "scr_region.h" +#include "scr_arena.h" #include "scr_itable.h" #include "scr_slots.h" #include "scr_async.h" diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 71d629941..62a155bd1 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -706,31 +706,42 @@ static inline size_t scr_utf16_units_short(const char *data, size_t len) { } /* A fresh concatenation (no head, the template-literal shape): numbers - * format straight into the result, which reserves scr_f64_to_str's whole - * 32-byte buffer for each. */ + * format into a stack buffer first, so the result is allocated at its exact + * length. */ #define SCR_F64_TEXT_MAX 32 static __attribute__((noinline)) ScrStr *scr_str_concat_fresh(ScrStr *const *parts, const double *nums, size_t count) { - size_t reserve = 0; + char texts[SCR_STR_MIXED_MAX * SCR_F64_TEXT_MAX]; + size_t text_len[SCR_STR_MIXED_MAX]; + size_t len = 0, used = 0; bool ascii = true; for (size_t i = 0; i < count; i++) { - size_t n = parts[i] ? parts[i]->len : SCR_F64_TEXT_MAX; - if (n > SCR_STR_MAX_CAP - reserve) scr_oom(); - reserve += n; - if (parts[i]) ascii &= parts[i]->ascii; + size_t n; + if (parts[i]) { + n = parts[i]->len; + ascii &= parts[i]->ascii; + } else { + n = scr_f64_to_str(nums[i], texts + used); + text_len[i] = n; + used += n; + } + if (n > SCR_STR_MAX_CAP - len) scr_oom(); + len += n; } - ScrStr *out = scr_str_take_spare(reserve); - if (!out) out = scr_str_alloc(reserve, reserve); + ScrStr *out = scr_str_take_spare(len); + if (!out) out = scr_str_alloc(len, len); char *w = out->data; + const char *text = texts; for (size_t i = 0; i < count; i++) { if (parts[i]) { scr_copy_bytes(w, parts[i]->data, parts[i]->len); w += parts[i]->len; } else { - w += scr_f64_to_str(nums[i], w); + scr_copy_bytes(w, text, text_len[i]); + w += text_len[i]; + text += text_len[i]; } } - size_t len = (size_t)(w - out->data); out->len = len; out->data[len] = '\0'; out->ascii = ascii; @@ -2172,11 +2183,19 @@ double scr_str_char_code_at(ScrStr *s, double i) { } double scr_str_index_of(ScrStr *s, ScrStr *needle, double fromIndex) { - /* A search from the start of a proven-ASCII receiver: byte offsets are - * UTF-16 indexes, and a needle with other bytes is never found. A start - * index that is not positive (NaN, -0 or negative) clamps to 0. */ - if (s->ascii && !(fromIndex > 0)) { - const char *found = scr_byte_find(s->data, s->len, needle->data, needle->len); + /* A proven-ASCII receiver: byte offsets are UTF-16 indexes, and a needle + * with other bytes is never found. A start index that is not positive + * (NaN, -0 or negative) clamps to 0, a positive one truncates + * (ToIntegerOrInfinity) and clamps to the length, where only the empty + * needle is found. */ + if (s->ascii) { + size_t start = 0; + if (fromIndex > 0) { + if (fromIndex >= (double)s->len) return needle->len == 0 ? (double)s->len : -1.0; + start = (size_t)fromIndex; + } + const char *found = + scr_byte_find(s->data + start, s->len - start, needle->data, needle->len); return found ? (double)(found - s->data) : -1.0; } double pos = scr_to_integer_or_infinity(fromIndex); diff --git a/packages/runtime/src/scr_symbol.c b/packages/runtime/src/scr_symbol.c index 7e573fc28..372e60702 100644 --- a/packages/runtime/src/scr_symbol.c +++ b/packages/runtime/src/scr_symbol.c @@ -333,7 +333,7 @@ static void scr_dirent_init(void) { scr_dyn_obj_set(proto, methods[i].name, strlen(methods[i].name), scr_dyn_new_func(scr_closure_new(NULL, 0), methods[i].thunk, 0, "native:fs.Dirent", methods[i].name)); - proto->v.obj.entries[proto->v.obj.len - 1].enumerable = false; + scr_dyn_obj_set_attrs_at(proto, scr_dyn_obj_count(proto) - 1, true, false, true); } ScrStr *desc = scr_str_new("type", 4); ScrSym *sym = scr_sym_new(desc); @@ -350,15 +350,14 @@ void scr_dyn_dirent_finish(ScrDyn *obj) { scr_dyn_region_store(obj, NULL); /* a prototype store (ScrDynRegion) */ scr_dyn_release(obj->prototype); obj->prototype = scr_dyn_retain(scr_dirent_prototype); - for (size_t i = 0; i < obj->v.obj.len; i++) { - ScrDynEntry *ent = &obj->v.obj.entries[i]; - if (ent->key_len != 6 || memcmp(ent->key, "%dtype", 6) != 0) continue; - ent->enumerable = false; + size_t i = scr_dyn_obj_index_of(obj, "%dtype", 6); + if (i != SIZE_MAX) { + ScrDynProp ent = scr_dyn_obj_prop_at(obj, i); + scr_dyn_obj_set_attrs_at(obj, i, ent.writable, false, ent.configurable); if (!obj->symbol_properties && !obj->symbol_keys) - scr_dyn_symbol_init_own(obj, scr_dirent_type_key, scr_dirent_type_storage, ent->value); + scr_dyn_symbol_init_own(obj, scr_dirent_type_key, scr_dirent_type_storage, ent.value); else - scr_dyn_symbol_key_set(obj, scr_dirent_type_key, ent->value); - break; + scr_dyn_symbol_key_set(obj, scr_dirent_type_key, ent.value); } } diff --git a/packages/runtime/src/scr_tls.c b/packages/runtime/src/scr_tls.c index 547fef27a..76ec48e6c 100644 --- a/packages/runtime/src/scr_tls.c +++ b/packages/runtime/src/scr_tls.c @@ -2597,21 +2597,21 @@ static bool scr_tls_srv_opts_walk_ex(const ScrDyn *opts, const char *api, ScrByt } if (!scr_tls_opts_validate(opts)) return false; /* Node's typed errors first */ bool ok = true; - for (size_t i = 0; ok && i < opts->v.obj.len; i++) { - const ScrDynEntry *e = &opts->v.obj.entries[i]; - const ScrDyn *v = e->value; + for (size_t i = 0; ok && i < scr_dyn_obj_count(opts); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(opts, i); + const ScrDyn *v = e.value; if (v->kind == SCR_DYN_UNDEF) continue; /* an undefined member reads as absent, like Node */ - if (strcmp(e->key, "cert") == 0 || strcmp(e->key, "key") == 0) { + if (strcmp(e.key, "cert") == 0 || strcmp(e.key, "key") == 0) { char what[96]; - snprintf(what, sizeof what, "a %s '%s' option", api, e->key); + snprintf(what, sizeof what, "a %s '%s' option", api, e.key); ScrBytes *b = scr_tls_pem_dyn(v, what, false); if (b == NULL) { ok = false; break; } - ScrBytes **slot = e->key[0] == 'c' ? cert_out : key_out; + ScrBytes **slot = e.key[0] == 'c' ? cert_out : key_out; scr_bytes_release(*slot); *slot = b; continue; } - if (strcmp(e->key, "ca") == 0) { + if (strcmp(e.key, "ca") == 0) { /* the server's `ca` verifies client certificates and completes the * server's own chain (scr_tls_srv_client_auth); a walk without * client certificates drops it */ @@ -2623,7 +2623,7 @@ static bool scr_tls_srv_opts_walk_ex(const ScrDyn *opts, const char *api, ScrByt } continue; } - if (strcmp(e->key, "requestCert") == 0) { + if (strcmp(e.key, "requestCert") == 0) { if (auth != NULL) { auth->request = scr_tls_dyn_truthy(v); continue; @@ -2635,22 +2635,22 @@ static bool scr_tls_srv_opts_walk_ex(const ScrDyn *opts, const char *api, ScrByt } continue; } - if (strcmp(e->key, "rejectUnauthorized") == 0) { + if (strcmp(e.key, "rejectUnauthorized") == 0) { if (auth != NULL) auth->reject = scr_tls_dyn_not_false(v); continue; /* without client certificates it has no effect */ } - if (server_ok && (strcmp(e->key, "ALPNProtocols") == 0 || strcmp(e->key, "minVersion") == 0 || - strcmp(e->key, "maxVersion") == 0)) + if (server_ok && (strcmp(e.key, "ALPNProtocols") == 0 || strcmp(e.key, "minVersion") == 0 || + strcmp(e.key, "maxVersion") == 0)) continue; - if (strcmp(e->key, "SNICallback") == 0) { + if (strcmp(e.key, "SNICallback") == 0) { scr_tls_opt_fence(api, "SNICallback", "SNICallback lowers only on http2.createSecureServer({ allowHTTP1: true, ... }) with a literal options object"); ok = false; continue; } for (size_t k = 0; SCR_TLS_SRV_FENCED_OPTIONS[k] != NULL; k++) { - if (strcmp(e->key, SCR_TLS_SRV_FENCED_OPTIONS[k]) == 0) { - scr_tls_opt_fence(api, e->key, + if (strcmp(e.key, SCR_TLS_SRV_FENCED_OPTIONS[k]) == 0) { + scr_tls_opt_fence(api, e.key, "cert and key (PEM strings or Buffers) are the honestly-implemented members of a runtime options record"); ok = false; break; @@ -2957,17 +2957,17 @@ static const char *const SCR_TLS_WRAP_FENCED_OPTIONS[] = { * certificates are not modeled), SNICallback and ALPNCallback fence. * False with the exception pending. */ static bool scr_tls_wrap_socket_opts_check(const ScrDyn *record, const char *api) { - for (size_t i = 0; record != NULL && i < record->v.obj.len; i++) { - const ScrDynEntry *e = &record->v.obj.entries[i]; - if (e->value->kind == SCR_DYN_UNDEF) continue; - if (strcmp(e->key, "requestCert") == 0) { - if (!scr_tls_dyn_truthy(e->value)) continue; + for (size_t i = 0; record != NULL && i < scr_dyn_obj_count(record); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(record, i); + if (e.value->kind == SCR_DYN_UNDEF) continue; + if (strcmp(e.key, "requestCert") == 0) { + if (!scr_tls_dyn_truthy(e.value)) continue; scr_tls_opt_fence(api, "requestCert", "requestCert lowers with cert and key options, not beside a secureContext"); return false; } - if (strcmp(e->key, "SNICallback") == 0 || strcmp(e->key, "ALPNCallback") == 0) { - scr_tls_opt_fence(api, e->key, + if (strcmp(e.key, "SNICallback") == 0 || strcmp(e.key, "ALPNCallback") == 0) { + scr_tls_opt_fence(api, e.key, "isServer, cert, key, secureContext, ALPNProtocols, minVersion and maxVersion are the implemented members"); return false; } @@ -2980,21 +2980,21 @@ ScrNetSocket *scr_tls_wrap_socket(ScrNetSocket *raw, const ScrDyn *opts /*borrow static const char api[] = "new tls.TLSSocket"; bool is_server = false; const ScrDyn *record = opts != NULL && opts->kind == SCR_DYN_OBJ ? opts : NULL; - for (size_t i = 0; record != NULL && i < record->v.obj.len; i++) { - const ScrDynEntry *e = &record->v.obj.entries[i]; - if (e->value->kind == SCR_DYN_UNDEF) continue; - if (strcmp(e->key, "isServer") == 0) { - is_server = scr_tls_dyn_truthy(e->value); + for (size_t i = 0; record != NULL && i < scr_dyn_obj_count(record); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(record, i); + if (e.value->kind == SCR_DYN_UNDEF) continue; + if (strcmp(e.key, "isServer") == 0) { + is_server = scr_tls_dyn_truthy(e.value); continue; } - if (strcmp(e->key, "secureContext") == 0) { + if (strcmp(e.key, "secureContext") == 0) { scr_tls_opt_fence(api, "secureContext", "secureContext lowers as a member of an options object literal"); return NULL; } for (size_t k = 0; SCR_TLS_WRAP_FENCED_OPTIONS[k] != NULL; k++) { - if (strcmp(e->key, SCR_TLS_WRAP_FENCED_OPTIONS[k]) == 0) { - scr_tls_opt_fence(api, e->key, + if (strcmp(e.key, SCR_TLS_WRAP_FENCED_OPTIONS[k]) == 0) { + scr_tls_opt_fence(api, e.key, "isServer, cert, key, secureContext, ALPNProtocols, minVersion and maxVersion are the implemented members"); return NULL; } @@ -3267,10 +3267,10 @@ static ScrNetSocket *scr_tls_connect_plain(double port, ScrStr *host /*borrowed, /* createSecureContext's version checks come before its other options * (Node's SecureContext constructor). */ if (ok && !scr_tls_versions_dyn(opts, &min_version, &max_version)) ok = false; - for (size_t i = 0; ok && i < opts->v.obj.len; i++) { - const ScrDynEntry *e = &opts->v.obj.entries[i]; - const ScrDyn *v = e->value; - if (strcmp(e->key, "checkServerIdentity") == 0) { + for (size_t i = 0; ok && i < scr_dyn_obj_count(opts); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(opts, i); + const ScrDyn *v = e.value; + if (strcmp(e.key, "checkServerIdentity") == 0) { /* Node spreads user options over the defaults, so a PRESENT key * replaces the builtin verifier even when its value is undefined * — validateFunction then throws for anything non-callable. A @@ -3287,7 +3287,7 @@ static ScrNetSocket *scr_tls_connect_plain(double port, ScrStr *host /*borrowed, continue; } if (v->kind == SCR_DYN_UNDEF) continue; - if (strcmp(e->key, "port") == 0) { + if (strcmp(e.key, "port") == 0) { if (port >= 0) continue; /* the argument form wins, like Node */ if (v->kind == SCR_DYN_NUM) port = v->v.num; else if (v->kind == SCR_DYN_STR) port = strtod(v->v.str->data, NULL); /* Node coerces numeric strings */ @@ -3297,7 +3297,7 @@ static ScrNetSocket *scr_tls_connect_plain(double port, ScrStr *host /*borrowed, } continue; } - if (strcmp(e->key, "host") == 0) { + if (strcmp(e.key, "host") == 0) { if (v->kind == SCR_DYN_STR) opt_host = v->v.str; else { scr_dyn_arg_type_fail("options.host", "of type string", v); @@ -3305,34 +3305,34 @@ static ScrNetSocket *scr_tls_connect_plain(double port, ScrStr *host /*borrowed, } continue; } - if (strcmp(e->key, "rejectUnauthorized") == 0) { + if (strcmp(e.key, "rejectUnauthorized") == 0) { reject = scr_tls_dyn_not_false(v); continue; } - if (strcmp(e->key, "ca") == 0) { + if (strcmp(e.key, "ca") == 0) { scr_bytes_release(ca); if (!scr_tls_ca_dyn(v, "a tls.connect 'ca' option", &ca)) ok = false; continue; } - if (strcmp(e->key, "cert") == 0 || strcmp(e->key, "key") == 0) { - ScrBytes **slot = e->key[0] == 'c' ? &own_cert : &own_key; + if (strcmp(e.key, "cert") == 0 || strcmp(e.key, "key") == 0) { + ScrBytes **slot = e.key[0] == 'c' ? &own_cert : &own_key; scr_bytes_release(*slot); - *slot = scr_tls_pem_dyn(v, e->key[0] == 'c' ? "a tls.connect 'cert' option" : "a tls.connect 'key' option", + *slot = scr_tls_pem_dyn(v, e.key[0] == 'c' ? "a tls.connect 'cert' option" : "a tls.connect 'key' option", false); if (*slot == NULL) ok = false; continue; } - if (strcmp(e->key, "ALPNProtocols") == 0) { + if (strcmp(e.key, "ALPNProtocols") == 0) { alpn_value = v; /* converted after the context options, as TLSSocket does */ continue; } - if (strcmp(e->key, "minVersion") == 0 || strcmp(e->key, "maxVersion") == 0) continue; - if (strcmp(e->key, "timeout") == 0) { + if (strcmp(e.key, "minVersion") == 0 || strcmp(e.key, "maxVersion") == 0) continue; + if (strcmp(e.key, "timeout") == 0) { /* `if (options.timeout) tlssock.setTimeout(options.timeout)` */ timeout_value = scr_tls_dyn_truthy(v) ? v : NULL; continue; } - if (strcmp(e->key, "socket") == 0) { + if (strcmp(e.key, "socket") == 0) { if (v->kind == SCR_DYN_HANDLE && v->v.handle.tag == SCR_DYNH_NET_SOCKET) { wrap = (ScrNetSocket *)v->v.handle.ptr; } else { @@ -3341,7 +3341,7 @@ static ScrNetSocket *scr_tls_connect_plain(double port, ScrStr *host /*borrowed, } continue; } - if (strcmp(e->key, "servername") == 0) { + if (strcmp(e.key, "servername") == 0) { if (v->kind == SCR_DYN_STR) servername = v->v.str; else { scr_dyn_arg_type_fail("options.servername", "of type string", v); @@ -3350,8 +3350,8 @@ static ScrNetSocket *scr_tls_connect_plain(double port, ScrStr *host /*borrowed, continue; } for (size_t k = 0; SCR_TLS_CONNECT_FENCED_OPTIONS[k] != NULL; k++) { - if (strcmp(e->key, SCR_TLS_CONNECT_FENCED_OPTIONS[k]) == 0) { - scr_tls_opt_fence("tls.connect", e->key, + if (strcmp(e.key, SCR_TLS_CONNECT_FENCED_OPTIONS[k]) == 0) { + scr_tls_opt_fence("tls.connect", e.key, "port/host/socket/rejectUnauthorized/ca/servername/ALPNProtocols/minVersion/maxVersion/timeout are the implemented members of a runtime options record"); ok = false; break; diff --git a/packages/runtime/src/scr_util_compare.c b/packages/runtime/src/scr_util_compare.c index 5a15b662f..4c2cb9d16 100644 --- a/packages/runtime/src/scr_util_compare.c +++ b/packages/runtime/src/scr_util_compare.c @@ -26,15 +26,15 @@ static bool util_deep_equal(const ScrDyn *, const ScrDyn *, bool, const ScrDeepF static size_t util_deep_key_count(const ScrDyn *table) { size_t count = 0; - if (table) for (size_t i = 0; i < table->v.obj.len; i++) count += table->v.obj.entries[i].enumerable; + if (table) for (size_t i = 0; i < scr_dyn_obj_count(table); i++) count += scr_dyn_obj_prop_at(table, i).enumerable; return count; } static bool util_deep_same_keys(const ScrDyn *a, const ScrDyn *b) { if (util_deep_key_count(a) != util_deep_key_count(b)) return false; - if (a) for (size_t i = 0; i < a->v.obj.len; i++) { - const ScrDynEntry *e = &a->v.obj.entries[i]; - if (e->enumerable && (!b || !scr_dyn_obj_enumerable(b, e->key, e->key_len))) return false; + if (a) for (size_t i = 0; i < scr_dyn_obj_count(a); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(a, i); + if (e.enumerable && (!b || !scr_dyn_obj_enumerable(b, e.key, e.key_len))) return false; } return true; } @@ -42,14 +42,14 @@ static bool util_deep_same_keys(const ScrDyn *a, const ScrDyn *b) { static ScrDyn *util_deep_constructor(const ScrDyn *value) { for (const ScrDyn *p = value; p; p = p->prototype) { const ScrDyn *table = p->kind == SCR_DYN_OBJ ? p : p->kind == SCR_DYN_ARR ? p->v.arr.properties : NULL; - if (table) for (size_t i = 0; i < table->v.obj.len; i++) { - const ScrDynEntry *e = &table->v.obj.entries[i]; - if (e->key_len == 11 && !memcmp(e->key, "constructor", 11)) { - if (e->accessor) { util_deep_refused("native accessor properties"); return NULL; } - if (p == value && e->value->kind == SCR_DYN_FUNC) { + if (table) for (size_t i = 0; i < scr_dyn_obj_count(table); i++) { + ScrDynProp e = scr_dyn_obj_prop_at(table, i); + if (e.key_len == 11 && !memcmp(e.key, "constructor", 11)) { + if (e.accessor) { util_deep_refused("native accessor properties"); return NULL; } + if (p == value && e.value->kind == SCR_DYN_FUNC) { util_deep_refused("native own constructor functions"); return NULL; } - return scr_dyn_retain(e->value); + return scr_dyn_retain(e.value); } } if (!p->prototype) { @@ -77,8 +77,8 @@ static ScrDyn *util_deep_tag(const ScrDyn *value) { if (!scr_dyn_same_value(keys->v.arr.items[i], key)) continue; // General native getters retain an explicit boundary, including // non-enumerable Symbol.toStringTag descriptors read by Node. - for (size_t j = 0; j < p->symbol_properties->v.obj.len; j++) { - if (p->symbol_properties->v.obj.entries[j].accessor) { + for (size_t j = 0; j < scr_dyn_obj_count(p->symbol_properties); j++) { + if (scr_dyn_obj_prop_at(p->symbol_properties, j).accessor) { scr_dyn_release(key); util_deep_refused("native accessor properties"); return NULL; } } @@ -95,20 +95,21 @@ static bool util_deep_properties(const ScrDyn *a, const ScrDyn *b, bool skip, const ScrDeepFrame *frame) { if (util_deep_key_count(a) != util_deep_key_count(b)) return false; if (!a) return true; - for (size_t i = 0; i < a->v.obj.len; i++) { - const ScrDynEntry *x = &a->v.obj.entries[i]; - if (!x->enumerable) continue; - const ScrDynEntry *y = NULL; - if (b) for (size_t j = 0; j < b->v.obj.len; j++) { - const ScrDynEntry *candidate = &b->v.obj.entries[j]; - if (candidate->enumerable && candidate->key_len == x->key_len && - !memcmp(candidate->key, x->key, x->key_len)) { y = candidate; break; } + for (size_t i = 0; i < scr_dyn_obj_count(a); i++) { + ScrDynProp x = scr_dyn_obj_prop_at(a, i); + if (!x.enumerable) continue; + ScrDynProp yp; + const ScrDynProp *y = NULL; + if (b) for (size_t j = 0; j < scr_dyn_obj_count(b); j++) { + ScrDynProp candidate = scr_dyn_obj_prop_at(b, j); + if (candidate.enumerable && candidate.key_len == x.key_len && + !memcmp(candidate.key, x.key, x.key_len)) { yp = candidate; y = &yp; break; } } if (!y) return false; - if (x->accessor || y->accessor) return util_deep_refused("native accessor properties"); + if (x.accessor || y->accessor) return util_deep_refused("native accessor properties"); // Materializing a cyclic typed child can refresh its ancestor table. // Retain the values before that refresh replaces descriptor storage. - ScrDyn *xv = scr_dyn_retain(x->value), *yv = scr_dyn_retain(y->value); + ScrDyn *xv = scr_dyn_retain(x.value), *yv = scr_dyn_retain(y->value); bool same = util_deep_equal(xv, yv, skip, frame); scr_dyn_release(xv); scr_dyn_release(yv); if (!same) return false; diff --git a/packages/runtime/src/scr_worker_events.c b/packages/runtime/src/scr_worker_events.c index a23395c91..7669c4fb9 100644 --- a/packages/runtime/src/scr_worker_events.c +++ b/packages/runtime/src/scr_worker_events.c @@ -1385,17 +1385,17 @@ static ScrDyn *scr_worker_new_options(double root, ScrStr *filename, ScrDyn *opt scr_dyn_arg_type_fail("options", "object", options); return NULL; } - for (size_t i = 0; i < options->v.obj.len; i++) { - const ScrDynEntry *entry = &options->v.obj.entries[i]; - if (entry->accessor) return scr_worker_refusal("Worker option accessors are not supported yet"); - if (entry->key_len == 10 && !memcmp(entry->key, "workerData", 10)) data = entry->value; - else if (entry->key_len == 4 && !memcmp(entry->key, "argv", 4)) argv = entry->value; - else if (entry->key_len == 12 && !memcmp(entry->key, "transferList", 12)) transfer_list = entry->value; - else if (entry->key_len == 14 && !memcmp(entry->key, "resourceLimits", 14)) limits = entry->value; - else if (entry->key_len == 4 && !memcmp(entry->key, "name", 4)) name = entry->value; - else if (entry->key_len == 6 && !memcmp(entry->key, "stdout", 6)) capture |= scr_dyn_truthy(entry->value) ? 1 : 0; - else if (entry->key_len == 6 && !memcmp(entry->key, "stderr", 6)) capture |= scr_dyn_truthy(entry->value) ? 2 : 0; - else if (entry->key_len == 5 && !memcmp(entry->key, "stdin", 5)) capture |= scr_dyn_truthy(entry->value) ? 4 : 0; + for (size_t i = 0; i < scr_dyn_obj_count(options); i++) { + ScrDynProp entry = scr_dyn_obj_prop_at(options, i); + if (entry.accessor) return scr_worker_refusal("Worker option accessors are not supported yet"); + if (entry.key_len == 10 && !memcmp(entry.key, "workerData", 10)) data = entry.value; + else if (entry.key_len == 4 && !memcmp(entry.key, "argv", 4)) argv = entry.value; + else if (entry.key_len == 12 && !memcmp(entry.key, "transferList", 12)) transfer_list = entry.value; + else if (entry.key_len == 14 && !memcmp(entry.key, "resourceLimits", 14)) limits = entry.value; + else if (entry.key_len == 4 && !memcmp(entry.key, "name", 4)) name = entry.value; + else if (entry.key_len == 6 && !memcmp(entry.key, "stdout", 6)) capture |= scr_dyn_truthy(entry.value) ? 1 : 0; + else if (entry.key_len == 6 && !memcmp(entry.key, "stderr", 6)) capture |= scr_dyn_truthy(entry.value) ? 2 : 0; + else if (entry.key_len == 5 && !memcmp(entry.key, "stdin", 5)) capture |= scr_dyn_truthy(entry.value) ? 4 : 0; else return scr_worker_refusal("This Worker constructor option is not supported yet"); } } diff --git a/packages/runtime/src/scr_zlib.c b/packages/runtime/src/scr_zlib.c index 8756c3d17..bc1254f29 100644 --- a/packages/runtime/src/scr_zlib.c +++ b/packages/runtime/src/scr_zlib.c @@ -1,13 +1,10 @@ /* node:zlib, the lowered slice: the one-shot zlib/raw-DEFLATE/gzip codecs * over u8 bytes with Node's DEFAULT options. Compiled ONLY when the program * uses zlib (native-toolchain.ts gates it exactly like scr_regex.c/libregexp), - * so zlib-free binaries keep their historical link line. Host builds link - * the system -lz; cross targets link the vendored zlib built per target - * (ensureZlibObjects in native-toolchain.ts). - * - * Compressed OUTPUT bytes are zlib-version-dependent — the differential - * corpus tests round-trips and fixed-blob inflation, never raw deflate - * output. deflate of valid input cannot fail (OOM aborts); inflate of + * so zlib-free binaries keep their historical link line. Every build links + * the vendored zlib (packages/runtime/vendor/zlib), the copy Node bundles, + * so compressed bytes match Node's for the same parameters. deflate of valid + * input cannot fail (OOM aborts); inflate of * corrupt input THROWS Node's catchable Error (zlib's own msg text: * "incorrect header check", ...). */ #include "scr_runtime.h" diff --git a/packages/runtime/test/array.test.ts b/packages/runtime/test/array.test.ts index 4b42b2beb..ffdafb5e0 100644 --- a/packages/runtime/test/array.test.ts +++ b/packages/runtime/test/array.test.ts @@ -24,6 +24,7 @@ beforeAll(async () => { bin, join(testDir, "test_array.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), // Checked-value collection equality shares the native JSON value runtime. join(testDir, "../src/scr_json.c"), join(testDir, "../src/scr_map.c"), diff --git a/packages/runtime/test/bigint.test.ts b/packages/runtime/test/bigint.test.ts index ea0a8b79c..7b6a5e51a 100644 --- a/packages/runtime/test/bigint.test.ts +++ b/packages/runtime/test/bigint.test.ts @@ -32,6 +32,7 @@ beforeAll(async () => { // which pull in the closure/array/box machinery join(testDir, "../src/scr_union.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), // Checked-value collection equality shares the native JSON value runtime. join(testDir, "../src/scr_json.c"), join(testDir, "../src/scr_bigint.c"), diff --git a/packages/runtime/test/bytes.test.ts b/packages/runtime/test/bytes.test.ts index 7ca0606ec..05123bf5f 100644 --- a/packages/runtime/test/bytes.test.ts +++ b/packages/runtime/test/bytes.test.ts @@ -40,6 +40,7 @@ beforeAll(async () => { join(testDir, "../src/scr_hash.c"), join(testDir, "../src/scr_string.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), join(testDir, "../src/scr_map.c"), join(testDir, "../src/scr_exception.c"), join(testDir, "../src/scr_error.c"), diff --git a/packages/runtime/test/cipher.test.ts b/packages/runtime/test/cipher.test.ts index 6b44a4b4b..44bbc250c 100644 --- a/packages/runtime/test/cipher.test.ts +++ b/packages/runtime/test/cipher.test.ts @@ -20,6 +20,7 @@ const runtimeSources = [ "scr_hash.c", "scr_string.c", "scr_array.c", + "scr_arena.c", "scr_map.c", "scr_exception.c", "scr_error.c", diff --git a/packages/runtime/test/closure.test.ts b/packages/runtime/test/closure.test.ts index 96fad6c10..e4c6200dc 100644 --- a/packages/runtime/test/closure.test.ts +++ b/packages/runtime/test/closure.test.ts @@ -25,6 +25,7 @@ test("closure/box runtime: RC cascades clean under ASan + audit", async () => { join(srcDir, "scr_closure.c"), join(srcDir, "scr_string.c"), join(srcDir, "scr_array.c"), + join(srcDir, "scr_arena.c"), join(srcDir, "scr_json.c"), join(srcDir, "scr_bigint.c"), join(srcDir, "scr_map.c"), diff --git a/packages/runtime/test/dh.test.ts b/packages/runtime/test/dh.test.ts index d402d0465..1f324ffb0 100644 --- a/packages/runtime/test/dh.test.ts +++ b/packages/runtime/test/dh.test.ts @@ -20,6 +20,7 @@ const runtimeSources = [ "scr_hash.c", "scr_string.c", "scr_array.c", + "scr_arena.c", "scr_map.c", "scr_exception.c", "scr_error.c", diff --git a/packages/runtime/test/glob.test.ts b/packages/runtime/test/glob.test.ts index 92292787a..9900382ea 100644 --- a/packages/runtime/test/glob.test.ts +++ b/packages/runtime/test/glob.test.ts @@ -35,6 +35,7 @@ beforeAll(async () => { join(testDir, "../src/scr_string.c"), join(testDir, "../src/scr_map.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), join(testDir, "../src/scr_json.c"), join(testDir, "../src/scr_bigint.c"), join(testDir, "../src/scr_exception.c"), diff --git a/packages/runtime/test/inspect.test.ts b/packages/runtime/test/inspect.test.ts index 772afe4ef..40d812e53 100644 --- a/packages/runtime/test/inspect.test.ts +++ b/packages/runtime/test/inspect.test.ts @@ -35,6 +35,7 @@ beforeAll(async () => { join(testDir, "../src/scr_string.c"), join(testDir, "../src/scr_number.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), join(testDir, "../src/scr_bytes.c"), join(testDir, "../src/scr_shared.c"), join(testDir, "../src/scr_lib.c"), diff --git a/packages/runtime/test/json.test.ts b/packages/runtime/test/json.test.ts index b08f2fdd3..b049e6922 100644 --- a/packages/runtime/test/json.test.ts +++ b/packages/runtime/test/json.test.ts @@ -29,6 +29,7 @@ beforeAll(async () => { join(testDir, "../src/scr_bigint.c"), join(testDir, "../src/scr_string.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), join(testDir, "../src/scr_map.c"), join(testDir, "../src/scr_exception.c"), join(testDir, "../src/scr_error.c"), diff --git a/packages/runtime/test/lib.test.ts b/packages/runtime/test/lib.test.ts index 2a32202b7..d1e9e8c90 100644 --- a/packages/runtime/test/lib.test.ts +++ b/packages/runtime/test/lib.test.ts @@ -35,6 +35,7 @@ beforeAll(async () => { join(testDir, "../src/scr_path.c"), join(testDir, "../src/scr_string.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), join(testDir, "../src/scr_map.c"), join(testDir, "../src/scr_exception.c"), join(testDir, "../src/scr_error.c"), diff --git a/packages/runtime/test/map.test.ts b/packages/runtime/test/map.test.ts index 4db1889ac..4a56c3055 100644 --- a/packages/runtime/test/map.test.ts +++ b/packages/runtime/test/map.test.ts @@ -33,6 +33,7 @@ beforeAll(async () => { // which pull in the closure/array/box machinery join(testDir, "../src/scr_union.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), // Checked-value collection equality shares the native JSON value runtime. join(testDir, "../src/scr_json.c"), join(testDir, "../src/scr_bigint.c"), diff --git a/packages/runtime/test/object-core.test.ts b/packages/runtime/test/object-core.test.ts new file mode 100644 index 000000000..ce9e1e3e0 --- /dev/null +++ b/packages/runtime/test/object-core.test.ts @@ -0,0 +1,24 @@ +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { expect, test } from "vitest"; + +const src = join(import.meta.dirname, "..", "src"); + +/* Ordinary objects keep their properties behind the object core: units + * outside it read and adjust them through the scr_dyn.h own-property + * accessors (scr_dyn_obj_count, scr_dyn_obj_prop_at, ...), so the core can + * change how it stores them. */ +const CORE = new Set(["scr_json.c", "scr_js_object.c", "scr_dyn.h"]); + +test("only the object core reads ordinary objects' storage", () => { + const offenders: string[] = []; + for (const name of readdirSync(src)) { + if (!/\.(c|h)$/.test(name) || name.endsWith(".test.c") || CORE.has(name)) continue; + const lines = readFileSync(join(src, name), "utf8").split("\n"); + lines.forEach((line, i) => { + if (/\bv\.obj\.(entries|slots|layout|len|cap|index)\b/.test(line)) + offenders.push(`${name}:${i + 1}: ${line.trim()}`); + }); + } + expect(offenders).toEqual([]); +}); diff --git a/packages/runtime/test/path.test.ts b/packages/runtime/test/path.test.ts index 7f0da7841..79d34c1c3 100644 --- a/packages/runtime/test/path.test.ts +++ b/packages/runtime/test/path.test.ts @@ -29,6 +29,7 @@ beforeAll(async () => { join(testDir, "../src/scr_string.c"), join(testDir, "../src/scr_number.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), // Checked-value collection equality shares the native JSON value runtime. join(testDir, "../src/scr_json.c"), join(testDir, "../src/scr_map.c"), diff --git a/packages/runtime/test/regex.test.ts b/packages/runtime/test/regex.test.ts index 47853d960..cbcbc868b 100644 --- a/packages/runtime/test/regex.test.ts +++ b/packages/runtime/test/regex.test.ts @@ -36,6 +36,7 @@ beforeAll(async () => { join(testDir, "../src/scr_string.c"), join(testDir, "../src/scr_map.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), join(testDir, "../src/scr_json.c"), join(testDir, "../src/scr_bigint.c"), join(testDir, "../src/scr_exception.c"), diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts index 11bdf75b4..5b115f868 100644 --- a/packages/runtime/test/runtime.test.ts +++ b/packages/runtime/test/runtime.test.ts @@ -13,6 +13,7 @@ const RUNTIME_SOURCES = [ "scr_bigint.c", "scr_string.c", "scr_array.c", + "scr_arena.c", "scr_bytes.c", "scr_shared.c", "scr_bytes_io.c", diff --git a/packages/runtime/test/sha-kernels.test.ts b/packages/runtime/test/sha-kernels.test.ts index 6f7759348..4cb12a638 100644 --- a/packages/runtime/test/sha-kernels.test.ts +++ b/packages/runtime/test/sha-kernels.test.ts @@ -19,6 +19,7 @@ const runtimeSources = [ "scr_hash.c", "scr_string.c", "scr_array.c", + "scr_arena.c", "scr_map.c", "scr_exception.c", "scr_error.c", diff --git a/packages/runtime/test/sort.test.ts b/packages/runtime/test/sort.test.ts index 711a41f03..fed629b73 100644 --- a/packages/runtime/test/sort.test.ts +++ b/packages/runtime/test/sort.test.ts @@ -24,6 +24,7 @@ beforeAll(async () => { bin, join(testDir, "test_sort.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), // Checked-value collection equality shares the native JSON value runtime. join(testDir, "../src/scr_json.c"), join(testDir, "../src/scr_map.c"), diff --git a/packages/runtime/test/string.test.ts b/packages/runtime/test/string.test.ts index 72a479a10..cc6d4fbb6 100644 --- a/packages/runtime/test/string.test.ts +++ b/packages/runtime/test/string.test.ts @@ -30,6 +30,7 @@ beforeAll(async () => { // scr_str_split returns a real string[]: the array module and its // own dependencies join the link (the array.test.ts set). join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), // Checked-value collection equality shares the native JSON value runtime. join(testDir, "../src/scr_json.c"), join(testDir, "../src/scr_map.c"), @@ -87,6 +88,13 @@ test("string methods match Node on committed oracle cases", async () => { expect(r.stderr.trim()).toMatch(/^(\d+)\/\1 cases passed$/); }); +// A template literal of strings and numbers allocates exactly its length: +// numbers format before the result is sized, not into a reserved maximum. +test("template-literal concatenation allocates its exact length", async () => { + const { stderr } = await execFileAsync(bin, ["--concat-capacity"]); + expect(stderr.trim()).toBe("0 concat capacity failures"); +}); + // repeat(count) with count < 0 or Infinity is Node's catchable RangeError // ("Invalid count value: -1"); the runtime leaves it pending. test.each([ diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index 08eaecad4..961d43586 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -833,6 +833,96 @@ static void check_deferred_white_restoration(void) { * scheduling decision or pass would do. */ size_t scr_cyc_pass_at(unsigned gen); void scr_cyc_route_bypassed_at(void); + +/* A nursery pass over a dead ring, longer than the walk recurses, whose + * nodes hold edges into an older ring, and two live nodes the ring reaches + * that hold edges into another: only the dead ring's edges are paid off. */ +static void check_cross_generation_holders(void) { + enum { RING = 200 }; + /* The ring's candidate must wait for the pass below. */ + if (configured_nursery_threshold() < 8) return; + static ScrCycType cyc_type = { &node_trace, &node_free, 0 }; + scr_collect_cycles(); + size_t before = freed; + Node *targets[2]; + for (int r = 0; r < 2; r++) { + targets[r] = make_ring(2); + targets[r]->rc++; + release_live(targets[r]); + } + scr_collect_cycles(); /* both survive, out of the nursery */ + Node *live[2]; + for (int i = 0; i < 2; i++) { + live[i] = scr_cyc_alloc(sizeof(*live[i]), &cyc_type); + live[i]->rc = 3; /* itself, the ring's edge and an outside owner */ + live[i]->next = live[i]; + live[i]->other = targets[1]; + targets[1]->rc++; + } + Node *ring = make_ring(RING); + Node *node = ring; + for (size_t i = 0; i < RING; i++, node = node->next) { + node->other = i == 10 ? live[0] : i == 150 ? live[1] : targets[0]; + if (node->other == targets[0]) targets[0]->rc++; + } + release_live(ring); + scr_cyc_pass_at(SCR_CYC_NURSERY); + check(freed == before + RING, "nursery pass did not free only the dead ring"); + check(targets[0]->rc == 2, "the dead ring's edges into an older ring were not paid off"); + check(targets[1]->rc == 4, "a live node's edge into an older ring was paid off"); + release_live(live[0]); + release_live(live[1]); + scr_collect_cycles(); + check(freed == before + RING + 2, "the live nodes were not collected once dead"); + release_live(targets[0]); + release_live(targets[1]); + scr_collect_cycles(); + check(freed == before + RING + 6, "the older rings were not collected once dead"); +} + +/* A dispose observer (scr_cyc_free notifies it) reads the color of the + * objects a pass tears down. */ +static size_t observed_doomed, observed_other; +static void observe_disposal(void *obj) { + if (scr_cyc_color(scr_cyc_hdr(obj)) == SCR_CYC_DOOMED) observed_doomed++; + else observed_other++; +} + +static void check_dispose_observer_sees_doomed(void) { + enum { RING = 8 }; + size_t before = freed; + void (*hook)(void *) = scr_weak_dispose_hook; + scr_weak_dispose_hook = observe_disposal; + observed_doomed = observed_other = 0; + Node *ring = make_ring(RING); + release_live(ring); + scr_cyc_pass_at(SCR_CYC_NURSERY); + scr_weak_dispose_hook = hook; + check(freed == before + RING, "the dead ring was not freed"); + check(observed_doomed == RING && observed_other == 0, + "a dispose observer saw a collected object that was not DOOMED"); +} + +/* A full pass whose gray list fills: the dead rings walked before it filled + * stay in the list, and the scan from the candidates passes over them. */ +static void check_doomed_rings_before_full_gray_list(void) { + enum { RINGS = 4, RING = 8, LIVE = 70000 }; + if (configured_nursery_threshold() < 8) return; + size_t before = freed; + Node *dead[RINGS]; + for (int i = 0; i < RINGS; i++) dead[i] = make_ring(RING); + Node *live = make_ring(LIVE); + scr_collect_cycles(); /* re-arms the allocation trigger */ + for (int i = 0; i < RINGS; i++) release_live(dead[i]); + live->rc++; + release_live(live); + scr_collect_cycles(); + check(freed == before + RINGS * RING, "a pass with a full gray list did not free only the dead rings"); + release_live(live); + scr_collect_cycles(); + check(freed == before + RINGS * RING + LIVE, "the large ring was not collected once dead"); +} + static size_t bypass_traced; /* trace calls on bypass-kind nodes */ static size_t bypass_freed; /* bypass-kind nodes freed */ static void bypass_trace(void *obj, ScrTraceVisit visit, void *ctx) { @@ -1617,6 +1707,9 @@ int main(void) { check_deferred_white_restoration(); check_cross_generation_edges(); check_deep_cross_generation_chain(); + check_cross_generation_holders(); + check_dispose_observer_sees_doomed(); + check_doomed_rings_before_full_gray_list(); check_teardown_rebuffers_and_immortals(); check_growth_schedule_follows_yield(); check_growth_cap_bounds_floating_garbage(); diff --git a/packages/runtime/test/test_json.c b/packages/runtime/test/test_json.c index 865f1c757..6d492065d 100644 --- a/packages/runtime/test/test_json.c +++ b/packages/runtime/test/test_json.c @@ -571,10 +571,10 @@ static ScrDyn *define_from(ScrDyn *target, const char *key, ScrDyn *value, bool } static const ScrDynEntry *entry_of(const ScrDyn *obj, const char *key) { + const ScrDynEntry *entries = scr_dyn_obj_dict_entries(obj); for (size_t i = 0; i < obj->v.obj.len; i++) - if (obj->v.obj.entries[i].key_len == strlen(key) && - memcmp(obj->v.obj.entries[i].key, key, strlen(key)) == 0) - return &obj->v.obj.entries[i]; + if (entries[i].key_len == strlen(key) && memcmp(entries[i].key, key, strlen(key)) == 0) + return &entries[i]; return NULL; } @@ -618,7 +618,7 @@ static void test_copy_property_descriptors(void) { scr_dyn_obj_set(existing, "label", 5, scr_dyn_new_num(9)); scr_dyn_release(scr_dyn_copy_property_descriptors(existing, bag)); const ScrDynEntry *label = entry_of(existing, "label"); - check(!scr_exc_pending() && existing->v.obj.len == 5 && existing->v.obj.entries[0].key_len == 5 && + check(!scr_exc_pending() && existing->v.obj.len == 5 && scr_dyn_obj_dict_entries(existing)[0].key_len == 5 && label->value->v.num == 1, "copy redefines an existing property in place"); diff --git a/packages/runtime/test/test_string.c b/packages/runtime/test/test_string.c index f41c2de50..d0dbebf0d 100644 --- a/packages/runtime/test/test_string.c +++ b/packages/runtime/test/test_string.c @@ -982,7 +982,42 @@ static void ordering_asserts(void) { } } +/* Template-literal concatenations (no head) of strings and numbers: the + * result holds exactly its bytes, whatever the numbers format to. */ +static int check_concat_capacity(void) { + ScrStr *a = scr_str_new("f", 1), *b = scr_str_new("_d", 2), *c = scr_str_new("", 0); + struct { + ScrStr *parts[4]; + double nums[4]; + size_t count; + const char *expected; + } cases[] = { + {{a, NULL, b, NULL}, {0, 7, 0, 2230}, 4, "f7_d2230"}, + {{NULL, a}, {-1.5e-7, 0}, 2, "-1.5e-7f"}, + {{NULL, NULL, NULL}, {1.7976931348623157e308, 0.1, -0.0}, 3, "1.7976931348623157e+3080.10"}, + {{c, NULL, c}, {0, 42, 0}, 3, "42"}, + }; + int failed = 0; + for (size_t i = 0; i < sizeof cases / sizeof cases[0]; i++) { + ScrStr *r = scr_str_concat_mixed(NULL, cases[i].parts, cases[i].nums, cases[i].count); + size_t want = strlen(cases[i].expected); + if (r->len != want || memcmp(r->data, cases[i].expected, want) != 0 || r->data[want] != '\0' || + r->cap != r->len) { + fprintf(stderr, "concat case %zu: got \"%.*s\" len %zu cap %zu, want \"%s\" len %zu\n", i, + (int)r->len, r->data, r->len, (size_t)r->cap, cases[i].expected, want); + failed++; + } + scr_str_release(r); + } + scr_str_release(a); + scr_str_release(b); + scr_str_release(c); + fprintf(stderr, "%d concat capacity failures\n", failed); + return failed != 0; +} + int main(int argc, char **argv) { + if (argc > 1 && strcmp(argv[1], "--concat-capacity") == 0) return check_concat_capacity(); if (argc > 1 && strncmp(argv[1], "--crash-repeat", 14) == 0) { ScrStr *s = scr_str_new("ab", 2); double count = strcmp(argv[1], "--crash-repeat-inf") == 0 diff --git a/packages/runtime/test/tonumber.test.ts b/packages/runtime/test/tonumber.test.ts index 2a3c24575..0a95f2bb6 100644 --- a/packages/runtime/test/tonumber.test.ts +++ b/packages/runtime/test/tonumber.test.ts @@ -37,6 +37,7 @@ test("scr_string_to_number and scr_parse_float match Node on committed oracle ca // scr_string.c's own link closure (the string.test.ts set): // scr_str_split pulls the array module and its dependencies. join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), // Checked-value collection equality shares the native JSON value runtime. join(testDir, "../src/scr_json.c"), join(testDir, "../src/scr_map.c"), diff --git a/packages/runtime/test/url-legacy.test.ts b/packages/runtime/test/url-legacy.test.ts index a237382ae..0af2f7768 100644 --- a/packages/runtime/test/url-legacy.test.ts +++ b/packages/runtime/test/url-legacy.test.ts @@ -38,6 +38,7 @@ beforeAll(async () => { join(testDir, "../src/scr_string.c"), join(testDir, "../src/scr_map.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), join(testDir, "../src/scr_json.c"), join(testDir, "../src/scr_bigint.c"), join(testDir, "../src/scr_exception.c"), diff --git a/packages/runtime/test/url.test.ts b/packages/runtime/test/url.test.ts index 9876ec4e5..ea6948487 100644 --- a/packages/runtime/test/url.test.ts +++ b/packages/runtime/test/url.test.ts @@ -29,6 +29,7 @@ beforeAll(async () => { join(testDir, "../src/scr_string.c"), join(testDir, "../src/scr_number.c"), join(testDir, "../src/scr_array.c"), + join(testDir, "../src/scr_arena.c"), // Checked-value collection equality shares the native JSON value runtime. join(testDir, "../src/scr_json.c"), join(testDir, "../src/scr_map.c"), diff --git a/packages/runtime/vendor/README.md b/packages/runtime/vendor/README.md index 53238410e..cfb0d196f 100644 --- a/packages/runtime/vendor/README.md +++ b/packages/runtime/vendor/README.md @@ -38,13 +38,15 @@ The native runtime pack build produces `libscriptc-quickjs.a` for the compiler b ## zlib/ -- Project: https://github.com/madler/zlib -- Version: 1.3.1 (release tarball zlib-1.3.1.tar.gz, sha256 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23) +- Project: https://chromium.googlesource.com/chromium/src/third_party/zlib (Chromium's zlib, a fork of https://github.com/madler/zlib) +- Version: 1.3.2.1-motley-285e94b, the copy Node v24.22.0 bundles in `deps/zlib` (`process.versions.zlib`) - License: zlib (see zlib/LICENSE) -The CROSS-target arm of node:zlib (see packages/runtime/src/scr_zlib.c): host builds keep the historical system `-lz` link (macOS ships libz), but zig's cross sysroots have no libz, so SCRIPTC_TARGET builds compile this vendored copy per target instead. Only the flat `*.c`/`*.h` at the distribution root are vendored (LICENSE beside them) — contrib, tests, build machinery, and docs are not. No vendored file is modified; the gz* file-I/O TUs are vendored for faithfulness but never compiled (nothing references the gzFile API — see ZLIB_SOURCES in packages/compiler/src/backend/vendor-archives.ts). To update, re-fetch the release tarball, copy the same flat set, and bump `ZLIB_VERSION` in vendor-archives.ts. +The codec behind node:zlib's gzip, deflate and inflate functions and streams (see packages/runtime/src/scr_zlib.c) and the native fetch's gzip decoder. The flat `*.c`/`*.h` files at the root of Node's `deps/zlib`, `contrib/optimizations/`, `LICENSE` and `README.chromium` are vendored; `contrib/minizip`, the other contrib directories, tests, patches, Chromium's build files and docs are not. No vendored file is modified, and the files match Node's copy. The gz* file-I/O units are vendored but never compiled (nothing references the gzFile API). -The objects build lazily on the first zlib-using cross compile into the per-user build cache's `vendor/zlib----/` directory — one flavor per driver/target and compiler environment/identity (the lre-objects pattern). Zlib-free binaries never compile any of this; compressed OUTPUT bytes are zlib-version-dependent, which is why the corpus only compares round-trips and fixed-blob inflation. +Each target compiles the units and defines Node's `deps/zlib/zlib.gyp` selects for it: the SSSE3 Adler-32 and SSE2 inflate and slide-hash code on x86-64, the NEON code and the ARMv8 CRC-32 code on arm64 Linux, Android and macOS, the NEON inflate and slide-hash code on iOS and Windows arm64, and the portable code (`CPU_NO_SIMD`) elsewhere. The table is `zlibInputs` in packages/runtime-pack-common/scripts/zlib-inputs.mjs (runtime packs) and in packages/compiler/src/backend/vendor-inputs.ts (Zig cross builds and the sanitized runtime), and a unit test keeps the two identical. Android reads CPU features through `getauxval` as Linux does, where Node's Android build uses the NDK's cpu-features library. The SIMD units check the CPU at run time and give the same output as the portable code, so compressed bytes match Node's for the same parameters on every target. To update, copy the same files from the `deps/zlib` of the Node version scriptc follows and bump `ZLIB_VERSION` in vendor-inputs.ts. + +Runtime packs archive zlib at `-O3`, as Node builds it. Zig cross builds compile the objects lazily on the first zlib-using cross compile into the per-user build cache's `vendor/zlib----/` directory, one flavor per driver/target and compiler environment/identity (the lre-objects pattern). Zlib-free binaries compile and link none of it. ## brotli/ diff --git a/packages/runtime/vendor/zlib/LICENSE b/packages/runtime/vendor/zlib/LICENSE index ab8ee6f71..b7a69d058 100644 --- a/packages/runtime/vendor/zlib/LICENSE +++ b/packages/runtime/vendor/zlib/LICENSE @@ -1,6 +1,6 @@ Copyright notice: - (C) 1995-2022 Jean-loup Gailly and Mark Adler + (C) 1995-2026 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages diff --git a/packages/runtime/vendor/zlib/LICENSE.chromium b/packages/runtime/vendor/zlib/LICENSE.chromium new file mode 100644 index 000000000..2249a2865 --- /dev/null +++ b/packages/runtime/vendor/zlib/LICENSE.chromium @@ -0,0 +1,27 @@ +// Copyright 2015 The Chromium Authors +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/packages/runtime/vendor/zlib/README.chromium b/packages/runtime/vendor/zlib/README.chromium new file mode 100644 index 000000000..232ef66ae --- /dev/null +++ b/packages/runtime/vendor/zlib/README.chromium @@ -0,0 +1,38 @@ +Name: zlib +Short Name: zlib +URL: https://github.com/madler/zlib +Version: 1.3.2 +Revision: 09a1572aa624e5ddb6c075dc013880de70b1b9b9 +Update Mechanism: Manual (https://crbug.com/422348588) +CPEPrefix: cpe:/a:zlib:zlib:1.3.2 +Security Critical: yes +Shipped: yes +License: Zlib +License File: LICENSE +License Android Compatible: yes + +Description: +"A massively spiffy yet delicately unobtrusive compression library." + +zlib is a free, general-purpose, legally unencumbered lossless data-compression +library. zlib implements the "deflate" compression algorithm described by RFC +1951, which combines the LZ77 (Lempel-Ziv) algorithm with Huffman coding. zlib +also implements the zlib (RFC 1950) and gzip (RFC 1952) wrapper formats. + +Local Modifications: + - Only source code from the http://zlib.net distribution used to build the zlib + and minizip libraries are present. Many other files have been omitted. Only *.c + and *.h files from the upstream root directory, contrib/minizip and + examples/zpipe.c were imported. + - The files named '*simd*' are original x86/Arm/RISC-V specific optimizations. + - The contents of the google directory are original Chromium-specific + additions. + - The contents of the 'contrib' of directory are either Chromium-specific + additions or heavily patched zlib files (e.g. inffast_chunk*). + - Added chromeconf.h + - Plus the changes in 'patches' folder. + - Code in contrib/ other than contrib/minizip was added to match zlib's + contributor layout. + - In sync with 1.3.2 official release + - ZIP reader modified to allow for progress callbacks during extraction. + - ZIP reader modified to add detection of AES encrypted content. diff --git a/packages/runtime/vendor/zlib/adler32.c b/packages/runtime/vendor/zlib/adler32.c index 04b81d29b..de78b4e56 100644 --- a/packages/runtime/vendor/zlib/adler32.c +++ b/packages/runtime/vendor/zlib/adler32.c @@ -57,10 +57,27 @@ # define MOD63(a) a %= BASE #endif +#include "cpu_features.h" +#if defined(ADLER32_SIMD_SSSE3) || defined(ADLER32_SIMD_NEON) || defined(ADLER32_SIMD_RVV) +#include "adler32_simd.h" +#endif + /* ========================================================================= */ uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf, z_size_t len) { unsigned long sum2; unsigned n; + /* TODO(cavalcantii): verify if this lengths are optimal for current CPUs. */ +#if defined(ADLER32_SIMD_SSSE3) || defined(ADLER32_SIMD_NEON) \ + || defined(ADLER32_SIMD_RVV) +#if defined(ADLER32_SIMD_SSSE3) + if (buf != Z_NULL && len >= 64 && x86_cpu_enable_ssse3) +#elif defined(ADLER32_SIMD_NEON) + if (buf != Z_NULL && len >= 64) +#elif defined(ADLER32_SIMD_RVV) + if (buf != Z_NULL && len >= 32 && riscv_cpu_enable_rvv) +#endif + return adler32_simd_(adler, buf, len); +#endif /* split Adler-32 into component sums */ sum2 = (adler >> 16) & 0xffff; @@ -77,9 +94,25 @@ uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf, z_size_t len) { return adler | (sum2 << 16); } +#if defined(ADLER32_SIMD_SSSE3) || defined(ADLER32_SIMD_NEON) \ + || defined(RISCV_RVV) + /* + * Use SIMD to compute the adler32. Since this function can be + * freely used, check CPU features here. zlib convention is to + * call adler32(0, NULL, 0), before making calls to adler32(). + * So this is a good early (and infrequent) place to cache CPU + * features for those later, more interesting adler32() calls. + */ + if (buf == Z_NULL) { + if (!len) /* Assume user is calling adler32(0, NULL, 0); */ + cpu_check_features(); + return 1L; + } +#else /* initial Adler-32 value (deferred check for len == 1 speed) */ if (buf == Z_NULL) return 1L; +#endif /* in case short lengths are provided, keep it somewhat fast */ if (len < 16) { diff --git a/packages/runtime/vendor/zlib/adler32_simd.c b/packages/runtime/vendor/zlib/adler32_simd.c new file mode 100644 index 000000000..ce4ce7c57 --- /dev/null +++ b/packages/runtime/vendor/zlib/adler32_simd.c @@ -0,0 +1,459 @@ +/* adler32_simd.c + * + * Copyright 2017 The Chromium Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + * + * Per http://en.wikipedia.org/wiki/Adler-32 the adler32 A value (aka s1) is + * the sum of N input data bytes D1 ... DN, + * + * A = A0 + D1 + D2 + ... + DN + * + * where A0 is the initial value. + * + * SSE2 _mm_sad_epu8() can be used for byte sums (see http://bit.ly/2wpUOeD, + * for example) and accumulating the byte sums can use SSE shuffle-adds (see + * the "Integer" section of http://bit.ly/2erPT8t for details). Arm NEON has + * similar instructions. + * + * The adler32 B value (aka s2) sums the A values from each step: + * + * B0 + (A0 + D1) + (A0 + D1 + D2) + ... + (A0 + D1 + D2 + ... + DN) or + * + * B0 + N.A0 + N.D1 + (N-1).D2 + (N-2).D3 + ... + (N-(N-1)).DN + * + * B0 being the initial value. For 32 bytes (ideal for garden-variety SIMD): + * + * B = B0 + 32.A0 + [D1 D2 D3 ... D32] x [32 31 30 ... 1]. + * + * Adjacent blocks of 32 input bytes can be iterated with the expressions to + * compute the adler32 s1 s2 of M >> 32 input bytes [1]. + * + * As M grows, the s1 s2 sums grow. If left unchecked, they would eventually + * overflow the precision of their integer representation (bad). However, s1 + * and s2 also need to be computed modulo the adler BASE value (reduced). If + * at most NMAX bytes are processed before a reduce, s1 s2 _cannot_ overflow + * a uint32_t type (the NMAX constraint) [2]. + * + * [1] the iterative equations for s2 contain constant factors; these can be + * hoisted from the n-blocks do loop of the SIMD code. + * + * [2] zlib adler32_z() uses this fact to implement NMAX-block-based updates + * of the adler s1 s2 of uint32_t type (see adler32.c). + */ + +#include "adler32_simd.h" + +/* Definitions from adler32.c: largest prime smaller than 65536 */ +#define BASE 65521U +/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ +#define NMAX 5552 + +#if defined(ADLER32_SIMD_SSSE3) + +#include + +#if defined(__GNUC__) || defined(__clang__) +__attribute__((__target__("ssse3"))) +#endif +uint32_t ZLIB_INTERNAL adler32_simd_( /* SSSE3 */ + uint32_t adler, + const unsigned char *buf, + z_size_t len) +{ + /* + * Split Adler-32 into component sums. + */ + uint32_t s1 = adler & 0xffff; + uint32_t s2 = adler >> 16; + + /* + * Process the data in blocks. + */ + const unsigned BLOCK_SIZE = 1 << 5; + + z_size_t blocks = len / BLOCK_SIZE; + len -= blocks * BLOCK_SIZE; + + while (blocks) + { + unsigned n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */ + if (n > blocks) + n = (unsigned) blocks; + blocks -= n; + + const __m128i tap1 = + _mm_setr_epi8(32,31,30,29,28,27,26,25,24,23,22,21,20,19,18,17); + const __m128i tap2 = + _mm_setr_epi8(16,15,14,13,12,11,10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + const __m128i zero = + _mm_setr_epi8( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + const __m128i ones = + _mm_set_epi16( 1, 1, 1, 1, 1, 1, 1, 1); + + /* + * Process n blocks of data. At most NMAX data bytes can be + * processed before s2 must be reduced modulo BASE. + */ + __m128i v_ps = _mm_set_epi32(0, 0, 0, s1 * n); + __m128i v_s2 = _mm_set_epi32(0, 0, 0, s2); + __m128i v_s1 = _mm_set_epi32(0, 0, 0, 0); + + do { + /* + * Load 32 input bytes. + */ + const __m128i bytes1 = _mm_loadu_si128((__m128i*)(buf)); + const __m128i bytes2 = _mm_loadu_si128((__m128i*)(buf + 16)); + + /* + * Add previous block byte sum to v_ps. + */ + v_ps = _mm_add_epi32(v_ps, v_s1); + + /* + * Horizontally add the bytes for s1, multiply-adds the + * bytes by [ 32, 31, 30, ... ] for s2. + */ + v_s1 = _mm_add_epi32(v_s1, _mm_sad_epu8(bytes1, zero)); + const __m128i mad1 = _mm_maddubs_epi16(bytes1, tap1); + v_s2 = _mm_add_epi32(v_s2, _mm_madd_epi16(mad1, ones)); + + v_s1 = _mm_add_epi32(v_s1, _mm_sad_epu8(bytes2, zero)); + const __m128i mad2 = _mm_maddubs_epi16(bytes2, tap2); + v_s2 = _mm_add_epi32(v_s2, _mm_madd_epi16(mad2, ones)); + + buf += BLOCK_SIZE; + + } while (--n); + + v_s2 = _mm_add_epi32(v_s2, _mm_slli_epi32(v_ps, 5)); + + /* + * Sum epi32 ints v_s1(s2) and accumulate in s1(s2). + */ + +#define S23O1 _MM_SHUFFLE(2,3,0,1) /* A B C D -> B A D C */ +#define S1O32 _MM_SHUFFLE(1,0,3,2) /* A B C D -> C D A B */ + + v_s1 = _mm_add_epi32(v_s1, _mm_shuffle_epi32(v_s1, S23O1)); + v_s1 = _mm_add_epi32(v_s1, _mm_shuffle_epi32(v_s1, S1O32)); + + s1 += _mm_cvtsi128_si32(v_s1); + + v_s2 = _mm_add_epi32(v_s2, _mm_shuffle_epi32(v_s2, S23O1)); + v_s2 = _mm_add_epi32(v_s2, _mm_shuffle_epi32(v_s2, S1O32)); + + s2 = _mm_cvtsi128_si32(v_s2); + +#undef S23O1 +#undef S1O32 + + /* + * Reduce. + */ + s1 %= BASE; + s2 %= BASE; + } + + /* + * Handle leftover data. + */ + if (len) { + if (len >= 16) { + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + len -= 16; + } + + while (len--) { + s2 += (s1 += *buf++); + } + + if (s1 >= BASE) + s1 -= BASE; + s2 %= BASE; + } + + /* + * Return the recombined sums. + */ + return s1 | (s2 << 16); +} + +#elif defined(ADLER32_SIMD_NEON) + +#include + +uint32_t ZLIB_INTERNAL adler32_simd_( /* NEON */ + uint32_t adler, + const unsigned char *buf, + z_size_t len) +{ + /* + * Split Adler-32 into component sums. + */ + uint32_t s1 = adler & 0xffff; + uint32_t s2 = adler >> 16; + + /* + * Serially compute s1 & s2, until the data is 16-byte aligned. + */ + if ((uintptr_t)buf & 15) { + while ((uintptr_t)buf & 15) { + s2 += (s1 += *buf++); + --len; + } + + if (s1 >= BASE) + s1 -= BASE; + s2 %= BASE; + } + + /* + * Process the data in blocks. + */ + const unsigned BLOCK_SIZE = 1 << 5; + + z_size_t blocks = len / BLOCK_SIZE; + len -= blocks * BLOCK_SIZE; + + while (blocks) + { + unsigned n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */ + if (n > blocks) + n = (unsigned) blocks; + blocks -= n; + + /* + * Process n blocks of data. At most NMAX data bytes can be + * processed before s2 must be reduced modulo BASE. + */ + uint32x4_t v_s2 = (uint32x4_t) { 0, 0, 0, s1 * n }; + uint32x4_t v_s1 = (uint32x4_t) { 0, 0, 0, 0 }; + + uint16x8_t v_column_sum_1 = vdupq_n_u16(0); + uint16x8_t v_column_sum_2 = vdupq_n_u16(0); + uint16x8_t v_column_sum_3 = vdupq_n_u16(0); + uint16x8_t v_column_sum_4 = vdupq_n_u16(0); + + do { + /* + * Load 32 input bytes. + */ + const uint8x16_t bytes1 = vld1q_u8((uint8_t*)(buf)); + const uint8x16_t bytes2 = vld1q_u8((uint8_t*)(buf + 16)); + + /* + * Add previous block byte sum to v_s2. + */ + v_s2 = vaddq_u32(v_s2, v_s1); + + /* + * Horizontally add the bytes for s1. + */ + v_s1 = vpadalq_u16(v_s1, vpadalq_u8(vpaddlq_u8(bytes1), bytes2)); + + /* + * Vertically add the bytes for s2. + */ + v_column_sum_1 = vaddw_u8(v_column_sum_1, vget_low_u8 (bytes1)); + v_column_sum_2 = vaddw_u8(v_column_sum_2, vget_high_u8(bytes1)); + v_column_sum_3 = vaddw_u8(v_column_sum_3, vget_low_u8 (bytes2)); + v_column_sum_4 = vaddw_u8(v_column_sum_4, vget_high_u8(bytes2)); + + buf += BLOCK_SIZE; + + } while (--n); + + v_s2 = vshlq_n_u32(v_s2, 5); + + /* + * Multiply-add bytes by [ 32, 31, 30, ... ] for s2. + */ + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_1), + (uint16x4_t) { 32, 31, 30, 29 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_1), + (uint16x4_t) { 28, 27, 26, 25 }); + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_2), + (uint16x4_t) { 24, 23, 22, 21 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_2), + (uint16x4_t) { 20, 19, 18, 17 }); + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_3), + (uint16x4_t) { 16, 15, 14, 13 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_3), + (uint16x4_t) { 12, 11, 10, 9 }); + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_4), + (uint16x4_t) { 8, 7, 6, 5 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_4), + (uint16x4_t) { 4, 3, 2, 1 }); + + /* + * Sum epi32 ints v_s1(s2) and accumulate in s1(s2). + */ + uint32x2_t sum1 = vpadd_u32(vget_low_u32(v_s1), vget_high_u32(v_s1)); + uint32x2_t sum2 = vpadd_u32(vget_low_u32(v_s2), vget_high_u32(v_s2)); + uint32x2_t s1s2 = vpadd_u32(sum1, sum2); + + s1 += vget_lane_u32(s1s2, 0); + s2 += vget_lane_u32(s1s2, 1); + + /* + * Reduce. + */ + s1 %= BASE; + s2 %= BASE; + } + + /* + * Handle leftover data. + */ + if (len) { + if (len >= 16) { + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + len -= 16; + } + + while (len--) { + s2 += (s1 += *buf++); + } + + if (s1 >= BASE) + s1 -= BASE; + s2 %= BASE; + } + + /* + * Return the recombined sums. + */ + return s1 | (s2 << 16); +} + +#elif defined(ADLER32_SIMD_RVV) +#include + +/* + * Patch by Simon Hosie, from: + * https://github.com/cloudflare/zlib/pull/55 + */ + +uint32_t ZLIB_INTERNAL adler32_simd_( /* RVV */ + uint32_t adler, + const unsigned char *buf, + unsigned long len) +{ + size_t vl = __riscv_vsetvlmax_e8m2(); + const vuint16m4_t zero16 = __riscv_vmv_v_x_u16m4(0, vl); + vuint16m4_t a_sum = zero16; + vuint32m8_t b_sum = __riscv_vmv_v_x_u32m8(0, vl); + + /* Deal with the part which is not a multiple of vl first; because it's + * easier to zero-stuff the beginning of the checksum than it is to tweak the + * multipliers and sums for odd lengths afterwards. + */ + size_t head = len & (vl - 1); + if (head > 0) { + vuint8m2_t zero8 = __riscv_vmv_v_x_u8m2(0, vl); + vuint8m2_t in = __riscv_vle8_v_u8m2(buf, vl); + in = __riscv_vslideup(zero8, in, vl - head, vl); + vuint16m4_t in16 = __riscv_vwcvtu_x(in, vl); + a_sum = in16; + buf += head; + } + + /* We have a 32-bit accumulator, and in each iteration we add 22-times a + * 16-bit value, plus another 16-bit value. We periodically subtract up to + * 65535 times BASE to avoid overflow. b_overflow estimates how often we + * need to do this subtraction. + */ + const int b_overflow = BASE / 23; + int fixup = b_overflow; + ssize_t iters = (len - head) / vl; + while (iters > 0) { + const vuint16m4_t a_overflow = __riscv_vrsub(a_sum, BASE, vl); + int batch = iters < 22 ? iters : 22; + iters -= batch; + b_sum = __riscv_vwmaccu(b_sum, batch, a_sum, vl); + vuint16m4_t a_batch = zero16, b_batch = zero16; + + /* Do a short batch, where neither a_sum nor b_sum can overflow a 16-bit + * register. Then add them back into the main accumulators. + */ + while (batch-- > 0) { + vuint8m2_t in8 = __riscv_vle8_v_u8m2(buf, vl); + buf += vl; + b_batch = __riscv_vadd(b_batch, a_batch, vl); + a_batch = __riscv_vwaddu_wv(a_batch, in8, vl); + } + vbool4_t ov = __riscv_vmsgeu(a_batch, a_overflow, vl); + a_sum = __riscv_vadd(a_sum, a_batch, vl); + a_sum = __riscv_vadd_mu(ov, a_sum, a_sum, 65536 - BASE, vl); + b_sum = __riscv_vwaddu_wv(b_sum, b_batch, vl); + if (--fixup <= 0) { + b_sum = __riscv_vnmsac(b_sum, BASE, __riscv_vsrl(b_sum, 16, vl), vl); + fixup = b_overflow; + } + } + /* Adjust per-lane sums to have appropriate offsets from the end of the + * buffer. + */ + const vuint16m4_t off = __riscv_vrsub(__riscv_vid_v_u16m4(vl), vl, vl); + vuint16m4_t bsum16 = __riscv_vncvt_x(__riscv_vremu(b_sum, BASE, vl), vl); + b_sum = __riscv_vadd(__riscv_vwmulu(a_sum, off, vl), + __riscv_vwmulu(bsum16, vl, vl), vl); + bsum16 = __riscv_vncvt_x(__riscv_vremu(b_sum, BASE, vl), vl); + + /* And finally, do a horizontal sum across the registers for the final + * result. + */ + uint32_t a = adler & 0xffff; + uint32_t b = ((adler >> 16) + a * (len % BASE)) % BASE; + vuint32m1_t sca = __riscv_vmv_v_x_u32m1(a, 1); + vuint32m1_t scb = __riscv_vmv_v_x_u32m1(b, 1); + sca = __riscv_vwredsumu(a_sum, sca, vl); + scb = __riscv_vwredsumu(bsum16, scb, vl); + a = __riscv_vmv_x(sca); + b = __riscv_vmv_x(scb); + a %= BASE; + b %= BASE; + return (b << 16) | a; +} + +#endif /* ADLER32_SIMD_SSSE3 */ diff --git a/packages/runtime/vendor/zlib/adler32_simd.h b/packages/runtime/vendor/zlib/adler32_simd.h new file mode 100644 index 000000000..0b2361a51 --- /dev/null +++ b/packages/runtime/vendor/zlib/adler32_simd.h @@ -0,0 +1,16 @@ +/* adler32_simd.h + * + * Copyright 2017 The Chromium Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include + +#include "zconf.h" +#include "zutil.h" + +uint32_t ZLIB_INTERNAL adler32_simd_( + uint32_t adler, + const unsigned char *buf, + z_size_t len); diff --git a/packages/runtime/vendor/zlib/chromeconf.h b/packages/runtime/vendor/zlib/chromeconf.h new file mode 100644 index 000000000..77f0e29e0 --- /dev/null +++ b/packages/runtime/vendor/zlib/chromeconf.h @@ -0,0 +1,211 @@ +/* Copyright 2017 The Chromium Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. */ + +#ifndef THIRD_PARTY_ZLIB_CHROMECONF_H_ +#define THIRD_PARTY_ZLIB_CHROMECONF_H_ + +#if defined(COMPONENT_BUILD) +#if defined(WIN32) +#if defined(ZLIB_IMPLEMENTATION) +#define ZEXTERN __declspec(dllexport) +#else +#define ZEXTERN __declspec(dllimport) +#endif +#elif defined(ZLIB_IMPLEMENTATION) +#define ZEXTERN __attribute__((visibility("default"))) +#endif +#endif + +/* Rename all zlib names with a Cr_z_ prefix. This is based on the Z_PREFIX + * option from zconf.h, but with a custom prefix. Where zconf.h would rename + * both a macro and its underscore-suffixed internal implementation (such as + * deflateInit2 and deflateInit2_), only the implementation is renamed here. + * The Byte type is also omitted. + * + * To generate this list, run + * sed -rn -e 's/^# *define +([^ ]+) +(z_[^ ]+)$/#define \1 Cr_\2/p' zconf.h + * (use -E instead of -r on macOS). + * + * gzread is also addressed by modifications in gzread.c and zlib.h. */ + +#define Z_CR_PREFIX_SET + +#define _dist_code Cr_z__dist_code +#define _length_code Cr_z__length_code +#define _tr_align Cr_z__tr_align +#define _tr_flush_bits Cr_z__tr_flush_bits +#define _tr_flush_block Cr_z__tr_flush_block +#define _tr_init Cr_z__tr_init +#define _tr_stored_block Cr_z__tr_stored_block +#define _tr_tally Cr_z__tr_tally +#define adler32 Cr_z_adler32 +#define adler32_combine Cr_z_adler32_combine +#define adler32_combine64 Cr_z_adler32_combine64 +#define adler32_z Cr_z_adler32_z +#define compress Cr_z_compress +#define compress2 Cr_z_compress2 +#define compress_z Cr_z_compress_z +#define compress2_z Cr_z_compress2_z +#define compressBound Cr_z_compressBound +#define compressBound_z Cr_z_compressBound_z +#define crc32 Cr_z_crc32 +#define crc32_combine Cr_z_crc32_combine +#define crc32_combine64 Cr_z_crc32_combine64 +#define crc32_combine_gen Cr_z_crc32_combine_gen +#define crc32_combine_gen64 Cr_z_crc32_combine_gen64 +#define crc32_combine_op Cr_z_crc32_combine_op +#define crc32_z Cr_z_crc32_z +#define deflate Cr_z_deflate +#define deflateBound Cr_z_deflateBound +#define deflateBound_z Cr_z_deflateBound_z +#define deflateCopy Cr_z_deflateCopy +#define deflateEnd Cr_z_deflateEnd +#define deflateGetDictionary Cr_z_deflateGetDictionary +/* #undef deflateInit */ +/* #undef deflateInit2 */ +#define deflateInit2_ Cr_z_deflateInit2_ +#define deflateInit_ Cr_z_deflateInit_ +#define deflateParams Cr_z_deflateParams +#define deflatePending Cr_z_deflatePending +#define deflatePrime Cr_z_deflatePrime +#define deflateReset Cr_z_deflateReset +#define deflateResetKeep Cr_z_deflateResetKeep +#define deflateSetDictionary Cr_z_deflateSetDictionary +#define deflateSetHeader Cr_z_deflateSetHeader +#define deflateTune Cr_z_deflateTune +#define deflateUsed Cr_z_deflateUsed +#define deflate_copyright Cr_z_deflate_copyright +#define get_crc_table Cr_z_get_crc_table +#define gz_error Cr_z_gz_error +#define gz_intmax Cr_z_gz_intmax +#define gz_strwinerror Cr_z_gz_strwinerror +#define gzbuffer Cr_z_gzbuffer +#define gzclearerr Cr_z_gzclearerr +#define gzclose Cr_z_gzclose +#define gzclose_r Cr_z_gzclose_r +#define gzclose_w Cr_z_gzclose_w +#define gzdirect Cr_z_gzdirect +#define gzdopen Cr_z_gzdopen +#define gzeof Cr_z_gzeof +#define gzerror Cr_z_gzerror +#define gzflush Cr_z_gzflush +#define gzfread Cr_z_gzfread +#define gzfwrite Cr_z_gzfwrite +#define gzgetc Cr_z_gzgetc +#define gzgetc_ Cr_z_gzgetc_ +#define gzgets Cr_z_gzgets +#define gzoffset Cr_z_gzoffset +#define gzoffset64 Cr_z_gzoffset64 +#define gzopen Cr_z_gzopen +#define gzopen64 Cr_z_gzopen64 +#define gzopen_w Cr_z_gzopen_w +#define gzprintf Cr_z_gzprintf +#define gzputc Cr_z_gzputc +#define gzputs Cr_z_gzputs +#define gzread Cr_z_gzread +#define gzrewind Cr_z_gzrewind +#define gzseek Cr_z_gzseek +#define gzseek64 Cr_z_gzseek64 +#define gzsetparams Cr_z_gzsetparams +#define gztell Cr_z_gztell +#define gztell64 Cr_z_gztell64 +#define gzungetc Cr_z_gzungetc +#define gzvprintf Cr_z_gzvprintf +#define gzwrite Cr_z_gzwrite +#define inflate Cr_z_inflate +#define inflateBack Cr_z_inflateBack +#define inflateBackEnd Cr_z_inflateBackEnd +/* #undef inflateBackInit */ +#define inflateBackInit_ Cr_z_inflateBackInit_ +#define inflateCodesUsed Cr_z_inflateCodesUsed +#define inflateCopy Cr_z_inflateCopy +#define inflateEnd Cr_z_inflateEnd +#define inflateGetDictionary Cr_z_inflateGetDictionary +#define inflateGetHeader Cr_z_inflateGetHeader +/* #undef inflateInit */ +/* #undef inflateInit2 */ +#define inflateInit2_ Cr_z_inflateInit2_ +#define inflateInit_ Cr_z_inflateInit_ +#define inflateMark Cr_z_inflateMark +#define inflatePrime Cr_z_inflatePrime +#define inflateReset Cr_z_inflateReset +#define inflateReset2 Cr_z_inflateReset2 +#define inflateResetKeep Cr_z_inflateResetKeep +#define inflateSetDictionary Cr_z_inflateSetDictionary +#define inflateSync Cr_z_inflateSync +#define inflateSyncPoint Cr_z_inflateSyncPoint +#define inflateUndermine Cr_z_inflateUndermine +#define inflateValidate Cr_z_inflateValidate +#define inflate_copyright Cr_z_inflate_copyright +#define inflate_fast Cr_z_inflate_fast +#define inflate_table Cr_z_inflate_table +#define inflate_fixed Cr_z_inflate_fixed +#define uncompress Cr_z_uncompress +#define uncompress2 Cr_z_uncompress2 +#define uncompress_z Cr_z_uncompress_z +#define uncompress2_z Cr_z_uncompress2_z +#define zError Cr_z_zError +#define zcalloc Cr_z_zcalloc +#define zcfree Cr_z_zcfree +#define zlibCompileFlags Cr_z_zlibCompileFlags +#define zlibVersion Cr_z_zlibVersion +/* #undef Byte */ +#define Bytef Cr_z_Bytef +#define alloc_func Cr_z_alloc_func +#define charf Cr_z_charf +#define free_func Cr_z_free_func +#define gzFile Cr_z_gzFile +#define gz_header Cr_z_gz_header +#define gz_headerp Cr_z_gz_headerp +#define in_func Cr_z_in_func +#define intf Cr_z_intf +#define out_func Cr_z_out_func +#define uInt Cr_z_uInt +#define uIntf Cr_z_uIntf +#define uLong Cr_z_uLong +#define uLongf Cr_z_uLongf +#define voidp Cr_z_voidp +#define voidpc Cr_z_voidpc +#define voidpf Cr_z_voidpf +#define gz_header_s Cr_z_gz_header_s +/* #undef internal_state */ +/* #undef z_off64_t */ + +/* An exported symbol that isn't handled by Z_PREFIX in zconf.h */ +#define z_errmsg Cr_z_z_errmsg + +/* Symbols added in simd.patch */ +#define copy_with_crc Cr_z_copy_with_crc +#define crc_finalize Cr_z_crc_finalize +#define crc_fold_512to32 Cr_z_crc_fold_512to32 +#define crc_fold_copy Cr_z_crc_fold_copy +#define crc_fold_init Cr_z_crc_fold_init +#define crc_reset Cr_z_crc_reset +#define fill_window_sse Cr_z_fill_window_sse +#define deflate_read_buf Cr_z_deflate_read_buf +#define x86_check_features Cr_z_x86_check_features +#define x86_cpu_enable_simd Cr_z_x86_cpu_enable_simd + +/* Symbols added by adler_simd.c */ +#define adler32_simd_ Cr_z_adler32_simd_ +#define x86_cpu_enable_ssse3 Cr_z_x86_cpu_enable_ssse3 + +/* Symbols added by contrib/optimizations/inffast_chunk */ +#define inflate_fast_chunk_ Cr_z_inflate_fast_chunk_ + +/* Symbols added by crc32_simd.c */ +#define crc32_sse42_simd_ Cr_z_crc32_sse42_simd_ + +/* Symbols added by armv8_crc32 */ +#define arm_cpu_enable_crc32 Cr_z_arm_cpu_enable_crc32 +#define arm_cpu_enable_pmull Cr_z_arm_cpu_enable_pmull +#define arm_check_features Cr_z_arm_check_features +#define armv8_crc32_little Cr_z_armv8_crc32_little +#define armv8_crc32_pmull_little Cr_z_armv8_crc32_pmull_little + +/* Symbols added by cpu_features.c */ +#define cpu_check_features Cr_z_cpu_check_features +#define x86_cpu_enable_sse2 Cr_z_x86_cpu_enable_sse2 + +#endif /* THIRD_PARTY_ZLIB_CHROMECONF_H_ */ diff --git a/packages/runtime/vendor/zlib/compare256.h b/packages/runtime/vendor/zlib/compare256.h new file mode 100644 index 000000000..80061c3fe --- /dev/null +++ b/packages/runtime/vendor/zlib/compare256.h @@ -0,0 +1,56 @@ +/* compare256.h + * + * This does 256-byte match comparison for deflate's longest_match. + * + * Copyright 2026 The Chromium Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ +#ifndef COMPARE256_H +#define COMPARE256_H + +#include +#include +#if defined(_MSC_VER) && !defined(__clang__) +#include +#endif + +/* Safe unaligned 16-bit load. Compilers optimize this into a single load + * instruction. */ +static inline uint16_t read16(const void* p) { + uint16_t v; + memcpy(&v, p, sizeof(v)); + return v; +} + +/* Finds the byte offset (0..7) of the first difference in non-zero XOR mask + * `x`: + */ +static inline int compare256_diff(uint64_t x) { +#if defined(_MSC_VER) && !defined(__clang__) + unsigned long i; + _BitScanForward64(&i, x); + return (int)i / 8; +#else + return __builtin_ctzll(x) / 8; +#endif +} + +/* Returns the number of matching leading bytes (0 to 256) between src0 and + * src1. Compares 8 bytes per iteration with early exit. */ +static inline int compare256(const unsigned char* src0, + const unsigned char* src1) { + int len = 0; + do { + uint64_t a, b, x; + memcpy(&a, src0 + len, sizeof(a)); + memcpy(&b, src1 + len, sizeof(b)); + x = a ^ b; + if (x) + return len + compare256_diff(x); + len += (int)sizeof(a); + } while (len < 256); + return 256; +} + +#endif /* COMPARE256_H */ diff --git a/packages/runtime/vendor/zlib/compress.c b/packages/runtime/vendor/zlib/compress.c index f43bacf7a..e3a7516e4 100644 --- a/packages/runtime/vendor/zlib/compress.c +++ b/packages/runtime/vendor/zlib/compress.c @@ -1,5 +1,5 @@ /* compress.c -- compress a memory buffer - * Copyright (C) 1995-2005, 2014, 2016 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2026 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -18,13 +18,19 @@ compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer, Z_STREAM_ERROR if the level parameter is invalid. + + The _z versions of the functions take size_t length arguments. */ -int ZEXPORT compress2(Bytef *dest, uLongf *destLen, const Bytef *source, - uLong sourceLen, int level) { +int ZEXPORT compress2_z(Bytef *dest, z_size_t *destLen, const Bytef *source, + z_size_t sourceLen, int level) { z_stream stream; int err; const uInt max = (uInt)-1; - uLong left; + z_size_t left; + + if ((sourceLen > 0 && source == NULL) || + destLen == NULL || (*destLen > 0 && dest == NULL)) + return Z_STREAM_ERROR; left = *destLen; *destLen = 0; @@ -43,23 +49,36 @@ int ZEXPORT compress2(Bytef *dest, uLongf *destLen, const Bytef *source, do { if (stream.avail_out == 0) { - stream.avail_out = left > (uLong)max ? max : (uInt)left; + stream.avail_out = left > (z_size_t)max ? max : (uInt)left; left -= stream.avail_out; } if (stream.avail_in == 0) { - stream.avail_in = sourceLen > (uLong)max ? max : (uInt)sourceLen; + stream.avail_in = sourceLen > (z_size_t)max ? max : + (uInt)sourceLen; sourceLen -= stream.avail_in; } err = deflate(&stream, sourceLen ? Z_NO_FLUSH : Z_FINISH); } while (err == Z_OK); - *destLen = stream.total_out; + *destLen = (z_size_t)(stream.next_out - dest); deflateEnd(&stream); return err == Z_STREAM_END ? Z_OK : err; } - +int ZEXPORT compress2(Bytef *dest, uLongf *destLen, const Bytef *source, + uLong sourceLen, int level) { + int ret; + z_size_t got = *destLen; + ret = compress2_z(dest, &got, source, sourceLen, level); + *destLen = (uLong)got; + return ret; +} /* =========================================================================== */ +int ZEXPORT compress_z(Bytef *dest, z_size_t *destLen, const Bytef *source, + z_size_t sourceLen) { + return compress2_z(dest, destLen, source, sourceLen, + Z_DEFAULT_COMPRESSION); +} int ZEXPORT compress(Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen) { return compress2(dest, destLen, source, sourceLen, Z_DEFAULT_COMPRESSION); @@ -69,7 +88,23 @@ int ZEXPORT compress(Bytef *dest, uLongf *destLen, const Bytef *source, If the default memLevel or windowBits for deflateInit() is changed, then this function needs to be updated. */ +z_size_t ZEXPORT compressBound_z(z_size_t sourceLen) { + z_size_t bound = sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + + (sourceLen >> 25) + 13; + + /* FIXME(cavalcantii): usage of CRC32 Castagnoli as a hash function + * for the hash table of symbols used for compression has a side effect + * where for compression level [4, 5] it will increase the output buffer size + * by 0.1% (i.e. less than 1%) for a high entropy input (i.e. random data). + * To avoid a scenario where client code would fail, for safety we increase + * the expected output size by 0.8% (i.e. 8x more than the worst scenario). + * See: http://crbug.com/990489 + */ + bound += bound >> 7; // Equivalent to 1.0078125 + + return bound < sourceLen ? (z_size_t)-1 : bound; +} uLong ZEXPORT compressBound(uLong sourceLen) { - return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + - (sourceLen >> 25) + 13; + z_size_t bound = compressBound_z(sourceLen); + return (uLong)bound != bound ? (uLong)-1 : (uLong)bound; } diff --git a/packages/runtime/vendor/zlib/contrib/optimizations/chunkcopy.h b/packages/runtime/vendor/zlib/contrib/optimizations/chunkcopy.h new file mode 100644 index 000000000..97efff3a4 --- /dev/null +++ b/packages/runtime/vendor/zlib/contrib/optimizations/chunkcopy.h @@ -0,0 +1,564 @@ +/* chunkcopy.h -- fast chunk copy and set operations + * Copyright (C) 2017 ARM, Inc. + * Copyright 2017 The Chromium Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#ifndef CHUNKCOPY_H +#define CHUNKCOPY_H + +#include +#include "zutil.h" + +#define Z_STATIC_ASSERT(name, assert) typedef char name[(assert) ? 1 : -1] + +#if __STDC_VERSION__ >= 199901L +#define Z_RESTRICT restrict +#else +#define Z_RESTRICT +#endif + +#if defined(__clang__) || defined(__GNUC__) || defined(__llvm__) +#define Z_BUILTIN_MEMCPY __builtin_memcpy +#define Z_BUILTIN_MEMSET __builtin_memset +#else +#define Z_BUILTIN_MEMCPY zmemcpy +#define Z_BUILTIN_MEMSET zmemset +#endif + +#if defined(INFLATE_CHUNK_SIMD_NEON) +#include +typedef uint8x16_t z_vec128i_t; +#elif defined(INFLATE_CHUNK_SIMD_SSE2) +#include +typedef __m128i z_vec128i_t; +#elif defined(INFLATE_CHUNK_GENERIC) +typedef struct { uint8_t x[16]; } z_vec128i_t; +#else +#error chunkcopy.h inflate chunk SIMD is not defined for your build target +#endif + +/* + * Suppress MSan errors about copying uninitialized bytes (crbug.com/1376033). + */ +#define Z_DISABLE_MSAN +#if defined(__has_feature) + #if __has_feature(memory_sanitizer) + #undef Z_DISABLE_MSAN + #define Z_DISABLE_MSAN __attribute__((no_sanitize("memory"))) + #endif +#endif + +/* + * chunk copy type: the z_vec128i_t type size should be exactly 128-bits + * and equal to CHUNKCOPY_CHUNK_SIZE. + */ +#define CHUNKCOPY_CHUNK_SIZE sizeof(z_vec128i_t) + +Z_STATIC_ASSERT(vector_128_bits_wide, + CHUNKCOPY_CHUNK_SIZE == sizeof(int8_t) * 16); + +/* + * Ask the compiler to perform a wide, unaligned load with a machine + * instruction appropriate for the z_vec128i_t type. + */ +static inline z_vec128i_t loadchunk( + const unsigned char FAR* s) Z_DISABLE_MSAN { + z_vec128i_t v; + Z_BUILTIN_MEMCPY(&v, s, sizeof(v)); + return v; +} + +/* + * Ask the compiler to perform a wide, unaligned store with a machine + * instruction appropriate for the z_vec128i_t type. + */ +static inline void storechunk( + unsigned char FAR* d, + const z_vec128i_t v) { + Z_BUILTIN_MEMCPY(d, &v, sizeof(v)); +} + +/* + * Perform a memcpy-like operation, assuming that length is non-zero and that + * it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE bytes of output even if + * the length is shorter than this. + * + * It also guarantees that it will properly unroll the data if the distance + * between `out` and `from` is at least CHUNKCOPY_CHUNK_SIZE, which we rely on + * in chunkcopy_relaxed(). + * + * Aside from better memory bus utilisation, this means that short copies + * (CHUNKCOPY_CHUNK_SIZE bytes or fewer) will fall straight through the loop + * without iteration, which will hopefully make the branch prediction more + * reliable. + */ +static inline unsigned char FAR* chunkcopy_core( + unsigned char FAR* out, + const unsigned char FAR* from, + unsigned len) Z_DISABLE_MSAN { + const int bump = (--len % CHUNKCOPY_CHUNK_SIZE) + 1; + storechunk(out, loadchunk(from)); + out += bump; + from += bump; + len /= CHUNKCOPY_CHUNK_SIZE; + while (len-- > 0) { + storechunk(out, loadchunk(from)); + out += CHUNKCOPY_CHUNK_SIZE; + from += CHUNKCOPY_CHUNK_SIZE; + } + return out; +} + +/* + * Like chunkcopy_core(), but avoid writing beyond of legal output. + * + * Accepts an additional pointer to the end of safe output. A generic safe + * copy would use (out + len), but it's normally the case that the end of the + * output buffer is beyond the end of the current copy, and this can still be + * exploited. + */ +static inline unsigned char FAR* chunkcopy_core_safe( + unsigned char FAR* out, + const unsigned char FAR* from, + unsigned len, + unsigned char FAR* limit) { + Assert(out + len <= limit, "chunk copy exceeds safety limit"); + if ((limit - out) < (ptrdiff_t)CHUNKCOPY_CHUNK_SIZE) { + const unsigned char FAR* Z_RESTRICT rfrom = from; + Assert((uintptr_t)out - (uintptr_t)from >= len, + "invalid restrict in chunkcopy_core_safe"); + Assert((uintptr_t)from - (uintptr_t)out >= len, + "invalid restrict in chunkcopy_core_safe"); + if (len & 8) { + Z_BUILTIN_MEMCPY(out, rfrom, 8); + out += 8; + rfrom += 8; + } + if (len & 4) { + Z_BUILTIN_MEMCPY(out, rfrom, 4); + out += 4; + rfrom += 4; + } + if (len & 2) { + Z_BUILTIN_MEMCPY(out, rfrom, 2); + out += 2; + rfrom += 2; + } + if (len & 1) { + *out++ = *rfrom++; + } + return out; + } + return chunkcopy_core(out, from, len); +} + +/* + * Perform short copies until distance can be rewritten as being at least + * CHUNKCOPY_CHUNK_SIZE. + * + * Assumes it's OK to overwrite at least the first 2*CHUNKCOPY_CHUNK_SIZE + * bytes of output even if the copy is shorter than this. This assumption + * holds within zlib inflate_fast(), which starts every iteration with at + * least 258 bytes of output space available (258 being the maximum length + * output from a single token; see inffast.c). + */ +static inline unsigned char FAR* chunkunroll_relaxed( + unsigned char FAR* out, + unsigned FAR* dist, + unsigned FAR* len) Z_DISABLE_MSAN { + const unsigned char FAR* from = out - *dist; + while (*dist < *len && *dist < CHUNKCOPY_CHUNK_SIZE) { + storechunk(out, loadchunk(from)); + out += *dist; + *len -= *dist; + *dist += *dist; + } + return out; +} + +#if defined(INFLATE_CHUNK_SIMD_NEON) +/* + * v_load64_dup(): load *src as an unaligned 64-bit int and duplicate it in + * every 64-bit component of the 128-bit result (64-bit int splat). + */ +static inline z_vec128i_t v_load64_dup(const void* src) { + return vcombine_u8(vld1_u8(src), vld1_u8(src)); +} + +/* + * v_load32_dup(): load *src as an unaligned 32-bit int and duplicate it in + * every 32-bit component of the 128-bit result (32-bit int splat). + */ +static inline z_vec128i_t v_load32_dup(const void* src) { + int32_t i32; + Z_BUILTIN_MEMCPY(&i32, src, sizeof(i32)); + return vreinterpretq_u8_s32(vdupq_n_s32(i32)); +} + +/* + * v_load16_dup(): load *src as an unaligned 16-bit int and duplicate it in + * every 16-bit component of the 128-bit result (16-bit int splat). + */ +static inline z_vec128i_t v_load16_dup(const void* src) { + int16_t i16; + Z_BUILTIN_MEMCPY(&i16, src, sizeof(i16)); + return vreinterpretq_u8_s16(vdupq_n_s16(i16)); +} + +/* + * v_load8_dup(): load the 8-bit int *src and duplicate it in every 8-bit + * component of the 128-bit result (8-bit int splat). + */ +static inline z_vec128i_t v_load8_dup(const void* src) { + return vld1q_dup_u8((const uint8_t*)src); +} + +/* + * v_store_128(): store the 128-bit vec in a memory destination (that might + * not be 16-byte aligned) void* out. + */ +static inline void v_store_128(void* out, const z_vec128i_t vec) { + vst1q_u8(out, vec); +} + +#elif defined(INFLATE_CHUNK_SIMD_SSE2) +/* + * v_load64_dup(): load *src as an unaligned 64-bit int and duplicate it in + * every 64-bit component of the 128-bit result (64-bit int splat). + */ +static inline z_vec128i_t v_load64_dup(const void* src) { + int64_t i64; + Z_BUILTIN_MEMCPY(&i64, src, sizeof(i64)); + return _mm_set1_epi64x(i64); +} + +/* + * v_load32_dup(): load *src as an unaligned 32-bit int and duplicate it in + * every 32-bit component of the 128-bit result (32-bit int splat). + */ +static inline z_vec128i_t v_load32_dup(const void* src) { + int32_t i32; + Z_BUILTIN_MEMCPY(&i32, src, sizeof(i32)); + return _mm_set1_epi32(i32); +} + +/* + * v_load16_dup(): load *src as an unaligned 16-bit int and duplicate it in + * every 16-bit component of the 128-bit result (16-bit int splat). + */ +static inline z_vec128i_t v_load16_dup(const void* src) { + int16_t i16; + Z_BUILTIN_MEMCPY(&i16, src, sizeof(i16)); + return _mm_set1_epi16(i16); +} + +/* + * v_load8_dup(): load the 8-bit int *src and duplicate it in every 8-bit + * component of the 128-bit result (8-bit int splat). + */ +static inline z_vec128i_t v_load8_dup(const void* src) { + return _mm_set1_epi8(*(const char*)src); +} + +/* + * v_store_128(): store the 128-bit vec in a memory destination (that might + * not be 16-byte aligned) void* out. + */ +static inline void v_store_128(void* out, const z_vec128i_t vec) { + _mm_storeu_si128((__m128i*)out, vec); +} +#elif defined(INFLATE_CHUNK_GENERIC) +/* + * Default implementations for chunk-copy functions rely on memcpy() being + * inlined by the compiler for best performance. This is most likely to work + * as expected when the length argument is constant (as is the case here) and + * the target supports unaligned loads and stores. Since that's not always a + * safe assumption, this may need extra compiler arguments such as + * `-mno-strict-align` or `-munaligned-access`, or the availability of + * extensions like SIMD. + */ + +/* + * v_load64_dup(): load *src as an unaligned 64-bit int and duplicate it in + * every 64-bit component of the 128-bit result (64-bit int splat). + */ +static inline z_vec128i_t v_load64_dup(const void* src) { + int64_t in; + Z_BUILTIN_MEMCPY(&in, src, sizeof(in)); + z_vec128i_t out; + for (int i = 0; i < sizeof(out); i += sizeof(in)) { + Z_BUILTIN_MEMCPY((uint8_t*)&out + i, &in, sizeof(in)); + } + return out; +} + +/* + * v_load32_dup(): load *src as an unaligned 32-bit int and duplicate it in + * every 32-bit component of the 128-bit result (32-bit int splat). + */ +static inline z_vec128i_t v_load32_dup(const void* src) { + int32_t in; + Z_BUILTIN_MEMCPY(&in, src, sizeof(in)); + z_vec128i_t out; + for (int i = 0; i < sizeof(out); i += sizeof(in)) { + Z_BUILTIN_MEMCPY((uint8_t*)&out + i, &in, sizeof(in)); + } + return out; +} + +/* + * v_load16_dup(): load *src as an unaligned 16-bit int and duplicate it in + * every 16-bit component of the 128-bit result (16-bit int splat). + */ +static inline z_vec128i_t v_load16_dup(const void* src) { + int16_t in; + Z_BUILTIN_MEMCPY(&in, src, sizeof(in)); + z_vec128i_t out; + for (int i = 0; i < sizeof(out); i += sizeof(in)) { + Z_BUILTIN_MEMCPY((uint8_t*)&out + i, &in, sizeof(in)); + } + return out; +} + +/* + * v_load8_dup(): load the 8-bit int *src and duplicate it in every 8-bit + * component of the 128-bit result (8-bit int splat). + */ +static inline z_vec128i_t v_load8_dup(const void* src) { + int8_t in = *(const uint8_t*)src; + z_vec128i_t out; + Z_BUILTIN_MEMSET(&out, in, sizeof(out)); + return out; +} + +/* + * v_store_128(): store the 128-bit vec in a memory destination (that might + * not be 16-byte aligned) void* out. + */ +static inline void v_store_128(void* out, const z_vec128i_t vec) { + Z_BUILTIN_MEMCPY(out, &vec, sizeof(vec)); +} +#endif + +/* + * Perform an overlapping copy which behaves as a memset() operation, but + * supporting periods other than one, and assume that length is non-zero and + * that it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE*3 bytes of output + * even if the length is shorter than this. + */ +static inline unsigned char FAR* chunkset_core( + unsigned char FAR* out, + unsigned period, + unsigned len) { + z_vec128i_t v; + const int bump = ((len - 1) % sizeof(v)) + 1; + + switch (period) { + case 1: + v = v_load8_dup(out - 1); + v_store_128(out, v); + out += bump; + len -= bump; + while (len > 0) { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } + return out; + case 2: + v = v_load16_dup(out - 2); + v_store_128(out, v); + out += bump; + len -= bump; + if (len > 0) { + v = v_load16_dup(out - 2); + do { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } while (len > 0); + } + return out; + case 4: + v = v_load32_dup(out - 4); + v_store_128(out, v); + out += bump; + len -= bump; + if (len > 0) { + v = v_load32_dup(out - 4); + do { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } while (len > 0); + } + return out; + case 8: + v = v_load64_dup(out - 8); + v_store_128(out, v); + out += bump; + len -= bump; + if (len > 0) { + v = v_load64_dup(out - 8); + do { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } while (len > 0); + } + return out; + } + out = chunkunroll_relaxed(out, &period, &len); + return chunkcopy_core(out, out - period, len); +} + +/* + * Perform a memcpy-like operation, but assume that length is non-zero and that + * it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE bytes of output even if + * the length is shorter than this. + * + * Unlike chunkcopy_core() above, no guarantee is made regarding the behaviour + * of overlapping buffers, regardless of the distance between the pointers. + * This is reflected in the `restrict`-qualified pointers, allowing the + * compiler to re-order loads and stores. + */ +static inline unsigned char FAR* chunkcopy_relaxed( + unsigned char FAR* Z_RESTRICT out, + const unsigned char FAR* Z_RESTRICT from, + unsigned len) { + Assert((uintptr_t)out - (uintptr_t)from >= len, + "invalid restrict in chunkcopy_relaxed"); + Assert((uintptr_t)from - (uintptr_t)out >= len, + "invalid restrict in chunkcopy_relaxed"); + return chunkcopy_core(out, from, len); +} + +/* + * Like chunkcopy_relaxed(), but avoid writing beyond of legal output. + * + * Unlike chunkcopy_core_safe() above, no guarantee is made regarding the + * behaviour of overlapping buffers, regardless of the distance between the + * pointers. This is reflected in the `restrict`-qualified pointers, allowing + * the compiler to re-order loads and stores. + * + * Accepts an additional pointer to the end of safe output. A generic safe + * copy would use (out + len), but it's normally the case that the end of the + * output buffer is beyond the end of the current copy, and this can still be + * exploited. + */ +static inline unsigned char FAR* chunkcopy_safe( + unsigned char FAR* out, + const unsigned char FAR* Z_RESTRICT from, + unsigned len, + unsigned char FAR* limit) { + Assert(out + len <= limit, "chunk copy exceeds safety limit"); + Assert((uintptr_t)out - (uintptr_t)from >= len, + "invalid restrict in chunkcopy_safe"); + Assert((uintptr_t)from - (uintptr_t)out >= len, + "invalid restrict in chunkcopy_safe"); + + return chunkcopy_core_safe(out, from, len, limit); +} + +/* + * Perform chunky copy within the same buffer, where the source and destination + * may potentially overlap. + * + * Assumes that len > 0 on entry, and that it's safe to write at least + * CHUNKCOPY_CHUNK_SIZE*3 bytes to the output. + */ +static inline unsigned char FAR* chunkcopy_lapped_relaxed( + unsigned char FAR* out, + unsigned dist, + unsigned len) { + if (dist < len && dist < CHUNKCOPY_CHUNK_SIZE) { + return chunkset_core(out, dist, len); + } + return chunkcopy_core(out, out - dist, len); +} + +/* + * Behave like chunkcopy_lapped_relaxed(), but avoid writing beyond of legal + * output. + * + * Accepts an additional pointer to the end of safe output. A generic safe + * copy would use (out + len), but it's normally the case that the end of the + * output buffer is beyond the end of the current copy, and this can still be + * exploited. + */ +static inline unsigned char FAR* chunkcopy_lapped_safe( + unsigned char FAR* out, + unsigned dist, + unsigned len, + unsigned char FAR* limit) { + Assert(out + len <= limit, "chunk copy exceeds safety limit"); + if ((limit - out) < (ptrdiff_t)(3 * CHUNKCOPY_CHUNK_SIZE)) { + /* TODO(cavalcantii): try harder to optimise this */ + while (len-- > 0) { + *out = *(out - dist); + out++; + } + return out; + } + return chunkcopy_lapped_relaxed(out, dist, len); +} + +/* TODO(cavalcanti): see crbug.com/1110083. */ +static inline unsigned char FAR* chunkcopy_safe_ugly(unsigned char FAR* out, + unsigned dist, + unsigned len, + unsigned char FAR* limit) { +#if defined(__GNUC__) && !defined(__clang__) + /* Speed is the same as using chunkcopy_safe + w/ GCC on ARM (tested gcc 6.3 and 7.5) and avoids + undefined behavior. + */ + return chunkcopy_core_safe(out, out - dist, len, limit); +#elif defined(__clang__) && defined(ARMV8_OS_ANDROID) && !defined(__aarch64__) + /* Seems to perform better on 32bit (i.e. Android). */ + return chunkcopy_core_safe(out, out - dist, len, limit); +#else + /* Seems to perform better on 64bit. */ + return chunkcopy_lapped_safe(out, dist, len, limit); +#endif +} + +/* + * The chunk-copy code above deals with writing the decoded DEFLATE data to + * the output with SIMD methods to increase decode speed. Reading the input + * to the DEFLATE decoder with a wide, SIMD method can also increase decode + * speed. This option is supported on little endian machines, and reads the + * input data in 64-bit (8 byte) chunks. + */ + +#ifdef INFLATE_CHUNK_READ_64LE +/* + * Buffer the input in a uint64_t (8 bytes) in the wide input reading case. + */ +typedef uint64_t inflate_holder_t; + +/* + * Ask the compiler to perform a wide, unaligned load of a uint64_t using a + * machine instruction appropriate for the uint64_t type. + */ +static inline inflate_holder_t read64le(const unsigned char FAR *in) { + inflate_holder_t input; + Z_BUILTIN_MEMCPY(&input, in, sizeof(input)); + return input; +} +#else +/* + * Otherwise, buffer the input bits using zlib's default input buffer type. + */ +typedef unsigned long inflate_holder_t; + +#endif /* INFLATE_CHUNK_READ_64LE */ + +#undef Z_STATIC_ASSERT +#undef Z_RESTRICT +#undef Z_BUILTIN_MEMCPY +#undef Z_DISABLE_MSAN + +#endif /* CHUNKCOPY_H */ diff --git a/packages/runtime/vendor/zlib/contrib/optimizations/inffast_chunk.c b/packages/runtime/vendor/zlib/contrib/optimizations/inffast_chunk.c new file mode 100644 index 000000000..3c93da051 --- /dev/null +++ b/packages/runtime/vendor/zlib/contrib/optimizations/inffast_chunk.c @@ -0,0 +1,376 @@ +/* inffast_chunk.c -- fast decoding + * Copyright (C) 1995-2026 Mark Adler + * Copyright 2023 The Chromium Authors + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "contrib/optimizations/inffast_chunk.h" +#include "contrib/optimizations/chunkcopy.h" + +#ifdef ASMINF +# pragma message("Assembler code may have bugs -- use at your own risk") +#else + +/* + Decode literal, length, and distance codes and write out the resulting + literal and match bytes until either not enough input or output is + available, an end-of-block is encountered, or a data error is encountered. + When large enough input and output buffers are supplied to inflate(), for + example, a 16K input buffer and a 64K output buffer, more than 95% of the + inflate() execution time is spent in this routine. + + Entry assumptions: + + state->mode == LEN + strm->avail_in >= INFLATE_FAST_MIN_INPUT (6 or 8 bytes + 7 bytes) + strm->avail_out >= INFLATE_FAST_MIN_OUTPUT (258 bytes + 2 bytes) + start >= strm->avail_out + state->bits < 8 + (state->hold >> state->bits) == 0 + strm->next_out[0..strm->avail_out] does not overlap with + strm->next_in[0..strm->avail_in] + strm->state->window is allocated with an additional + CHUNKCOPY_CHUNK_SIZE-1 bytes of padding beyond strm->state->wsize + + On return, state->mode is one of: + + LEN -- ran out of enough output space or enough available input + TYPE -- reached end of block code, inflate() to interpret next block + BAD -- error in block data + + Notes: + + INFLATE_FAST_MIN_INPUT: 6 or 8 bytes + 7 bytes + + - The maximum input bits used by a length/distance pair is 15 bits for the + length code, 5 bits for the length extra, 15 bits for the distance code, + and 13 bits for the distance extra. This totals 48 bits, or six bytes. + Therefore if strm->avail_in >= 6, then there is enough input to avoid + checking for available input while decoding. + + - The wide input data reading option reads 64 input bits at a time. Thus, + if strm->avail_in >= 8, then there is enough input to avoid checking for + available input while decoding. Reading consumes the input with: + + hold |= read64le(in) << bits; + in += 6; + bits += 48; + + reporting 6 bytes of new input because |bits| is 0..15 (2 bytes rounded + up, worst case) and 6 bytes is enough to decode as noted above. At exit, + hold &= (1U << bits) - 1 drops excess input to keep the invariant: + + (state->hold >> state->bits) == 0 + + INFLATE_FAST_MIN_OUTPUT: 258 bytes + 2 bytes for literals = 260 bytes + + - The maximum bytes that a single length/distance pair can output is 258 + bytes, which is the maximum length that can be coded. inflate_fast() + requires strm->avail_out >= 260 for each loop to avoid checking for + available output space while decoding. + */ +void ZLIB_INTERNAL inflate_fast_chunk_(z_streamp strm, unsigned start) { + struct inflate_state FAR *state; + z_const unsigned char FAR *in; /* local strm->next_in */ + z_const unsigned char FAR *last; /* have enough input while in < last */ + unsigned char FAR *out; /* local strm->next_out */ + unsigned char FAR *beg; /* inflate()'s initial strm->next_out */ + unsigned char FAR *end; /* while out < end, enough space available */ + unsigned char FAR *limit; /* safety limit for chunky copies */ +#ifdef INFLATE_STRICT + unsigned dmax; /* maximum distance from zlib header */ +#endif + unsigned wsize; /* window size or zero if not using window */ + unsigned whave; /* valid bytes in the window */ + unsigned wnext; /* window write index */ + unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */ + inflate_holder_t hold; /* local strm->hold */ + unsigned bits; /* local strm->bits */ + code const FAR *lcode; /* local strm->lencode */ + code const FAR *dcode; /* local strm->distcode */ + unsigned lmask; /* mask for first level of length codes */ + unsigned dmask; /* mask for first level of distance codes */ + code const *here; /* retrieved table entry */ + unsigned op; /* code bits, operation, extra bits, or */ + /* window position, window bytes to copy */ + unsigned len; /* match length, unused bytes */ + unsigned dist; /* match distance */ + unsigned char FAR *from; /* where to copy match from */ + + /* copy state to local variables */ + state = (struct inflate_state FAR *)strm->state; + in = strm->next_in; + last = in + (strm->avail_in - (INFLATE_FAST_MIN_INPUT - 1)); + out = strm->next_out; + beg = out - (start - strm->avail_out); + end = out + (strm->avail_out - (INFLATE_FAST_MIN_OUTPUT - 1)); + limit = out + strm->avail_out; +#ifdef INFLATE_STRICT + dmax = state->dmax; +#endif + wsize = state->wsize; + whave = state->whave; + wnext = (state->wnext == 0 && whave >= wsize) ? wsize : state->wnext; + window = state->window; + hold = state->hold; + bits = state->bits; + lcode = state->lencode; + dcode = state->distcode; + lmask = (1U << state->lenbits) - 1; + dmask = (1U << state->distbits) - 1; + +#ifdef INFLATE_CHUNK_READ_64LE +#define REFILL() do { \ + Assert(bits < 64, "### Too many bits in inflate_fast."); \ + hold |= read64le(in) << bits; \ + in += 7; \ + in -= bits >> 3; \ + bits |= 56; \ + } while (0) +#endif + + /* decode literals and length/distances until end-of-block or not enough + input data or output space */ + do { +#ifdef INFLATE_CHUNK_READ_64LE + REFILL(); +#else + if (bits < 15) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; + } +#endif + here = lcode + (hold & lmask); +#ifdef INFLATE_CHUNK_READ_64LE + if (here->op == 0) { /* literal */ + Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here->val)); + *out++ = (unsigned char)(here->val); + hold >>= here->bits; + bits -= here->bits; + here = lcode + (hold & lmask); + if (here->op == 0) { /* literal */ + Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ? + "inflate: 2nd literal '%c'\n" : + "inflate: 2nd literal 0x%02x\n", here->val)); + *out++ = (unsigned char)(here->val); + hold >>= here->bits; + bits -= here->bits; + here = lcode + (hold & lmask); + } + } +#endif + dolen: + op = (unsigned)(here->bits); + hold >>= op; + bits -= op; + op = (unsigned)(here->op); + if (op == 0) { /* literal */ + Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here->val)); + *out++ = (unsigned char)(here->val); + } + else if (op & 16) { /* length base */ + len = (unsigned)(here->val); + op &= 15; /* number of extra bits */ + if (op) { +#ifndef INFLATE_CHUNK_READ_64LE + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + } +#endif + len += (unsigned)hold & ((1U << op) - 1); + hold >>= op; + bits -= op; + } + Tracevv((stderr, "inflate: length %u\n", len)); +#ifndef INFLATE_CHUNK_READ_64LE + if (bits < 15) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; + } +#endif + here = dcode + (hold & dmask); + dodist: + op = (unsigned)(here->bits); + hold >>= op; + bits -= op; + op = (unsigned)(here->op); + if (op & 16) { /* distance base */ + dist = (unsigned)(here->val); + op &= 15; /* number of extra bits */ + /* we have two fast-path loads: 10+10 + 15+5 + 15 = 55, + but we may need to refill here in the worst case */ + if (bits < op) { +#ifdef INFLATE_CHUNK_READ_64LE + REFILL(); +#else + hold += (unsigned long)(*in++) << bits; + bits += 8; + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + } +#endif + } + dist += (unsigned)hold & ((1U << op) - 1); +#ifdef INFLATE_STRICT + if (dist > dmax) { + strm->msg = (z_const char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + hold >>= op; + bits -= op; + Tracevv((stderr, "inflate: distance %u\n", dist)); + op = (unsigned)(out - beg); /* max distance in output */ + if (dist > op) { /* see if copy from window */ + op = dist - op; /* distance back in window */ + if (op > whave) { + if (state->sane) { + strm->msg = + (z_const char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + if (len <= op - whave) { + do { + *out++ = 0; + } while (--len); + continue; + } + len -= op - whave; + do { + *out++ = 0; + } while (--op > whave); + if (op == 0) { + from = out - dist; + do { + *out++ = *from++; + } while (--len); + continue; + } +#endif + } + from = window; + if (wnext >= op) { /* contiguous in window */ + from += wnext - op; + } + else { /* wrap around window */ + op -= wnext; + from += wsize - op; + if (op < len) { /* some from end of window */ + len -= op; + out = chunkcopy_safe(out, from, op, limit); + from = window; /* more from start of window */ + op = wnext; + /* This (rare) case can create a situation where + the first chunkcopy below must be checked. + */ + } + } + if (op < len) { /* still need some from output */ + out = chunkcopy_safe(out, from, op, limit); + len -= op; + /* When dist is small the amount of data that can be + copied from the window is also small, and progress + towards the dangerous end of the output buffer is + also small. This means that for trivial memsets and + for chunkunroll_relaxed() a safety check is + unnecessary. However, these conditions may not be + entered at all, and in that case it's possible that + the main copy is near the end. + */ + out = chunkunroll_relaxed(out, &dist, &len); + out = chunkcopy_safe_ugly(out, dist, len, limit); + } else { + /* from points to window, so there is no risk of + overlapping pointers requiring memset-like behaviour + */ + out = chunkcopy_safe(out, from, len, limit); + } + } + else { + /* Whole reference is in range of current output. No + range checks are necessary because we start with room + for at least 258 bytes of output, so unroll and roundoff + operations can write beyond `out+len` so long as they + stay within 258 bytes of `out`. + */ + out = chunkcopy_lapped_relaxed(out, dist, len); + } + } + else if ((op & 64) == 0) { /* 2nd level distance code */ + here = dcode + here->val + (hold & ((1U << op) - 1)); + goto dodist; + } + else { + strm->msg = (z_const char *)"invalid distance code"; + state->mode = BAD; + break; + } + } + else if ((op & 64) == 0) { /* 2nd level length code */ + here = lcode + here->val + (hold & ((1U << op) - 1)); + goto dolen; + } + else if (op & 32) { /* end-of-block */ + Tracevv((stderr, "inflate: end of block\n")); + state->mode = TYPE; + break; + } + else { + strm->msg = (z_const char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + } while (in < last && out < end); + + /* return unused bytes (on entry, bits < 8, so in won't go too far back) */ + len = bits >> 3; + in -= len; + bits -= len << 3; + hold &= (1U << bits) - 1; + + /* update state and return */ + strm->next_in = in; + strm->next_out = out; + strm->avail_in = (unsigned)(in < last ? + (INFLATE_FAST_MIN_INPUT - 1) + (last - in) : + (INFLATE_FAST_MIN_INPUT - 1) - (in - last)); + strm->avail_out = (unsigned)(out < end ? + (INFLATE_FAST_MIN_OUTPUT - 1) + (end - out) : + (INFLATE_FAST_MIN_OUTPUT - 1) - (out - end)); + state->hold = hold; + state->bits = bits; + + Assert((state->hold >> state->bits) == 0, "invalid input data state"); +} + +/* + inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe): + - Using bit fields for code structure + - Different op definition to avoid & for extra bits (do & for table bits) + - Three separate decoding do-loops for direct, window, and wnext == 0 + - Special case for distance > 1 copies to do overlapped load and store copy + - Explicit branch predictions (based on measured branch probabilities) + - Deferring match copy and interspersed it with decoding subsequent codes + - Swapping literal/length else + - Swapping window/direct else + - Larger unrolled copy loops (three is about right) + - Moving len -= 3 statement into middle of loop + */ + +#endif /* !ASMINF */ diff --git a/packages/runtime/vendor/zlib/contrib/optimizations/inffast_chunk.h b/packages/runtime/vendor/zlib/contrib/optimizations/inffast_chunk.h new file mode 100644 index 000000000..e75ee5e91 --- /dev/null +++ b/packages/runtime/vendor/zlib/contrib/optimizations/inffast_chunk.h @@ -0,0 +1,42 @@ +/* inffast_chunk.h -- header to use inffast_chunk.c + * Copyright (C) 1995-2003, 2010 Mark Adler + * Copyright (C) 2017 ARM, Inc. + * Copyright 2023 The Chromium Authors + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +#include "inffast.h" + +/* INFLATE_FAST_MIN_INPUT: + The minimum number of input bytes needed so that we can safely call + inflate_fast() with only one up-front bounds check. One + length/distance code pair (15 bits for the length code, 5 bits for length + extra, 15 bits for the distance code, 13 bits for distance extra) requires + reading up to 48 input bits. Additionally, in the same iteraction, we may + decode two literals from the root-table (requiring MIN_OUTPUT = 258 + 2). + + Each root-table entry is up to 10 bits, for a total of 68 input bits each + iteraction. + + The refill variant reads 8 bytes from the buffer at a time, and advances + the input pointer by up to 7 bytes, ensuring there are at least 56-bits + available in the bit-buffer. The technique was documented by Fabian Giesen + on his blog as variant 4 in the article 'Reading bits in far too many ways': + https://fgiesen.wordpress.com/2018/02/20/ + + In the worst case, we may refill twice in the same iteraction, requiring + MIN_INPUT = 8 + 7. +*/ +#ifdef INFLATE_CHUNK_READ_64LE +#undef INFLATE_FAST_MIN_INPUT +#define INFLATE_FAST_MIN_INPUT 15 +#undef INFLATE_FAST_MIN_OUTPUT +#define INFLATE_FAST_MIN_OUTPUT 260 +#endif + +void ZLIB_INTERNAL inflate_fast_chunk_(z_streamp strm, unsigned start); diff --git a/packages/runtime/vendor/zlib/contrib/optimizations/inflate.c b/packages/runtime/vendor/zlib/contrib/optimizations/inflate.c new file mode 100644 index 000000000..3268320e9 --- /dev/null +++ b/packages/runtime/vendor/zlib/contrib/optimizations/inflate.c @@ -0,0 +1,1562 @@ +/* inflate.c -- zlib decompression + * Copyright (C) 1995-2022 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* + * Change history: + * + * 1.2.beta0 24 Nov 2002 + * - First version -- complete rewrite of inflate to simplify code, avoid + * creation of window when not needed, minimize use of window when it is + * needed, make inffast.c even faster, implement gzip decoding, and to + * improve code readability and style over the previous zlib inflate code + * + * 1.2.beta1 25 Nov 2002 + * - Use pointers for available input and output checking in inffast.c + * - Remove input and output counters in inffast.c + * - Change inffast.c entry and loop from avail_in >= 7 to >= 6 + * - Remove unnecessary second byte pull from length extra in inffast.c + * - Unroll direct copy to three copies per loop in inffast.c + * + * 1.2.beta2 4 Dec 2002 + * - Change external routine names to reduce potential conflicts + * - Correct filename to inffixed.h for fixed tables in inflate.c + * - Make hbuf[] unsigned char to match parameter type in inflate.c + * - Change strm->next_out[-state->offset] to *(strm->next_out - state->offset) + * to avoid negation problem on Alphas (64 bit) in inflate.c + * + * 1.2.beta3 22 Dec 2002 + * - Add comments on state->bits assertion in inffast.c + * - Add comments on op field in inftrees.h + * - Fix bug in reuse of allocated window after inflateReset() + * - Remove bit fields--back to byte structure for speed + * - Remove distance extra == 0 check in inflate_fast()--only helps for lengths + * - Change post-increments to pre-increments in inflate_fast(), PPC biased? + * - Add compile time option, POSTINC, to use post-increments instead (Intel?) + * - Make MATCH copy in inflate() much faster for when inflate_fast() not used + * - Use local copies of stream next and avail values, as well as local bit + * buffer and bit count in inflate()--for speed when inflate_fast() not used + * + * 1.2.beta4 1 Jan 2003 + * - Split ptr - 257 statements in inflate_table() to avoid compiler warnings + * - Move a comment on output buffer sizes from inffast.c to inflate.c + * - Add comments in inffast.c to introduce the inflate_fast() routine + * - Rearrange window copies in inflate_fast() for speed and simplification + * - Unroll last copy for window match in inflate_fast() + * - Use local copies of window variables in inflate_fast() for speed + * - Pull out common wnext == 0 case for speed in inflate_fast() + * - Make op and len in inflate_fast() unsigned for consistency + * - Add FAR to lcode and dcode declarations in inflate_fast() + * - Simplified bad distance check in inflate_fast() + * - Added inflateBackInit(), inflateBack(), and inflateBackEnd() in new + * source file infback.c to provide a call-back interface to inflate for + * programs like gzip and unzip -- uses window as output buffer to avoid + * window copying + * + * 1.2.beta5 1 Jan 2003 + * - Improved inflateBack() interface to allow the caller to provide initial + * input in strm. + * - Fixed stored blocks bug in inflateBack() + * + * 1.2.beta6 4 Jan 2003 + * - Added comments in inffast.c on effectiveness of POSTINC + * - Typecasting all around to reduce compiler warnings + * - Changed loops from while (1) or do {} while (1) to for (;;), again to + * make compilers happy + * - Changed type of window in inflateBackInit() to unsigned char * + * + * 1.2.beta7 27 Jan 2003 + * - Changed many types to unsigned or unsigned short to avoid warnings + * - Added inflateCopy() function + * + * 1.2.0 9 Mar 2003 + * - Changed inflateBack() interface to provide separate opaque descriptors + * for the in() and out() functions + * - Changed inflateBack() argument and in_func typedef to swap the length + * and buffer address return values for the input function + * - Check next_in and next_out for Z_NULL on entry to inflate() + * + * The history for versions after 1.2.0 are in ChangeLog in zlib distribution. + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "contrib/optimizations/inffast_chunk.h" +#include "contrib/optimizations/chunkcopy.h" + +#ifdef MAKEFIXED +# ifndef BUILDFIXED +# define BUILDFIXED +# endif +#endif + +local int inflateStateCheck(z_streamp strm) { + struct inflate_state FAR *state; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + state = (struct inflate_state FAR *)strm->state; + if (state == Z_NULL || state->strm != strm || + state->mode < HEAD || state->mode > SYNC) + return 1; + return 0; +} + +int ZEXPORT inflateResetKeep(z_streamp strm) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + strm->total_in = strm->total_out = state->total = 0; + strm->msg = Z_NULL; + if (state->wrap) /* to support ill-conceived Java test suite */ + strm->adler = state->wrap & 1; + state->mode = HEAD; + state->last = 0; + state->havedict = 0; + state->flags = -1; + state->dmax = 32768U; + state->head = Z_NULL; + state->hold = 0; + state->bits = 0; + state->lencode = state->distcode = state->next = state->codes; + state->sane = 1; + state->back = -1; + Tracev((stderr, "inflate: reset\n")); + return Z_OK; +} + +int ZEXPORT inflateReset(z_streamp strm) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + state->wsize = 0; + state->whave = 0; + state->wnext = 0; + return inflateResetKeep(strm); +} + +int ZEXPORT inflateReset2(z_streamp strm, int windowBits) { + int wrap; + struct inflate_state FAR *state; + + /* get the state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* extract wrap request from windowBits parameter */ + if (windowBits < 0) { + if (windowBits < -15) + return Z_STREAM_ERROR; + wrap = 0; + windowBits = -windowBits; + } + else { + wrap = (windowBits >> 4) + 5; +#ifdef GUNZIP + if (windowBits < 48) + windowBits &= 15; +#endif + } + + /* set number of window bits, free window if different */ + if (windowBits && (windowBits < 8 || windowBits > 15)) + return Z_STREAM_ERROR; + if (state->window != Z_NULL && state->wbits != (unsigned)windowBits) { + ZFREE(strm, state->window); + state->window = Z_NULL; + } + + /* update state and reset the rest of it */ + state->wrap = wrap; + state->wbits = (unsigned)windowBits; + return inflateReset(strm); +} + +int ZEXPORT inflateInit2_(z_streamp strm, int windowBits, + const char *version, int stream_size) { + int ret; + struct inflate_state FAR *state; + + if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || + stream_size != (int)(sizeof(z_stream))) + return Z_VERSION_ERROR; + if (strm == Z_NULL) return Z_STREAM_ERROR; + strm->msg = Z_NULL; /* in case we return an error */ + if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zalloc = zcalloc; + strm->opaque = (voidpf)0; +#endif + } + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif + state = (struct inflate_state FAR *) + ZALLOC(strm, 1, sizeof(struct inflate_state)); + if (state == Z_NULL) return Z_MEM_ERROR; + Tracev((stderr, "inflate: allocated\n")); + strm->state = (struct internal_state FAR *)state; + state->strm = strm; + state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ + state->check = 1L; /* 1L is the result of adler32() zero length data */ + ret = inflateReset2(strm, windowBits); + if (ret != Z_OK) { + ZFREE(strm, state); + strm->state = Z_NULL; + } + return ret; +} + +int ZEXPORT inflateInit_(z_streamp strm, const char *version, + int stream_size) { + return inflateInit2_(strm, DEF_WBITS, version, stream_size); +} + +int ZEXPORT inflatePrime(z_streamp strm, int bits, int value) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + if (bits == 0) + return Z_OK; + state = (struct inflate_state FAR *)strm->state; + if (bits < 0) { + state->hold = 0; + state->bits = 0; + return Z_OK; + } + if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; + value &= (1L << bits) - 1; + state->hold += (unsigned)value << state->bits; + state->bits += (uInt)bits; + return Z_OK; +} + +/* + Return state with length and distance decoding tables and index sizes set to + fixed code decoding. Normally this returns fixed tables from inffixed.h. + If BUILDFIXED is defined, then instead this routine builds the tables the + first time it's called, and returns those tables the first time and + thereafter. This reduces the size of the code by about 2K bytes, in + exchange for a little execution time. However, BUILDFIXED should not be + used for threaded applications, since the rewriting of the tables and virgin + may not be thread-safe. + */ +local void fixedtables(struct inflate_state FAR *state) { +#ifdef BUILDFIXED + static int virgin = 1; + static code *lenfix, *distfix; + static code fixed[544]; + + /* build fixed huffman tables if first call (may not be thread safe) */ + if (virgin) { + unsigned sym, bits; + static code *next; + + /* literal/length table */ + sym = 0; + while (sym < 144) state->lens[sym++] = 8; + while (sym < 256) state->lens[sym++] = 9; + while (sym < 280) state->lens[sym++] = 7; + while (sym < 288) state->lens[sym++] = 8; + next = fixed; + lenfix = next; + bits = 9; + inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); + + /* distance table */ + sym = 0; + while (sym < 32) state->lens[sym++] = 5; + distfix = next; + bits = 5; + inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); + + /* do this just once */ + virgin = 0; + } +#else /* !BUILDFIXED */ +# include "inffixed.h" +#endif /* BUILDFIXED */ + state->lencode = lenfix; + state->lenbits = 9; + state->distcode = distfix; + state->distbits = 5; +} + +#ifdef MAKEFIXED +#include + +/* + Write out the inffixed.h that is #include'd above. Defining MAKEFIXED also + defines BUILDFIXED, so the tables are built on the fly. makefixed() writes + those tables to stdout, which would be piped to inffixed.h. A small program + can simply call makefixed to do this: + + void makefixed(void); + + int main(void) + { + makefixed(); + return 0; + } + + Then that can be linked with zlib built with MAKEFIXED defined and run: + + a.out > inffixed.h + */ +void makefixed(void) +{ + unsigned low, size; + struct inflate_state state; + + fixedtables(&state); + puts(" /* inffixed.h -- table for decoding fixed codes"); + puts(" * Generated automatically by makefixed()."); + puts(" */"); + puts(""); + puts(" /* WARNING: this file should *not* be used by applications."); + puts(" It is part of the implementation of this library and is"); + puts(" subject to change. Applications should only use zlib.h."); + puts(" */"); + puts(""); + size = 1U << 9; + printf(" static const code lenfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 7) == 0) printf("\n "); + printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, + state.lencode[low].bits, state.lencode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); + size = 1U << 5; + printf("\n static const code distfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 6) == 0) printf("\n "); + printf("{%u,%u,%d}", state.distcode[low].op, state.distcode[low].bits, + state.distcode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); +} +#endif /* MAKEFIXED */ + +/* + Update the window with the last wsize (normally 32K) bytes written before + returning. If window does not exist yet, create it. This is only called + when a window is already in use, or when output has been written during this + inflate call, but the end of the deflate stream has not been reached yet. + It is also called to create a window for dictionary data when a dictionary + is loaded. + + Providing output buffers larger than 32K to inflate() should provide a speed + advantage, since only the last 32K of output is copied to the sliding window + upon return from inflate(), and since all distances after the first 32K of + output will fall in the output data, making match copies simpler and faster. + The advantage may be dependent on the size of the processor's data caches. + */ +local int updatewindow(z_streamp strm, const Bytef *end, unsigned copy) { + struct inflate_state FAR *state; + unsigned dist; + + state = (struct inflate_state FAR *)strm->state; + + /* if it hasn't been done already, allocate space for the window */ + if (state->window == Z_NULL) { + unsigned wsize = 1U << state->wbits; + state->window = (unsigned char FAR *) + ZALLOC(strm, wsize + CHUNKCOPY_CHUNK_SIZE, + sizeof(unsigned char)); + if (state->window == Z_NULL) return 1; +#ifdef INFLATE_CLEAR_UNUSED_UNDEFINED + /* Copies from the overflow portion of this buffer are undefined and + may cause analysis tools to raise a warning if we don't initialize + it. However, this undefined data overwrites other undefined data + and is subsequently either overwritten or left deliberately + undefined at the end of decode; so there's really no point. + */ + zmemzero(state->window + wsize, CHUNKCOPY_CHUNK_SIZE); +#endif + } + + /* if window not in use yet, initialize */ + if (state->wsize == 0) { + state->wsize = 1U << state->wbits; + state->wnext = 0; + state->whave = 0; + } + + /* copy state->wsize or less output bytes into the circular window */ + if (copy >= state->wsize) { + zmemcpy(state->window, end - state->wsize, state->wsize); + state->wnext = 0; + state->whave = state->wsize; + } + else { + dist = state->wsize - state->wnext; + if (dist > copy) dist = copy; + zmemcpy(state->window + state->wnext, end - copy, dist); + copy -= dist; + if (copy) { + zmemcpy(state->window, end - copy, copy); + state->wnext = copy; + state->whave = state->wsize; + } + else { + state->wnext += dist; + if (state->wnext == state->wsize) state->wnext = 0; + if (state->whave < state->wsize) state->whave += dist; + } + } + return 0; +} + +/* Macros for inflate(): */ + +/* check function to use adler32() for zlib or crc32() for gzip */ +#ifdef GUNZIP +# define UPDATE_CHECK(check, buf, len) \ + (state->flags ? crc32(check, buf, len) : adler32(check, buf, len)) +#else +# define UPDATE_CHECK(check, buf, len) adler32(check, buf, len) +#endif + +/* check macros for header crc */ +#ifdef GUNZIP +# define CRC2(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + check = crc32(check, hbuf, 2); \ + } while (0) + +# define CRC4(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + hbuf[2] = (unsigned char)((word) >> 16); \ + hbuf[3] = (unsigned char)((word) >> 24); \ + check = crc32(check, hbuf, 4); \ + } while (0) +#endif + +/* Load registers with state in inflate() for speed */ +#define LOAD() \ + do { \ + put = strm->next_out; \ + left = strm->avail_out; \ + next = strm->next_in; \ + have = strm->avail_in; \ + hold = state->hold; \ + bits = state->bits; \ + } while (0) + +/* Restore state from registers in inflate() */ +#define RESTORE() \ + do { \ + strm->next_out = put; \ + strm->avail_out = left; \ + strm->next_in = next; \ + strm->avail_in = have; \ + state->hold = hold; \ + state->bits = bits; \ + } while (0) + +/* Clear the input bit accumulator */ +#define INITBITS() \ + do { \ + hold = 0; \ + bits = 0; \ + } while (0) + +/* Get a byte of input into the bit accumulator, or return from inflate() + if there is no input available. */ +#define PULLBYTE() \ + do { \ + if (have == 0) goto inf_leave; \ + have--; \ + hold += (unsigned long)(*next++) << bits; \ + bits += 8; \ + } while (0) + +/* Assure that there are at least n bits in the bit accumulator. If there is + not enough available input to do that, then return from inflate(). */ +#define NEEDBITS(n) \ + do { \ + while (bits < (unsigned)(n)) \ + PULLBYTE(); \ + } while (0) + +/* Return the low n bits of the bit accumulator (n < 16) */ +#define BITS(n) \ + ((unsigned)hold & ((1U << (n)) - 1)) + +/* Remove n bits from the bit accumulator */ +#define DROPBITS(n) \ + do { \ + hold >>= (n); \ + bits -= (unsigned)(n); \ + } while (0) + +/* Remove zero to seven bits as needed to go to a byte boundary */ +#define BYTEBITS() \ + do { \ + hold >>= bits & 7; \ + bits -= bits & 7; \ + } while (0) + +/* + inflate() uses a state machine to process as much input data and generate as + much output data as possible before returning. The state machine is + structured roughly as follows: + + for (;;) switch (state) { + ... + case STATEn: + if (not enough input data or output space to make progress) + return; + ... make progress ... + state = STATEm; + break; + ... + } + + so when inflate() is called again, the same case is attempted again, and + if the appropriate resources are provided, the machine proceeds to the + next state. The NEEDBITS() macro is usually the way the state evaluates + whether it can proceed or should return. NEEDBITS() does the return if + the requested bits are not available. The typical use of the BITS macros + is: + + NEEDBITS(n); + ... do something with BITS(n) ... + DROPBITS(n); + + where NEEDBITS(n) either returns from inflate() if there isn't enough + input left to load n bits into the accumulator, or it continues. BITS(n) + gives the low n bits in the accumulator. When done, DROPBITS(n) drops + the low n bits off the accumulator. INITBITS() clears the accumulator + and sets the number of available bits to zero. BYTEBITS() discards just + enough bits to put the accumulator on a byte boundary. After BYTEBITS() + and a NEEDBITS(8), then BITS(8) would return the next byte in the stream. + + NEEDBITS(n) uses PULLBYTE() to get an available byte of input, or to return + if there is no input available. The decoding of variable length codes uses + PULLBYTE() directly in order to pull just enough bytes to decode the next + code, and no more. + + Some states loop until they get enough input, making sure that enough + state information is maintained to continue the loop where it left off + if NEEDBITS() returns in the loop. For example, want, need, and keep + would all have to actually be part of the saved state in case NEEDBITS() + returns: + + case STATEw: + while (want < need) { + NEEDBITS(n); + keep[want++] = BITS(n); + DROPBITS(n); + } + state = STATEx; + case STATEx: + + As shown above, if the next state is also the next case, then the break + is omitted. + + A state may also return if there is not enough output space available to + complete that state. Those states are copying stored data, writing a + literal byte, and copying a matching string. + + When returning, a "goto inf_leave" is used to update the total counters, + update the check value, and determine whether any progress has been made + during that inflate() call in order to return the proper return code. + Progress is defined as a change in either strm->avail_in or strm->avail_out. + When there is a window, goto inf_leave will update the window with the last + output written. If a goto inf_leave occurs in the middle of decompression + and there is no window currently, goto inf_leave will create one and copy + output to the window for the next call of inflate(). + + In this implementation, the flush parameter of inflate() only affects the + return code (per zlib.h). inflate() always writes as much as possible to + strm->next_out, given the space available and the provided input--the effect + documented in zlib.h of Z_SYNC_FLUSH. Furthermore, inflate() always defers + the allocation of and copying into a sliding window until necessary, which + provides the effect documented in zlib.h for Z_FINISH when the entire input + stream available. So the only thing the flush parameter actually does is: + when flush is set to Z_FINISH, inflate() cannot return Z_OK. Instead it + will return Z_BUF_ERROR if it has not reached the end of the stream. + */ + +int ZEXPORT inflate(z_streamp strm, int flush) { + struct inflate_state FAR *state; + z_const unsigned char FAR *next; /* next input */ + unsigned char FAR *put; /* next output */ + unsigned have, left; /* available input and output */ + unsigned long hold; /* bit buffer */ + unsigned bits; /* bits in bit buffer */ + unsigned in, out; /* save starting available input and output */ + unsigned copy; /* number of stored or match bytes to copy */ + unsigned char FAR *from; /* where to copy match bytes from */ + code here; /* current decoding table entry */ + code last; /* parent table entry */ + unsigned len; /* length to copy for repeats, bits to drop */ + int ret; /* return code */ +#ifdef GUNZIP + unsigned char hbuf[4]; /* buffer for gzip header crc calculation */ +#endif + static const unsigned short order[19] = /* permutation of code lengths */ + {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + + if (inflateStateCheck(strm) || strm->next_out == Z_NULL || + (strm->next_in == Z_NULL && strm->avail_in != 0)) + return Z_STREAM_ERROR; + + state = (struct inflate_state FAR *)strm->state; + if (state->mode == TYPE) state->mode = TYPEDO; /* skip check */ + LOAD(); + in = have; + out = left; + ret = Z_OK; + for (;;) + switch (state->mode) { + case HEAD: + if (state->wrap == 0) { + state->mode = TYPEDO; + break; + } + NEEDBITS(16); +#ifdef GUNZIP + if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ + if (state->wbits == 0) + state->wbits = 15; + state->check = crc32(0L, Z_NULL, 0); + CRC2(state->check, hold); + INITBITS(); + state->mode = FLAGS; + break; + } + if (state->head != Z_NULL) + state->head->done = -1; + if (!(state->wrap & 1) || /* check if zlib header allowed */ +#else + if ( +#endif + ((BITS(8) << 8) + (hold >> 8)) % 31) { + strm->msg = (char *)"incorrect header check"; + state->mode = BAD; + break; + } + if (BITS(4) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + DROPBITS(4); + len = BITS(4) + 8; + if (state->wbits == 0) + state->wbits = len; + if (len > 15 || len > state->wbits) { + strm->msg = (char *)"invalid window size"; + state->mode = BAD; + break; + } + state->dmax = 1U << len; + state->flags = 0; /* indicate zlib header */ + Tracev((stderr, "inflate: zlib header ok\n")); + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = hold & 0x200 ? DICTID : TYPE; + INITBITS(); + break; +#ifdef GUNZIP + case FLAGS: + NEEDBITS(16); + state->flags = (int)(hold); + if ((state->flags & 0xff) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + if (state->flags & 0xe000) { + strm->msg = (char *)"unknown header flags set"; + state->mode = BAD; + break; + } + if (state->head != Z_NULL) + state->head->text = (int)((hold >> 8) & 1); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = TIME; + /* fallthrough */ + case TIME: + NEEDBITS(32); + if (state->head != Z_NULL) + state->head->time = hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC4(state->check, hold); + INITBITS(); + state->mode = OS; + /* fallthrough */ + case OS: + NEEDBITS(16); + if (state->head != Z_NULL) { + state->head->xflags = (int)(hold & 0xff); + state->head->os = (int)(hold >> 8); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = EXLEN; + /* fallthrough */ + case EXLEN: + if (state->flags & 0x0400) { + NEEDBITS(16); + state->length = (unsigned)(hold); + if (state->head != Z_NULL) + state->head->extra_len = (unsigned)hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + } + else if (state->head != Z_NULL) + state->head->extra = Z_NULL; + state->mode = EXTRA; + /* fallthrough */ + case EXTRA: + if (state->flags & 0x0400) { + copy = state->length; + if (copy > have) copy = have; + if (copy) { + if (state->head != Z_NULL && + state->head->extra != Z_NULL && + (len = state->head->extra_len - state->length) < + state->head->extra_max) { + zmemcpy(state->head->extra + len, next, + len + copy > state->head->extra_max ? + state->head->extra_max - len : copy); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + state->length -= copy; + } + if (state->length) goto inf_leave; + } + state->length = 0; + state->mode = NAME; + /* fallthrough */ + case NAME: + if (state->flags & 0x0800) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->name != Z_NULL && + state->length < state->head->name_max) + state->head->name[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->name = Z_NULL; + state->length = 0; + state->mode = COMMENT; + /* fallthrough */ + case COMMENT: + if (state->flags & 0x1000) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->comment != Z_NULL && + state->length < state->head->comm_max) + state->head->comment[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->comment = Z_NULL; + state->mode = HCRC; + /* fallthrough */ + case HCRC: + if (state->flags & 0x0200) { + NEEDBITS(16); + if ((state->wrap & 4) && hold != (state->check & 0xffff)) { + strm->msg = (char *)"header crc mismatch"; + state->mode = BAD; + break; + } + INITBITS(); + } + if (state->head != Z_NULL) { + state->head->hcrc = (int)((state->flags >> 9) & 1); + state->head->done = 1; + } + strm->adler = state->check = crc32(0L, Z_NULL, 0); + state->mode = TYPE; + break; +#endif + case DICTID: + NEEDBITS(32); + strm->adler = state->check = ZSWAP32(hold); + INITBITS(); + state->mode = DICT; + /* fallthrough */ + case DICT: + if (state->havedict == 0) { + RESTORE(); + return Z_NEED_DICT; + } + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = TYPE; + /* fallthrough */ + case TYPE: + if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave; + /* fallthrough */ + case TYPEDO: + if (state->last) { + BYTEBITS(); + state->mode = CHECK; + break; + } + NEEDBITS(3); + state->last = BITS(1); + DROPBITS(1); + switch (BITS(2)) { + case 0: /* stored block */ + Tracev((stderr, "inflate: stored block%s\n", + state->last ? " (last)" : "")); + state->mode = STORED; + break; + case 1: /* fixed block */ + fixedtables(state); + Tracev((stderr, "inflate: fixed codes block%s\n", + state->last ? " (last)" : "")); + state->mode = LEN_; /* decode codes */ + if (flush == Z_TREES) { + DROPBITS(2); + goto inf_leave; + } + break; + case 2: /* dynamic block */ + Tracev((stderr, "inflate: dynamic codes block%s\n", + state->last ? " (last)" : "")); + state->mode = TABLE; + break; + case 3: + strm->msg = (char *)"invalid block type"; + state->mode = BAD; + } + DROPBITS(2); + break; + case STORED: + BYTEBITS(); /* go to byte boundary */ + NEEDBITS(32); + if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { + strm->msg = (char *)"invalid stored block lengths"; + state->mode = BAD; + break; + } + state->length = (unsigned)hold & 0xffff; + Tracev((stderr, "inflate: stored length %u\n", + state->length)); + INITBITS(); + state->mode = COPY_; + if (flush == Z_TREES) goto inf_leave; + /* fallthrough */ + case COPY_: + state->mode = COPY; + /* fallthrough */ + case COPY: + copy = state->length; + if (copy) { + if (copy > have) copy = have; + if (copy > left) copy = left; + if (copy == 0) goto inf_leave; + zmemcpy(put, next, copy); + have -= copy; + next += copy; + left -= copy; + put += copy; + state->length -= copy; + break; + } + Tracev((stderr, "inflate: stored end\n")); + state->mode = TYPE; + break; + case TABLE: + NEEDBITS(14); + state->nlen = BITS(5) + 257; + DROPBITS(5); + state->ndist = BITS(5) + 1; + DROPBITS(5); + state->ncode = BITS(4) + 4; + DROPBITS(4); +#ifndef PKZIP_BUG_WORKAROUND + if (state->nlen > 286 || state->ndist > 30) { + strm->msg = (char *)"too many length or distance symbols"; + state->mode = BAD; + break; + } +#endif + Tracev((stderr, "inflate: table sizes ok\n")); + state->have = 0; + state->mode = LENLENS; + /* fallthrough */ + case LENLENS: + while (state->have < state->ncode) { + NEEDBITS(3); + state->lens[order[state->have++]] = (unsigned short)BITS(3); + DROPBITS(3); + } + while (state->have < 19) + state->lens[order[state->have++]] = 0; + state->next = state->codes; + state->lencode = state->distcode = (const code FAR *)(state->next); + state->lenbits = 7; + ret = inflate_table(CODES, state->lens, 19, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid code lengths set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: code lengths ok\n")); + state->have = 0; + state->mode = CODELENS; + /* fallthrough */ + case CODELENS: + while (state->have < state->nlen + state->ndist) { + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.val < 16) { + DROPBITS(here.bits); + state->lens[state->have++] = here.val; + } + else { + if (here.val == 16) { + NEEDBITS(here.bits + 2); + DROPBITS(here.bits); + if (state->have == 0) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + len = state->lens[state->have - 1]; + copy = 3 + BITS(2); + DROPBITS(2); + } + else if (here.val == 17) { + NEEDBITS(here.bits + 3); + DROPBITS(here.bits); + len = 0; + copy = 3 + BITS(3); + DROPBITS(3); + } + else { + NEEDBITS(here.bits + 7); + DROPBITS(here.bits); + len = 0; + copy = 11 + BITS(7); + DROPBITS(7); + } + if (state->have + copy > state->nlen + state->ndist) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + while (copy--) + state->lens[state->have++] = (unsigned short)len; + } + } + + /* handle error breaks in while */ + if (state->mode == BAD) break; + + /* check for end-of-block code (better have one) */ + if (state->lens[256] == 0) { + strm->msg = (char *)"invalid code -- missing end-of-block"; + state->mode = BAD; + break; + } + + /* build code tables -- note: do not change the lenbits or distbits + values here (10 and 9) without reading the comments in inftrees.h + concerning the ENOUGH constants, which depend on those values */ + state->next = state->codes; + state->lencode = (const code FAR *)(state->next); + state->lenbits = 10; + ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid literal/lengths set"; + state->mode = BAD; + break; + } + state->distcode = (const code FAR *)(state->next); + state->distbits = 9; + ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, + &(state->next), &(state->distbits), state->work); + if (ret) { + strm->msg = (char *)"invalid distances set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: codes ok\n")); + state->mode = LEN_; + if (flush == Z_TREES) goto inf_leave; + /* fallthrough */ + case LEN_: + state->mode = LEN; + /* fallthrough */ + case LEN: + if (have >= INFLATE_FAST_MIN_INPUT && + left >= INFLATE_FAST_MIN_OUTPUT) { + RESTORE(); + inflate_fast_chunk_(strm, out); + LOAD(); + if (state->mode == TYPE) + state->back = -1; + break; + } + state->back = 0; + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.op && (here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->lencode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + state->length = (unsigned)here.val; + if ((int)(here.op) == 0) { + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here.val)); + state->mode = LIT; + break; + } + if (here.op & 32) { + Tracevv((stderr, "inflate: end of block\n")); + state->back = -1; + state->mode = TYPE; + break; + } + if (here.op & 64) { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + state->extra = (unsigned)(here.op) & 15; + state->mode = LENEXT; + /* fallthrough */ + case LENEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->length += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } + Tracevv((stderr, "inflate: length %u\n", state->length)); + state->was = state->length; + state->mode = DIST; + /* fallthrough */ + case DIST: + for (;;) { + here = state->distcode[BITS(state->distbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if ((here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->distcode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + if (here.op & 64) { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + state->offset = (unsigned)here.val; + state->extra = (unsigned)(here.op) & 15; + state->mode = DISTEXT; + /* fallthrough */ + case DISTEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->offset += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } +#ifdef INFLATE_STRICT + if (state->offset > state->dmax) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + Tracevv((stderr, "inflate: distance %u\n", state->offset)); + state->mode = MATCH; + /* fallthrough */ + case MATCH: + if (left == 0) goto inf_leave; + copy = out - left; + if (state->offset > copy) { /* copy from window */ + copy = state->offset - copy; + if (copy > state->whave) { + if (state->sane) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + Trace((stderr, "inflate.c too far\n")); + copy -= state->whave; + if (copy > state->length) copy = state->length; + if (copy > left) copy = left; + left -= copy; + state->length -= copy; + do { + *put++ = 0; + } while (--copy); + if (state->length == 0) state->mode = LEN; + break; +#endif + } + if (copy > state->wnext) { + copy -= state->wnext; + from = state->window + (state->wsize - copy); + } + else + from = state->window + (state->wnext - copy); + if (copy > state->length) copy = state->length; + if (copy > left) copy = left; + put = chunkcopy_safe(put, from, copy, put + left); + } + else { /* copy from output */ + copy = state->length; + if (copy > left) copy = left; + put = chunkcopy_lapped_safe(put, state->offset, copy, put + left); + } + left -= copy; + state->length -= copy; + if (state->length == 0) state->mode = LEN; + break; + case LIT: + if (left == 0) goto inf_leave; + *put++ = (unsigned char)(state->length); + left--; + state->mode = LEN; + break; + case CHECK: + if (state->wrap) { + NEEDBITS(32); + out -= left; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE_CHECK(state->check, put - out, out); + out = left; + if ((state->wrap & 4) && ( +#ifdef GUNZIP + state->flags ? hold : +#endif + ZSWAP32(hold)) != state->check) { + strm->msg = (char *)"incorrect data check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: check matches trailer\n")); + } +#ifdef GUNZIP + state->mode = LENGTH; + /* fallthrough */ + case LENGTH: + if (state->wrap && state->flags) { + NEEDBITS(32); + if ((state->wrap & 4) && hold != (state->total & 0xffffffff)) { + strm->msg = (char *)"incorrect length check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: length matches trailer\n")); + } +#endif + state->mode = DONE; + /* fallthrough */ + case DONE: + ret = Z_STREAM_END; + goto inf_leave; + case BAD: + ret = Z_DATA_ERROR; + goto inf_leave; + case MEM: + return Z_MEM_ERROR; + case SYNC: + /* fallthrough */ + default: + return Z_STREAM_ERROR; + } + + /* + Return from inflate(), updating the total counts and the check value. + If there was no progress during the inflate() call, return a buffer + error. Call updatewindow() to create and/or update the window state. + Note: a memory error from inflate() is non-recoverable. + */ + inf_leave: +#if defined(ZLIB_DEBUG) + /* XXX(cavalcantii): I put this in place back in 2017 to help debug faulty + * client code relying on undefined behavior when chunk_copy first landed. + * + * It is save to say after all these years that Chromium code is well + * behaved and works fine with the optimization, therefore we can enable + * this only for DEBUG builds. + * + * We write a defined value in the unused space to help mark + * where the stream has ended. We don't use zeros as that can + * mislead clients relying on undefined behavior (i.e. assuming + * that the data is over when the buffer has a zero/null value). + * + * The basic idea is that if client code is not relying on the zlib context + * to inform the amount of decompressed data, but instead reads the output + * buffer until a zero/null is found, it will fail faster and harder + * when the remaining of the buffer is marked with a symbol (e.g. 0x55). + */ + if (left >= CHUNKCOPY_CHUNK_SIZE) + memset(put, 0x55, CHUNKCOPY_CHUNK_SIZE); + else + memset(put, 0x55, left); +#endif + RESTORE(); + if (state->wsize || (out != strm->avail_out && state->mode < BAD && + (state->mode < CHECK || flush != Z_FINISH))) + if (updatewindow(strm, strm->next_out, out - strm->avail_out)) { + state->mode = MEM; + return Z_MEM_ERROR; + } + in -= strm->avail_in; + out -= strm->avail_out; + strm->total_in += in; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE_CHECK(state->check, strm->next_out - out, out); + strm->data_type = (int)state->bits + (state->last ? 64 : 0) + + (state->mode == TYPE ? 128 : 0) + + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); + if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) + ret = Z_BUF_ERROR; + return ret; +} + +int ZEXPORT inflateEnd(z_streamp strm) { + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->window != Z_NULL) ZFREE(strm, state->window); + ZFREE(strm, strm->state); + strm->state = Z_NULL; + Tracev((stderr, "inflate: end\n")); + return Z_OK; +} + +int ZEXPORT inflateGetDictionary(z_streamp strm, Bytef *dictionary, + uInt *dictLength) { + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* copy dictionary */ + if (state->whave && dictionary != Z_NULL) { + zmemcpy(dictionary, state->window + state->wnext, + state->whave - state->wnext); + zmemcpy(dictionary + state->whave - state->wnext, + state->window, state->wnext); + } + if (dictLength != Z_NULL) + *dictLength = state->whave; + return Z_OK; +} + +int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef *dictionary, + uInt dictLength) { + struct inflate_state FAR *state; + unsigned long dictid; + int ret; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->wrap != 0 && state->mode != DICT) + return Z_STREAM_ERROR; + + /* check for correct dictionary identifier */ + if (state->mode == DICT) { + dictid = adler32(0L, Z_NULL, 0); + dictid = adler32(dictid, dictionary, dictLength); + if (dictid != state->check) + return Z_DATA_ERROR; + } + + /* copy dictionary to window using updatewindow(), which will amend the + existing dictionary if appropriate */ + ret = updatewindow(strm, dictionary + dictLength, dictLength); + if (ret) { + state->mode = MEM; + return Z_MEM_ERROR; + } + state->havedict = 1; + Tracev((stderr, "inflate: dictionary set\n")); + return Z_OK; +} + +int ZEXPORT inflateGetHeader(z_streamp strm, gz_headerp head) { + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; + + /* save header structure */ + state->head = head; + head->done = 0; + return Z_OK; +} + +/* + Search buf[0..len-1] for the pattern: 0, 0, 0xff, 0xff. Return when found + or when out of input. When called, *have is the number of pattern bytes + found in order so far, in 0..3. On return *have is updated to the new + state. If on return *have equals four, then the pattern was found and the + return value is how many bytes were read including the last byte of the + pattern. If *have is less than four, then the pattern has not been found + yet and the return value is len. In the latter case, syncsearch() can be + called again with more data and the *have state. *have is initialized to + zero for the first call. + */ +local unsigned syncsearch(unsigned FAR *have, const unsigned char FAR *buf, + unsigned len) { + unsigned got; + unsigned next; + + got = *have; + next = 0; + while (next < len && got < 4) { + if ((int)(buf[next]) == (got < 2 ? 0 : 0xff)) + got++; + else if (buf[next]) + got = 0; + else + got = 4 - got; + next++; + } + *have = got; + return next; +} + +int ZEXPORT inflateSync(z_streamp strm) { + unsigned len; /* number of bytes to look at or looked at */ + int flags; /* temporary to save header status */ + unsigned long in, out; /* temporary to save total_in and total_out */ + unsigned char buf[4]; /* to restore bit buffer to byte string */ + struct inflate_state FAR *state; + + /* check parameters */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; + + /* if first time, start search in bit buffer */ + if (state->mode != SYNC) { + state->mode = SYNC; + state->hold >>= state->bits & 7; + state->bits -= state->bits & 7; + len = 0; + while (state->bits >= 8) { + buf[len++] = (unsigned char)(state->hold); + state->hold >>= 8; + state->bits -= 8; + } + state->have = 0; + syncsearch(&(state->have), buf, len); + } + + /* search available input */ + len = syncsearch(&(state->have), strm->next_in, strm->avail_in); + strm->avail_in -= len; + strm->next_in += len; + strm->total_in += len; + + /* return no joy or set up to restart inflate() on a new block */ + if (state->have != 4) return Z_DATA_ERROR; + if (state->flags == -1) + state->wrap = 0; /* if no header yet, treat as raw */ + else + state->wrap &= ~4; /* no point in computing a check value now */ + flags = state->flags; + in = strm->total_in; out = strm->total_out; + inflateReset(strm); + strm->total_in = in; strm->total_out = out; + state->flags = flags; + state->mode = TYPE; + return Z_OK; +} + +/* + Returns true if inflate is currently at the end of a block generated by + Z_SYNC_FLUSH or Z_FULL_FLUSH. This function is used by one PPP + implementation to provide an additional safety check. PPP uses + Z_SYNC_FLUSH but removes the length bytes of the resulting empty stored + block. When decompressing, PPP checks that at the end of input packet, + inflate is waiting for these length bytes. + */ +int ZEXPORT inflateSyncPoint(z_streamp strm) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + return state->mode == STORED && state->bits == 0; +} + +int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) { + struct inflate_state FAR *state; + struct inflate_state FAR *copy; + unsigned char FAR *window; + unsigned wsize; + + /* check input */ + if (inflateStateCheck(source) || dest == Z_NULL) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)source->state; + + /* allocate space */ + copy = (struct inflate_state FAR *) + ZALLOC(source, 1, sizeof(struct inflate_state)); + if (copy == Z_NULL) return Z_MEM_ERROR; + window = Z_NULL; + if (state->window != Z_NULL) { + window = (unsigned char FAR *)ZALLOC( + source, (1U << state->wbits) + CHUNKCOPY_CHUNK_SIZE, + sizeof(unsigned char)); + if (window == Z_NULL) { + ZFREE(source, copy); + return Z_MEM_ERROR; + } + } + + /* copy state */ + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); + zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + copy->strm = dest; + if (state->lencode >= state->codes && + state->lencode <= state->codes + ENOUGH - 1) { + copy->lencode = copy->codes + (state->lencode - state->codes); + copy->distcode = copy->codes + (state->distcode - state->codes); + } + copy->next = copy->codes + (state->next - state->codes); + if (window != Z_NULL) { + wsize = 1U << state->wbits; + zmemcpy(window, state->window, wsize); + } + copy->window = window; + dest->state = (struct internal_state FAR *)copy; + return Z_OK; +} + +int ZEXPORT inflateUndermine(z_streamp strm, int subvert) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = !subvert; + return Z_OK; +#else + (void)subvert; + state->sane = 1; + return Z_DATA_ERROR; +#endif +} + +int ZEXPORT inflateValidate(z_streamp strm, int check) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (check && state->wrap) + state->wrap |= 4; + else + state->wrap &= ~4; + return Z_OK; +} + +long ZEXPORT inflateMark(z_streamp strm) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) + return -(1L << 16); + state = (struct inflate_state FAR *)strm->state; + return (long)(((unsigned long)((long)state->back)) << 16) + + (state->mode == COPY ? state->length : + (state->mode == MATCH ? state->was - state->length : 0)); +} + +unsigned long ZEXPORT inflateCodesUsed(z_streamp strm) { + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) return (unsigned long)-1; + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); +} diff --git a/packages/runtime/vendor/zlib/contrib/optimizations/insert_string.h b/packages/runtime/vendor/zlib/contrib/optimizations/insert_string.h new file mode 100644 index 000000000..260b82680 --- /dev/null +++ b/packages/runtime/vendor/zlib/contrib/optimizations/insert_string.h @@ -0,0 +1,80 @@ +/* insert_string.h + * + * Copyright 2019 The Chromium Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#ifndef INSERT_STRING_H +#define INSERT_STRING_H + +#ifndef INLINE +#if defined(_MSC_VER) && !defined(__clang__) +#define INLINE __inline +#else +#define INLINE inline +#endif +#endif + +#include + +/** + * Some applications need to match zlib DEFLATE output exactly [3]. Use the + * canonical zlib Rabin-Karp rolling hash [1,2] in that case. + * + * [1] For a description of the Rabin and Karp algorithm, see "Algorithms" + * book by R. Sedgewick, Addison-Wesley, p252. + * [2] https://www.euccas.me/zlib/#zlib_rabin_karp and also "rolling hash" + * https://en.wikipedia.org/wiki/Rolling_hash + * [3] crbug.com/1316541 AOSP incremental client APK package OTA upgrades. + */ +#ifdef CHROMIUM_ZLIB_NO_CASTAGNOLI +#define USE_ZLIB_RABIN_KARP_ROLLING_HASH +#endif + +/* =========================================================================== + * Update a hash value with the given input byte (Rabin-Karp rolling hash). + * IN assertion: all calls to UPDATE_HASH are made with consecutive input + * characters, so that a running hash key can be computed from the previous + * key instead of complete recalculation each time. + */ +#define UPDATE_HASH(s, h, c) (h = (((h) << s->hash_shift) ^ (c)) & s->hash_mask) + +/* =========================================================================== + * Insert string str in the dictionary and set match_head to the previous head + * of the hash chain (the most recent string with same hash key). Return + * the previous length of the hash chain. + * If this file is compiled with -DFASTEST, the compression level is forced + * to 1, and no hash chains are maintained. + * IN assertion: all calls to INSERT_STRING are made with consecutive input + * characters and the first MIN_MATCH bytes of str are valid (except for + * the last MIN_MATCH-1 bytes of the input file). + */ +local INLINE Pos insert_string(deflate_state* const s, const Pos str) { + Pos ret; +/* insert_string dictionary insertion: ANZAC++ hasher + * significantly improves data compression speed. + * + * Note: the generated compressed output is a valid DEFLATE stream, but will + * differ from canonical zlib output. + */ +#if defined(USE_ZLIB_RABIN_KARP_ROLLING_HASH) + UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH - 1)]); +#else + uint32_t value; + // Validated for little endian archs (i.e. x86, Arm). YMMV for big endian. + zmemcpy(&value, &s->window[str], sizeof(value)); + s->ins_h = ((value * 66521 + 66521) >> 16) & s->hash_mask; +#endif + +#ifdef FASTEST + ret = s->head[s->ins_h]; +#else + ret = s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = str; + + return ret; +} + +#endif /* INSERT_STRING_H */ diff --git a/packages/runtime/vendor/zlib/cpu_features.c b/packages/runtime/vendor/zlib/cpu_features.c new file mode 100644 index 000000000..6613a65e3 --- /dev/null +++ b/packages/runtime/vendor/zlib/cpu_features.c @@ -0,0 +1,218 @@ +/* cpu_features.c -- Processor features detection. + * + * Copyright 2018 The Chromium Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include "cpu_features.h" +#include "zutil.h" + +#include +#if defined(_MSC_VER) +#include +#elif defined(ADLER32_SIMD_SSSE3) +#include +#endif + +/* TODO(cavalcantii): remove checks for x86_flags on deflate. + */ +#if defined(ARMV8_OS_MACOS) +/* Crypto extensions (crc32/pmull) are a baseline feature in ARMv8.1-A, and + * OSX running on arm64 is new enough that these can be assumed without + * runtime detection. + */ +int ZLIB_INTERNAL arm_cpu_enable_crc32 = 1; +int ZLIB_INTERNAL arm_cpu_enable_pmull = 1; +#else +int ZLIB_INTERNAL arm_cpu_enable_crc32 = 0; +int ZLIB_INTERNAL arm_cpu_enable_pmull = 0; +#endif +int ZLIB_INTERNAL x86_cpu_enable_sse2 = 0; +int ZLIB_INTERNAL x86_cpu_enable_ssse3 = 0; +int ZLIB_INTERNAL x86_cpu_enable_simd = 0; +int ZLIB_INTERNAL x86_cpu_enable_avx512 = 0; + +int ZLIB_INTERNAL riscv_cpu_enable_rvv = 0; +int ZLIB_INTERNAL riscv_cpu_enable_vclmul = 0; + +#ifndef CPU_NO_SIMD + +#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || \ + defined(ARMV8_OS_FUCHSIA) || defined(ARMV8_OS_IOS) +#include +#endif + +#if defined(ARMV8_OS_ANDROID) +#include +#elif defined(ARMV8_OS_LINUX) +#include +#include +#elif defined(ARMV8_OS_FUCHSIA) +#include +#include +#include +#elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) +#include +#elif defined(ARMV8_OS_IOS) +#include +#elif !defined(_MSC_VER) +#include +#else +#error cpu_features.c CPU feature detection in not defined for your platform +#endif + +#if !defined(CPU_NO_SIMD) && !defined(ARMV8_OS_MACOS) +static void _cpu_check_features(void); +#endif + +#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || \ + defined(ARMV8_OS_MACOS) || defined(ARMV8_OS_FUCHSIA) || \ + defined(X86_NOT_WINDOWS) || defined(ARMV8_OS_IOS) || \ + defined(RISCV_RVV) +#if !defined(ARMV8_OS_MACOS) +// _cpu_check_features() doesn't need to do anything on mac/arm since all +// features are known at build time, so don't call it. +// Do provide cpu_check_features() (with a no-op implementation) so that we +// don't have to make all callers of it check for mac/arm. +static pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; +#endif +void ZLIB_INTERNAL cpu_check_features(void) +{ +#if !defined(ARMV8_OS_MACOS) + pthread_once(&cpu_check_inited_once, _cpu_check_features); +#endif +} +#elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) +static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; +static BOOL CALLBACK _cpu_check_features_forwarder(PINIT_ONCE once, PVOID param, PVOID* context) +{ + _cpu_check_features(); + return TRUE; +} +void ZLIB_INTERNAL cpu_check_features(void) +{ + InitOnceExecuteOnce(&cpu_check_inited_once, _cpu_check_features_forwarder, + NULL, NULL); +} +#endif + +#if (defined(__ARM_NEON__) || defined(__ARM_NEON)) +#if !defined(ARMV8_OS_MACOS) +/* + * See http://bit.ly/2CcoEsr for run-time detection of ARM features and also + * crbug.com/931275 for android_getCpuFeatures() use in the Android sandbox. + */ +static void _cpu_check_features(void) +{ +#if defined(ARMV8_OS_ANDROID) && defined(__aarch64__) + uint64_t features = android_getCpuFeatures(); + arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM64_FEATURE_CRC32); + arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM64_FEATURE_PMULL); +#elif defined(ARMV8_OS_ANDROID) /* aarch32 */ + uint64_t features = android_getCpuFeatures(); + arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM_FEATURE_CRC32); + arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM_FEATURE_PMULL); +#elif defined(ARMV8_OS_LINUX) && defined(__aarch64__) + unsigned long features = getauxval(AT_HWCAP); + arm_cpu_enable_crc32 = !!(features & HWCAP_CRC32); + arm_cpu_enable_pmull = !!(features & HWCAP_PMULL); +#elif defined(ARMV8_OS_LINUX) && (defined(__ARM_NEON) || defined(__ARM_NEON__)) + /* Query HWCAP2 for ARMV8-A SoCs running in aarch32 mode */ + unsigned long features = getauxval(AT_HWCAP2); + arm_cpu_enable_crc32 = !!(features & HWCAP2_CRC32); + arm_cpu_enable_pmull = !!(features & HWCAP2_PMULL); +#elif defined(ARMV8_OS_FUCHSIA) + uint32_t features; + zx_status_t rc = zx_system_get_features(ZX_FEATURE_KIND_CPU, &features); + if (rc != ZX_OK || (features & ZX_ARM64_FEATURE_ISA_ASIMD) == 0) + return; /* Report nothing if ASIMD(NEON) is missing */ + arm_cpu_enable_crc32 = !!(features & ZX_ARM64_FEATURE_ISA_CRC32); + arm_cpu_enable_pmull = !!(features & ZX_ARM64_FEATURE_ISA_PMULL); +#elif defined(ARMV8_OS_WINDOWS) + arm_cpu_enable_crc32 = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE); + arm_cpu_enable_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); +#elif defined(ARMV8_OS_IOS) + // Determine what features are supported dynamically. This code is applicable to macOS + // as well if we wish to do that dynamically on that platform in the future. + // See https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics + int val = 0; + size_t len = sizeof(val); + arm_cpu_enable_crc32 = sysctlbyname("hw.optional.armv8_crc32", &val, &len, 0, 0) == 0 + && val != 0; + val = 0; + len = sizeof(val); + arm_cpu_enable_pmull = sysctlbyname("hw.optional.arm.FEAT_PMULL", &val, &len, 0, 0) == 0 + && val != 0; +#endif +} +#endif +#elif defined(X86_NOT_WINDOWS) || defined(X86_WINDOWS) +/* + * iOS@x86 (i.e. emulator) is another special case where we disable + * SIMD optimizations. + */ +#ifndef CPU_NO_SIMD +/* On x86 we simply use a instruction to check the CPU features. + * (i.e. CPUID). + */ +#ifdef CRC32_SIMD_AVX512_PCLMUL +#include +#include +#endif +/* _xgetbv() below needs the xsave ISA. Annotate it here so the build does not + * need -mxsave. */ +#if defined(CRC32_SIMD_AVX512_PCLMUL) && (defined(__GNUC__) || defined(__clang__)) +__attribute__((__target__("xsave"))) +#endif +static void _cpu_check_features(void) +{ + int x86_cpu_has_sse2; + int x86_cpu_has_ssse3; + int x86_cpu_has_sse42; + int x86_cpu_has_pclmulqdq; + int abcd[4]; + +#ifdef _MSC_VER + __cpuid(abcd, 1); +#else + __cpuid(1, abcd[0], abcd[1], abcd[2], abcd[3]); +#endif + + x86_cpu_has_sse2 = abcd[3] & 0x4000000; + x86_cpu_has_ssse3 = abcd[2] & 0x000200; + x86_cpu_has_sse42 = abcd[2] & 0x100000; + x86_cpu_has_pclmulqdq = abcd[2] & 0x2; + + x86_cpu_enable_sse2 = x86_cpu_has_sse2; + + x86_cpu_enable_ssse3 = x86_cpu_has_ssse3; + + x86_cpu_enable_simd = x86_cpu_has_sse2 && + x86_cpu_has_sse42 && + x86_cpu_has_pclmulqdq; + +#ifdef CRC32_SIMD_AVX512_PCLMUL + x86_cpu_enable_avx512 = _xgetbv(0) & 0x00000040; +#endif +} +#endif // x86 & NO_SIMD + +#elif defined(RISCV_RVV) +#include + +#ifndef ZLIB_HWCAP_RVV +#define ZLIB_HWCAP_RVV (1 << ('v' - 'a')) +#endif + +/* TODO(cavalcantii) + * - add support for Android@RISCV i.e. __riscv_hwprobe(). + * - detect vclmul (crypto extensions). + */ +static void _cpu_check_features(void) +{ + unsigned long features = getauxval(AT_HWCAP); + riscv_cpu_enable_rvv = !!(features & ZLIB_HWCAP_RVV); +} +#endif // ARM | x86 | RISCV +#endif // NO SIMD CPU diff --git a/packages/runtime/vendor/zlib/cpu_features.h b/packages/runtime/vendor/zlib/cpu_features.h new file mode 100644 index 000000000..6092c7e85 --- /dev/null +++ b/packages/runtime/vendor/zlib/cpu_features.h @@ -0,0 +1,22 @@ +/* cpu_features.h -- Processor features detection. + * + * Copyright 2018 The Chromium Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include "zlib.h" + +/* TODO(cavalcantii): remove checks for x86_flags on deflate. + */ +extern int arm_cpu_enable_crc32; +extern int arm_cpu_enable_pmull; +extern int x86_cpu_enable_sse2; +extern int x86_cpu_enable_ssse3; +extern int x86_cpu_enable_simd; +extern int x86_cpu_enable_avx512; + +extern int riscv_cpu_enable_rvv; +extern int riscv_cpu_enable_vclmul; + +void cpu_check_features(void); diff --git a/packages/runtime/vendor/zlib/crc32.c b/packages/runtime/vendor/zlib/crc32.c index 6c38f5c04..2ec5ab5ce 100644 --- a/packages/runtime/vendor/zlib/crc32.c +++ b/packages/runtime/vendor/zlib/crc32.c @@ -1,5 +1,5 @@ /* crc32.c -- compute the CRC-32 of a data stream - * Copyright (C) 1995-2022 Mark Adler + * Copyright (C) 1995-2026 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h * * This interleaved implementation of a CRC makes use of pipelined multiple @@ -24,11 +24,24 @@ # include # ifndef DYNAMIC_CRC_TABLE # define DYNAMIC_CRC_TABLE -# endif /* !DYNAMIC_CRC_TABLE */ -#endif /* MAKECRCH */ +# endif +#endif +#ifdef DYNAMIC_CRC_TABLE +# define Z_ONCE +#endif +#include "deflate.h" +#include "cpu_features.h" #include "zutil.h" /* for Z_U4, Z_U8, z_crc_t, and FAR definitions */ +#ifdef HAVE_S390X_VX +# include "contrib/crc32vx/crc32_vx_hooks.h" +#endif + +#if defined(CRC32_SIMD_SSE42_PCLMUL) || defined(CRC32_ARMV8_CRC32) +#include "crc32_simd.h" +#endif + /* A CRC of a message is computed on N braids of words in the message, where each word consists of W bytes (4 or 8). If N is 3, for example, then three @@ -99,11 +112,12 @@ #endif /* If available, use the ARM processor CRC32 instruction. */ -#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) && W == 8 -# define ARMCRC32 +#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) && \ + defined(W) && W == 8 && defined(USE_CANONICAL_ARMV8_CRC32) +# define ARMCRC32_CANONICAL_ZLIB #endif -#if defined(W) && (!defined(ARMCRC32) || defined(DYNAMIC_CRC_TABLE)) +#if defined(W) && (!defined(ARMCRC32_CANONICAL_ZLIB) || defined(DYNAMIC_CRC_TABLE)) /* Swap the bytes in a z_word_t to convert between little and big endian. Any self-respecting compiler will optimize this to a single machine byte-swap @@ -152,10 +166,10 @@ local z_word_t byte_swap(z_word_t word) { Return a(x) multiplied by b(x) modulo p(x), where p(x) is the CRC polynomial, reflected. For speed, this requires that a not be zero. */ -local z_crc_t multmodp(z_crc_t a, z_crc_t b) { - z_crc_t m, p; +local uLong multmodp(uLong a, uLong b) { + uLong m, p; - m = (z_crc_t)1 << 31; + m = (uLong)1 << 31; p = 0; for (;;) { if (a & m) { @@ -171,12 +185,12 @@ local z_crc_t multmodp(z_crc_t a, z_crc_t b) { /* Return x^(n * 2^k) modulo p(x). Requires that x2n_table[] has been - initialized. + initialized. n must not be negative. */ -local z_crc_t x2nmodp(z_off64_t n, unsigned k) { - z_crc_t p; +local uLong x2nmodp(z_off64_t n, unsigned k) { + uLong p; - p = (z_crc_t)1 << 31; /* x^0 == 1 */ + p = (uLong)1 << 31; /* x^0 == 1 */ while (n) { if (n & 1) p = multmodp(x2n_table[k & 31], p); @@ -204,83 +218,8 @@ local z_crc_t FAR crc_table[256]; local void write_table64(FILE *, const z_word_t FAR *, int); #endif /* MAKECRCH */ -/* - Define a once() function depending on the availability of atomics. If this is - compiled with DYNAMIC_CRC_TABLE defined, and if CRCs will be computed in - multiple threads, and if atomics are not available, then get_crc_table() must - be called to initialize the tables and must return before any threads are - allowed to compute or combine CRCs. - */ - -/* Definition of once functionality. */ -typedef struct once_s once_t; - -/* Check for the availability of atomics. */ -#if defined(__STDC__) && __STDC_VERSION__ >= 201112L && \ - !defined(__STDC_NO_ATOMICS__) - -#include - -/* Structure for once(), which must be initialized with ONCE_INIT. */ -struct once_s { - atomic_flag begun; - atomic_int done; -}; -#define ONCE_INIT {ATOMIC_FLAG_INIT, 0} - -/* - Run the provided init() function exactly once, even if multiple threads - invoke once() at the same time. The state must be a once_t initialized with - ONCE_INIT. - */ -local void once(once_t *state, void (*init)(void)) { - if (!atomic_load(&state->done)) { - if (atomic_flag_test_and_set(&state->begun)) - while (!atomic_load(&state->done)) - ; - else { - init(); - atomic_store(&state->done, 1); - } - } -} - -#else /* no atomics */ - -/* Structure for once(), which must be initialized with ONCE_INIT. */ -struct once_s { - volatile int begun; - volatile int done; -}; -#define ONCE_INIT {0, 0} - -/* Test and set. Alas, not atomic, but tries to minimize the period of - vulnerability. */ -local int test_and_set(int volatile *flag) { - int was; - - was = *flag; - *flag = 1; - return was; -} - -/* Run the provided init() function once. This is not thread-safe. */ -local void once(once_t *state, void (*init)(void)) { - if (!state->done) { - if (test_and_set(&state->begun)) - while (!state->done) - ; - else { - init(); - state->done = 1; - } - } -} - -#endif - /* State for once(). */ -local once_t made = ONCE_INIT; +local z_once_t made = Z_ONCE_INIT; /* Generate tables for a byte-wise 32-bit CRC calculation on the polynomial: @@ -306,8 +245,8 @@ local once_t made = ONCE_INIT; information needed to generate CRCs on data a byte at a time for all combinations of CRC register values and incoming bytes. */ - -local void make_crc_table(void) { +local void make_crc_table(void) +{ unsigned i, j, n; z_crc_t p; @@ -326,7 +265,7 @@ local void make_crc_table(void) { p = (z_crc_t)1 << 30; /* x^1 */ x2n_table[0] = p; for (n = 1; n < 32; n++) - x2n_table[n] = p = multmodp(p, p); + x2n_table[n] = p = (z_crc_t)multmodp(p, p); #ifdef W /* initialize the braiding tables -- needs x2n_table[] */ @@ -529,11 +468,11 @@ local void braid(z_crc_t ltl[][256], z_word_t big[][256], int n, int w) { int k; z_crc_t i, p, q; for (k = 0; k < w; k++) { - p = x2nmodp((n * w + 3 - k) << 3, 0); + p = (z_crc_t)x2nmodp((n * w + 3 - k) << 3, 0); ltl[k][0] = 0; big[w - 1 - k][0] = 0; for (i = 1; i < 256; i++) { - ltl[k][i] = q = multmodp(i << 24, p); + ltl[k][i] = q = (z_crc_t)multmodp(i << 24, p); big[w - 1 - k][i] = byte_swap(q); } } @@ -548,7 +487,7 @@ local void braid(z_crc_t ltl[][256], z_word_t big[][256], int n, int w) { */ const z_crc_t FAR * ZEXPORT get_crc_table(void) { #ifdef DYNAMIC_CRC_TABLE - once(&made, make_crc_table); + z_once(&made, make_crc_table); #endif /* DYNAMIC_CRC_TABLE */ return (const z_crc_t FAR *)crc_table; } @@ -562,7 +501,7 @@ const z_crc_t FAR * ZEXPORT get_crc_table(void) { * -march=armv8-a+crc, or -march=native if the compile machine has the crc32 * instructions. */ -#ifdef ARMCRC32 +#if ARMCRC32_CANONICAL_ZLIB /* Constants empirically determined to maximize speed. These values are from @@ -572,9 +511,8 @@ const z_crc_t FAR * ZEXPORT get_crc_table(void) { #define Z_BATCH_ZEROS 0xa10d3d0c /* computed from Z_BATCH = 3990 */ #define Z_BATCH_MIN 800 /* fewest words in a final batch */ -unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, - z_size_t len) { - z_crc_t val; +uLong ZEXPORT crc32_z(uLong crc, const unsigned char FAR *buf, z_size_t len) { + uLong val; z_word_t crc1, crc2; const z_word_t *word; z_word_t val0, val1, val2; @@ -585,7 +523,7 @@ unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, if (buf == Z_NULL) return 0; #ifdef DYNAMIC_CRC_TABLE - once(&made, make_crc_table); + z_once(&made, make_crc_table); #endif /* DYNAMIC_CRC_TABLE */ /* Pre-condition the CRC */ @@ -640,7 +578,7 @@ unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, } word += 3 * last; num -= 3 * last; - val = x2nmodp(last, 6); + val = x2nmodp((int)last, 6); crc = multmodp(val, crc) ^ crc1; crc = multmodp(val, crc) ^ crc2; } @@ -691,15 +629,79 @@ local z_word_t crc_word_big(z_word_t data) { #endif /* ========================================================================= */ -unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, - z_size_t len) { - /* Return initial CRC, if requested. */ - if (buf == Z_NULL) return 0; +uLong ZEXPORT crc32_z(uLong crc, const unsigned char FAR *buf, z_size_t len) { + + /* If no optimizations are enabled, do it as canonical zlib. */ +#if !defined(CRC32_SIMD_SSE42_PCLMUL) && !defined(CRC32_ARMV8_CRC32) && \ + !defined(RISCV_RVV) && !defined(CRC32_SIMD_AVX512_PCLMUL) + if (buf == Z_NULL) { + return 0UL; + } +#else + /* + * zlib convention is to call crc32(0, NULL, 0); before making + * calls to crc32(). So this is a good, early (and infrequent) + * place to cache CPU features if needed for those later, more + * interesting crc32() calls. + */ + if (buf == Z_NULL) { + if (!len) + cpu_check_features(); + return 0UL; + } +#endif + /* If AVX-512 is enabled, we will use it for longer inputs and fallback + * to SSE4.2 and eventually the portable implementation to handle the tail. + */ +#if defined(CRC32_SIMD_AVX512_PCLMUL) + if (x86_cpu_enable_avx512 && len >= Z_CRC32_AVX512_MINIMUM_LENGTH) { + /* crc32 64-byte chunks */ + z_size_t chunk_size = len & ~Z_CRC32_AVX512_CHUNKSIZE_MASK; + crc = ~crc32_avx512_simd_(buf, chunk_size, ~(uint32_t)crc); + /* check remaining data */ + len -= chunk_size; + if (!len) + return crc; + /* Fall into the default crc32 for the remaining data. */ + buf += chunk_size; + } +#endif +#if defined(CRC32_SIMD_SSE42_PCLMUL) + if (x86_cpu_enable_simd && len >= Z_CRC32_SSE42_MINIMUM_LENGTH) { + /* crc32 16-byte chunks */ + z_size_t chunk_size = len & ~Z_CRC32_SSE42_CHUNKSIZE_MASK; + crc = ~crc32_sse42_simd_(buf, chunk_size, ~(uint32_t)crc); + /* check remaining data */ + len -= chunk_size; + if (!len) + return crc; + /* Fall into the default crc32 for the remaining data. */ + buf += chunk_size; + } +#elif defined(CRC32_ARMV8_CRC32) + if (arm_cpu_enable_crc32) { +#if defined(__aarch64__) + /* PMULL is 64bit only, plus code needs at least a 64 bytes buffer. */ + if (arm_cpu_enable_pmull && (len > Z_CRC32_PMULL_MINIMUM_LENGTH)) { + const size_t chunk_size = len & ~Z_CRC32_PMULL_CHUNKSIZE_MASK; + crc = ~armv8_crc32_pmull_little(buf, chunk_size, ~(uint32_t)crc); + /* Check remaining data. */ + len -= chunk_size; + if (!len) + return crc; + + /* Fall through for the remaining data. */ + buf += chunk_size; + } +#endif + /* This is scalar and self contained, used on Armv8@32bit or tail. */ + return armv8_crc32_little(buf, len, crc); + } +#endif /* CRC32_SIMD */ #ifdef DYNAMIC_CRC_TABLE - once(&made, make_crc_table); + z_once(&made, make_crc_table); #endif /* DYNAMIC_CRC_TABLE */ - /* Pre-condition the CRC */ crc = (~crc) & 0xffffffff; @@ -1012,28 +1014,56 @@ unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, #endif /* ========================================================================= */ -unsigned long ZEXPORT crc32(unsigned long crc, const unsigned char FAR *buf, - uInt len) { - return crc32_z(crc, buf, len); -} +uLong ZEXPORT crc32(uLong crc, const unsigned char FAR *buf, uInt len) { + #ifdef HAVE_S390X_VX + return crc32_z_hook(crc, buf, len); + #endif -/* ========================================================================= */ -uLong ZEXPORT crc32_combine64(uLong crc1, uLong crc2, z_off64_t len2) { -#ifdef DYNAMIC_CRC_TABLE - once(&made, make_crc_table); -#endif /* DYNAMIC_CRC_TABLE */ - return multmodp(x2nmodp(len2, 3), crc1) ^ (crc2 & 0xffffffff); -} + /* Some bots compile with optimizations disabled, others will emulate + * ARM on x86 and other weird combinations. + */ +#if defined(CRC32_SIMD_SSE42_PCLMUL) || defined(CRC32_ARMV8_CRC32) \ + || defined(RISCV_RVV) + /* We got to verify CPU features, so exploit the common usage pattern + * of calling this function with Z_NULL for an initial valid crc value. + * This allows to cache the result of the feature check and avoid extraneous + * function calls. + */ + if (buf == Z_NULL) { + if (!len) /* Assume user is calling crc32(0, NULL, 0); */ + cpu_check_features(); + return 0UL; + } +#endif -/* ========================================================================= */ -uLong ZEXPORT crc32_combine(uLong crc1, uLong crc2, z_off_t len2) { - return crc32_combine64(crc1, crc2, (z_off64_t)len2); +#if defined(CRC32_ARMV8_CRC32) + if (arm_cpu_enable_crc32) { +#if defined(__aarch64__) + /* PMULL is 64bit only, plus code needs at least a 64 bytes buffer. */ + if (arm_cpu_enable_pmull && (len > Z_CRC32_PMULL_MINIMUM_LENGTH)) { + const size_t chunk_size = len & ~Z_CRC32_PMULL_CHUNKSIZE_MASK; + crc = ~armv8_crc32_pmull_little(buf, chunk_size, ~(uint32_t)crc); + /* Check remaining data. */ + len -= chunk_size; + if (!len) + return crc; + + /* Fall through for the remaining data. */ + buf += chunk_size; + } +#endif + return armv8_crc32_little(buf, len, crc); /* Armv8@32bit or tail. */ + } +#endif + return crc32_z(crc, buf, len); /* Armv7 or Armv8 w/o crypto extensions. */ } /* ========================================================================= */ uLong ZEXPORT crc32_combine_gen64(z_off64_t len2) { + if (len2 < 0) + return 0; #ifdef DYNAMIC_CRC_TABLE - once(&made, make_crc_table); + z_once(&made, make_crc_table); #endif /* DYNAMIC_CRC_TABLE */ return x2nmodp(len2, 3); } @@ -1042,8 +1072,55 @@ uLong ZEXPORT crc32_combine_gen64(z_off64_t len2) { uLong ZEXPORT crc32_combine_gen(z_off_t len2) { return crc32_combine_gen64((z_off64_t)len2); } - /* ========================================================================= */ uLong ZEXPORT crc32_combine_op(uLong crc1, uLong crc2, uLong op) { - return multmodp(op, crc1) ^ (crc2 & 0xffffffff); + if (op == 0) + return 0; + return multmodp(op, crc1 & 0xffffffff) ^ (crc2 & 0xffffffff); +} + +/* ========================================================================= */ +uLong ZEXPORT crc32_combine64(uLong crc1, uLong crc2, z_off64_t len2) { + return crc32_combine_op(crc1, crc2, crc32_combine_gen64(len2)); +} + +/* ========================================================================= */ +uLong ZEXPORT crc32_combine(uLong crc1, uLong crc2, z_off_t len2) { + return crc32_combine64(crc1, crc2, (z_off64_t)len2); +} + +ZLIB_INTERNAL void crc_reset(deflate_state *const s) +{ +#ifdef CRC32_SIMD_SSE42_PCLMUL + if (x86_cpu_enable_simd) { + crc_fold_init(s); + return; + } +#endif + s->strm->adler = crc32(0L, Z_NULL, 0); +} + +ZLIB_INTERNAL void crc_finalize(deflate_state *const s) +{ +#ifdef QAT_COMPRESSION_ENABLED + if (s->qat_s) { + return; + } +#endif +#ifdef CRC32_SIMD_SSE42_PCLMUL + if (x86_cpu_enable_simd) + s->strm->adler = crc_fold_512to32(s); +#endif +} + +ZLIB_INTERNAL void copy_with_crc(z_streamp strm, Bytef *dst, long size) +{ +#ifdef CRC32_SIMD_SSE42_PCLMUL + if (x86_cpu_enable_simd) { + crc_fold_copy(strm->state, dst, strm->next_in, size); + return; + } +#endif + zmemcpy(dst, strm->next_in, size); + strm->adler = crc32(strm->adler, dst, size); } diff --git a/packages/runtime/vendor/zlib/crc32_simd.c b/packages/runtime/vendor/zlib/crc32_simd.c new file mode 100644 index 000000000..f9d52a631 --- /dev/null +++ b/packages/runtime/vendor/zlib/crc32_simd.c @@ -0,0 +1,628 @@ +/* crc32_simd.c + * + * Copyright 2017 The Chromium Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include "crc32_simd.h" +#if defined(CRC32_SIMD_AVX512_PCLMUL) + +/* + * crc32_avx512_simd_(): compute the crc32 of the buffer, where the buffer + * length must be at least 256, and a multiple of 64. Based on: + * + * "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" + * V. Gopal, E. Ozturk, et al., 2009, http://intel.ly/2ySEwL0 + */ + +#include +#include +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +__attribute__((__target__("avx512f,pclmul,vpclmulqdq"))) +#endif +uint32_t ZLIB_INTERNAL crc32_avx512_simd_( /* AVX512+PCLMUL */ + const unsigned char *buf, + z_size_t len, + uint32_t crc) +{ + /* + * Definitions of the bit-reflected domain constants k1,k2,k3,k4 + * are similar to those given at the end of the paper, and remaining + * constants and CRC32+Barrett polynomials remain unchanged. + * + * Replace the index of x from 128 to 512. As follows: + * k1 = ( x ^ ( 512 * 4 + 32 ) mod P(x) << 32 )' << 1 = 0x011542778a + * k2 = ( x ^ ( 512 * 4 - 32 ) mod P(x) << 32 )' << 1 = 0x01322d1430 + * k3 = ( x ^ ( 512 + 32 ) mod P(x) << 32 )' << 1 = 0x0154442bd4 + * k4 = ( x ^ ( 512 - 32 ) mod P(x) << 32 )' << 1 = 0x01c6e41596 + */ + static const uint64_t zalign(64) k1k2[] = { 0x011542778a, 0x01322d1430, + 0x011542778a, 0x01322d1430, + 0x011542778a, 0x01322d1430, + 0x011542778a, 0x01322d1430 }; + static const uint64_t zalign(64) k3k4[] = { 0x0154442bd4, 0x01c6e41596, + 0x0154442bd4, 0x01c6e41596, + 0x0154442bd4, 0x01c6e41596, + 0x0154442bd4, 0x01c6e41596 }; + static const uint64_t zalign(16) k5k6[] = { 0x01751997d0, 0x00ccaa009e }; + static const uint64_t zalign(16) k7k8[] = { 0x0163cd6124, 0x0000000000 }; + static const uint64_t zalign(16) poly[] = { 0x01db710641, 0x01f7011641 }; + __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; + __m128i a0, a1, a2, a3; + + /* + * There's at least one block of 256. + */ + x1 = _mm512_loadu_si512((__m512i *)(buf + 0x00)); + x2 = _mm512_loadu_si512((__m512i *)(buf + 0x40)); + x3 = _mm512_loadu_si512((__m512i *)(buf + 0x80)); + x4 = _mm512_loadu_si512((__m512i *)(buf + 0xC0)); + + x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc))); + + x0 = _mm512_load_si512((__m512i *)k1k2); + + buf += 256; + len -= 256; + + /* + * Parallel fold blocks of 256, if any. + */ + while (len >= 256) + { + x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); + x6 = _mm512_clmulepi64_epi128(x2, x0, 0x00); + x7 = _mm512_clmulepi64_epi128(x3, x0, 0x00); + x8 = _mm512_clmulepi64_epi128(x4, x0, 0x00); + + + x1 = _mm512_clmulepi64_epi128(x1, x0, 0x11); + x2 = _mm512_clmulepi64_epi128(x2, x0, 0x11); + x3 = _mm512_clmulepi64_epi128(x3, x0, 0x11); + x4 = _mm512_clmulepi64_epi128(x4, x0, 0x11); + + y5 = _mm512_loadu_si512((__m512i *)(buf + 0x00)); + y6 = _mm512_loadu_si512((__m512i *)(buf + 0x40)); + y7 = _mm512_loadu_si512((__m512i *)(buf + 0x80)); + y8 = _mm512_loadu_si512((__m512i *)(buf + 0xC0)); + + x1 = _mm512_xor_si512(x1, x5); + x2 = _mm512_xor_si512(x2, x6); + x3 = _mm512_xor_si512(x3, x7); + x4 = _mm512_xor_si512(x4, x8); + + x1 = _mm512_xor_si512(x1, y5); + x2 = _mm512_xor_si512(x2, y6); + x3 = _mm512_xor_si512(x3, y7); + x4 = _mm512_xor_si512(x4, y8); + + buf += 256; + len -= 256; + } + + /* + * Fold into 512-bits. + */ + x0 = _mm512_load_si512((__m512i *)k3k4); + + x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); + x1 = _mm512_clmulepi64_epi128(x1, x0, 0x11); + x1 = _mm512_xor_si512(x1, x2); + x1 = _mm512_xor_si512(x1, x5); + + x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); + x1 = _mm512_clmulepi64_epi128(x1, x0, 0x11); + x1 = _mm512_xor_si512(x1, x3); + x1 = _mm512_xor_si512(x1, x5); + + x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); + x1 = _mm512_clmulepi64_epi128(x1, x0, 0x11); + x1 = _mm512_xor_si512(x1, x4); + x1 = _mm512_xor_si512(x1, x5); + + /* + * Single fold blocks of 64, if any. + */ + while (len >= 64) + { + x2 = _mm512_loadu_si512((__m512i *)buf); + + x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); + x1 = _mm512_clmulepi64_epi128(x1, x0, 0x11); + x1 = _mm512_xor_si512(x1, x2); + x1 = _mm512_xor_si512(x1, x5); + + buf += 64; + len -= 64; + } + + /* + * Fold 512-bits to 384-bits. + */ + a0 = _mm_load_si128((__m128i *)k5k6); + + a1 = _mm512_extracti32x4_epi32(x1, 0); + a2 = _mm512_extracti32x4_epi32(x1, 1); + + a3 = _mm_clmulepi64_si128(a1, a0, 0x00); + a1 = _mm_clmulepi64_si128(a1, a0, 0x11); + + a1 = _mm_xor_si128(a1, a3); + a1 = _mm_xor_si128(a1, a2); + + /* + * Fold 384-bits to 256-bits. + */ + a2 = _mm512_extracti32x4_epi32(x1, 2); + a3 = _mm_clmulepi64_si128(a1, a0, 0x00); + a1 = _mm_clmulepi64_si128(a1, a0, 0x11); + a1 = _mm_xor_si128(a1, a3); + a1 = _mm_xor_si128(a1, a2); + + /* + * Fold 256-bits to 128-bits. + */ + a2 = _mm512_extracti32x4_epi32(x1, 3); + a3 = _mm_clmulepi64_si128(a1, a0, 0x00); + a1 = _mm_clmulepi64_si128(a1, a0, 0x11); + a1 = _mm_xor_si128(a1, a3); + a1 = _mm_xor_si128(a1, a2); + + /* + * Fold 128-bits to 64-bits. + */ + a2 = _mm_clmulepi64_si128(a1, a0, 0x10); + a3 = _mm_setr_epi32(~0, 0, ~0, 0); + a1 = _mm_srli_si128(a1, 8); + a1 = _mm_xor_si128(a1, a2); + + a0 = _mm_loadl_epi64((__m128i*)k7k8); + a2 = _mm_srli_si128(a1, 4); + a1 = _mm_and_si128(a1, a3); + a1 = _mm_clmulepi64_si128(a1, a0, 0x00); + a1 = _mm_xor_si128(a1, a2); + + /* + * Barret reduce to 32-bits. + */ + a0 = _mm_load_si128((__m128i*)poly); + + a2 = _mm_and_si128(a1, a3); + a2 = _mm_clmulepi64_si128(a2, a0, 0x10); + a2 = _mm_and_si128(a2, a3); + a2 = _mm_clmulepi64_si128(a2, a0, 0x00); + a1 = _mm_xor_si128(a1, a2); + + /* + * Return the crc32. + */ + return _mm_extract_epi32(a1, 1); +} + +#endif +#if defined(CRC32_SIMD_SSE42_PCLMUL) + +/* + * crc32_sse42_simd_(): compute the crc32 of the buffer, where the buffer + * length must be at least 64, and a multiple of 16. + */ + +#include +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +__attribute__((__target__("sse4.2,pclmul"))) +#endif +uint32_t ZLIB_INTERNAL crc32_sse42_simd_( /* SSE4.2+PCLMUL */ + const unsigned char *buf, + z_size_t len, + uint32_t crc) +{ + /* + * Definitions of the bit-reflected domain constants k1,k2,k3, etc and + * the CRC32+Barrett polynomials given at the end of the paper. + */ + static const uint64_t zalign(16) k1k2[] = { 0x0154442bd4, 0x01c6e41596 }; + static const uint64_t zalign(16) k3k4[] = { 0x01751997d0, 0x00ccaa009e }; + static const uint64_t zalign(16) k5k0[] = { 0x0163cd6124, 0x0000000000 }; + static const uint64_t zalign(16) poly[] = { 0x01db710641, 0x01f7011641 }; + + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; + + /* + * There's at least one block of 64. + */ + x1 = _mm_loadu_si128((__m128i *)(buf + 0x00)); + x2 = _mm_loadu_si128((__m128i *)(buf + 0x10)); + x3 = _mm_loadu_si128((__m128i *)(buf + 0x20)); + x4 = _mm_loadu_si128((__m128i *)(buf + 0x30)); + + x1 = _mm_xor_si128(x1, _mm_cvtsi32_si128(crc)); + + x0 = _mm_load_si128((__m128i *)k1k2); + + buf += 64; + len -= 64; + + /* + * Parallel fold blocks of 64, if any. + */ + while (len >= 64) + { + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x6 = _mm_clmulepi64_si128(x2, x0, 0x00); + x7 = _mm_clmulepi64_si128(x3, x0, 0x00); + x8 = _mm_clmulepi64_si128(x4, x0, 0x00); + + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x2 = _mm_clmulepi64_si128(x2, x0, 0x11); + x3 = _mm_clmulepi64_si128(x3, x0, 0x11); + x4 = _mm_clmulepi64_si128(x4, x0, 0x11); + + y5 = _mm_loadu_si128((__m128i *)(buf + 0x00)); + y6 = _mm_loadu_si128((__m128i *)(buf + 0x10)); + y7 = _mm_loadu_si128((__m128i *)(buf + 0x20)); + y8 = _mm_loadu_si128((__m128i *)(buf + 0x30)); + + x1 = _mm_xor_si128(x1, x5); + x2 = _mm_xor_si128(x2, x6); + x3 = _mm_xor_si128(x3, x7); + x4 = _mm_xor_si128(x4, x8); + + x1 = _mm_xor_si128(x1, y5); + x2 = _mm_xor_si128(x2, y6); + x3 = _mm_xor_si128(x3, y7); + x4 = _mm_xor_si128(x4, y8); + + buf += 64; + len -= 64; + } + + /* + * Fold into 128-bits. + */ + x0 = _mm_load_si128((__m128i *)k3k4); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x5); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x5); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x4); + x1 = _mm_xor_si128(x1, x5); + + /* + * Single fold blocks of 16, if any. + */ + while (len >= 16) + { + x2 = _mm_loadu_si128((__m128i *)buf); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x5); + + buf += 16; + len -= 16; + } + + /* + * Fold 128-bits to 64-bits. + */ + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x3 = _mm_setr_epi32(~0, 0, ~0, 0); + x1 = _mm_srli_si128(x1, 8); + x1 = _mm_xor_si128(x1, x2); + + x0 = _mm_loadl_epi64((__m128i*)k5k0); + + x2 = _mm_srli_si128(x1, 4); + x1 = _mm_and_si128(x1, x3); + x1 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_xor_si128(x1, x2); + + /* + * Barret reduce to 32-bits. + */ + x0 = _mm_load_si128((__m128i*)poly); + + x2 = _mm_and_si128(x1, x3); + x2 = _mm_clmulepi64_si128(x2, x0, 0x10); + x2 = _mm_and_si128(x2, x3); + x2 = _mm_clmulepi64_si128(x2, x0, 0x00); + x1 = _mm_xor_si128(x1, x2); + + /* + * Return the crc32. + */ + return _mm_extract_epi32(x1, 1); +} + +#elif defined(CRC32_ARMV8_CRC32) + +/* CRC32 checksums using ARMv8-a crypto instructions. + */ + +#if defined(__clang__) +/* We need some extra types for using PMULL. + */ +#if defined(__aarch64__) +#include +#include +#endif + +/* CRC32 intrinsics are #ifdef'ed out of arm_acle.h unless we build with an + * armv8 target, which is incompatible with ThinLTO optimizations on Android. + * (Namely, mixing and matching different module-level targets makes ThinLTO + * warn, and Android defaults to armv7-a. This restriction does not apply to + * function-level `target`s, however.) + * + * Since we only need four crc intrinsics, and since clang's implementation of + * those are just wrappers around compiler builtins, it's simplest to #define + * those builtins directly. If this #define list grows too much (or we depend on + * an intrinsic that isn't a trivial wrapper), we may have to find a better way + * to go about this. + * + * NOTE: clang currently complains that "'+soft-float-abi' is not a recognized + * feature for this target (ignoring feature)." This appears to be a harmless + * bug in clang. + * + * These definitions must appear *after* including arm_acle.h otherwise that + * header may end up defining functions named __builtin_arm_crc32* that call + * themselves, creating an infinite loop when the intrinsic is called. + */ +/* XXX: Cannot hook into builtins with XCode for arm64. */ +#if !defined(ARMV8_OS_MACOS) +#define __crc32b __builtin_arm_crc32b +#define __crc32d __builtin_arm_crc32d +#define __crc32w __builtin_arm_crc32w +#define __crc32cw __builtin_arm_crc32cw +#endif + +#if defined(__aarch64__) +#define TARGET_ARMV8_WITH_CRC __attribute__((target("arch=armv8-a+aes+crc"))) +#else // !defined(__aarch64__) +#define TARGET_ARMV8_WITH_CRC __attribute__((target("crc"))) +#endif // defined(__aarch64__) + +#elif defined(__GNUC__) +/* For GCC, we are setting CRC extensions at module level, so ThinLTO is not + * allowed. We can just include arm_acle.h. + */ +#include +#include +#define TARGET_ARMV8_WITH_CRC __attribute__((target("arch=armv8-a+crc+crypto"))) +#else // !defined(__GNUC__) && !defined(_aarch64__) +#error ARM CRC32 SIMD extensions only supported for Clang and GCC +#endif + +TARGET_ARMV8_WITH_CRC +uint32_t ZLIB_INTERNAL armv8_crc32_little( + const unsigned char *buf, + z_size_t len, + uint32_t crc) +{ + uint32_t c = (uint32_t) ~crc; + + while (len && ((uintptr_t)buf & 7)) { + c = __crc32b(c, *buf++); + --len; + } + + const uint64_t *buf8 = (const uint64_t *)buf; + + while (len >= 64) { + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + len -= 64; + } + + while (len >= 8) { + c = __crc32d(c, *buf8++); + len -= 8; + } + + buf = (const unsigned char *)buf8; + + while (len--) { + c = __crc32b(c, *buf++); + } + + return ~c; +} + +#if defined(__aarch64__) || defined(ARMV8_OS_MACOS) /* aarch64 specific code. */ + +/* + * crc32_pmull_simd_(): compute the crc32 of the buffer, where the buffer + * length must be at least 64, and a multiple of 16. Based on: + * + * "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" + * V. Gopal, E. Ozturk, et al., 2009, http://intel.ly/2ySEwL0 + */ +TARGET_ARMV8_WITH_CRC +static inline uint8x16_t pmull_lo(const uint64x2_t a, const uint64x2_t b) +{ + uint8x16_t r; + __asm__ __volatile__ ("pmull %0.1q, %1.1d, %2.1d \n\t" + : "=w" (r) : "w" (a), "w" (b) ); + return r; +} + +TARGET_ARMV8_WITH_CRC +static inline uint8x16_t pmull_01(const uint64x2_t a, const uint64x2_t b) +{ + uint8x16_t r; + __asm__ __volatile__ ("pmull %0.1q, %1.1d, %2.1d \n\t" + : "=w" (r) : "w" (a), "w" (vgetq_lane_u64(b, 1)) ); + return r; +} + +TARGET_ARMV8_WITH_CRC +static inline uint8x16_t pmull_hi(const uint64x2_t a, const uint64x2_t b) +{ + uint8x16_t r; + __asm__ __volatile__ ("pmull2 %0.1q, %1.2d, %2.2d \n\t" + : "=w" (r) : "w" (a), "w" (b) ); + return r; +} + +TARGET_ARMV8_WITH_CRC +uint32_t ZLIB_INTERNAL armv8_crc32_pmull_little( + const unsigned char *buf, + z_size_t len, + uint32_t crc) +{ + /* + * Definitions of the bit-reflected domain constants k1,k2,k3, etc and + * the CRC32+Barrett polynomials given at the end of the paper. + */ + static const uint64_t zalign(16) k1k2[] = { 0x0154442bd4, 0x01c6e41596 }; + static const uint64_t zalign(16) k3k4[] = { 0x01751997d0, 0x00ccaa009e }; + static const uint64_t zalign(16) k5k0[] = { 0x0163cd6124, 0x0000000000 }; + static const uint64_t zalign(16) poly[] = { 0x01db710641, 0x01f7011641 }; + + uint64x2_t x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; + + /* + * There's at least one block of 64. + */ + x1 = vld1q_u64((const uint64_t *)(buf + 0x00)); + x2 = vld1q_u64((const uint64_t *)(buf + 0x10)); + x3 = vld1q_u64((const uint64_t *)(buf + 0x20)); + x4 = vld1q_u64((const uint64_t *)(buf + 0x30)); + + x1 = veorq_u64(x1, (uint64x2_t) vsetq_lane_u32(crc, vdupq_n_u32(0), 0)); + + x0 = vld1q_u64(k1k2); + + buf += 64; + len -= 64; + + /* + * Parallel fold blocks of 64, if any. + */ + while (len >= 64) + { + x5 = (uint64x2_t) pmull_lo(x1, x0); + x6 = (uint64x2_t) pmull_lo(x2, x0); + x7 = (uint64x2_t) pmull_lo(x3, x0); + x8 = (uint64x2_t) pmull_lo(x4, x0); + + y5 = vld1q_u64((const uint64_t *)(buf + 0x00)); + y6 = vld1q_u64((const uint64_t *)(buf + 0x10)); + y7 = vld1q_u64((const uint64_t *)(buf + 0x20)); + y8 = vld1q_u64((const uint64_t *)(buf + 0x30)); + + x1 = (uint64x2_t) pmull_hi(x1, x0); + x2 = (uint64x2_t) pmull_hi(x2, x0); + x3 = (uint64x2_t) pmull_hi(x3, x0); + x4 = (uint64x2_t) pmull_hi(x4, x0); + + x1 = veorq_u64(x1, x5); + x2 = veorq_u64(x2, x6); + x3 = veorq_u64(x3, x7); + x4 = veorq_u64(x4, x8); + + x1 = veorq_u64(x1, y5); + x2 = veorq_u64(x2, y6); + x3 = veorq_u64(x3, y7); + x4 = veorq_u64(x4, y8); + + buf += 64; + len -= 64; + } + + /* + * Fold into 128-bits. + */ + x0 = vld1q_u64(k3k4); + + x5 = (uint64x2_t) pmull_lo(x1, x0); + x1 = (uint64x2_t) pmull_hi(x1, x0); + x1 = veorq_u64(x1, x2); + x1 = veorq_u64(x1, x5); + + x5 = (uint64x2_t) pmull_lo(x1, x0); + x1 = (uint64x2_t) pmull_hi(x1, x0); + x1 = veorq_u64(x1, x3); + x1 = veorq_u64(x1, x5); + + x5 = (uint64x2_t) pmull_lo(x1, x0); + x1 = (uint64x2_t) pmull_hi(x1, x0); + x1 = veorq_u64(x1, x4); + x1 = veorq_u64(x1, x5); + + /* + * Single fold blocks of 16, if any. + */ + while (len >= 16) + { + x2 = vld1q_u64((const uint64_t *)buf); + + x5 = (uint64x2_t) pmull_lo(x1, x0); + x1 = (uint64x2_t) pmull_hi(x1, x0); + x1 = veorq_u64(x1, x2); + x1 = veorq_u64(x1, x5); + + buf += 16; + len -= 16; + } + + /* + * Fold 128-bits to 64-bits. + */ + static uint32_t zalign(16) mask[] = { ~0u, 0u, ~0u, 0u }; + + x2 = (uint64x2_t) pmull_01(x1, x0); + x1 = (uint64x2_t) vextq_u8(vreinterpretq_u8_u64(x1), vdupq_n_u8(0), 8); + x3 = (uint64x2_t) vld1q_u32(mask); + x1 = veorq_u64(x1, x2); + + x0 = vld1q_u64(k5k0); + + x2 = (uint64x2_t) pmull_01(x2, x0); + x2 = (uint64x2_t) vextq_u8(vreinterpretq_u8_u64(x1), vdupq_n_u8(0), 4); + x1 = vandq_u64(x1, x3); + x1 = (uint64x2_t) pmull_lo(x1, x0); + x1 = veorq_u64(x1, x2); + + /* + * Barret reduce to 32-bits. + */ + x0 = vld1q_u64(poly); + + x2 = vandq_u64(x1, x3); + x2 = (uint64x2_t) pmull_01(x2, x0); + x2 = vandq_u64(x2, x3); + x2 = (uint64x2_t) pmull_lo(x2, x0); + x1 = veorq_u64(x1, x2); + + /* + * Return the crc32. + */ + return vgetq_lane_u32(vreinterpretq_u32_u64(x1), 1); +} +#endif /* aarch64 specific code. */ + +#endif diff --git a/packages/runtime/vendor/zlib/crc32_simd.h b/packages/runtime/vendor/zlib/crc32_simd.h new file mode 100644 index 000000000..84624647b --- /dev/null +++ b/packages/runtime/vendor/zlib/crc32_simd.h @@ -0,0 +1,57 @@ +/* crc32_simd.h + * + * Copyright 2017 The Chromium Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include + +#include "zconf.h" +#include "zutil.h" +#include "deflate.h" + +/* + * crc32_sse42_simd_(): compute the crc32 of the buffer, where the buffer + * length must be at least 64, and a multiple of 16. + */ +uint32_t ZLIB_INTERNAL crc32_sse42_simd_(const unsigned char* buf, + z_size_t len, + uint32_t crc); + +uint32_t ZLIB_INTERNAL crc32_avx512_simd_(const unsigned char* buf, + z_size_t len, + uint32_t crc); + +/* + * crc32_sse42_simd_ buffer size constraints: see the use in zlib/crc32.c + * for computing the crc32 of an arbitrary length buffer. + */ +#define Z_CRC32_SSE42_MINIMUM_LENGTH 64 +#define Z_CRC32_SSE42_CHUNKSIZE_MASK 15 +#define Z_CRC32_AVX512_MINIMUM_LENGTH 256 +#define Z_CRC32_AVX512_CHUNKSIZE_MASK 63 + +/* + * CRC32 checksums using ARMv8-a crypto instructions. + */ +uint32_t ZLIB_INTERNAL armv8_crc32_little(const unsigned char* buf, + z_size_t len, + uint32_t crc); + +/* aarch64 specific code. */ +#if defined(__aarch64__) + +/* 128 is the sweet spot at the time of coding (late 2020). */ +#define Z_CRC32_PMULL_MINIMUM_LENGTH 128 +#define Z_CRC32_PMULL_CHUNKSIZE_MASK 15 + +/* + * CRC32 checksums using ARMv8-a PMULL instructions, where the buffer + * length must be at least 64, and a multiple of 16. + */ +uint32_t ZLIB_INTERNAL armv8_crc32_pmull_little(const unsigned char* buf, + z_size_t len, + uint32_t crc); + +#endif diff --git a/packages/runtime/vendor/zlib/crc_folding.c b/packages/runtime/vendor/zlib/crc_folding.c new file mode 100644 index 000000000..3e31ae4a7 --- /dev/null +++ b/packages/runtime/vendor/zlib/crc_folding.c @@ -0,0 +1,513 @@ +/* + * Compute the CRC32 using a parallelized folding approach with the PCLMULQDQ + * instruction. + * + * A white paper describing this algorithm can be found at: + * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf + * + * Copyright (C) 2013 Intel Corporation. All rights reserved. + * Authors: + * Wajdi Feghali + * Jim Guilford + * Vinodh Gopal + * Erdinc Ozturk + * Jim Kukunas + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "deflate.h" + +#ifdef CRC32_SIMD_SSE42_PCLMUL + +#include +#include +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +#define TARGET_SSE42_PCLMUL __attribute__((__target__("sse4.2,pclmul"))) +#else +#define TARGET_SSE42_PCLMUL +#endif + +#define CRC_LOAD(s) \ + do { \ + __m128i xmm_crc0 = _mm_loadu_si128((__m128i *)s->crc0 + 0);\ + __m128i xmm_crc1 = _mm_loadu_si128((__m128i *)s->crc0 + 1);\ + __m128i xmm_crc2 = _mm_loadu_si128((__m128i *)s->crc0 + 2);\ + __m128i xmm_crc3 = _mm_loadu_si128((__m128i *)s->crc0 + 3);\ + __m128i xmm_crc_part = _mm_loadu_si128((__m128i *)s->crc0 + 4); + +#define CRC_SAVE(s) \ + _mm_storeu_si128((__m128i *)s->crc0 + 0, xmm_crc0);\ + _mm_storeu_si128((__m128i *)s->crc0 + 1, xmm_crc1);\ + _mm_storeu_si128((__m128i *)s->crc0 + 2, xmm_crc2);\ + _mm_storeu_si128((__m128i *)s->crc0 + 3, xmm_crc3);\ + _mm_storeu_si128((__m128i *)s->crc0 + 4, xmm_crc_part);\ + } while (0); + +TARGET_SSE42_PCLMUL +ZLIB_INTERNAL void crc_fold_init(deflate_state *const s) +{ + CRC_LOAD(s) + + xmm_crc0 = _mm_cvtsi32_si128(0x9db42487); + xmm_crc1 = _mm_setzero_si128(); + xmm_crc2 = _mm_setzero_si128(); + xmm_crc3 = _mm_setzero_si128(); + + CRC_SAVE(s) + + s->strm->adler = 0; +} + +TARGET_SSE42_PCLMUL +local void fold_1(deflate_state *const s, + __m128i *xmm_crc0, __m128i *xmm_crc1, + __m128i *xmm_crc2, __m128i *xmm_crc3) +{ + const __m128i xmm_fold4 = _mm_set_epi32( + 0x00000001, 0x54442bd4, + 0x00000001, 0xc6e41596); + + __m128i x_tmp3; + __m128 ps_crc0, ps_crc3, ps_res; + + x_tmp3 = *xmm_crc3; + + *xmm_crc3 = *xmm_crc0; + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); + ps_res = _mm_xor_ps(ps_crc0, ps_crc3); + + *xmm_crc0 = *xmm_crc1; + *xmm_crc1 = *xmm_crc2; + *xmm_crc2 = x_tmp3; + *xmm_crc3 = _mm_castps_si128(ps_res); +} + +TARGET_SSE42_PCLMUL +local void fold_2(deflate_state *const s, + __m128i *xmm_crc0, __m128i *xmm_crc1, + __m128i *xmm_crc2, __m128i *xmm_crc3) +{ + const __m128i xmm_fold4 = _mm_set_epi32( + 0x00000001, 0x54442bd4, + 0x00000001, 0xc6e41596); + + __m128i x_tmp3, x_tmp2; + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res31, ps_res20; + + x_tmp3 = *xmm_crc3; + x_tmp2 = *xmm_crc2; + + *xmm_crc3 = *xmm_crc1; + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); + ps_res31= _mm_xor_ps(ps_crc3, ps_crc1); + + *xmm_crc2 = *xmm_crc0; + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); + ps_res20= _mm_xor_ps(ps_crc0, ps_crc2); + + *xmm_crc0 = x_tmp2; + *xmm_crc1 = x_tmp3; + *xmm_crc2 = _mm_castps_si128(ps_res20); + *xmm_crc3 = _mm_castps_si128(ps_res31); +} + +TARGET_SSE42_PCLMUL +local void fold_3(deflate_state *const s, + __m128i *xmm_crc0, __m128i *xmm_crc1, + __m128i *xmm_crc2, __m128i *xmm_crc3) +{ + const __m128i xmm_fold4 = _mm_set_epi32( + 0x00000001, 0x54442bd4, + 0x00000001, 0xc6e41596); + + __m128i x_tmp3; + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res32, ps_res21, ps_res10; + + x_tmp3 = *xmm_crc3; + + *xmm_crc3 = *xmm_crc2; + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); + ps_res32 = _mm_xor_ps(ps_crc2, ps_crc3); + + *xmm_crc2 = *xmm_crc1; + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); + ps_res21= _mm_xor_ps(ps_crc1, ps_crc2); + + *xmm_crc1 = *xmm_crc0; + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); + ps_res10= _mm_xor_ps(ps_crc0, ps_crc1); + + *xmm_crc0 = x_tmp3; + *xmm_crc1 = _mm_castps_si128(ps_res10); + *xmm_crc2 = _mm_castps_si128(ps_res21); + *xmm_crc3 = _mm_castps_si128(ps_res32); +} + +TARGET_SSE42_PCLMUL +local void fold_4(deflate_state *const s, + __m128i *xmm_crc0, __m128i *xmm_crc1, + __m128i *xmm_crc2, __m128i *xmm_crc3) +{ + const __m128i xmm_fold4 = _mm_set_epi32( + 0x00000001, 0x54442bd4, + 0x00000001, 0xc6e41596); + + __m128i x_tmp0, x_tmp1, x_tmp2, x_tmp3; + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3; + __m128 ps_t0, ps_t1, ps_t2, ps_t3; + __m128 ps_res0, ps_res1, ps_res2, ps_res3; + + x_tmp0 = *xmm_crc0; + x_tmp1 = *xmm_crc1; + x_tmp2 = *xmm_crc2; + x_tmp3 = *xmm_crc3; + + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); + x_tmp0 = _mm_clmulepi64_si128(x_tmp0, xmm_fold4, 0x10); + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); + ps_t0 = _mm_castsi128_ps(x_tmp0); + ps_res0 = _mm_xor_ps(ps_crc0, ps_t0); + + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); + x_tmp1 = _mm_clmulepi64_si128(x_tmp1, xmm_fold4, 0x10); + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); + ps_t1 = _mm_castsi128_ps(x_tmp1); + ps_res1 = _mm_xor_ps(ps_crc1, ps_t1); + + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); + x_tmp2 = _mm_clmulepi64_si128(x_tmp2, xmm_fold4, 0x10); + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); + ps_t2 = _mm_castsi128_ps(x_tmp2); + ps_res2 = _mm_xor_ps(ps_crc2, ps_t2); + + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x01); + x_tmp3 = _mm_clmulepi64_si128(x_tmp3, xmm_fold4, 0x10); + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); + ps_t3 = _mm_castsi128_ps(x_tmp3); + ps_res3 = _mm_xor_ps(ps_crc3, ps_t3); + + *xmm_crc0 = _mm_castps_si128(ps_res0); + *xmm_crc1 = _mm_castps_si128(ps_res1); + *xmm_crc2 = _mm_castps_si128(ps_res2); + *xmm_crc3 = _mm_castps_si128(ps_res3); +} + +local const unsigned zalign(32) pshufb_shf_table[60] = { + 0x84838281,0x88878685,0x8c8b8a89,0x008f8e8d, /* shl 15 (16 - 1)/shr1 */ + 0x85848382,0x89888786,0x8d8c8b8a,0x01008f8e, /* shl 14 (16 - 3)/shr2 */ + 0x86858483,0x8a898887,0x8e8d8c8b,0x0201008f, /* shl 13 (16 - 4)/shr3 */ + 0x87868584,0x8b8a8988,0x8f8e8d8c,0x03020100, /* shl 12 (16 - 4)/shr4 */ + 0x88878685,0x8c8b8a89,0x008f8e8d,0x04030201, /* shl 11 (16 - 5)/shr5 */ + 0x89888786,0x8d8c8b8a,0x01008f8e,0x05040302, /* shl 10 (16 - 6)/shr6 */ + 0x8a898887,0x8e8d8c8b,0x0201008f,0x06050403, /* shl 9 (16 - 7)/shr7 */ + 0x8b8a8988,0x8f8e8d8c,0x03020100,0x07060504, /* shl 8 (16 - 8)/shr8 */ + 0x8c8b8a89,0x008f8e8d,0x04030201,0x08070605, /* shl 7 (16 - 9)/shr9 */ + 0x8d8c8b8a,0x01008f8e,0x05040302,0x09080706, /* shl 6 (16 -10)/shr10*/ + 0x8e8d8c8b,0x0201008f,0x06050403,0x0a090807, /* shl 5 (16 -11)/shr11*/ + 0x8f8e8d8c,0x03020100,0x07060504,0x0b0a0908, /* shl 4 (16 -12)/shr12*/ + 0x008f8e8d,0x04030201,0x08070605,0x0c0b0a09, /* shl 3 (16 -13)/shr13*/ + 0x01008f8e,0x05040302,0x09080706,0x0d0c0b0a, /* shl 2 (16 -14)/shr14*/ + 0x0201008f,0x06050403,0x0a090807,0x0e0d0c0b /* shl 1 (16 -15)/shr15*/ +}; + +TARGET_SSE42_PCLMUL +local void partial_fold(deflate_state *const s, const size_t len, + __m128i *xmm_crc0, __m128i *xmm_crc1, + __m128i *xmm_crc2, __m128i *xmm_crc3, + __m128i *xmm_crc_part) +{ + + const __m128i xmm_fold4 = _mm_set_epi32( + 0x00000001, 0x54442bd4, + 0x00000001, 0xc6e41596); + const __m128i xmm_mask3 = _mm_set1_epi32(0x80808080); + + __m128i xmm_shl, xmm_shr, xmm_tmp1, xmm_tmp2, xmm_tmp3; + __m128i xmm_a0_0, xmm_a0_1; + __m128 ps_crc3, psa0_0, psa0_1, ps_res; + + xmm_shl = _mm_load_si128((__m128i *)pshufb_shf_table + (len - 1)); + xmm_shr = xmm_shl; + xmm_shr = _mm_xor_si128(xmm_shr, xmm_mask3); + + xmm_a0_0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shl); + + *xmm_crc0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shr); + xmm_tmp1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shl); + *xmm_crc0 = _mm_or_si128(*xmm_crc0, xmm_tmp1); + + *xmm_crc1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shr); + xmm_tmp2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shl); + *xmm_crc1 = _mm_or_si128(*xmm_crc1, xmm_tmp2); + + *xmm_crc2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shr); + xmm_tmp3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shl); + *xmm_crc2 = _mm_or_si128(*xmm_crc2, xmm_tmp3); + + *xmm_crc3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shr); + *xmm_crc_part = _mm_shuffle_epi8(*xmm_crc_part, xmm_shl); + *xmm_crc3 = _mm_or_si128(*xmm_crc3, *xmm_crc_part); + + xmm_a0_1 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x10); + xmm_a0_0 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x01); + + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); + psa0_0 = _mm_castsi128_ps(xmm_a0_0); + psa0_1 = _mm_castsi128_ps(xmm_a0_1); + + ps_res = _mm_xor_ps(ps_crc3, psa0_0); + ps_res = _mm_xor_ps(ps_res, psa0_1); + + *xmm_crc3 = _mm_castps_si128(ps_res); +} + +TARGET_SSE42_PCLMUL +ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s, + unsigned char *dst, const unsigned char *src, long len) +{ + unsigned long algn_diff; + __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; + + CRC_LOAD(s) + + if (len < 16) { + if (len == 0) + return; + goto partial; + } + + algn_diff = (0 - (uintptr_t)src) & 0xF; + if (algn_diff) { + xmm_crc_part = _mm_loadu_si128((__m128i *)src); + _mm_storeu_si128((__m128i *)dst, xmm_crc_part); + + dst += algn_diff; + src += algn_diff; + len -= algn_diff; + + partial_fold(s, algn_diff, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, + &xmm_crc_part); + } + + while ((len -= 64) >= 0) { + xmm_t0 = _mm_load_si128((__m128i *)src); + xmm_t1 = _mm_load_si128((__m128i *)src + 1); + xmm_t2 = _mm_load_si128((__m128i *)src + 2); + xmm_t3 = _mm_load_si128((__m128i *)src + 3); + + fold_4(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); + + _mm_storeu_si128((__m128i *)dst, xmm_t0); + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); + _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); + _mm_storeu_si128((__m128i *)dst + 3, xmm_t3); + + xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0); + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1); + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3); + + src += 64; + dst += 64; + } + + /* + * len = num bytes left - 64 + */ + if (len + 16 >= 0) { + len += 16; + + xmm_t0 = _mm_load_si128((__m128i *)src); + xmm_t1 = _mm_load_si128((__m128i *)src + 1); + xmm_t2 = _mm_load_si128((__m128i *)src + 2); + + fold_3(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); + + _mm_storeu_si128((__m128i *)dst, xmm_t0); + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); + _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); + + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0); + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2); + + if (len == 0) + goto done; + + dst += 48; + src += 48; + } else if (len + 32 >= 0) { + len += 32; + + xmm_t0 = _mm_load_si128((__m128i *)src); + xmm_t1 = _mm_load_si128((__m128i *)src + 1); + + fold_2(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); + + _mm_storeu_si128((__m128i *)dst, xmm_t0); + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); + + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1); + + if (len == 0) + goto done; + + dst += 32; + src += 32; + } else if (len + 48 >= 0) { + len += 48; + + xmm_t0 = _mm_load_si128((__m128i *)src); + + fold_1(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); + + _mm_storeu_si128((__m128i *)dst, xmm_t0); + + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t0); + + if (len == 0) + goto done; + + dst += 16; + src += 16; + } else { + len += 64; + if (len == 0) + goto done; + } + +partial: + +#if defined(_MSC_VER) + /* VS does not permit the use of _mm_set_epi64x in 32-bit builds */ + { + int32_t parts[4] = {0, 0, 0, 0}; + memcpy(&parts, src, len); + xmm_crc_part = _mm_set_epi32(parts[3], parts[2], parts[1], parts[0]); + } +#else + { + int64_t parts[2] = {0, 0}; + memcpy(&parts, src, len); + xmm_crc_part = _mm_set_epi64x(parts[1], parts[0]); + } +#endif + + zmemcpy(dst, src, len); /* TODO: Possibly generate more efficient code. */ + partial_fold(s, len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, + &xmm_crc_part); +done: + CRC_SAVE(s) +} + +local const unsigned zalign(16) crc_k[] = { + 0xccaa009e, 0x00000000, /* rk1 */ + 0x751997d0, 0x00000001, /* rk2 */ + 0xccaa009e, 0x00000000, /* rk5 */ + 0x63cd6124, 0x00000001, /* rk6 */ + 0xf7011640, 0x00000001, /* rk7 */ + 0xdb710640, 0x00000001 /* rk8 */ +}; + +local const unsigned zalign(16) crc_mask[4] = { + 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000 +}; + +local const unsigned zalign(16) crc_mask2[4] = { + 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF +}; + +TARGET_SSE42_PCLMUL +unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) +{ + const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask); + const __m128i xmm_mask2 = _mm_load_si128((__m128i *)crc_mask2); + + unsigned crc; + __m128i x_tmp0, x_tmp1, x_tmp2, crc_fold; + + __m128i xmm_crc0 = _mm_loadu_si128((__m128i *)s->crc0 + 0); + __m128i xmm_crc1 = _mm_loadu_si128((__m128i *)s->crc0 + 1); + __m128i xmm_crc2 = _mm_loadu_si128((__m128i *)s->crc0 + 2); + __m128i xmm_crc3 = _mm_loadu_si128((__m128i *)s->crc0 + 3); + + /* + * k1 + */ + crc_fold = _mm_load_si128((__m128i *)crc_k); + + x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10); + xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01); + xmm_crc1 = _mm_xor_si128(xmm_crc1, x_tmp0); + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_crc0); + + x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10); + xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01); + xmm_crc2 = _mm_xor_si128(xmm_crc2, x_tmp1); + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_crc1); + + x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10); + xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01); + xmm_crc3 = _mm_xor_si128(xmm_crc3, x_tmp2); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); + + /* + * k5 + */ + crc_fold = _mm_load_si128((__m128i *)crc_k + 1); + + xmm_crc0 = xmm_crc3; + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); + xmm_crc0 = _mm_srli_si128(xmm_crc0, 8); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); + + xmm_crc0 = xmm_crc3; + xmm_crc3 = _mm_slli_si128(xmm_crc3, 4); + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); + xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask2); + + /* + * k7 + */ + xmm_crc1 = xmm_crc3; + xmm_crc2 = xmm_crc3; + crc_fold = _mm_load_si128((__m128i *)crc_k + 2); + + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); + xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask); + + xmm_crc2 = xmm_crc3; + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1); + + crc = _mm_extract_epi32(xmm_crc3, 2); + return ~crc; +} + +#endif /* CRC32_SIMD_SSE42_PCLMUL */ diff --git a/packages/runtime/vendor/zlib/deflate.c b/packages/runtime/vendor/zlib/deflate.c index 012ea8148..b98a985e5 100644 --- a/packages/runtime/vendor/zlib/deflate.c +++ b/packages/runtime/vendor/zlib/deflate.c @@ -1,5 +1,5 @@ /* deflate.c -- compress data using the deflation algorithm - * Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler + * Copyright (C) 1995-2026 Jean-loup Gailly and Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -37,7 +37,7 @@ * REFERENCES * * Deutsch, L.P.,"DEFLATE Compressed Data Format Specification". - * Available in http://tools.ietf.org/html/rfc1951 + * Available at https://datatracker.ietf.org/doc/html/rfc1951 * * A description of the Rabin and Karp algorithm is given in the book * "Algorithms" by R. Sedgewick, Addison-Wesley, p252. @@ -48,11 +48,30 @@ */ /* @(#) $Id$ */ - +#include #include "deflate.h" +#include "cpu_features.h" + +#if defined(DEFLATE_SLIDE_HASH_SSE2) || defined(DEFLATE_SLIDE_HASH_NEON) +#include "slide_hash_simd.h" +#endif + +#include "compare256.h" + +#if defined(QAT_COMPRESSION_ENABLED) +#include "contrib/qat/deflate_qat.h" +#endif + +#include "contrib/optimizations/insert_string.h" + +#ifdef FASTEST +/* See http://crbug.com/1113596 */ +#error "FASTEST is not supported in Chromium's zlib." +#endif + const char deflate_copyright[] = - " deflate 1.3.1 Copyright 1995-2024 Jean-loup Gailly and Mark Adler "; + " deflate 1.3.2.1 Copyright 1995-2026 Jean-loup Gailly and Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -78,6 +97,11 @@ local block_state deflate_slow(deflate_state *s, int flush); local block_state deflate_rle(deflate_state *s, int flush); local block_state deflate_huff(deflate_state *s, int flush); +/* From crc32.c */ +extern void ZLIB_INTERNAL crc_reset(deflate_state *const s); +extern void ZLIB_INTERNAL crc_finalize(deflate_state *const s); +extern void ZLIB_INTERNAL copy_with_crc(z_streamp strm, Bytef *dst, long size); + /* =========================================================================== * Local data */ @@ -132,46 +156,17 @@ local const config configuration_table[10] = { /* rank Z_BLOCK between Z_NO_FLUSH and Z_PARTIAL_FLUSH */ #define RANK(f) (((f) * 2) - ((f) > 4 ? 9 : 0)) -/* =========================================================================== - * Update a hash value with the given input byte - * IN assertion: all calls to UPDATE_HASH are made with consecutive input - * characters, so that a running hash key can be computed from the previous - * key instead of complete recalculation each time. - */ -#define UPDATE_HASH(s,h,c) (h = (((h) << s->hash_shift) ^ (c)) & s->hash_mask) - - -/* =========================================================================== - * Insert string str in the dictionary and set match_head to the previous head - * of the hash chain (the most recent string with same hash key). Return - * the previous length of the hash chain. - * If this file is compiled with -DFASTEST, the compression level is forced - * to 1, and no hash chains are maintained. - * IN assertion: all calls to INSERT_STRING are made with consecutive input - * characters and the first MIN_MATCH bytes of str are valid (except for - * the last MIN_MATCH-1 bytes of the input file). - */ -#ifdef FASTEST -#define INSERT_STRING(s, str, match_head) \ - (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ - match_head = s->head[s->ins_h], \ - s->head[s->ins_h] = (Pos)(str)) -#else -#define INSERT_STRING(s, str, match_head) \ - (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ - match_head = s->prev[(str) & s->w_mask] = s->head[s->ins_h], \ - s->head[s->ins_h] = (Pos)(str)) -#endif - /* =========================================================================== * Initialize the hash table (avoiding 64K overflow for 16 bit systems). * prev[] will be initialized on the fly. + * TODO(cavalcantii): optimization opportunity, check comments on: + * https://chromium-review.googlesource.com/c/chromium/src/+/3561506/ */ #define CLEAR_HASH(s) \ do { \ s->head[s->hash_size - 1] = NIL; \ - zmemzero((Bytef *)s->head, \ - (unsigned)(s->hash_size - 1)*sizeof(*s->head)); \ + zmemzero(s->head, (unsigned)(s->hash_size - 1)*sizeof(*s->head)); \ + s->slid = 0; \ } while (0) /* =========================================================================== @@ -185,6 +180,12 @@ local const config configuration_table[10] = { # endif #endif local void slide_hash(deflate_state *s) { +#if defined(DEFLATE_SLIDE_HASH_SSE2) || defined(DEFLATE_SLIDE_HASH_NEON) + slide_hash_simd(s->head, s->prev, s->w_size, s->hash_size); + s->slid = 1; + return; +#endif + unsigned n, m; Posf *p; uInt wsize = s->w_size; @@ -195,8 +196,8 @@ local void slide_hash(deflate_state *s) { m = *--p; *p = (Pos)(m >= wsize ? m - wsize : NIL); } while (--n); - n = wsize; #ifndef FASTEST + n = wsize; p = &s->prev[n]; do { m = *--p; @@ -206,6 +207,7 @@ local void slide_hash(deflate_state *s) { */ } while (--n); #endif + s->slid = 1; } /* =========================================================================== @@ -223,15 +225,19 @@ local unsigned read_buf(z_streamp strm, Bytef *buf, unsigned size) { strm->avail_in -= len; - zmemcpy(buf, strm->next_in, len); - if (strm->state->wrap == 1) { - strm->adler = adler32(strm->adler, buf, len); - } + /* TODO(cavalcantii): verify if we can remove 'copy_with_crc', it is legacy + * of the Intel optimizations dating back to 2015. + */ #ifdef GZIP - else if (strm->state->wrap == 2) { - strm->adler = crc32(strm->adler, buf, len); - } + if (strm->state->wrap == 2) + copy_with_crc(strm, buf, len); + else #endif + { + zmemcpy(buf, strm->next_in, len); + if (strm->state->wrap == 1) + strm->adler = adler32(strm->adler, buf, len); + } strm->next_in += len; strm->total_in += len; @@ -259,7 +265,14 @@ local void fill_window(deflate_state *s) { more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); /* Deal with !@#$% 64K limit: */ +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4127) +#endif if (sizeof(int) <= 2) { +#ifdef _MSC_VER +#pragma warning(pop) +#endif if (more == 0 && s->strstart == 0 && s->lookahead == 0) { more = wsize; @@ -303,6 +316,20 @@ local void fill_window(deflate_state *s) { n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); s->lookahead += n; + /* Initialize the hash value now that we have some input: */ + if (s->chromium_zlib_hash) { + /* chromium hash reads 4 bytes */ + if (s->lookahead + s->insert > MIN_MATCH) { + uInt str = s->strstart - s->insert; + while (s->insert) { + insert_string(s, str); + str++; + s->insert--; + if (s->lookahead + s->insert <= MIN_MATCH) + break; + } + } + } else /* Initialize the hash value now that we have some input: */ if (s->lookahead + s->insert >= MIN_MATCH) { uInt str = s->strstart - s->insert; @@ -375,6 +402,8 @@ int ZEXPORT deflateInit_(z_streamp strm, int level, const char *version, /* To do: ignore strm->next_in if we use it as window */ } +#define WINDOW_PADDING 8 + /* ========================================================================= */ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy, @@ -383,6 +412,15 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, int wrap = 1; static const char my_version[] = ZLIB_VERSION; + // Needed to activate optimized insert_string() that helps compression + // for all wrapper formats (e.g. RAW, ZLIB, GZIP). + // Feature detection is not triggered while using RAW mode (i.e. we never + // call crc32() with a NULL buffer). +#if defined(CRC32_ARMV8_CRC32) || defined(CRC32_SIMD_SSE42_PCLMUL) \ + || defined(RISCV_RVV) + cpu_check_features(); +#endif + if (version == Z_NULL || version[0] != my_version[0] || stream_size != sizeof(z_stream)) { return Z_VERSION_ERROR; @@ -431,6 +469,7 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, if (windowBits == 8) windowBits = 9; /* until 256-byte window bug fixed */ s = (deflate_state *) ZALLOC(strm, 1, sizeof(deflate_state)); if (s == Z_NULL) return Z_MEM_ERROR; + zmemzero(s, sizeof(deflate_state)); strm->state = (struct internal_state FAR *)s; s->strm = strm; s->status = INIT_STATE; /* to pass state test in deflateReset() */ @@ -441,12 +480,23 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, s->w_size = 1 << s->w_bits; s->w_mask = s->w_size - 1; - s->hash_bits = (uInt)memLevel + 7; + s->chromium_zlib_hash = 1; +#if defined(USE_ZLIB_RABIN_KARP_ROLLING_HASH) + s->chromium_zlib_hash = 0; +#endif + + s->hash_bits = memLevel + 7; + if (s->chromium_zlib_hash && s->hash_bits < 15) { + s->hash_bits = 15; + } + s->hash_size = 1 << s->hash_bits; s->hash_mask = s->hash_size - 1; s->hash_shift = ((s->hash_bits + MIN_MATCH-1) / MIN_MATCH); - s->window = (Bytef *) ZALLOC(strm, s->w_size, 2*sizeof(Byte)); + s->window = (Bytef *) ZALLOC(strm, + s->w_size + WINDOW_PADDING, + 2*sizeof(Byte)); s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); @@ -492,8 +542,11 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, * the compressed data for a dynamic block also cannot overwrite the * symbols from which it is being constructed. */ - - s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, LIT_BUFS); +#ifdef LIT_MEM + s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, 5); +#else + s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, 4); +#endif s->pending_buf_size = (ulg)s->lit_bufsize * 4; if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL || @@ -503,6 +556,13 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, deflateEnd (strm); return Z_MEM_ERROR; } + /* Avoid use of unitialized values in the window, see crbug.com/1137613 and + * crbug.com/1144420 */ + zmemzero(s->window, (s->w_size + WINDOW_PADDING) * (2 * sizeof(Byte))); + /* Avoid use of uninitialized value, see: + * https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11360 + */ + zmemzero(s->prev, s->w_size * sizeof(Pos)); #ifdef LIT_MEM s->d_buf = (ushf *)(s->pending_buf + (s->lit_bufsize << 1)); s->l_buf = s->pending_buf + (s->lit_bufsize << 2); @@ -520,6 +580,13 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, s->strategy = strategy; s->method = (Byte)method; +#if defined(QAT_COMPRESSION_ENABLED) + s->qat_s = NULL; + if (s->level && qat_deflate_init() == Z_OK) { + s->qat_s = qat_deflate_state_init(s->level, s->wrap); + } +#endif + return deflateReset(strm); } @@ -589,11 +656,7 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef *dictionary, str = s->strstart; n = s->lookahead - (MIN_MATCH-1); do { - UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); -#ifndef FASTEST - s->prev[str & s->w_mask] = s->head[s->ins_h]; -#endif - s->head[s->ins_h] = (Pos)str; + insert_string(s, str); str++; } while (--n); s->strstart = str; @@ -712,10 +775,23 @@ int ZEXPORT deflateSetHeader(z_streamp strm, gz_headerp head) { /* ========================================================================= */ int ZEXPORT deflatePending(z_streamp strm, unsigned *pending, int *bits) { if (deflateStateCheck(strm)) return Z_STREAM_ERROR; - if (pending != Z_NULL) - *pending = strm->state->pending; if (bits != Z_NULL) *bits = strm->state->bi_valid; + if (pending != Z_NULL) { + *pending = (unsigned)strm->state->pending; + if (*pending != strm->state->pending) { + *pending = (unsigned)-1; + return Z_BUF_ERROR; + } + } + return Z_OK; +} + +/* ========================================================================= */ +int ZEXPORT deflateUsed(z_streamp strm, int *bits) { + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; + if (bits != Z_NULL) + *bits = strm->state->bi_used; return Z_OK; } @@ -739,7 +815,11 @@ int ZEXPORT deflatePrime(z_streamp strm, int bits, int value) { put = Buf_size - s->bi_valid; if (put > bits) put = bits; +#if defined(DEFLATE_CHUNK_WRITE_64LE) + s->bi_buf |= (uint64_t)((value & ((1ULL << put) - 1)) << s->bi_valid); +#else s->bi_buf |= (ush)((value & ((1 << put) - 1)) << s->bi_valid); +#endif /* DEFLATE_CHUNK_WRITE_64LE */ s->bi_valid += put; _tr_flush_bits(s); value >>= put; @@ -831,28 +911,34 @@ int ZEXPORT deflateTune(z_streamp strm, int good_length, int max_lazy, * * Shifts are used to approximate divisions, for speed. */ -uLong ZEXPORT deflateBound(z_streamp strm, uLong sourceLen) { +z_size_t ZEXPORT deflateBound_z(z_streamp strm, z_size_t sourceLen) { deflate_state *s; - uLong fixedlen, storelen, wraplen; + z_size_t fixedlen, storelen, wraplen, bound; /* upper bound for fixed blocks with 9-bit literals and length 255 (memLevel == 2, which is the lowest that may not use stored blocks) -- ~13% overhead plus a small constant */ fixedlen = sourceLen + (sourceLen >> 3) + (sourceLen >> 8) + (sourceLen >> 9) + 4; + if (fixedlen < sourceLen) + fixedlen = (z_size_t)-1; /* upper bound for stored blocks with length 127 (memLevel == 1) -- ~4% overhead plus a small constant */ storelen = sourceLen + (sourceLen >> 5) + (sourceLen >> 7) + (sourceLen >> 11) + 7; + if (storelen < sourceLen) + storelen = (z_size_t)-1; - /* if can't get parameters, return larger bound plus a zlib wrapper */ - if (deflateStateCheck(strm)) - return (fixedlen > storelen ? fixedlen : storelen) + 6; + /* if can't get parameters, return larger bound plus a wrapper */ + if (deflateStateCheck(strm)) { + bound = fixedlen > storelen ? fixedlen : storelen; + return bound + 18 < bound ? (z_size_t)-1 : bound + 18; + } /* compute wrapper length */ s = strm->state; - switch (s->wrap) { + switch (s->wrap < 0 ? -s->wrap : s->wrap) { case 0: /* raw deflate */ wraplen = 0; break; @@ -882,18 +968,33 @@ uLong ZEXPORT deflateBound(z_streamp strm, uLong sourceLen) { break; #endif default: /* for compiler happiness */ - wraplen = 6; + wraplen = 18; + } + + /* With Chromium's hashing, s->hash_bits may not correspond to the + memLevel, making the computations below incorrect. Return the + conservative bound. */ + if (s->chromium_zlib_hash) { + bound = fixedlen > storelen ? fixedlen : storelen; + return bound + wraplen < bound ? (z_size_t)-1 : bound + wraplen; } /* if not default parameters, return one of the conservative bounds */ - if (s->w_bits != 15 || s->hash_bits != 8 + 7) - return (s->w_bits <= s->hash_bits && s->level ? fixedlen : storelen) + - wraplen; + if (s->w_bits != 15 || s->hash_bits != 8 + 7) { + bound = s->w_bits <= s->hash_bits && s->level ? fixedlen : + storelen; + return bound + wraplen < bound ? (z_size_t)-1 : bound + wraplen; + } /* default settings: return tight bound for that case -- ~0.03% overhead plus a small constant */ - return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + - (sourceLen >> 25) + 13 - 6 + wraplen; + bound = sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + + (sourceLen >> 25) + 13 - 6 + wraplen; + return bound < sourceLen ? (z_size_t)-1 : bound; +} +uLong ZEXPORT deflateBound(z_streamp strm, uLong sourceLen) { + z_size_t bound = deflateBound_z(strm, sourceLen); + return (uLong)bound != bound ? (uLong)-1 : (uLong)bound; } /* ========================================================================= @@ -916,9 +1017,15 @@ local void flush_pending(z_streamp strm) { unsigned len; deflate_state *s = strm->state; +#if defined(QAT_COMPRESSION_ENABLED) + if (s->qat_s) { + qat_flush_pending(s); + } +#endif + _tr_flush_bits(s); - len = s->pending; - if (len > strm->avail_out) len = strm->avail_out; + len = s->pending > strm->avail_out ? strm->avail_out : + (unsigned)s->pending; if (len == 0) return; zmemcpy(strm->next_out, s->pending_out, len); @@ -938,8 +1045,8 @@ local void flush_pending(z_streamp strm) { #define HCRC_UPDATE(beg) \ do { \ if (s->gzhead->hcrc && s->pending > (beg)) \ - strm->adler = crc32(strm->adler, s->pending_buf + (beg), \ - s->pending - (beg)); \ + strm->adler = crc32_z(strm->adler, s->pending_buf + (beg), \ + s->pending - (beg)); \ } while (0) /* ========================================================================= */ @@ -1030,7 +1137,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { #ifdef GZIP if (s->status == GZIP_STATE) { /* gzip header */ - strm->adler = crc32(0L, Z_NULL, 0); + crc_reset(s); put_byte(s, 31); put_byte(s, 139); put_byte(s, 8); @@ -1073,8 +1180,8 @@ int ZEXPORT deflate(z_streamp strm, int flush) { put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); } if (s->gzhead->hcrc) - strm->adler = crc32(strm->adler, s->pending_buf, - s->pending); + strm->adler = crc32_z(strm->adler, s->pending_buf, + s->pending); s->gzindex = 0; s->status = EXTRA_STATE; } @@ -1082,9 +1189,9 @@ int ZEXPORT deflate(z_streamp strm, int flush) { if (s->status == EXTRA_STATE) { if (s->gzhead->extra != Z_NULL) { ulg beg = s->pending; /* start of bytes to update crc */ - uInt left = (s->gzhead->extra_len & 0xffff) - s->gzindex; + ulg left = (s->gzhead->extra_len & 0xffff) - s->gzindex; while (s->pending + left > s->pending_buf_size) { - uInt copy = s->pending_buf_size - s->pending; + ulg copy = s->pending_buf_size - s->pending; zmemcpy(s->pending_buf + s->pending, s->gzhead->extra + s->gzindex, copy); s->pending = s->pending_buf_size; @@ -1231,6 +1338,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { /* Write the trailer */ #ifdef GZIP if (s->wrap == 2) { + crc_finalize(s); put_byte(s, (Byte)(strm->adler & 0xff)); put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); put_byte(s, (Byte)((strm->adler >> 16) & 0xff)); @@ -1268,6 +1376,12 @@ int ZEXPORT deflateEnd(z_streamp strm) { TRY_FREE(strm, strm->state->prev); TRY_FREE(strm, strm->state->window); +#if defined(QAT_COMPRESSION_ENABLED) + if (strm->state->qat_s) { + qat_deflate_state_free(strm->state); + } +#endif + ZFREE(strm, strm->state); strm->state = Z_NULL; @@ -1295,18 +1409,25 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { ss = source->state; - zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); + zmemcpy(dest, source, sizeof(z_stream)); ds = (deflate_state *) ZALLOC(dest, 1, sizeof(deflate_state)); if (ds == Z_NULL) return Z_MEM_ERROR; + zmemzero(ds, sizeof(deflate_state)); dest->state = (struct internal_state FAR *) ds; - zmemcpy((voidpf)ds, (voidpf)ss, sizeof(deflate_state)); + zmemcpy(ds, ss, sizeof(deflate_state)); ds->strm = dest; - ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte)); + ds->window = (Bytef *) ZALLOC(dest, + ds->w_size + WINDOW_PADDING, + 2*sizeof(Byte)); ds->prev = (Posf *) ZALLOC(dest, ds->w_size, sizeof(Pos)); ds->head = (Posf *) ZALLOC(dest, ds->hash_size, sizeof(Pos)); - ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, LIT_BUFS); +#ifdef LIT_MEM + ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, 5); +#else + ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, 4); +#endif if (ds->window == Z_NULL || ds->prev == Z_NULL || ds->head == Z_NULL || ds->pending_buf == Z_NULL) { @@ -1314,23 +1435,36 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { return Z_MEM_ERROR; } /* following zmemcpy do not work for 16-bit MSDOS */ - zmemcpy(ds->window, ss->window, ds->w_size * 2 * sizeof(Byte)); - zmemcpy((voidpf)ds->prev, (voidpf)ss->prev, ds->w_size * sizeof(Pos)); - zmemcpy((voidpf)ds->head, (voidpf)ss->head, ds->hash_size * sizeof(Pos)); - zmemcpy(ds->pending_buf, ss->pending_buf, ds->lit_bufsize * LIT_BUFS); + zmemcpy(ds->window, ss->window, ss->high_water); + zmemcpy(ds->prev, ss->prev, + (ss->slid || ss->strstart - ss->insert > ds->w_size ? ds->w_size : + ss->strstart - ss->insert) * sizeof(Pos)); + zmemcpy(ds->head, ss->head, ds->hash_size * sizeof(Pos)); ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf); + zmemcpy(ds->pending_out, ss->pending_out, ss->pending); #ifdef LIT_MEM ds->d_buf = (ushf *)(ds->pending_buf + (ds->lit_bufsize << 1)); ds->l_buf = ds->pending_buf + (ds->lit_bufsize << 2); + zmemcpy(ds->d_buf, ss->d_buf, ss->sym_next * sizeof(ush)); + zmemcpy(ds->l_buf, ss->l_buf, ss->sym_next); #else ds->sym_buf = ds->pending_buf + ds->lit_bufsize; + zmemcpy(ds->sym_buf, ss->sym_buf, ss->sym_next); #endif ds->l_desc.dyn_tree = ds->dyn_ltree; ds->d_desc.dyn_tree = ds->dyn_dtree; ds->bl_desc.dyn_tree = ds->bl_tree; +#if defined(QAT_COMPRESSION_ENABLED) + if(ss->qat_s) { + ds->qat_s = qat_deflate_copy(ss); + if (!ds->qat_s) + return Z_MEM_ERROR; + } +#endif + return Z_OK; #endif /* MAXSEG_64K */ } @@ -1347,9 +1481,9 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { */ local uInt longest_match(deflate_state *s, IPos cur_match) { unsigned chain_length = s->max_chain_length;/* max hash chain length */ - register Bytef *scan = s->window + s->strstart; /* current string */ - register Bytef *match; /* matched string */ - register int len; /* length of current match */ + Bytef *scan = s->window + s->strstart; /* current string */ + Bytef *match; /* matched string */ + int len; /* length of current match */ int best_len = (int)s->prev_length; /* best match length so far */ int nice_match = s->nice_match; /* stop if match long enough */ IPos limit = s->strstart > (IPos)MAX_DIST(s) ? @@ -1360,17 +1494,23 @@ local uInt longest_match(deflate_state *s, IPos cur_match) { Posf *prev = s->prev; uInt wmask = s->w_mask; -#ifdef UNALIGNED_OK +#if defined(DEFLATE_COMPARE256_64LE) + /* Quickly reject non-matches with the 2-byte prefilter + * (scan_start/scan_end); candidates that pass are extended 8 bytes at a + * time with compare256(). */ + uInt scan_start = read16(scan); + uInt scan_end = read16(scan + best_len - 1); +#elif defined(UNALIGNED_OK) /* Compare two bytes at a time. Note: this is not always beneficial. * Try with and without -DUNALIGNED_OK to check. */ - register Bytef *strend = s->window + s->strstart + MAX_MATCH - 1; - register ush scan_start = *(ushf*)scan; - register ush scan_end = *(ushf*)(scan + best_len - 1); + Bytef *strend = s->window + s->strstart + MAX_MATCH - 1; + ush scan_start = *(ushf*)scan; + ush scan_end = *(ushf*)(scan + best_len - 1); #else - register Bytef *strend = s->window + s->strstart + MAX_MATCH; - register Byte scan_end1 = scan[best_len - 1]; - register Byte scan_end = scan[best_len]; + Bytef *strend = s->window + s->strstart + MAX_MATCH; + Byte scan_end1 = scan[best_len - 1]; + Byte scan_end = scan[best_len]; #endif /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16. @@ -1402,7 +1542,13 @@ local uInt longest_match(deflate_state *s, IPos cur_match) { * However the length of the match is limited to the lookahead, so * the output of deflate is not affected by the uninitialized values. */ -#if (defined(UNALIGNED_OK) && MAX_MATCH == 258) +#if defined(DEFLATE_COMPARE256_64LE) + if (read16(match + best_len - 1) != scan_end || + read16(match) != scan_start) continue; + /* scan_start matched, so bytes 0..1 are equal; compare the remaining + * up to 256 bytes so the length can still reach MAX_MATCH (258). */ + len = 2 + compare256(scan + 2, match + 2); +#elif (defined(UNALIGNED_OK) && MAX_MATCH == 258) /* This code assumes sizeof(unsigned short) == 2. Do not use * UNALIGNED_OK if your compiler uses a different size. */ @@ -1418,7 +1564,16 @@ local uInt longest_match(deflate_state *s, IPos cur_match) { * necessary to put more guard bytes at the end of the window, or * to check more often for insufficient lookahead. */ - Assert(scan[2] == match[2], "scan[2]?"); + if (!s->chromium_zlib_hash) { + Assert(scan[2] == match[2], "scan[2]?"); + } else { + /* When using CRC hashing, scan[2] and match[2] may mismatch, but in + * that case at least one of the other hashed bytes will mismatch + * also. Bytes 0 and 1 were already checked above, and we know there + * are at least four bytes to check otherwise the mismatch would have + * been found by the scan_end comparison above, so: */ + Assert(scan[2] == match[2] || scan[3] != match[3], "scan[2]??"); + } scan++, match++; do { } while (*(ushf*)(scan += 2) == *(ushf*)(match += 2) && @@ -1429,7 +1584,7 @@ local uInt longest_match(deflate_state *s, IPos cur_match) { /* The funny "do {}" generates better code on most compilers */ /* Here, scan <= window + strstart + 257 */ - Assert(scan <= s->window + (unsigned)(s->window_size - 1), + Assert(scan <= s->window+(unsigned)(s->window_size - 1), "wild scan"); if (*scan == *match) scan++; @@ -1438,10 +1593,10 @@ local uInt longest_match(deflate_state *s, IPos cur_match) { #else /* UNALIGNED_OK */ - if (match[best_len] != scan_end || + if (match[best_len] != scan_end || match[best_len - 1] != scan_end1 || - *match != *scan || - *++match != scan[1]) continue; + *match != *scan || + *++match != scan[1]) continue; /* The check at best_len - 1 can be removed because it will be made * again later. (This heuristic is not always a win.) @@ -1450,7 +1605,16 @@ local uInt longest_match(deflate_state *s, IPos cur_match) { * the hash keys are equal and that HASH_BITS >= 8. */ scan += 2, match++; - Assert(*scan == *match, "match[2]?"); + if (!s->chromium_zlib_hash) { + Assert(*scan == *match, "match[2]?"); + } else { + /* When using CRC hashing, scan[2] and match[2] may mismatch, but in + * that case at least one of the other hashed bytes will mismatch + * also. Bytes 0 and 1 were already checked above, and we know there + * are at least four bytes to check otherwise the mismatch would have + * been found by the scan_end comparison above, so: */ + Assert(*scan == *match || scan[1] != match[1], "match[2]??"); + } /* We check for insufficient lookahead only every 8th comparison; * the 256th check will be made at strstart + 258. @@ -1474,7 +1638,9 @@ local uInt longest_match(deflate_state *s, IPos cur_match) { s->match_start = cur_match; best_len = len; if (len >= nice_match) break; -#ifdef UNALIGNED_OK +#if defined(DEFLATE_COMPARE256_64LE) + scan_end = read16(scan + best_len - 1); +#elif defined(UNALIGNED_OK) scan_end = *(ushf*)(scan + best_len - 1); #else scan_end1 = scan[best_len - 1]; @@ -1494,10 +1660,10 @@ local uInt longest_match(deflate_state *s, IPos cur_match) { * Optimized version for FASTEST only */ local uInt longest_match(deflate_state *s, IPos cur_match) { - register Bytef *scan = s->window + s->strstart; /* current string */ - register Bytef *match; /* matched string */ - register int len; /* length of current match */ - register Bytef *strend = s->window + s->strstart + MAX_MATCH; + Bytef *scan = s->window + s->strstart; /* current string */ + Bytef *match; /* matched string */ + int len; /* length of current match */ + Bytef *strend = s->window + s->strstart + MAX_MATCH; /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16. * It is easy to get rid of this optimization if necessary. @@ -1557,7 +1723,7 @@ local uInt longest_match(deflate_state *s, IPos cur_match) { local void check_match(deflate_state *s, IPos start, IPos match, int length) { /* check that the match is indeed a match */ Bytef *back = s->window + (int)match, *here = s->window + start; - IPos len = length; + IPos len = (IPos)length; if (match == (IPos)-1) { /* match starts one byte before the current window -- just compare the subsequent length-1 bytes */ @@ -1629,13 +1795,14 @@ local block_state deflate_stored(deflate_state *s, int flush) { * this is 32K. This can be as small as 507 bytes for memLevel == 1. For * large input and output buffers, the stored block size will be larger. */ - unsigned min_block = MIN(s->pending_buf_size - 5, s->w_size); + unsigned min_block = (unsigned)(MIN(s->pending_buf_size - 5, s->w_size)); /* Copy as many min_block or larger stored blocks directly to next_out as * possible. If flushing, copy the remaining available input to next_out as * stored blocks, if there is enough space. */ - unsigned len, left, have, last = 0; + int last = 0; + unsigned len, left, have; unsigned used = s->strm->avail_in; do { /* Set len to the maximum size block that we can copy directly with the @@ -1643,12 +1810,12 @@ local block_state deflate_stored(deflate_state *s, int flush) { * would be copied from what's left in the window. */ len = MAX_STORED; /* maximum deflate stored block length */ - have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + have = ((unsigned)s->bi_valid + 42) >> 3; /* bytes in header */ if (s->strm->avail_out < have) /* need room for header */ break; /* maximum stored block length that will fit in avail_out: */ have = s->strm->avail_out - have; - left = s->strstart - s->block_start; /* bytes left in window */ + left = (unsigned)(s->strstart - s->block_start); /* window bytes */ if (len > (ulg)left + s->strm->avail_in) len = left + s->strm->avail_in; /* limit len to the input */ if (len > have) @@ -1671,10 +1838,10 @@ local block_state deflate_stored(deflate_state *s, int flush) { _tr_stored_block(s, (char *)0, 0L, last); /* Replace the lengths in the dummy stored block with len. */ - s->pending_buf[s->pending - 4] = len; - s->pending_buf[s->pending - 3] = len >> 8; - s->pending_buf[s->pending - 2] = ~len; - s->pending_buf[s->pending - 1] = ~len >> 8; + s->pending_buf[s->pending - 4] = (Bytef)len; + s->pending_buf[s->pending - 3] = (Bytef)(len >> 8); + s->pending_buf[s->pending - 2] = (Bytef)~len; + s->pending_buf[s->pending - 1] = (Bytef)(~len >> 8); /* Write the stored block header bytes. */ flush_pending(s->strm); @@ -1745,8 +1912,10 @@ local block_state deflate_stored(deflate_state *s, int flush) { s->high_water = s->strstart; /* If the last block was written to next_out, then done. */ - if (last) + if (last) { + s->bi_used = 8; return finish_done; + } /* If flushing and all input has been consumed, then done. */ if (flush != Z_NO_FLUSH && flush != Z_FINISH && @@ -1754,7 +1923,7 @@ local block_state deflate_stored(deflate_state *s, int flush) { return block_done; /* Fill the window with any remaining input. */ - have = s->window_size - s->strstart; + have = (unsigned)(s->window_size - s->strstart); if (s->strm->avail_in > have && s->block_start >= (long)s->w_size) { /* Slide the window down. */ s->block_start -= s->w_size; @@ -1781,11 +1950,11 @@ local block_state deflate_stored(deflate_state *s, int flush) { * have enough input for a worthy block, or if flushing and there is enough * room for the remaining input as a stored block in the pending buffer. */ - have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + have = ((unsigned)s->bi_valid + 42) >> 3; /* bytes in header */ /* maximum stored block length that will fit in pending: */ - have = MIN(s->pending_buf_size - have, MAX_STORED); + have = (unsigned)MIN(s->pending_buf_size - have, MAX_STORED); min_block = MIN(have, s->w_size); - left = s->strstart - s->block_start; + left = (unsigned)(s->strstart - s->block_start); if (left >= min_block || ((left || flush == Z_FINISH) && flush != Z_NO_FLUSH && s->strm->avail_in == 0 && left <= have)) { @@ -1798,6 +1967,8 @@ local block_state deflate_stored(deflate_state *s, int flush) { } /* We've done all we can with the available input and output. */ + if (last) + s->bi_used = 8; return last ? finish_started : need_more; } @@ -1812,6 +1983,24 @@ local block_state deflate_fast(deflate_state *s, int flush) { IPos hash_head; /* head of the hash chain */ int bflush; /* set if current block must be flushed */ +#if defined(QAT_COMPRESSION_ENABLED) + if (s->qat_s) { + qat_block_state qat_block = qat_deflate_step(s, flush); + switch (qat_block) { + case qat_block_need_more: + return need_more; + case qat_block_done: + return block_done; + case qat_block_finish_started: + return finish_started; + case qat_block_finish_done: + return finish_done; + case qat_failure: + break; + } + } +#endif + for (;;) { /* Make sure that we always have enough lookahead, except * at the end of the input file. We need MAX_MATCH bytes @@ -1831,7 +2020,7 @@ local block_state deflate_fast(deflate_state *s, int flush) { */ hash_head = NIL; if (s->lookahead >= MIN_MATCH) { - INSERT_STRING(s, s->strstart, hash_head); + hash_head = insert_string(s, s->strstart); } /* Find the longest match, discarding those <= prev_length. @@ -1846,7 +2035,7 @@ local block_state deflate_fast(deflate_state *s, int flush) { /* longest_match() sets match_start */ } if (s->match_length >= MIN_MATCH) { - check_match(s, s->strstart, s->match_start, s->match_length); + check_match(s, s->strstart, s->match_start, (int)s->match_length); _tr_tally_dist(s, s->strstart - s->match_start, s->match_length - MIN_MATCH, bflush); @@ -1862,7 +2051,7 @@ local block_state deflate_fast(deflate_state *s, int flush) { s->match_length--; /* string at strstart already in table */ do { s->strstart++; - INSERT_STRING(s, s->strstart, hash_head); + hash_head = insert_string(s, s->strstart); /* strstart never exceeds WSIZE-MAX_MATCH, so there are * always MIN_MATCH bytes ahead. */ @@ -1873,14 +2062,17 @@ local block_state deflate_fast(deflate_state *s, int flush) { { s->strstart += s->match_length; s->match_length = 0; - s->ins_h = s->window[s->strstart]; - UPDATE_HASH(s, s->ins_h, s->window[s->strstart + 1]); + + if (!s->chromium_zlib_hash) { + s->ins_h = s->window[s->strstart]; + UPDATE_HASH(s, s->ins_h, s->window[s->strstart + 1]); #if MIN_MATCH != 3 - Call UPDATE_HASH() MIN_MATCH-3 more times + Call UPDATE_HASH() MIN_MATCH-3 more times #endif - /* If lookahead < MIN_MATCH, ins_h is garbage, but it does not - * matter since it will be recomputed at next deflate call. - */ + /* If lookahead < MIN_MATCH, ins_h is garbage, but it does not + * matter since it will be recomputed at next deflate call. + */ + } } } else { /* No match, output a literal byte */ @@ -1911,6 +2103,24 @@ local block_state deflate_slow(deflate_state *s, int flush) { IPos hash_head; /* head of hash chain */ int bflush; /* set if current block must be flushed */ +#if defined(QAT_COMPRESSION_ENABLED) + if (s->qat_s) { + qat_block_state qat_block = qat_deflate_step(s, flush); + switch (qat_block) { + case qat_block_need_more: + return need_more; + case qat_block_done: + return block_done; + case qat_block_finish_started: + return finish_started; + case qat_block_finish_done: + return finish_done; + case qat_failure: + break; + } + } +#endif + /* Process the input block. */ for (;;) { /* Make sure that we always have enough lookahead, except @@ -1931,7 +2141,7 @@ local block_state deflate_slow(deflate_state *s, int flush) { */ hash_head = NIL; if (s->lookahead >= MIN_MATCH) { - INSERT_STRING(s, s->strstart, hash_head); + hash_head = insert_string(s, s->strstart); } /* Find the longest match, discarding those <= prev_length. @@ -1968,7 +2178,7 @@ local block_state deflate_slow(deflate_state *s, int flush) { uInt max_insert = s->strstart + s->lookahead - MIN_MATCH; /* Do not insert strings in hash table beyond this. */ - check_match(s, s->strstart - 1, s->prev_match, s->prev_length); + check_match(s, s->strstart - 1, s->prev_match, (int)s->prev_length); _tr_tally_dist(s, s->strstart - 1 - s->prev_match, s->prev_length - MIN_MATCH, bflush); @@ -1982,7 +2192,7 @@ local block_state deflate_slow(deflate_state *s, int flush) { s->prev_length -= 2; do { if (++s->strstart <= max_insert) { - INSERT_STRING(s, s->strstart, hash_head); + hash_head = insert_string(s, s->strstart); } } while (--s->prev_length != 0); s->match_available = 0; @@ -2076,7 +2286,7 @@ local block_state deflate_rle(deflate_state *s, int flush) { /* Emit match if have run of MIN_MATCH or longer, else emit literal */ if (s->match_length >= MIN_MATCH) { - check_match(s, s->strstart, s->strstart - 1, s->match_length); + check_match(s, s->strstart, s->strstart - 1, (int)s->match_length); _tr_tally_dist(s, 1, s->match_length - MIN_MATCH, bflush); diff --git a/packages/runtime/vendor/zlib/deflate.h b/packages/runtime/vendor/zlib/deflate.h index 300c6ada6..c55fd8f6a 100644 --- a/packages/runtime/vendor/zlib/deflate.h +++ b/packages/runtime/vendor/zlib/deflate.h @@ -1,5 +1,5 @@ /* deflate.h -- internal compression state - * Copyright (C) 1995-2024 Jean-loup Gailly + * Copyright (C) 1995-2026 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -13,6 +13,10 @@ #ifndef DEFLATE_H #define DEFLATE_H +#if defined(DEFLATE_CHUNK_WRITE_64LE) +#include +#endif + #include "zutil.h" /* define NO_GZIP when compiling if you want to disable gzip header and @@ -25,7 +29,7 @@ /* define LIT_MEM to slightly increase the speed of deflate (order 1% to 2%) at the cost of a larger memory footprint */ -/* #define LIT_MEM */ +#define LIT_MEM /* =========================================================================== * Internal compression state. @@ -52,7 +56,11 @@ #define MAX_BITS 15 /* All codes must not exceed MAX_BITS bits */ +#if defined(DEFLATE_CHUNK_WRITE_64LE) +#define Buf_size 64 +#else #define Buf_size 16 +#endif /* size of bit buffer in bi_buf */ #define INIT_STATE 42 /* zlib header -> BUSY_STATE */ @@ -113,8 +121,8 @@ typedef struct internal_state { ulg gzindex; /* where in extra, name, or comment */ Byte method; /* can only be DEFLATED */ int last_flush; /* value of flush param for previous deflate call */ - - /* used by deflate.c: */ + unsigned crc0[4 * 5]; + /* used by deflate.c: */ uInt w_size; /* LZ77 window size (32K by default) */ uInt w_bits; /* log2(w_size) (8..16) */ @@ -222,11 +230,9 @@ typedef struct internal_state { */ #ifdef LIT_MEM -# define LIT_BUFS 5 ushf *d_buf; /* buffer for distances */ uchf *l_buf; /* buffer for literals/lengths */ #else -# define LIT_BUFS 4 uchf *sym_buf; /* buffer for distances and literals/lengths */ #endif @@ -263,7 +269,11 @@ typedef struct internal_state { ulg bits_sent; /* bit length of compressed data sent mod 2^32 */ #endif +#if defined(DEFLATE_CHUNK_WRITE_64LE) + uint64_t bi_buf; +#else ush bi_buf; +#endif /* Output buffer. bits are inserted starting at the bottom (least * significant bits). */ @@ -271,6 +281,9 @@ typedef struct internal_state { /* Number of valid bits in bi_buf. All bits above the last valid bit * are always zero. */ + int bi_used; + /* Last number of used bits when going to a byte boundary. + */ ulg high_water; /* High water mark offset in window for initialized bytes -- bytes above @@ -279,6 +292,21 @@ typedef struct internal_state { * updated to the new high water mark. */ + uInt chromium_zlib_hash; + /* 0 if Rabin-Karp rolling hash is enabled, non-zero if chromium zlib + * hash is enabled. + */ + +#if defined(QAT_COMPRESSION_ENABLED) + /* Pointer to a struct that contains the current state of the QAT + * stream. + */ + struct qat_deflate *qat_s; +#endif + + int slid; + /* True if the hash table has been slid since it was cleared. */ + } FAR deflate_state; /* Output a byte on the stream. @@ -374,4 +402,12 @@ void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, flush = _tr_tally(s, distance, length) #endif +/* Functions that are SIMD optimised on x86 */ +void ZLIB_INTERNAL crc_fold_init(deflate_state* const s); +void ZLIB_INTERNAL crc_fold_copy(deflate_state* const s, + unsigned char* dst, + const unsigned char* src, + long len); +unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state* const s); + #endif /* DEFLATE_H */ diff --git a/packages/runtime/vendor/zlib/gzguts.h b/packages/runtime/vendor/zlib/gzguts.h index eba72085b..f7946bdc5 100644 --- a/packages/runtime/vendor/zlib/gzguts.h +++ b/packages/runtime/vendor/zlib/gzguts.h @@ -1,5 +1,5 @@ /* gzguts.h -- zlib internal header definitions for gz* operations - * Copyright (C) 2004-2024 Mark Adler + * Copyright (C) 2004-2026 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -17,6 +17,18 @@ # define ZLIB_INTERNAL #endif +#if defined(_WIN32) +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# ifndef _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_WARNINGS +# endif +# ifndef _CRT_NONSTDC_NO_DEPRECATE +# define _CRT_NONSTDC_NO_DEPRECATE +# endif +#endif + #include #include "zlib.h" #ifdef STDC @@ -25,8 +37,8 @@ # include #endif -#ifndef _POSIX_SOURCE -# define _POSIX_SOURCE +#ifndef _POSIX_C_SOURCE +# define _POSIX_C_SOURCE 200112L #endif #include @@ -36,19 +48,13 @@ #if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32) # include +# include #endif -#if defined(_WIN32) +#if defined(_WIN32) && !defined(WIDECHAR) # define WIDECHAR #endif -#ifdef WINAPI_FAMILY -# define open _open -# define read _read -# define write _write -# define close _close -#endif - #ifdef NO_DEFLATE /* for compatibility with old definition */ # define NO_GZCOMPRESS #endif @@ -72,33 +78,28 @@ #endif #ifndef HAVE_VSNPRINTF -# ifdef MSDOS +# if !defined(NO_vsnprintf) && \ + (defined(MSDOS) || defined(__TURBOC__) || defined(__SASC) || \ + defined(VMS) || defined(__OS400) || defined(__MVS__)) /* vsnprintf may exist on some MS-DOS compilers (DJGPP?), but for now we just assume it doesn't. */ # define NO_vsnprintf # endif -# ifdef __TURBOC__ -# define NO_vsnprintf -# endif # ifdef WIN32 /* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ -# if !defined(vsnprintf) && !defined(NO_vsnprintf) -# if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 ) -# define vsnprintf _vsnprintf +# if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 ) +# ifndef vsnprintf +# define vsnprintf _vsnprintf # endif # endif -# endif -# ifdef __SASC -# define NO_vsnprintf -# endif -# ifdef VMS -# define NO_vsnprintf -# endif -# ifdef __OS400__ -# define NO_vsnprintf -# endif -# ifdef __MVS__ -# define NO_vsnprintf +# elif !defined(__STDC_VERSION__) || __STDC_VERSION__-0 < 199901L +/* Otherwise if C89/90, assume no C99 snprintf() or vsnprintf() */ +# ifndef NO_snprintf +# define NO_snprintf +# endif +# ifndef NO_vsnprintf +# define NO_vsnprintf +# endif # endif #endif @@ -135,6 +136,11 @@ # endif #endif +/* some environments don't define EWOULDBLOCK */ +#ifndef EWOULDBLOCK +# define EWOULDBLOCK EAGAIN +#endif + /* provide prototypes for these when building zlib without LFS */ #if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0 ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *); @@ -182,7 +188,9 @@ typedef struct { unsigned char *out; /* output buffer (double-sized when reading) */ int direct; /* 0 if processing gzip, 1 if transparent */ /* just for reading */ + int junk; /* -1 = start, 1 = junk candidate, 0 = in gzip */ int how; /* 0: get header, 1: copy, 2: decompress */ + int again; /* true if EAGAIN or EWOULDBLOCK on last i/o */ z_off64_t start; /* where the gzip data started, for rewinding */ int eof; /* true if end of input file reached */ int past; /* true if read requested past end */ @@ -192,7 +200,6 @@ typedef struct { int reset; /* true if a reset is pending after a Z_FINISH */ /* seek request */ z_off64_t skip; /* amount to skip (already rewound if backwards) */ - int seek; /* true if seek request pending */ /* error information */ int err; /* error code */ char *msg; /* error message */ diff --git a/packages/runtime/vendor/zlib/gzlib.c b/packages/runtime/vendor/zlib/gzlib.c index 983153cc8..2db29c8f0 100644 --- a/packages/runtime/vendor/zlib/gzlib.c +++ b/packages/runtime/vendor/zlib/gzlib.c @@ -1,18 +1,21 @@ /* gzlib.c -- zlib functions common to reading and writing gzip files - * Copyright (C) 2004-2024 Mark Adler + * Copyright (C) 2004-2026 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ #include "gzguts.h" -#if defined(_WIN32) && !defined(__BORLANDC__) +#if defined(__DJGPP__) +# define LSEEK llseek +#elif defined(_WIN32) && !defined(__BORLANDC__) && !defined(UNDER_CE) # define LSEEK _lseeki64 -#else -#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 +# define OPEN open +#elif defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 # define LSEEK lseek64 +# define OPEN open64 #else # define LSEEK lseek -#endif +# define OPEN open #endif #if defined UNDER_CE @@ -52,7 +55,7 @@ char ZLIB_INTERNAL *gz_strwinerror(DWORD error) { msgbuf[chars] = 0; } - wcstombs(buf, msgbuf, chars + 1); + wcstombs(buf, msgbuf, chars + 1); /* assumes buf is big enough */ LocalFree(msgbuf); } else { @@ -72,10 +75,12 @@ local void gz_reset(gz_statep state) { state->eof = 0; /* not at end of file */ state->past = 0; /* have not read past end yet */ state->how = LOOK; /* look for gzip header */ + state->junk = -1; /* mark first member */ } else /* for writing ... */ state->reset = 0; /* no deflateReset pending */ - state->seek = 0; /* no seek request pending */ + state->again = 0; /* no stalled i/o yet */ + state->skip = 0; /* no seek request pending */ gz_error(state, Z_OK, NULL); /* clear error */ state->x.pos = 0; /* no uncompressed data yet */ state->strm.avail_in = 0; /* no input data yet */ @@ -85,16 +90,13 @@ local void gz_reset(gz_statep state) { local gzFile gz_open(const void *path, int fd, const char *mode) { gz_statep state; z_size_t len; - int oflag; -#ifdef O_CLOEXEC - int cloexec = 0; -#endif + int oflag = 0; #ifdef O_EXCL int exclusive = 0; #endif /* check input */ - if (path == NULL) + if (path == NULL || mode == NULL) return NULL; /* allocate gzFile structure to return */ @@ -103,6 +105,7 @@ local gzFile gz_open(const void *path, int fd, const char *mode) { return NULL; state->size = 0; /* no buffers allocated yet */ state->want = GZBUFSIZE; /* requested buffer size */ + state->err = Z_OK; /* no error yet */ state->msg = NULL; /* no error message yet */ /* interpret mode */ @@ -133,7 +136,7 @@ local gzFile gz_open(const void *path, int fd, const char *mode) { break; #ifdef O_CLOEXEC case 'e': - cloexec = 1; + oflag |= O_CLOEXEC; break; #endif #ifdef O_EXCL @@ -153,6 +156,14 @@ local gzFile gz_open(const void *path, int fd, const char *mode) { case 'F': state->strategy = Z_FIXED; break; + case 'G': + state->direct = -1; + break; +#ifdef O_NONBLOCK + case 'N': + oflag |= O_NONBLOCK; + break; +#endif case 'T': state->direct = 1; break; @@ -168,22 +179,30 @@ local gzFile gz_open(const void *path, int fd, const char *mode) { return NULL; } - /* can't force transparent read */ + /* direct is 0, 1 if "T", or -1 if "G" (last "G" or "T" wins) */ if (state->mode == GZ_READ) { - if (state->direct) { + if (state->direct == 1) { + /* can't force a transparent read */ free(state); return NULL; } - state->direct = 1; /* for empty file */ + if (state->direct == 0) + /* default when reading is auto-detect of gzip vs. transparent -- + start with a transparent assumption in case of an empty file */ + state->direct = 1; } + else if (state->direct == -1) { + /* "G" has no meaning when writing -- disallow it */ + free(state); + return NULL; + } + /* if reading, direct == 1 for auto-detect, -1 for gzip only; if writing or + appending, direct == 0 for gzip, 1 for transparent (copy in to out) */ /* save the path name for error messages */ #ifdef WIDECHAR - if (fd == -2) { + if (fd == -2) len = wcstombs(NULL, path, 0); - if (len == (z_size_t)-1) - len = 0; - } else #endif len = strlen((const char *)path); @@ -193,29 +212,29 @@ local gzFile gz_open(const void *path, int fd, const char *mode) { return NULL; } #ifdef WIDECHAR - if (fd == -2) + if (fd == -2) { if (len) wcstombs(state->path, path, len + 1); else *(state->path) = 0; + } else #endif + { #if !defined(NO_snprintf) && !defined(NO_vsnprintf) (void)snprintf(state->path, len + 1, "%s", (const char *)path); #else strcpy(state->path, path); #endif + } /* compute the flags for open() */ - oflag = + oflag |= #ifdef O_LARGEFILE O_LARGEFILE | #endif #ifdef O_BINARY O_BINARY | -#endif -#ifdef O_CLOEXEC - (cloexec ? O_CLOEXEC : 0) | #endif (state->mode == GZ_READ ? O_RDONLY : @@ -228,11 +247,23 @@ local gzFile gz_open(const void *path, int fd, const char *mode) { O_APPEND))); /* open the file with the appropriate flags (or just use fd) */ - state->fd = fd > -1 ? fd : ( + if (fd == -1) + state->fd = OPEN((const char *)path, oflag, 0666); #ifdef WIDECHAR - fd == -2 ? _wopen(path, oflag, 0666) : + else if (fd == -2) + state->fd = _wopen(path, oflag, _S_IREAD | _S_IWRITE); #endif - open((const char *)path, oflag, 0666)); + else { +#ifdef O_NONBLOCK + if (oflag & O_NONBLOCK) + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK); +#endif +#ifdef O_CLOEXEC + if (oflag & O_CLOEXEC) + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | O_CLOEXEC); +#endif + state->fd = fd; + } if (state->fd == -1) { free(state->path); free(state); @@ -359,9 +390,10 @@ z_off64_t ZEXPORT gzseek64(gzFile file, z_off64_t offset, int whence) { /* normalize offset to a SEEK_CUR specification */ if (whence == SEEK_SET) offset -= state->x.pos; - else if (state->seek) - offset += state->skip; - state->seek = 0; + else { + offset += state->past ? 0 : state->skip; + state->skip = 0; + } /* if within raw area while reading, just go there */ if (state->mode == GZ_READ && state->how == COPY && @@ -372,7 +404,7 @@ z_off64_t ZEXPORT gzseek64(gzFile file, z_off64_t offset, int whence) { state->x.have = 0; state->eof = 0; state->past = 0; - state->seek = 0; + state->skip = 0; gz_error(state, Z_OK, NULL); state->strm.avail_in = 0; state->x.pos += offset; @@ -401,10 +433,7 @@ z_off64_t ZEXPORT gzseek64(gzFile file, z_off64_t offset, int whence) { } /* request skip (if not zero) */ - if (offset) { - state->seek = 1; - state->skip = offset; - } + state->skip = offset; return state->x.pos + offset; } @@ -428,7 +457,7 @@ z_off64_t ZEXPORT gztell64(gzFile file) { return -1; /* return position */ - return state->x.pos + (state->seek ? state->skip : 0); + return state->x.pos + (state->past ? 0 : state->skip); } /* -- see zlib.h -- */ @@ -535,7 +564,7 @@ void ZLIB_INTERNAL gz_error(gz_statep state, int err, const char *msg) { } /* if fatal, set state->x.have to 0 so that the gzgetc() macro fails */ - if (err != Z_OK && err != Z_BUF_ERROR) + if (err != Z_OK && err != Z_BUF_ERROR && !state->again) state->x.have = 0; /* set error code, and if no message, then done */ @@ -572,6 +601,7 @@ unsigned ZLIB_INTERNAL gz_intmax(void) { return INT_MAX; #else unsigned p = 1, q; + do { q = p; p <<= 1; diff --git a/packages/runtime/vendor/zlib/gzread.c b/packages/runtime/vendor/zlib/gzread.c index 4168cbc88..bf41d63bc 100644 --- a/packages/runtime/vendor/zlib/gzread.c +++ b/packages/runtime/vendor/zlib/gzread.c @@ -1,5 +1,5 @@ /* gzread.c -- zlib functions for reading gzip files - * Copyright (C) 2004-2017 Mark Adler + * Copyright (C) 2004-2026 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -8,23 +8,36 @@ /* Use read() to load a buffer -- return -1 on error, otherwise 0. Read from state->fd, and update state->eof, state->err, and state->msg as appropriate. This function needs to loop on read(), since read() is not guaranteed to - read the number of bytes requested, depending on the type of descriptor. */ + read the number of bytes requested, depending on the type of descriptor. It + also needs to loop to manage the fact that read() returns an int. If the + descriptor is non-blocking and read() returns with no data in order to avoid + blocking, then gz_load() will return 0 if some data has been read, or -1 if + no data has been read. Either way, state->again is set true to indicate a + non-blocking event. If errno is non-zero on return, then there was an error + signaled from read(). *have is set to the number of bytes read. */ local int gz_load(gz_statep state, unsigned char *buf, unsigned len, unsigned *have) { int ret; unsigned get, max = ((unsigned)-1 >> 2) + 1; + state->again = 0; + errno = 0; *have = 0; do { get = len - *have; if (get > max) get = max; - ret = read(state->fd, buf + *have, get); + ret = (int)read(state->fd, buf + *have, get); if (ret <= 0) break; *have += (unsigned)ret; } while (*have < len); if (ret < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + state->again = 1; + if (*have != 0) + return 0; + } gz_error(state, Z_ERRNO, zstrerror()); return -1; } @@ -50,10 +63,14 @@ local int gz_avail(gz_statep state) { if (strm->avail_in) { /* copy what's there to the start */ unsigned char *p = state->in; unsigned const char *q = strm->next_in; - unsigned n = strm->avail_in; - do { - *p++ = *q++; - } while (--n); + + if (q != p) { + unsigned n = strm->avail_in; + + do { + *p++ = *q++; + } while (--n); + } } if (gz_load(state, state->in + strm->avail_in, state->size - strm->avail_in, &got) == -1) @@ -104,39 +121,44 @@ local int gz_look(gz_statep state) { } } - /* get at least the magic bytes in the input buffer */ - if (strm->avail_in < 2) { - if (gz_avail(state) == -1) - return -1; - if (strm->avail_in == 0) - return 0; - } - - /* look for gzip magic bytes -- if there, do gzip decoding (note: there is - a logical dilemma here when considering the case of a partially written - gzip file, to wit, if a single 31 byte is written, then we cannot tell - whether this is a single-byte file, or just a partially written gzip - file -- for here we assume that if a gzip file is being written, then - the header will be written in a single operation, so that reading a - single byte is sufficient indication that it is not a gzip file) */ - if (strm->avail_in > 1 && - strm->next_in[0] == 31 && strm->next_in[1] == 139) { + /* if transparent reading is disabled, which would only be at the start, or + if we're looking for a gzip member after the first one, which is not at + the start, then proceed directly to look for a gzip member next */ + if (state->direct == -1 || state->junk == 0) { inflateReset(strm); state->how = GZIP; + state->junk = state->junk != -1; state->direct = 0; return 0; } - /* no gzip header -- if we were decoding gzip before, then this is trailing - garbage. Ignore the trailing garbage and finish. */ - if (state->direct == 0) { - strm->avail_in = 0; - state->eof = 1; - state->x.have = 0; + /* otherwise we're at the start with auto-detect -- we check to see if the + first four bytes could be gzip header in order to decide whether or not + this will be a transparent read */ + + /* load any header bytes into the input buffer -- if the input is empty, + then it's not an error as this is a transparent read of zero bytes */ + if (gz_avail(state) == -1) + return -1; + if (strm->avail_in == 0 || (state->again && strm->avail_in < 4)) + /* if non-blocking input stalled before getting four bytes, then + return and wait until a later call has accumulated enough */ + return 0; + + /* see if this is (likely) gzip input -- if the first four bytes are + consistent with a gzip header, then go look for the first gzip member, + otherwise proceed to copy the input transparently */ + if (strm->avail_in > 3 && + strm->next_in[0] == 31 && strm->next_in[1] == 139 && + strm->next_in[2] == 8 && strm->next_in[3] < 32) { + inflateReset(strm); + state->how = GZIP; + state->junk = 1; + state->direct = 0; return 0; } - /* doing raw i/o, copy any leftover input to output -- this assumes that + /* doing raw i/o: copy any leftover input to output -- this assumes that the output buffer is larger than the input buffer, which also assures space for gzungetc() */ state->x.next = state->out; @@ -144,15 +166,17 @@ local int gz_look(gz_statep state) { state->x.have = strm->avail_in; strm->avail_in = 0; state->how = COPY; - state->direct = 1; return 0; } /* Decompress from input to the provided next_out and avail_out in the state. On return, state->x.have and state->x.next point to the just decompressed - data. If the gzip stream completes, state->how is reset to LOOK to look for - the next gzip stream or raw data, once state->x.have is depleted. Returns 0 - on success, -1 on failure. */ + data. If the gzip stream completes, state->how is reset to LOOK to look for + the next gzip stream or raw data, once state->x.have is depleted. Returns 0 + on success, -1 on failure. If EOF is reached when looking for more input to + complete the gzip member, then an unexpected end of file error is raised. + If there is no more input, but state->again is true, then EOF has not been + reached, and no error is raised. */ local int gz_decomp(gz_statep state) { int ret = Z_OK; unsigned had; @@ -162,28 +186,41 @@ local int gz_decomp(gz_statep state) { had = strm->avail_out; do { /* get more input for inflate() */ - if (strm->avail_in == 0 && gz_avail(state) == -1) - return -1; + if (strm->avail_in == 0 && gz_avail(state) == -1) { + ret = state->err; + break; + } if (strm->avail_in == 0) { - gz_error(state, Z_BUF_ERROR, "unexpected end of file"); + if (!state->again) + gz_error(state, Z_BUF_ERROR, "unexpected end of file"); break; } /* decompress and handle errors */ ret = inflate(strm, Z_NO_FLUSH); + if (strm->avail_out < had) + /* any decompressed data marks this as a real gzip stream */ + state->junk = 0; if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT) { gz_error(state, Z_STREAM_ERROR, "internal error: inflate stream corrupt"); - return -1; + break; } if (ret == Z_MEM_ERROR) { gz_error(state, Z_MEM_ERROR, "out of memory"); - return -1; + break; } if (ret == Z_DATA_ERROR) { /* deflate stream invalid */ + if (state->junk == 1) { /* trailing garbage is ok */ + strm->avail_in = 0; + state->eof = 1; + state->how = LOOK; + ret = Z_OK; + break; + } gz_error(state, Z_DATA_ERROR, strm->msg == NULL ? "compressed data error" : strm->msg); - return -1; + break; } } while (strm->avail_out && ret != Z_STREAM_END); @@ -192,11 +229,14 @@ local int gz_decomp(gz_statep state) { state->x.next = strm->next_out - state->x.have; /* if the gzip stream completed successfully, look for another */ - if (ret == Z_STREAM_END) + if (ret == Z_STREAM_END) { + state->junk = 0; state->how = LOOK; + return 0; + } - /* good decompression */ - return 0; + /* return decompression status */ + return ret != Z_OK ? -1 : 0; } /* Fetch data and put it in the output buffer. Assumes state->x.have is 0. @@ -227,25 +267,31 @@ local int gz_fetch(gz_statep state) { strm->next_out = state->out; if (gz_decomp(state) == -1) return -1; + break; + default: + gz_error(state, Z_STREAM_ERROR, "state corrupt"); + return -1; } } while (state->x.have == 0 && (!state->eof || strm->avail_in)); return 0; } -/* Skip len uncompressed bytes of output. Return -1 on error, 0 on success. */ -local int gz_skip(gz_statep state, z_off64_t len) { +/* Skip state->skip (> 0) uncompressed bytes of output. Return -1 on error, 0 + on success. */ +local int gz_skip(gz_statep state) { unsigned n; /* skip over len bytes or reach end-of-file, whichever comes first */ - while (len) + do { /* skip over whatever is in output buffer */ if (state->x.have) { - n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > len ? - (unsigned)len : state->x.have; + n = GT_OFF(state->x.have) || + (z_off64_t)state->x.have > state->skip ? + (unsigned)state->skip : state->x.have; state->x.have -= n; state->x.next += n; state->x.pos += n; - len -= n; + state->skip -= n; } /* output buffer empty -- return if we're at the end of the input */ @@ -258,30 +304,32 @@ local int gz_skip(gz_statep state, z_off64_t len) { if (gz_fetch(state) == -1) return -1; } + } while (state->skip); return 0; } /* Read len bytes into buf from file, or less than len up to the end of the - input. Return the number of bytes read. If zero is returned, either the - end of file was reached, or there was an error. state->err must be - consulted in that case to determine which. */ + input. Return the number of bytes read. If zero is returned, either the end + of file was reached, or there was an error. state->err must be consulted in + that case to determine which. If there was an error, but some uncompressed + bytes were read before the error, then that count is returned. The error is + still recorded, and so is deferred until the next call. */ local z_size_t gz_read(gz_statep state, voidp buf, z_size_t len) { z_size_t got; unsigned n; + int err; /* if len is zero, avoid unnecessary operations */ if (len == 0) return 0; /* process a skip request */ - if (state->seek) { - state->seek = 0; - if (gz_skip(state, state->skip) == -1) - return 0; - } + if (state->skip && gz_skip(state) == -1) + return 0; /* get len bytes to buf, or less than len if at the end */ got = 0; + err = 0; do { /* set n to the maximum amount of len that fits in an unsigned int */ n = (unsigned)-1; @@ -295,37 +343,36 @@ local z_size_t gz_read(gz_statep state, voidp buf, z_size_t len) { memcpy(buf, state->x.next, n); state->x.next += n; state->x.have -= n; + if (state->err != Z_OK) + /* caught deferred error from gz_fetch() */ + err = -1; } /* output buffer empty -- return if we're at the end of the input */ - else if (state->eof && state->strm.avail_in == 0) { - state->past = 1; /* tried to read past end */ + else if (state->eof && state->strm.avail_in == 0) break; - } /* need output data -- for small len or new stream load up our output - buffer */ + buffer, so that gzgetc() can be fast */ else if (state->how == LOOK || n < (state->size << 1)) { /* get more output, looking for header if required */ - if (gz_fetch(state) == -1) - return 0; + if (gz_fetch(state) == -1 && state->x.have == 0) + /* if state->x.have != 0, error will be caught after copy */ + err = -1; continue; /* no progress yet -- go back to copy above */ /* the copy above assures that we will leave with space in the output buffer, allowing at least one gzungetc() to succeed */ } /* large len -- read directly into user buffer */ - else if (state->how == COPY) { /* read directly */ - if (gz_load(state, (unsigned char *)buf, n, &n) == -1) - return 0; - } + else if (state->how == COPY) /* read directly */ + err = gz_load(state, (unsigned char *)buf, n, &n); /* large len -- decompress directly into user buffer */ else { /* state->how == GZIP */ state->strm.avail_out = n; state->strm.next_out = (unsigned char *)buf; - if (gz_decomp(state) == -1) - return 0; + err = gz_decomp(state); n = state->x.have; state->x.have = 0; } @@ -335,7 +382,11 @@ local z_size_t gz_read(gz_statep state, voidp buf, z_size_t len) { buf = (char *)buf + n; got += n; state->x.pos += n; - } while (len); + } while (len && !err); + + /* note read past eof */ + if (len && state->eof) + state->past = 1; /* return number of bytes read into user buffer */ return got; @@ -345,16 +396,18 @@ local z_size_t gz_read(gz_statep state, voidp buf, z_size_t len) { int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { gz_statep state; - /* get internal structure */ + /* get internal structure and check that it's for reading */ if (file == NULL) return -1; state = (gz_statep)file; - - /* check that we're reading and that there's no (serious) error */ - if (state->mode != GZ_READ || - (state->err != Z_OK && state->err != Z_BUF_ERROR)) + if (state->mode != GZ_READ) return -1; + /* check that there was no (serious) error */ + if (state->err != Z_OK && state->err != Z_BUF_ERROR && !state->again) + return -1; + gz_error(state, Z_OK, NULL); + /* since an int is returned, make sure len fits in one, otherwise return with an error (this avoids a flaw in the interface) */ if ((int)len < 0) { @@ -366,28 +419,40 @@ int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { len = (unsigned)gz_read(state, buf, len); /* check for an error */ - if (len == 0 && state->err != Z_OK && state->err != Z_BUF_ERROR) - return -1; + if (len == 0) { + if (state->err != Z_OK && state->err != Z_BUF_ERROR) + return -1; + if (state->again) { + /* non-blocking input stalled after some input was read, but no + uncompressed bytes were produced -- let the application know + this isn't EOF */ + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + } - /* return the number of bytes read (this is assured to fit in an int) */ + /* return the number of bytes read */ return (int)len; } /* -- see zlib.h -- */ -z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, gzFile file) { +z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, + gzFile file) { z_size_t len; gz_statep state; - /* get internal structure */ + /* get internal structure and check that it's for reading */ if (file == NULL) return 0; state = (gz_statep)file; - - /* check that we're reading and that there's no (serious) error */ - if (state->mode != GZ_READ || - (state->err != Z_OK && state->err != Z_BUF_ERROR)) + if (state->mode != GZ_READ) return 0; + /* check that there was no (serious) error */ + if (state->err != Z_OK && state->err != Z_BUF_ERROR && !state->again) + return 0; + gz_error(state, Z_OK, NULL); + /* compute bytes to read -- error on overflow */ len = nitems * size; if (size && len / size != nitems) { @@ -404,21 +469,27 @@ z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, gzFile file) # undef z_gzgetc #else # undef gzgetc +# ifdef Z_CR_PREFIX_SET +# define gzgetc Cr_z_gzgetc +# endif #endif + int ZEXPORT gzgetc(gzFile file) { unsigned char buf[1]; gz_statep state; - /* get internal structure */ + /* get internal structure and check that it's for reading */ if (file == NULL) return -1; state = (gz_statep)file; - - /* check that we're reading and that there's no (serious) error */ - if (state->mode != GZ_READ || - (state->err != Z_OK && state->err != Z_BUF_ERROR)) + if (state->mode != GZ_READ) return -1; + /* check that there was no (serious) error */ + if (state->err != Z_OK && state->err != Z_BUF_ERROR && !state->again) + return -1; + gz_error(state, Z_OK, NULL); + /* try output buffer (no need to check for skip request) */ if (state->x.have) { state->x.have--; @@ -438,26 +509,25 @@ int ZEXPORT gzgetc_(gzFile file) { int ZEXPORT gzungetc(int c, gzFile file) { gz_statep state; - /* get internal structure */ + /* get internal structure and check that it's for reading */ if (file == NULL) return -1; state = (gz_statep)file; - - /* in case this was just opened, set up the input buffer */ - if (state->mode == GZ_READ && state->how == LOOK && state->x.have == 0) - (void)gz_look(state); - - /* check that we're reading and that there's no (serious) error */ - if (state->mode != GZ_READ || - (state->err != Z_OK && state->err != Z_BUF_ERROR)) + if (state->mode != GZ_READ) return -1; + /* in case this was just opened, set up the input buffer */ + if (state->how == LOOK && state->x.have == 0) + (void)gz_look(state); + + /* check that there was no (serious) error */ + if (state->err != Z_OK && state->err != Z_BUF_ERROR && !state->again) + return -1; + gz_error(state, Z_OK, NULL); + /* process a skip request */ - if (state->seek) { - state->seek = 0; - if (gz_skip(state, state->skip) == -1) - return -1; - } + if (state->skip && gz_skip(state) == -1) + return -1; /* can't push EOF */ if (c < 0) @@ -483,6 +553,7 @@ int ZEXPORT gzungetc(int c, gzFile file) { if (state->x.next == state->out) { unsigned char *src = state->out + state->x.have; unsigned char *dest = state->out + (state->size << 1); + while (src > state->out) *--dest = *--src; state->x.next = dest; @@ -502,32 +573,31 @@ char * ZEXPORT gzgets(gzFile file, char *buf, int len) { unsigned char *eol; gz_statep state; - /* check parameters and get internal structure */ + /* check parameters, get internal structure, and check that it's for + reading */ if (file == NULL || buf == NULL || len < 1) return NULL; state = (gz_statep)file; - - /* check that we're reading and that there's no (serious) error */ - if (state->mode != GZ_READ || - (state->err != Z_OK && state->err != Z_BUF_ERROR)) + if (state->mode != GZ_READ) return NULL; - /* process a skip request */ - if (state->seek) { - state->seek = 0; - if (gz_skip(state, state->skip) == -1) - return NULL; - } + /* check that there was no (serious) error */ + if (state->err != Z_OK && state->err != Z_BUF_ERROR && !state->again) + return NULL; + gz_error(state, Z_OK, NULL); - /* copy output bytes up to new line or len - 1, whichever comes first -- - append a terminating zero to the string (we don't check for a zero in - the contents, let the user worry about that) */ + /* process a skip request */ + if (state->skip && gz_skip(state) == -1) + return NULL; + + /* copy output up to a new line, len-1 bytes, or there is no more output, + whichever comes first */ str = buf; left = (unsigned)len - 1; if (left) do { /* assure that something is in the output buffer */ if (state->x.have == 0 && gz_fetch(state) == -1) - return NULL; /* error */ + break; /* error */ if (state->x.have == 0) { /* end of file */ state->past = 1; /* read past end */ break; /* return what we have */ @@ -548,7 +618,9 @@ char * ZEXPORT gzgets(gzFile file, char *buf, int len) { buf += n; } while (left && eol == NULL); - /* return terminated string, or if nothing, end of file */ + /* append a terminating zero to the string (we don't check for a zero in + the contents, let the user worry about that) -- return the terminated + string, or if nothing was read, NULL */ if (buf == str) return NULL; buf[0] = 0; @@ -570,7 +642,7 @@ int ZEXPORT gzdirect(gzFile file) { (void)gz_look(state); /* return 1 if transparent, 0 if processing a gzip stream */ - return state->direct; + return state->direct == 1; } /* -- see zlib.h -- */ @@ -578,12 +650,10 @@ int ZEXPORT gzclose_r(gzFile file) { int ret, err; gz_statep state; - /* get internal structure */ + /* get internal structure and check that it's for reading */ if (file == NULL) return Z_STREAM_ERROR; state = (gz_statep)file; - - /* check that we're reading */ if (state->mode != GZ_READ) return Z_STREAM_ERROR; diff --git a/packages/runtime/vendor/zlib/gzwrite.c b/packages/runtime/vendor/zlib/gzwrite.c index 435b4621b..13a3700a8 100644 --- a/packages/runtime/vendor/zlib/gzwrite.c +++ b/packages/runtime/vendor/zlib/gzwrite.c @@ -1,5 +1,5 @@ /* gzwrite.c -- zlib functions for writing gzip files - * Copyright (C) 2004-2019 Mark Adler + * Copyright (C) 2004-2026 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -74,9 +74,13 @@ local int gz_comp(gz_statep state, int flush) { /* write directly if requested */ if (state->direct) { while (strm->avail_in) { + errno = 0; + state->again = 0; put = strm->avail_in > max ? max : strm->avail_in; - writ = write(state->fd, strm->next_in, put); + writ = (int)write(state->fd, strm->next_in, put); if (writ < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) + state->again = 1; gz_error(state, Z_ERRNO, zstrerror()); return -1; } @@ -88,8 +92,9 @@ local int gz_comp(gz_statep state, int flush) { /* check for a pending reset */ if (state->reset) { - /* don't start a new gzip member unless there is data to write */ - if (strm->avail_in == 0) + /* don't start a new gzip member unless there is data to write and + we're not flushing */ + if (strm->avail_in == 0 && flush == Z_NO_FLUSH) return 0; deflateReset(strm); state->reset = 0; @@ -103,10 +108,14 @@ local int gz_comp(gz_statep state, int flush) { if (strm->avail_out == 0 || (flush != Z_NO_FLUSH && (flush != Z_FINISH || ret == Z_STREAM_END))) { while (strm->next_out > state->x.next) { + errno = 0; + state->again = 0; put = strm->next_out - state->x.next > (int)max ? max : (unsigned)(strm->next_out - state->x.next); - writ = write(state->fd, state->x.next, put); + writ = (int)write(state->fd, state->x.next, put); if (writ < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) + state->again = 1; gz_error(state, Z_ERRNO, zstrerror()); return -1; } @@ -138,10 +147,12 @@ local int gz_comp(gz_statep state, int flush) { return 0; } -/* Compress len zeros to output. Return -1 on a write error or memory - allocation failure by gz_comp(), or 0 on success. */ -local int gz_zero(gz_statep state, z_off64_t len) { - int first; +/* Compress state->skip (> 0) zeros to output. Return -1 on a write error or + memory allocation failure by gz_comp(), or 0 on success. state->skip is + updated with the number of successfully written zeros, in case there is a + stall on a non-blocking write destination. */ +local int gz_zero(gz_statep state) { + int first, ret; unsigned n; z_streamp strm = &(state->strm); @@ -149,29 +160,34 @@ local int gz_zero(gz_statep state, z_off64_t len) { if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) return -1; - /* compress len zeros (len guaranteed > 0) */ + /* compress state->skip zeros */ first = 1; - while (len) { - n = GT_OFF(state->size) || (z_off64_t)state->size > len ? - (unsigned)len : state->size; + do { + n = GT_OFF(state->size) || (z_off64_t)state->size > state->skip ? + (unsigned)state->skip : state->size; if (first) { memset(state->in, 0, n); first = 0; } strm->avail_in = n; strm->next_in = state->in; + ret = gz_comp(state, Z_NO_FLUSH); + n -= strm->avail_in; state->x.pos += n; - if (gz_comp(state, Z_NO_FLUSH) == -1) + state->skip -= n; + if (ret == -1) return -1; - len -= n; - } + } while (state->skip); return 0; } /* Write len bytes from buf to file. Return the number of bytes written. If - the returned value is less than len, then there was an error. */ + the returned value is less than len, then there was an error. If the error + was a non-blocking stall, then the number of bytes consumed is returned. + For any other error, 0 is returned. */ local z_size_t gz_write(gz_statep state, voidpc buf, z_size_t len) { z_size_t put = len; + int ret; /* if len is zero, avoid unnecessary operations */ if (len == 0) @@ -182,16 +198,13 @@ local z_size_t gz_write(gz_statep state, voidpc buf, z_size_t len) { return 0; /* check for seek request */ - if (state->seek) { - state->seek = 0; - if (gz_zero(state, state->skip) == -1) - return 0; - } + if (state->skip && gz_zero(state) == -1) + return 0; /* for small len, copy to input buffer, otherwise compress directly */ if (len < state->size) { /* copy to input buffer, compress when full */ - do { + for (;;) { unsigned have, copy; if (state->strm.avail_in == 0) @@ -206,9 +219,11 @@ local z_size_t gz_write(gz_statep state, voidpc buf, z_size_t len) { state->x.pos += copy; buf = (const char *)buf + copy; len -= copy; - if (len && gz_comp(state, Z_NO_FLUSH) == -1) - return 0; - } while (len); + if (len == 0) + break; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return state->again ? put - len : 0; + } } else { /* consume whatever's left in the input buffer */ @@ -219,13 +234,16 @@ local z_size_t gz_write(gz_statep state, voidpc buf, z_size_t len) { state->strm.next_in = (z_const Bytef *)buf; do { unsigned n = (unsigned)-1; + if (n > len) n = (unsigned)len; state->strm.avail_in = n; + ret = gz_comp(state, Z_NO_FLUSH); + n -= state->strm.avail_in; state->x.pos += n; - if (gz_comp(state, Z_NO_FLUSH) == -1) - return 0; len -= n; + if (ret == -1) + return state->again ? put - len : 0; } while (len); } @@ -242,9 +260,10 @@ int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len) { return 0; state = (gz_statep)file; - /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK) + /* check that we're writing and that there's no (serious) error */ + if (state->mode != GZ_WRITE || (state->err != Z_OK && !state->again)) return 0; + gz_error(state, Z_OK, NULL); /* since an int is returned, make sure len fits in one, otherwise return with an error (this avoids a flaw in the interface) */ @@ -268,9 +287,10 @@ z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size, z_size_t nitems, return 0; state = (gz_statep)file; - /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK) + /* check that we're writing and that there's no (serious) error */ + if (state->mode != GZ_WRITE || (state->err != Z_OK && !state->again)) return 0; + gz_error(state, Z_OK, NULL); /* compute bytes to read -- error on overflow */ len = nitems * size; @@ -296,16 +316,14 @@ int ZEXPORT gzputc(gzFile file, int c) { state = (gz_statep)file; strm = &(state->strm); - /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK) + /* check that we're writing and that there's no (serious) error */ + if (state->mode != GZ_WRITE || (state->err != Z_OK && !state->again)) return -1; + gz_error(state, Z_OK, NULL); /* check for seek request */ - if (state->seek) { - state->seek = 0; - if (gz_zero(state, state->skip) == -1) - return -1; - } + if (state->skip && gz_zero(state) == -1) + return -1; /* try writing to input buffer for speed (state->size == 0 if buffer not initialized) */ @@ -338,9 +356,10 @@ int ZEXPORT gzputs(gzFile file, const char *s) { return -1; state = (gz_statep)file; - /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK) + /* check that we're writing and that there's no (serious) error */ + if (state->mode != GZ_WRITE || (state->err != Z_OK && !state->again)) return -1; + gz_error(state, Z_OK, NULL); /* write string */ len = strlen(s); @@ -349,16 +368,47 @@ int ZEXPORT gzputs(gzFile file, const char *s) { return -1; } put = gz_write(state, s, len); - return put < len ? -1 : (int)len; + return len && put == 0 ? -1 : (int)put; } +#if (((!defined(STDC) && !defined(Z_HAVE_STDARG_H)) || !defined(NO_vsnprintf)) && \ + (defined(STDC) || defined(Z_HAVE_STDARG_H) || !defined(NO_snprintf))) || \ + defined(ZLIB_INSECURE) +/* If the second half of the input buffer is occupied, write out the contents. + If there is any input remaining due to a non-blocking stall on write, move + it to the start of the buffer. Return true if this did not open up the + second half of the buffer. state->err should be checked after this to + handle a gz_comp() error. */ +local int gz_vacate(gz_statep state) { + z_streamp strm; + + strm = &(state->strm); + if (strm->next_in + strm->avail_in <= state->in + state->size) + return 0; + (void)gz_comp(state, Z_NO_FLUSH); + if (strm->avail_in == 0) { + strm->next_in = state->in; + return 0; + } + memmove(state->in, strm->next_in, strm->avail_in); + strm->next_in = state->in; + return strm->avail_in > state->size; +} +#endif + #if defined(STDC) || defined(Z_HAVE_STDARG_H) #include /* -- see zlib.h -- */ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) { - int len; - unsigned left; +#if defined(NO_vsnprintf) && !defined(ZLIB_INSECURE) +#warning "vsnprintf() not available -- gzprintf() stub returns Z_STREAM_ERROR" +#warning "you can recompile with ZLIB_INSECURE defined to use vsprintf()" + /* prevent use of insecure vsprintf(), unless purposefully requested */ + (void)file, (void)format, (void)va; + return Z_STREAM_ERROR; +#else + int len, ret; char *next; gz_statep state; z_streamp strm; @@ -369,24 +419,34 @@ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) { state = (gz_statep)file; strm = &(state->strm); - /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK) + /* check that we're writing and that there's no (serious) error */ + if (state->mode != GZ_WRITE || (state->err != Z_OK && !state->again)) return Z_STREAM_ERROR; + gz_error(state, Z_OK, NULL); /* make sure we have some buffer space */ if (state->size == 0 && gz_init(state) == -1) return state->err; /* check for seek request */ - if (state->seek) { - state->seek = 0; - if (gz_zero(state, state->skip) == -1) - return state->err; - } + if (state->skip && gz_zero(state) == -1) + return state->err; /* do the printf() into the input buffer, put length in len -- the input - buffer is double-sized just for this function, so there is guaranteed to - be state->size bytes available after the current contents */ + buffer is double-sized just for this function, so there should be + state->size bytes available after the current contents */ + ret = gz_vacate(state); + if (state->err) { + if (ret && state->again) { + /* There was a non-blocking stall on write, resulting in the part + of the second half of the output buffer being occupied. Return + a Z_BUF_ERROR to let the application know that this gzprintf() + needs to be retried. */ + gz_error(state, Z_BUF_ERROR, "stalled write on gzprintf"); + } + if (!state->again) + return state->err; + } if (strm->avail_in == 0) strm->next_in = state->in; next = (char *)(state->in + (strm->next_in - state->in) + strm->avail_in); @@ -412,19 +472,16 @@ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) { if (len == 0 || (unsigned)len >= state->size || next[state->size - 1] != 0) return 0; - /* update buffer and position, compress first half if past that */ + /* update buffer and position */ strm->avail_in += (unsigned)len; state->x.pos += len; - if (strm->avail_in >= state->size) { - left = strm->avail_in - state->size; - strm->avail_in = state->size; - if (gz_comp(state, Z_NO_FLUSH) == -1) - return state->err; - memmove(state->in, state->in + state->size, left); - strm->next_in = state->in; - strm->avail_in = left; - } + + /* write out buffer if more than half is occupied */ + ret = gz_vacate(state); + if (state->err && !state->again) + return state->err; return len; +#endif } int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) { @@ -444,6 +501,17 @@ int ZEXPORTVA gzprintf(gzFile file, const char *format, int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8, int a9, int a10, int a11, int a12, int a13, int a14, int a15, int a16, int a17, int a18, int a19, int a20) { +#if defined(NO_snprintf) && !defined(ZLIB_INSECURE) +#warning "snprintf() not available -- gzprintf() stub returns Z_STREAM_ERROR" +#warning "you can recompile with ZLIB_INSECURE defined to use sprintf()" + /* prevent use of insecure sprintf(), unless purposefully requested */ + (void)file, (void)format, (void)a1, (void)a2, (void)a3, (void)a4, (void)a5, + (void)a6, (void)a7, (void)a8, (void)a9, (void)a10, (void)a11, (void)a12, + (void)a13, (void)a14, (void)a15, (void)a16, (void)a17, (void)a18, + (void)a19, (void)a20; + return Z_STREAM_ERROR; +#else + int ret; unsigned len, left; char *next; gz_statep state; @@ -459,24 +527,34 @@ int ZEXPORTVA gzprintf(gzFile file, const char *format, int a1, int a2, int a3, if (sizeof(int) != sizeof(void *)) return Z_STREAM_ERROR; - /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK) + /* check that we're writing and that there's no (serious) error */ + if (state->mode != GZ_WRITE || (state->err != Z_OK && !state->again)) return Z_STREAM_ERROR; + gz_error(state, Z_OK, NULL); /* make sure we have some buffer space */ if (state->size == 0 && gz_init(state) == -1) - return state->error; + return state->err; /* check for seek request */ - if (state->seek) { - state->seek = 0; - if (gz_zero(state, state->skip) == -1) - return state->error; - } + if (state->skip && gz_zero(state) == -1) + return state->err; /* do the printf() into the input buffer, put length in len -- the input buffer is double-sized just for this function, so there is guaranteed to be state->size bytes available after the current contents */ + ret = gz_vacate(state); + if (state->err) { + if (ret && state->again) { + /* There was a non-blocking stall on write, resulting in the part + of the second half of the output buffer being occupied. Return + a Z_BUF_ERROR to let the application know that this gzprintf() + needs to be retried. */ + gz_error(state, Z_BUF_ERROR, "stalled write on gzprintf"); + } + if (!state->again) + return state->err; + } if (strm->avail_in == 0) strm->next_in = state->in; next = (char *)(strm->next_in + strm->avail_in); @@ -510,16 +588,13 @@ int ZEXPORTVA gzprintf(gzFile file, const char *format, int a1, int a2, int a3, /* update buffer and position, compress first half if past that */ strm->avail_in += len; state->x.pos += len; - if (strm->avail_in >= state->size) { - left = strm->avail_in - state->size; - strm->avail_in = state->size; - if (gz_comp(state, Z_NO_FLUSH) == -1) - return state->err; - memmove(state->in, state->in + state->size, left); - strm->next_in = state->in; - strm->avail_in = left; - } + + /* write out buffer if more than half is occupied */ + ret = gz_vacate(state); + if (state->err && !state->again) + return state->err; return (int)len; +#endif } #endif @@ -533,20 +608,18 @@ int ZEXPORT gzflush(gzFile file, int flush) { return Z_STREAM_ERROR; state = (gz_statep)file; - /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK) + /* check that we're writing and that there's no (serious) error */ + if (state->mode != GZ_WRITE || (state->err != Z_OK && !state->again)) return Z_STREAM_ERROR; + gz_error(state, Z_OK, NULL); /* check flush parameter */ if (flush < 0 || flush > Z_FINISH) return Z_STREAM_ERROR; /* check for seek request */ - if (state->seek) { - state->seek = 0; - if (gz_zero(state, state->skip) == -1) - return state->err; - } + if (state->skip && gz_zero(state) == -1) + return state->err; /* compress remaining data with requested flush */ (void)gz_comp(state, flush); @@ -564,20 +637,19 @@ int ZEXPORT gzsetparams(gzFile file, int level, int strategy) { state = (gz_statep)file; strm = &(state->strm); - /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK || state->direct) + /* check that we're compressing and that there's no (serious) error */ + if (state->mode != GZ_WRITE || (state->err != Z_OK && !state->again) || + state->direct) return Z_STREAM_ERROR; + gz_error(state, Z_OK, NULL); /* if no change is requested, then do nothing */ if (level == state->level && strategy == state->strategy) return Z_OK; /* check for seek request */ - if (state->seek) { - state->seek = 0; - if (gz_zero(state, state->skip) == -1) - return state->err; - } + if (state->skip && gz_zero(state) == -1) + return state->err; /* change compression parameters for subsequent input */ if (state->size) { @@ -606,11 +678,8 @@ int ZEXPORT gzclose_w(gzFile file) { return Z_STREAM_ERROR; /* check for seek request */ - if (state->seek) { - state->seek = 0; - if (gz_zero(state, state->skip) == -1) - ret = state->err; - } + if (state->skip && gz_zero(state) == -1) + ret = state->err; /* flush, free memory, and close file */ if (gz_comp(state, Z_FINISH) == -1) diff --git a/packages/runtime/vendor/zlib/infback.c b/packages/runtime/vendor/zlib/infback.c index e7b25b307..7ffb7b044 100644 --- a/packages/runtime/vendor/zlib/infback.c +++ b/packages/runtime/vendor/zlib/infback.c @@ -1,5 +1,5 @@ /* infback.c -- inflate using a call-back interface - * Copyright (C) 1995-2022 Mark Adler + * Copyright (C) 1995-2026 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -46,7 +46,7 @@ int ZEXPORT inflateBackInit_(z_streamp strm, int windowBits, #ifdef Z_SOLO return Z_STREAM_ERROR; #else - strm->zfree = zcfree; + strm->zfree = zcfree; #endif state = (struct inflate_state FAR *)ZALLOC(strm, 1, sizeof(struct inflate_state)); @@ -63,57 +63,6 @@ int ZEXPORT inflateBackInit_(z_streamp strm, int windowBits, return Z_OK; } -/* - Return state with length and distance decoding tables and index sizes set to - fixed code decoding. Normally this returns fixed tables from inffixed.h. - If BUILDFIXED is defined, then instead this routine builds the tables the - first time it's called, and returns those tables the first time and - thereafter. This reduces the size of the code by about 2K bytes, in - exchange for a little execution time. However, BUILDFIXED should not be - used for threaded applications, since the rewriting of the tables and virgin - may not be thread-safe. - */ -local void fixedtables(struct inflate_state FAR *state) { -#ifdef BUILDFIXED - static int virgin = 1; - static code *lenfix, *distfix; - static code fixed[544]; - - /* build fixed huffman tables if first call (may not be thread safe) */ - if (virgin) { - unsigned sym, bits; - static code *next; - - /* literal/length table */ - sym = 0; - while (sym < 144) state->lens[sym++] = 8; - while (sym < 256) state->lens[sym++] = 9; - while (sym < 280) state->lens[sym++] = 7; - while (sym < 288) state->lens[sym++] = 8; - next = fixed; - lenfix = next; - bits = 9; - inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); - - /* distance table */ - sym = 0; - while (sym < 32) state->lens[sym++] = 5; - distfix = next; - bits = 5; - inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); - - /* do this just once */ - virgin = 0; - } -#else /* !BUILDFIXED */ -# include "inffixed.h" -#endif /* BUILDFIXED */ - state->lencode = lenfix; - state->lenbits = 9; - state->distcode = distfix; - state->distbits = 5; -} - /* Macros for inflateBack(): */ /* Load returned state from inflate_fast() */ @@ -293,7 +242,7 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, state->mode = STORED; break; case 1: /* fixed block */ - fixedtables(state); + inflate_fixed(state); Tracev((stderr, "inflate: fixed codes block%s\n", state->last ? " (last)" : "")); state->mode = LEN; /* decode codes */ @@ -303,8 +252,8 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, state->last ? " (last)" : "")); state->mode = TABLE; break; - case 3: - strm->msg = (char *)"invalid block type"; + default: + strm->msg = (z_const char *)"invalid block type"; state->mode = BAD; } DROPBITS(2); @@ -315,7 +264,7 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, BYTEBITS(); /* go to byte boundary */ NEEDBITS(32); if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { - strm->msg = (char *)"invalid stored block lengths"; + strm->msg = (z_const char *)"invalid stored block lengths"; state->mode = BAD; break; } @@ -353,7 +302,8 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, DROPBITS(4); #ifndef PKZIP_BUG_WORKAROUND if (state->nlen > 286 || state->ndist > 30) { - strm->msg = (char *)"too many length or distance symbols"; + strm->msg = (z_const char *) + "too many length or distance symbols"; state->mode = BAD; break; } @@ -375,7 +325,7 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, ret = inflate_table(CODES, state->lens, 19, &(state->next), &(state->lenbits), state->work); if (ret) { - strm->msg = (char *)"invalid code lengths set"; + strm->msg = (z_const char *)"invalid code lengths set"; state->mode = BAD; break; } @@ -398,7 +348,8 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, NEEDBITS(here.bits + 2); DROPBITS(here.bits); if (state->have == 0) { - strm->msg = (char *)"invalid bit length repeat"; + strm->msg = (z_const char *) + "invalid bit length repeat"; state->mode = BAD; break; } @@ -421,7 +372,8 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, DROPBITS(7); } if (state->have + copy > state->nlen + state->ndist) { - strm->msg = (char *)"invalid bit length repeat"; + strm->msg = (z_const char *) + "invalid bit length repeat"; state->mode = BAD; break; } @@ -435,30 +387,31 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, /* check for end-of-block code (better have one) */ if (state->lens[256] == 0) { - strm->msg = (char *)"invalid code -- missing end-of-block"; + strm->msg = (z_const char *) + "invalid code -- missing end-of-block"; state->mode = BAD; break; } /* build code tables -- note: do not change the lenbits or distbits - values here (9 and 6) without reading the comments in inftrees.h + values here (10 and 9) without reading the comments in inftrees.h concerning the ENOUGH constants, which depend on those values */ state->next = state->codes; state->lencode = (code const FAR *)(state->next); - state->lenbits = 9; + state->lenbits = 10; ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), &(state->lenbits), state->work); if (ret) { - strm->msg = (char *)"invalid literal/lengths set"; + strm->msg = (z_const char *)"invalid literal/lengths set"; state->mode = BAD; break; } state->distcode = (code const FAR *)(state->next); - state->distbits = 6; + state->distbits = 9; ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, &(state->next), &(state->distbits), state->work); if (ret) { - strm->msg = (char *)"invalid distances set"; + strm->msg = (z_const char *)"invalid distances set"; state->mode = BAD; break; } @@ -468,10 +421,9 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, case LEN: /* use inflate_fast() if we have enough input and output */ - if (have >= 6 && left >= 258) { + if (have >= INFLATE_FAST_MIN_INPUT && + left >= INFLATE_FAST_MIN_OUTPUT) { RESTORE(); - if (state->whave < state->wsize) - state->whave = state->wsize - left; inflate_fast(strm, state->wsize); LOAD(); break; @@ -517,7 +469,7 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, /* invalid code */ if (here.op & 64) { - strm->msg = (char *)"invalid literal/length code"; + strm->msg = (z_const char *)"invalid literal/length code"; state->mode = BAD; break; } @@ -549,7 +501,7 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, } DROPBITS(here.bits); if (here.op & 64) { - strm->msg = (char *)"invalid distance code"; + strm->msg = (z_const char *)"invalid distance code"; state->mode = BAD; break; } @@ -564,7 +516,7 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, } if (state->offset > state->wsize - (state->whave < state->wsize ? left : 0)) { - strm->msg = (char *)"invalid distance too far back"; + strm->msg = (z_const char *)"invalid distance too far back"; state->mode = BAD; break; } diff --git a/packages/runtime/vendor/zlib/inffast.c b/packages/runtime/vendor/zlib/inffast.c index 9354676e7..58fc5afc0 100644 --- a/packages/runtime/vendor/zlib/inffast.c +++ b/packages/runtime/vendor/zlib/inffast.c @@ -1,5 +1,5 @@ /* inffast.c -- fast decoding - * Copyright (C) 1995-2017 Mark Adler + * Copyright (C) 1995-2026 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -18,13 +18,13 @@ available, an end-of-block is encountered, or a data error is encountered. When large enough input and output buffers are supplied to inflate(), for example, a 16K input buffer and a 64K output buffer, more than 95% of the - inflate execution time is spent in this routine. + inflate() execution time is spent in this routine. Entry assumptions: state->mode == LEN - strm->avail_in >= 6 - strm->avail_out >= 258 + strm->avail_in >= INFLATE_FAST_MIN_INPUT (6 bytes) + strm->avail_out >= INFLATE_FAST_MIN_OUTPUT (258 bytes) start >= strm->avail_out state->bits < 8 @@ -36,16 +36,20 @@ Notes: + INFLATE_FAST_MIN_INPUT: 6 bytes + - The maximum input bits used by a length/distance pair is 15 bits for the length code, 5 bits for the length extra, 15 bits for the distance code, and 13 bits for the distance extra. This totals 48 bits, or six bytes. Therefore if strm->avail_in >= 6, then there is enough input to avoid checking for available input while decoding. + INFLATE_FAST_MIN_OUTPUT: 258 bytes + - The maximum bytes that a single length/distance pair can output is 258 bytes, which is the maximum length that can be coded. inflate_fast() requires strm->avail_out >= 258 for each loop to avoid checking for - output space. + available output space while decoding. */ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) { struct inflate_state FAR *state; @@ -77,10 +81,10 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) { /* copy state to local variables */ state = (struct inflate_state FAR *)strm->state; in = strm->next_in; - last = in + (strm->avail_in - 5); + last = in + (strm->avail_in - (INFLATE_FAST_MIN_INPUT - 1)); out = strm->next_out; beg = out - (start - strm->avail_out); - end = out + (strm->avail_out - 257); + end = out + (strm->avail_out - (INFLATE_FAST_MIN_OUTPUT - 1)); #ifdef INFLATE_STRICT dmax = state->dmax; #endif @@ -155,7 +159,8 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) { dist += (unsigned)hold & ((1U << op) - 1); #ifdef INFLATE_STRICT if (dist > dmax) { - strm->msg = (char *)"invalid distance too far back"; + strm->msg = (z_const char *) + "invalid distance too far back"; state->mode = BAD; break; } @@ -168,8 +173,8 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) { op = dist - op; /* distance back in window */ if (op > whave) { if (state->sane) { - strm->msg = - (char *)"invalid distance too far back"; + strm->msg = (z_const char *) + "invalid distance too far back"; state->mode = BAD; break; } @@ -265,7 +270,7 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) { goto dodist; } else { - strm->msg = (char *)"invalid distance code"; + strm->msg = (z_const char *)"invalid distance code"; state->mode = BAD; break; } @@ -280,7 +285,7 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) { break; } else { - strm->msg = (char *)"invalid literal/length code"; + strm->msg = (z_const char *)"invalid literal/length code"; state->mode = BAD; break; } @@ -295,9 +300,12 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) { /* update state and return */ strm->next_in = in; strm->next_out = out; - strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last)); + strm->avail_in = (unsigned)(in < last ? + (INFLATE_FAST_MIN_INPUT - 1) + (last - in) : + (INFLATE_FAST_MIN_INPUT - 1) - (in - last)); strm->avail_out = (unsigned)(out < end ? - 257 + (end - out) : 257 - (out - end)); + (INFLATE_FAST_MIN_OUTPUT - 1) + (end - out) : + (INFLATE_FAST_MIN_OUTPUT - 1) - (out - end)); state->hold = hold; state->bits = bits; return; diff --git a/packages/runtime/vendor/zlib/inffast.h b/packages/runtime/vendor/zlib/inffast.h index 49c6d156c..9ff6ce3b6 100644 --- a/packages/runtime/vendor/zlib/inffast.h +++ b/packages/runtime/vendor/zlib/inffast.h @@ -8,4 +8,19 @@ subject to change. Applications should only use zlib.h. */ +/* INFLATE_FAST_MIN_INPUT: the minimum number of input bytes needed so that + we can safely call inflate_fast() with only one up-front bounds check. One + length/distance code pair (15 bits for the length code, 5 bits for length + extra, 15 bits for the distance code, 13 bits for distance extra) requires + reading up to 48 input bits (6 bytes). +*/ +#define INFLATE_FAST_MIN_INPUT 6 + +/* INFLATE_FAST_MIN_OUTPUT: the minimum number of output bytes needed so that + we can safely call inflate_fast() with only one up-front bounds check. One + length/distance code pair can output up to 258 bytes, which is the maximum + length that can be coded. + */ +#define INFLATE_FAST_MIN_OUTPUT 258 + void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start); diff --git a/packages/runtime/vendor/zlib/inffixed.h b/packages/runtime/vendor/zlib/inffixed.h index d62832776..05ce88e4f 100644 --- a/packages/runtime/vendor/zlib/inffixed.h +++ b/packages/runtime/vendor/zlib/inffixed.h @@ -1,94 +1,94 @@ - /* inffixed.h -- table for decoding fixed codes - * Generated automatically by makefixed(). - */ +/* inffixed.h -- table for decoding fixed codes + * Generated automatically by makefixed(). + */ - /* WARNING: this file should *not* be used by applications. - It is part of the implementation of this library and is - subject to change. Applications should only use zlib.h. - */ +/* WARNING: this file should *not* be used by applications. + It is part of the implementation of this library and is + subject to change. Applications should only use zlib.h. + */ - static const code lenfix[512] = { - {96,7,0},{0,8,80},{0,8,16},{20,8,115},{18,7,31},{0,8,112},{0,8,48}, - {0,9,192},{16,7,10},{0,8,96},{0,8,32},{0,9,160},{0,8,0},{0,8,128}, - {0,8,64},{0,9,224},{16,7,6},{0,8,88},{0,8,24},{0,9,144},{19,7,59}, - {0,8,120},{0,8,56},{0,9,208},{17,7,17},{0,8,104},{0,8,40},{0,9,176}, - {0,8,8},{0,8,136},{0,8,72},{0,9,240},{16,7,4},{0,8,84},{0,8,20}, - {21,8,227},{19,7,43},{0,8,116},{0,8,52},{0,9,200},{17,7,13},{0,8,100}, - {0,8,36},{0,9,168},{0,8,4},{0,8,132},{0,8,68},{0,9,232},{16,7,8}, - {0,8,92},{0,8,28},{0,9,152},{20,7,83},{0,8,124},{0,8,60},{0,9,216}, - {18,7,23},{0,8,108},{0,8,44},{0,9,184},{0,8,12},{0,8,140},{0,8,76}, - {0,9,248},{16,7,3},{0,8,82},{0,8,18},{21,8,163},{19,7,35},{0,8,114}, - {0,8,50},{0,9,196},{17,7,11},{0,8,98},{0,8,34},{0,9,164},{0,8,2}, - {0,8,130},{0,8,66},{0,9,228},{16,7,7},{0,8,90},{0,8,26},{0,9,148}, - {20,7,67},{0,8,122},{0,8,58},{0,9,212},{18,7,19},{0,8,106},{0,8,42}, - {0,9,180},{0,8,10},{0,8,138},{0,8,74},{0,9,244},{16,7,5},{0,8,86}, - {0,8,22},{64,8,0},{19,7,51},{0,8,118},{0,8,54},{0,9,204},{17,7,15}, - {0,8,102},{0,8,38},{0,9,172},{0,8,6},{0,8,134},{0,8,70},{0,9,236}, - {16,7,9},{0,8,94},{0,8,30},{0,9,156},{20,7,99},{0,8,126},{0,8,62}, - {0,9,220},{18,7,27},{0,8,110},{0,8,46},{0,9,188},{0,8,14},{0,8,142}, - {0,8,78},{0,9,252},{96,7,0},{0,8,81},{0,8,17},{21,8,131},{18,7,31}, - {0,8,113},{0,8,49},{0,9,194},{16,7,10},{0,8,97},{0,8,33},{0,9,162}, - {0,8,1},{0,8,129},{0,8,65},{0,9,226},{16,7,6},{0,8,89},{0,8,25}, - {0,9,146},{19,7,59},{0,8,121},{0,8,57},{0,9,210},{17,7,17},{0,8,105}, - {0,8,41},{0,9,178},{0,8,9},{0,8,137},{0,8,73},{0,9,242},{16,7,4}, - {0,8,85},{0,8,21},{16,8,258},{19,7,43},{0,8,117},{0,8,53},{0,9,202}, - {17,7,13},{0,8,101},{0,8,37},{0,9,170},{0,8,5},{0,8,133},{0,8,69}, - {0,9,234},{16,7,8},{0,8,93},{0,8,29},{0,9,154},{20,7,83},{0,8,125}, - {0,8,61},{0,9,218},{18,7,23},{0,8,109},{0,8,45},{0,9,186},{0,8,13}, - {0,8,141},{0,8,77},{0,9,250},{16,7,3},{0,8,83},{0,8,19},{21,8,195}, - {19,7,35},{0,8,115},{0,8,51},{0,9,198},{17,7,11},{0,8,99},{0,8,35}, - {0,9,166},{0,8,3},{0,8,131},{0,8,67},{0,9,230},{16,7,7},{0,8,91}, - {0,8,27},{0,9,150},{20,7,67},{0,8,123},{0,8,59},{0,9,214},{18,7,19}, - {0,8,107},{0,8,43},{0,9,182},{0,8,11},{0,8,139},{0,8,75},{0,9,246}, - {16,7,5},{0,8,87},{0,8,23},{64,8,0},{19,7,51},{0,8,119},{0,8,55}, - {0,9,206},{17,7,15},{0,8,103},{0,8,39},{0,9,174},{0,8,7},{0,8,135}, - {0,8,71},{0,9,238},{16,7,9},{0,8,95},{0,8,31},{0,9,158},{20,7,99}, - {0,8,127},{0,8,63},{0,9,222},{18,7,27},{0,8,111},{0,8,47},{0,9,190}, - {0,8,15},{0,8,143},{0,8,79},{0,9,254},{96,7,0},{0,8,80},{0,8,16}, - {20,8,115},{18,7,31},{0,8,112},{0,8,48},{0,9,193},{16,7,10},{0,8,96}, - {0,8,32},{0,9,161},{0,8,0},{0,8,128},{0,8,64},{0,9,225},{16,7,6}, - {0,8,88},{0,8,24},{0,9,145},{19,7,59},{0,8,120},{0,8,56},{0,9,209}, - {17,7,17},{0,8,104},{0,8,40},{0,9,177},{0,8,8},{0,8,136},{0,8,72}, - {0,9,241},{16,7,4},{0,8,84},{0,8,20},{21,8,227},{19,7,43},{0,8,116}, - {0,8,52},{0,9,201},{17,7,13},{0,8,100},{0,8,36},{0,9,169},{0,8,4}, - {0,8,132},{0,8,68},{0,9,233},{16,7,8},{0,8,92},{0,8,28},{0,9,153}, - {20,7,83},{0,8,124},{0,8,60},{0,9,217},{18,7,23},{0,8,108},{0,8,44}, - {0,9,185},{0,8,12},{0,8,140},{0,8,76},{0,9,249},{16,7,3},{0,8,82}, - {0,8,18},{21,8,163},{19,7,35},{0,8,114},{0,8,50},{0,9,197},{17,7,11}, - {0,8,98},{0,8,34},{0,9,165},{0,8,2},{0,8,130},{0,8,66},{0,9,229}, - {16,7,7},{0,8,90},{0,8,26},{0,9,149},{20,7,67},{0,8,122},{0,8,58}, - {0,9,213},{18,7,19},{0,8,106},{0,8,42},{0,9,181},{0,8,10},{0,8,138}, - {0,8,74},{0,9,245},{16,7,5},{0,8,86},{0,8,22},{64,8,0},{19,7,51}, - {0,8,118},{0,8,54},{0,9,205},{17,7,15},{0,8,102},{0,8,38},{0,9,173}, - {0,8,6},{0,8,134},{0,8,70},{0,9,237},{16,7,9},{0,8,94},{0,8,30}, - {0,9,157},{20,7,99},{0,8,126},{0,8,62},{0,9,221},{18,7,27},{0,8,110}, - {0,8,46},{0,9,189},{0,8,14},{0,8,142},{0,8,78},{0,9,253},{96,7,0}, - {0,8,81},{0,8,17},{21,8,131},{18,7,31},{0,8,113},{0,8,49},{0,9,195}, - {16,7,10},{0,8,97},{0,8,33},{0,9,163},{0,8,1},{0,8,129},{0,8,65}, - {0,9,227},{16,7,6},{0,8,89},{0,8,25},{0,9,147},{19,7,59},{0,8,121}, - {0,8,57},{0,9,211},{17,7,17},{0,8,105},{0,8,41},{0,9,179},{0,8,9}, - {0,8,137},{0,8,73},{0,9,243},{16,7,4},{0,8,85},{0,8,21},{16,8,258}, - {19,7,43},{0,8,117},{0,8,53},{0,9,203},{17,7,13},{0,8,101},{0,8,37}, - {0,9,171},{0,8,5},{0,8,133},{0,8,69},{0,9,235},{16,7,8},{0,8,93}, - {0,8,29},{0,9,155},{20,7,83},{0,8,125},{0,8,61},{0,9,219},{18,7,23}, - {0,8,109},{0,8,45},{0,9,187},{0,8,13},{0,8,141},{0,8,77},{0,9,251}, - {16,7,3},{0,8,83},{0,8,19},{21,8,195},{19,7,35},{0,8,115},{0,8,51}, - {0,9,199},{17,7,11},{0,8,99},{0,8,35},{0,9,167},{0,8,3},{0,8,131}, - {0,8,67},{0,9,231},{16,7,7},{0,8,91},{0,8,27},{0,9,151},{20,7,67}, - {0,8,123},{0,8,59},{0,9,215},{18,7,19},{0,8,107},{0,8,43},{0,9,183}, - {0,8,11},{0,8,139},{0,8,75},{0,9,247},{16,7,5},{0,8,87},{0,8,23}, - {64,8,0},{19,7,51},{0,8,119},{0,8,55},{0,9,207},{17,7,15},{0,8,103}, - {0,8,39},{0,9,175},{0,8,7},{0,8,135},{0,8,71},{0,9,239},{16,7,9}, - {0,8,95},{0,8,31},{0,9,159},{20,7,99},{0,8,127},{0,8,63},{0,9,223}, - {18,7,27},{0,8,111},{0,8,47},{0,9,191},{0,8,15},{0,8,143},{0,8,79}, - {0,9,255} - }; +static const code lenfix[512] = { + {96,7,0},{0,8,80},{0,8,16},{20,8,115},{18,7,31},{0,8,112},{0,8,48}, + {0,9,192},{16,7,10},{0,8,96},{0,8,32},{0,9,160},{0,8,0},{0,8,128}, + {0,8,64},{0,9,224},{16,7,6},{0,8,88},{0,8,24},{0,9,144},{19,7,59}, + {0,8,120},{0,8,56},{0,9,208},{17,7,17},{0,8,104},{0,8,40},{0,9,176}, + {0,8,8},{0,8,136},{0,8,72},{0,9,240},{16,7,4},{0,8,84},{0,8,20}, + {21,8,227},{19,7,43},{0,8,116},{0,8,52},{0,9,200},{17,7,13},{0,8,100}, + {0,8,36},{0,9,168},{0,8,4},{0,8,132},{0,8,68},{0,9,232},{16,7,8}, + {0,8,92},{0,8,28},{0,9,152},{20,7,83},{0,8,124},{0,8,60},{0,9,216}, + {18,7,23},{0,8,108},{0,8,44},{0,9,184},{0,8,12},{0,8,140},{0,8,76}, + {0,9,248},{16,7,3},{0,8,82},{0,8,18},{21,8,163},{19,7,35},{0,8,114}, + {0,8,50},{0,9,196},{17,7,11},{0,8,98},{0,8,34},{0,9,164},{0,8,2}, + {0,8,130},{0,8,66},{0,9,228},{16,7,7},{0,8,90},{0,8,26},{0,9,148}, + {20,7,67},{0,8,122},{0,8,58},{0,9,212},{18,7,19},{0,8,106},{0,8,42}, + {0,9,180},{0,8,10},{0,8,138},{0,8,74},{0,9,244},{16,7,5},{0,8,86}, + {0,8,22},{64,8,0},{19,7,51},{0,8,118},{0,8,54},{0,9,204},{17,7,15}, + {0,8,102},{0,8,38},{0,9,172},{0,8,6},{0,8,134},{0,8,70},{0,9,236}, + {16,7,9},{0,8,94},{0,8,30},{0,9,156},{20,7,99},{0,8,126},{0,8,62}, + {0,9,220},{18,7,27},{0,8,110},{0,8,46},{0,9,188},{0,8,14},{0,8,142}, + {0,8,78},{0,9,252},{96,7,0},{0,8,81},{0,8,17},{21,8,131},{18,7,31}, + {0,8,113},{0,8,49},{0,9,194},{16,7,10},{0,8,97},{0,8,33},{0,9,162}, + {0,8,1},{0,8,129},{0,8,65},{0,9,226},{16,7,6},{0,8,89},{0,8,25}, + {0,9,146},{19,7,59},{0,8,121},{0,8,57},{0,9,210},{17,7,17},{0,8,105}, + {0,8,41},{0,9,178},{0,8,9},{0,8,137},{0,8,73},{0,9,242},{16,7,4}, + {0,8,85},{0,8,21},{16,8,258},{19,7,43},{0,8,117},{0,8,53},{0,9,202}, + {17,7,13},{0,8,101},{0,8,37},{0,9,170},{0,8,5},{0,8,133},{0,8,69}, + {0,9,234},{16,7,8},{0,8,93},{0,8,29},{0,9,154},{20,7,83},{0,8,125}, + {0,8,61},{0,9,218},{18,7,23},{0,8,109},{0,8,45},{0,9,186},{0,8,13}, + {0,8,141},{0,8,77},{0,9,250},{16,7,3},{0,8,83},{0,8,19},{21,8,195}, + {19,7,35},{0,8,115},{0,8,51},{0,9,198},{17,7,11},{0,8,99},{0,8,35}, + {0,9,166},{0,8,3},{0,8,131},{0,8,67},{0,9,230},{16,7,7},{0,8,91}, + {0,8,27},{0,9,150},{20,7,67},{0,8,123},{0,8,59},{0,9,214},{18,7,19}, + {0,8,107},{0,8,43},{0,9,182},{0,8,11},{0,8,139},{0,8,75},{0,9,246}, + {16,7,5},{0,8,87},{0,8,23},{64,8,0},{19,7,51},{0,8,119},{0,8,55}, + {0,9,206},{17,7,15},{0,8,103},{0,8,39},{0,9,174},{0,8,7},{0,8,135}, + {0,8,71},{0,9,238},{16,7,9},{0,8,95},{0,8,31},{0,9,158},{20,7,99}, + {0,8,127},{0,8,63},{0,9,222},{18,7,27},{0,8,111},{0,8,47},{0,9,190}, + {0,8,15},{0,8,143},{0,8,79},{0,9,254},{96,7,0},{0,8,80},{0,8,16}, + {20,8,115},{18,7,31},{0,8,112},{0,8,48},{0,9,193},{16,7,10},{0,8,96}, + {0,8,32},{0,9,161},{0,8,0},{0,8,128},{0,8,64},{0,9,225},{16,7,6}, + {0,8,88},{0,8,24},{0,9,145},{19,7,59},{0,8,120},{0,8,56},{0,9,209}, + {17,7,17},{0,8,104},{0,8,40},{0,9,177},{0,8,8},{0,8,136},{0,8,72}, + {0,9,241},{16,7,4},{0,8,84},{0,8,20},{21,8,227},{19,7,43},{0,8,116}, + {0,8,52},{0,9,201},{17,7,13},{0,8,100},{0,8,36},{0,9,169},{0,8,4}, + {0,8,132},{0,8,68},{0,9,233},{16,7,8},{0,8,92},{0,8,28},{0,9,153}, + {20,7,83},{0,8,124},{0,8,60},{0,9,217},{18,7,23},{0,8,108},{0,8,44}, + {0,9,185},{0,8,12},{0,8,140},{0,8,76},{0,9,249},{16,7,3},{0,8,82}, + {0,8,18},{21,8,163},{19,7,35},{0,8,114},{0,8,50},{0,9,197},{17,7,11}, + {0,8,98},{0,8,34},{0,9,165},{0,8,2},{0,8,130},{0,8,66},{0,9,229}, + {16,7,7},{0,8,90},{0,8,26},{0,9,149},{20,7,67},{0,8,122},{0,8,58}, + {0,9,213},{18,7,19},{0,8,106},{0,8,42},{0,9,181},{0,8,10},{0,8,138}, + {0,8,74},{0,9,245},{16,7,5},{0,8,86},{0,8,22},{64,8,0},{19,7,51}, + {0,8,118},{0,8,54},{0,9,205},{17,7,15},{0,8,102},{0,8,38},{0,9,173}, + {0,8,6},{0,8,134},{0,8,70},{0,9,237},{16,7,9},{0,8,94},{0,8,30}, + {0,9,157},{20,7,99},{0,8,126},{0,8,62},{0,9,221},{18,7,27},{0,8,110}, + {0,8,46},{0,9,189},{0,8,14},{0,8,142},{0,8,78},{0,9,253},{96,7,0}, + {0,8,81},{0,8,17},{21,8,131},{18,7,31},{0,8,113},{0,8,49},{0,9,195}, + {16,7,10},{0,8,97},{0,8,33},{0,9,163},{0,8,1},{0,8,129},{0,8,65}, + {0,9,227},{16,7,6},{0,8,89},{0,8,25},{0,9,147},{19,7,59},{0,8,121}, + {0,8,57},{0,9,211},{17,7,17},{0,8,105},{0,8,41},{0,9,179},{0,8,9}, + {0,8,137},{0,8,73},{0,9,243},{16,7,4},{0,8,85},{0,8,21},{16,8,258}, + {19,7,43},{0,8,117},{0,8,53},{0,9,203},{17,7,13},{0,8,101},{0,8,37}, + {0,9,171},{0,8,5},{0,8,133},{0,8,69},{0,9,235},{16,7,8},{0,8,93}, + {0,8,29},{0,9,155},{20,7,83},{0,8,125},{0,8,61},{0,9,219},{18,7,23}, + {0,8,109},{0,8,45},{0,9,187},{0,8,13},{0,8,141},{0,8,77},{0,9,251}, + {16,7,3},{0,8,83},{0,8,19},{21,8,195},{19,7,35},{0,8,115},{0,8,51}, + {0,9,199},{17,7,11},{0,8,99},{0,8,35},{0,9,167},{0,8,3},{0,8,131}, + {0,8,67},{0,9,231},{16,7,7},{0,8,91},{0,8,27},{0,9,151},{20,7,67}, + {0,8,123},{0,8,59},{0,9,215},{18,7,19},{0,8,107},{0,8,43},{0,9,183}, + {0,8,11},{0,8,139},{0,8,75},{0,9,247},{16,7,5},{0,8,87},{0,8,23}, + {64,8,0},{19,7,51},{0,8,119},{0,8,55},{0,9,207},{17,7,15},{0,8,103}, + {0,8,39},{0,9,175},{0,8,7},{0,8,135},{0,8,71},{0,9,239},{16,7,9}, + {0,8,95},{0,8,31},{0,9,159},{20,7,99},{0,8,127},{0,8,63},{0,9,223}, + {18,7,27},{0,8,111},{0,8,47},{0,9,191},{0,8,15},{0,8,143},{0,8,79}, + {0,9,255} +}; - static const code distfix[32] = { - {16,5,1},{23,5,257},{19,5,17},{27,5,4097},{17,5,5},{25,5,1025}, - {21,5,65},{29,5,16385},{16,5,3},{24,5,513},{20,5,33},{28,5,8193}, - {18,5,9},{26,5,2049},{22,5,129},{64,5,0},{16,5,2},{23,5,385}, - {19,5,25},{27,5,6145},{17,5,7},{25,5,1537},{21,5,97},{29,5,24577}, - {16,5,4},{24,5,769},{20,5,49},{28,5,12289},{18,5,13},{26,5,3073}, - {22,5,193},{64,5,0} - }; +static const code distfix[32] = { + {16,5,1},{23,5,257},{19,5,17},{27,5,4097},{17,5,5},{25,5,1025}, + {21,5,65},{29,5,16385},{16,5,3},{24,5,513},{20,5,33},{28,5,8193}, + {18,5,9},{26,5,2049},{22,5,129},{64,5,0},{16,5,2},{23,5,385}, + {19,5,25},{27,5,6145},{17,5,7},{25,5,1537},{21,5,97},{29,5,24577}, + {16,5,4},{24,5,769},{20,5,49},{28,5,12289},{18,5,13},{26,5,3073}, + {22,5,193},{64,5,0} +}; diff --git a/packages/runtime/vendor/zlib/inflate.c b/packages/runtime/vendor/zlib/inflate.c index 94ecff015..0742ce10e 100644 --- a/packages/runtime/vendor/zlib/inflate.c +++ b/packages/runtime/vendor/zlib/inflate.c @@ -1,5 +1,5 @@ /* inflate.c -- zlib decompression - * Copyright (C) 1995-2022 Mark Adler + * Copyright (C) 1995-2026 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -85,12 +85,6 @@ #include "inflate.h" #include "inffast.h" -#ifdef MAKEFIXED -# ifndef BUILDFIXED -# define BUILDFIXED -# endif -#endif - local int inflateStateCheck(z_streamp strm) { struct inflate_state FAR *state; if (strm == Z_NULL || @@ -110,6 +104,7 @@ int ZEXPORT inflateResetKeep(z_streamp strm) { state = (struct inflate_state FAR *)strm->state; strm->total_in = strm->total_out = state->total = 0; strm->msg = Z_NULL; + strm->data_type = 0; if (state->wrap) /* to support ill-conceived Java test suite */ strm->adler = state->wrap & 1; state->mode = HEAD; @@ -202,11 +197,13 @@ int ZEXPORT inflateInit2_(z_streamp strm, int windowBits, state = (struct inflate_state FAR *) ZALLOC(strm, 1, sizeof(struct inflate_state)); if (state == Z_NULL) return Z_MEM_ERROR; + zmemzero(state, sizeof(struct inflate_state)); Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state FAR *)state; state->strm = strm; state->window = Z_NULL; state->mode = HEAD; /* to pass state test in inflateReset2() */ + state->check = 1L; /* 1L is the result of adler32() zero length data */ ret = inflateReset2(strm, windowBits); if (ret != Z_OK) { ZFREE(strm, state); @@ -234,123 +231,11 @@ int ZEXPORT inflatePrime(z_streamp strm, int bits, int value) { } if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; value &= (1L << bits) - 1; - state->hold += (unsigned)value << state->bits; + state->hold += (unsigned long)value << state->bits; state->bits += (uInt)bits; return Z_OK; } -/* - Return state with length and distance decoding tables and index sizes set to - fixed code decoding. Normally this returns fixed tables from inffixed.h. - If BUILDFIXED is defined, then instead this routine builds the tables the - first time it's called, and returns those tables the first time and - thereafter. This reduces the size of the code by about 2K bytes, in - exchange for a little execution time. However, BUILDFIXED should not be - used for threaded applications, since the rewriting of the tables and virgin - may not be thread-safe. - */ -local void fixedtables(struct inflate_state FAR *state) { -#ifdef BUILDFIXED - static int virgin = 1; - static code *lenfix, *distfix; - static code fixed[544]; - - /* build fixed huffman tables if first call (may not be thread safe) */ - if (virgin) { - unsigned sym, bits; - static code *next; - - /* literal/length table */ - sym = 0; - while (sym < 144) state->lens[sym++] = 8; - while (sym < 256) state->lens[sym++] = 9; - while (sym < 280) state->lens[sym++] = 7; - while (sym < 288) state->lens[sym++] = 8; - next = fixed; - lenfix = next; - bits = 9; - inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); - - /* distance table */ - sym = 0; - while (sym < 32) state->lens[sym++] = 5; - distfix = next; - bits = 5; - inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); - - /* do this just once */ - virgin = 0; - } -#else /* !BUILDFIXED */ -# include "inffixed.h" -#endif /* BUILDFIXED */ - state->lencode = lenfix; - state->lenbits = 9; - state->distcode = distfix; - state->distbits = 5; -} - -#ifdef MAKEFIXED -#include - -/* - Write out the inffixed.h that is #include'd above. Defining MAKEFIXED also - defines BUILDFIXED, so the tables are built on the fly. makefixed() writes - those tables to stdout, which would be piped to inffixed.h. A small program - can simply call makefixed to do this: - - void makefixed(void); - - int main(void) - { - makefixed(); - return 0; - } - - Then that can be linked with zlib built with MAKEFIXED defined and run: - - a.out > inffixed.h - */ -void makefixed(void) -{ - unsigned low, size; - struct inflate_state state; - - fixedtables(&state); - puts(" /* inffixed.h -- table for decoding fixed codes"); - puts(" * Generated automatically by makefixed()."); - puts(" */"); - puts(""); - puts(" /* WARNING: this file should *not* be used by applications."); - puts(" It is part of the implementation of this library and is"); - puts(" subject to change. Applications should only use zlib.h."); - puts(" */"); - puts(""); - size = 1U << 9; - printf(" static const code lenfix[%u] = {", size); - low = 0; - for (;;) { - if ((low % 7) == 0) printf("\n "); - printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, - state.lencode[low].bits, state.lencode[low].val); - if (++low == size) break; - putchar(','); - } - puts("\n };"); - size = 1U << 5; - printf("\n static const code distfix[%u] = {", size); - low = 0; - for (;;) { - if ((low % 6) == 0) printf("\n "); - printf("{%u,%u,%d}", state.distcode[low].op, state.distcode[low].bits, - state.distcode[low].val); - if (++low == size) break; - putchar(','); - } - puts("\n };"); -} -#endif /* MAKEFIXED */ - /* Update the window with the last wsize (normally 32K) bytes written before returning. If window does not exist yet, create it. This is only called @@ -642,12 +527,12 @@ int ZEXPORT inflate(z_streamp strm, int flush) { if ( #endif ((BITS(8) << 8) + (hold >> 8)) % 31) { - strm->msg = (char *)"incorrect header check"; + strm->msg = (z_const char *)"incorrect header check"; state->mode = BAD; break; } if (BITS(4) != Z_DEFLATED) { - strm->msg = (char *)"unknown compression method"; + strm->msg = (z_const char *)"unknown compression method"; state->mode = BAD; break; } @@ -656,7 +541,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { if (state->wbits == 0) state->wbits = len; if (len > 15 || len > state->wbits) { - strm->msg = (char *)"invalid window size"; + strm->msg = (z_const char *)"invalid window size"; state->mode = BAD; break; } @@ -672,12 +557,12 @@ int ZEXPORT inflate(z_streamp strm, int flush) { NEEDBITS(16); state->flags = (int)(hold); if ((state->flags & 0xff) != Z_DEFLATED) { - strm->msg = (char *)"unknown compression method"; + strm->msg = (z_const char *)"unknown compression method"; state->mode = BAD; break; } if (state->flags & 0xe000) { - strm->msg = (char *)"unknown header flags set"; + strm->msg = (z_const char *)"unknown header flags set"; state->mode = BAD; break; } @@ -793,7 +678,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { if (state->flags & 0x0200) { NEEDBITS(16); if ((state->wrap & 4) && hold != (state->check & 0xffff)) { - strm->msg = (char *)"header crc mismatch"; + strm->msg = (z_const char *)"header crc mismatch"; state->mode = BAD; break; } @@ -840,7 +725,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { state->mode = STORED; break; case 1: /* fixed block */ - fixedtables(state); + inflate_fixed(state); Tracev((stderr, "inflate: fixed codes block%s\n", state->last ? " (last)" : "")); state->mode = LEN_; /* decode codes */ @@ -854,8 +739,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { state->last ? " (last)" : "")); state->mode = TABLE; break; - case 3: - strm->msg = (char *)"invalid block type"; + default: + strm->msg = (z_const char *)"invalid block type"; state->mode = BAD; } DROPBITS(2); @@ -864,7 +749,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { BYTEBITS(); /* go to byte boundary */ NEEDBITS(32); if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { - strm->msg = (char *)"invalid stored block lengths"; + strm->msg = (z_const char *)"invalid stored block lengths"; state->mode = BAD; break; } @@ -905,7 +790,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { DROPBITS(4); #ifndef PKZIP_BUG_WORKAROUND if (state->nlen > 286 || state->ndist > 30) { - strm->msg = (char *)"too many length or distance symbols"; + strm->msg = (z_const char *) + "too many length or distance symbols"; state->mode = BAD; break; } @@ -923,12 +809,12 @@ int ZEXPORT inflate(z_streamp strm, int flush) { while (state->have < 19) state->lens[order[state->have++]] = 0; state->next = state->codes; - state->lencode = (const code FAR *)(state->next); + state->lencode = state->distcode = (const code FAR *)(state->next); state->lenbits = 7; ret = inflate_table(CODES, state->lens, 19, &(state->next), &(state->lenbits), state->work); if (ret) { - strm->msg = (char *)"invalid code lengths set"; + strm->msg = (z_const char *)"invalid code lengths set"; state->mode = BAD; break; } @@ -952,7 +838,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { NEEDBITS(here.bits + 2); DROPBITS(here.bits); if (state->have == 0) { - strm->msg = (char *)"invalid bit length repeat"; + strm->msg = (z_const char *) + "invalid bit length repeat"; state->mode = BAD; break; } @@ -975,7 +862,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { DROPBITS(7); } if (state->have + copy > state->nlen + state->ndist) { - strm->msg = (char *)"invalid bit length repeat"; + strm->msg = (z_const char *) + "invalid bit length repeat"; state->mode = BAD; break; } @@ -989,30 +877,31 @@ int ZEXPORT inflate(z_streamp strm, int flush) { /* check for end-of-block code (better have one) */ if (state->lens[256] == 0) { - strm->msg = (char *)"invalid code -- missing end-of-block"; + strm->msg = (z_const char *) + "invalid code -- missing end-of-block"; state->mode = BAD; break; } /* build code tables -- note: do not change the lenbits or distbits - values here (9 and 6) without reading the comments in inftrees.h + values here (10 and 9) without reading the comments in inftrees.h concerning the ENOUGH constants, which depend on those values */ state->next = state->codes; state->lencode = (const code FAR *)(state->next); - state->lenbits = 9; + state->lenbits = 10; ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), &(state->lenbits), state->work); if (ret) { - strm->msg = (char *)"invalid literal/lengths set"; + strm->msg = (z_const char *)"invalid literal/lengths set"; state->mode = BAD; break; } state->distcode = (const code FAR *)(state->next); - state->distbits = 6; + state->distbits = 9; ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, &(state->next), &(state->distbits), state->work); if (ret) { - strm->msg = (char *)"invalid distances set"; + strm->msg = (z_const char *)"invalid distances set"; state->mode = BAD; break; } @@ -1024,7 +913,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { state->mode = LEN; /* fallthrough */ case LEN: - if (have >= 6 && left >= 258) { + if (have >= INFLATE_FAST_MIN_INPUT && + left >= INFLATE_FAST_MIN_OUTPUT) { RESTORE(); inflate_fast(strm, out); LOAD(); @@ -1066,7 +956,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { break; } if (here.op & 64) { - strm->msg = (char *)"invalid literal/length code"; + strm->msg = (z_const char *)"invalid literal/length code"; state->mode = BAD; break; } @@ -1104,7 +994,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { DROPBITS(here.bits); state->back += here.bits; if (here.op & 64) { - strm->msg = (char *)"invalid distance code"; + strm->msg = (z_const char *)"invalid distance code"; state->mode = BAD; break; } @@ -1121,7 +1011,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { } #ifdef INFLATE_STRICT if (state->offset > state->dmax) { - strm->msg = (char *)"invalid distance too far back"; + strm->msg = (z_const char *)"invalid distance too far back"; state->mode = BAD; break; } @@ -1136,7 +1026,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { copy = state->offset - copy; if (copy > state->whave) { if (state->sane) { - strm->msg = (char *)"invalid distance too far back"; + strm->msg = (z_const char *) + "invalid distance too far back"; state->mode = BAD; break; } @@ -1195,7 +1086,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { state->flags ? hold : #endif ZSWAP32(hold)) != state->check) { - strm->msg = (char *)"incorrect data check"; + strm->msg = (z_const char *)"incorrect data check"; state->mode = BAD; break; } @@ -1209,7 +1100,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { if (state->wrap && state->flags) { NEEDBITS(32); if ((state->wrap & 4) && hold != (state->total & 0xffffffff)) { - strm->msg = (char *)"incorrect length check"; + strm->msg = (z_const char *)"incorrect length check"; state->mode = BAD; break; } @@ -1440,7 +1331,6 @@ int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) { struct inflate_state FAR *state; struct inflate_state FAR *copy; unsigned char FAR *window; - unsigned wsize; /* check input */ if (inflateStateCheck(source) || dest == Z_NULL) @@ -1451,6 +1341,7 @@ int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) { copy = (struct inflate_state FAR *) ZALLOC(source, 1, sizeof(struct inflate_state)); if (copy == Z_NULL) return Z_MEM_ERROR; + zmemzero(copy, sizeof(struct inflate_state)); window = Z_NULL; if (state->window != Z_NULL) { window = (unsigned char FAR *) @@ -1462,8 +1353,8 @@ int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) { } /* copy state */ - zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); - zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + zmemcpy(dest, source, sizeof(z_stream)); + zmemcpy(copy, state, sizeof(struct inflate_state)); copy->strm = dest; if (state->lencode >= state->codes && state->lencode <= state->codes + ENOUGH - 1) { @@ -1471,10 +1362,8 @@ int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) { copy->distcode = copy->codes + (state->distcode - state->codes); } copy->next = copy->codes + (state->next - state->codes); - if (window != Z_NULL) { - wsize = 1U << state->wbits; - zmemcpy(window, state->window, wsize); - } + if (window != Z_NULL) + zmemcpy(window, state->window, state->whave); copy->window = window; dest->state = (struct internal_state FAR *)copy; return Z_OK; diff --git a/packages/runtime/vendor/zlib/inflate.h b/packages/runtime/vendor/zlib/inflate.h index f127b6b1f..f758e0dcc 100644 --- a/packages/runtime/vendor/zlib/inflate.h +++ b/packages/runtime/vendor/zlib/inflate.h @@ -100,7 +100,7 @@ struct inflate_state { unsigned char FAR *window; /* allocated sliding window, if needed */ /* bit accumulator */ unsigned long hold; /* input bit accumulator */ - unsigned bits; /* number of bits in "in" */ + unsigned bits; /* number of bits in hold */ /* for string and stored block copying */ unsigned length; /* literal or length of data to copy */ unsigned offset; /* distance back to copy string from */ diff --git a/packages/runtime/vendor/zlib/inftrees.c b/packages/runtime/vendor/zlib/inftrees.c index 98cfe1644..99d8d7730 100644 --- a/packages/runtime/vendor/zlib/inftrees.c +++ b/packages/runtime/vendor/zlib/inftrees.c @@ -1,15 +1,29 @@ /* inftrees.c -- generate Huffman trees for efficient decoding - * Copyright (C) 1995-2024 Mark Adler + * Copyright (C) 1995-2026 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ +#ifdef MAKEFIXED +# ifndef BUILDFIXED +# define BUILDFIXED +# endif +#endif +#ifdef BUILDFIXED +# define Z_ONCE +#endif + #include "zutil.h" #include "inftrees.h" +#include "inflate.h" + +#ifndef NULL +# define NULL 0 +#endif #define MAXBITS 15 const char inflate_copyright[] = - " inflate 1.3.1 Copyright 1995-2024 Mark Adler "; + " inflate 1.3.2.1 Copyright 1995-2026 Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -47,9 +61,9 @@ int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, unsigned mask; /* mask for low root bits */ code here; /* table entry for duplication */ code FAR *next; /* next available space in table */ - const unsigned short FAR *base; /* base value table to use */ - const unsigned short FAR *extra; /* extra bits table to use */ - unsigned match; /* use base and extra for symbol >= match */ + const unsigned short FAR *base = NULL; /* base value table to use */ + const unsigned short FAR *extra = NULL; /* extra bits table to use */ + unsigned match = 0; /* use base and extra for symbol >= match */ unsigned short count[MAXBITS+1]; /* number of codes of each length */ unsigned short offs[MAXBITS+1]; /* offsets in table for each length */ static const unsigned short lbase[31] = { /* Length codes 257..285 base */ @@ -57,7 +71,7 @@ int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; static const unsigned short lext[31] = { /* Length codes 257..285 extra */ 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, - 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 203, 77}; + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 68, 193}; static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, @@ -175,7 +189,6 @@ int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, /* set up for code type */ switch (type) { case CODES: - base = extra = work; /* dummy value--not used */ match = 20; break; case LENS: @@ -183,10 +196,9 @@ int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, extra = lext; match = 257; break; - default: /* DISTS */ + case DISTS: base = dbase; extra = dext; - match = 0; } /* initialize state for loop */ @@ -297,3 +309,116 @@ int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, *bits = root; return 0; } + +#ifdef BUILDFIXED +/* + If this is compiled with BUILDFIXED defined, and if inflate will be used in + multiple threads, and if atomics are not available, then inflate() must be + called with a fixed block (e.g. 0x03 0x00) to initialize the tables and must + return before any other threads are allowed to call inflate. + */ + +static code *lenfix, *distfix; +static code fixed[544]; + +/* State for z_once(). */ +local z_once_t built = Z_ONCE_INIT; + +local void buildtables(void) { + unsigned sym, bits; + static code *next; + unsigned short lens[288], work[288]; + + /* literal/length table */ + sym = 0; + while (sym < 144) lens[sym++] = 8; + while (sym < 256) lens[sym++] = 9; + while (sym < 280) lens[sym++] = 7; + while (sym < 288) lens[sym++] = 8; + next = fixed; + lenfix = next; + bits = 9; + inflate_table(LENS, lens, 288, &(next), &(bits), work); + + /* distance table */ + sym = 0; + while (sym < 32) lens[sym++] = 5; + distfix = next; + bits = 5; + inflate_table(DISTS, lens, 32, &(next), &(bits), work); +} +#else /* !BUILDFIXED */ +# include "inffixed.h" +#endif /* BUILDFIXED */ + +/* + Return state with length and distance decoding tables and index sizes set to + fixed code decoding. Normally this returns fixed tables from inffixed.h. + If BUILDFIXED is defined, then instead this routine builds the tables the + first time it's called, and returns those tables the first time and + thereafter. This reduces the size of the code by about 2K bytes, in + exchange for a little execution time. However, BUILDFIXED should not be + used for threaded applications if atomics are not available, as it will + not be thread-safe. + */ +void inflate_fixed(struct inflate_state FAR *state) { +#ifdef BUILDFIXED + z_once(&built, buildtables); +#endif /* BUILDFIXED */ + state->lencode = lenfix; + state->lenbits = 9; + state->distcode = distfix; + state->distbits = 5; +} + +#ifdef MAKEFIXED +#include + +/* + Write out the inffixed.h that will be #include'd above. Defining MAKEFIXED + also defines BUILDFIXED, so the tables are built on the fly. main() writes + those tables to stdout, which would directed to inffixed.h. Compile this + along with zutil.c: + + cc -DMAKEFIXED -o fix inftrees.c zutil.c + ./fix > inffixed.h + */ +int main(void) { + unsigned low, size; + struct inflate_state state; + + inflate_fixed(&state); + puts("/* inffixed.h -- table for decoding fixed codes"); + puts(" * Generated automatically by makefixed()."); + puts(" */"); + puts(""); + puts("/* WARNING: this file should *not* be used by applications."); + puts(" It is part of the implementation of this library and is"); + puts(" subject to change. Applications should only use zlib.h."); + puts(" */"); + puts(""); + size = 1U << 9; + printf("static const code lenfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 7) == 0) printf("\n "); + printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, + state.lencode[low].bits, state.lencode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n};"); + size = 1U << 5; + printf("\nstatic const code distfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 6) == 0) printf("\n "); + printf("{%u,%u,%d}", state.distcode[low].op, state.distcode[low].bits, + state.distcode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n};"); + return 0; +} +#endif /* MAKEFIXED */ diff --git a/packages/runtime/vendor/zlib/inftrees.h b/packages/runtime/vendor/zlib/inftrees.h index 396f74b5d..c572f49d5 100644 --- a/packages/runtime/vendor/zlib/inftrees.h +++ b/packages/runtime/vendor/zlib/inftrees.h @@ -1,5 +1,5 @@ /* inftrees.h -- header to use inftrees.c - * Copyright (C) 1995-2005, 2010 Mark Adler + * Copyright (C) 1995-2026 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -36,17 +36,17 @@ typedef struct { */ /* Maximum size of the dynamic table. The maximum number of code structures is - 1444, which is the sum of 852 for literal/length codes and 592 for distance + 1924, which is the sum of 1332 for literal/length codes and 592 for distance codes. These values were found by exhaustive searches using the program examples/enough.c found in the zlib distribution. The arguments to that program are the number of symbols, the initial root table size, and the - maximum bit length of a code. "enough 286 9 15" for literal/length codes - returns 852, and "enough 30 6 15" for distance codes returns 592. The - initial root table size (9 or 6) is found in the fifth argument of the + maximum bit length of a code. "enough 286 10 15" for literal/length codes + returns 1332, and "enough 30 9 15" for distance codes returns 592. + The initial root table size (10 or 9) is found in the fifth argument of the inflate_table() calls in inflate.c and infback.c. If the root table size is changed, then these maximum sizes would be need to be recalculated and updated. */ -#define ENOUGH_LENS 852 +#define ENOUGH_LENS 1332 #define ENOUGH_DISTS 592 #define ENOUGH (ENOUGH_LENS+ENOUGH_DISTS) @@ -60,3 +60,5 @@ typedef enum { int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, unsigned codes, code FAR * FAR *table, unsigned FAR *bits, unsigned short FAR *work); +struct inflate_state; +void ZLIB_INTERNAL inflate_fixed(struct inflate_state FAR *state); diff --git a/packages/runtime/vendor/zlib/slide_hash_simd.h b/packages/runtime/vendor/zlib/slide_hash_simd.h new file mode 100644 index 000000000..1000b774c --- /dev/null +++ b/packages/runtime/vendor/zlib/slide_hash_simd.h @@ -0,0 +1,116 @@ +/* slide_hash_simd.h + * + * Copyright 2022 The Chromium Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#ifndef SLIDE_HASH_SIMD_H +#define SLIDE_HASH_SIMD_H + +#include "deflate.h" + +#ifndef INLINE +#if defined(_MSC_VER) && !defined(__clang__) +#define INLINE __inline +#else +#define INLINE inline +#endif +#endif + +#if defined(CPU_NO_SIMD) + +#error SIMD has been disabled for your build target + +#elif defined(DEFLATE_SLIDE_HASH_SSE2) + +#include /* SSE2 */ + +#define Z_SLIDE_INIT_SIMD(wsize) _mm_set1_epi16((ush)(wsize)) + +#define Z_SLIDE_HASH_SIMD(table, size, vector_wsize) \ + for (const Posf* const end = table + size; table != end;) { \ + __m128i vO = _mm_loadu_si128((__m128i *)(table + 0)); \ + vO = _mm_subs_epu16(vO, vector_wsize); \ + _mm_storeu_si128((__m128i *)(table + 0), vO); \ + table += 8; \ + } + +typedef __m128i z_vec128i_u16x8_t; + +#elif defined(DEFLATE_SLIDE_HASH_NEON) + +#include /* NEON */ + +#define Z_SLIDE_INIT_SIMD(wsize) vdupq_n_u16((ush)(wsize)) + +#define Z_SLIDE_HASH_SIMD(table, size, vector_wsize) \ + for (const Posf* const end = table + size; table != end;) { \ + uint16x8_t vO = vld1q_u16(table + 0); \ + uint16x8_t v8 = vld1q_u16(table + 8); \ + vO = vqsubq_u16(vO, vector_wsize); \ + v8 = vqsubq_u16(v8, vector_wsize); \ + vst1q_u16(table + 0, vO); \ + vst1q_u16(table + 8, v8); \ + table += 8 + 8; \ + } + +typedef uint16x8_t z_vec128i_u16x8_t; + +#else + +#error slide_hash_simd is not defined for your build target + +#endif + +/* =========================================================================== + * Slide the hash table when sliding the window down (could be avoided with 32 + * bit values at the expense of memory usage). We slide even when level == 0 to + * keep the hash table consistent if we switch back to level > 0 later. + */ +local INLINE void slide_hash_simd( + Posf *head, Posf *prev, const uInt w_size, const uInt hash_size) { + /* + * The SIMD implementation of the hash table slider assumes: + * + * 1. hash chain offset is 2 bytes. Should be true as Pos is "ush" type. + */ + Assert(sizeof(Pos) == 2, "Pos type size error: should be 2 bytes"); + Assert(sizeof(ush) == 2, "ush type size error: should be 2 bytes"); + + Assert(hash_size <= (1 << 16), "Hash table maximum size error"); + Assert(hash_size >= (1 << 8), "Hash table minimum size error"); + Assert(w_size == (ush)w_size, "Prev table size error"); + + /* + * 2. The hash & prev table sizes are a multiple of 32 bytes (256 bits), + * since the NEON table slider moves two 128-bit items per loop (loop is + * unrolled on NEON for performance, see http://crbug.com/863257). + */ + Assert(!((hash_size * sizeof(head[0])) & (32 - 1)), + "Hash table size error: should be a multiple of 32 bytes"); + Assert(!((w_size * sizeof(prev[0])) & (32 - 1)), + "Prev table size error: should be a multiple of 32 bytes"); + + /* + * Duplicate (ush)w_size in each uint16_t component of a 128-bit vector. + */ + const z_vec128i_u16x8_t vec_wsize = Z_SLIDE_INIT_SIMD(w_size); + + /* + * Slide {head,prev} hash chain values: subtracts (ush)w_size from every + * value with a saturating SIMD subtract, to clamp the result to 0(NIL), + * to implement slide_hash() `(m >= wsize ? m - wsize : NIL);` code. + */ + Z_SLIDE_HASH_SIMD(head, hash_size, vec_wsize); +#ifndef FASTEST + Z_SLIDE_HASH_SIMD(prev, w_size, vec_wsize); +#endif + +} + +#undef z_vec128i_u16x8_t +#undef Z_SLIDE_HASH_SIMD +#undef Z_SLIDE_INIT_SIMD + +#endif /* SLIDE_HASH_SIMD_H */ diff --git a/packages/runtime/vendor/zlib/trees.c b/packages/runtime/vendor/zlib/trees.c index 6a523ef34..1aa155102 100644 --- a/packages/runtime/vendor/zlib/trees.c +++ b/packages/runtime/vendor/zlib/trees.c @@ -1,5 +1,5 @@ /* trees.c -- output deflated data using Huffman coding - * Copyright (C) 1995-2024 Jean-loup Gailly + * Copyright (C) 1995-2026 Jean-loup Gailly * detect_data_type() function provided freely by Cosmin Truta, 2006 * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -112,7 +112,7 @@ local int base_dist[D_CODES]; #else # include "trees.h" -#endif /* GEN_TREES_H */ +#endif /* defined(GEN_TREES_H) || !defined(STDC) */ struct static_tree_desc_s { const ct_data *static_tree; /* static tree or NULL */ @@ -141,18 +141,25 @@ local TCONST static_tree_desc static_bl_desc = * Output a short LSB first on the stream. * IN assertion: there is enough room in pendingBuf. */ +#if defined(DEFLATE_CHUNK_WRITE_64LE) +#define put_short(s, w) { \ + uint16_t data = (uint16_t)(w); \ + zmemcpy(&s->pending_buf[s->pending], &data, sizeof(data)); \ + s->pending += 2; \ + } +#else #define put_short(s, w) { \ put_byte(s, (uch)((w) & 0xff)); \ put_byte(s, (uch)((ush)(w) >> 8)); \ } - +#endif /* DEFLATE_CHUNK_WRITE_64LE */ /* =========================================================================== * Reverse the first len bits of a code, using straightforward code (a faster * method would use a table) * IN assertion: 1 <= len <= 15 */ local unsigned bi_reverse(unsigned code, int len) { - register unsigned res = 0; + unsigned res = 0; do { res |= code & 1; code >>= 1, res <<= 1; @@ -164,6 +171,13 @@ local unsigned bi_reverse(unsigned code, int len) { * Flush the bit buffer, keeping at most 7 bits in it. */ local void bi_flush(deflate_state *s) { +#if defined(DEFLATE_CHUNK_WRITE_64LE) + while (s->bi_valid >= 16) { + put_short(s, s->bi_buf); + s->bi_buf >>= 16; + s->bi_valid -= 16; + } +#endif /* DEFLATE_CHUNK_WRITE_64LE */ if (s->bi_valid == 16) { put_short(s, s->bi_buf); s->bi_buf = 0; @@ -173,21 +187,31 @@ local void bi_flush(deflate_state *s) { s->bi_buf >>= 8; s->bi_valid -= 8; } + + Assert(s->bi_valid >= 0 && s->bi_valid <= 7, "bad bi_flush"); } /* =========================================================================== * Flush the bit buffer and align the output on a byte boundary */ local void bi_windup(deflate_state *s) { +#if defined(DEFLATE_CHUNK_WRITE_64LE) + while (s->bi_valid >= 16) { + put_short(s, s->bi_buf); + s->bi_buf >>= 16; + s->bi_valid -= 16; + } +#endif /* DEFLATE_CHUNK_WRITE_64LE */ if (s->bi_valid > 8) { put_short(s, s->bi_buf); } else if (s->bi_valid > 0) { put_byte(s, (Byte)s->bi_buf); } + s->bi_used = ((s->bi_valid - 1) & 7) + 1; s->bi_buf = 0; s->bi_valid = 0; #ifdef ZLIB_DEBUG - s->bits_sent = (s->bits_sent + 7) & ~7; + s->bits_sent = (s->bits_sent + 7) & ~(ulg)7; #endif } @@ -248,7 +272,28 @@ local void gen_trees_header(void); * Send a value on a given number of bits. * IN assertion: length <= 16 and value fits in length bits. */ +#if defined(DEFLATE_CHUNK_WRITE_64LE) +local void send_bits(deflate_state *s, uint64_t value, int length) +{ + Tracevv((stderr," l %2d v %4llx ", length, (unsigned long long)value)); + Assert(s->bi_valid >= 0 && s->bi_valid <= 63, "invalid bi_valid"); + Assert(length > 0 && length <= 15, "invalid length"); #ifdef ZLIB_DEBUG + s->bits_sent += (ulg)length; +#endif + + s->bi_buf |= (value << s->bi_valid); + s->bi_valid += length; + + if (s->bi_valid >= Buf_size) { + zmemcpy(&s->pending_buf[s->pending], &s->bi_buf, sizeof(s->bi_buf)); + s->pending += 8; + s->bi_valid -= Buf_size; + s->bi_buf = value >> (length - s->bi_valid); + } + Assert(s->bi_valid >= 0 && s->bi_valid <= 63, "invalid bi_valid"); +} +#elif defined(ZLIB_DEBUG) local void send_bits(deflate_state *s, int value, int length) { Tracevv((stderr," l %2d v %4x ", length, value)); Assert(length > 0 && length <= 15, "invalid length"); @@ -268,8 +313,7 @@ local void send_bits(deflate_state *s, int value, int length) { s->bi_valid += length; } } -#else /* !ZLIB_DEBUG */ - +#else /* !ZLIB_DEBUG && !DEFLATE_CHUNK_WRITE_64LE */ #define send_bits(s, value, length) \ { int len = length;\ if (s->bi_valid > (int)Buf_size - len) {\ @@ -283,7 +327,7 @@ local void send_bits(deflate_state *s, int value, int length) { s->bi_valid += len;\ }\ } -#endif /* ZLIB_DEBUG */ +#endif /* DEFLATE_CHUNK_WRITE_64LE */ /* the arguments must not have side effects */ @@ -466,6 +510,7 @@ void ZLIB_INTERNAL _tr_init(deflate_state *s) { s->bi_buf = 0; s->bi_valid = 0; + s->bi_used = 0; #ifdef ZLIB_DEBUG s->compressed_len = 0L; s->bits_sent = 0L; @@ -724,7 +769,7 @@ local void scan_tree(deflate_state *s, ct_data *tree, int max_code) { if (++count < max_count && curlen == nextlen) { continue; } else if (count < min_count) { - s->bl_tree[curlen].Freq += count; + s->bl_tree[curlen].Freq += (ush)count; } else if (curlen != 0) { if (curlen != prevlen) s->bl_tree[curlen].Freq++; s->bl_tree[REP_3_6].Freq++; @@ -817,7 +862,7 @@ local int build_bl_tree(deflate_state *s) { } /* Update opt_len to include the bit length tree and counts */ s->opt_len += 3*((ulg)max_blindex + 1) + 5 + 5 + 4; - Tracev((stderr, "\ndyn trees: dyn %ld, stat %ld", + Tracev((stderr, "\ndyn trees: dyn %lu, stat %lu", s->opt_len, s->static_len)); return max_blindex; @@ -843,13 +888,13 @@ local void send_all_trees(deflate_state *s, int lcodes, int dcodes, Tracev((stderr, "\nbl code %2d ", bl_order[rank])); send_bits(s, s->bl_tree[bl_order[rank]].Len, 3); } - Tracev((stderr, "\nbl tree: sent %ld", s->bits_sent)); + Tracev((stderr, "\nbl tree: sent %lu", s->bits_sent)); send_tree(s, (ct_data *)s->dyn_ltree, lcodes - 1); /* literal tree */ - Tracev((stderr, "\nlit tree: sent %ld", s->bits_sent)); + Tracev((stderr, "\nlit tree: sent %lu", s->bits_sent)); send_tree(s, (ct_data *)s->dyn_dtree, dcodes - 1); /* distance tree */ - Tracev((stderr, "\ndist tree: sent %ld", s->bits_sent)); + Tracev((stderr, "\ndist tree: sent %lu", s->bits_sent)); } /* =========================================================================== @@ -932,13 +977,14 @@ local void compress_block(deflate_state *s, const ct_data *ltree, extra = extra_dbits[code]; if (extra != 0) { dist -= (unsigned)base_dist[code]; - send_bits(s, dist, extra); /* send the extra distance bits */ + send_bits(s, (int)dist, extra); /* send the extra bits */ } } /* literal or match pair ? */ /* Check for no overlay of pending_buf on needed symbols */ #ifdef LIT_MEM - Assert(s->pending < 2 * (s->lit_bufsize + sx), "pendingBuf overflow"); + Assert(s->pending < (s->lit_bufsize << 1) + (sx << 1), + "pendingBuf overflow"); #else Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow"); #endif @@ -1006,11 +1052,11 @@ void ZLIB_INTERNAL _tr_flush_block(deflate_state *s, charf *buf, /* Construct the literal and distance trees */ build_tree(s, (tree_desc *)(&(s->l_desc))); - Tracev((stderr, "\nlit data: dyn %ld, stat %ld", s->opt_len, + Tracev((stderr, "\nlit data: dyn %lu, stat %lu", s->opt_len, s->static_len)); build_tree(s, (tree_desc *)(&(s->d_desc))); - Tracev((stderr, "\ndist data: dyn %ld, stat %ld", s->opt_len, + Tracev((stderr, "\ndist data: dyn %lu, stat %lu", s->opt_len, s->static_len)); /* At this point, opt_len and static_len are the total bit lengths of * the compressed block data, excluding the tree representations. @@ -1083,7 +1129,7 @@ void ZLIB_INTERNAL _tr_flush_block(deflate_state *s, charf *buf, #endif } Tracev((stderr,"\ncomprlen %lu(%lu) ", s->compressed_len >> 3, - s->compressed_len - 7*last)); + s->compressed_len - 7*(ulg)last)); } /* =========================================================================== diff --git a/packages/runtime/vendor/zlib/uncompr.c b/packages/runtime/vendor/zlib/uncompr.c index 5e256663b..2195e7855 100644 --- a/packages/runtime/vendor/zlib/uncompr.c +++ b/packages/runtime/vendor/zlib/uncompr.c @@ -1,5 +1,5 @@ /* uncompr.c -- decompress a memory buffer - * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2026 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -23,24 +23,24 @@ memory, Z_BUF_ERROR if there was not enough room in the output buffer, or Z_DATA_ERROR if the input data was corrupted, including if the input data is an incomplete zlib stream. + + The _z versions of the functions take size_t length arguments. */ -int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source, - uLong *sourceLen) { +int ZEXPORT uncompress2_z(Bytef *dest, z_size_t *destLen, const Bytef *source, + z_size_t *sourceLen) { z_stream stream; int err; const uInt max = (uInt)-1; - uLong len, left; - Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */ + z_size_t len, left; + + if (sourceLen == NULL || (*sourceLen > 0 && source == NULL) || + destLen == NULL || (*destLen > 0 && dest == NULL)) + return Z_STREAM_ERROR; len = *sourceLen; - if (*destLen) { - left = *destLen; - *destLen = 0; - } - else { - left = 1; - dest = buf; - } + left = *destLen; + if (left == 0 && dest == Z_NULL) + dest = (Bytef *)&stream.reserved; /* next_out cannot be NULL */ stream.next_in = (z_const Bytef *)source; stream.avail_in = 0; @@ -56,30 +56,46 @@ int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source, do { if (stream.avail_out == 0) { - stream.avail_out = left > (uLong)max ? max : (uInt)left; + stream.avail_out = left > (z_size_t)max ? max : (uInt)left; left -= stream.avail_out; } if (stream.avail_in == 0) { - stream.avail_in = len > (uLong)max ? max : (uInt)len; + stream.avail_in = len > (z_size_t)max ? max : (uInt)len; len -= stream.avail_in; } err = inflate(&stream, Z_NO_FLUSH); } while (err == Z_OK); - *sourceLen -= len + stream.avail_in; - if (dest != buf) - *destLen = stream.total_out; - else if (stream.total_out && err == Z_BUF_ERROR) - left = 1; + /* Set len and left to the unused input data and unused output space. Set + *sourceLen to the amount of input consumed. Set *destLen to the amount + of data produced. */ + len += stream.avail_in; + left += stream.avail_out; + *sourceLen -= len; + *destLen -= left; inflateEnd(&stream); return err == Z_STREAM_END ? Z_OK : err == Z_NEED_DICT ? Z_DATA_ERROR : - err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR : + err == Z_BUF_ERROR && len == 0 ? Z_DATA_ERROR : err; } - +int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source, + uLong *sourceLen) { + int ret; + z_size_t got = *destLen, used = *sourceLen; + ret = uncompress2_z(dest, &got, source, &used); + *sourceLen = (uLong)used; + *destLen = (uLong)got; + return ret; +} +int ZEXPORT uncompress_z(Bytef *dest, z_size_t *destLen, const Bytef *source, + z_size_t sourceLen) { + z_size_t used = sourceLen; + return uncompress2_z(dest, destLen, source, &used); +} int ZEXPORT uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen) { - return uncompress2(dest, destLen, source, &sourceLen); + uLong used = sourceLen; + return uncompress2(dest, destLen, source, &used); } diff --git a/packages/runtime/vendor/zlib/zconf.h b/packages/runtime/vendor/zlib/zconf.h index 62adc8d84..4d64a6530 100644 --- a/packages/runtime/vendor/zlib/zconf.h +++ b/packages/runtime/vendor/zlib/zconf.h @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2026 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -8,6 +8,18 @@ #ifndef ZCONF_H #define ZCONF_H +/* + * This library is also built as a part of AOSP, which does not need to include + * chromeconf.h. This config does not want chromeconf.h, so it can set this + * macro to opt out. While this works today, there's no guarantee that building + * zlib outside of Chromium keeps working in the future. + */ +#if !defined(CHROMIUM_ZLIB_NO_CHROMECONF) +/* This include does prefixing as below, but with an updated set of names. Also + * sets up export macros in component builds. */ +//#include "chromeconf.h" +#endif + /* * If you *really* need a unique prefix for all types and library functions, * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. @@ -33,7 +45,10 @@ # ifndef Z_SOLO # define compress z_compress # define compress2 z_compress2 +# define compress_z z_compress_z +# define compress2_z z_compress2_z # define compressBound z_compressBound +# define compressBound_z z_compressBound_z # endif # define crc32 z_crc32 # define crc32_combine z_crc32_combine @@ -44,6 +59,7 @@ # define crc32_z z_crc32_z # define deflate z_deflate # define deflateBound z_deflateBound +# define deflateBound_z z_deflateBound_z # define deflateCopy z_deflateCopy # define deflateEnd z_deflateEnd # define deflateGetDictionary z_deflateGetDictionary @@ -59,6 +75,7 @@ # define deflateSetDictionary z_deflateSetDictionary # define deflateSetHeader z_deflateSetHeader # define deflateTune z_deflateTune +# define deflateUsed z_deflateUsed # define deflate_copyright z_deflate_copyright # define get_crc_table z_get_crc_table # ifndef Z_SOLO @@ -128,9 +145,12 @@ # define inflate_copyright z_inflate_copyright # define inflate_fast z_inflate_fast # define inflate_table z_inflate_table +# define inflate_fixed z_inflate_fixed # ifndef Z_SOLO # define uncompress z_uncompress # define uncompress2 z_uncompress2 +# define uncompress_z z_uncompress_z +# define uncompress2_z z_uncompress2_z # endif # define zError z_zError # ifndef Z_SOLO @@ -234,10 +254,12 @@ # endif #endif -#if defined(ZLIB_CONST) && !defined(z_const) -# define z_const const -#else -# define z_const +#ifndef z_const +# ifdef ZLIB_CONST +# define z_const const +# else +# define z_const +# endif #endif #ifdef Z_SOLO @@ -388,6 +410,9 @@ #ifndef FAR # define FAR #endif +#ifndef far +# define far +#endif #if !defined(__MACTYPES__) typedef unsigned char Byte; /* 8 bits */ @@ -433,11 +458,11 @@ typedef uLong FAR uLongf; typedef unsigned long z_crc_t; #endif -#ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ +#if !defined(_WIN32) # define Z_HAVE_UNISTD_H #endif -#ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ +#if defined(HAVE_STDARG_H) && HAVE_STDARG_H-0 # define Z_HAVE_STDARG_H #endif @@ -470,12 +495,8 @@ typedef uLong FAR uLongf; #endif #ifndef Z_HAVE_UNISTD_H -# ifdef __WATCOMC__ -# define Z_HAVE_UNISTD_H -# endif -#endif -#ifndef Z_HAVE_UNISTD_H -# if defined(_LARGEFILE64_SOURCE) && !defined(_WIN32) +# if defined(__WATCOMC__) || defined(__GO32__) || \ + (defined(_LARGEFILE64_SOURCE) && !defined(_WIN32)) # define Z_HAVE_UNISTD_H # endif #endif @@ -510,17 +531,19 @@ typedef uLong FAR uLongf; #endif #ifndef z_off_t -# define z_off_t long +# define z_off_t long long #endif #if !defined(_WIN32) && defined(Z_LARGE64) # define z_off64_t off64_t +#elif defined(__MINGW32__) +# define z_off64_t long long +#elif defined(_WIN32) && !defined(__GNUC__) +# define z_off64_t __int64 +#elif defined(__GO32__) +# define z_off64_t offset_t #else -# if defined(_WIN32) && !defined(__GNUC__) -# define z_off64_t __int64 -# else -# define z_off64_t z_off_t -# endif +# define z_off64_t z_off_t #endif /* MVS linker does not support external names larger than 8 bytes */ diff --git a/packages/runtime/vendor/zlib/zlib.h b/packages/runtime/vendor/zlib/zlib.h index 8d4b932ea..13909f2f8 100644 --- a/packages/runtime/vendor/zlib/zlib.h +++ b/packages/runtime/vendor/zlib/zlib.h @@ -1,7 +1,7 @@ /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.3.1, January 22nd, 2024 + version 1.3.2.1, February xxth, 2026 - Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler + Copyright (C) 1995-2026 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -24,25 +24,29 @@ The data format used by the zlib library is described by RFCs (Request for - Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950 + Comments) 1950 to 1952 at https://datatracker.ietf.org/doc/html/rfc1950 (zlib format), rfc1951 (deflate format) and rfc1952 (gzip format). */ #ifndef ZLIB_H #define ZLIB_H -#include "zconf.h" +#ifdef ZLIB_BUILD +# include +#else +# include "zconf.h" +#endif #ifdef __cplusplus extern "C" { #endif -#define ZLIB_VERSION "1.3.1" -#define ZLIB_VERNUM 0x1310 +#define ZLIB_VERSION "1.3.2.1-motley" +#define ZLIB_VERNUM 0x1321 #define ZLIB_VER_MAJOR 1 #define ZLIB_VER_MINOR 3 -#define ZLIB_VER_REVISION 1 -#define ZLIB_VER_SUBREVISION 0 +#define ZLIB_VER_REVISION 2 +#define ZLIB_VER_SUBREVISION 1 /* The 'zlib' compression library provides in-memory compression and @@ -359,7 +363,6 @@ ZEXTERN int ZEXPORT deflate(z_streamp strm, int flush); continue compressing. */ - ZEXTERN int ZEXPORT deflateEnd(z_streamp strm); /* All dynamically allocated data structures for this stream are freed. @@ -441,7 +444,7 @@ ZEXTERN int ZEXPORT inflate(z_streamp strm, int flush); The Z_BLOCK option assists in appending to or combining deflate streams. To assist in this, on return inflate() always sets strm->data_type to the - number of unused bits in the last byte taken from strm->next_in, plus 64 if + number of unused bits in the input taken from strm->next_in, plus 64 if inflate() is currently decoding the last block in the deflate stream, plus 128 if inflate() returned immediately after decoding an end-of-block code or decoding the complete header up to just before the first byte of the deflate @@ -587,18 +590,21 @@ ZEXTERN int ZEXPORT deflateInit2(z_streamp strm, The strategy parameter is used to tune the compression algorithm. Use the value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a - filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no - string match), or Z_RLE to limit match distances to one (run-length - encoding). Filtered data consists mostly of small values with a somewhat - random distribution. In this case, the compression algorithm is tuned to - compress them better. The effect of Z_FILTERED is to force more Huffman - coding and less string matching; it is somewhat intermediate between - Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as - fast as Z_HUFFMAN_ONLY, but give better compression for PNG image data. The - strategy parameter only affects the compression ratio but not the - correctness of the compressed output even if it is not set appropriately. - Z_FIXED prevents the use of dynamic Huffman codes, allowing for a simpler - decoder for special applications. + filter (or predictor), Z_RLE to limit match distances to one (run-length + encoding), or Z_HUFFMAN_ONLY to force Huffman encoding only (no string + matching). Filtered data consists mostly of small values with a somewhat + random distribution, as produced by the PNG filters. In this case, the + compression algorithm is tuned to compress them better. The effect of + Z_FILTERED is to force more Huffman coding and less string matching than the + default; it is intermediate between Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY. + Z_RLE is almost as fast as Z_HUFFMAN_ONLY, but should give better + compression for PNG image data than Huffman only. The degree of string + matching from most to none is: Z_DEFAULT_STRATEGY, Z_FILTERED, Z_RLE, then + Z_HUFFMAN_ONLY. The strategy parameter affects the compression ratio but + never the correctness of the compressed output, even if it is not set + optimally for the given data. Z_FIXED uses the default string matching, but + prevents the use of dynamic Huffman codes, allowing for a simpler decoder + for special applications. deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_STREAM_ERROR if any parameter is invalid (such as an invalid @@ -758,8 +764,8 @@ ZEXTERN int ZEXPORT deflateTune(z_streamp strm, returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream. */ -ZEXTERN uLong ZEXPORT deflateBound(z_streamp strm, - uLong sourceLen); +ZEXTERN uLong ZEXPORT deflateBound(z_streamp strm, uLong sourceLen); +ZEXTERN z_size_t ZEXPORT deflateBound_z(z_streamp strm, z_size_t sourceLen); /* deflateBound() returns an upper bound on the compressed size after deflation of sourceLen bytes. It must be called after deflateInit() or @@ -771,6 +777,9 @@ ZEXTERN uLong ZEXPORT deflateBound(z_streamp strm, to return Z_STREAM_END. Note that it is possible for the compressed size to be larger than the value returned by deflateBound() if flush options other than Z_FINISH or Z_NO_FLUSH are used. + + delfateBound_z() is the same, but takes and returns a size_t length. Note + that a long is 32 bits on Windows. */ ZEXTERN int ZEXPORT deflatePending(z_streamp strm, @@ -785,6 +794,21 @@ ZEXTERN int ZEXPORT deflatePending(z_streamp strm, or bits are Z_NULL, then those values are not set. deflatePending returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. If an int is 16 bits and memLevel is 9, then + it is possible for the number of pending bytes to not fit in an unsigned. In + that case Z_BUF_ERROR is returned and *pending is set to the maximum value + of an unsigned. + */ + +ZEXTERN int ZEXPORT deflateUsed(z_streamp strm, + int *bits); +/* + deflateUsed() returns in *bits the most recent number of deflate bits used + in the last byte when flushing to a byte boundary. The result is in 1..8, or + 0 if there has not yet been a flush. This helps determine the location of + the last bit of a deflate stream. + + deflateUsed returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent. */ @@ -987,13 +1011,15 @@ ZEXTERN int ZEXPORT inflatePrime(z_streamp strm, int bits, int value); /* - This function inserts bits in the inflate input stream. The intent is - that this function is used to start inflating at a bit position in the - middle of a byte. The provided bits will be used before any bytes are used - from next_in. This function should only be used with raw inflate, and - should be used before the first inflate() call after inflateInit2() or - inflateReset(). bits must be less than or equal to 16, and that many of the - least significant bits of value will be inserted in the input. + This function inserts bits in the inflate input stream. The intent is to + use inflatePrime() to start inflating at a bit position in the middle of a + byte. The provided bits will be used before any bytes are used from + next_in. This function should be used with raw inflate, before the first + inflate() call, after inflateInit2() or inflateReset(). It can also be used + after an inflate() return indicates the end of a deflate block or header + when using Z_BLOCK. bits must be less than or equal to 16, and that many of + the least significant bits of value will be inserted in the input. The + other bits in value can be non-zero, and will be ignored. If bits is negative, then the input stream bit buffer is emptied. Then inflatePrime() can be called again to put bits in the buffer. This is used @@ -1001,7 +1027,15 @@ ZEXTERN int ZEXPORT inflatePrime(z_streamp strm, to feeding inflate codes. inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source - stream state was inconsistent. + stream state was inconsistent, or if bits is out of range. If inflate was + in the middle of processing a header, trailer, or stored block lengths, then + it is possible for there to be only eight bits available in the bit buffer. + In that case, bits > 8 is considered out of range. However, when used as + outlined above, there will always be 16 bits available in the buffer for + insertion. As noted in its documentation above, inflate records the number + of bits in the bit buffer on return in data_type. 32 minus that is the + number of bits available for insertion. inflatePrime does not update + data_type with the new number of bits in buffer. */ ZEXTERN long ZEXPORT inflateMark(z_streamp strm); @@ -1047,20 +1081,22 @@ ZEXTERN int ZEXPORT inflateGetHeader(z_streamp strm, The text, time, xflags, and os fields are filled in with the gzip header contents. hcrc is set to true if there is a header CRC. (The header CRC - was valid if done is set to one.) If extra is not Z_NULL, then extra_max - contains the maximum number of bytes to write to extra. Once done is true, - extra_len contains the actual extra field length, and extra contains the - extra field, or that field truncated if extra_max is less than extra_len. - If name is not Z_NULL, then up to name_max characters are written there, - terminated with a zero unless the length is greater than name_max. If - comment is not Z_NULL, then up to comm_max characters are written there, - terminated with a zero unless the length is greater than comm_max. When any - of extra, name, or comment are not Z_NULL and the respective field is not - present in the header, then that field is set to Z_NULL to signal its - absence. This allows the use of deflateSetHeader() with the returned - structure to duplicate the header. However if those fields are set to - allocated memory, then the application will need to save those pointers - elsewhere so that they can be eventually freed. + was valid if done is set to one.) The extra, name, and comment pointers + much each be either Z_NULL or point to space to store that information from + the header. If extra is not Z_NULL, then extra_max contains the maximum + number of bytes that can be written to extra. Once done is true, extra_len + contains the actual extra field length, and extra contains the extra field, + or that field truncated if extra_max is less than extra_len. If name is not + Z_NULL, then up to name_max characters, including the terminating zero, are + written there. If comment is not Z_NULL, then up to comm_max characters, + including the terminating zero, are written there. The application can tell + that the name or comment did not fit in the provided space by the absence of + a terminating zero. If any of extra, name, or comment are not present in + the header, then that field's pointer is set to Z_NULL. This allows the use + of deflateSetHeader() with the returned structure to duplicate the header. + Note that if those fields initially pointed to allocated memory, then the + application will need to save them elsewhere so that they can be eventually + freed. If inflateGetHeader is not used, then the header information is simply discarded. The header is always checked for validity, including the header @@ -1208,13 +1244,14 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags(void); 21: FASTEST -- deflate algorithm with only one, lowest compression level 22,23: 0 (reserved) - The sprintf variant used by gzprintf (zero is best): + The sprintf variant used by gzprintf (all zeros is best): 24: 0 = vs*, 1 = s* -- 1 means limited to 20 arguments after the format - 25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() not secure! + 25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() is not secure! 26: 0 = returns value, 1 = void -- 1 means inferred string length returned + 27: 0 = gzprintf() present, 1 = not -- 1 means gzprintf() returns an error Remainder: - 27-31: 0 (reserved) + 28-31: 0 (reserved) */ #ifndef Z_SOLO @@ -1226,11 +1263,14 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags(void); stream-oriented functions. To simplify the interface, some default options are assumed (compression level and memory usage, standard memory allocation functions). The source code of these utility functions can be modified if - you need special options. + you need special options. The _z versions of the functions use the size_t + type for lengths. Note that a long is 32 bits on Windows. */ -ZEXTERN int ZEXPORT compress(Bytef *dest, uLongf *destLen, +ZEXTERN int ZEXPORT compress(Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen); +ZEXTERN int ZEXPORT compress_z(Bytef *dest, z_size_t *destLen, + const Bytef *source, z_size_t sourceLen); /* Compresses the source buffer into the destination buffer. sourceLen is the byte length of the source buffer. Upon entry, destLen is the total size @@ -1244,9 +1284,12 @@ ZEXTERN int ZEXPORT compress(Bytef *dest, uLongf *destLen, buffer. */ -ZEXTERN int ZEXPORT compress2(Bytef *dest, uLongf *destLen, +ZEXTERN int ZEXPORT compress2(Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen, int level); +ZEXTERN int ZEXPORT compress2_z(Bytef *dest, z_size_t *destLen, + const Bytef *source, z_size_t sourceLen, + int level); /* Compresses the source buffer into the destination buffer. The level parameter has the same meaning as in deflateInit. sourceLen is the byte @@ -1261,21 +1304,24 @@ ZEXTERN int ZEXPORT compress2(Bytef *dest, uLongf *destLen, */ ZEXTERN uLong ZEXPORT compressBound(uLong sourceLen); +ZEXTERN z_size_t ZEXPORT compressBound_z(z_size_t sourceLen); /* compressBound() returns an upper bound on the compressed size after compress() or compress2() on sourceLen bytes. It would be used before a compress() or compress2() call to allocate the destination buffer. */ -ZEXTERN int ZEXPORT uncompress(Bytef *dest, uLongf *destLen, +ZEXTERN int ZEXPORT uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen); +ZEXTERN int ZEXPORT uncompress_z(Bytef *dest, z_size_t *destLen, + const Bytef *source, z_size_t sourceLen); /* Decompresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total size + the byte length of the source buffer. On entry, *destLen is the total size of the destination buffer, which must be large enough to hold the entire uncompressed data. (The size of the uncompressed data must have been saved previously by the compressor and transmitted to the decompressor by some - mechanism outside the scope of this compression library.) Upon exit, destLen + mechanism outside the scope of this compression library.) On exit, *destLen is the actual size of the uncompressed data. uncompress returns Z_OK if success, Z_MEM_ERROR if there was not @@ -1285,8 +1331,10 @@ ZEXTERN int ZEXPORT uncompress(Bytef *dest, uLongf *destLen, buffer with the uncompressed data up to that point. */ -ZEXTERN int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, +ZEXTERN int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source, uLong *sourceLen); +ZEXTERN int ZEXPORT uncompress2_z(Bytef *dest, z_size_t *destLen, + const Bytef *source, z_size_t *sourceLen); /* Same as uncompress, except that sourceLen is a pointer, where the length of the source is *sourceLen. On return, *sourceLen is the number of @@ -1314,13 +1362,17 @@ ZEXTERN gzFile ZEXPORT gzopen(const char *path, const char *mode); 'R' for run-length encoding as in "wb1R", or 'F' for fixed code compression as in "wb9F". (See the description of deflateInit2 for more information about the strategy parameter.) 'T' will request transparent writing or - appending with no compression and not using the gzip format. + appending with no compression and not using the gzip format. 'T' cannot be + used to force transparent reading. Transparent reading is automatically + performed if there is no gzip header at the start. Transparent reading can + be disabled with the 'G' option, which will instead return an error if there + is no gzip header. 'N' will open the file in non-blocking mode. - "a" can be used instead of "w" to request that the gzip stream that will - be written be appended to the file. "+" will result in an error, since + 'a' can be used instead of 'w' to request that the gzip stream that will + be written be appended to the file. '+' will result in an error, since reading and writing to the same gzip file is not supported. The addition of - "x" when writing will create the file exclusively, which fails if the file - already exists. On systems that support it, the addition of "e" when + 'x' when writing will create the file exclusively, which fails if the file + already exists. On systems that support it, the addition of 'e' when reading or writing will set the flag to close the file on an execve() call. These functions, as well as gzip, will read and decode a sequence of gzip @@ -1339,14 +1391,22 @@ ZEXTERN gzFile ZEXPORT gzopen(const char *path, const char *mode); insufficient memory to allocate the gzFile state, or if an invalid mode was specified (an 'r', 'w', or 'a' was not provided, or '+' was provided). errno can be checked to determine if the reason gzopen failed was that the - file could not be opened. + file could not be opened. Note that if 'N' is in mode for non-blocking, the + open() itself can fail in order to not block. In that case gzopen() will + return NULL and errno will be EAGAIN or ENONBLOCK. The call to gzopen() can + then be re-tried. If the application would like to block on opening the + file, then it can use open() without O_NONBLOCK, and then gzdopen() with the + resulting file descriptor and 'N' in the mode, which will set it to non- + blocking. */ ZEXTERN gzFile ZEXPORT gzdopen(int fd, const char *mode); /* Associate a gzFile with the file descriptor fd. File descriptors are obtained from calls like open, dup, creat, pipe or fileno (if the file has - been previously opened with fopen). The mode parameter is as in gzopen. + been previously opened with fopen). The mode parameter is as in gzopen. An + 'e' in mode will set fd's flag to close the file on an execve() call. An 'N' + in mode will set fd's non-blocking flag. The next call of gzclose on the returned gzFile will also close the file descriptor fd, just like fclose(fdopen(fd, mode)) closes the file descriptor @@ -1416,10 +1476,16 @@ ZEXTERN int ZEXPORT gzread(gzFile file, voidp buf, unsigned len); stream. Alternatively, gzerror can be used before gzclose to detect this case. + gzread can be used to read a gzip file on a non-blocking device. If the + input stalls and there is no uncompressed data to return, then gzread() will + return -1, and errno will be EAGAIN or EWOULDBLOCK. gzread() can then be + called again. + gzread returns the number of uncompressed bytes actually read, less than len for end of file, or -1 for error. If len is too large to fit in an int, then nothing is read, -1 is returned, and the error state is set to - Z_STREAM_ERROR. + Z_STREAM_ERROR. If some data was read before an error, then that data is + returned until exhausted, after which the next call will signal the error. */ ZEXTERN z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, @@ -1443,15 +1509,20 @@ ZEXTERN z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, multiple of size, then the final partial item is nevertheless read into buf and the end-of-file flag is set. The length of the partial item read is not provided, but could be inferred from the result of gztell(). This behavior - is the same as the behavior of fread() implementations in common libraries, - but it prevents the direct use of gzfread() to read a concurrently written - file, resetting and retrying on end-of-file, when size is not 1. + is the same as that of fread() implementations in common libraries. This + could result in data loss if used with size != 1 when reading a concurrently + written file or a non-blocking file. In that case, use size == 1 or gzread() + instead. */ ZEXTERN int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len); /* Compress and write the len uncompressed bytes at buf to file. gzwrite - returns the number of uncompressed bytes written or 0 in case of error. + returns the number of uncompressed bytes written, or 0 in case of error or + if len is 0. If the write destination is non-blocking, then gzwrite() may + return a number of bytes written that is not 0 and less than len. + + If len does not fit in an int, then 0 is returned and nothing is written. */ ZEXTERN z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size, @@ -1466,9 +1537,18 @@ ZEXTERN z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size, if there was an error. If the multiplication of size and nitems overflows, i.e. the product does not fit in a z_size_t, then nothing is written, zero is returned, and the error state is set to Z_STREAM_ERROR. + + If writing a concurrently read file or a non-blocking file with size != 1, + a partial item could be written, with no way of knowing how much of it was + not written, resulting in data loss. In that case, use size == 1 or + gzwrite() instead. */ +#if defined(STDC) || defined(Z_HAVE_STDARG_H) ZEXTERN int ZEXPORTVA gzprintf(gzFile file, const char *format, ...); +#else +ZEXTERN int ZEXPORTVA gzprintf(); +#endif /* Convert, format, compress, and write the arguments (...) to file under control of the string format, as in fprintf. gzprintf returns the number of @@ -1476,11 +1556,19 @@ ZEXTERN int ZEXPORTVA gzprintf(gzFile file, const char *format, ...); of error. The number of uncompressed bytes written is limited to 8191, or one less than the buffer size given to gzbuffer(). The caller should assure that this limit is not exceeded. If it is exceeded, then gzprintf() will - return an error (0) with nothing written. In this case, there may also be a - buffer overflow with unpredictable consequences, which is possible only if - zlib was compiled with the insecure functions sprintf() or vsprintf(), - because the secure snprintf() or vsnprintf() functions were not available. - This can be determined using zlibCompileFlags(). + return an error (0) with nothing written. + + In that last case, there may also be a buffer overflow with unpredictable + consequences, which is possible only if zlib was compiled with the insecure + functions sprintf() or vsprintf(), because the secure snprintf() and + vsnprintf() functions were not available. That would only be the case for + a non-ANSI C compiler. zlib may have been built without gzprintf() because + secure functions were not available and having gzprintf() be insecure was + not an option, in which case, gzprintf() returns Z_STREAM_ERROR. All of + these possibilities can be determined using zlibCompileFlags(). + + If a Z_BUF_ERROR is returned, then nothing was written due to a stall on + the non-blocking write destination. */ ZEXTERN int ZEXPORT gzputs(gzFile file, const char *s); @@ -1489,6 +1577,11 @@ ZEXTERN int ZEXPORT gzputs(gzFile file, const char *s); the terminating null character. gzputs returns the number of characters written, or -1 in case of error. + The number of characters written may be less than the length of the string + if the write destination is non-blocking. + + If the length of the string does not fit in an int, then -1 is returned + and nothing is written. */ ZEXTERN char * ZEXPORT gzgets(gzFile file, char *buf, int len); @@ -1501,8 +1594,13 @@ ZEXTERN char * ZEXPORT gzgets(gzFile file, char *buf, int len); left untouched. gzgets returns buf which is a null-terminated string, or it returns NULL - for end-of-file or in case of error. If there was an error, the contents at - buf are indeterminate. + for end-of-file or in case of error. If some data was read before an error, + then that data is returned until exhausted, after which the next call will + return NULL to signal the error. + + gzgets can be used on a file being concurrently written, and on a non- + blocking device, both as for gzread(). However lines may be broken in the + middle, leaving it up to the application to reassemble them as needed. */ ZEXTERN int ZEXPORT gzputc(gzFile file, int c); @@ -1513,11 +1611,19 @@ ZEXTERN int ZEXPORT gzputc(gzFile file, int c); ZEXTERN int ZEXPORT gzgetc(gzFile file); /* - Read and decompress one byte from file. gzgetc returns this byte or -1 - in case of end of file or error. This is implemented as a macro for speed. - As such, it does not do all of the checking the other functions do. I.e. - it does not check to see if file is NULL, nor whether the structure file - points to has been clobbered or not. + Read and decompress one byte from file. gzgetc returns this byte or -1 in + case of end of file or error. If some data was read before an error, then + that data is returned until exhausted, after which the next call will return + -1 to signal the error. + + This is implemented as a macro for speed. As such, it does not do all of + the checking the other functions do. I.e. it does not check to see if file + is NULL, nor whether the structure file points to has been clobbered or not. + + gzgetc can be used to read a gzip file on a non-blocking device. If the + input stalls and there is no uncompressed data to return, then gzgetc() will + return -1, and errno will be EAGAIN or EWOULDBLOCK. gzread() can then be + called again. */ ZEXTERN int ZEXPORT gzungetc(int c, gzFile file); @@ -1530,6 +1636,11 @@ ZEXTERN int ZEXPORT gzungetc(int c, gzFile file); output buffer size of pushed characters is allowed. (See gzbuffer above.) The pushed character will be discarded if the stream is repositioned with gzseek() or gzrewind(). + + gzungetc(-1, file) will force any pending seek to execute. Then gztell() + will report the position, even if the requested seek reached end of file. + This can be used to determine the number of uncompressed bytes in a gzip + file without having to read it into a buffer. */ ZEXTERN int ZEXPORT gzflush(gzFile file, int flush); @@ -1559,7 +1670,8 @@ ZEXTERN z_off_t ZEXPORT gzseek(gzFile file, If the file is opened for reading, this function is emulated but can be extremely slow. If the file is opened for writing, only forward seeks are supported; gzseek then compresses a sequence of zeroes up to the new - starting position. + starting position. For reading or writing, any actual seeking is deferred + until the next read or write operation, or close operation when writing. gzseek returns the resulting offset location as measured in bytes from the beginning of the uncompressed stream, or -1 in case of error, in @@ -1567,7 +1679,7 @@ ZEXTERN z_off_t ZEXPORT gzseek(gzFile file, would be before the current position. */ -ZEXTERN int ZEXPORT gzrewind(gzFile file); +ZEXTERN int ZEXPORT gzrewind(gzFile file); /* Rewind file. This function is supported only for reading. @@ -1575,7 +1687,7 @@ ZEXTERN int ZEXPORT gzrewind(gzFile file); */ /* -ZEXTERN z_off_t ZEXPORT gztell(gzFile file); +ZEXTERN z_off_t ZEXPORT gztell(gzFile file); Return the starting position for the next gzread or gzwrite on file. This position represents a number of bytes in the uncompressed data stream, @@ -1620,8 +1732,11 @@ ZEXTERN int ZEXPORT gzdirect(gzFile file); If gzdirect() is used immediately after gzopen() or gzdopen() it will cause buffers to be allocated to allow reading the file to determine if it - is a gzip file. Therefore if gzbuffer() is used, it should be called before - gzdirect(). + is a gzip file. Therefore if gzbuffer() is used, it should be called before + gzdirect(). If the input is being written concurrently or the device is non- + blocking, then gzdirect() may give a different answer once four bytes of + input have been accumulated, which is what is needed to confirm or deny a + gzip header. Before this, gzdirect() will return true (1). When writing, gzdirect() returns true (1) if transparent writing was requested ("wT" for the gzopen() mode), or false (0) otherwise. (Note: @@ -1631,7 +1746,7 @@ ZEXTERN int ZEXPORT gzdirect(gzFile file); gzip file reading and decompression, which may not be desired.) */ -ZEXTERN int ZEXPORT gzclose(gzFile file); +ZEXTERN int ZEXPORT gzclose(gzFile file); /* Flush all pending output for file, if necessary, close file and deallocate the (de)compression state. Note that once file is closed, you @@ -1659,9 +1774,10 @@ ZEXTERN int ZEXPORT gzclose_w(gzFile file); ZEXTERN const char * ZEXPORT gzerror(gzFile file, int *errnum); /* Return the error message for the last error which occurred on file. - errnum is set to zlib error number. If an error occurred in the file system - and not in the compression library, errnum is set to Z_ERRNO and the - application may consult errno to get the exact error code. + If errnum is not NULL, *errnum is set to zlib error number. If an error + occurred in the file system and not in the compression library, *errnum is + set to Z_ERRNO and the application may consult errno to get the exact error + code. The application must not modify the returned string. Future calls to this function may invalidate the previously returned string. If file is @@ -1712,7 +1828,8 @@ ZEXTERN uLong ZEXPORT adler32(uLong adler, const Bytef *buf, uInt len); ZEXTERN uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf, z_size_t len); /* - Same as adler32(), but with a size_t length. + Same as adler32(), but with a size_t length. Note that a long is 32 bits + on Windows. */ /* @@ -1748,7 +1865,8 @@ ZEXTERN uLong ZEXPORT crc32(uLong crc, const Bytef *buf, uInt len); ZEXTERN uLong ZEXPORT crc32_z(uLong crc, const Bytef *buf, z_size_t len); /* - Same as crc32(), but with a size_t length. + Same as crc32(), but with a size_t length. Note that a long is 32 bits on + Windows. */ /* @@ -1758,14 +1876,14 @@ ZEXTERN uLong ZEXPORT crc32_combine(uLong crc1, uLong crc2, z_off_t len2); seq1 and seq2 with lengths len1 and len2, CRC-32 check values were calculated for each, crc1 and crc2. crc32_combine() returns the CRC-32 check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and - len2. len2 must be non-negative. + len2. len2 must be non-negative, otherwise zero is returned. */ /* ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t len2); Return the operator corresponding to length len2, to be used with - crc32_combine_op(). len2 must be non-negative. + crc32_combine_op(). len2 must be non-negative, otherwise zero is returned. */ ZEXTERN uLong ZEXPORT crc32_combine_op(uLong crc1, uLong crc2, uLong op); @@ -1844,6 +1962,11 @@ ZEXTERN int ZEXPORT gzgetc_(gzFile file); /* backward compatibility */ # undef z_gzgetc # define z_gzgetc(g) \ ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +#elif defined(Z_CR_PREFIX_SET) +# undef gzgetc +# define gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) \ + : (Cr_z_gzgetc)(g)) #else # define gzgetc(g) \ ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) @@ -1875,22 +1998,47 @@ ZEXTERN int ZEXPORT gzgetc_(gzFile file); /* backward compatibility */ # define z_crc32_combine z_crc32_combine64 # define z_crc32_combine_gen z_crc32_combine_gen64 # else +# ifdef gzopen +# undef gzopen +# endif # define gzopen gzopen64 +# ifdef gzseek +# undef gzseek +# endif # define gzseek gzseek64 +# ifdef gztell +# undef gztell +# endif # define gztell gztell64 +# ifdef gzoffset +# undef gzoffset +# endif # define gzoffset gzoffset64 +# ifdef adler32_combine +# undef adler32_combine +# endif # define adler32_combine adler32_combine64 -# define crc32_combine crc32_combine64 -# define crc32_combine_gen crc32_combine_gen64 +# ifdef crc32_combine +# undef crc32_combine +# endif +# ifdef crc32_combine64 +# undef crc32_combine64 +# endif +# ifdef crc32_combine_gen +# undef crc32_combine_gen +# endif +# ifdef crc32_combine_op +# undef crc32_combine_op +# endif # endif # ifndef Z_LARGE64 ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *); ZEXTERN z_off_t ZEXPORT gzseek64(gzFile, z_off_t, int); ZEXTERN z_off_t ZEXPORT gztell64(gzFile); ZEXTERN z_off_t ZEXPORT gzoffset64(gzFile); - ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off_t); + ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off64_t); + ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off64_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off64_t); # endif #else ZEXTERN gzFile ZEXPORT gzopen(const char *, const char *); diff --git a/packages/runtime/vendor/zlib/zutil.c b/packages/runtime/vendor/zlib/zutil.c index b1c5d2d3c..0e30c5664 100644 --- a/packages/runtime/vendor/zlib/zutil.c +++ b/packages/runtime/vendor/zlib/zutil.c @@ -1,5 +1,5 @@ /* zutil.c -- target dependent utility functions for the compression library - * Copyright (C) 1995-2017 Jean-loup Gailly + * Copyright (C) 1995-2026 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -86,28 +86,36 @@ uLong ZEXPORT zlibCompileFlags(void) { flags += 1L << 21; #endif #if defined(STDC) || defined(Z_HAVE_STDARG_H) -# ifdef NO_vsnprintf - flags += 1L << 25; -# ifdef HAS_vsprintf_void - flags += 1L << 26; -# endif -# else -# ifdef HAS_vsnprintf_void - flags += 1L << 26; -# endif -# endif +# ifdef NO_vsnprintf +# ifdef ZLIB_INSECURE + flags += 1L << 25; +# else + flags += 1L << 27; +# endif +# ifdef HAS_vsprintf_void + flags += 1L << 26; +# endif +# else +# ifdef HAS_vsnprintf_void + flags += 1L << 26; +# endif +# endif #else flags += 1L << 24; -# ifdef NO_snprintf - flags += 1L << 25; -# ifdef HAS_sprintf_void - flags += 1L << 26; -# endif -# else -# ifdef HAS_snprintf_void - flags += 1L << 26; -# endif -# endif +# ifdef NO_snprintf +# ifdef ZLIB_INSECURE + flags += 1L << 25; +# else + flags += 1L << 27; +# endif +# ifdef HAS_sprintf_void + flags += 1L << 26; +# endif +# else +# ifdef HAS_snprintf_void + flags += 1L << 26; +# endif +# endif #endif return flags; } @@ -142,28 +150,33 @@ const char * ZEXPORT zError(int err) { #ifndef HAVE_MEMCPY -void ZLIB_INTERNAL zmemcpy(Bytef* dest, const Bytef* source, uInt len) { - if (len == 0) return; - do { - *dest++ = *source++; /* ??? to be unrolled */ - } while (--len != 0); +void ZLIB_INTERNAL zmemcpy(void FAR *dst, const void FAR *src, z_size_t n) { + uchf *p = dst; + const uchf *q = src; + while (n) { + *p++ = *q++; + n--; + } } -int ZLIB_INTERNAL zmemcmp(const Bytef* s1, const Bytef* s2, uInt len) { - uInt j; - - for (j = 0; j < len; j++) { - if (s1[j] != s2[j]) return 2*(s1[j] > s2[j])-1; +int ZLIB_INTERNAL zmemcmp(const void FAR *s1, const void FAR *s2, z_size_t n) { + const uchf *p = s1, *q = s2; + while (n) { + if (*p++ != *q++) + return (int)p[-1] - (int)q[-1]; + n--; } return 0; } -void ZLIB_INTERNAL zmemzero(Bytef* dest, uInt len) { - if (len == 0) return; - do { - *dest++ = 0; /* ??? to be unrolled */ - } while (--len != 0); +void ZLIB_INTERNAL zmemzero(void FAR *b, z_size_t len) { + uchf *p = b; + while (len) { + *p++ = 0; + len--; + } } + #endif #ifndef Z_SOLO diff --git a/packages/runtime/vendor/zlib/zutil.h b/packages/runtime/vendor/zlib/zutil.h index 48dd7feba..64cd7a41e 100644 --- a/packages/runtime/vendor/zlib/zutil.h +++ b/packages/runtime/vendor/zlib/zutil.h @@ -1,5 +1,5 @@ /* zutil.h -- internal interface and configuration of the compression library - * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2026 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -28,6 +28,21 @@ # include # include #endif +#ifdef NO_ERRNO_H +# ifdef _WIN32_WCE + /* The Microsoft C Run-Time Library for Windows CE doesn't have + * errno. We define it as a global variable to simplify porting. + * Its value is always 0 and should not be used. We rename it to + * avoid conflict with other libraries that use the same workaround. + */ +# define errno z_errno +# endif + extern int errno; +#else +# ifndef _WIN32_WCE +# include +# endif +#endif #ifndef local # define local static @@ -36,6 +51,10 @@ define "local" for the non-static meaning of "static", for readability (compile with -Dlocal if your debugger can't find static symbols) */ +extern const char deflate_copyright[]; +extern const char inflate_copyright[]; +extern const char inflate9_copyright[]; + typedef unsigned char uch; typedef uch FAR uchf; typedef unsigned short ush; @@ -48,6 +67,8 @@ typedef unsigned long ulg; # define Z_U8 unsigned long # elif (ULLONG_MAX == 0xffffffffffffffff) # define Z_U8 unsigned long long +# elif (ULONG_LONG_MAX == 0xffffffffffffffff) +# define Z_U8 unsigned long long # elif (UINT_MAX == 0xffffffffffffffff) # define Z_U8 unsigned # endif @@ -63,7 +84,9 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* To be used only when the state is known to be valid */ /* common constants */ - +#if MAX_WBITS < 9 || MAX_WBITS > 15 +# error MAX_WBITS must be in 9..15 +#endif #ifndef DEF_WBITS # define DEF_WBITS MAX_WBITS #endif @@ -141,7 +164,7 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # define OS_CODE 7 #endif -#ifdef __acorn +#if defined(__acorn) || defined(__riscos) # define OS_CODE 13 #endif @@ -168,11 +191,10 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #endif /* provide prototypes for these when building zlib without LFS */ -#if !defined(_WIN32) && \ - (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0) - ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off_t); +#ifndef Z_LARGE64 + ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off64_t); + ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off64_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off64_t); #endif /* common defaults */ @@ -211,9 +233,9 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # define zmemzero(dest, len) memset(dest, 0, len) # endif #else - void ZLIB_INTERNAL zmemcpy(Bytef* dest, const Bytef* source, uInt len); - int ZLIB_INTERNAL zmemcmp(const Bytef* s1, const Bytef* s2, uInt len); - void ZLIB_INTERNAL zmemzero(Bytef* dest, uInt len); + void ZLIB_INTERNAL zmemcpy(void FAR *, const void FAR *, z_size_t); + int ZLIB_INTERNAL zmemcmp(const void FAR *, const void FAR *, z_size_t); + void ZLIB_INTERNAL zmemzero(void FAR *, z_size_t); #endif /* Diagnostic functions */ @@ -251,4 +273,80 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) +#ifdef _MSC_VER +#define zalign(x) __declspec(align(x)) +#else +#define zalign(x) __attribute__((aligned((x)))) +#endif + +#ifdef Z_ONCE +/* + Create a local z_once() function depending on the availability of atomics. + */ + +/* Check for the availability of atomics. */ +#if defined(__STDC__) && __STDC_VERSION__ >= 201112L && \ + !defined(__STDC_NO_ATOMICS__) + +#include +typedef struct { + atomic_flag begun; + atomic_int done; +} z_once_t; +#define Z_ONCE_INIT {ATOMIC_FLAG_INIT, 0} + +/* + Run the provided init() function exactly once, even if multiple threads + invoke once() at the same time. The state must be a once_t initialized with + Z_ONCE_INIT. + */ +local void z_once(z_once_t *state, void (*init)(void)) { + if (!atomic_load(&state->done)) { + if (atomic_flag_test_and_set(&state->begun)) + while (!atomic_load(&state->done)) + ; + else { + init(); + atomic_store(&state->done, 1); + } + } +} + +#else /* no atomics */ + +#warning zlib not thread-safe + +typedef struct z_once_s { + volatile int begun; + volatile int done; +} z_once_t; +#define Z_ONCE_INIT {0, 0} + +/* Test and set. Alas, not atomic, but tries to limit the period of + vulnerability. */ +local int test_and_set(int volatile *flag) { + int was; + + was = *flag; + *flag = 1; + return was; +} + +/* Run the provided init() function once. This is not thread-safe. */ +local void z_once(z_once_t *state, void (*init)(void)) { + if (!state->done) { + if (test_and_set(&state->begun)) + while (!state->done) + ; + else { + init(); + state->done = 1; + } + } +} + +#endif /* ?atomics */ + +#endif /* Z_ONCE */ + #endif /* ZUTIL_H */ diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index 3dbe22bf3..22b9967ba 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -31,10 +31,10 @@ const sanitize = process.env["SCRIPTC_SAN"] === "1"; // SCRIPTC_NATIVE_PROFILE_USE builds without one. const profileGenerate = process.env["SCRIPTC_NATIVE_PROFILE_GENERATE"] || undefined; // Below this share of matching functions, the build warns that the profile -// has lost most of its gain. A profile refreshed on the same commit matched -// 92.7% of the native compiler's profiled functions, one refreshed 1.5 days -// and about 200 commits earlier 87.2%, and that one kept about half of the -// fresh profile's gain on a cold tsc-ts build. +// has lost most of its gain. A profile refreshed on the same commit, with +// the same layout, matches all of the native compiler's profiled functions. +// One refreshed 1.5 days and about 200 commits earlier matched 87.2%, and +// kept about half of the fresh profile's gain on a cold tsc-ts build. const PROFILE_MATCH_WARNING = 0.9; const committedProfile = join(root, "packages/compiler/native/cli.profdata.gz"); const requestedProfile = process.env["SCRIPTC_NATIVE_PROFILE_USE"]; @@ -46,11 +46,15 @@ const useProfile = (requestedProfile !== undefined || existsSync(committedProfile)); // Builds lay out the compiler's functions with the committed layout // (scripts/refresh-cli-layout.mjs), or the one SCRIPTC_NATIVE_LAYOUT names; -// an empty SCRIPTC_NATIVE_LAYOUT, and instrumented builds, use none. +// an empty SCRIPTC_NATIVE_LAYOUT uses none. Instrumented builds use the same +// layout as the builds their profile is for. The layout decides which +// functions share a partition, and a function whose partition differs from +// the instrumented build's is optimized differently before its control flow +// is hashed, so the profile's record for it no longer matches. const committedLayout = join(root, "packages/compiler/native/cli.layout.gz"); const requestedLayout = process.env["SCRIPTC_NATIVE_LAYOUT"]; const functionLayout = - profileGenerate !== undefined || requestedLayout === "" + requestedLayout === "" ? undefined : (requestedLayout ?? (existsSync(committedLayout) ? committedLayout : undefined)); let target = nativeCodegenTarget(); diff --git a/scripts/generate-ir-fields.mjs b/scripts/generate-ir-fields.mjs index 5b5ad011a..95ce5d217 100644 --- a/scripts/generate-ir-fields.mjs +++ b/scripts/generate-ir-fields.mjs @@ -14,202 +14,283 @@ const require = createRequire(join(root, "package.json")); const ts = require("typescript"); const irPath = join(root, "packages/compiler/src/ir/ir.ts"); const target = join(root, "packages/compiler/src/ir/fields.generated.ts"); +const samePath = (a, b) => a.replace(/\\/g, "/") === b.replace(/\\/g, "/"); -const program = ts.createProgram([irPath], { strict: true, target: ts.ScriptTarget.ES2022 }); -const checker = program.getTypeChecker(); -const source = program.getSourceFile(irPath); -const aliases = new Map(); -ts.forEachChild(source, (node) => { - if (ts.isTypeAliasDeclaration(node)) aliases.set(node.name.text, node); -}); +/** The text of fields.generated.ts for `irText`, the source of ir.ts. */ +export function generateIrFields(irText = readFileSync(irPath, "utf8")) { + const options = { strict: true, target: ts.ScriptTarget.ES2022 }; + const host = ts.createCompilerHost(options); + const readSourceFile = host.getSourceFile; + // ir.ts is read from `irText`; the files it imports from disk. + host.getSourceFile = (file, languageVersion, ...rest) => + samePath(file, irPath) + ? ts.createSourceFile(file, irText, languageVersion) + : readSourceFile.call(host, file, languageVersion, ...rest); + const program = ts.createProgram([irPath], options, host); + const checker = program.getTypeChecker(); + const source = program.getSourceFile(irPath); + const aliases = new Map(); + ts.forEachChild(source, (node) => { + if (ts.isTypeAliasDeclaration(node)) aliases.set(node.name.text, node); + }); -/** Whether a value of `type` can be a string. */ -function holdsString(type) { - if (type.isUnion()) return type.types.some(holdsString); - return (type.flags & (ts.TypeFlags.StringLike | ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0; -} - -/** Whether every value of `type` is a string. */ -function onlyString(type) { - if (type.isUnion()) return type.types.every(onlyString); - return (type.flags & ts.TypeFlags.StringLike) !== 0; -} - -/** Each kind's string fields, in declaration order, merged over the unions: - * a kind both unions declare lists the fields of both. */ -const kinds = new Map(); -for (const name of ["IrExpr", "IrStmt"]) { - const alias = aliases.get(name); - const union = checker.getTypeAtLocation(alias.name); - for (const member of union.isUnion() ? union.types : [union]) { - const kindSymbol = member.getProperty("kind"); - const kindType = checker.getTypeOfSymbolAtLocation(kindSymbol, alias); - if (!kindType.isStringLiteral()) throw new Error(`${name} member without a literal kind`); - const fields = kinds.get(kindType.value) ?? []; - for (const property of member.getProperties()) { - const type = checker.getTypeOfSymbolAtLocation(property, alias); - if (!holdsString(type)) continue; - const known = fields.find((field) => field.name === property.name); - const always = onlyString(type) && (property.flags & ts.SymbolFlags.Optional) === 0; - if (known === undefined) fields.push({ name: property.name, always }); - else known.always &&= always; - } - kinds.set(kindType.value, fields); + /** Whether a value of `type` can be a string. */ + function holdsString(type) { + if (type.isUnion()) return type.types.some(holdsString); + return (type.flags & (ts.TypeFlags.StringLike | ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0; } -} -const lines = [ - "// Generated from packages/compiler/src/ir/ir.ts by scripts/generate-ir-fields.mjs.", - "// Run `pnpm generate:ir-fields` after changing the fields of an IR node.", - 'import type { IrExpr, IrStmt, IrType } from "./ir.js";', - "", - "/** Calls `visit` with each string field of `node` (including `kind`) and its", - " * name: the string values `for (const key in node)` finds on a node, read", - " * by name in the order ir.ts declares them. */", - "export function eachStringField(", - " node: IrExpr | IrStmt,", - " visit: (value: string, field: string) => void,", - "): void {", - " switch (node.kind) {", -]; -for (const [kind, fields] of [...kinds].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) { - lines.push(` case ${JSON.stringify(kind)}: {`); - for (const { name, always } of fields) { - const read = `node.${name}`; - if (always) lines.push(` visit(${read}, ${JSON.stringify(name)});`); - else - lines.push(` if (typeof ${read} === "string") visit(${read}, ${JSON.stringify(name)});`); + /** Whether every value of `type` is a string. */ + function onlyString(type) { + if (type.isUnion()) return type.types.every(onlyString); + return (type.flags & ts.TypeFlags.StringLike) !== 0; } - lines.push(" return;", " }"); -} -lines.push(" }", "}", ""); -// The deep walkers: every field of every object under an IR node, as a -// recursive key walk finds them, read by name. -const members = new Map(); -for (const name of ["IrExpr", "IrStmt", "IrType"]) { - const union = checker.getTypeAtLocation(aliases.get(name).name); - members.set(name, new Set(union.isUnion() ? union.types : [union])); -} -const nullish = ts.TypeFlags.Null | ts.TypeFlags.Undefined | ts.TypeFlags.Void; -const scalar = - ts.TypeFlags.NumberLike | - ts.TypeFlags.BooleanLike | - ts.TypeFlags.BigIntLike | - ts.TypeFlags.ESSymbolLike; - -/** The walker an IR union member belongs to, or null. */ -function irKind(type) { - for (const [name, set] of members) if (set.has(type)) return name; - return null; -} - -let temp = 0; -/** Lines that walk `expr` of `type`: strings to `walker.string`, IR nodes to - * their walkers, arrays by element and other objects by field. */ -function walkLines(expr, type, indent, depth) { - if (depth > 12) throw new Error(`field type nests too deep at ${expr}`); - const parts = type.isUnion() ? type.types : [type]; - const present = parts.filter((part) => (part.flags & nullish) === 0); - const optional = present.length !== parts.length; - const strings = present.filter((part) => (part.flags & ts.TypeFlags.StringLike) !== 0); - const objects = present.filter((part) => (part.flags & (ts.TypeFlags.StringLike | scalar)) === 0); - for (const part of present) - if ((part.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) - throw new Error(`untyped field ${expr}`); - const body = []; - if (objects.length > 0) { - const kinds = new Set(objects.map(irKind)); - if (kinds.size === 1 && !kinds.has(null)) { - const [kind] = kinds; - body.push(`walk${kind.slice(2)}Fields(${expr}, walker, skipType);`); - } else if (objects.every((part) => checker.isArrayType(part))) { - if (objects.length !== 1) throw new Error(`several array types at ${expr}`); - const item = `item${temp++}`; - body.push(`for (const ${item} of ${expr}) {`); - body.push(...walkLines(item, checker.getTypeArguments(objects[0])[0], " ", depth + 1)); - body.push("}"); - } else if (kinds.has(null) && objects.length === 1) { - for (const property of checker.getPropertiesOfType(objects[0])) { - if (property.name === "loc") continue; - const fieldType = checker.getTypeOfSymbol(property); - const field = `${expr}.${property.name}`; - const lines = walkLines(field, fieldType, "", depth + 1); - if (lines.length === 0) continue; - if (property.name === "type") - body.push("if (!skipType) {", ...lines.map((l) => ` ${l}`), "}"); - else body.push(...lines); + /** Each kind's string fields, in declaration order, merged over the unions: + * a kind both unions declare lists the fields of both. */ + const kinds = new Map(); + for (const name of ["IrExpr", "IrStmt"]) { + const alias = aliases.get(name); + const union = checker.getTypeAtLocation(alias.name); + for (const member of union.isUnion() ? union.types : [union]) { + const kindSymbol = member.getProperty("kind"); + const kindType = checker.getTypeOfSymbolAtLocation(kindSymbol, alias); + if (!kindType.isStringLiteral()) throw new Error(`${name} member without a literal kind`); + const fields = kinds.get(kindType.value) ?? []; + for (const property of member.getProperties()) { + const type = checker.getTypeOfSymbolAtLocation(property, alias); + if (!holdsString(type)) continue; + const known = fields.find((field) => field.name === property.name); + const always = onlyString(type) && (property.flags & ts.SymbolFlags.Optional) === 0; + if (known === undefined) fields.push({ name: property.name, always }); + else known.always &&= always; } - } else throw new Error(`mixed object types at ${expr}: ${checker.typeToString(type)}`); + kinds.set(kindType.value, fields); + } } - let lines = body; - if (strings.length > 0 && objects.length > 0) - lines = [ - `if (typeof ${expr} === "string") walker.string(${expr});`, - ...(body.length > 0 ? [`else {`, ...body.map((l) => ` ${l}`), "}"] : []), - ]; - else if (strings.length > 0) lines = [`walker.string(${expr});`]; - if (lines.length > 0 && optional && !(strings.length > 0 && objects.length > 0)) - lines = [ - `if (${expr} !== undefined && ${expr} !== null) {`, - ...lines.map((l) => ` ${l}`), - "}", - ]; - else if (lines.length > 0 && optional) - lines = [ - `if (${expr} !== undefined && ${expr} !== null) {`, - ...lines.map((l) => ` ${l}`), - "}", - ]; - return lines.map((line) => indent + line); -} -lines.push( - "/** What a field walk reports (walkExprFields): each expression and", - " * statement node before its fields, and each string a field holds. */", - "export interface IrFieldWalker {", - " node(node: IrExpr | IrStmt): void;", - " string(value: string): void;", - "}", - "", -); -for (const name of ["IrExpr", "IrStmt", "IrType"]) { - const short = name.slice(2); - const nodeCall = name === "IrType" ? [] : [" walker.node(node);"]; - lines.push( - `/** Walks every field of \`node\` but \`loc\` (and \`type\` with \`skipType\`), and`, - " * every field of each object under it: the strings and nodes a recursive", - " * `for (const key in value)` walk with the same skips finds, read by name in", - " * the order ir.ts declares the fields. */", - `export function walk${short}Fields(node: ${name}, walker: IrFieldWalker, skipType: boolean): void {`, - ...nodeCall, + const lines = [ + "// Generated from packages/compiler/src/ir/ir.ts by scripts/generate-ir-fields.mjs.", + "// Run `pnpm generate:ir-fields` after changing the fields of an IR node.", + 'import type { IrExpr, IrStmt, IrType } from "./ir.js";', + "", + "/** Calls `visit` with each string field of `node` (including `kind`) and its", + " * name: the string values `for (const key in node)` finds on a node, read", + " * by name in the order ir.ts declares them. */", + "export function eachStringField(", + " node: IrExpr | IrStmt,", + " visit: (value: string, field: string) => void,", + "): void {", " switch (node.kind) {", - ); - const byKind = new Map(); - for (const member of members.get(name)) { - const kind = checker.getTypeOfSymbol(member.getProperty("kind")).value; - if (byKind.has(kind)) throw new Error(`${name} kind ${kind} declared twice`); - byKind.set(kind, member); - } - for (const [kind, member] of [...byKind].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) { + ]; + for (const [kind, fields] of [...kinds].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) { lines.push(` case ${JSON.stringify(kind)}: {`); - for (const property of member.getProperties()) { - if (property.name === "loc") continue; - const fieldType = checker.getTypeOfSymbol(property); - const field = `node.${property.name}`; - const walk = walkLines(field, fieldType, "", 0); - if (walk.length === 0) continue; - if (property.name === "type") - lines.push(" if (!skipType) {", ...walk.map((l) => ` ${l}`), " }"); - else lines.push(...walk.map((l) => ` ${l}`)); + for (const { name, always } of fields) { + const read = `node.${name}`; + if (always) lines.push(` visit(${read}, ${JSON.stringify(name)});`); + else + lines.push( + ` if (typeof ${read} === "string") visit(${read}, ${JSON.stringify(name)});`, + ); } lines.push(" return;", " }"); } lines.push(" }", "}", ""); -} -const text = lines.join("\n"); -if (process.argv.includes("--check")) { - if (readFileSync(target, "utf8") !== text) - throw new Error("fields.generated.ts is stale; run node scripts/generate-ir-fields.mjs"); -} else writeFileSync(target, text); + // The deep walkers: every field of every object under an IR node, as a + // recursive key walk finds them, read by name. + const members = new Map(); + for (const name of ["IrExpr", "IrStmt", "IrType"]) { + const union = checker.getTypeAtLocation(aliases.get(name).name); + members.set(name, new Set(union.isUnion() ? union.types : [union])); + } + const nullish = ts.TypeFlags.Null | ts.TypeFlags.Undefined | ts.TypeFlags.Void; + const scalar = + ts.TypeFlags.NumberLike | + ts.TypeFlags.BooleanLike | + ts.TypeFlags.BigIntLike | + ts.TypeFlags.ESSymbolLike; + + /** Whether every value of `type` is a number, boolean, bigint, symbol, null + * or undefined. A record of such fields walks to nothing too, but it is an + * object: its own fields are checked instead. */ + function onlyScalar(type) { + const parts = type.isUnion() ? type.types : [type]; + return parts.every((part) => (part.flags & (scalar | nullish)) !== 0); + } + + /** The walker an IR union member belongs to, or null. */ + function irKind(type) { + for (const [name, set] of members) if (set.has(type)) return name; + return null; + } + + let temp = 0; + /** The fields each walked object type declares, for the type-level check at + * the end of the file: `typeExpr` names the object type, `fields` every + * field it had when the file was generated and `scalars` the fields that held + * no string and no object then. */ + const checks = []; + /** Records `property list` of `object` (named `typeExpr`) for the check. */ + function noteFields(typeExpr, properties, scalarNames) { + checks.push({ + typeExpr, + fields: properties.map((property) => property.name), + scalars: scalarNames, + }); + } + /** Lines that walk `expr` of `type` (named `typeExpr` in the generated file): + * strings to `walker.string`, IR nodes to their walkers, arrays by element and + * other objects by field. */ + function walkLines(expr, type, typeExpr, indent, depth) { + if (depth > 12) throw new Error(`field type nests too deep at ${expr}`); + const parts = type.isUnion() ? type.types : [type]; + const present = parts.filter((part) => (part.flags & nullish) === 0); + const optional = present.length !== parts.length; + const strings = present.filter((part) => (part.flags & ts.TypeFlags.StringLike) !== 0); + const objects = present.filter( + (part) => (part.flags & (ts.TypeFlags.StringLike | scalar)) === 0, + ); + for (const part of present) + if ((part.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) + throw new Error(`untyped field ${expr}`); + const body = []; + if (objects.length > 0) { + const kinds = new Set(objects.map(irKind)); + if (kinds.size === 1 && !kinds.has(null)) { + const [kind] = kinds; + body.push(`walk${kind.slice(2)}Fields(${expr}, walker, skipType);`); + } else if (objects.every((part) => checker.isArrayType(part))) { + if (objects.length !== 1) throw new Error(`several array types at ${expr}`); + const item = `item${temp++}`; + body.push(`for (const ${item} of ${expr}) {`); + const itemType = `Extract, readonly unknown[]>[number]`; + body.push( + ...walkLines(item, checker.getTypeArguments(objects[0])[0], itemType, " ", depth + 1), + ); + body.push("}"); + } else if (kinds.has(null) && objects.length === 1) { + const recordType = `Extract, object>`; + const properties = checker.getPropertiesOfType(objects[0]); + const scalarNames = []; + for (const property of properties) { + if (property.name === "loc") continue; + const fieldType = checker.getTypeOfSymbol(property); + const field = `${expr}.${property.name}`; + const fieldTypeExpr = `${recordType}[${JSON.stringify(property.name)}]`; + const lines = walkLines(field, fieldType, fieldTypeExpr, "", depth + 1); + if (onlyScalar(fieldType)) scalarNames.push(property.name); + if (lines.length === 0) continue; + if (property.name === "type") + body.push("if (!skipType) {", ...lines.map((l) => ` ${l}`), "}"); + else body.push(...lines); + } + noteFields(recordType, properties, scalarNames); + } else throw new Error(`mixed object types at ${expr}: ${checker.typeToString(type)}`); + } + let lines = body; + if (strings.length > 0 && objects.length > 0) + lines = [ + `if (typeof ${expr} === "string") walker.string(${expr});`, + ...(body.length > 0 ? [`else {`, ...body.map((l) => ` ${l}`), "}"] : []), + ]; + else if (strings.length > 0) lines = [`walker.string(${expr});`]; + if (lines.length > 0 && optional && !(strings.length > 0 && objects.length > 0)) + lines = [ + `if (${expr} !== undefined && ${expr} !== null) {`, + ...lines.map((l) => ` ${l}`), + "}", + ]; + else if (lines.length > 0 && optional) + lines = [ + `if (${expr} !== undefined && ${expr} !== null) {`, + ...lines.map((l) => ` ${l}`), + "}", + ]; + return lines.map((line) => indent + line); + } + + lines.push( + "/** What a field walk reports (walkExprFields): each expression and", + " * statement node before its fields, and each string a field holds. */", + "export interface IrFieldWalker {", + " node(node: IrExpr | IrStmt): void;", + " string(value: string): void;", + "}", + "", + ); + for (const name of ["IrExpr", "IrStmt", "IrType"]) { + const short = name.slice(2); + const nodeCall = name === "IrType" ? [] : [" walker.node(node);"]; + lines.push( + `/** Walks every field of \`node\` but \`loc\` (and \`type\` with \`skipType\`), and`, + " * every field of each object under it: the strings and nodes a recursive", + " * `for (const key in value)` walk with the same skips finds, read by name in", + " * the order ir.ts declares the fields. */", + `export function walk${short}Fields(node: ${name}, walker: IrFieldWalker, skipType: boolean): void {`, + ...nodeCall, + " switch (node.kind) {", + ); + const byKind = new Map(); + for (const member of members.get(name)) { + const kind = checker.getTypeOfSymbol(member.getProperty("kind")).value; + if (byKind.has(kind)) throw new Error(`${name} kind ${kind} declared twice`); + byKind.set(kind, member); + } + for (const [kind, member] of [...byKind].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) { + lines.push(` case ${JSON.stringify(kind)}: {`); + const memberType = `Extract<${name}, { kind: ${JSON.stringify(kind)} }>`; + const scalarNames = []; + for (const property of member.getProperties()) { + if (property.name === "loc") continue; + const fieldType = checker.getTypeOfSymbol(property); + const field = `node.${property.name}`; + const fieldTypeExpr = `${memberType}[${JSON.stringify(property.name)}]`; + const walk = walkLines(field, fieldType, fieldTypeExpr, "", 0); + if (onlyScalar(fieldType)) scalarNames.push(property.name); + if (walk.length === 0) continue; + if (property.name === "type") + lines.push(" if (!skipType) {", ...walk.map((l) => ` ${l}`), " }"); + else lines.push(...walk.map((l) => ` ${l}`)); + } + lines.push(" return;", " }"); + noteFields(memberType, member.getProperties(), scalarNames); + } + lines.push(" }", "}", ""); + } + + // Type checking fails below when ir.ts gives a walked object a field this + // file does not know, or lets a field that held no string and no object hold + // one before this file is regenerated. + const quoted = (names) => names.map((name) => JSON.stringify(name)).join(" | "); + lines.push( + "/** A field ir.ts declares that this file does not read. */", + "type FieldsNotRead = Exclude;", + "/** The string or object values a field read as a scalar can hold. */", + "type ValuesNotRead = Extract, string | object>;", + "/** Fails type checking unless `T` is never: run `pnpm generate:ir-fields`. */", + "type Unread = T;", + "", + "/** Every walked object type's fields as ir.ts declared them when this file", + " * was generated. */", + "export type IrFieldsCurrent = [", + ); + for (const { typeExpr, fields, scalars } of checks) { + lines.push( + ` Unread 0 ? quoted(fields) : "never"}>>,`, + ); + for (const scalarName of scalars) + lines.push(` Unread>,`); + } + lines.push("];", ""); + return lines.join("\n"); +} + +if (process.argv[1] !== undefined && samePath(process.argv[1], fileURLToPath(import.meta.url))) { + const text = generateIrFields(); + + if (process.argv.includes("--check")) { + if (readFileSync(target, "utf8") !== text) + throw new Error("fields.generated.ts is stale; run node scripts/generate-ir-fields.mjs"); + } else writeFileSync(target, text); +} diff --git a/scripts/refresh-cli-profile.mjs b/scripts/refresh-cli-profile.mjs index dde4e21b9..a9436b505 100644 --- a/scripts/refresh-cli-profile.mjs +++ b/scripts/refresh-cli-profile.mjs @@ -6,7 +6,7 @@ * * Builds an instrumented native compiler for this Linux host, runs it on the * training builds below, and merges the raw profiles it writes into one - * sparse indexed profile. The training builds are tsc-ts (a cold `speed` + * indexed profile. The training builds are tsc-ts (a cold `speed` * build, then a rebuild after an edit), every seventh top-level corpus * program (by number, or by a hash of the name of a program without one) at * the default optimization and a further slice in `dev`, and the runtime @@ -14,6 +14,11 @@ * move much compiler code; a stale profile only loses part of its gain. A * refresh takes about 11 minutes on 16 cores. * + * The instrumented compiler is built with the committed function layout, or + * the one SCRIPTC_NATIVE_LAYOUT names, and the profile matches builds that + * use the same layout. When refreshing both, refresh the layout first + * (scripts/refresh-cli-layout.mjs). + * * node scripts/refresh-cli-profile.mjs --tsc-ts [--jobs 8] */ import { spawn, spawnSync } from "node:child_process"; @@ -210,12 +215,15 @@ const profiles = readdirSync(raw) .map((name) => join(raw, name)); rmSync(builds, { recursive: true, force: true }); const indexed = join(work, "cli.profdata"); -const merged = spawnSync(scriptc, ["merge-profiles", "--sparse", "-o", indexed, ...profiles], { +// The profile keeps the functions that never ran in training, with zero +// counts. Builds optimize those as cold and place them after the trained code, +// which keeps the compiler's .text smaller than a sparse profile does. +const merged = spawnSync(scriptc, ["merge-profiles", "-o", indexed, ...profiles], { stdio: "inherit", env: { ...trainingEnv, LLVM_PROFILE_FILE: join(work, "merge.profraw") }, }); if (merged.status !== 0) throw new Error("merge-profiles failed"); -// The indexed profile compresses to about an eighth of its size. +// The indexed profile compresses to about a sixth of its size. writeFileSync( profileOutput, gzipSync(readFileSync(indexed), { level: constants.Z_BEST_COMPRESSION }), diff --git a/scripts/sandbox-bootstrap.sh b/scripts/sandbox-bootstrap.sh index ac0a4558a..ea50a7574 100644 --- a/scripts/sandbox-bootstrap.sh +++ b/scripts/sandbox-bootstrap.sh @@ -32,6 +32,7 @@ sudo env DEBIAN_FRONTEND=noninteractive apt-get install --quiet=2 --yes --no-ins libclang-rt-22-dev \ libpolly-22-dev \ libzstd-dev \ + lld-22 \ llvm-22-dev \ ninja-build \ xz-utils \ @@ -40,6 +41,9 @@ sudo rm -f /usr/local/bin/clang /usr/local/bin/clang++ sudo ln -sf clang-22 /usr/bin/clang sudo ln -sf clang++-22 /usr/bin/clang++ test "$(llvm-config-22 --version)" = 22.1.8 +# CI's pinned LLVM ships ld.lld and the compiler links with it when the +# driver finds it (beside clang's real executable): match that here. +test -x "$(dirname "$(readlink -f "$(command -v clang)")")/ld.lld" node_archive="node-v${node_version}-linux-x64.tar.xz" curl --fail --silent --show-error --location \ diff --git a/scripts/sandbox-config.mjs b/scripts/sandbox-config.mjs index 9ab1f6fa1..9b64eba57 100644 --- a/scripts/sandbox-config.mjs +++ b/scripts/sandbox-config.mjs @@ -291,3 +291,22 @@ export function sandboxTestSourceConfig(env) { createArgs: ["--image", image.sandboxImage], }; } + +/** A built checkout inside a prepared snapshot whose native artifacts a + * lane may reuse when their inputs match (SCRIPTC_SANDBOX_NATIVE_REFERENCE). + * It needs the snapshot's tools, so it is ignored for the managed image. */ +export function sandboxNativeReference(env) { + loadLocalEnv(); + const source = env ?? process.env; + const reference = source.SCRIPTC_SANDBOX_NATIVE_REFERENCE?.trim(); + if (!reference) return undefined; + if (!sandboxTestSourceConfig(source).prepared) { + throw new Error( + "SCRIPTC_SANDBOX_NATIVE_REFERENCE needs SCRIPTC_SANDBOX_SNAPSHOT or SCRIPTC_SANDBOX_IMAGE", + ); + } + if (!/^\/[\w./-]+$/.test(reference) || reference.split("/").includes("..")) { + throw new Error("SCRIPTC_SANDBOX_NATIVE_REFERENCE must be an absolute path without '..'"); + } + return reference.replace(/\/+$/, ""); +} diff --git a/scripts/sandbox-test.mjs b/scripts/sandbox-test.mjs index 602ef033e..3fe063075 100644 --- a/scripts/sandbox-test.mjs +++ b/scripts/sandbox-test.mjs @@ -9,6 +9,7 @@ import { parseArgs } from "node:util"; import { fileURLToPath } from "node:url"; import { sandboxBootstrapCommand, + sandboxNativeReference, sandboxTestSourceConfig, sandboxRunnerConfig, sandboxEffectWorkerAllocation, @@ -112,6 +113,38 @@ const hostInvariantFiles = [ "packages/runtime/test/tonumber.test.ts", "packages/runtime/test/url.test.ts", ]; +// Native artifacts a lane needs and every source path their builds read. +// Output paths are excluded from the input comparison by these names. +const NATIVE_OUTPUT_NAMES = [ + "bin", + "artifacts", + "runtime-pack.json", + ".runtime-pack-*", + "node_modules", + "dist", + "build", + ".cache", +]; +const LLVM_HELPER_INPUTS = ["native/llvm-codegen", "packages/llvm-linux-x64-gnu"]; +const NATIVE_ARTIFACTS = { + llvm: { + inputs: LLVM_HELPER_INPUTS, + outputs: ["packages/llvm-linux-x64-gnu/bin/scriptc-llvm-codegen"], + }, + runtime: { + inputs: [ + ...LLVM_HELPER_INPUTS, + "packages/runtime", + "packages/runtime-pack-common", + "packages/runtime-linux-x64-gnu", + "packages/runtime-darwin-arm64/scripts", + ], + outputs: [ + "packages/runtime-linux-x64-gnu/artifacts", + "packages/runtime-linux-x64-gnu/runtime-pack.json", + ], + }, +}; const hostSchedule = sandboxHostSchedule(process.platform, hostInvariantFiles); const nativeHostInvariantFiles = hostSchedule.localInvariantFiles; const remoteWorkspaceReset = workspaceResetCommand("/workspace"); @@ -188,6 +221,8 @@ Environment: VERCEL_TOKEN Access-token fallback; also set VERCEL_TEAM_ID + VERCEL_PROJECT_ID SCRIPTC_SANDBOX_IMAGE Optional fully qualified VCR image (default: vercel/sandbox/universal) SCRIPTC_SANDBOX_SNAPSHOT Optional prepared snapshot with the pinned toolchain (exclusive with IMAGE) + SCRIPTC_SANDBOX_NATIVE_REFERENCE Optional built checkout inside the snapshot; its LLVM helper + and runtime pack are reused when their inputs are identical SCRIPTC_SANDBOX_VCPUS vCPUs per sandbox (default: 8) SCRIPTC_SANDBOX_TIMEOUT sandbox and command timeout (default: 45m) SCRIPTC_TEST_WORKERS Vitest workers per sandbox (default: 4) @@ -197,6 +232,7 @@ Environment: } const sourceConfig = sandboxTestSourceConfig(); +const nativeReference = sandboxNativeReference(); const bootstrapCommand = sandboxBootstrapCommand(sourceConfig.prepared); const vercelConfig = sandboxVercelConfig(); const vercelProcessEnv = sandboxVercelEnvironment(vercelConfig); @@ -496,6 +532,42 @@ const execIn = async ( } }; +/** The command that provides one native artifact (`llvm`: the Linux LLVM + * helper, `runtime`: the Linux runtime pack). It builds the artifact unless + * SCRIPTC_SANDBOX_NATIVE_REFERENCE names a built checkout inside the Sandbox + * (a prepared snapshot's) whose inputs for that artifact are identical; then + * it copies the reference's artifact. The runtime build runs the helper, so + * its inputs include the helper's. */ +function nativeArtifactCommand(artifact) { + const pkg = + artifact === "llvm" ? "@scriptc/llvm-linux-x64-gnu" : "@scriptc/runtime-linux-x64-gnu"; + if (nativeReference === undefined) return ["pnpm", ["--filter", pkg, "build:native"]]; + const build = `pnpm --filter ${pkg} build:native`; + const { inputs, outputs } = NATIVE_ARTIFACTS[artifact]; + const excludes = NATIVE_OUTPUT_NAMES.map((name) => `-x '${name}'`).join(" "); + const ref = nativeReference; + const script = [ + "set -eu", + `ref='${ref}'`, + "same=1", + ...outputs.map((output) => `[ -e "$ref/${output}" ] || same=0`), + ...inputs.map( + (input) => + `[ "$same" = 0 ] || diff -rq ${excludes} "$ref/${input}" "${input}" >/dev/null 2>&1 || same=0`, + ), + 'if [ "$same" = 1 ]; then', + ...outputs.map( + (output) => + ` rm -rf "${output}" && mkdir -p "$(dirname "${output}")" && cp -R "$ref/${output}" "${output}"`, + ), + ` echo "reused the ${artifact} artifact from $ref: identical inputs"`, + "else", + ` ${build}`, + "fi", + ].join("\n"); + return ["sh", ["-c", script]]; +} + async function preflight() { console.log("Sandbox preflight:"); console.log(` auth: ${vercelConfig.authSource}`); @@ -582,7 +654,10 @@ async function createWorker(worker) { vcpus, "--non-persistent", ]; - for (let attempt = 1; attempt <= 2; attempt++) { + // Sandbox creation is rate limited per team in vCPUs per minute, so a + // large fan-out (many shards, large Sandboxes) retries with backoff. + const attempts = 6; + for (let attempt = 1; attempt <= attempts; attempt++) { try { await vercel(args, { idleTimeoutMs: 60_000, @@ -598,8 +673,12 @@ async function createWorker(worker) { await execIn(worker, "true", [], {}, "create status", 60_000, "/"); return; } catch (error) { - if (attempt === 2) throw error; - console.warn(`[${worker.label}] Sandbox is not reachable; retrying creation once...`); + if (attempt === attempts) throw error; + const delay = 15_000 * attempt; + console.warn( + `[${worker.label}] Sandbox is not reachable; retrying creation in ${delay / 1000}s...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); } } } @@ -781,19 +860,13 @@ try { // including TypeScript's incremental metadata. Rebuild both together. await execIn(worker, "pnpm", ["build:fresh"], {}, "", 2 * 60_000); // Workspace builds deliberately do not rebuild packaged native artifacts. - // Every remote lane needs the Linux helper and runtime from this worktree. + // Every remote lane needs the Linux helper and runtime from this worktree, + // built from its sources or reused from a reference checkout whose + // native inputs are identical. + await execIn(worker, ...nativeArtifactCommand("llvm"), {}, "LLVM helper", 5 * 60_000); await execIn( worker, - "pnpm", - ["--filter", "@scriptc/llvm-linux-x64-gnu", "build:native"], - {}, - "LLVM helper", - 5 * 60_000, - ); - await execIn( - worker, - "pnpm", - ["--filter", "@scriptc/runtime-linux-x64-gnu", "build:native"], + ...nativeArtifactCommand("runtime"), { CC: "zig", AR: "zig" }, "runtime pack", 5 * 60_000, diff --git a/scripts/zero-cost-compare.mjs b/scripts/zero-cost-compare.mjs index affe4ee34..697f43812 100644 --- a/scripts/zero-cost-compare.mjs +++ b/scripts/zero-cost-compare.mjs @@ -6,6 +6,7 @@ * the first unit that differs. The self-hosted compiler's IR and LLVM exceed * V8's maximum string length, so splitting works on Buffers and hashes each * unit instead of decoding the artifact. */ +import assert from "node:assert/strict"; import { createHash } from "node:crypto"; const hash = (bytes) => createHash("sha1").update(bytes).digest("hex"); @@ -412,7 +413,9 @@ export function nodeIslandSymbols(names) { * bench/dynamic-properties are untyped JavaScript. bench/regex-replacers, * also added later, passes checked-dynamic values to replace callbacks * whose parameters the lib types `any`: captures a match can leave unset - * and results the checker types `any`. Every other bench workload is fully typed and must link none. The gate and + * and results the checker types `any`. bench/record-untyped-writes, also + * added later, writes typed records through `unknown` and `Object.assign` + * views. Every other bench workload is fully typed and must link none. The gate and * tests/harness/bench-dynamic-tier.test.ts enforce that, and the test also * fails when a listed workload stops linking them, so remove a workload here * when a leak fix makes its binary clean. */ @@ -436,6 +439,7 @@ export const DYNAMIC_WORKLOADS = new Set([ "bench/json-unions", "bench/throw-chains", "bench/regex-replacers", + "bench/record-untyped-writes", ]); /** Names of the functions (text and weak symbols) in `nm` output. Mach-O @@ -498,6 +502,57 @@ export function compareCounts(baseline, candidate) { }; } +/** The report lines the gate prints for one program. */ +export function programLines(entry) { + const lines = []; + const pct = (ratio) => `${ratio >= 1 ? "+" : ""}${((ratio - 1) * 100).toFixed(2)}%`; + const label = { + identical: "identical", + allowed: "allowed ", + differs: "DIFFERS ", + "build-error": "ERROR ", + "both-fail": "both-fail", + }[entry.verdict]; + const notes = []; + for (const [kind, result] of Object.entries(entry.artifacts)) + if (!result.identical) { + const counts = result.changed + ? ` (${result.changed.length} changed, ${result.added.length} added, ${result.removed.length} removed)` + : ""; + notes.push(`${kind}: ${result.first}${counts}`); + } + const size = entry.artifacts.exe?.bytes; + if (size && size.baseline !== size.candidate) + notes.push( + `size ${size.baseline} -> ${size.candidate} B (${pct(size.candidate / size.baseline)})`, + ); + if (entry.cachegrind?.total) { + const { total, deltas } = entry.cachegrind; + notes.push( + `Ir ${total.baseline} -> ${total.candidate} (${pct(total.candidate / Math.max(1, total.baseline))})` + + (deltas[0] + ? `, largest ${deltas[0].name} ${deltas[0].delta > 0 ? "+" : ""}${deltas[0].delta}` + : ""), + ); + } else if (entry.cachegrind?.skipped) + notes.push(`cachegrind skipped: ${entry.cachegrind.skipped}`); + for (const failure of entry.dynamicTier?.failures ?? []) notes.push(`SYMBOLS: ${failure}`); + lines.push(`${label} ${entry.id}${notes.length ? ` ${notes.join("; ")}` : ""}`); + // A build error's first line is often a warning; print all of it. + const errors = + entry.baselineError !== undefined && entry.baselineError === entry.candidateError + ? [["baseline and candidate", entry.baselineError]] + : [ + ["baseline", entry.baselineError], + ["candidate", entry.candidateError], + ].filter(([, error]) => error !== undefined); + for (const [side, error] of errors) { + lines.push(` ${side}:`); + for (const line of error.split("\n")) lines.push(` ${line}`); + } + return lines; +} + /* ── verdicts ──────────────────────────────────────────────────────────── */ /** One program's verdict from its build results and artifact comparisons. @@ -550,3 +605,142 @@ export function timeRatio(rows, resamples = 2000) { high: ratios[Math.floor(resamples * 0.975)], }; } + +/** The compile-time verdict for one group of timing rows (`bench` or + * `self`): it fails when the 95% interval lies entirely above the allowed + * growth, and is inconclusive when the interval straddles it. */ +export function timingVerdict(name, rows, maxGrowth) { + const result = timeRatio(rows); + const fails = result.low > 1 + maxGrowth; + const verdict = fails + ? "SLOWER" + : result.high > 1 + maxGrowth + ? "inconclusive (rerun with more runs)" + : "within tolerance"; + const pct = (ratio) => `${ratio >= 1 ? "+" : ""}${((ratio - 1) * 100).toFixed(2)}%`; + return { + result, + fails, + line: + `time ${name} ${rows.length > 1 ? "geomean " : ""}${pct(result.ratio)} ` + + `[95% CI ${pct(result.low)}, ${pct(result.high)}]: ${verdict}`, + failure: fails + ? `${name} compile time grew ${pct(result.ratio)} (CI low ${pct(result.low)})` + : null, + }; +} + +/** Timing rows grouped as the gate judges them. */ +export function timingGroups(rows) { + return { + bench: rows.filter((r) => r.id.startsWith("bench/")), + self: rows.filter((r) => r.id === "self/cli"), + }; +} + +/* ── baseline cache ────────────────────────────────────────────────────── */ + +/** A content hash of one artifact file, as the baseline cache records it. */ +export const artifactHash = hash; + +/** The hash of what a program's build reads, as the baseline cache records + * it: `files` (the entry and its relative imports) by path relative to + * `root` and content, read through `read`, and the optimization level. */ +export function programInputsHash(root, files, optimization, read) { + const h = createHash("sha1").update(`${optimization}\0`); + for (const file of [...files].sort()) + h.update(`${file.startsWith(root + "/") ? file.slice(root.length + 1) : file}\0`) + .update(read(file)) + .update("\0"); + return h.digest("hex"); +} + +/** The baseline cache entry recorded for program `id`, or undefined when the + * cache has none or recorded it for other inputs (`programInputsHash`): a + * program or an import that changed since the record must build its + * baseline, or a recorded build error or artifact hash of the old sources + * would settle the new sources' comparison. */ +export function recordedBaseline(cache, id, inputs) { + const entry = cache?.programs[id]; + return entry !== undefined && entry.inputs === inputs ? entry : undefined; +} + +/** The comparison a cached baseline entry settles without building the + * baseline, or null when the baseline must be built: identical artifacts + * (every recorded hash equals the candidate's) or the same build error on + * both sides. `candidate` holds the candidate's build result and `hashes` + * its artifact hashes. */ +export function cachedComparison(entry, candidate, hashes) { + if (entry === undefined) return null; + if (!entry.ok || !candidate.ok) + return !entry.ok && !candidate.ok && entry.error === candidate.error + ? { + baseline: { ok: false, error: entry.error, ms: entry.ms ?? null, cached: true }, + artifacts: {}, + } + : null; + const kinds = Object.keys(entry.hashes); + const differs = (kind) => entry.hashes[kind] === null || entry.hashes[kind] !== hashes[kind]; + if (kinds.length === 0 || kinds.some(differs)) return null; + const artifacts = {}; + for (const kind of kinds) artifacts[kind] = { identical: true }; + if (artifacts.exe) artifacts.exe.bytes = { baseline: entry.exeBytes, candidate: entry.exeBytes }; + return { baseline: { ok: true, ms: entry.ms ?? null }, artifacts }; +} + +/* ── sharded runs ──────────────────────────────────────────────────────── */ + +/** Merge the --json reports of gate shards that split one good-code set + * (--shard, --set) into a single report with one verdict. Program entries + * and their failures are concatenated; the compile-time groups and the + * "nothing was compared" check are recomputed over all shards. */ +export function mergeReports(reports) { + assert(reports.length > 0, "no gate reports to merge"); + const [first] = reports; + for (const report of reports) { + assert(report.schema === first.schema, "gate reports have different schemas"); + for (const label of ["baseline", "candidate"]) + assert( + report.trees[label].revision === first.trees[label].revision, + `gate reports compare different ${label} revisions`, + ); + } + const globals = new Set(reports.flatMap((r) => r.globalFailures ?? [])); + const merged = { + ...first, + shards: reports.length, + programs: reports.flatMap((r) => r.programs), + failures: reports.flatMap((r) => r.failures.filter((f) => !globals.has(f))), + globalFailures: [], + timing: null, + candidateSelf: reports.find((r) => r.candidateSelf)?.candidateSelf, + elapsedMs: Math.max(...reports.map((r) => r.elapsedMs ?? 0)), + }; + const ids = new Set(); + for (const p of merged.programs) { + assert(!ids.has(p.id), `program ${p.id} appears in more than one gate report`); + ids.add(p.id); + } + const timed = reports.filter((r) => r.timing !== null); + const lines = []; + if (timed.length > 0) { + const rows = timed.flatMap((r) => r.timing.rows); + const maxGrowth = timed[0].timing.maxGrowth; + merged.timing = { runs: timed[0].timing.runs, maxGrowth, rows, groups: {} }; + for (const [name, group] of Object.entries(timingGroups(rows))) { + if (group.length === 0) continue; + const { result, line, failure } = timingVerdict(name, group, maxGrowth); + merged.timing.groups[name] = result; + lines.push(line); + if (failure !== null) merged.globalFailures.push(failure); + } + } + const counts = {}; + for (const p of merged.programs) counts[p.verdict] = (counts[p.verdict] ?? 0) + 1; + if (!merged.programs.some((p) => ["identical", "allowed", "differs"].includes(p.verdict))) + merged.globalFailures.push("no program built with both checkouts, so nothing was compared"); + merged.failures.push(...merged.globalFailures); + merged.counts = counts; + merged.verdict = merged.failures.length > 0 ? "fail" : "pass"; + return { report: merged, timingLines: lines }; +} diff --git a/scripts/zero-cost-gate.mjs b/scripts/zero-cost-gate.mjs index 053c2f0d0..75724bd7d 100644 --- a/scripts/zero-cost-gate.mjs +++ b/scripts/zero-cost-gate.mjs @@ -13,6 +13,7 @@ * pattern signature) may differ; any other difference fails the gate. */ import assert from "node:assert/strict"; import { spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { existsSync, mkdirSync, @@ -23,12 +24,14 @@ import { statSync, writeFileSync, } from "node:fs"; -import { availableParallelism } from "node:os"; +import { availableParallelism, totalmem } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; import { readWorkloads } from "../benchmarks/runtime/workloads.mjs"; import { + artifactHash, + cachedComparison, cachegrindCounts, compareArtifact, compareCounts, @@ -39,8 +42,12 @@ import { median, nodeIslandSymbols, nmOutput, + programInputsHash, + programLines, programVerdict, - timeRatio, + recordedBaseline, + timingGroups, + timingVerdict, } from "./zero-cost-compare.mjs"; const repoRoot = fileURLToPath(new URL("..", import.meta.url)); @@ -62,6 +69,8 @@ const { values } = parseArgs({ cachegrind: { type: "boolean", default: false }, jobs: { type: "string", default: String(Math.max(1, availableParallelism())) }, json: { type: "string" }, + "baseline-cache": { type: "string" }, + "record-baseline-cache": { type: "string" }, keep: { type: "boolean", default: false }, help: { type: "boolean", default: false }, }, @@ -73,6 +82,7 @@ if (values.help || values.baseline === undefined) { [--timing-runs=N] [--self-timing-runs=N] [--max-time-growth=2] [--max-size-growth=0] [--cachegrind] [--jobs=N] [--json=] [--keep] + [--baseline-cache=] [--record-baseline-cache=] Each checkout must be built (packages/cli/dist/bootstrap.js and this host's runtime pack). Both compile the same program files, taken from --programs @@ -83,7 +93,9 @@ benchmarks/builds applications), self (packages/compiler/src/native/cli.ts, LLVM only; the candidate's own compiler source must also build with the candidate), corpus (tests/corpus TypeScript programs; --corpus=all adds JavaScript entries). --quick is --set=bench --timing-runs=0, for CI. ---shard=i/n keeps every n-th corpus program; --only filters program ids. +--shard=i/n keeps every n-th bench and corpus program; the self program runs +in shard 1. scripts/zero-cost-merge.mjs merges the shards' --json reports +into one verdict. --only filters program ids. --allow= is the change's pattern signature: programs whose source files match it may differ (reported as "allowed"). Any other difference in @@ -107,6 +119,18 @@ cachegrind (bench workloads with their arguments, corpus programs without) and compares instruction counts per function of the executable's own symbols, excluding the allocator's prefault thread. +--record-baseline-cache= builds only the baseline and records each +program's artifact hashes (or build error) in , with a hash of the +program's inputs (its entry, its relative imports and its optimization +level), then exits. +--baseline-cache= reads such a record: a program whose candidate +artifacts hash the same as the recorded baseline's is identical without a +baseline build, and the same build error on both sides is both-fail; any +other program builds the baseline as usual, and so does a program whose +inputs changed since the record. The record applies only to the +baseline revision and toolchain it was made with; otherwise it is ignored. +The self program always builds both sides, since its builds are timed. + Exit status: 0 when the gate passes, 1 when it fails.`); process.exit(values.help ? 0 : 2); } @@ -201,13 +225,7 @@ function goodCodeSet() { if (main !== undefined) entries.push(main); } else if (exts.some((ext) => name.endsWith(`.${ext}`))) entries.push(path); } - let selected = entries; - if (values.shard) { - const [index, count] = values.shard.split("/").map(Number); - assert.ok(index >= 1 && index <= count, "--shard must be i/n with 1 <= i <= n"); - selected = entries.filter((_, k) => k % count === index - 1); - } - for (const entry of selected) { + for (const entry of entries) { const head = readFileSync(entry, "utf8").split("\n", 2); const speed = head.some((line) => /^\/\/ @optimization:\s*speed\s*$/.test(line)); programs.push({ @@ -218,7 +236,17 @@ function goodCodeSet() { }); } } - return programs.filter((p) => only === null || only.test(p.id)); + const selected = programs.filter((p) => only === null || only.test(p.id)); + if (!values.shard) return selected; + const [index, count] = values.shard.split("/").map(Number); + assert.ok( + Number.isInteger(index) && Number.isInteger(count) && index >= 1 && index <= count, + "--shard must be i/n with 1 <= i <= n", + ); + // The self program takes minutes and pairs its own timing: shard 1 owns it. + const self = selected.filter((p) => p.kind === "source"); + const rest = selected.filter((p) => p.kind !== "source"); + return [...(index === 1 ? self : []), ...rest.filter((_, k) => k % count === index - 1)]; } /* ── builds ────────────────────────────────────────────────────────────── */ @@ -420,7 +448,80 @@ function revision(root) { const fmtMs = (ms) => `${(ms / 1000).toFixed(2)}s`; const pct = (ratio) => `${ratio >= 1 ? "+" : ""}${((ratio - 1) * 100).toFixed(2)}%`; -const programs = goodCodeSet(); +/* ── baseline cache ────────────────────────────────────────────────────── */ + +/** Every file under `dir`, recursively, in a stable order. */ +function filesUnder(dir) { + if (!existsSync(dir)) return []; + return readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((d) => d.isFile()) + .map((d) => join(d.parentPath, d.name)) + .sort(); +} + +/** What a baseline build depends on besides the program: the checkout's + * built compiler and CLI, its runtime packs and LLVM helpers, Node, the + * host and the program and checkout paths that reach the artifacts. */ +function toolchainFingerprint(root) { + const h = createHash("sha1"); + h.update(JSON.stringify([process.platform, process.arch, process.version, root, programsRoot])); + const clang = spawnSync("clang", ["--version"], { encoding: "utf8" }); + h.update(clang.stdout ?? ""); + // Linux links use ld.lld when the driver finds it, else the system linker. + const lld = spawnSync("clang", ["-print-prog-name=ld.lld"], { encoding: "utf8" }); + const lldPath = (lld.stdout ?? "").trim(); + h.update(lldPath); + if (lldPath.startsWith("/") && existsSync(lldPath)) + h.update(spawnSync(lldPath, ["--version"], { encoding: "utf8" }).stdout ?? ""); + const packages = join(root, "packages"); + for (const name of readdirSync(packages).sort()) { + const dir = join(packages, name); + const files = [ + ...filesUnder(join(dir, "dist")), + ...(name.startsWith("runtime-") + ? [join(dir, "runtime-pack.json"), ...filesUnder(join(dir, "artifacts"))] + : []), + ...(name.startsWith("llvm-") ? filesUnder(join(dir, "bin")) : []), + ]; + for (const file of files) + if (existsSync(file)) h.update(relative(root, file)).update(readFileSync(file)); + } + return h.digest("hex"); +} + +/** The hash of `program`'s inputs, which its baseline cache entry records. */ +function inputsHash(program) { + return programInputsHash( + programsRoot, + programInputs(program.entry), + program.kind === "source" ? "source" : (program.optimization ?? "release"), + (file) => readFileSync(file), + ); +} + +/** The content hashes of one build's artifacts (null for a missing one). */ +function buildHashes(build) { + const hashes = {}; + for (const kind of ["ir", "llvm", "exe"]) + if (kind in build) + hashes[kind] = build[kind] === null ? null : artifactHash(readFileSync(build[kind])); + return hashes; +} + +const recordPath = values["record-baseline-cache"]; +let baselineCache = null; +if (recordPath === undefined && values["baseline-cache"] !== undefined) { + const file = resolve(values["baseline-cache"]); + const cache = existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : null; + const fingerprint = cache === null ? null : toolchainFingerprint(trees.baseline); + if (cache?.schema === 2 && cache.fingerprint === fingerprint) baselineCache = cache; + else + console.log( + `baseline cache ${file} ${cache === null ? "is missing" : "was made with another baseline or toolchain"}; building every baseline`, + ); +} + +const programs = goodCodeSet().filter((p) => recordPath === undefined || p.kind !== "source"); assert.ok(programs.length > 0, "the selected good-code set is empty"); const report = { schema: 1, @@ -434,6 +535,13 @@ const report = { programs: [], timing: null, failures: [], + // Failures judged over the whole good-code set (compile time, nothing + // compared); scripts/zero-cost-merge.mjs recomputes them across shards. + globalFailures: [], + baselineCache: + baselineCache === null + ? null + : { programs: Object.keys(baselineCache.programs).length, used: 0 }, }; console.log( `zero-cost gate: ${programs.length} programs (${[...sets].join(", ")}), ` + @@ -442,18 +550,95 @@ console.log( let failed = false; const started = Date.now(); + +if (recordPath !== undefined) { + const record = { + schema: 2, + date: report.date, + fingerprint: toolchainFingerprint(trees.baseline), + baseline: report.trees.baseline, + programs: {}, + }; + await pool( + programs.map((program) => ({ + run: async () => { + const build = await buildProgram(program, "baseline"); + const inputs = inputsHash(program); + record.programs[program.id] = build.ok + ? { + inputs, + ok: true, + ms: build.ms, + hashes: buildHashes(build), + ...(build.exe ? { exeBytes: statSync(build.exe).size } : {}), + } + : { inputs, ok: false, ms: build.ms ?? null, error: build.error }; + if (build.dir) rmSync(build.dir, { recursive: true, force: true }); + }, + })), + ); + mkdirSync(dirname(resolve(recordPath)), { recursive: true }); + writeFileSync(recordPath, JSON.stringify(record) + "\n"); + rmSync(work, { recursive: true, force: true }); + const failures = Object.values(record.programs).filter((p) => !p.ok).length; + console.log( + `zero-cost gate: recorded ${programs.length} baseline programs (${failures} build errors) ` + + `in ${recordPath} in ${fmtMs(Date.now() - started)}`, + ); + process.exit(0); +} + +/** The candidate's own compiler source must still build with the candidate + * (see below). On a host with room for a third self-hosted build it runs + * alongside the comparison builds instead of after them. */ +const checkCandidateSelf = + sets.has("self") && programsRoot !== trees.candidate && programs.some((p) => p.kind === "source"); +const buildCandidateSelf = () => + buildProgram( + { + id: "self/candidate-source", + entry: join(trees.candidate, "packages/compiler/src/native/cli.ts"), + kind: "source", + }, + "candidate", + ); +const earlyCandidateSelf = + checkCandidateSelf && totalmem() >= 40 * 1024 ** 3 ? buildCandidateSelf() : null; + await pool( programs.map((program) => ({ heavy: program.kind === "source", run: async () => { - const [baseline, candidate] = await Promise.all([ - buildProgram(program, "baseline"), - buildProgram(program, "candidate"), - ]); + const recorded = + program.kind === "source" || baselineCache === null + ? undefined + : recordedBaseline(baselineCache, program.id, inputsHash(program)); + let baseline; + let candidate; + let settled = null; + if (recorded === undefined) + [baseline, candidate] = await Promise.all([ + buildProgram(program, "baseline"), + buildProgram(program, "candidate"), + ]); + else { + candidate = await buildProgram(program, "candidate"); + settled = cachedComparison(recorded, candidate, candidate.ok ? buildHashes(candidate) : {}); + // A cached identical baseline has the candidate's artifacts. + baseline = + settled === null + ? await buildProgram(program, "baseline") + : settled.baseline.ok + ? { ...candidate, ms: settled.baseline.ms, cached: true } + : settled.baseline; + if (settled !== null) report.baselineCache.used++; + } const allowed = allow !== null && programInputs(program.entry).some((f) => allow.test(readFileSync(f, "utf8"))); - const artifacts = baseline.ok && candidate.ok ? compareBuilds(baseline, candidate) : {}; + const artifacts = + settled?.artifacts ?? + (baseline.ok && candidate.ok ? compareBuilds(baseline, candidate) : {}); // The benchmark workloads and the self-hosted compiler always build. const required = !program.id.startsWith("corpus/"); const { verdict, fails } = programVerdict({ @@ -470,6 +655,7 @@ await pool( buildMs: { baseline: baseline.ms ?? null, candidate: candidate.ms ?? null }, artifacts, }; + if (settled !== null) entry.cachedBaseline = true; if (!baseline.ok) entry.baselineError = baseline.error; if (!candidate.ok) entry.candidateError = candidate.error; const size = artifacts.exe?.bytes; @@ -511,7 +697,8 @@ await pool( nmCache.delete(baseline.exe); nmCache.delete(candidate.exe); if (!values.keep) { - if (baseline.dir) rmSync(baseline.dir, { recursive: true, force: true }); + if (baseline.dir && !baseline.cached) + rmSync(baseline.dir, { recursive: true, force: true }); if (candidate.dir) rmSync(candidate.dir, { recursive: true, force: true }); } }, @@ -519,50 +706,7 @@ await pool( ); function printProgram(entry) { - const label = { - identical: "identical", - allowed: "allowed ", - differs: "DIFFERS ", - "build-error": "ERROR ", - "both-fail": "both-fail", - }[entry.verdict]; - const notes = []; - for (const [kind, result] of Object.entries(entry.artifacts)) - if (!result.identical) { - const counts = result.changed - ? ` (${result.changed.length} changed, ${result.added.length} added, ${result.removed.length} removed)` - : ""; - notes.push(`${kind}: ${result.first}${counts}`); - } - const size = entry.artifacts.exe?.bytes; - if (size && size.baseline !== size.candidate) - notes.push( - `size ${size.baseline} -> ${size.candidate} B (${pct(size.candidate / size.baseline)})`, - ); - if (entry.cachegrind?.total) { - const { total, deltas } = entry.cachegrind; - notes.push( - `Ir ${total.baseline} -> ${total.candidate} (${pct(total.candidate / Math.max(1, total.baseline))})` + - (deltas[0] - ? `, largest ${deltas[0].name} ${deltas[0].delta > 0 ? "+" : ""}${deltas[0].delta}` - : ""), - ); - } else if (entry.cachegrind?.skipped) - notes.push(`cachegrind skipped: ${entry.cachegrind.skipped}`); - for (const failure of entry.dynamicTier?.failures ?? []) notes.push(`SYMBOLS: ${failure}`); - console.log(`${label} ${entry.id}${notes.length ? ` ${notes.join("; ")}` : ""}`); - // A build error's first line is often a warning; print all of it. - const errors = - entry.baselineError !== undefined && entry.baselineError === entry.candidateError - ? [["baseline and candidate", entry.baselineError]] - : [ - ["baseline", entry.baselineError], - ["candidate", entry.candidateError], - ].filter(([, error]) => error !== undefined); - for (const [side, error] of errors) { - console.log(` ${side}:`); - for (const line of error.split("\n")) console.log(` ${line}`); - } + for (const line of programLines(entry)) console.log(line); } /* Compile time: paired --emit=llvm builds, the baseline and the candidate @@ -608,29 +752,15 @@ if (timingRuns > 0) { ); } report.timing = { runs: timingRuns, maxGrowth: maxTimeGrowth, rows, groups: {} }; - const groups = { - bench: rows.filter((r) => r.id.startsWith("bench/")), - self: rows.filter((r) => r.id === "self/cli"), - }; - for (const [name, group] of Object.entries(groups)) { + for (const [name, group] of Object.entries(timingGroups(rows))) { if (group.length === 0) continue; - const result = timeRatio(group); + const { result, line, failure } = timingVerdict(name, group, maxTimeGrowth); report.timing.groups[name] = result; - const verdict = - result.low > 1 + maxTimeGrowth - ? "SLOWER" - : result.high > 1 + maxTimeGrowth - ? "inconclusive (rerun with more runs)" - : "within tolerance"; - console.log( - `time ${name} ${group.length > 1 ? "geomean " : ""}${pct(result.ratio)} ` + - `[95% CI ${pct(result.low)}, ${pct(result.high)}]: ${verdict}`, - ); - if (result.low > 1 + maxTimeGrowth) { + console.log(line); + if (failure !== null) { failed = true; - report.failures.push( - `${name} compile time grew ${pct(result.ratio)} (CI low ${pct(result.low)})`, - ); + report.failures.push(failure); + report.globalFailures.push(failure); } } } @@ -639,12 +769,8 @@ if (timingRuns > 0) { // from one checkout), so a change that adds compiler code the self-hosted // frontend cannot lower would pass unnoticed. The candidate's own compiler // source must still build with the candidate (LLVM only, not compared). -if (sets.has("self") && programsRoot !== trees.candidate) { - const entry = join(trees.candidate, "packages/compiler/src/native/cli.ts"); - const result = await buildProgram( - { id: "self/candidate-source", entry, kind: "source" }, - "candidate", - ); +if (checkCandidateSelf) { + const result = await (earlyCandidateSelf ?? buildCandidateSelf()); report.candidateSelf = result.ok ? { ok: true, buildMs: result.ms } : { ok: false, error: result.error }; @@ -663,7 +789,9 @@ const counts = {}; for (const p of report.programs) counts[p.verdict] = (counts[p.verdict] ?? 0) + 1; if (!report.programs.some((p) => ["identical", "allowed", "differs"].includes(p.verdict))) { failed = true; - report.failures.push("no program built with both checkouts, so nothing was compared"); + const failure = "no program built with both checkouts, so nothing was compared"; + report.failures.push(failure); + report.globalFailures.push(failure); } report.counts = counts; report.verdict = failed ? "fail" : "pass"; @@ -677,7 +805,8 @@ else console.log(`artifacts kept in ${work}`); console.log( `zero-cost gate: ${failed ? "FAIL" : "PASS"} — ${Object.entries(counts) .map(([k, v]) => `${v} ${k}`) - .join(", ")} in ${fmtMs(report.elapsedMs)}`, + .join(", ")} in ${fmtMs(report.elapsedMs)}` + + (report.baselineCache ? ` (${report.baselineCache.used} baselines from the cache)` : ""), ); for (const failure of report.failures) console.log(` ${failure}`); process.exit(failed ? 1 : 0); diff --git a/scripts/zero-cost-merge.mjs b/scripts/zero-cost-merge.mjs new file mode 100644 index 000000000..d32022dd9 --- /dev/null +++ b/scripts/zero-cost-merge.mjs @@ -0,0 +1,52 @@ +/* Merge zero-cost gate shards into one verdict. + * + * A sharded gate runs scripts/zero-cost-gate.mjs several times over one + * good-code set (--shard=i/n for the bench and corpus programs, the self + * program in shard 1 or its own --set=self run), each with --json. This + * script merges those reports: program verdicts and failures are + * concatenated, and the compile-time verdicts and the "nothing was + * compared" check are recomputed over every shard's rows. */ +import { readFileSync, writeFileSync } from "node:fs"; +import { parseArgs } from "node:util"; +import { mergeReports, programLines } from "./zero-cost-compare.mjs"; + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + json: { type: "string" }, + quiet: { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + }, +}); +if (values.help || positionals.length === 0) { + console.log(`Usage: node scripts/zero-cost-merge.mjs [--json=] [--quiet] ... + +Merges the --json reports of zero-cost gate shards and prints one summary: +every program that is not identical, the compile-time verdicts and the +failures. --quiet prints only the summary and failures. --json writes the +merged report. Exit status: 0 when the merged gate passes, 1 when it fails.`); + process.exit(values.help ? 0 : 2); +} + +const reports = positionals.map((file) => JSON.parse(readFileSync(file, "utf8"))); +const { report, timingLines } = mergeReports(reports); +if (!values.quiet) + for (const entry of report.programs) + if (entry.verdict !== "identical") for (const line of programLines(entry)) console.log(line); +for (const line of timingLines) console.log(line); +if (report.candidateSelf) + console.log( + report.candidateSelf.ok + ? `self candidate compiler source builds with the candidate (${(report.candidateSelf.buildMs / 1000).toFixed(2)}s)` + : "self candidate compiler source FAILS to build with the candidate", + ); +const cached = report.programs.filter((p) => p.cachedBaseline).length; +console.log( + `zero-cost gate: ${report.verdict === "pass" ? "PASS" : "FAIL"} — ${Object.entries(report.counts) + .map(([k, v]) => `${v} ${k}`) + .join(", ")} over ${report.shards} shards (longest ${(report.elapsedMs / 1000).toFixed(2)}s` + + `${cached ? `, ${cached} baselines from the cache` : ""})`, +); +for (const failure of report.failures) console.log(` ${failure}`); +if (values.json) writeFileSync(values.json, JSON.stringify(report, null, 2) + "\n"); +process.exit(report.verdict === "pass" ? 0 : 1); diff --git a/tests/corpus/7393-non-null-element-operands.ts b/tests/corpus/7393-non-null-element-operands.ts new file mode 100644 index 000000000..0fd7190a3 --- /dev/null +++ b/tests/corpus/7393-non-null-element-operands.ts @@ -0,0 +1,81 @@ +// Non-null assertions on array element reads whose index is out of bounds +// or names a hole, used as operands of operators that convert them. `!` +// performs no runtime check, so the read is undefined, as in Node: NaN in +// arithmetic, false in comparisons, "undefined" in strings. + +const values = [3, 1, 4, 1, 5]; +const words = ["alpha", "beta"]; +const flags = [true, false]; +const holey: number[] = []; +holey[3] = 7; + +function show(v: unknown): string { + return Object.is(v, -0) ? "-0" : String(v); +} + +function line(label: string, parts: unknown[]): void { + console.log(label.padEnd(10), parts.map(show).join(" ")); +} + +for (const k of [0, 4, 5, -1, 1.5, 100]) { + line(`k=${k}`, [ + values[k]! + 1, + values[k]! * 2, + -values[k]!, + values[k]! | 0, + values[k]! < 3, + values[k]! >= 1, + values[k]! === 1, + values[k]! !== 1, + `${values[k]!}`, + "v=" + values[k]!, + Math.max(values[k]!, 0), + words[k]! + "!", + `<${words[k]!}>`, + words[k]! < "b", + !flags[k]!, + ]); +} + +line("holes", [holey[0]! + 1, holey[3]! + 1, `${holey[1]!}`, holey[2]! === undefined, holey[9]! > 0]); + +// `+` with an untyped partner concatenates or adds when it runs. +const untypedText: any = "x"; +const untypedNumber: any = 5; +for (const k of [1, 9]) { + line(`untyped ${k}`, [ + values[k]! + untypedText, + untypedText + values[k]!, + values[k]! + untypedNumber, + words[k]! + untypedNumber, + words[k]! + untypedText, + ]); +} + +let sum = 0; +let text = ""; +for (let i = 0; i <= values.length; i++) { + sum += values[i]!; + text += words[i % 3]!; + text += ","; +} +line("compound", [sum, text]); + +// A counter array: an index past the end reads NaN, and the store widens +// the array to doubles. +function counter(n: number, m: number): number[] { + const a = new Array(n).fill(0) as number[]; + for (let i = 0; i < 6000; i++) a[i % m] = a[i % m]! + 1; + return [a[0]!, a[m - 1]!, a.length, a[n + 1]! + 0]; +} +line("counter", counter(5000, 5000)); +line("past", counter(5000, 5003)); +line("zero", counter(4, 0)); + +const grid = [ + [1, 2], + [3, 4], +]; +let total = 0; +for (let r = 0; r < 3; r++) for (let c = 0; c < 3; c++) total += grid[r % 2]![c]! * 1; +line("grid", [total]); diff --git a/tests/corpus/7395-non-null-element-dereference.ts b/tests/corpus/7395-non-null-element-dereference.ts new file mode 100644 index 000000000..67d9249d1 --- /dev/null +++ b/tests/corpus/7395-non-null-element-dereference.ts @@ -0,0 +1,142 @@ +// Member reads and writes through non-null asserted array element reads. +// `!` performs no runtime check: a missing element is undefined, so the +// member access throws Node's TypeError, which the program can catch. + +class Cell { + count = 0; + label: string; + constructor(label: string) { + this.label = label; + } + bump(): number { + return ++this.count; + } +} + +function cell(label: string): Cell { + return new Cell(label); +} + +const cells = [cell("a"), cell("b")]; +const words = ["alpha", "beta"]; +const grid = [ + [1, 2], + [3, 4], +]; +const maps = [new Map([["x", 1]])]; +const holey: Cell[] = []; +holey[2] = cell("z"); +let reads = 0; +function at(n: number): number { + reads++; + return n; +} + +function attempt(label: string, f: () => unknown): void { + try { + console.log(label.padEnd(12), "ok", f()); + } catch (e) { + const error = e as Error; + console.log(label.padEnd(12), error instanceof TypeError, error.message); + } +} + +for (const k of [0, 1, 2, -1]) { + attempt(`label ${k}`, () => cells[k]!.label); + attempt(`bump ${k}`, () => cells[k]!.bump()); + attempt(`set ${k}`, () => (cells[k]!.label = "set")); + attempt(`count++ ${k}`, () => cells[k]!.count++); + attempt(`length ${k}`, () => words[k]!.length); + attempt(`upper ${k}`, () => words[k]!.toUpperCase()); + attempt(`grid ${k}`, () => grid[k]![0]); + attempt(`row ${k}`, () => { + const j = 1; + return grid[k]![j]; + }); + attempt(`push ${k}`, () => grid[k]!.push(9)); + attempt(`map ${k}`, () => maps[k]!.get("x")); +} + +attempt("hole", () => holey[0]!.label); +attempt("hole value", () => holey[2]!.label); +attempt("order", () => cells[at(7)]!.count + at(1)); +console.log("reads", reads); + +let total = 0; +try { + for (let i = 0; i < 4; i++) total += grid[i]![0]!; +} catch (e) { + console.log("loop", (e as Error).message, total); +} +console.log(cells.map((c) => `${c.label}:${c.count}`).join(" ")); + +// A write through a missing element evaluates the key and the value before +// it throws; an error the value throws is the one that propagates. +const events: string[] = []; +const note = (event: string, value: number): number => { + events.push(event); + return value; +}; +attempt("write order", () => { + cells[note("index", 7)]!.count = note("value", 1); +}); +attempt("write key order", () => { + grid[note("row", 9)]![note("column", 0)] = note("cell", 5); +}); +attempt("write literal key", () => { + grid[9]![0] = note("cell 0", 5); +}); +attempt("value throws", () => { + cells[7]!.count = (() => { + throw new Error("from the value"); + })(); +}); +attempt("present write", () => { + cells[0]!.count = note("present", 41) + 1; + return cells[0]!.count; +}); +console.log("events", events.join(",")); + +// The array is read before an index that replaces it. +let current = [new Cell("first")]; +const replace = (): number => { + current = []; + return 0; +}; +attempt("replaced read", () => current[replace()]!.label); +current = [new Cell("second")]; +attempt("replaced write", () => { + current[replace()]!.label = "written"; +}); +let names = ["kept"]; +attempt("replaced string", () => `${names[(names = [], 0)]!}`); +console.log(current.length, names.length); + +// The element is read before the key and the value: a value that replaces, +// removes or adds it writes to, or misses, the element the target read, and +// a key the value reassigns is the one read before it. +const slots = [new Cell("p"), new Cell("q")]; +const original = slots[0]!; +attempt("value replaces", () => { + slots[0]!.label = ((slots[0] = new Cell("r")), "set"); + return `${original.label} ${slots[0]!.label}`; +}); +const tail = slots[1]!; +attempt("value removes", () => { + slots[1]!.label = (slots.pop(), "gone"); + return `${tail.label} ${slots.length}`; +}); +attempt("value adds", () => { + slots[1]!.label = (slots.push(new Cell("s")), "added"); +}); +const table: number[][] = [[1, 2]]; +let column = 0; +attempt("value moves key", () => { + table[0]![column] = ((column = 1), 5); + return table[0]!.join(","); +}); +column = 0; +attempt("value moves missing key", () => { + table[4]![column] = ((column = 1), 5); +}); +console.log(slots.length, column); diff --git a/tests/corpus/8370-stack-overflow-cycles/main.ts b/tests/corpus/8370-stack-overflow-cycles/main.ts new file mode 100644 index 000000000..3b9ef5ddc --- /dev/null +++ b/tests/corpus/8370-stack-overflow-cycles/main.ts @@ -0,0 +1,69 @@ +// Unbounded recursion through each kind of call cycle raises Node's +// RangeError, and the program recovers and recurses again afterwards. +import { Even, Odd, Tree } from "./shapes.ts"; + +function probe(name: string, run: (n: number) => number): void { + try { + console.log(name, "returned", run(10_000_000)); + } catch (e) { + console.log(name, e instanceof RangeError, (e as Error).message); + } + console.log(name, "after", run(100)); +} + +function down(n: number): number { + return n === 0 ? 0 : 1 + down(n - 1); +} +probe("direct", down); + +function ping(n: number): number { + return n === 0 ? 0 : 1 + pong(n - 1); +} +function pong(n: number): number { + return n === 0 ? 0 : 2 + ping(n - 1); +} +probe("mutual", ping); + +function first(n: number): number { + return n === 0 ? 0 : 1 + second(n - 1); +} +function second(n: number): number { + return n === 0 ? 0 : 1 + third(n - 1); +} +function third(n: number): number { + return n === 0 ? 0 : 1 + fourth(n - 1); +} +function fourth(n: number): number { + return n === 0 ? 0 : 1 + fifth(n - 1); +} +function fifth(n: number): number { + return n === 0 ? 0 : 1 + first(n - 1); +} +probe("ring", first); + +function viaValue(n: number, again: (m: number) => number): number { + return n === 0 ? 0 : 1 + again(n - 1); +} +const callback = (m: number): number => viaValue(m, callback); +probe("closure", callback); + +function viaMap(n: number): number { + return n === 0 ? 0 : 1 + [n - 1].map(viaMap)[0]!; +} +probe("library callback", viaMap); + +const even = new Even(); +const odd = new Odd(); +even.next = odd; +odd.next = even; +probe("virtual", (n) => even.depth(n)); + +probe("constructor", (n) => { + let tree: Tree | null = new Tree(n); + let count = 0; + while (tree !== null) { + count++; + tree = tree.child; + } + return count; +}); diff --git a/tests/corpus/8370-stack-overflow-cycles/shapes.ts b/tests/corpus/8370-stack-overflow-cycles/shapes.ts new file mode 100644 index 000000000..6f5f6fada --- /dev/null +++ b/tests/corpus/8370-stack-overflow-cycles/shapes.ts @@ -0,0 +1,24 @@ +// Classes in their own module, so their names carry the module prefix. +export abstract class Link { + next: Link | null = null; + abstract depth(n: number): number; +} + +export class Even extends Link { + depth(n: number): number { + return n === 0 ? 0 : 1 + this.next!.depth(n - 1); + } +} + +export class Odd extends Link { + depth(n: number): number { + return n === 0 ? 0 : 2 + this.next!.depth(n - 1); + } +} + +export class Tree { + child: Tree | null; + constructor(n: number) { + this.child = n === 0 ? null : new Tree(n - 1); + } +} diff --git a/tests/corpus/8380-untyped-generator-loops.mjs b/tests/corpus/8380-untyped-generator-loops.mjs new file mode 100644 index 000000000..a6dd20ebe --- /dev/null +++ b/tests/corpus/8380-untyped-generator-loops.mjs @@ -0,0 +1,162 @@ +// Untyped for-of and for await loops over generator objects: values, +// completion values, early exits, errors, other iterables through the same +// loop, and the microtask order of each step. +function* numbers(n) { + for (let i = 0; i < n; i++) yield i; + return "done"; +} +function* mixed() { + yield "a"; + yield { k: 1 }; + yield undefined; + yield [1, 2]; + yield Promise.resolve(7); + return 99; +} +async function* asyncNumbers(n) { + for (let i = 0; i < n; i++) yield i * 10; + return "async done"; +} +async function* asyncMixed() { + yield "x"; + yield Promise.resolve(8); + yield null; +} + +function collect(it) { + const out = []; + for (const x of it) out.push(x); + return out; +} +async function collectAsync(it) { + const out = []; + for await (const x of it) out.push(x); + return out; +} +const show = (v) => JSON.stringify(v, (k, x) => (x instanceof Promise ? "promise" : x === undefined ? "undef" : x)); + +async function main() { + // The same loops over generators, arrays, Maps, strings and custom iterators. + console.log("sync", show(collect(numbers(4))), show(collect(mixed()))); + console.log("sync other", show(collect([1, 2])), show(collect(new Map([[1, 2]]))), show(collect("hé"))); + const custom = { + [Symbol.iterator]() { + let i = 0; + return { next: () => ({ value: i * i, done: i++ > 2 }) }; + }, + }; + console.log("sync custom", show(collect(custom))); + console.log("async", show(await collectAsync(asyncNumbers(3))), show(await collectAsync(asyncMixed()))); + console.log("async over sync", show(await collectAsync(mixed())), show(await collectAsync(numbers(3)))); + console.log("async other", show(await collectAsync([Promise.resolve(1), 2])), show(await collectAsync(custom))); + const asyncCustom = { + [Symbol.asyncIterator]() { + let i = 0; + return { next: async () => ({ value: `c${i}`, done: i++ > 1 }) }; + }, + }; + console.log("async custom", show(await collectAsync(asyncCustom))); + // Symbol.asyncIterator returning a sync generator: each result is awaited, + // its value is not, so a yielded promise reaches the loop as a promise. + const viaAsyncIterator = { [Symbol.asyncIterator]: () => mixed() }; + const seen = []; + for await (const x of viaAsyncIterator) seen.push(x instanceof Promise ? `promise ${await x}` : x); + console.log("async iterator sync", show(seen)); + + // A generator that has started keeps its position; one that finished stays done. + const started = numbers(5); + started.next(); + console.log("started", show(collect(started)), show(started.next())); + + // Early exits close the generator: finally blocks run once. + function* guarded(label) { + try { + yield 1; + yield 2; + } finally { + console.log(label, "finally"); + } + } + const pick = (v) => v; + for (const x of pick(guarded("break"))) { + console.log("break at", x); + break; + } + try { + for (const x of pick(guarded("throw"))) throw new Error(`body ${x}`); + } catch (e) { + console.log("caught", e.message); + } + const g = pick(guarded("after")); + for (const x of g) { + if (x === 1) break; + } + console.log("after break", show(g.next())); + async function* aguarded(label) { + try { + yield 1; + yield 2; + } finally { + console.log(label, "finally"); + } + } + for await (const x of pick(aguarded("abreak"))) { + console.log("abreak at", x); + break; + } + for await (const x of pick(guarded("sync abreak"))) { + console.log("sync abreak at", x); + break; + } + + // Errors from the generator body reach the loop. + function* failing() { + yield 1; + throw new TypeError("generator failed"); + } + async function* afailing() { + yield 1; + throw new RangeError("async generator failed"); + } + try { + for (const x of pick(failing())) console.log("failing got", x); + } catch (e) { + console.log("caught", e.name, e.message); + } + try { + for await (const x of pick(afailing())) console.log("afailing got", x); + } catch (e) { + console.log("caught", e.name, e.message); + } + // Microtask order: each step against a promise chain ticking alongside. + const ticker = (label, n) => { + let p = Promise.resolve(); + for (let i = 0; i < n; i++) p = p.then(() => console.log(label, "tick", i)); + return p; + }; + const consume = async (label, it) => { + for await (const x of it) console.log(label, "value", x); + console.log(label, "end"); + }; + await Promise.all([consume("sync", pick(numbers(3))), ticker("sync", 10)]); + await Promise.all([consume("async", pick(asyncNumbers(3))), ticker("async", 10)]); + await Promise.all([consume("promises", pick(mixed())), ticker("promises", 14)]); + + // Stages chained through untyped parameters. + async function* double(src) { + for await (const x of src) yield x * 2; + } + async function* keepOdd(src) { + for await (const x of src) if (x % 4 !== 0) yield x; + } + let sum = 0; + for await (const x of keepOdd(double(asyncNumbers(50)))) sum += x; + console.log("stages", sum); + let syncSum = 0; + function* inc(src) { + for (const x of src) yield x + 1; + } + for (const x of inc(inc(numbers(100)))) syncSum += x; + console.log("sync stages", syncSum); +} +main(); diff --git a/tests/corpus/8381-untyped-generator-stages.mjs b/tests/corpus/8381-untyped-generator-stages.mjs new file mode 100644 index 000000000..d326a00c1 --- /dev/null +++ b/tests/corpus/8381-untyped-generator-stages.mjs @@ -0,0 +1,82 @@ +// Untyped stages chained through generator objects: a sync generator source +// consumed with for await, async generator transforms, an async function +// consumer, and the same stages over arrays and custom iterables. +function run(source, transform, destination) { + return destination(transform(source())); +} +const N = 2000; +const total = await run( + function* () { + for (let i = 0; i < N; i++) yield i; + }, + async function* (src) { + for await (const n of src) yield n * 2; + }, + async function (src) { + let sum = 0; + for await (const n of src) sum += n; + return sum; + }, +); +console.log("total", total); + +async function* map(it, f) { + for await (const x of it) yield await f(x); +} +async function* take(it, n) { + if (n <= 0) return; + for await (const x of it) { + yield x; + if (--n === 0) return; + } +} +async function* source(n) { + try { + for (let i = 0; i < n; i++) yield i; + } finally { + console.log("source closed"); + } +} +async function toArray(it) { + const out = []; + for await (const x of it) out.push(x); + return out; +} +const half = async (x) => x / 2; +console.log(await toArray(take(map(source(10), half), 3))); +console.log(await toArray(map([1, 2, 3], half))); +console.log(await toArray(map(new Set(["a", "b"]), async (s) => s.toUpperCase()))); +const words = { + *[Symbol.iterator]() { + yield "x"; + yield "y"; + }, +}; +console.log(await toArray(map(words, async (s) => s + s))); + +// A stage that switches iterables between calls of the same loop. +function* letters() { + yield "p"; + yield "q"; +} +async function* numbersAsync() { + yield 1; + yield 2; +} +const sources = [letters(), numbersAsync(), ["z"], letters()]; +for (const s of sources) console.log(await toArray(s)); + +// Sync stages: generator to generator through untyped parameters. +function* inc(src) { + for (const x of src) yield x + 1; +} +function* evens(src) { + for (const x of src) if (x % 2 === 0) yield x; +} +function* range(n) { + for (let i = 0; i < n; i++) yield i; + return "range done"; +} +let s = 0; +for (const x of evens(inc(range(1000)))) s += x; +console.log("sync", s, [...inc([10, 20])], Array.from(evens(range(7)))); diff --git a/tests/corpus/8382-stackless-async-generator-steps.mjs b/tests/corpus/8382-stackless-async-generator-steps.mjs new file mode 100644 index 000000000..507040de0 --- /dev/null +++ b/tests/corpus/8382-stackless-async-generator-steps.mjs @@ -0,0 +1,172 @@ +// Async generators whose bodies suspend only at their own awaits and yields: +// requests queued while the body awaits, return() and throw() at a yield and +// during an await, finally blocks that await, yield* into another async +// generator, awaits of plain values and of settled and pending promises, a +// deep chain of stages, and the microtask order around them. +const log = []; +const tick = (label) => Promise.resolve().then(() => log.push(label)); +const later = (value) => new Promise((resolve) => setTimeout(() => resolve(value), 1)); + +async function* counter(n) { + try { + for (let i = 0; i < n; i++) { + const v = await (i % 2 === 0 ? i : Promise.resolve(i)); + log.push(`yield ${v}`); + yield v; + } + return "done"; + } finally { + log.push("counter finally"); + await later(0); + log.push("counter finally after await"); + } +} + +// Requests queued before the body settles the first one. +{ + const it = counter(3); + const requests = []; + for (let i = 0; i < 5; i++) requests.push(it.next()); + const results = []; + for (const request of requests) results.push(await request); + console.log(JSON.stringify(results)); + console.log(log.splice(0).join(" | ")); +} + +// return() at a yield runs the finally block, which awaits. +{ + const it = counter(5); + console.log(JSON.stringify(await it.next())); + const ret = it.return("early"); + tick("tick after return()"); + console.log(JSON.stringify(await ret)); + console.log(JSON.stringify(await it.next())); + console.log(log.splice(0).join(" | ")); +} + +// return() queued while the body awaits a pending promise. +async function* slow() { + try { + const v = await later(7); + yield v; + yield v + 1; + return "slow done"; + } finally { + log.push("slow finally"); + } +} +{ + const it = slow(); + const first = it.next(); + const ret = it.return("stopped"); + console.log(JSON.stringify(await first), JSON.stringify(await ret)); + console.log(log.splice(0).join(" | ")); +} + +// throw() at a yield, caught by the body, which keeps yielding. +async function* catcher() { + for (let i = 0; ; i++) { + try { + yield i; + } catch (e) { + log.push(`caught ${e}`); + await null; + yield `recovered ${i}`; + return; + } + } +} +{ + const it = catcher(); + console.log(JSON.stringify(await it.next())); + console.log(JSON.stringify(await it.throw("boom"))); + console.log(JSON.stringify(await it.next())); + console.log(log.splice(0).join(" | ")); +} + +// A body that throws after an await rejects its request and completes. +async function* failing() { + yield 1; + await later(0); + throw new Error("failed after await"); +} +{ + const it = failing(); + console.log(JSON.stringify(await it.next())); + try { + await it.next(); + } catch (e) { + console.log("rejected:", e.message); + } + console.log(JSON.stringify(await it.next())); +} + +// yield* into another async generator, and break out of for await. +async function* inner() { + try { + yield "a"; + yield await later("b"); + yield "c"; + } finally { + log.push("inner finally"); + } +} +async function* outer() { + yield "start"; + const r = yield* inner(); + log.push(`inner returned ${r}`); + yield "end"; +} +{ + const seen = []; + for await (const x of outer()) { + seen.push(x); + if (x === "b") break; + } + console.log(seen.join(",")); + console.log(log.splice(0).join(" | ")); +} + +// Stages chained through untyped values, interleaved with a promise chain. +async function* source(n) { + for (let i = 0; i < n; i++) yield i; +} +async function* map(it, f) { + for await (const x of it) yield await f(x); +} +async function* filter(it, p) { + for await (const x of it) if (p(x)) yield x; +} +{ + const order = []; + const chain = (async () => { + for await (const x of filter(map(source(6), async (x) => x * 3), (v) => v % 2 === 0)) order.push(`item ${x}`); + })(); + Promise.resolve() + .then(() => order.push("p1")) + .then(() => order.push("p2")) + .then(() => order.push("p3")); + await chain; + console.log(order.join(" ")); +} + +// A deep chain of pass-through stages. +async function* pass(it) { + for await (const x of it) yield x; +} +function stack(depth) { + return depth === 0 ? source(4) : pass(stack(depth - 1)); +} +{ + let sum = 0; + for await (const x of stack(500)) sum += x; + console.log("deep sum", sum); +} + +// Generator objects keep their tag and protocol. +{ + const g = counter(1); + console.log(Object.prototype.toString.call(g)); + console.log(JSON.stringify(await g.next()), JSON.stringify(await g.next())); + log.splice(0); +} diff --git a/tests/corpus/8383-stackless-async-generator-steps.ts b/tests/corpus/8383-stackless-async-generator-steps.ts new file mode 100644 index 000000000..f07e1d0f7 --- /dev/null +++ b/tests/corpus/8383-stackless-async-generator-steps.ts @@ -0,0 +1,99 @@ +// Typed async generators whose bodies suspend only at their own awaits and +// yields: number, string and object channels, values passed into next(), +// return() and throw() while the body awaits, finally blocks that await, +// yield* delegation, and an async function consuming them. +const log: string[] = []; +const later = (value: T): Promise => new Promise((resolve) => setTimeout(() => resolve(value), 1)); + +async function* numbers(n: number): AsyncGenerator { + try { + for (let i = 0; i < n; i++) { + yield i % 2 === 0 ? i : await later(i); + } + return "numbers done"; + } finally { + await later(0); + log.push("numbers finally"); + } +} + +async function* accumulate(): AsyncGenerator { + let total = 0; + for (;;) { + const v = yield total; + if (v < 0) return total; + total += await Promise.resolve(v); + } +} + +interface Point { + x: number; + y: number; +} +async function* points(n: number): AsyncGenerator { + for (let i = 0; i < n; i++) { + await null; + yield { x: i, y: i * i }; + } +} + +async function* labels(): AsyncGenerator { + yield "first"; + yield* (async function* () { + yield "inner one"; + yield await later("inner two"); + })(); + yield "last"; +} + +async function sum(it: AsyncGenerator): Promise { + let s = 0; + for await (const v of it) s += v; + return s; +} + +async function main(): Promise { + console.log("sum", await sum(numbers(6))); + console.log(log.splice(0).join(" | ")); + + const acc = accumulate(); + console.log(JSON.stringify(await acc.next(0))); + for (const v of [1, 2, 3]) console.log(JSON.stringify(await acc.next(v))); + console.log(JSON.stringify(await acc.next(-1))); + console.log(JSON.stringify(await acc.next(5))); + + for await (const p of points(3)) console.log(p.x, p.y); + + const words: string[] = []; + for await (const w of labels()) words.push(w); + console.log(words.join(", ")); + + const it = numbers(10); + console.log(JSON.stringify(await it.next())); + const pending = it.next(); + const ret = it.return("returned"); + console.log(JSON.stringify(await pending), JSON.stringify(await ret)); + console.log(log.splice(0).join(" | ")); + + const thrower = numbers(4); + await thrower.next(); + try { + await thrower.throw(new Error("thrown in")); + } catch (e) { + console.log("rejected:", (e as Error).message); + } + console.log(JSON.stringify(await thrower.next())); + console.log(log.splice(0).join(" | ")); + + const order: string[] = []; + const run = (async () => { + for await (const v of numbers(3)) order.push(`v${v}`); + })(); + void Promise.resolve() + .then(() => order.push("m1")) + .then(() => order.push("m2")); + await run; + console.log(order.join(" ")); + log.splice(0); +} +await main(); diff --git a/tests/corpus/8405-zlib-compressed-bytes.ts b/tests/corpus/8405-zlib-compressed-bytes.ts new file mode 100644 index 000000000..bfada812d --- /dev/null +++ b/tests/corpus/8405-zlib-compressed-bytes.ts @@ -0,0 +1,96 @@ +// Compressed zlib bytes over the vendored zlib, the copy Node bundles: deflate, +// raw deflate and gzip output match Node's for every level, strategy, memLevel, +// windowBits and dictionary, for text, binary and a 1 MiB input that slides the +// window, and for streams that flush and change parameters between writes. +import { createHash } from "node:crypto"; +import { + constants, + createDeflate, + createGzip, + deflateRawSync, + deflateSync, + gunzipSync, + gzipSync, + inflateRawSync, + inflateSync, + type Deflate, + type Gzip, + type ZlibOptions, +} from "node:zlib"; + +const digest = (bytes: Buffer): string => + `${bytes.length} ${createHash("sha256").update(bytes).digest("hex").slice(0, 16)}`; + +const words = ["stream", "pipeline", "émetteur", "π", "buffer", "deflate", "\u0000", "window"]; +const parts: string[] = []; +for (let i = 0; i < 3000; i++) parts.push(`${words[i % words.length]}-${(i * 7919) % 1000}`); +const text = Buffer.from(parts.join(" "), "utf8"); + +let seed = 0x2545f491; +const binary = Buffer.alloc(20000); +for (let i = 0; i < binary.length; i++) { + seed = (seed * 1103515245 + 12345) >>> 0; + binary[i] = i % 3 === 0 ? seed >>> 24 : (i >> 4) & 0xff; +} + +const large = Buffer.alloc(1 << 20); +for (let i = 0; i < large.length; i++) large[i] = text[(i * 31) % text.length]! ^ ((i >> 12) & 7); + +for (const level of [-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) { + const deflated = deflateSync(text, { level }); + console.log("deflate", level, digest(deflated), inflateSync(deflated).equals(text)); + console.log("raw", level, digest(deflateRawSync(binary, { level }))); + console.log("gzip", level, digest(gzipSync(text, { level }))); +} + +const variants: [string, ZlibOptions][] = [ + ["filtered", { strategy: constants.Z_FILTERED }], + ["huffman only", { strategy: constants.Z_HUFFMAN_ONLY }], + ["rle", { strategy: constants.Z_RLE }], + ["fixed", { strategy: constants.Z_FIXED }], + ["memLevel 1", { memLevel: 1 }], + ["memLevel 9", { memLevel: 9 }], + ["windowBits 9", { windowBits: 9 }], + ["windowBits 12", { windowBits: 12 }], + ["level 9 memLevel 9", { level: 9, memLevel: 9 }], + ["dictionary", { dictionary: Buffer.from("stream pipeline buffer deflate window") }], +]; +for (const [label, options] of variants) { + const deflated = deflateSync(text, options); + console.log(label, digest(deflated), digest(deflateRawSync(binary, options))); + console.log(label, "inflates", inflateSync(deflated, options).equals(text)); +} + +for (const level of [1, 6, 9]) { + const gz = gzipSync(large, { level }); + console.log("1 MiB gzip", level, digest(gz), gunzipSync(gz).equals(large)); + const raw = deflateRawSync(large, { level, windowBits: 10 }); + console.log("1 MiB raw windowBits 10", level, digest(raw), inflateRawSync(raw, { windowBits: 10 }).equals(large)); +} + +function collect(stream: Deflate | Gzip, label: string, done: () => void): void { + const chunks: Buffer[] = []; + stream.on("data", (chunk: Buffer) => chunks.push(chunk)); + stream.on("end", () => { + console.log(label, digest(Buffer.concat(chunks))); + done(); + }); +} + +const flushed = createDeflate({ level: 6 }); +collect(flushed, "stream sync flush", () => { + const changed = createGzip({ level: 1 }); + collect(changed, "stream params", () => console.log("done")); + changed.write(text.subarray(0, 8000)); + changed.flush(constants.Z_FULL_FLUSH, () => { + changed.params(9, constants.Z_DEFAULT_STRATEGY, () => { + changed.write(text.subarray(8000)); + changed.end(binary); + }); + }); +}); +flushed.write(text.subarray(0, 5000)); +flushed.flush(constants.Z_SYNC_FLUSH, () => { + flushed.write(binary); + flushed.end(text.subarray(5000)); +}); diff --git a/tests/corpus/8406-zlib-inflate-chunks.cjs b/tests/corpus/8406-zlib-inflate-chunks.cjs new file mode 100644 index 000000000..f8fade68f --- /dev/null +++ b/tests/corpus/8406-zlib-inflate-chunks.cjs @@ -0,0 +1,69 @@ +// Inflation over the vendored zlib's chunked inflate: inputs fed in pieces of +// 1 to 4096 bytes, output chunks of 64 bytes to 64 KiB, raw windowBits 9 to 15, +// concatenated gzip members, back-references across a 1 MiB window, and a +// truncated input. The output and the error match Node's. +const zlib = require("node:zlib"); +const { createHash } = require("node:crypto"); + +const digest = (bytes) => `${bytes.length} ${createHash("sha256").update(bytes).digest("hex").slice(0, 16)}`; + +const parts = []; +for (let i = 0; i < 20000; i++) parts.push(`row ${i % 977} ${"abcdefgh".slice(i % 8)} ${(i * 2654435761) % 4093}`); +const text = Buffer.from(parts.join("\n")); +const large = Buffer.alloc(1 << 20); +for (let i = 0; i < large.length; i++) large[i] = text[(i * 13) % text.length] ^ ((i >> 15) & 3); + +const gz = zlib.gzipSync(large, { level: 6 }); +for (const chunkSize of [64, 1024, 16384, 65536]) { + console.log("gunzip chunkSize", chunkSize, digest(zlib.gunzipSync(gz, { chunkSize }))); +} +for (const windowBits of [9, 10, 12, 15]) { + const raw = zlib.deflateRawSync(text, { windowBits, level: 9 }); + console.log("raw windowBits", windowBits, digest(zlib.inflateRawSync(raw, { windowBits }))); +} +const members = Buffer.concat([zlib.gzipSync(text.subarray(0, 1000)), zlib.gzipSync(text.subarray(1000, 5000))]); +console.log("members", digest(zlib.gunzipSync(members))); +try { + zlib.inflateSync(zlib.deflateSync(text).subarray(0, 2000)); +} catch (e) { + console.log("truncated", e.code, e.errno, e.message); +} + +function feed(input, pieceSize, outSize, next) { + const inflate = zlib.createGunzip({ chunkSize: outSize }); + const hash = createHash("sha256"); + let length = 0; + inflate.on("data", (chunk) => { + hash.update(chunk); + length += chunk.length; + }); + inflate.on("end", () => { + console.log("stream piece", pieceSize, "chunk", outSize, length, hash.digest("hex").slice(0, 16)); + next(); + }); + let offset = 0; + const pump = () => { + while (offset < input.length) { + const piece = input.subarray(offset, offset + pieceSize); + offset += pieceSize; + if (!inflate.write(piece)) { + inflate.once("drain", pump); + return; + } + } + inflate.end(); + }; + pump(); +} + +const head = zlib.gzipSync(large.subarray(0, 1 << 16), { level: 6 }); +const runs = [ + [head, 1, 64], + [gz, 7, 1024], + [gz, 333, 4096], + [gz, 4096, 65536], +]; +const step = (i) => { + if (i < runs.length) feed(runs[i][0], runs[i][1], runs[i][2], () => step(i + 1)); +}; +step(0); diff --git a/tests/corpus/8407-string-index-of-from-ascii.ts b/tests/corpus/8407-string-index-of-from-ascii.ts new file mode 100644 index 000000000..04582b7cd --- /dev/null +++ b/tests/corpus/8407-string-index-of-from-ascii.ts @@ -0,0 +1,38 @@ +// indexOf with a start index on ASCII receivers: integral, fractional, +// negative, -0, NaN, Infinity and out-of-range starts; empty needles, which +// are found at the clamped start; needles longer than the rest; needles with +// non-ASCII characters and lone surrogates, which an ASCII receiver never +// contains; and receivers built at run time (slices, concatenations, readline +// style splits) next to non-ASCII ones. The results match Node's. +const line = `10.1.2.3 - - [24/Oct/2026:10:00:00 +0000] "GET /api/items/7?page=3 HTTP/1.1" 200 5120 "agent"`; +const starts = [0, 1, 2.7, -1, -0, NaN, Infinity, -Infinity, line.length - 1, line.length, line.length + 5, 1e21]; +const needles = ["", '"', " ", "/", "api", "HTTP/1.1", line, line + "x", "é", "\ud83d", "\udca9", "💩"]; + +for (const needle of needles) { + const found = starts.map((start) => line.indexOf(needle, start)); + console.log(JSON.stringify(needle), found.join(" ")); +} + +const quote = line.indexOf('"'); +const space = line.indexOf(" ", quote); +const path = line.slice(space + 1, line.indexOf("/", line.indexOf("/", space) + 1)); +const close = line.indexOf('"', space); +console.log(quote, space, path, close, line.slice(close + 2, close + 5)); + +const built = ["alpha", "beta", "gamma"].join("/") + "/" + String(42); +for (let i = 0; i <= built.length + 1; i++) { + console.log(i, built.indexOf("/", i), built.indexOf("a", i), built.indexOf("", i), built.indexOf("42", i)); +} + +const mixed = "café/naïve/ascii/end"; +const ascii = mixed.slice(mixed.indexOf("ascii")); +console.log(ascii, ascii.indexOf("/", 1), ascii.indexOf("end", 3), mixed.indexOf("/", 5), mixed.indexOf("ascii", 6)); + +let count = 0; +let at = 0; +const csv = Array.from({ length: 200 }, (_, i) => `${i},${i * 3},row${i}`).join(","); +while ((at = csv.indexOf(",", at)) !== -1) { + count++; + at += 1.5; +} +console.log("commas", count); diff --git a/tests/corpus/8410-dynamic-class-instance-reads.js b/tests/corpus/8410-dynamic-class-instance-reads.js new file mode 100644 index 000000000..83b9cad9b --- /dev/null +++ b/tests/corpus/8410-dynamic-class-instance-reads.js @@ -0,0 +1,62 @@ +// Untyped reads and writes of class instances at property sites: base and +// subclass instances, subclass instances returned where the base class is +// the inferred type, a subclass that redeclares an inherited field, a +// getter, fields set after construction and fields never set. +class Point { + constructor(x, y) { + this.x = x; + this.y = y; + } +} +class Tagged extends Point { + constructor(x, y, tag) { + super(x, y); + this.tag = tag; + } +} +class Shifted extends Point { + label = "s"; + x = 100; +} +class Gauge { + get x() { + return 9; + } +} +class Late { + constructor(set) { + if (set) this.x = "late"; + } +} +function make(i) { + if (i % 3 === 0) return new Point(i, -i); + if (i % 3 === 1) return new Tagged(i, i * 2, "t" + i); + return new Shifted(i, 0); +} +function readX(o) { + return o.x; +} +function readY(o) { + return o.y; +} +function writeX(o, v) { + o.x = v; +} +const values = []; +for (let i = 0; i < 9; i++) values.push(make(i)); +values.push(new Gauge(), new Late(true), new Late(false), { x: "plain", y: 1 }, { y: 2, x: "swapped" }); +let sum = 0; +for (let round = 0; round < 3; round++) { + for (const v of values) { + const x = readX(v); + sum += typeof x === "number" ? x : x === undefined ? 1000 : x.length; + const y = readY(v); + if (typeof y === "number") sum += y; + } + for (let i = 0; i < 9; i++) writeX(values[i], round * 10 + i); +} +console.log(sum); +console.log(values.map((v) => String(readX(v))).join(" ")); +console.log(values.map((v) => String(readY(v))).join(" ")); +writeX(values[1], "changed"); +console.log(readX(values[1]), readX(values[2]), values[2].label, values[1].tag); diff --git a/tests/corpus/8411-dynamic-base-keyed-instances/main.ts b/tests/corpus/8411-dynamic-base-keyed-instances/main.ts new file mode 100644 index 000000000..5c0caa936 --- /dev/null +++ b/tests/corpus/8411-dynamic-base-keyed-instances/main.ts @@ -0,0 +1,32 @@ +// Untyped property reads of class instances that typed code passes as their +// base class: subclass instances in a Point[] read their inherited field and +// their own field through an untyped module. +import { createRequire } from "node:module"; +const require = createRequire(import.meta.url); +const reader: { + readX(o: unknown): unknown; + readTag(o: unknown): unknown; + sum(list: unknown[]): number; +} = require("./reader.cjs"); + +class Point { + x: any; + constructor(x: any) { + this.x = x; + } +} +class Tagged extends Point { + tag: any; + constructor(x: any, tag: any) { + super(x); + this.tag = tag; + } +} +function make(i: number): Point { + return i % 2 === 0 ? new Point(i) : new Tagged(i * 10, "t" + i); +} +const list: Point[] = []; +for (let i = 0; i < 6; i++) list.push(make(i)); +console.log(list.map((p) => String(reader.readX(p))).join(" ")); +console.log(list.map((p) => String(reader.readTag(p))).join(" ")); +console.log(reader.sum(list)); diff --git a/tests/corpus/8411-dynamic-base-keyed-instances/package.json b/tests/corpus/8411-dynamic-base-keyed-instances/package.json new file mode 100644 index 000000000..5ffd9800b --- /dev/null +++ b/tests/corpus/8411-dynamic-base-keyed-instances/package.json @@ -0,0 +1 @@ +{ "type": "module" } diff --git a/tests/corpus/8411-dynamic-base-keyed-instances/reader.cjs b/tests/corpus/8411-dynamic-base-keyed-instances/reader.cjs new file mode 100644 index 000000000..941299ed5 --- /dev/null +++ b/tests/corpus/8411-dynamic-base-keyed-instances/reader.cjs @@ -0,0 +1,11 @@ +exports.readX = function (o) { + return o.x; +}; +exports.readTag = function (o) { + return o.tag; +}; +exports.sum = function (list) { + let s = 0; + for (let i = 0; i < list.length; i++) s += list[i].x; + return s; +}; diff --git a/tests/corpus/arena-array-subclass.ts b/tests/corpus/arena-array-subclass.ts new file mode 100644 index 000000000..281d07823 --- /dev/null +++ b/tests/corpus/arena-array-subclass.ts @@ -0,0 +1,63 @@ +// Builders whose groups hold instances of a class that extends Array, +// beside a builder whose group takes an arena. An Array subclass's +// instance is an array, so a group that allocates one keeps its objects on +// the heap; the results and their teardown match Node either way. +class Row extends Array { + sum(): number { + let s = 0; + for (const v of this) s += v; + return s; + } +} + +class Branch { + row: Row; + left: Branch | null; + right: Branch | null; + constructor(row: Row, left: Branch | null, right: Branch | null) { + this.row = row; + this.left = left; + this.right = right; + } + total(): number { + return this.row.sum() + (this.left ? this.left.total() : 0) + (this.right ? this.right.total() : 0); + } +} + +class Pair { + left: Pair | null; + right: Pair | null; + constructor(left: Pair | null, right: Pair | null) { + this.left = left; + this.right = right; + } + count(): number { + return 1 + (this.left ? this.left.count() : 0) + (this.right ? this.right.count() : 0); + } +} + +function row(n: number): Row { + const r = new Row(); + for (let j = 0; j <= n; j++) r.push(j); + return r; +} + +function branches(depth: number): Branch { + return depth === 0 + ? new Branch(row(depth), null, null) + : new Branch(row(depth), branches(depth - 1), branches(depth - 1)); +} + +function pairs(depth: number): Pair { + return depth === 0 ? new Pair(null, null) : new Pair(pairs(depth - 1), pairs(depth - 1)); +} + +let checks = 0; +for (let round = 0; round < 20; round++) { + const b = branches(6); + const p = pairs(8); + const r = row(round); + checks += b.total() + p.count() + r.sum() + r.length; + if (round === 0) console.log(b.total(), b.row.length, b.row instanceof Row, p.count(), r.sum()); +} +console.log(checks); diff --git a/tests/corpus/arena-arrays.ts b/tests/corpus/arena-arrays.ts new file mode 100644 index 000000000..fb2134231 --- /dev/null +++ b/tests/corpus/arena-arrays.ts @@ -0,0 +1,49 @@ +// Groups with arrays: child lists that grow past their first storage, +// back references to parents and siblings, and lists of lists. +class Node { + parent: Node | undefined = undefined; + children: Node[] = []; + sibling: Node | undefined = undefined; + weight: number; + constructor(weight: number) { + this.weight = weight; + } +} + +function wide(depth: number, fan: number, parent: Node | undefined, weight: number): Node { + const node = new Node(weight); + node.parent = parent; + if (depth > 0) { + for (let i = 0; i < fan; i++) node.children.push(wide(depth - 1, fan, node, weight * 3 + i)); + for (let i = 0; i < fan; i++) node.children[i]!.sibling = node.children[(i + 1) % fan]!; + } + return node; +} + +function weigh(node: Node): number { + let total = node.weight; + for (const child of node.children) total += weigh(child); + return total % 1000003; +} + +let w = 0; +for (let round = 0; round < 30; round++) w = (w + weigh(wide(2 + (round % 3), 3 + (round % 20), undefined, round))) % 1000003; +console.log("weights", w); + +function grid(rows: number, cols: number): number[][] { + const out: number[][] = []; + for (let r = 0; r < rows; r++) { + const row: number[] = []; + for (let c = 0; c < cols; c++) row.push(r * cols + c); + out.push(row); + } + return out; +} +let cells = 0; +for (let k = 1; k < 40; k++) { + const g = grid(k, 70); + cells += g.length + g[k - 1]![69]! + g[0]!.length; +} +console.log("cells", cells); +const t = wide(2, 4, undefined, 1); +console.log(t.children.length, t.children[1]!.parent === t, t.children[3]!.sibling === t.children[0], t.children[2]!.children[3]!.weight); diff --git a/tests/corpus/arena-cap.ts b/tests/corpus/arena-cap.ts new file mode 100644 index 000000000..1ab1f43aa --- /dev/null +++ b/tests/corpus/arena-cap.ts @@ -0,0 +1,30 @@ +// A group too large for one arena: past its byte cap the rest of the +// group goes to the heap, and both parts are released together. +interface Cell { + index: number; + label: string; + next: Cell | null; +} + +function list(n: number): Cell { + let head: Cell = { index: 0, label: "c0", next: null }; + const first = head; + for (let i = 1; i < n; i++) { + const cell: Cell = { index: i, label: i % 1000 === 0 ? `c${i}` : "c", next: null }; + head.next = cell; + head = cell; + } + return first; +} + +function walk(c: Cell | null): number { + let total = 0; + let labels = 0; + for (let at = c; at !== null; at = at.next) { + total += at.index; + if (at.label.length > 1) labels++; + } + return total + labels; +} + +for (let round = 0; round < 3; round++) console.log("cells", walk(list(900000 + round))); diff --git a/tests/corpus/arena-escapes.ts b/tests/corpus/arena-escapes.ts new file mode 100644 index 000000000..f65a34e87 --- /dev/null +++ b/tests/corpus/arena-escapes.ts @@ -0,0 +1,72 @@ +// Values built in a group that leave it: handed to callbacks that keep +// them, stored by a method into a module list, kept from a getter, pushed +// into a caller's list, or captured by a returned closure. None of those +// groups takes an arena; the one whose values stay in the frame does. +class Node { + kids: Node[] = []; + label: string; + constructor(label: string) { + this.label = label; + } + get first(): Node { + return this.kids[0]!; + } + register(): void { + registry.push(this); + } +} +const registry: Node[] = []; +const saved: Node[] = []; +const held = new Map(); +function build(depth: number, label: string): Node { + const n = new Node(label); + for (let i = 0; depth > 0 && i < 3; i++) n.kids.push(build(depth - 1, `${label}.${i}`)); + return n; +} +function each(depth: number, cb: (n: Node) => void): number { + const t = build(depth, "e"); + cb(t); + return t.kids.length; +} +function visit(n: Node, cb: (x: Node) => void): void { + cb(n); + for (const k of n.kids) visit(k, cb); +} +function forward(depth: number, cb: (x: Node) => void): number { + const t = build(depth, "f"); + visit(t, cb); + return t.kids.length; +} +function buildInto(sink: Node[], depth: number): number { + const n = build(depth, "s"); + sink.push(n.kids[0]!); + return n.kids.length; +} +function closureOf(depth: number): () => string { + const n = build(depth, "c"); + return () => n.kids[1]!.label; +} +let total = 0; +for (let r = 0; r < 8; r++) { + total += each(2, (n) => saved.push(n.kids[1]!)); + total += forward(2, (x) => { + if (x.label.endsWith(".2")) saved.push(x); + }); + const t = build(2, `t${r}`); + t.kids.forEach((k) => { + if (k.label.endsWith(".0")) saved.push(k); + }); + const u = build(2, `u${r}`); + for (const k of u.kids) if (k.label.endsWith(".1")) saved.push(k); + const v = build(2, `v${r}`); + held.set(r, v.first); + const a = build(2, `a${r}`); + if (r % 3 === 0) a.kids[0]!.register(); + total += buildInto(saved, 2); + const w = build(2, `w${r}`); + total += w.kids.length + w.first.kids.length; +} +console.log(total, saved.length, saved.map((n) => n.label + "/" + n.kids.length).slice(0, 10).join(",")); +const f = closureOf(2); +console.log(f(), registry.map((n) => n.label + n.kids.length).join(",")); +console.log([...held.values()].map((n) => n.label + n.kids[0]!.label).join(",")); diff --git a/tests/corpus/arena-mixed.ts b/tests/corpus/arena-mixed.ts new file mode 100644 index 000000000..ff6c8e76c --- /dev/null +++ b/tests/corpus/arena-mixed.ts @@ -0,0 +1,80 @@ +// Groups built through virtual calls and sorted with callbacks, nested +// groups inside a group, and temporaries the group does not keep. +abstract class Shape { + abstract area(): number; + abstract scaled(k: number): Shape; +} +class Square extends Shape { + side: number; + constructor(side: number) { + super(); + this.side = side; + } + area(): number { + return this.side * this.side; + } + scaled(k: number): Shape { + return new Square(this.side * k); + } +} +class Circle extends Shape { + radius: number; + constructor(radius: number) { + super(); + this.radius = radius; + } + area(): number { + return Math.round(Math.PI * this.radius * this.radius * 100) / 100; + } + scaled(k: number): Shape { + return new Circle(this.radius * k); + } +} + +interface Box { + label: string; + shapes: Shape[]; +} + +function measure(s: Shape): { area: number } { + return { area: s.area() }; +} + +function boxes(n: number): Box[] { + const out: Box[] = []; + for (let i = 0; i < n; i++) { + const shapes: Shape[] = []; + for (let j = 0; j < 4; j++) { + const base: Shape = j % 2 === 0 ? new Square(j + i) : new Circle(j + 1); + shapes.push(base.scaled(2)); + } + // Temporaries the group does not keep. + let total = 0; + for (const s of shapes) total += measure(s).area; + shapes.sort((a, b) => a.area() - b.area()); + out.push({ label: `box${i}:${Math.round(total)}`, shapes }); + } + return out; +} + +function summarize(n: number): string { + const list = boxes(n); + let largest = ""; + let best = -1; + for (const b of list) { + const top = b.shapes[b.shapes.length - 1]!.area(); + if (top > best) { + best = top; + largest = b.label; + } + } + return `${largest}@${best}`; +} + +let out = ""; +for (let round = 1; round < 30; round++) { + const all = boxes(round); + out = `${all.length} ${all[0]!.label} ${all[round - 1]!.shapes[0]!.area()} ${summarize(round)}`; + if (round % 7 === 0) console.log(out); +} +console.log(out); diff --git a/tests/corpus/arena-outside.ts b/tests/corpus/arena-outside.ts new file mode 100644 index 000000000..cc965d5d3 --- /dev/null +++ b/tests/corpus/arena-outside.ts @@ -0,0 +1,49 @@ +// Groups whose members refer to objects outside them: a long-lived +// registry, strings built along the way, and a parameter's object. The +// arena's drop releases those references; the outside objects stay usable. +class Category { + members = 0; + name: string; + constructor(name: string) { + this.name = name; + } +} + +class Member { + next: Member | undefined = undefined; + category: Category; + label: string; + rank: number; + constructor(category: Category, label: string, rank: number) { + this.category = category; + this.label = label; + this.rank = rank; + category.members++; + } +} + +function ring(categories: Category[], size: number, seed: number): Member { + const first = new Member(categories[seed % categories.length]!, `m${seed}`, seed); + let last = first; + for (let i = 1; i < size; i++) { + const member = new Member(categories[(seed + i * 7) % categories.length]!, `m${seed}-${i}`, seed + i); + last.next = member; + last = member; + } + last.next = first; + return first; +} + +const categories: Category[] = []; +for (let i = 0; i < 5; i++) categories.push(new Category(`cat-${i}`)); + +let ranks = 0; +let labels = ""; +for (let round = 0; round < 200; round++) { + const r = ring(categories, 30, round); + ranks += r.next!.rank + r.next!.next!.rank; + if (round % 50 === 0) labels += `${r.label}:${r.category.name}:${r.next!.label} `; +} +console.log("ranks", ranks); +console.log("labels", labels.trim()); +for (const c of categories) console.log(c.name, c.members); diff --git a/tests/corpus/arena-region-throws.ts b/tests/corpus/arena-region-throws.ts new file mode 100644 index 000000000..073dca966 --- /dev/null +++ b/tests/corpus/arena-region-throws.ts @@ -0,0 +1,33 @@ +// Exceptions inside call regions: the region's arena is dropped as the +// exception leaves the call, and the caller's own objects are untouched. +interface Entry { + key: string; + value: number; + next: Entry | null; +} + +function tally(keys: string[], limit: number): number { + const entries: Entry[] = []; + for (const k of keys) { + entries.push({ key: k, value: k.length, next: entries.length > 0 ? entries[entries.length - 1]! : null }); + if (entries.length > limit) throw new RangeError(`too many entries: ${entries.length}`); + } + let sum = 0; + for (const e of entries) sum += e.value + (e.next === null ? 0 : e.next.value); + return sum; +} + +const kept: Entry[] = [{ key: "outside", value: 1, next: null }]; +const keys = "alpha beta gamma delta epsilon zeta eta theta iota kappa".split(" "); +let total = 0; +let errors = 0; +for (let round = 0; round < 40; round++) { + try { + total += tally(keys, 3 + (round % 10)); + } catch (e) { + errors++; + if (round < 6) console.log((e as Error).name, (e as Error).message); + } + kept.push({ key: `k${round}`, value: round, next: kept[kept.length - 1]! }); +} +console.log("total", total, "errors", errors, "kept", kept.length, kept[kept.length - 1]!.next!.key); diff --git a/tests/corpus/arena-regions.ts b/tests/corpus/arena-regions.ts new file mode 100644 index 000000000..cd1aa9679 --- /dev/null +++ b/tests/corpus/arena-regions.ts @@ -0,0 +1,44 @@ +// Call regions: calls of functions that build what they need and return +// a number, so everything they allocate is dead when they return. +interface Point { + x: number; + y: number; +} +interface Segment { + from: Point; + to: Point; +} + +function perimeter(n: number, scale: number): number { + const points: Point[] = []; + for (let i = 0; i < n; i++) points.push({ x: Math.cos(i) * scale, y: Math.sin(i) * scale }); + const segments: Segment[] = []; + for (let i = 0; i < n; i++) segments.push({ from: points[i]!, to: points[(i + 1) % n]! }); + let total = 0; + for (const s of segments) total += Math.hypot(s.to.x - s.from.x, s.to.y - s.from.y); + return Math.round(total * 1000) / 1000; +} + +let sum = 0; +for (let k = 3; k < 300; k++) sum += perimeter(k, k / 10); +console.log("perimeters", sum.toFixed(3)); + +function histogram(words: string[]): number { + const buckets: { word: string; count: number }[] = []; + for (const w of words) { + let found = false; + for (const b of buckets) + if (b.word === w) { + b.count++; + found = true; + } + if (!found) buckets.push({ word: w, count: 1 }); + } + let best = 0; + for (const b of buckets) best = Math.max(best, b.count * 100 + b.word.length); + return best; +} +const words = "the quick brown fox jumps over the lazy dog the end".split(" "); +let h = 0; +for (let i = 0; i < 200; i++) h += histogram(words); +console.log("histogram", h); diff --git a/tests/corpus/arena-thrown-member.ts b/tests/corpus/arena-thrown-member.ts new file mode 100644 index 000000000..68382075c --- /dev/null +++ b/tests/corpus/arena-thrown-member.ts @@ -0,0 +1,36 @@ +// A thrown value that holds an object built in a group: the group cannot +// take an arena, because the value outlives it in the catch handler. +class Node { + label: string; + weight: number; + constructor(label: string, weight: number) { + this.label = label; + this.weight = weight; + } +} +class BuildError { + node: Node; + constructor(node: Node) { + this.node = node; + } +} +function build(n: number, failAt: number): Node[] { + const out: Node[] = []; + for (let i = 0; i < n; i++) { + const node = new Node(`n${i}`, i); + out.push(node); + if (i === failAt) throw new BuildError(node); + } + return out; +} +let total = 0; +const saved: string[] = []; +for (let r = 0; r < 20; r++) { + try { + total += build(50, r % 4 === 0 ? 30 : -1).length; + } catch (e) { + const b = e as BuildError; + saved.push(b.node.label + ":" + b.node.weight); + } +} +console.log(total, saved.join(",")); diff --git a/tests/corpus/arena-throws.ts b/tests/corpus/arena-throws.ts new file mode 100644 index 000000000..b3dbd3c16 --- /dev/null +++ b/tests/corpus/arena-throws.ts @@ -0,0 +1,40 @@ +// A builder that throws part way: the arena of the group it was building +// is dropped as the exception leaves, and building goes on afterwards. +interface Item { + id: number; + tags: string[]; + next: Item | null; +} + +function chain(n: number, failAt: number): Item { + let head: Item = { id: 0, tags: ["head"], next: null }; + const first = head; + for (let i = 1; i < n; i++) { + if (i === failAt) throw new Error(`failed at ${i} of ${n}`); + const item: Item = { id: i, tags: [`t${i}`, `u${i % 3}`], next: null }; + head.next = item; + head = item; + } + return first; +} + +function length(item: Item | null): number { + let n = 0; + for (let at = item; at !== null; at = at.next) n++; + return n; +} + +let built = 0; +let failed = 0; +for (let round = 0; round < 60; round++) { + try { + const c = chain(20 + (round % 7), round % 5 === 0 ? 10 + (round % 3) : -1); + built += length(c) + c.next!.tags.length; + } catch (e) { + failed++; + if (round < 20) console.log("caught", (e as Error).message); + } +} +console.log("built", built, "failed", failed); +const last = chain(5, -1); +console.log(last.id, last.next!.id, last.next!.tags.join("/"), length(last)); diff --git a/tests/corpus/arena-trees.ts b/tests/corpus/arena-trees.ts new file mode 100644 index 000000000..eb736dbab --- /dev/null +++ b/tests/corpus/arena-trees.ts @@ -0,0 +1,70 @@ +// Recursive builders whose results their callers only read: each tree is a +// group that lives in an arena until the statement (or the loop body that +// holds it) ends. +class TreeNode { + left: TreeNode | null; + right: TreeNode | null; + value: number; + constructor(left: TreeNode | null, right: TreeNode | null, value: number) { + this.left = left; + this.right = right; + this.value = value; + } + sum(): number { + return this.value + (this.left === null ? 0 : this.left.sum()) + (this.right === null ? 0 : this.right.sum()); + } + depth(): number { + return 1 + Math.max(this.left === null ? 0 : this.left.depth(), this.right === null ? 0 : this.right.depth()); + } +} + +function build(depth: number, value: number): TreeNode { + return depth === 0 + ? new TreeNode(null, null, value) + : new TreeNode(build(depth - 1, value * 2), build(depth - 1, value * 2 + 1), value); +} + +let total = 0; +for (let d = 0; d < 12; d++) total += build(d, 1).sum(); +console.log("sums", total); + +for (let d = 2; d <= 4; d++) { + const t = build(d, 1); + console.log("tree", d, t.sum(), t.depth(), t.left!.value, t.right!.right!.value); +} + +function deepest(n: number): number { + let best = 0; + for (let i = 0; i < n; i++) { + const t = build(i % 6, i); + best = Math.max(best, t.depth() * 1000 + (t.sum() % 1000)); + } + return best; +} +console.log("deepest", deepest(40)); + +// Record-literal trees: a root call whose result its caller only reads +// builds the whole tree, its root node included, in the group's arena. +interface Node2 { + left: Node2 | null; + right: Node2 | null; + value: number; +} + +function buildRecord(depth: number, value: number): Node2 { + return depth === 0 + ? { left: null, right: null, value } + : { left: buildRecord(depth - 1, value * 2), right: buildRecord(depth - 1, value * 2 + 1), value }; +} + +function sumRecord(node: Node2): number { + return node.value + (node.left === null ? 0 : sumRecord(node.left)) + (node.right === null ? 0 : sumRecord(node.right)); +} + +let recordTotal = 0; +for (let d = 0; d < 12; d++) recordTotal += sumRecord(buildRecord(d, 1)); +console.log("record sums", recordTotal); +for (let d = 1; d <= 3; d++) { + const t = buildRecord(d, 1); + console.log("record tree", d, sumRecord(t), t.value, t.left!.value, t.right!.value); +} diff --git a/tests/corpus/arena-workers.ts b/tests/corpus/arena-workers.ts new file mode 100644 index 000000000..b08a49748 --- /dev/null +++ b/tests/corpus/arena-workers.ts @@ -0,0 +1,75 @@ +// Arena groups in worker threads: each worker builds and reads trees of +// classes and record literals, groups that live in the worker's own arena +// until the statement or loop body that holds them ends. The main thread +// builds its own groups meanwhile and keeps one tree for the whole run. +import { isMainThread, parentPort, Worker, workerData } from "node:worker_threads"; + +class Pair { + left: Pair | null; + right: Pair | null; + label: string; + constructor(left: Pair | null, right: Pair | null, label: string) { + this.left = left; + this.right = right; + this.label = label; + } +} + +interface Leaf { + left: Leaf | null; + right: Leaf | null; + weight: number; +} + +function pairs(depth: number, label: string): Pair { + return depth === 0 ? new Pair(null, null, label) : new Pair(pairs(depth - 1, label + "l"), pairs(depth - 1, label + "r"), label); +} + +function leaves(depth: number, weight: number): Leaf { + return depth === 0 + ? { left: null, right: null, weight } + : { left: leaves(depth - 1, weight + 1), right: leaves(depth - 1, weight * 2), weight }; +} + +function countPairs(p: Pair): number { + return 1 + (p.left === null ? 0 : countPairs(p.left)) + (p.right === null ? 0 : countPairs(p.right)); +} + +function rightmost(p: Pair): string { + return p.right === null ? p.label : rightmost(p.right); +} + +function weigh(l: Leaf): number { + return l.weight + (l.left === null ? 0 : weigh(l.left)) + (l.right === null ? 0 : weigh(l.right)); +} + +function task(depth: number, rounds: number): string { + let nodes = 0; + let total = 0; + for (let i = 0; i < rounds; i++) { + nodes += countPairs(pairs(depth, "p")); + total += weigh(leaves(depth, i)); + } + const kept = pairs(depth, "k"); + return `depth ${depth}: ${nodes} pairs, weight ${total}, rightmost ${rightmost(kept)}, kept ${countPairs(kept)}`; +} + +if (isMainThread) { + const longLived = leaves(10, 1); + const depths = [3, 5, 7, 9]; + const lines: string[] = new Array(depths.length).fill(""); + let pending = depths.length; + depths.forEach((depth, index) => { + const worker = new Worker(new URL(import.meta.url), { workerData: depth }); + worker.on("message", (line: string) => { + lines[index] = line; + if (--pending === 0) { + for (const l of lines) console.log(l); + console.log("main", task(6, 20)); + console.log("long lived", weigh(longLived)); + } + }); + }); +} else { + parentPort!.postMessage(task(workerData as number, 50)); +} diff --git a/tests/corpus/array-subclass-element-cycles.ts b/tests/corpus/array-subclass-element-cycles.ts new file mode 100644 index 000000000..375fe6faf --- /dev/null +++ b/tests/corpus/array-subclass-element-cycles.ts @@ -0,0 +1,60 @@ +// A class that extends Array whose elements can lead back to it only +// through a field of plain array type (TreeNode.kids holds a Kids, typed as +// an array of TreeNode): its instances carry the collector header that +// arrays of its element type carry, so the collector can visit them through +// that field. Forest's elements are Kids instances, which carry one too. +class Kids extends Array { + label = "kids"; + total(): number { + let sum = 0; + for (const kid of this) sum += kid.size(); + return sum; + } +} + +class TreeNode { + kids = new Kids(); + value: number; + constructor(value: number) { + this.value = value; + } + size(): number { + return 1 + this.kids.total(); + } +} + +class Forest extends Array { + name = "forest"; +} + +function grow(depth: number): TreeNode { + const node = new TreeNode(depth); + if (depth > 0) for (let i = 0; i < 3; i++) node.kids.push(grow(depth - 1)); + return node; +} + +// Garbage cycles through the field of array type make the collector run +// while the trees are alive. +function churn(rounds: number): number { + let made = 0; + for (let i = 0; i < rounds; i++) { + const node = new TreeNode(i); + node.kids.push(node); + made++; + } + return made; +} + +let checks = 0; +for (let round = 0; round < 20; round++) { + checks += grow(4).size(); + churn(500); +} +console.log(checks); + +const forest = new Forest(); +for (let i = 0; i < 4; i++) forest.push(grow(2).kids); +churn(5000); +let leaves = 0; +for (const kids of forest) leaves += kids.total(); +console.log(forest.name, forest.length, leaves, forest[0]!.label, forest[0] instanceof Kids); diff --git a/tests/corpus/array-subclass-element-instanceof.ts b/tests/corpus/array-subclass-element-instanceof.ts new file mode 100644 index 000000000..f520e55a1 --- /dev/null +++ b/tests/corpus/array-subclass-element-instanceof.ts @@ -0,0 +1,37 @@ +// Instances of classes that extend Array read back from an array of them +// (an element read is `Row | undefined`) still answer instanceof for their +// own class and Array, false for another Array subclass, and plain arrays +// answer false. +class Row extends Array { + sum(): number { + let s = 0; + for (const v of this) s += v; + return s; + } +} +class Column extends Array {} + +function rows(n: number): Row[] { + const out: Row[] = []; + for (let i = 0; i < n; i++) { + const row = new Row(); + for (let j = 0; j <= i; j++) row.push(j); + out.push(row); + } + return out; +} + +const lines: number[][] = [new Row(), new Column(), [1, 2]]; + +const table = rows(6); +console.log(table.length, table[5]!.sum(), table[5] instanceof Row, table[5] instanceof Column); +console.log(table[4] instanceof Row, table[4] instanceof Array); +console.log(table.at(-1) instanceof Row, table.at(10) instanceof Row); +const plain: number[][] = [[1, 2]]; +console.log(plain[0] instanceof Row, plain[0] instanceof Array); +const kinds = lines.map((r) => (r instanceof Column ? "column" : r instanceof Row ? "row" : "plain")); +console.log(kinds.join(",")); +let checks = 0; +for (let round = 0; round < 20; round++) + checks += rows(6).reduce((s, r) => s + (r instanceof Row ? r.sum() : 0), 0); +console.log(checks); diff --git a/tests/corpus/discarded-logical-void-conditionals.mjs b/tests/corpus/discarded-logical-void-conditionals.mjs new file mode 100644 index 000000000..e804a28e3 --- /dev/null +++ b/tests/corpus/discarded-logical-void-conditionals.mjs @@ -0,0 +1,22 @@ +// React's development warnings discard a logical whose right operand is a +// conditional of console.error calls. +function warnIfMissingRender(instance, Component) { + instance.render || + (Component.prototype && typeof Component.prototype.render === "function" + ? console.error("%s(...): No `render` method found on the returned component.", "App") + : console.error("%s(...): No `render` method found, and its prototype has none.", "App")); +} +warnIfMissingRender({ render: null }, { prototype: { render() {} } }); +warnIfMissingRender({ render: null }, { prototype: {} }); +warnIfMissingRender({ render() {} }, { prototype: {} }); + +let warned = 0; +function warn(message) { + warned++; + console.log("warn:", message); +} +for (const props of [{ key: 1 }, {}, { key: "a", ref: null }]) { + "key" in props && (typeof props.key === "string" ? warn("string key") : warn("number key")); + props.ref === undefined || (props.ref === null ? warn("null ref") : warn("ref")); +} +console.log("warned", warned); diff --git a/tests/corpus/discarded-logical-void-conditionals.ts b/tests/corpus/discarded-logical-void-conditionals.ts new file mode 100644 index 000000000..1437957da --- /dev/null +++ b/tests/corpus/discarded-logical-void-conditionals.ts @@ -0,0 +1,69 @@ +// A logical expression whose result is discarded and whose right operand is +// a conditional of void calls (`skip || (flag ? warnA() : warnB())`) runs as +// a branch: the left operand decides whether the conditional runs, and the +// conditional runs one arm. +const order: string[] = []; +function check(label: string, value: boolean): boolean { + order.push(label); + return value; +} +function emit(label: string): void { + order.push(label); +} + +for (const left of [false, true]) { + for (const flag of [false, true]) { + check("and", left) && (check("flag", flag) ? emit("and:yes") : emit("and:no")); + check("or", left) || (check("flag", flag) ? emit("or:yes") : emit("or:no")); + left || (flag && (check("inner", !flag) ? emit("nested:yes") : emit("nested:no"))); + left || (order.push("comma"), flag ? emit("comma:yes") : emit("comma:no")); + !left && (flag ? (left ? emit("deep:a") : emit("deep:b")) : emit("deep:c")); + } +} +console.log(order.join(" ")); +order.length = 0; + +// A for loop's initializer and update discard their results too. +let steps = 0; +for ( + check("init", false) || (check("init flag", true) ? emit("init:yes") : emit("init:no")); + steps < 2; + check("update", true) && (steps > 1 ? emit("update:last") : emit("update:next")) +) { + steps++; +} +console.log(steps, order.join(" ")); +order.length = 0; + +// The left operand narrows the conditional. +function describe(value: string | number): void { + typeof value === "string" && (value.length > 2 ? emit(value.toUpperCase()) : emit(value)); + typeof value === "number" || (value === "" ? emit("empty") : emit(`text:${value}`)); +} +describe("abc"); +describe("ab"); +describe(""); +describe(7); +console.log(order.join(" ")); +order.length = 0; + +// A generic callback: the conditional's type is T, void for the first two +// calls and number for the last. +function choose(skip: boolean, flag: boolean, a: () => T, b: () => T): void { + skip || (flag ? a() : b()); +} +choose(false, true, () => emit("generic:a"), () => emit("generic:b")); +choose(false, false, () => emit("generic:a"), () => emit("generic:b")); +choose(true, true, () => emit("generic:skipped"), () => emit("generic:skipped")); +choose(false, true, () => order.push("number:a"), () => order.push("number:b")); +console.log(order.join(" ")); +order.length = 0; + +// A throw from the chosen arm propagates. +try { + choose(false, false, () => emit("unused"), () => { + throw new Error("from b"); + }); +} catch (error) { + console.log((error as Error).message); +} diff --git a/tests/corpus/frozen-record-untyped-writes.ts b/tests/corpus/frozen-record-untyped-writes.ts new file mode 100644 index 000000000..02ed6b5c6 --- /dev/null +++ b/tests/corpus/frozen-record-untyped-writes.ts @@ -0,0 +1,35 @@ +// A frozen native record or array seen through an untyped reference keeps +// its frozen state: strict-mode writes throw Node's TypeErrors and leave +// the value unchanged, and Object.isFrozen answers true through any name. +export {}; + +function write(target: unknown, key: string, value: unknown): void { + try { + (target as Record)[key] = value; + console.log("wrote", key); + } catch (e) { + console.log("threw", (e as Error).message); + } +} + +const literal = Object.freeze({ a: 1, b: "x" }); +function make(n: number) { + return Object.freeze({ a: n, tags: ["t"] }); +} +const made = make(2); +write(literal, "a", 5); +write(literal, "z", 5); +write(made, "a", 7); +write(made, "tags", []); +made.tags.push("u"); +console.log(literal.a, made.a, JSON.stringify(literal), JSON.stringify(made)); +console.log(Object.isFrozen(literal), Object.isFrozen(made), Object.isFrozen(made.tags)); + +const list = Object.freeze([1, 2]); +write(list, "0", 9); +write(list, "5", 9); +console.log(list[0], list.length, Object.isFrozen(list)); + +const open = { a: 1 }; +write(open, "a", 3); +console.log(open.a); diff --git a/tests/corpus/frozen-untyped-writes-sloppy.cjs b/tests/corpus/frozen-untyped-writes-sloppy.cjs new file mode 100644 index 000000000..c80f185a7 --- /dev/null +++ b/tests/corpus/frozen-untyped-writes-sloppy.cjs @@ -0,0 +1,43 @@ +// Sloppy-mode code ignores writes that strict mode refuses: assignments to +// frozen objects and arrays and to non-extensible objects leave them +// unchanged without an error, while strict functions still throw. +function write(target, key, value) { + target[key] = value; + console.log("after", key, JSON.stringify(target)); +} +function strictWrite(target, key, value) { + "use strict"; + try { + target[key] = value; + console.log("wrote", key); + } catch (e) { + console.log("threw", e.message); + } +} +const frozen = Object.freeze({ a: 1, b: "x" }); +write(frozen, "a", 5); +write(frozen, "z", 5); +strictWrite(frozen, "a", 5); +const closed = Object.preventExtensions(JSON.parse('{"a":1}')); +write(closed, "a", 2); +write(closed, "b", 2); +strictWrite(closed, "b", 2); +const list = Object.freeze([1, 2]); +write(list, "0", 9); +strictWrite(list, "0", 9); +const holder = {}; +Object.defineProperty(holder, "fixed", { value: 1, writable: false, enumerable: true }); +write(holder, "fixed", 2); +strictWrite(holder, "fixed", 2); +const withSetter = { + /** @param {number} x */ + set v(x) { + throw new Error("setter " + x); + }, +}; +try { + write(withSetter, "v", 1); +} catch (e) { + console.log("setter threw", e.message); +} +console.log(frozen.a, closed.b, list[0], holder.fixed); diff --git a/tests/corpus/integer-clone-int32-slots.ts b/tests/corpus/integer-clone-int32-slots.ts new file mode 100644 index 000000000..5c34ada67 --- /dev/null +++ b/tests/corpus/integer-clone-int32-slots.ts @@ -0,0 +1,40 @@ +// A recursive function over a number array runs as an integer clone. +// sumFrom only receives arrays that hold int32s, which keep 4-byte int32 +// slots, and the clone reads them at that width, near their end too. +// sumWide receives an array that also holds a fraction, with 8-byte slots. +function sumFrom(xs: number[], i: number): number { + if (i >= xs.length) return 0; + return xs[i]! + sumFrom(xs, i + 1); +} + +function masked(n: number, start: number): number { + const a: number[] = []; + for (let i = 0; i < n; i++) a.push(i & 7); + return sumFrom(a, start); +} + +function halves(n: number, start: number): number { + const a: number[] = []; + for (let i = 0; i < n; i++) a.push(i < n / 2 ? 0 : 3); + return sumFrom(a, start); +} + +function sumWide(xs: number[], i: number): number { + if (i >= xs.length) return 0; + return xs[i]! + sumWide(xs, i + 1); +} + +function widened(n: number, start: number): number { + const a: number[] = []; + for (let i = 0; i < n; i++) a.push(i % 5); + const before = sumWide(a, start); + a.push(0.5); + return before * 10 + sumWide(a, start); +} + +const { n } = JSON.parse('{"n": 1000}') as { n: number }; +for (const start of [0, 1, n / 2, n - 9, n - 2, n - 1, n]) + console.log(start, masked(n | 0, start | 0), halves(n | 0, start | 0)); +for (const size of [63, 64, 65]) + console.log(size, halves(size, size - 24), halves(size, size - 1), masked(size, size - 3)); +console.log(widened(40, 0), widened(40, 37)); diff --git a/tests/corpus/native-worker-terminate-short-inner-loops.ts b/tests/corpus/native-worker-terminate-short-inner-loops.ts new file mode 100644 index 000000000..33a213ea8 --- /dev/null +++ b/tests/corpus/native-worker-terminate-short-inner-loops.ts @@ -0,0 +1,96 @@ +// worker.terminate() stops a worker that spends its time in short runs of +// inner loops: counted and paired loops of a few iterations inside loops +// without a condition, runs at the length where an inner loop starts to +// poll on its own, and a short inner loop whose body calls a slow function. +// Each terminate resolves promptly. +import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads"; + +function flips(n: number): number { + const p = new Int32Array(n), + q = new Int32Array(n), + count = new Int32Array(n); + for (let i = 0; i < n; i++) p[i] = i; + let total = 0; + for (let round = 0; ; round++) { + for (let i = 0; i < n; i++) q[i] = p[i]!; + let first = q[0]!; + for (;;) { + if (first === 0) break; + for (let i = 0, j = first; i < j; i++, j--) { + const t = q[i]!; + q[i] = q[j]!; + q[j] = t; + } + total++; + first = q[0]!; + } + // The next permutation: rotate the first k + 1 elements for the + // smallest k whose count has not run out. + for (let k = 1; ; k++) { + if (k === n) { + for (let i = 0; i < n; i++) p[i] = i; + count.fill(0); + break; + } + const head = p[0]!; + for (let i = 0; i < k; i++) p[i] = p[i + 1]!; + p[k] = head; + if (++count[k]! <= k) break; + count[k] = 0; + } + if (round === 2000) parentPort!.postMessage("ready"); + } + return total; +} + +function thresholds(values: number[]): number { + let acc = 0; + for (let round = 0; ; round++) { + for (const run of [63, 64, 65]) { + for (let i = 0; i < run; i++) acc = (acc + values[i]! * round) % 1000003; + for (let i = 0, j = run; i < j; i++, j--) acc = (acc + values[i]! + values[j]!) % 1000003; + } + if (round === 2000) parentPort!.postMessage("ready"); + } + return acc; +} + +function slow(seed: number): number { + let acc = seed; + for (let i = 0; i < 20000; i++) acc = (acc * 31 + i) % 1000003; + return acc; +} + +function slowBodies(): number { + let acc = 0; + for (let round = 0; ; round++) { + for (let i = 0; i < 4; i++) acc = (acc + slow(acc + i)) % 1000003; + if (round === 50) parentPort!.postMessage("ready"); + } + return acc; +} + +if (isMainThread) { + for (const kind of ["flips", "thresholds", "slow"]) { + await new Promise((resolve) => { + const worker = new Worker(new URL(import.meta.url), { workerData: kind }); + worker.once("message", () => { + const asked = Date.now(); + void worker.terminate().then((code: number) => { + console.log(kind, "terminate", code, Date.now() - asked < 1000 ? "prompt" : "late"); + }); + }); + worker.once("exit", (code: number) => { + console.log(kind, "exit", code); + resolve(); + }); + }); + } +} else { + let result = 0; + if (workerData === "flips") result = flips(9); + else if (workerData === "thresholds") + result = thresholds(Array.from({ length: 70 }, (_, i) => i)); + else result = slowBodies(); + console.log("unexpected result", result); +} diff --git a/tests/corpus/record-dynamic-writes.ts b/tests/corpus/record-dynamic-writes.ts new file mode 100644 index 000000000..58a47116b --- /dev/null +++ b/tests/corpus/record-dynamic-writes.ts @@ -0,0 +1,62 @@ +// Writes through an untyped view of a native record store into the record +// itself: typed reads see them, the view keeps answering from the record +// (no snapshot), and typed writes show through the view again. +type Point = { x: number; y: number; label: string; tag?: string }; +const p: Point = { x: 1, y: 2, label: "a" }; +const d = p as unknown as Record; +d.x = 10; +d.label = "b"; +console.log(p.x, p.label, JSON.stringify(d), Object.keys(d).join(",")); +p.y = 7; +console.log(d.y, JSON.stringify(p), "y" in d); +d.tag = "t"; +console.log(p.tag, JSON.stringify(d), Object.keys(p).join(",")); +d.tag = "u"; +console.log(p.tag, d.tag); +d.label = "c"; +console.log(p.label, JSON.stringify(Object.entries(d))); + +// Nested values written through the view are the record's own. +type Nested = { inner: { v: number }; list: number[]; flag: boolean }; +const n: Nested = { inner: { v: 1 }, list: [1], flag: false }; +const dn = n as unknown as { inner: unknown; list: unknown; flag: unknown }; +const fresh = { v: 2 }; +dn.inner = fresh; +fresh.v = 3; +console.log(n.inner.v, JSON.stringify(dn)); +dn.list = [4, 5]; +n.list.push(6); +console.log(n.list.length, n.list[1], JSON.stringify(dn.list)); +dn.flag = true; +console.log(n.flag, dn.flag); + +// A counter updated through the view in a loop. +const acc: { total: number; count: number } = { total: 0, count: 0 }; +const da = acc as unknown as { total: number; count: number }; +for (let i = 0; i < 100000; i++) { + da.total = da.total + i; + da.count += 1; +} +console.log(acc.total, acc.count); + +// Object.assign onto the view. +type Cfg = { host: string; port: number; debug: boolean; tags?: string[] }; +const c: Cfg = { host: "a", port: 1, debug: false, tags: ["x"] }; +const dc = c as unknown as object; +Object.assign(dc, { port: 2, debug: true }); +console.log(c.port, c.debug, JSON.stringify(dc)); +Object.assign(dc, { host: "z", extra: 1 }); +console.log(c.host, JSON.stringify(dc), Object.keys(dc).length); +c.port = 3; +console.log(JSON.stringify(dc)); + +// Writes through a function that takes the record as unknown. +function bump(value: unknown, key: string): void { + const object = value as Record; + object[key] = object[key]! + 1; +} +const counts = { a: 1, b: 2 }; +bump(counts, "a"); +bump(counts, "b"); +bump(counts, "b"); +console.log(counts.a, counts.b, JSON.stringify(counts)); diff --git a/tests/corpus/record-prefix-identity.ts b/tests/corpus/record-prefix-identity.ts new file mode 100644 index 000000000..b1747da10 --- /dev/null +++ b/tests/corpus/record-prefix-identity.ts @@ -0,0 +1,52 @@ +// A record value converted from an untyped value into a narrower record +// type whose fields begin the wider record's storage is the same object: +// a write through either name shows through the other, untyped views of +// the narrower name see every property, and identity holds. +type Wide = { a: number; b: string; c: boolean; d: number }; +type Narrow = { a: number; b: string }; +type Head = { a: number }; +function make(): unknown { + return { a: 1, b: "x", c: true, d: 4 } satisfies Wide; +} +const src = make(); +const w = src as Wide; +const n = src as Narrow; +n.a = 10; +console.log(w.a, (src as Head).a); +w.b = "y"; +console.log(n.b); +console.log(JSON.stringify(src), JSON.stringify(n)); +const n2 = src as Narrow; +console.log(n2 === n, n2.a); +(src as Record).a = 30; +console.log(w.a, n.a, (src as Head).a); +console.log(Object.keys(n).join(","), JSON.stringify({ n }), (n as unknown) === src); +console.log(n); + +// A native record boxed and narrowed in a function. +function narrow(value: unknown): Narrow { + return value as Narrow; +} +const wide: Wide = { a: 1, b: "p", c: false, d: 2 }; +const seen = narrow(wide); +seen.b = "q"; +wide.a = 5; +console.log(wide.b, seen.a, JSON.stringify(seen)); + +// Narrowing to a leading field of a record whose other fields are scalars. +type Sample = { a: number; b: number; c: boolean }; +const items: Sample[] = [ + { a: 1, b: 10, c: true }, + { a: 2, b: 20, c: false }, +]; +function asHead(value: unknown): Head { + return value as Head; +} +const heads = items.map((item) => asHead(item)); +for (const head of heads) head.a *= 10; +items[0]!.a += 1; +console.log(JSON.stringify(items), JSON.stringify(heads), heads[0]!.a); +const last = items[items.length - 1]!; +const second = asHead(last); +second.a = 7; +console.log(last.a, (second as unknown) === (last as unknown), Object.keys(second).join(",")); diff --git a/tests/corpus/typed-array-nested-loop-versions.ts b/tests/corpus/typed-array-nested-loop-versions.ts new file mode 100644 index 000000000..19e408d30 --- /dev/null +++ b/tests/corpus/typed-array-nested-loop-versions.ts @@ -0,0 +1,68 @@ +// Loops whose own typed-array accesses run once per pass around nested +// loops that read typed arrays: row stores past the end, reads at NaN, +// negative and fractional rows, rows that read a typed array the nested +// loop writes, and a nested loop that reads the row's own result. +function rowSums(u: Float64Array, out: Float64Array, start: number, end: number): void { + for (let i = start; i < end; i++) { + let t = out[i] ?? -1; + for (let j = 0; j < u.length; j++) t += u[j]! * (i + 1); + out[i] = t; + } +} + +function scaled(u: Float64Array, scale: Float64Array, out: Float64Array, rows: number): void { + for (let i = 0; i < rows; i++) { + const s = scale[i * 2] ?? 0.5; + for (let j = 0; j < u.length; j++) out[j] = out[j]! + u[j]! * s; + out[i] = out[i]! - 1; + } +} + +function chained(v: Int32Array, rows: number): number { + let total = 0; + for (let i = 0; i < rows; i++) { + let k = 0; + while (k < v.length) { + v[k] = v[k]! + (v[i] ?? 7); + k++; + } + total += v[i + 1] ?? -3; + } + return total; +} + +const show = (label: string, v: Float64Array | Int32Array): void => { + const parts: string[] = []; + for (let k = 0; k < v.length; k++) parts.push(Object.is(v[k], -0) ? "-0" : String(v[k])); + console.log(label, parts.join(" ")); +}; + +const u = new Float64Array([0.5, -1.25, 3, 7.75]); +for (const [start, end] of [ + [0, 4], + [0, 7], + [2, 9], + [-2, 3], + [0.5, 3.5], + [NaN, 3], + [3, 1], +]) { + const out = new Float64Array(5).fill(2); + rowSums(u, out, start!, end!); + show(`rows ${start} ${end}`, out); +} + +for (const rows of [0, 1, 3, 6]) { + const out = new Float64Array(4).fill(1); + scaled(u, new Float64Array([2, -1, 0.25, 4, 8]), out, rows); + show(`scaled ${rows}`, out); + const same = new Float64Array([1, 2, 3, 4]); + scaled(same, same, same, rows); + show(`aliased ${rows}`, same); +} + +for (const rows of [0, 2, 4, 9]) { + const v = new Int32Array([1, -2, 3, 2147483647]); + const total = chained(v, rows); + show(`chained ${rows} ${total}`, v); +} diff --git a/tests/corpus/untyped-nullish-scalar-defaults.mjs b/tests/corpus/untyped-nullish-scalar-defaults.mjs new file mode 100644 index 000000000..507945456 --- /dev/null +++ b/tests/corpus/untyped-nullish-scalar-defaults.mjs @@ -0,0 +1,50 @@ +// `??` in untyped JavaScript where the left is an optional number, boolean or +// string and the default has the same type: Map reads, find(), a function that +// may return undefined, and defaults of another type, null or an object. +const words = ["alpha", "beta", "gamma", "beta", "delta", "beta", "alpha"]; +const counts = new Map(); +for (const word of words) counts.set(word, (counts.get(word) ?? 0) + 1); +for (const [word, n] of counts) console.log(word, n); + +const odd = new Map(); +odd.set("nan", NaN); +odd.set("zero", -0); +odd.set("one", 1); +for (const key of ["nan", "zero", "one", "missing"]) { + const value = odd.get(key) ?? 7; + console.log(key, value, Object.is(value, -0), Number.isNaN(value), typeof value); +} + +const flags = new Map(); +flags.set("on", true); +flags.set("off", false); +for (const key of ["on", "off", "missing"]) console.log(key, flags.get(key) ?? true); + +const names = new Map(); +names.set("a", ""); +names.set("b", "bee"); +for (const key of ["a", "b", "c"]) console.log(JSON.stringify(names.get(key) ?? "none")); + +const values = [3, 8, 13, 21]; +console.log(values.find((v) => v > 10) ?? -1, values.find((v) => v > 100) ?? -1); + +function lookup(table, key) { + return key in table ? table[key] : undefined; +} +const table = { x: 1.5, y: 0 }; +console.log(lookup(table, "x") ?? 0, lookup(table, "y") ?? 9, lookup(table, "z") ?? 9); + +// Defaults of another type keep the left's value or take the default as is. +console.log(counts.get("beta") ?? "none", counts.get("omega") ?? "none"); +console.log(counts.get("omega") ?? null, counts.get("beta") ?? null); +const fallback = { n: 0 }; +console.log(counts.get("omega") ?? fallback, counts.get("alpha") ?? fallback); + +// The result is an ordinary value in later arithmetic and comparisons. +let total = 0; +for (const word of ["alpha", "beta", "omega"]) total += (counts.get(word) ?? 0) * 10; +console.log(total, (counts.get("omega") ?? 0) === 0, (odd.get("nan") ?? 0) === (odd.get("nan") ?? 0)); + +// A default of the same type that can itself be undefined keeps its value. +const spare = [4, 5]; +console.log(counts.get("omega") ?? spare[9], counts.get("beta") ?? spare[9], counts.get("omega") ?? spare[1]); diff --git a/tests/harness/corpus-costs/7393-non-null-element-operands.ts.json b/tests/harness/corpus-costs/7393-non-null-element-operands.ts.json new file mode 100644 index 000000000..e053c13c8 --- /dev/null +++ b/tests/harness/corpus-costs/7393-non-null-element-operands.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3800 } diff --git a/tests/harness/corpus-costs/7395-non-null-element-dereference.ts.json b/tests/harness/corpus-costs/7395-non-null-element-dereference.ts.json new file mode 100644 index 000000000..e0d5a6a6d --- /dev/null +++ b/tests/harness/corpus-costs/7395-non-null-element-dereference.ts.json @@ -0,0 +1 @@ +{ "plain": 3585, "san": 5768 } diff --git a/tests/harness/corpus-costs/8370-stack-overflow-cycles/main.ts.json b/tests/harness/corpus-costs/8370-stack-overflow-cycles/main.ts.json new file mode 100644 index 000000000..29769db15 --- /dev/null +++ b/tests/harness/corpus-costs/8370-stack-overflow-cycles/main.ts.json @@ -0,0 +1 @@ +{ "plain": 2780, "san": 53539 } diff --git a/tests/harness/corpus-costs/8405-zlib-compressed-bytes.ts.json b/tests/harness/corpus-costs/8405-zlib-compressed-bytes.ts.json new file mode 100644 index 000000000..9c291a72c --- /dev/null +++ b/tests/harness/corpus-costs/8405-zlib-compressed-bytes.ts.json @@ -0,0 +1 @@ +{ "plain": 5000, "san": 3200 } diff --git a/tests/harness/corpus-costs/8406-zlib-inflate-chunks.cjs.json b/tests/harness/corpus-costs/8406-zlib-inflate-chunks.cjs.json new file mode 100644 index 000000000..2202e2f02 --- /dev/null +++ b/tests/harness/corpus-costs/8406-zlib-inflate-chunks.cjs.json @@ -0,0 +1 @@ +{ "plain": 9000, "san": 13300 } diff --git a/tests/harness/corpus-costs/8407-string-index-of-from-ascii.ts.json b/tests/harness/corpus-costs/8407-string-index-of-from-ascii.ts.json new file mode 100644 index 000000000..054b3c2b4 --- /dev/null +++ b/tests/harness/corpus-costs/8407-string-index-of-from-ascii.ts.json @@ -0,0 +1 @@ +{ "plain": 3900, "san": 5000 } diff --git a/tests/harness/corpus-costs/arena-array-subclass.ts.json b/tests/harness/corpus-costs/arena-array-subclass.ts.json new file mode 100644 index 000000000..a4a33f447 --- /dev/null +++ b/tests/harness/corpus-costs/arena-array-subclass.ts.json @@ -0,0 +1 @@ +{ "plain": 3100 } diff --git a/tests/harness/corpus-costs/arena-workers.ts.json b/tests/harness/corpus-costs/arena-workers.ts.json new file mode 100644 index 000000000..2180ff94b --- /dev/null +++ b/tests/harness/corpus-costs/arena-workers.ts.json @@ -0,0 +1 @@ +{ "plain": 3600, "san": 18500 } diff --git a/tests/harness/corpus-costs/array-subclass-element-cycles.ts.json b/tests/harness/corpus-costs/array-subclass-element-cycles.ts.json new file mode 100644 index 000000000..371d0c7d1 --- /dev/null +++ b/tests/harness/corpus-costs/array-subclass-element-cycles.ts.json @@ -0,0 +1 @@ +{ "plain": 3200, "san": 4800 } diff --git a/tests/harness/corpus-costs/array-subclass-element-instanceof.ts.json b/tests/harness/corpus-costs/array-subclass-element-instanceof.ts.json new file mode 100644 index 000000000..d2c83f967 --- /dev/null +++ b/tests/harness/corpus-costs/array-subclass-element-instanceof.ts.json @@ -0,0 +1 @@ +{ "plain": 2500, "san": 3300 } diff --git a/tests/harness/corpus-costs/discarded-logical-void-conditionals.mjs.json b/tests/harness/corpus-costs/discarded-logical-void-conditionals.mjs.json new file mode 100644 index 000000000..3b70bc600 --- /dev/null +++ b/tests/harness/corpus-costs/discarded-logical-void-conditionals.mjs.json @@ -0,0 +1 @@ +{ "plain": 4136, "san": 4906 } diff --git a/tests/harness/corpus-costs/discarded-logical-void-conditionals.ts.json b/tests/harness/corpus-costs/discarded-logical-void-conditionals.ts.json new file mode 100644 index 000000000..85c20d025 --- /dev/null +++ b/tests/harness/corpus-costs/discarded-logical-void-conditionals.ts.json @@ -0,0 +1 @@ +{ "plain": 3559, "san": 5110 } diff --git a/tests/harness/corpus-costs/frozen-record-untyped-writes.ts.json b/tests/harness/corpus-costs/frozen-record-untyped-writes.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/frozen-record-untyped-writes.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/frozen-untyped-writes-sloppy.cjs.json b/tests/harness/corpus-costs/frozen-untyped-writes-sloppy.cjs.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/frozen-untyped-writes-sloppy.cjs.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/integer-clone-int32-slots.ts.json b/tests/harness/corpus-costs/integer-clone-int32-slots.ts.json new file mode 100644 index 000000000..91deb4042 --- /dev/null +++ b/tests/harness/corpus-costs/integer-clone-int32-slots.ts.json @@ -0,0 +1 @@ +{ "plain": 3474, "san": 38229 } diff --git a/tests/harness/corpus-costs/native-worker-terminate-short-inner-loops.ts.json b/tests/harness/corpus-costs/native-worker-terminate-short-inner-loops.ts.json new file mode 100644 index 000000000..c85984a8b --- /dev/null +++ b/tests/harness/corpus-costs/native-worker-terminate-short-inner-loops.ts.json @@ -0,0 +1 @@ +{ "plain": 4906, "san": 6969 } diff --git a/tests/harness/corpus-costs/record-dynamic-writes.ts.json b/tests/harness/corpus-costs/record-dynamic-writes.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/record-dynamic-writes.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/record-prefix-identity.ts.json b/tests/harness/corpus-costs/record-prefix-identity.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/record-prefix-identity.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/typed-array-nested-loop-versions.ts.json b/tests/harness/corpus-costs/typed-array-nested-loop-versions.ts.json new file mode 100644 index 000000000..1bee89d1a --- /dev/null +++ b/tests/harness/corpus-costs/typed-array-nested-loop-versions.ts.json @@ -0,0 +1 @@ +{ "plain": 6500, "san": 8800 } diff --git a/tests/harness/corpus-costs/untyped-nullish-scalar-defaults.mjs.json b/tests/harness/corpus-costs/untyped-nullish-scalar-defaults.mjs.json new file mode 100644 index 000000000..e822c3bda --- /dev/null +++ b/tests/harness/corpus-costs/untyped-nullish-scalar-defaults.mjs.json @@ -0,0 +1 @@ +{ "plain": 3711, "san": 5463 } diff --git a/tests/harness/ir-fields-generator.test.ts b/tests/harness/ir-fields-generator.test.ts new file mode 100644 index 000000000..f3cba57a1 --- /dev/null +++ b/tests/harness/ir-fields-generator.test.ts @@ -0,0 +1,60 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import ts from "typescript"; +import { expect, test } from "vitest"; +import { generateIrFields } from "../../scripts/generate-ir-fields.mjs"; + +const compilerDir = join(import.meta.dirname, "../../packages/compiler"); +const irPath = join(compilerDir, "src/ir/ir.ts"); +const generatedPath = join(compilerDir, "src/ir/fields.generated.ts"); +const irText = readFileSync(irPath, "utf8"); +const arrayGet = '{ kind: "arrayGet"; arr: IrExpr; index: IrExpr; type: IrType; loc: SrcLoc }'; + +/** ir.ts with `field` added to the arrayGet node. */ +function withArrayGetField(field: string): string { + expect(irText).toContain(arrayGet); + return irText.replace(arrayGet, arrayGet.replace("loc: SrcLoc }", `loc: SrcLoc; ${field} }`)); +} + +/** The type errors of `generated` as fields.generated.ts, with `ir` as ir.ts, + * under the compiler package's options. */ +function generatedErrors(ir: string, generated: string): string[] { + const parsed = ts.getParsedCommandLineOfConfigFile( + join(compilerDir, "tsconfig.json"), + { noEmit: true, incremental: false }, + { ...ts.sys, onUnRecoverableConfigFileDiagnostic: () => {} }, + )!; + const host = ts.createCompilerHost(parsed.options); + const readSourceFile = host.getSourceFile; + host.getSourceFile = (file, languageVersion, ...rest) => + file === irPath + ? ts.createSourceFile(file, ir, languageVersion) + : file === generatedPath + ? ts.createSourceFile(file, generated, languageVersion) + : readSourceFile.call(host, file, languageVersion, ...rest); + const program = ts.createProgram([generatedPath], parsed.options, host); + return ts + .getPreEmitDiagnostics(program, program.getSourceFile(generatedPath)) + .map((d) => ts.flattenDiagnosticMessageText(d.messageText, "\n")); +} + +test("a nested record of scalar fields passes the checks once the readers are regenerated", () => { + const ir = withArrayGetField("counts?: { hits: number; misses: number | undefined }"); + expect(generatedErrors(ir, generateIrFields(ir))).toEqual([]); +}); + +test("the record's own fields are checked: an added field or a widened scalar fails with its name", () => { + const generated = generateIrFields( + withArrayGetField("counts?: { hits: number; misses: number | undefined }"), + ); + const added = generatedErrors( + withArrayGetField("counts?: { hits: number; misses: number | undefined; label: string }"), + generated, + ); + expect(added).toEqual([`Type '"label"' does not satisfy the constraint 'never'.`]); + const widened = generatedErrors( + withArrayGetField("counts?: { hits: number | string; misses: number | undefined }"), + generated, + ); + expect(widened).toEqual([`Type 'string' does not satisfy the constraint 'never'.`]); +}); diff --git a/tests/harness/runtime-tree-shaking.test.ts b/tests/harness/runtime-tree-shaking.test.ts index b33a0d2f9..92584a59e 100644 --- a/tests/harness/runtime-tree-shaking.test.ts +++ b/tests/harness/runtime-tree-shaking.test.ts @@ -142,6 +142,9 @@ nativeToolchainTest( "scr_url_", "scr_json_parse", "scr_date_", + // The generator runtime installs its tag reader with its first + // generator; a load-time constructor would retain it everywhere. + "scr_gen_tag", ]) { expect(helloSymbols, `hello retains ${family}`).not.toContain(family); } @@ -159,8 +162,10 @@ nativeToolchainTest( // Symbol absence is the primary reachability contract. Keep a deliberately // roomy, platform-specific hello-world ceiling too: it catches losing - // section GC without pinning an exact linker/SDK byte count. The canonical - // Linux build is about 84KB and current Mach-O builds are about 70KB. Most + // section GC without pinning an exact linker/SDK byte count. The Linux + // build is about 65KB with lld (CI's pinned LLVM provides it) and about + // 94KB with GNU ld, which keeps a unit's string literals whole once any of + // them is reachable; current Mach-O builds are about 70KB. Most // of Linux hello's runtime code is the cycle collector, which its exit // path reaches (scr_lib_cleanup releases an array). ELF segments are page // aligned, so the Linux size moves in 4 KiB steps whenever .text crosses diff --git a/tests/harness/sandbox-config.test.ts b/tests/harness/sandbox-config.test.ts index cedaef600..532fb5c4e 100644 --- a/tests/harness/sandbox-config.test.ts +++ b/tests/harness/sandbox-config.test.ts @@ -5,6 +5,7 @@ import { sandboxBootstrapCommand, sandboxEffectWorkerAllocation, sandboxImageConfig, + sandboxNativeReference, sandboxRunnerConfig, sandboxTestSourceConfig, sandboxTestWorkerAllocation, @@ -97,6 +98,10 @@ test("the managed-image bootstrap pins the custom image's Node, pnpm, and LLVM", expect(bootstrap).toContain("libclang-rt-22-dev"); expect(bootstrap).toContain("llvm-22-dev"); expect(bootstrap).toContain("ninja-build"); + // CI links with ld.lld; without it Sandbox binaries link with GNU ld. + expect(bootstrap).toContain("lld-22"); + const dockerfile = readFileSync(new URL("../../Dockerfile.sandbox", import.meta.url), "utf8"); + expect(dockerfile).toContain("lld-22"); }); test("a custom image does not select the Sandbox team or project", () => { @@ -154,6 +159,24 @@ test("ambiguous or malformed snapshot sources fail before creating workers", () ); }); +test("a native reference checkout needs a prepared source and an absolute path", () => { + const snapshot = { SCRIPTC_SANDBOX_SNAPSHOT: "snap_ABC123" }; + expect(sandboxNativeReference(snapshot)).toBeUndefined(); + expect( + sandboxNativeReference({ + ...snapshot, + SCRIPTC_SANDBOX_NATIVE_REFERENCE: " /vercel/sandbox/prev/ ", + }), + ).toBe("/vercel/sandbox/prev"); + expect(() => + sandboxNativeReference({ SCRIPTC_SANDBOX_NATIVE_REFERENCE: "/vercel/sandbox/prev" }), + ).toThrow("needs SCRIPTC_SANDBOX_SNAPSHOT"); + for (const path of ["prev", "/a/../b", "/a b", "/a;rm"]) + expect(() => + sandboxNativeReference({ ...snapshot, SCRIPTC_SANDBOX_NATIVE_REFERENCE: path }), + ).toThrow("absolute path"); +}); + test("OIDC authentication wins and supplies its own Sandbox scope", () => { const config = sandboxVercelConfig({ VERCEL_OIDC_TOKEN: "header.payload.signature", diff --git a/tests/harness/self-hosting-emission.test.ts b/tests/harness/self-hosting-emission.test.ts index 3977ede29..e6364b855 100644 --- a/tests/harness/self-hosting-emission.test.ts +++ b/tests/harness/self-hosting-emission.test.ts @@ -59,7 +59,9 @@ test("the production layout and metadata stage lowers statically", async () => { }); for (const backend of ["llvm"] as const) { - test(`native LLVM layout and metadata emission (${backend})`, async () => { + // The sanitized dev build of the stage takes 170 to 300 s of CI time beside + // its shard's other tests. + test(`native LLVM layout and metadata emission (${backend})`, { timeout: 600_000 }, async () => { const dir = mkdtempSync( join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-emission-"), ); diff --git a/tests/harness/wasi-differential.test.ts b/tests/harness/wasi-differential.test.ts index 66c2934f2..68d4b1807 100644 --- a/tests/harness/wasi-differential.test.ts +++ b/tests/harness/wasi-differential.test.ts @@ -316,6 +316,8 @@ describe.skipIf(!zigOnPath())("wasm32-wasi differential", () => { // Weak-collection hashes mix 32-bit pointers in 64 bits. "4591-weak-keys-union-records.ts", "7742-weak-collections-many-keys.ts", + // zlib's portable build (CPU_NO_SIMD) compresses to Node's bytes. + "8405-zlib-compressed-bytes.ts", ])("the LLVM target runs %s byte-identically to Node", async (fixture) => { const entry = join(repoRoot, "tests/corpus", fixture); const outDir = await mkdtemp("/tmp/scriptc-wasi-"); diff --git a/tests/harness/zero-cost-gate.test.ts b/tests/harness/zero-cost-gate.test.ts index 565f8ba71..e99eca8d1 100644 --- a/tests/harness/zero-cost-gate.test.ts +++ b/tests/harness/zero-cost-gate.test.ts @@ -1,14 +1,20 @@ import { spawnSync } from "node:child_process"; import { expect, test } from "vitest"; import { + cachedComparison, cachegrindCounts, compareArtifact, compareCounts, compareExecutables, definedFunctionNames, dynamicTierSymbols, + mergeReports, nmOutput, + programInputsHash, + programLines, programVerdict, + recordedBaseline, + timingVerdict, } from "../../scripts/zero-cost-compare.mjs"; const ir = (functions: object[], globals: object[]) => @@ -191,3 +197,105 @@ test("the zero-cost gate fails differing programs unless they match the signatur expect(verdict({}, false, failed, failed, true)).toEqual({ verdict: "both-fail", fails: true }); expect(verdict({}, true, ok, failed)).toEqual({ verdict: "build-error", fails: true }); }); + +test("a recorded baseline settles identical programs and shared build errors", () => { + const hashes = { ir: "a", llvm: "b", exe: "c" }; + const entry = { ok: true, ms: 12, hashes, exeBytes: 100 }; + expect(cachedComparison(entry, { ok: true }, { ...hashes })).toEqual({ + baseline: { ok: true, ms: 12 }, + artifacts: { + ir: { identical: true }, + llvm: { identical: true }, + exe: { identical: true, bytes: { baseline: 100, candidate: 100 } }, + }, + }); + // Any other hash, a missing artifact or no record builds the baseline. + expect(cachedComparison(entry, { ok: true }, { ...hashes, exe: "d" })).toBeNull(); + expect( + cachedComparison({ ...entry, hashes: { ...hashes, ir: null } }, { ok: true }, hashes), + ).toBeNull(); + expect(cachedComparison(undefined, { ok: true }, hashes)).toBeNull(); + expect(cachedComparison(entry, { ok: false, error: "SC1090" }, {})).toBeNull(); + const failed = { ok: false, ms: 3, error: "SC1090" }; + expect(cachedComparison(failed, { ok: false, error: "SC1090" }, {})).toEqual({ + baseline: { ok: false, error: "SC1090", ms: 3, cached: true }, + artifacts: {}, + }); + expect(cachedComparison(failed, { ok: false, error: "SC1091" }, {})).toBeNull(); + expect(cachedComparison(failed, { ok: true }, hashes)).toBeNull(); +}); + +test("a recorded baseline applies only to the program inputs it was recorded for", () => { + const files: Record = { + "/p/tests/corpus/a/main.ts": 'import { f } from "./lib.ts";', + "/p/tests/corpus/a/lib.ts": "export const f = 1;", + }; + const read = (file: string) => Buffer.from(files[file]!); + const paths = Object.keys(files); + const inputs = programInputsHash("/p", paths, "release", read); + expect(programInputsHash("/p", [...paths].reverse(), "release", read)).toBe(inputs); + expect(programInputsHash("/p", paths, "speed", read)).not.toBe(inputs); + expect(programInputsHash("/q", paths, "release", read)).not.toBe(inputs); + // An edit to an import changes the hash as an edit to the entry does. + files["/p/tests/corpus/a/lib.ts"] = "export const f = 2;"; + const edited = programInputsHash("/p", paths, "release", read); + expect(edited).not.toBe(inputs); + const entry = { inputs, ok: false, ms: 3, error: "SC1090" }; + const cache = { programs: { "corpus/a/main.ts": entry } }; + expect(recordedBaseline(cache, "corpus/a/main.ts", inputs)).toBe(entry); + // A record of the old sources must not settle the new sources as both-fail or identical. + expect(recordedBaseline(cache, "corpus/a/main.ts", edited)).toBeUndefined(); + expect(recordedBaseline(cache, "corpus/b/main.ts", inputs)).toBeUndefined(); + expect(recordedBaseline(null, "corpus/a/main.ts", inputs)).toBeUndefined(); +}); + +test("merged gate shards judge compile time and comparisons over every shard", () => { + const trees = { baseline: { revision: "a" }, candidate: { revision: "b" } }; + const row = (id: string, ratio: number) => ({ + id, + baselineMs: [100, 100, 100], + candidateMs: [100 * ratio, 100 * ratio, 100 * ratio], + ratios: [ratio, ratio, ratio], + }); + const shard = ( + programs: object[], + rows: object[], + failures: string[] = [], + globals: string[] = [], + ) => ({ + schema: 1, + trees, + programs, + failures: [...failures, ...globals], + globalFailures: globals, + timing: { runs: 3, maxGrowth: 0.02, rows, groups: {} }, + elapsedMs: 10, + }); + const identical = (id: string) => ({ id, verdict: "identical", artifacts: {} }); + const fast = shard([identical("bench/a")], [row("bench/a", 1.0)]); + // Alone, this shard's bench is slower; with the other shard it is not. + const slow = shard( + [ + { + id: "corpus/x.ts", + verdict: "differs", + artifacts: { exe: { identical: false, first: "resized f" } }, + }, + ], + [row("bench/b", 1.03)], + ["corpus/x.ts: differs"], + ["bench compile time grew +3.00% (CI low +3.00%)"], + ); + const { report, timingLines } = mergeReports([fast, slow]); + expect(report.counts).toEqual({ identical: 1, differs: 1 }); + expect(report.failures).toEqual(["corpus/x.ts: differs"]); + expect(report.verdict).toBe("fail"); + expect(timingLines).toEqual([timingVerdict("bench", report.timing.rows, 0.02).line]); + expect(timingVerdict("bench", report.timing.rows, 0.02).fails).toBe(false); + expect(programLines(report.programs[1])).toEqual(["DIFFERS corpus/x.ts exe: resized f"]); + expect(mergeReports([fast]).report.verdict).toBe("pass"); + expect(mergeReports([shard([], [])]).report.failures).toEqual([ + "no program built with both checkouts, so nothing was compared", + ]); + expect(() => mergeReports([fast, fast])).toThrow(/more than one gate report/); +});