trending/scriptc
1
0
Fork 0
mirror of https://github.com/vercel-labs/scriptc.git synced 2026-10-29 07:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge main

Browse Source
This commit is contained in:
Chris Tate
2026-10-27 16:45:14 -05:00
parent f4a4cf8652 3c7f47e6aa
commit 1a86f94239
213 changed files with 12641 additions and 950 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/release.yml
+3 -2
View File
@@ -111,7 +111,8 @@ jobs:
needs: check-release
if: needs.check-release.outputs.should_release == 'true' && needs.check-release.outputs.reuse_artifacts != 'true'
runs-on: ubuntu-24.04
timeout-minutes: 75
# The layout and the profile's two phases each build an instrumented compiler.
timeout-minutes: 120
continue-on-error: true
permissions:
contents: read
@@ -148,7 +149,7 @@ jobs:
env:
SCRIPTC_NATIVE_LAYOUT: ${{ runner.temp }}/cli-profile/cli.layout.gz
run: >-
node scripts/refresh-cli-profile.mjs --tsc-ts "$RUNNER_TEMP/tsc-ts" --jobs 4
node scripts/refresh-cli-profile.mjs --tsc-ts "$RUNNER_TEMP/tsc-ts" --jobs 4 --context-sensitive
--output "$RUNNER_TEMP/cli-profile/cli.profdata.gz" --work "$RUNNER_TEMP/cli-profile-work"
- uses: actions/upload-artifact@v4
with:
docs/content/docs/cli.mdx
+27 -3
View File
@@ -151,6 +151,9 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
<dt><code>--strip</code></dt>
<dd>Remove symbol and debug payload when linking an executable, including cross-compiled targets. This flag preserves code optimization.</dd>
<dt><code>--static</code></dt>
<dd>On Linux, link the C library and the other system libraries into the executable. The executable starts without the dynamic loader, which shortens its startup, and doesn't depend on the run host's glibc version. glibc executables link as static PIE, are about 1 MB larger, and need glibc's static libraries on the build machine (<code>libc6-dev</code> on Debian and Ubuntu, <code>glibc-static</code> on Fedora); they link only for the build machine's own glibc target. A program whose executable includes host, user, or group name lookups, which <code>dns.lookup</code>, <code>net</code> and <code>http</code> connections, <code>fetch</code>, and <code>os.userInfo()</code> use, is refused with <code>SC3002</code>: a static glibc executable loads the NSS modules for those lookups from the run host, which glibc supports only for the glibc version it was linked with. Build such a program without <code>--static</code>, or for an <code>&lt;arch&gt;-linux-musl</code> target. musl executables are always linked statically, and accept the flag. Valid only for executable builds.</dd>
<dt><code>--optimization &lt;release|dev|speed&gt;</code></dt>
<dd>Select native optimization for executable, assembly, or object output. The default is <code>release</code> (<code>-O2</code>), which emits string retains and the retains and releases of dynamic values inline. <code>dev</code> uses <code>-O0</code> and emits source locations for TypeScript and JavaScript breakpoints and stack frames. LLVM builds include source variable names and native storage descriptions; see <a href="/docs/limitations#native-debugging">Native debugging</a> for the supported representations. macOS executable builds also produce an adjacent <code>.dSYM</code> bundle. <code>speed</code> is <code>-O2</code> plus optimizations that trade binary size and build time for run-time speed: it emits the remaining reference counting inline and, on x86-64, starts every runtime function on a cache-line boundary. Expect larger executables and longer cold builds than <code>release</code>; how much faster a program runs depends on how much of its time is spent in the runtime. Where a runtime pack has no speed runtime, <code>speed</code> links the release runtime and keeps inline reference counting. Selecting <code>speed</code> never changes what <code>release</code> builds produce, and the two never share cached artifacts. Library profiles accept <code>"optimization": "speed"</code> for inline reference counting. In executables large enough to be divided into partitions, <code>dev</code> compiles each partition on its own, and <code>release</code> and <code>speed</code> add a step that lets partitions import small functions from each other; see <a href="/docs/how-it-works#build-parallelism">Build parallelism</a>.</dd>
@@ -170,10 +173,10 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
<dd>On x86-64 targets, compile the program's code twice, for the baseline and for <code>x86-64-v3</code>, and choose the copy when the program starts. The executable runs on any x86-64 CPU and uses the <code>x86-64-v3</code> code on CPUs that support it. Not used with a <code>--target-cpu</code> level above <code>baseline</code>. Valid only for executable builds. See <a href="#runtime-cpu-dispatch">Runtime CPU dispatch</a>.</dd>
<dt><code>--profile-generate &lt;pattern&gt;</code></dt>
<dd>On Linux, build an instrumented executable for profile-guided optimization. Each run writes a raw profile to the path pattern, where <code>%p</code> expands to the process ID; the <code>LLVM_PROFILE_FILE</code> environment variable overrides the pattern at run time. A relative pattern resolves from the current working directory. Instrumented programs run slower. Valid only for executable builds with <code>--optimization=release</code> or <code>speed</code>.</dd>
<dd>On Linux, build an instrumented executable for profile-guided optimization. Each run writes a raw profile to the path pattern, where <code>%p</code> expands to the process ID; the <code>LLVM_PROFILE_FILE</code> environment variable overrides the pattern at run time. A relative pattern resolves from the current working directory. Instrumented programs run slower. With <code>--profile-use</code>, the build is optimized by that profile and instrumented after inlining, the second phase of a <a href="#context-sensitive-profiles">context-sensitive profile</a>. Valid only for executable builds with <code>--optimization=release</code> or <code>speed</code>.</dd>
<dt><code>--profile-use &lt;file.profdata&gt;</code></dt>
<dd>Optimize the executable with a profile from <code>scriptc merge-profiles</code>: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain. A new or changed profile compiles every partition of the program again. <code>scriptc build</code> prints how the profile matched the program's functions on stderr; see <a href="#profile-guided-optimization">Profile-guided optimization</a>. Valid only for executable builds with <code>--optimization=release</code> or <code>speed</code>.</dd>
<dd>Optimize the executable with a profile from <code>scriptc merge-profiles</code>: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain. A new or changed profile compiles every partition of the program again. <code>scriptc build</code> prints how the profile matched the program's functions on stderr; see <a href="#profile-guided-optimization">Profile-guided optimization</a>. A profile with <a href="#context-sensitive-profiles">context-sensitive counts</a> also orders each function's blocks by them on Linux. Valid only for executable builds with <code>--optimization=release</code> or <code>speed</code>.</dd>
<dt><code>--npm-static &lt;pkg[,pkg…]|auto&gt;</code></dt>
<dd>Name npm packages to compile even if they don't ship their own type declarations (repeatable). A trailing <code>...</code>, as in <code>fs-extra...</code>, also names every package that package depends on. <code>auto</code> uses the default automatic selection. Packages that can't compile produce diagnostics. See <a href="/docs/dependencies#explicit-package-selection">npm Dependencies</a>.</dd>
@@ -243,7 +246,7 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
<dd>The TypeScript 7 API server that checks the program: <code>tsc-ts</code> (the default), which scriptc includes, or <code>tsgo</code>, from the <code>typescript</code> package scriptc also includes. Both give the same diagnostics and generated code.</dd>
<dt><code>SCRIPTC_CHECKER_SERVER</code></dt>
<dd>Set to <code>1</code> on Linux and macOS to check with a tsc-ts server that later builds share. It takes effect with a tsc-ts release that serves <code>tsc-ts --api --listen</code>. The tsc-ts 0.0.9 that this version of scriptc includes does not, so each build checks with its own tsc-ts process, as without the variable. A build connects to the server for its working directory, starting it when none answers, and the server keeps the files it parsed, so the next build of the same program loads it faster. A build that finds the server in use by another build, or cannot start or reach it, checks with its own tsc-ts process. The server exits after 5 minutes without a build, or after a build when its resident memory is above 2048 MB (<code>TSC_TS_LISTEN_MEMORY</code> sets another limit in megabytes). Diagnostics and generated code are the same with and without it. It applies to the default checker, tsc-ts.</dd>
<dd>On Linux and macOS, builds check with a tsc-ts server that later builds share. The first build in a working directory starts it: a <code>tsc-ts</code> process, started with <code>--api</code> and <code>--listen</code>, that keeps running in the background with no terminal and keeps the files it parsed, so the next build in that directory loads its program faster. Each user, working directory and scriptc installation gets its own server, and a build connects only to the server of its own scriptc and tsc-ts. The socket is in <code>$XDG_RUNTIME_DIR/scriptc</code>, or <code>/tmp/scriptc-&lt;uid&gt;</code>, a directory only that user can access. A build that finds the server in use by another build, cannot start or reach it, or loses its connection to it, checks with its own tsc-ts process. The server exits after 10 minutes without a build, or after a build when its resident memory is above 2048 MB (<code>TSC_TS_LISTEN_MEMORY</code> sets another limit in megabytes). To stop it sooner, end that process, for example with <code>pkill -f 'tsc-ts --api .*--listen'</code>. Set to <code>0</code> to check with a tsc-ts process of each build's own. When <code>CI</code> is set to a value other than <code>0</code> or <code>false</code>, builds use their own process unless this variable is <code>1</code>. Diagnostics and generated code are the same with and without the server. It applies to the default checker, tsc-ts.</dd>
</dl>
```console
@@ -284,6 +287,23 @@ scriptc: warning: 3739 of the 8947 functions app.profdata describes (41.8%) chan
The build still succeeds. Collecting the profile again with the current program and scriptc version restores the counts.
### Context-sensitive profiles
A profile from the steps above counts each function's branches before the optimizer inlines calls. A second phase counts them again after inlining, separately for every place a function was inlined into:
```console
$ scriptc build app.ts --optimization=speed --profile-use app.profdata --profile-generate profiles-cs/%p.profraw -o app-instrumented-cs
$ ./app-instrumented-cs typical-input.json
$ scriptc merge-profiles -o app-cs.profdata app.profdata profiles-cs/*.profraw
$ scriptc build app.ts --optimization=speed --profile-use app-cs.profdata
```
- **First phase:** `--profile-generate` alone. It instruments the program before inlining and gives the profile of the steps above.
- **Second phase:** `--profile-use` with the first profile and `--profile-generate` together. The build is optimized by the first profile and instrumented after inlining, and `scriptc merge-profiles` adds its counts to the first profile.
- **Optimized build:** `--profile-use` with the merged profile. On Linux, it also orders each function's blocks by the counts and moves blocks the runs never reached into separate sections, so the code that runs sits together.
A profile that already holds context-sensitive counts cannot start a second phase. Merge the second phase's raw profiles into the first-phase profile instead. Both phases run the same workloads.
## Target CPU
By default, x86-64 executables run on any x86-64 CPU, so the compiler uses only the baseline instruction set (SSE2). `--target-cpu` compiles the program's code for a higher x86-64 microarchitecture level. Each level includes the ones below it:
@@ -373,6 +393,10 @@ For source breakpoints in Xcode or LLDB, build with <code>--optimization=dev</co
<td><code>scriptc run</code> on <code>wasm32-wasi</code></td>
<td>Node.js 24 or later on <code>PATH</code></td>
</tr>
<tr>
<td><code>--static</code> on glibc Linux</td>
<td>glibc's static libraries (<code>libc6-dev</code> on Debian and Ubuntu, <code>glibc-static</code> on Fedora)</td>
</tr>
<tr>
<td><code>--sanitize</code></td>
<td>A C compiler and LLVM toolchain</td>
docs/content/docs/platforms.mdx
+1 -1
View File
@@ -49,7 +49,7 @@ $ file fib-linux
fib-linux: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 2.0.0, with debug_info, not stripped
```
`x86_64-linux-gnu.2.36` works the same way. For Alpine and other musl-based systems, use `aarch64-linux-musl` or `x86_64-linux-musl`; these produce statically linked executables.
`x86_64-linux-gnu.2.36` works the same way. For Alpine and other musl-based systems, use `aarch64-linux-musl` or `x86_64-linux-musl`; these produce statically linked executables. On a glibc-based Linux machine, [`--static`](/docs/cli#options) links a static executable for the machine's own target.
Linux executables use the native Linux implementations of the event loop (epoll), servers, TLS (with the system's CA bundle), and `fs.watch`, and they are tested against Node.js on Linux, including Alpine.
docs/content/limitations/builtin-apis/streams-and-readline.mdx
+5 -1
View File
@@ -60,4 +60,8 @@
The `readline/promises` `Readline` class writes to the same outputs.
`readline.emitKeypressEvents` emits `'keypress'` events on `process.stdin` or a readable stream, shared with terminal interfaces on that stream as in Node; a listener's first parameter must admit `undefined`, which it receives for escape sequences.
Chained listener registration is unsupported.
`node:stream/web` exports the same `ReadableStream` class as the global; its other classes are unsupported.
`node:stream/web` exports the same `ReadableStream` class as the global; in TypeScript, its other classes are unsupported.
In JavaScript, `ReadableStream`, `ReadableStreamDefaultReader`, `ReadableStreamDefaultController`, `CountQueuingStrategy`, and `ByteLengthQueuingStrategy`, as globals and as `node:stream/web` exports, are constructor values with Node's prototypes, brand checks, and errors.
Their underlying sources (`start`, `pull`, `cancel`), queuing strategies with a `size` function, readers, `tee`, `ReadableStream.from`, and async iteration settle promises in Node's order.
A JavaScript `ReadableStream` given as a `Response` or `Request` body, or as `fetch`'s body, is read through a stream of its own: `response.body` is that stream rather than the one passed in, and the stream passed in is locked from the start.
In JavaScript, a byte stream (`type: 'bytes'`), `pipeTo`, `pipeThrough`, `Object.getPrototypeOf` on these streams, readers, controllers, and strategies, their use as `WeakMap` and `WeakSet` keys, `ReadableStream.from` of a function that is not iterable, and the other `node:stream/web` classes are unsupported.
internal/compatibility/generated/node-v24-internal.json
+2
View File
@@ -111928,6 +111928,7 @@
"tests/corpus/7743-transform-stream-identity.ts",
"tests/corpus/stream-web-readable.ts",
"tests/corpus/stream-web-require.cjs",
"tests/corpus/web-stream-readable-untyped.mjs",
"tests/harness/fetch-conformance.test.ts"
]
},
@@ -112200,6 +112201,7 @@
"evidence": "compiler-feature:webstreams.ReadableStream",
"tests": [
"tests/corpus/stream-web-readable.ts",
"tests/corpus/web-stream-readable-untyped.mjs",
"tests/corpus/stream-web-require.cjs",
"tests/corpus/2862-web-readable-async-iteration.ts"
]
internal/compatibility/static-support.json
+1 -1
View File
@@ -433,7 +433,7 @@
{ "chapter": "zlib", "signatures": ["`zlib.constants`"], "status": "partial", "evidence": ["tests/corpus/5376-zlib-constants.cjs", "tests/corpus/5374-zlib-stream-flush-params.ts"] },
{ "chapter": "dns", "symbols": ["dns.lookup", "dns.lookupService", "dns.getServers", "dns.getDefaultResultOrder", "dns.setDefaultResultOrder"], "status": "partial", "evidence": ["tests/corpus/5648-dns-lookup-forms.ts", "tests/corpus/5649-dns-service-servers.ts", "tests/corpus/5650-dns-lookup-errors-untyped.cjs"] },
{ "chapter": "dns", "symbols": ["dnsPromises.lookup", "dnsPromises.lookupService", "dnsPromises.getDefaultResultOrder"], "status": "partial", "evidence": ["tests/corpus/dns-promises-lookup.ts", "tests/corpus/dns-promises-untyped.cjs"] },
{ "chapter": "webstreams", "symbols": ["ReadableStream"], "status": "partial", "evidence": ["tests/corpus/stream-web-readable.ts", "tests/corpus/stream-web-require.cjs", "tests/corpus/2862-web-readable-async-iteration.ts"] },
{ "chapter": "webstreams", "symbols": ["ReadableStream"], "status": "partial", "evidence": ["tests/corpus/stream-web-readable.ts", "tests/corpus/web-stream-readable-untyped.mjs", "tests/corpus/stream-web-require.cjs", "tests/corpus/2862-web-readable-async-iteration.ts"] },
{ "chapter": "webstreams", "symbols": ["TransformStream"], "status": "partial", "evidence": ["tests/corpus/7743-transform-stream-identity.ts"] },
{ "chapter": "globals", "signatures": ["Class: `TransformStream`"], "status": "partial", "evidence": ["tests/corpus/7743-transform-stream-identity.ts"] }
],
native/llvm-codegen/src/emit.cpp
+54 -10
View File
@@ -111,9 +111,6 @@ std::optional<EmitOptions> parseEmitOptions(int Argc, char **Argv) {
}
if (Options.Input.empty() || Options.Outputs.empty())
return std::nullopt;
// A build either collects a profile or uses one.
if (!Options.ProfileGenerate.empty() && !Options.ProfileUse.empty())
return std::nullopt;
return Options;
}
@@ -150,20 +147,48 @@ PipelineTuningOptions tuningFor(OptimizationLevel Level) {
return Tuning;
}
// The profile-guided optimization a build asks for: IR instrumentation that
// writes raw profiles to the ProfileGenerate pattern, or annotation from the
// indexed ProfileUse profile. Functions the profile does not describe, or
// whose control flow changed since it was collected, are optimized as without
// one, so a stale profile loses only its gain.
// Whether the ProfileUse profile also holds context-sensitive counts, which
// a second instrumentation after inlining recorded (inspectProfile).
bool ContextProfile = false;
// The profile-guided optimization a build asks for:
// - IR instrumentation that writes raw profiles to the ProfileGenerate
// pattern;
// - annotation from the indexed ProfileUse profile, and from its
// context-sensitive counts after inlining when it has them;
// - with both, annotation from ProfileUse and a second instrumentation after
// inlining that writes context-sensitive counts to the ProfileGenerate
// pattern. `scriptc merge-profiles` adds them to the first profile.
// Functions the profile does not describe, or whose control flow changed
// since it was collected, are optimized as without one, so a stale profile
// loses only its gain.
std::optional<PGOOptions> profileOptions(const EmitOptions &Options) {
if (!Options.ProfileGenerate.empty() && !Options.ProfileUse.empty())
return PGOOptions(Options.ProfileUse, Options.ProfileGenerate, "", "",
PGOOptions::IRUse, PGOOptions::CSIRInstr);
if (!Options.ProfileGenerate.empty())
return PGOOptions(Options.ProfileGenerate, "", "", "",
PGOOptions::IRInstr);
if (!Options.ProfileUse.empty())
return PGOOptions(Options.ProfileUse, "", "", "", PGOOptions::IRUse);
return PGOOptions(Options.ProfileUse, "", "", "", PGOOptions::IRUse,
ContextProfile ? PGOOptions::CSIRUse
: PGOOptions::NoCSAction);
return std::nullopt;
}
// Block placement by the extended TSP model, which needs the exact block
// counts a context-sensitive profile gives. The option is process-wide; the
// helper sets it once.
void enableExtTspPlacement() {
static const bool Enabled = [] {
if (cl::Option *Option = cl::getRegisteredOptions().lookup(
"enable-ext-tsp-block-placement"))
Option->addOccurrence(0, "enable-ext-tsp-block-placement", "true");
return true;
}();
(void)Enabled;
}
// With a profile, the thin link imports the callees of hot call edges up to
// 10 times, and of the hottest edges up to 100 times, the usual size limit,
// into every partition that calls them. On large programs that triples the
@@ -318,7 +343,8 @@ void reportProfileMatch(const ProfileMatch &Match) {
}
// Reads the profile's header once, before any partition needs it: it must
// hold IR-level counts.
// hold IR-level counts. Context-sensitive counts lay out the code by them;
// a profile that has them cannot take a second set.
std::optional<EmitFailure> inspectProfile(const EmitOptions &Options) {
if (Options.ProfileUse.empty())
return std::nullopt;
@@ -333,6 +359,16 @@ std::optional<EmitFailure> inspectProfile(const EmitOptions &Options) {
return EmitFailure{"profile_read_failed",
Options.ProfileUse +
": not an IR-level instrumentation profile"};
ContextProfile = (*Reader)->hasCSIRLevelProfile();
if (ContextProfile && !Options.ProfileGenerate.empty())
return EmitFailure{"profile_read_failed",
Options.ProfileUse +
": already holds context-sensitive counts; "
"collect them on top of the IR-level profile"};
if (ContextProfile && Triple(Options.Target).isOSBinFormatELF()) {
enableProfileLayout();
enableExtTspPlacement();
}
return std::nullopt;
}
@@ -804,6 +840,14 @@ compileOptimizedPartitions(const EmitOptions &Options,
: Level == OptimizationLevel::O3 ? 3
: 2;
Config.PTO = tuningFor(Level);
// The thin backend instruments or annotates context-sensitive counts after
// its inlining (see profileOptions).
if (!Options.ProfileGenerate.empty() && !Options.ProfileUse.empty()) {
Config.RunCSIRInstr = true;
Config.CSIRProfile = Options.ProfileGenerate;
} else if (ContextProfile) {
Config.CSIRProfile = Options.ProfileUse;
}
Config.CGFileType = CodeGenFileType::ObjectFile;
Config.DefaultTriple = Options.Target;
Config.DiagHandler = [&](const DiagnosticInfo &Info) {
native/llvm-codegen/src/emit.h
+3 -1
View File
@@ -22,7 +22,9 @@ struct EmitOptions {
std::string CacheDirectory;
// Instruments the program for IR profiling. The instrumented executable
// writes raw profiles to this path pattern (LLVM_PROFILE_FILE overrides it
// at run time). Empty: no instrumentation.
// at run time). Empty: no instrumentation. With ProfileUse, the program is
// optimized by that profile and instrumented again after inlining, which
// records context-sensitive counts.
std::string ProfileGenerate;
// Indexed profile (.profdata) that guides optimization. Empty: none.
std::string ProfileUse;
native/llvm-codegen/src/target.cpp
+8
View File
@@ -118,11 +118,19 @@ CodeGenOptLevel codeGenLevel(StringRef Level) {
return CodeGenOptLevel::Default;
}
static bool ProfileLayout = false;
void enableProfileLayout() { ProfileLayout = true; }
TargetOptions targetOptions(const Triple &TargetTriple) {
TargetOptions Options;
// The Android API 26 runtime pack uses the NDK's emulated TLS ABI.
// Program globals and runtime thread instances must use the same ABI.
Options.EmulatedTLS = TargetTriple.isAndroid();
if (ProfileLayout && TargetTriple.isOSBinFormatELF()) {
Options.FunctionSections = true;
Options.EnableMachineFunctionSplitter = true;
}
return Options;
}
native/llvm-codegen/src/target.h
+4
View File
@@ -40,6 +40,10 @@ bool supportsTarget(llvm::StringRef Triple);
// The target options and code generation level every helper target machine
// uses, shared with the partitioned optimizer's configuration.
llvm::TargetOptions targetOptions(const llvm::Triple &TargetTriple);
// Lays out code by a profile's context-sensitive counts from now on: one
// section per function, and cold blocks split into .text.split sections.
// ELF targets only; the helper calls it once, before any target machine.
void enableProfileLayout();
llvm::CodeGenOptLevel codeGenLevel(llvm::StringRef OptLevel);
std::unique_ptr<llvm::TargetMachine>
createTargetMachine(llvm::StringRef Triple, llvm::StringRef OptLevel,
package.json
+1
View File
@@ -22,6 +22,7 @@
"test:test262:fetch": "node tests/test262/fetch.mjs",
"test:cache-identity": "node tests/harness/cache-identity.mjs",
"test:lowering-order": "node scripts/lowering-order.mjs --fail-on names",
"test:recorded-lowering": "node scripts/recorded-lowering.mjs",
"test:sandbox": "node scripts/sandbox-test.mjs",
"test:sandbox:image": "node scripts/sandbox-image.mjs",
"test:watch": "vitest",
packages/cli/src/bootstrap.ts
+2
View File
@@ -144,6 +144,7 @@ async function tryFastPath(): Promise<number | null> {
backend: "llvm",
...(optimization === "dev" || optimization === "speed" ? { optimization } : {}),
...(values.strip ? { strip: true as const } : {}),
...(values.static ? { staticLink: true as const } : {}),
...(values["detached-exit"] ? { detachedExit: true as const } : {}),
...(values["large-pages"] ? { largePages: true as const } : {}),
...(values["cpu-dispatch"] === true ? { cpuDispatch: true as const } : {}),
@@ -231,6 +232,7 @@ async function coverageFastPath(
(failOn !== undefined && failOn !== "blockers" && failOn !== "divergences") ||
values.emit !== undefined ||
values.strip ||
values.static ||
values["detached-exit"] ||
values["large-pages"] ||
values["target-cpu"] !== undefined ||
packages/cli/test/output-options.test.ts
+12 -1
View File
@@ -46,6 +46,8 @@ describe("output option compatibility", () => {
[{ emit: "llvm", windowsSubsystem: "gui" }, /windows-subsystem/],
[{ emit: "obj", strip: true }, /strip/],
[{ emit: "llvm", strip: true }, /strip/],
[{ emit: "obj", staticLink: true }, /--static/],
[{ emit: "llvm", staticLink: true }, /--static/],
[{ emit: "obj", detachedExit: true }, /detached-exit/],
[{ emit: "llvm", detachedExit: true }, /detached-exit/],
[{ emit: "obj", largePages: true }, /large-pages/],
@@ -53,7 +55,6 @@ describe("output option compatibility", () => {
[{ emit: "asm", cpuDispatch: true }, /cpu-dispatch/],
[{ emit: "obj", profileGenerate: "/tmp/%p.profraw" }, /profile-generate/],
[{ emit: "llvm", profileUse: "/tmp/a.profdata" }, /profile-use/],
[{ profileGenerate: "/tmp/%p.profraw", profileUse: "/tmp/a.profdata" }, /cannot be combined/],
] as const)("rejects %j", (override, message) => {
const result = resolveOutputOptions("build", { ...BASE, ...override });
expect(result).toEqual({ ok: false, message: expect.stringMatching(message) });
@@ -118,4 +119,14 @@ describe("output option compatibility", () => {
outputKind: "exe",
});
});
test("a second profile phase combines --profile-use and --profile-generate", () => {
expect(
resolveOutputOptions("build", {
...BASE,
profileGenerate: "/tmp/%p.profraw",
profileUse: "/tmp/a.profdata",
}),
).toMatchObject({ ok: true, outputKind: "exe" });
});
});
packages/cli/test/profile.test.ts
-4
View File
@@ -109,10 +109,6 @@ test("profile options are refused where they cannot apply", async () => {
await writeFile(entry, PROGRAM);
for (const [args, message] of [
[["build", entry, "--emit=llvm", "--profile-use", "a.profdata"], "--emit=exe"],
[
["build", entry, "--profile-generate", "%p.profraw", "--profile-use", "a.profdata"],
"cannot be combined",
],
[["coverage", entry, "--profile-use", "a.profdata"], "build/run option"],
[["merge-profiles", "a.profraw"], "merge-profiles takes -o"],
[["merge-profiles", "-o", "out.profdata"], "merge-profiles takes -o"],
packages/cli/test/static-link.test.ts
+138
View File
@@ -0,0 +1,138 @@
import { execFile, execFileSync } from "node:child_process";
import { createRequire } from "node:module";
import { existsSync } from "node:fs";
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, isAbsolute, join } from "node:path";
import { promisify } from "node:util";
import { expect, test } from "vitest";
const execFileAsync = promisify(execFile);
const require = createRequire(import.meta.url);
const repoRoot = join(import.meta.dirname, "../../..");
const cliEntry = join(repoRoot, "packages/cli/src/main.ts");
const bootstrapEntry = join(repoRoot, "packages/cli/dist/bootstrap.js");
const tsxLoader = join(dirname(require.resolve("tsx/package.json")), "dist/loader.mjs");
const glibcHost =
process.platform === "linux" &&
typeof (process.report?.getReport() as { header?: { glibcVersionRuntime?: unknown } })?.header
?.glibcVersionRuntime === "string";
/** Whether the host's clang finds glibc's static C library. */
function hostHasStaticLibc(): boolean {
try {
const path = execFileSync("clang", ["-print-file-name=libc.a"], { encoding: "utf8" }).trim();
return isAbsolute(path) && existsSync(path);
} catch {
return false;
}
}
/** The ELF type and whether the executable names a program interpreter. */
async function elfLinkage(path: string): Promise<{ type: number; interpreter: boolean }> {
const bytes = await readFile(path);
const type = bytes.readUInt16LE(16);
const phoff = Number(bytes.readBigUInt64LE(32));
const phentsize = bytes.readUInt16LE(54);
const phnum = bytes.readUInt16LE(56);
let interpreter = false;
for (let i = 0; i < phnum; i++) {
if (bytes.readUInt32LE(phoff + i * phentsize) === 3 /* PT_INTERP */) interpreter = true;
}
return { type, interpreter };
}
async function withProject<T>(run: (dir: string, env: NodeJS.ProcessEnv) => Promise<T>) {
const dir = await mkdtemp(
join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-static-"),
);
try {
return await run(dir, { ...process.env, SCRIPTC_CACHE_DIR: join(dir, "cache") });
} finally {
await rm(dir, { recursive: true, force: true });
}
}
function cli(args: string[], env: NodeJS.ProcessEnv, bootstrap = false) {
return execFileAsync(
process.execPath,
[...(bootstrap ? [bootstrapEntry] : ["--import", tsxLoader, cliEntry]), ...args],
{ env, maxBuffer: 4 * 1024 * 1024 },
);
}
test.skipIf(!glibcHost || !hostHasStaticLibc())(
"--static links a glibc executable as static PIE without mixing cache variants",
async () => {
await withProject(async (dir, env) => {
const entry = join(dir, "main.ts");
const outPath = join(dir, "program");
// The NSS module file name in a string of the program's own is no lookup.
await writeFile(
entry,
'console.log("static-ok", [1, 2, 3].map((n) => n * 2).join(","), "/lib/libnss_files.so.2");\n',
);
const build = async (staticLink: boolean, bootstrap = false) => {
await cli(
["build", entry, "-o", outPath, ...(staticLink ? ["--static"] : [])],
env,
bootstrap,
);
expect((await execFileAsync(outPath)).stdout).toBe(
"static-ok 2,4,6 /lib/libnss_files.so.2\n",
);
return elfLinkage(outPath);
};
// ET_DYN with a program interpreter: the default dynamic PIE link.
expect(await build(false)).toEqual({ type: 3, interpreter: true });
// ET_DYN without one: static PIE.
expect(await build(true)).toEqual({ type: 3, interpreter: false });
expect(await build(true, true)).toEqual({ type: 3, interpreter: false });
expect(await build(false, true)).toEqual({ type: 3, interpreter: true });
});
},
180_000,
);
test.skipIf(!glibcHost || !hostHasStaticLibc())(
"--static refuses a glibc executable that includes user name lookups",
async () => {
await withProject(async (dir, env) => {
const entry = join(dir, "main.ts");
const outPath = join(dir, "program");
await writeFile(
entry,
'import { userInfo } from "node:os";\nconsole.log(userInfo().username);\n',
);
const failure = await cli(["build", entry, "-o", outPath, "--static"], env).then(
() => null,
(error: { code?: number; stderr?: string }) => error,
);
expect(failure?.code).toBe(1);
expect(failure?.stderr).toContain("error SC3002: --static cannot link this program");
expect(existsSync(outPath)).toBe(false);
// The same program links without --static.
await cli(["build", entry, "-o", outPath], env);
expect((await execFileAsync(outPath)).stdout).not.toBe("");
});
},
180_000,
);
test.skipIf(process.platform === "linux")(
"--static is refused for executables that do not target Linux",
async () => {
await withProject(async (dir, env) => {
const entry = join(dir, "main.ts");
await writeFile(entry, 'console.log("x");\n');
const failure = await cli(["build", entry, "-o", join(dir, "program"), "--static"], env).then(
() => null,
(error: { code?: number; stderr?: string }) => error,
);
expect(failure?.code).toBe(1);
expect(failure?.stderr).toContain("--static is only supported for Linux executables");
});
},
180_000,
);
packages/compiler/ambient/node-fallback/stream-web.d.ts
Vendored
+22 -3
View File
@@ -1,13 +1,32 @@
/* node:stream/web — the module exports the web stream globals themselves
* (require("stream/web").ReadableStream === globalThis.ReadableStream).
* ReadableStream is the implemented class; its reader and controller are
* exported as types. */
* (require("stream/web").ReadableStream === globalThis.ReadableStream). */
declare module "stream/web" {
export const ReadableStream: typeof globalThis.ReadableStream;
export type ReadableStream<T = unknown> = globalThis.ReadableStream<T>;
export const ReadableStreamDefaultReader: typeof globalThis.ReadableStreamDefaultReader;
export type ReadableStreamDefaultReader<T = unknown> = globalThis.ReadableStreamDefaultReader<T>;
export const ReadableStreamDefaultController: typeof globalThis.ReadableStreamDefaultController;
export type ReadableStreamDefaultController<T = unknown> =
globalThis.ReadableStreamDefaultController<T>;
export const WritableStream: typeof globalThis.WritableStream;
export type WritableStream<W = unknown> = globalThis.WritableStream<W>;
export const WritableStreamDefaultWriter: typeof globalThis.WritableStreamDefaultWriter;
export type WritableStreamDefaultWriter<W = unknown> = globalThis.WritableStreamDefaultWriter<W>;
export const WritableStreamDefaultController: typeof globalThis.WritableStreamDefaultController;
export type WritableStreamDefaultController = globalThis.WritableStreamDefaultController;
export const TransformStream: typeof globalThis.TransformStream;
export type TransformStream<I = unknown, O = unknown> = globalThis.TransformStream<I, O>;
export const TransformStreamDefaultController: typeof globalThis.TransformStreamDefaultController;
export type TransformStreamDefaultController<O = unknown> =
globalThis.TransformStreamDefaultController<O>;
export const CountQueuingStrategy: typeof globalThis.CountQueuingStrategy;
export type CountQueuingStrategy = globalThis.CountQueuingStrategy;
export const ByteLengthQueuingStrategy: typeof globalThis.ByteLengthQueuingStrategy;
export type ByteLengthQueuingStrategy = globalThis.ByteLengthQueuingStrategy;
export const TextEncoderStream: typeof globalThis.TextEncoderStream;
export type TextEncoderStream = globalThis.TextEncoderStream;
export const TextDecoderStream: typeof globalThis.TextDecoderStream;
export type TextDecoderStream = globalThis.TextDecoderStream;
export type UnderlyingSource<T = unknown> = globalThis.UnderlyingSource<T>;
export type QueuingStrategy<T = unknown> = globalThis.QueuingStrategy<T>;
}
packages/compiler/ambient/scriptc-node-fallback.d.ts
Vendored
+86
View File
@@ -1084,6 +1084,12 @@ interface ReadableStreamDefaultController<T = unknown> {
close(): void;
error(reason?: unknown): void;
}
interface StreamPipeOptions {
preventAbort?: boolean;
preventCancel?: boolean;
preventClose?: boolean;
signal?: AbortSignal;
}
interface UnderlyingSource<T = unknown> {
start?(controller: ReadableStreamDefaultController<T>): unknown;
pull?(controller: ReadableStreamDefaultController<T>): unknown;
@@ -1093,6 +1099,12 @@ interface ReadableStream<T = unknown> {
readonly locked: boolean;
cancel(reason?: unknown): Promise<void>;
getReader(): ReadableStreamDefaultReader<T>;
tee(): [ReadableStream<T>, ReadableStream<T>];
pipeTo(destination: WritableStream<T>, options?: StreamPipeOptions): Promise<void>;
pipeThrough<O>(
transform: { readable: ReadableStream<O>; writable: WritableStream<T> },
options?: StreamPipeOptions,
): ReadableStream<O>;
[Symbol.asyncIterator](options?: { preventCancel?: boolean }): AsyncIterableIterator<T>;
values(options?: { preventCancel?: boolean }): AsyncIterableIterator<T>;
}
@@ -1131,6 +1143,79 @@ declare var TransformStream: {
new <I = unknown, O = I>(): TransformStream<I, O>;
readonly prototype: TransformStream<unknown, unknown>;
};
/* The other web stream interfaces. JavaScript sources construct and read
* them as checked-dynamic values (scr_webstreams.c); TypeScript sources
* have no static lowering for them. */
declare var ReadableStreamDefaultReader: {
new <T = unknown>(stream: ReadableStream<T>): ReadableStreamDefaultReader<T>;
readonly prototype: ReadableStreamDefaultReader<unknown>;
};
declare var ReadableStreamDefaultController: {
new (): ReadableStreamDefaultController<unknown>;
readonly prototype: ReadableStreamDefaultController<unknown>;
};
interface WritableStreamDefaultController {
readonly signal: AbortSignal;
error(reason?: unknown): void;
}
declare var WritableStream: {
new <W = unknown>(sink?: object, strategy?: QueuingStrategy<W>): WritableStream<W>;
readonly prototype: WritableStream<unknown>;
};
declare var WritableStreamDefaultWriter: {
new <W = unknown>(stream: WritableStream<W>): WritableStreamDefaultWriter<W>;
readonly prototype: WritableStreamDefaultWriter<unknown>;
};
declare var WritableStreamDefaultController: {
new (): WritableStreamDefaultController;
readonly prototype: WritableStreamDefaultController;
};
interface TransformStreamDefaultController<O = unknown> {
readonly desiredSize: number | null;
enqueue(chunk?: O): void;
error(reason?: unknown): void;
terminate(): void;
}
declare var TransformStreamDefaultController: {
new (): TransformStreamDefaultController<unknown>;
readonly prototype: TransformStreamDefaultController<unknown>;
};
interface CountQueuingStrategy extends QueuingStrategy<unknown> {
readonly highWaterMark: number;
readonly size: QueuingStrategySize<unknown>;
}
declare var CountQueuingStrategy: {
new (init: { highWaterMark: number }): CountQueuingStrategy;
readonly prototype: CountQueuingStrategy;
};
interface ByteLengthQueuingStrategy extends QueuingStrategy<ArrayBufferView> {
readonly highWaterMark: number;
readonly size: QueuingStrategySize<ArrayBufferView>;
}
declare var ByteLengthQueuingStrategy: {
new (init: { highWaterMark: number }): ByteLengthQueuingStrategy;
readonly prototype: ByteLengthQueuingStrategy;
};
interface TextEncoderStream {
readonly encoding: "utf-8";
readonly readable: ReadableStream<Uint8Array>;
readonly writable: WritableStream<string>;
}
declare var TextEncoderStream: {
new (): TextEncoderStream;
readonly prototype: TextEncoderStream;
};
interface TextDecoderStream {
readonly encoding: string;
readonly fatal: boolean;
readonly ignoreBOM: boolean;
readonly readable: ReadableStream<string>;
readonly writable: WritableStream<Uint8Array>;
}
declare var TextDecoderStream: {
new (label?: string, options?: { fatal?: boolean; ignoreBOM?: boolean }): TextDecoderStream;
readonly prototype: TextDecoderStream;
};
/* fetch, typed as the native JSON + readable-stream slice: one URL, an
* optional init with the members CLI requests carry, and a Response
@@ -1665,4 +1750,5 @@ declare module "node:tty" {
interface ErrorConstructor {
stackTraceLimit: number;
captureStackTrace(targetObject: object, constructorOpt?: Function): void;
}
packages/compiler/native/ts7-process-child.test.c
+7 -5
View File
@@ -20,8 +20,9 @@
/* `--listen PATH --idle-exit SECONDS`: a stand-in for a shared server. In
* mode "listen-exit" it exits as an executable without --listen does. In
* mode "listen" it answers each client, in order, with its client number,
* its listen arguments and TSC_TS_LISTEN_MEMORY, then echoes until the
* client closes, and exits after SECONDS (at most 5) without a client. */
* its listen arguments, TSC_TS_LISTEN_MEMORY and its working directory, then
* echoes until the client closes, and exits after SECONDS (at most 5) without
* a client. */
static int listen_main(const char *mode, char **argv) {
if (strcmp(mode, "listen-exit") == 0) return 2;
if (strcmp(mode, "listen") != 0) return 73;
@@ -42,9 +43,10 @@ static int listen_main(const char *mode, char **argv) {
if (poll(&entry, 1, idle * 1000) <= 0) break;
int fd = accept(listener, NULL, NULL);
if (fd < 0) return 79;
char line[512];
int size = snprintf(line, sizeof(line), "client %d\n%s %s\n%s\n", client, argv[0], argv[2],
memory == NULL ? "unset" : memory);
char line[1024], cwd[256];
if (getcwd(cwd, sizeof(cwd)) == NULL) return 81;
int size = snprintf(line, sizeof(line), "client %d\n%s %s\n%s\n%s\n", client, argv[0], argv[2],
memory == NULL ? "unset" : memory, cwd);
if (size <= 0 || write(fd, line, (size_t)size) != size) return 80;
uint8_t bytes[17];
ssize_t count;
packages/compiler/native/ts7-process.c
+5 -3
View File
@@ -414,8 +414,10 @@ static int connect_server(const char *path) {
/* Starts the server in its own process group, so that a terminal's signals
* to the build do not reach it, with its standard streams on /dev/null and
* no other descriptor of this process, so that it holds no pipe a caller of
* the build waits on. `limit` (TSC_TS_LISTEN_MEMORY=...) is added to the
* environment unless it sets TSC_TS_LISTEN_MEMORY. */
* the build waits on. It runs in the root directory, so that it does not
* keep the build's directory in use; its working directory for the build's
* files is the --cwd argument. `limit` (TSC_TS_LISTEN_MEMORY=...) is added
* to the environment unless it sets TSC_TS_LISTEN_MEMORY. */
static pid_t start_server(char **args, char *limit) {
#ifdef __APPLE__
char **inherited = *_NSGetEnviron();
@@ -440,7 +442,7 @@ static pid_t start_server(char **args, char *limit) {
// Only async-signal-safe calls until exec.
int null = open("/dev/null", O_RDWR);
if (setpgid(0, 0) != 0 || null < 0 || dup2(null, STDIN_FILENO) < 0 || dup2(null, STDOUT_FILENO) < 0 ||
dup2(null, STDERR_FILENO) < 0)
dup2(null, STDERR_FILENO) < 0 || chdir("/") != 0)
_exit(127);
#if defined(__linux__) && defined(SYS_close_range)
if (syscall(SYS_close_range, 3U, ~0U, 0U) != 0)
packages/compiler/native/ts7-process.test.c
+4 -3
View File
@@ -208,9 +208,9 @@ static void attach_cases(void) {
uint8_t bytes[4];
snprintf(directory, sizeof(directory), "%s/scriptc", runtime);
snprintf(socket_path, sizeof(socket_path), "%s/server.sock", directory);
// The first build starts the server with its idle time and memory limit.
// The server holds none of the build's descriptors, such as an output
// pipe its caller reads to the end.
// The first build starts the server with its idle time and memory limit,
// in the root directory. The server holds none of the build's descriptors,
// such as an output pipe its caller reads to the end.
int output[2];
assert(pipe(output) == 0);
uint32_t first = attach("server", "listen", 123);
@@ -224,6 +224,7 @@ static void attach_cases(void) {
snprintf(expected, sizeof(expected), "%s 1\n", socket_path);
expect_line(first, expected);
expect_line(first, "123\n");
expect_line(first, "/\n");
write_exact(first, (const uint8_t *)"ping", 4, 0, 4);
read_exact(first, bytes, 4);
assert(memcmp(bytes, "ping", 4) == 0);
packages/compiler/src/backend/executable-features.ts
+2
View File
@@ -37,6 +37,8 @@ export function executableLinkFeatures(mod: IrModule, largePages = false): Nativ
...(features.dynForAwait ? { dynForAwait: true } : {}),
...(features.eventsOnceSignal ? { eventsOnceSignal: true } : {}),
...(features.mime ? { mime: true } : {}),
...(features.webStreams ? { webStreams: true } : {}),
...(features.abortSignal ? { abortSignal: true } : {}),
...(features.utilTiming ? { utilTiming: true } : {}),
...(features.dynBuffer ? { dynBuffer: true } : {}),
...(features.builtinValues ? { builtinValues: true } : {}),
packages/compiler/src/backend/link-plan-core.ts
+8 -1
View File
@@ -2,12 +2,13 @@
import type { NativeOptimization } from "./optimization.js";
import {
executableOptimizationLinkerArgs,
executableStaticLinkerArgs,
executableStripLinkerArgs,
windowsSubsystemLinkerArgs,
type NativeTargetSpec,
type WindowsSubsystem,
} from "./targets.js";
import { profileGenerateLinkerArgs } from "./native-profile.js";
import { profileGenerateLinkerArgs, profileLayoutLinkerArgs } from "./native-profile.js";
export interface ExecutableLinkInputs {
inputs: string[];
@@ -28,9 +29,13 @@ export function executableLinkInputs(options: {
runtimeSystemLibraries: readonly string[];
optimization: NativeOptimization;
strip?: boolean;
/** Link the C library and the other system libraries statically (`--static`). */
staticLink?: boolean;
windowsSubsystem?: WindowsSubsystem;
/** Instrumented for profile-guided optimization. */
profileGenerate?: boolean;
/** Laid out by a profile's context-sensitive counts (nativeProfileLayout). */
profileLayout?: boolean;
}): ExecutableLinkInputs {
if (options.ffiFrameworks.length !== 0 && options.target.platform !== "darwin")
throw new Error("FFI frameworks require a Darwin target");
@@ -65,8 +70,10 @@ export function executableLinkInputs(options: {
),
...executableOptimizationLinkerArgs(options.target.platform, options.optimization),
...executableStripLinkerArgs(options.target.platform, options.strip ?? false),
...executableStaticLinkerArgs(options.target, options.staticLink ?? false),
...windowsSubsystemLinkerArgs(options.target.platform, options.windowsSubsystem),
...profileGenerateLinkerArgs(options.profileGenerate ?? false),
...profileLayoutLinkerArgs(options.target, options.profileLayout ?? false),
],
};
}
packages/compiler/src/backend/link-plan.ts
+7
View File
@@ -28,6 +28,8 @@ export interface NativeLinkPlan {
programObjectDependencies: NativeArtifactDependency[];
runtimePack: RuntimePackSelection;
darwinDebugSymbols?: boolean;
/** Linked with `--static`: the linked executable is checked before use. */
staticLink?: boolean;
}
export async function createNativeLinkPlan(options: {
@@ -39,8 +41,10 @@ export async function createNativeLinkPlan(options: {
ffi: FfiProfile | null;
optimization: NativeOptimization;
strip?: boolean;
staticLink?: boolean;
windowsSubsystem?: WindowsSubsystem;
profileGenerate?: boolean;
profileLayout?: boolean;
programObjectDependencies?: readonly NativeArtifactDependency[];
env?: NodeJS.ProcessEnv;
resolver?: (specifier: string) => string;
@@ -70,13 +74,16 @@ export async function createNativeLinkPlan(options: {
runtimeSystemLibraries: runtimePack.systemLibraries,
optimization: options.optimization,
...(options.strip === undefined ? {} : { strip: options.strip }),
...(options.staticLink === true ? { staticLink: true } : {}),
...(options.windowsSubsystem === undefined
? {}
: { windowsSubsystem: options.windowsSubsystem }),
...(options.profileGenerate === true ? { profileGenerate: true } : {}),
...(options.profileLayout === true ? { profileLayout: true } : {}),
}),
dependencyPaths: [...runtimePack.dependencyPaths, ...(options.ffi?.libraries ?? [])],
programObjectDependencies: [...(options.programObjectDependencies ?? [])],
runtimePack,
...(options.staticLink === true ? { staticLink: true } : {}),
};
}
packages/compiler/src/backend/linker.ts
+18 -3
View File
@@ -7,11 +7,13 @@
import { execFile } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
import { existsSync, realpathSync, statSync } from "node:fs";
import { mkdtemp, rename, rm, stat } from "node:fs/promises";
import { mkdtemp, readFile, rename, rm, stat } from "node:fs/promises";
import { delimiter } from "node:path";
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
import { promisify } from "node:util";
import { CcCompileError, subprocessFailureDetail } from "./native/process.js";
import { NativeCodegenError } from "./native-codegen-core.js";
import { staticExecutableRefusal, staticLinkFailureNote, staticLinkMapArgs } from "./targets.js";
import { nativeArtifactDependenciesStillMatch } from "./native/artifact-stamps.js";
import { parseLinkTraceFiles } from "./native/linker-fingerprint.js";
import {
@@ -238,6 +240,8 @@ export async function linkNativeExecutable(
join(dirname(plan.outputPath), ".scriptc-runtime-pack-link-"),
);
const privateOut = join(privateOutRoot, basename(plan.outputPath));
const linkMap = join(privateOutRoot, "link.map");
const mapArgs = staticLinkMapArgs(plan.target, plan.staticLink === true, linkMap);
let stagedRoot: string | null = null;
try {
const staged = await stageRuntimePackArtifacts(plan.runtimePack);
@@ -250,6 +254,7 @@ export async function linkNativeExecutable(
...(explicitLinker === undefined ? plan.target.defaultLinkerArgs : []),
...(withLld ? lld : []),
...plan.driverFlags,
...mapArgs,
...(withLld ? inputs : inputsWithoutLld(plan.target.platform, inputs)),
...systemLibraryLinkerArgs(plan.target, plan.systemLibraries),
"-o",
@@ -293,6 +298,10 @@ export async function linkNativeExecutable(
});
const output = await stat(privateOut);
if (!output.isFile() || output.size === 0) throw new Error("linker produced no executable");
if (mapArgs.length !== 0) {
const refusal = staticExecutableRefusal(plan.target, true, await readFile(linkMap, "utf8"));
if (refusal !== null) throw new NativeCodegenError("SC3002", refusal);
}
if (plan.darwinDebugSymbols) {
await createDarwinDebugSymbols(privateOut);
await installDarwinDebugSymbols(await readDarwinDebugSymbols(privateOut), plan.outputPath);
@@ -312,12 +321,18 @@ export async function linkNativeExecutable(
await options.onArtifactReady({ dependencies: preLinkDependencies }).catch(() => undefined);
}
} catch (error) {
if (error instanceof CcCompileError || error instanceof RuntimePackError) throw error;
if (
error instanceof CcCompileError ||
error instanceof RuntimePackError ||
error instanceof NativeCodegenError
)
throw error;
const detail = subprocessFailureDetail(error);
throw new CcCompileError(
linker,
detail,
`${linker} failed linking ${basename(plan.outputPath)} from the precompiled runtime pack.\n${detail}`,
`${linker} failed linking ${basename(plan.outputPath)} from the precompiled runtime pack.\n${detail}` +
staticLinkFailureNote(plan.target, plan.staticLink === true),
);
} finally {
await Promise.all([
packages/compiler/src/backend/llvm/arenas.test.ts
+114
View File
@@ -0,0 +1,114 @@
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, test } from "vitest";
import { compile } from "../../index.js";
async function llvmFor(source: string): Promise<string> {
const dir = await mkdtemp(join(tmpdir(), "scriptc-arenas-"));
try {
const entry = join(dir, "main.ts"),
output = join(dir, "main.ll");
await writeFile(entry, source);
const result = await compile(entry, { outDir: dir, outPath: output, outputKind: "llvm" });
if (!result.ok)
throw new Error(result.diagnostics.map((item) => `${item.code}: ${item.message}`).join("\n"));
return await readFile(output, "utf8");
} finally {
await rm(dir, { recursive: true, force: true });
}
}
/** The bodies of the module's arena-aware constructors. */
function arenaConstructors(llvm: string): string[] {
return [...llvm.matchAll(/^define internal ptr @[^(]*\.r\(\)[^\n]*\n[\s\S]*?^}/gm)].map(
(m) => m[0],
);
}
const TREE = `
class Node {
left: Node | null;
right: Node | null;
constructor(left: Node | null, right: Node | null) {
this.left = left;
this.right = right;
}
count(): number {
return 1 + (this.left === null ? 0 : this.left.count()) + (this.right === null ? 0 : this.right.count());
}
}
function tree(depth: number): Node {
return depth === 0 ? new Node(null, null) : new Node(tree(depth - 1), tree(depth - 1));
}
interface Leaf {
left: Leaf | null;
right: Leaf | null;
}
function leaves(depth: number): Leaf {
return depth === 0 ? { left: null, right: null } : { left: leaves(depth - 1), right: leaves(depth - 1) };
}
function size(leaf: Leaf): number {
return leaf.left === null || leaf.right === null ? 1 : 1 + size(leaf.left) + size(leaf.right);
}
`;
test("when every arena is closed, arena-aware constructors take no dispose list", async () => {
const llvm = await llvmFor(`${TREE}
let total = 0;
for (let d = 0; d < 4; d++) total += tree(d).count() + size(leaves(d));
console.log(total);
`);
expect(llvm).toMatch(/@scr_arena_open\(ptr %[\w.]+, i32 1\)/);
expect(llvm).not.toMatch(/@scr_arena_open\(ptr %[\w.]+, i32 0\)/);
const constructors = arenaConstructors(llvm);
expect(constructors.length).toBe(2);
for (const body of constructors) {
expect(body).not.toContain("ScrArena.closed");
expect(body).not.toContain("@scr_arena_disposal");
}
});
test("an arena that is not closed keeps the dispose list in every arena-aware constructor", async () => {
// The rings hold strings and outside categories, so their groups are not closed.
const llvm = await llvmFor(`${TREE}
class Category {
name: string;
constructor(name: string) {
this.name = name;
}
}
class Member {
next: Member | undefined = undefined;
category: Category;
label: string;
constructor(category: Category, label: string) {
this.category = category;
this.label = label;
}
}
function ring(categories: Category[], size: number, seed: number): Member {
const first = new Member(categories[seed % categories.length]!, \`m\${seed}\`);
let last = first;
for (let i = 1; i < size; i++) {
const member = new Member(categories[(seed + i) % categories.length]!, \`m\${seed}-\${i}\`);
last.next = member;
last = member;
}
last.next = first;
return first;
}
const categories = [new Category("a"), new Category("b")];
let total = 0;
for (let round = 0; round < 4; round++) {
const r = ring(categories, 30, round);
total += r.next!.label.length + r.category.name.length + tree(round).count();
}
console.log(total);
`);
expect(llvm).toMatch(/@scr_arena_open\(ptr %[\w.]+, i32 1\)/);
expect(llvm).toMatch(/@scr_arena_open\(ptr %[\w.]+, i32 0\)/);
const withFields = arenaConstructors(llvm).filter((body) => body.includes(".dispose"));
expect(withFields.length).toBeGreaterThan(0);
for (const body of withFields) expect(body).toContain("ScrArena.closed");
});
packages/compiler/src/backend/llvm/arenas.ts
+6 -1
View File
@@ -47,6 +47,10 @@ export interface ArenaPlan {
/** Record shapes and classes that get an arena-aware constructor. */
shapes: Set<string>;
classes: Set<string>;
/** Every root and call region opens a closed arena, so the current arena
* is always closed or absent, and the arena-aware constructors of record
* shapes and classes add no dispose entries. */
allClosed: boolean;
}
export interface ArenaEligibility {
@@ -150,7 +154,8 @@ export function planArenas(mod: IrModule, eligible: ArenaEligibility): ArenaPlan
};
const keptSite = (e: Node): boolean => sites.arenaSites.has(e) || groups.roots.has(e);
for (const r of analyzeClosedGroups(mod, groups, arenaType, keptSite)) groups.closed.add(r);
return { groups, sites, shapes, classes };
const allClosed = [...groups.roots, ...groups.calls].every((r) => groups.closed.has(r));
return { groups, sites, shapes, classes, allClosed };
}
/** The functions in `reachable` that can allocate a cycle-capable object
packages/compiler/src/backend/llvm/classes.ts
+23 -8
View File
@@ -863,6 +863,20 @@ function undefFieldInits(
host: ClassHost,
meta: LlClassMeta,
nullable: NullableRefFields | undefined,
): string[] {
return classUndefinedFieldInits(host, meta, nullable, "%o", "%uf");
}
/** The stores that start an instance's union and dyn fields at undefined,
* for the instance at `object`, with temporaries named `<prefix><index>`
* and `<prefix>v<index>`. A nullable-pointer field's zeroed slot already is
* its unit arm. */
export function classUndefinedFieldInits(
host: ClassHost,
meta: LlClassMeta,
nullable: NullableRefFields | undefined,
object: string,
prefix: string,
): string[] {
const out: string[] = [];
meta.def.fields.forEach((f, i) => {
@@ -880,17 +894,17 @@ function undefFieldInits(
const tag = undefinedArmTag(f.type, host.unionsById);
if (tag < 0) return;
out.push(
` %uf${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr %o, i64 0, i32 ${index}, i32 0`,
` store i32 ${tag}, ptr %uf${i} ; ${llvmCommentText(f.name)} starts undefined`,
` ${prefix}${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr ${object}, i64 0, i32 ${index}, i32 0`,
` store i32 ${tag}, ptr ${prefix}${i} ; ${llvmCommentText(f.name)} starts undefined`,
);
return;
}
if (f.type.kind === "dyn") {
host.declare("declare ptr @scr_dyn_undefined()");
out.push(
` %ufv${i} = call ptr @scr_dyn_undefined()`,
` %uf${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr %o, i64 0, i32 ${index}`,
` store ptr %ufv${i}, ptr %uf${i} ; ${llvmCommentText(f.name)} starts undefined`,
` ${prefix}v${i} = call ptr @scr_dyn_undefined()`,
` ${prefix}${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr ${object}, i64 0, i32 ${index}`,
` store ptr ${prefix}v${i}, ptr ${prefix}${i} ; ${llvmCommentText(f.name)} starts undefined`,
);
return;
}
@@ -898,8 +912,8 @@ function undefFieldInits(
const tag = undefinedArmTag(f.type, host.unionsById);
if (tag < 0) return;
out.push(
` %uf${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr %o, i64 0, i32 ${index}`,
` store ptr ${host.unitInstanceRef(f.type.unionId, tag)}, ptr %uf${i} ; ${llvmCommentText(f.name)} starts undefined`,
` ${prefix}${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr ${object}, i64 0, i32 ${index}`,
` store ptr ${host.unitInstanceRef(f.type.unionId, tag)}, ptr ${prefix}${i} ; ${llvmCommentText(f.name)} starts undefined`,
);
});
return out;
@@ -1527,7 +1541,8 @@ export function emitClassShapes(
? ` store i64 ${(BigInt(meta.pre + 1) * 2n ** 32n + BigInt(ARENA_RC)).toString()}, ptr %o.vt`
: ` store ${host.sizeType} ${ARENA_RC}, ptr %o`;
const newSym = mangleClassNew(cls.name);
const dispose = refFields.length > 0 ? arenaDisposeSym(newSym) : null;
const dispose =
refFields.length > 0 && host.arenasClosed !== true ? arenaDisposeSym(newSym) : null;
defs.push(
...arenaNewLines(
host,
packages/compiler/src/backend/llvm/dyn.ts
+53 -70
View File
@@ -1,5 +1,5 @@
import { emitPerKeyOrder } from "./key-orders.js";
import { rcMembers, typedRefConstructor } from "./shapes.js";
import { typedRefConstructor } from "./shapes.js";
import { bindLiveArrayOps } from "./expr-stream-bridges.js";
import {
BYTES_ELEM_NUM,
@@ -45,7 +45,6 @@ import {
DYN_HANDLE_KINDS,
isDirentShape,
KEY_ORDER_FIELD,
shapeHasAccessorSlots,
shapeKeyOrders,
isDynTypedRefType,
isRefCounted,
@@ -261,7 +260,7 @@ export class LlDyn {
/** Emitted function definitions, in interning order. */
readonly defs: string[] = [];
private readonly prefixSources = new Map<string, string[]>();
private readonly recordViewTables = new Map<string, string>();
constructor(private readonly host: DynHost) {}
@@ -1161,51 +1160,47 @@ export class LlDyn {
);
}
/** The record shapes whose values a pointer of shape `targetId` may
* reference unchanged: their struct begins with exactly the target's
* fields (same names, types and representations, so the same offsets),
* and everything the target's release, teardown and collector support
* would get wrong on them is absent. The extra fields hold no
* references (the target's release frees the block without visiting
* them; frees are size-class agnostic), neither shape carries a shape
* word, a cycle header, an overflow map, key order variants or accessor
* slots, and both freeze alike. A record that keeps undeclared source
* properties is refused at run time (scr_dyn_typed_ref_prefix_view). */
private prefixViewSources(targetId: string): string[] {
const memo = this.prefixSources.get(targetId);
if (memo) return memo;
/** `scr_dyn_record_view(d, sources, n, key, key_len, new)` for
* view target `targetId`: the view (+1) of the record a capsule `%d`
* holds when its type is one of the target's sources, else NULL. The
* sources table lists each source's type key and the byte offsets of
* the target's fields in it. */
private recordViewCall(B: BlockBuilder, targetId: string): string {
const host = this.host;
const plain = (shape: IrRecordShape): boolean =>
!shape.tuple &&
!shape.indexValue &&
!isDirentShape(shape) &&
!shapeHasAccessorSlots(shape) &&
shapeKeyOrders(shape).length === 1 &&
!shape.fields.some((field) => field.name === KEY_ORDER_FIELD) &&
host.nullableUnions.shapeNumber(shape.id) === 0 &&
!host.tracedShapes.has(`record:${shape.id}`);
const target = host.recordsById.get(targetId);
const out: string[] = [];
if (target && plain(target) && target.fields.length > 0) {
const base = host.nullableUnions.fieldBase(targetId);
for (const shape of host.recordsById.values()) {
if (shape.id === targetId || shape.fields.length <= target.fields.length) continue;
if (!plain(shape) || shape.freezable !== target.freezable) continue;
if (host.nullableUnions.fieldBase(shape.id) !== base) continue;
const prefix = target.fields.every(
(field, i) =>
shape.fields[i]!.name === field.name &&
typeKey(shape.fields[i]!.type) === typeKey(field.type),
let table = this.recordViewTables.get(targetId);
const sources = host.recordViews!.sources(targetId);
const target = host.recordsById.get(targetId)!;
if (table === undefined) {
table = `@sc_rv_${this.recordViewTables.size}`;
this.recordViewTables.set(targetId, table);
const S = this.S;
const entries = sources.map((sourceId, k) => {
const source = host.recordsById.get(sourceId)!;
const struct = mangleRecordStruct(sourceId);
const base = host.nullableUnions.fieldBase(sourceId);
const offsets = target.fields.map((field) => {
const index = source.fields.findIndex((f) => f.name === field.name) + base;
return `${S} ptrtoint (ptr getelementptr (%${struct}, ptr null, i32 0, i32 ${index}) to ${S})`;
});
const offsetTable = `${table}_${k}`;
this.defs.push(
`${offsetTable} = internal constant [${offsets.length} x ${S}] [${offsets.join(", ")}]`,
);
if (!prefix) continue;
const extra = rcMembers(shape, host.inlineUnions, base).some(
(member) => member.index >= target.fields.length + base && isRefCounted(member.type),
);
if (!extra) out.push(shape.id);
}
const key = typeKey({ kind: "record", shapeId: sourceId });
return `{ ptr, ${S}, ptr } { ptr ${host.cstr(key)}, ${S} ${Buffer.byteLength(key, "utf8")}, ptr ${offsetTable} }`;
});
this.defs.push(
`${table} = internal constant [${entries.length} x { ptr, ${S}, ptr }] [${entries.join(", ")}]`,
``,
);
}
this.prefixSources.set(targetId, out);
return out;
const key = typeKey({ kind: "record", shapeId: targetId });
host.declare(`declare ptr @scr_dyn_record_view(ptr, ptr, ${this.S}, ptr, ${this.S}, ptr)`);
const view = B.tmp();
B.line(
`${view} = call ptr @scr_dyn_record_view(ptr %d, ptr ${table}, ${this.S} ${sources.length}, ptr ${host.cstr(key)}, ${this.S} ${Buffer.byteLength(key, "utf8")}, ptr @${mangleRecordNew(targetId)})`,
);
return view;
}
private typedRefMatchesAny(B: BlockBuilder, types: readonly IrType[]): string {
@@ -2222,31 +2217,19 @@ export class LlDyn {
B.line(`${ref} = call ptr @scr_dyn_typed_ref_unbox(ptr %d)`);
B.terminate(`ret ptr ${ref}`);
B.startBlock(lNext);
if (t.kind === "record") {
// A record whose storage begins with exactly this type's storage
// is this type's value too: the conversion passes the pointer, so
// both names keep one object (decision D3).
const sources = this.prefixViewSources(t.shapeId);
if (sources.length > 0) {
const viewed = this.typedRefMatchesAny(
B,
sources.map((shapeId) => ({ kind: "record", shapeId })),
);
const lView = B.newLabel("dc.prefix");
const lOther = B.newLabel("dc.noprefix");
B.condBr(viewed, lView, lOther);
B.startBlock(lView);
host.declare(`declare ptr @scr_dyn_typed_ref_prefix_view(ptr)`);
const view = B.tmp();
B.line(`${view} = call ptr @scr_dyn_typed_ref_prefix_view(ptr %d)`);
const passed = B.tmp();
B.line(`${passed} = icmp ne ptr ${view}, null`);
const lPass = B.newLabel("dc.prefix.pass");
B.condBr(passed, lPass, lOther);
B.startBlock(lPass);
B.terminate(`ret ptr ${view}`);
B.startBlock(lOther);
}
if (t.kind === "record" && host.recordViews?.targets.has(t.shapeId) === true) {
// A capsule of a record with all of this type's fields converts
// to a view of that record: both names keep one object
// (record-views.ts, decision D3).
const view = this.recordViewCall(B, t.shapeId);
const viewed = B.tmp();
B.line(`${viewed} = icmp ne ptr ${view}, null`);
const lView = B.newLabel("dc.view");
const lOther = B.newLabel("dc.noview");
B.condBr(viewed, lView, lOther);
B.startBlock(lView);
B.terminate(`ret ptr ${view}`);
B.startBlock(lOther);
}
// A constructor-function stream (scr_dyn_builtin_stream.c) carries
// its runtime stream.
packages/compiler/src/backend/llvm/emit-statements.ts
+19 -4
View File
@@ -153,7 +153,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
break;
}
const stackObject = emitter.localStackObjects.get(s.localId);
if (stackObject) {
if (stackObject && b.kind !== "boxed") {
const value = emitStackObject(emitter, stackObject, (fields) =>
emitter.ownScopeFields(fields),
);
@@ -222,7 +222,11 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
B.line(`store ptr ${stack.box}, ptr ${b.slot}`);
if (stack.owner) emitter.scopes[emitter.scopes.length - 1]!.push(stack.owner);
if (s.init) {
const value = emitter.emitExpr(s.init);
const value = stackObject
? emitter.markImmortal(
emitStackObject(emitter, stackObject, (fields) => emitter.ownScopeFields(fields)),
)
: emitter.emitExpr(s.init);
if (isRefCounted(value.type)) emitter.toFrame(value);
emitter.boxSet(stack.box, b.type, value.name);
}
@@ -253,7 +257,13 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
if (sentinel !== null) B.line(`store ptr ${sentinel}, ptr ${emitter.boxSlot(box)}`);
break;
}
const v = emitter.emitExpr(s.init);
// A frame object's binding box holds its frame storage, which the
// closures that capture the box never keep (frame-escapes.ts).
const v = stackObject
? emitter.markImmortal(
emitStackObject(emitter, stackObject, (fields) => emitter.ownScopeFields(fields)),
)
: emitter.emitExpr(s.init);
if (isRefCounted(v.type)) emitter.toHeap(v); // the box takes ownership
if (b.local!.tdz === true && boxAccess(b.type) !== "ref") {
emitter.tdzScalarInit(box, b.type, v.name);
@@ -2992,7 +3002,12 @@ function emitProvenLoop(
}
const deferred = new Set<string>();
for (const id of plan.locals.keys()) deferred.add(id);
emitter.speculation = { writes: plan.writes, resume: new Map(), deferred };
emitter.speculation = {
writes: plan.writes,
resume: new Map(),
deferred,
increments: plan.increments,
};
try {
const leave = B.newLabel("integer.leave");
emitCopy(leave);
packages/compiler/src/backend/llvm/emitter.ts
+52 -4
View File
@@ -2,6 +2,7 @@ import type { TargetCpu } from "../target-cpu.js";
import { loopEntryInlines } from "./loop-entry-versions.js";
import type { FunctionLayout } from "./function-layout.js";
import { relieveHeapPressure } from "../../heap-pressure.js";
import { analyzeFrameEscapes, type FrameEscapes } from "./frame-escapes.js";
import { wasmBoundsChecks } from "./wasm-modules.js";
import { jsTruthy } from "./jsval.js";
import { JsIcSites } from "../../ir/js-ic.js";
@@ -357,6 +358,7 @@ import {
import { children, type Node } from "./region-ir.js";
import { LlDyn, recordDynKeepsIdentity, type DynHost } from "./dyn.js";
import { VirtualBorrows } from "./virtual-borrows.js";
import { planRecordViews, recordViewFieldPtr } from "./record-views.js";
import { LlvmUnsupportedError } from "./unsupported.js";
import { LlWalkers } from "./walkers.js";
import { isObjectArm, NULLABLE_ABSENT, NULLABLE_NULL, NullableUnions } from "./nullable-unions.js";
@@ -1400,6 +1402,8 @@ export class LlEmitter {
readonly dyn: LlDyn;
/** Explicit callback boundary for standalone layout and walker stages. */
readonly shapeHost: DynHost;
/** The record view plan's target set (record-views.ts), shared by reference. */
private recordViewTargets: Set<string> | null = null;
/** External declarations, in first-use order. */
readonly decls = new Set<string>();
/** Declared functions referenced as values: each needs an env-signature
@@ -1444,6 +1448,9 @@ export class LlEmitter {
/** The current function's fused string-key Map get-then-set pairs (map-entry-fusion.ts). */
mapEntryFusion: MapEntryFusion | null = null;
readonly callLifetimes: CallLifetimes;
/** The locals whose value never leaves its frame (frame-escapes.ts), for
* class instances with reference fields in frame storage. */
frameEscapes: FrameEscapes | null = null;
readonly constantCallbacks: Map<string, Map<string, string>>;
currentConstantCallbacks: ReadonlyMap<string, string> = new Map();
/** The current function's running maximum and minimum updates, whose
@@ -1779,6 +1786,9 @@ export class LlEmitter {
ran?: string;
/** Writes that speculate a rewritten value (rewriteSpeculable). */
values?: ReadonlyMap<IrStmt, SpeculativeValue>;
/** A proven integer loop's expression-position `++`/`--` of its
* shadowed bindings (ProvenLoopPlan.increments). */
increments?: ReadonlySet<IrExpr>;
} | null = null;
/** Speculative loop sites numbered so far in the module. */
speculativeLoopSites = 0;
@@ -2236,6 +2246,7 @@ export class LlEmitter {
},
(wrap) => this.borrowableNullableWrap(wrap) === "ref",
);
if (this.stackObjects) this.frameEscapes = analyzeFrameEscapes(this.mod);
if (this.debug === null) {
const strings = findLentStringParameters(this.fnByName, this.callLifetimes);
this.lendingFacts = {
@@ -2406,6 +2417,20 @@ export class LlEmitter {
}
}
}
// Planned before any host copies the shape host (record-views.ts).
{
const views = planRecordViews(mod, {
traced: (id) => this.tracedShapes.has(`record:${id}`),
// A tagged shape's values share field reads with the other arms of
// its shape union.
excluded: (id) =>
this.nullableUnions.shapeNumber(id) !== 0 ||
this.publishedTypes?.recordGuarded(id) === true ||
mod.workers === true,
});
this.shapeHost.recordViews = views;
this.recordViewTargets = views.targets as Set<string>;
}
this.walkers = new LlWalkers(this.shapeHost);
this.dyn = new LlDyn(this.shapeHost);
for (const g of mod.globals ?? []) {
@@ -2525,8 +2550,16 @@ export class LlEmitter {
if (this.arenas !== null) {
this.shapeHost.arenaShapes = this.arenas.shapes;
this.shapeHost.arenaClasses = this.arenas.classes;
this.shapeHost.arenasClosed = this.arenas.allClosed;
}
}
// Arena shapes are known only now; the plan's target set is shared by
// every host that copied the shape host, so dropping them here reaches
// all of them.
const arenaShapes = this.shapeHost.arenaShapes;
const viewTargets = this.recordViewTargets;
if (arenaShapes !== undefined && viewTargets !== null)
for (const id of arenaShapes) viewTargets.delete(id);
for (const [name, rec] of RUNTIME_ERROR_CLASSES) {
const meta = this.classMeta.get(name);
if (!meta) break; // hand-written IR without the builtin defs: no stamps
@@ -4848,10 +4881,25 @@ export class LlEmitter {
throw new InternalCompilerError(
`llvm emitter bug: unknown field ${field} on shape ${shapeId}`,
);
const p = this.B.tmp();
this.B.line(
`${p} = getelementptr inbounds %${mangleRecordStruct(shapeId)}, ptr ${objName}, i64 0, i32 ${idx + this.nullableUnions.fieldBase(shapeId)}`,
);
const base = this.nullableUnions.fieldBase(shapeId);
let p: string;
if (this.shapeHost.recordViews?.targets.has(shapeId) === true) {
p = recordViewFieldPtr(
this.B,
this.sizeType,
mangleRecordStruct(shapeId),
objName,
idx + base,
idx,
(decl) => this.declare(decl),
this.rcMode === undefined,
);
} else {
p = this.B.tmp();
this.B.line(
`${p} = getelementptr inbounds %${mangleRecordStruct(shapeId)}, ptr ${objName}, i64 0, i32 ${idx + base}`,
);
}
const type = shape.fields[idx]!.type;
return { ptr: p, type, inline: this.inlineUnions.record(type) };
}
packages/compiler/src/backend/llvm/expr-calls.ts
+19 -1
View File
@@ -18,7 +18,12 @@ import { liveRefView } from "./expr-stream-bridges.js";
import { typedRefConstructor, vAdapters } from "./shapes.js";
import { INT32_ABI_SUFFIX, SCALAR_UNION_RESULT } from "./int32-abi.js";
import { dispatchesCall, INTEGER_DISPATCH_SUFFIX } from "./integer-clones.js";
import { canStackObject, emitStackObject } from "./stack-objects.js";
import {
canStackObject,
emitStackObject,
frameClass,
type StackObjectSource,
} from "./stack-objects.js";
import { collectFfiRetainedOps, parseFfiCallbackKey } from "../ffi-callbacks.js";
import {
mangleAsyncAdoptCall,
@@ -129,6 +134,19 @@ export function emitCallExpr(
if (borrowed) host.frames.push([]);
let suffixes: Writes[] | undefined;
const args = e.args.map((a, index) => {
// A fresh instance of a frame class that the parameter keeps in the
// callee's frame (frame-escapes.ts) is built in the caller's frame;
// the call's statement frame releases its reference fields.
if (
a.kind === "new" &&
host.frameEscapes?.parameterStays(e.callee, index) === true &&
frameClass(host, a.className)
) {
const value = emitStackObject(host, a as StackObjectSource, (fields) => {
for (const field of fields) host.ownSlot(field.ptr, field.type);
});
return host.markImmortal(value);
}
if (borrowed?.has(index)) {
// A heap-borrowing helper may retain or return its argument.
// Only projection-only parameters may see an unowned stack box,
packages/compiler/src/backend/llvm/expr-primitives.ts
+23
View File
@@ -517,6 +517,29 @@ export function emitOperatorExpr(
// Expression-position ++/-- over an f64 binding (locals, module
// globals, capture boxes): read, write ±1, yield old (postfix) or
// new (prefix).
const shadow = host.integerLoopBindings.get(e.localId);
if (shadow !== undefined && host.speculation?.increments?.has(e)) {
// A proven integer loop's shadow: the loop range holds every value
// the binding takes, so the step cannot overflow.
const old = B.tmp();
const next = B.tmp();
const number = B.tmp();
B.line(`${old} = load i64, ptr ${shadow.slot}`);
B.line(`${next} = ${e.op === "+" ? "add" : "sub"} nsw i64 ${old}, 1`);
B.line(`store i64 ${next}, ptr ${shadow.slot}`);
const value = e.prefix ? next : old;
B.line(`${number} = sitofp i64 ${value} to double`);
return {
name: number,
type: e.type,
integer: {
name: value,
type: "i64",
signed: true,
range: shadow.range ?? { min: -Number.MAX_SAFE_INTEGER, max: Number.MAX_SAFE_INTEGER },
},
};
}
const b = host.binding(e.localId);
if (host.int32Locals.has(e.localId)) {
if (!host.int32LocalWrites.has(e))
packages/compiler/src/backend/llvm/frame-escapes.ts
+271
View File
@@ -0,0 +1,271 @@
/* Frame escapes (stack-objects.ts): the locals whose value provably dies
* before their function returns, so that an object bound to one can live in
* the frame.
*
* A local stays in the frame when every use of its value does: field and
* element reads and writes through it (storing into it is fine), length and
* search operations, identity comparisons, a statement that discards it, and
* passing it to a parameter that itself stays in its callee's frame
* (region-ir.ts keepsInFrame). A value that a conditional, a union wrap or
* narrow, a cast or a sequence passes on takes that expression's fate. A
* `const` binding of the value in the same function shares the local's fate:
* it is declared inside the local's scope, so it cannot outlive it.
*
* A closure that captures the local keeps it in the frame when the closure
* itself stays in the frame (it is only invoked, or passed to a parameter
* that stays in its callee's frame) and its body keeps the captured binding
* in the frame. The binding's box then never outlives the frame either.
*
* Everything else escapes: returning or throwing it, storing the value
* itself anywhere (a field, an element, a collection, a module global or
* another binding by assignment), capturing it in a closure that escapes or
* in a local class, converting it, and every call whose target is not a
* known function with a proven parameter.
*
* Parameter and capture facts are solved optimistically, like
* CallLifetimes: every reference parameter stays in its callee's frame, and
* every captured binding in its closure's, until a use proves otherwise.
* They are solved on demand: a query explores only the facts its answer
* depends on (a parameter's fact depends on the calls its own value reaches),
* and each function's body is walked once into a summary of those uses.
* Async functions, generators and module code have no facts, since their
* frames can outlive a call or are the program's own. */
import type { IrExpr, IrFunction, IrModule, IrStmt } from "../../ir/ir.js";
import { everyExprChild, everyStmtChild } from "../../ir/traverse.js";
import {
isExpr,
indexOfNode,
keepsInFrame,
same,
refType,
transparent,
type Node,
} from "./region-ir.js";
export interface FrameEscapes {
/** The reference locals of `fn` whose value never leaves its frame:
* never assigned after its declaration, captured only by closures that
* keep it in the frame, and every use keeping it in the frame. */
locals(fn: IrFunction): ReadonlySet<string>;
/** Whether argument `index` of a call of `callee` stays in the callee's
* frame. */
parameterStays(callee: string, index: number): boolean;
}
function suspends(fn: IrFunction): boolean {
return fn.async === true || fn.generator !== undefined || /^%(init\.\d+|main)$/.test(fn.name);
}
/** A use whose fate a fact decides: argument `index` of a call of `fn`
* (`index` >= 0), or a binding `id` that a closure of lifted function `fn`
* captures. */
interface FactUse {
fn: string;
index: number;
id: string;
}
/** What one walk of a function's body finds, by value group: a `const`
* binding of a value shares its fate, and a closure that of the locals it
* captures. */
interface Summary {
/** The representative of each grouped value. */
root: Map<string, string>;
/** Groups that escape whatever the facts say. */
escaping: Set<string>;
/** Per group, the uses whose facts decide it. */
uses: Map<string, FactUse[]>;
}
const NO_USES: readonly FactUse[] = [];
const NO_PARAMETERS: ReadonlyMap<string, Set<number>> = new Map();
/** Walks `fn` once. */
function summarize(fn: IrFunction): Summary {
const escapes = new Set<string>();
const facts: { id: string; use: FactUse }[] = [];
const own = new Set(fn.locals.map((l) => l.id));
const group = new Map<string, string>();
const find = (id: string): string => {
let root = id;
while (group.has(root)) root = group.get(root)!;
return root;
};
const unite = (a: string, b: string): void => {
const ra = find(a),
rb = find(b);
if (ra !== rb) group.set(ra, rb);
};
const declared = new Map<string, number>();
// Captured bindings that a closure node of this function captures.
const closureCaptured = new Set<string>();
let closures = 0;
const parents: Node[] = [];
/** The fate of the value `node` produces: shared with the binding a
* `const` declaration gives it, kept by its consumer, decided by a
* parameter fact, or escaping. */
const fate = (node: Node, id: string): void => {
let value: Node = node;
let at = parents.length - 1;
while (at >= 0 && transparent(value, parents[at])) value = parents[at--]!;
const parent = at >= 0 ? parents[at] : undefined;
if (
parent?.kind === "varDecl" &&
parent.init !== null &&
same(parent.init, value) &&
own.has(parent.localId) &&
parent.localId !== id
)
unite(id, parent.localId);
else if (parent?.kind === "call") {
const index = indexOfNode(parent.args, value);
if (index >= 0) facts.push({ id, use: { fn: parent.callee, index, id: "" } });
else escapes.add(id);
} else if (!keepsInFrame(value, parent, NO_PARAMETERS)) escapes.add(id);
};
const walk = (node: Node): void => {
if (node.kind === "varDecl") declared.set(node.localId, (declared.get(node.localId) ?? 0) + 1);
if (node.kind === "assign" || node.kind === "assignExpr" || node.kind === "incDec")
escapes.add(node.localId);
if (node.kind === "classRef") for (const id of node.captures ?? []) escapes.add(id);
if (node.kind === "closure") {
const closure = `%closure.${closures++}`;
fate(node, closure);
for (const id of node.captures ?? []) {
closureCaptured.add(id);
unite(id, closure);
facts.push({ id, use: { fn: node.fnName, index: -1, id } });
}
}
if (node.kind === "varRef") {
fate(node, node.localId);
return;
}
parents.push(node);
if (isExpr(node)) everyExprChild(node, visitExpr, visitStmt);
else everyStmtChild(node, visitExpr, visitStmt);
parents.pop();
};
const visitExpr = (child: IrExpr): boolean => {
walk(child);
return true;
};
const visitStmt = (child: IrStmt): boolean => {
walk(child);
return true;
};
for (const stmt of fn.body) walk(stmt);
const lifted = new Set((fn.captures ?? []).map((c) => c.localId));
for (const local of fn.locals) {
if (local.boxed === true && !lifted.has(local.id) && !closureCaptured.has(local.id))
escapes.add(local.id);
if ((declared.get(local.id) ?? 0) > 1) escapes.add(local.id);
}
const root = new Map<string, string>();
for (const id of group.keys()) root.set(id, find(id));
const rootOf = (id: string): string => root.get(id) ?? id;
const escaping = new Set<string>();
for (const id of escapes) escaping.add(rootOf(id));
const uses = new Map<string, FactUse[]>();
for (const { id, use } of facts) {
const r = rootOf(id);
if (escaping.has(r)) continue;
let list = uses.get(r);
if (!list) uses.set(r, (list = []));
list.push(use);
}
return { root, escaping, uses };
}
export function analyzeFrameEscapes(mod: IrModule): FrameEscapes {
const fns = new Map(mod.functions.map((fn) => [fn.name, fn]));
const summaries = new Map<string, Summary>();
const summaryOf = (fn: IrFunction): Summary => {
let summary = summaries.get(fn.name);
if (summary === undefined) {
summary = summarize(fn);
summaries.set(fn.name, summary);
}
return summary;
};
/** A fact's key, the binding in its function that it is about, or null
* when the fact is false from the start. */
const subject = (use: FactUse): { key: string; fn: IrFunction; id: string } | null => {
const fn = fns.get(use.fn);
if (fn === undefined || suspends(fn)) return null;
if (use.index >= 0) {
const param = fn.params[use.index];
if (param === undefined || !refType(param.type)) return null;
return { key: `${use.fn}#${use.index}`, fn, id: param.localId };
}
if (!(fn.captures ?? []).some((c) => c.localId === use.id)) return null;
return { key: `${use.fn}@${use.id}`, fn, id: use.id };
};
const solved = new Map<string, boolean>();
/** The facts `uses` decide, solved over the facts they depend on that are
* not solved yet: a greatest fixpoint on that part of the graph. */
const solve = (uses: readonly FactUse[]): void => {
const pending = new Map<string, { ok: boolean; deps: string[] }>();
const stack: FactUse[] = [...uses];
while (stack.length > 0) {
const use = stack.pop()!;
const s = subject(use);
if (s === null || solved.has(s.key) || pending.has(s.key)) continue;
const summary = summaryOf(s.fn);
const r = summary.root.get(s.id) ?? s.id;
const node = { ok: !summary.escaping.has(r), deps: [] as string[] };
pending.set(s.key, node);
if (!node.ok) continue;
for (const dep of summary.uses.get(r) ?? NO_USES) {
const d = subject(dep);
if (d === null) {
node.ok = false;
break;
}
node.deps.push(d.key);
stack.push(dep);
}
}
for (let changed = true; changed;) {
changed = false;
for (const node of pending.values())
if (node.ok && node.deps.some((d) => !(solved.get(d) ?? pending.get(d)!.ok))) {
node.ok = false;
changed = true;
}
}
for (const [key, node] of pending) solved.set(key, node.ok);
};
/** Whether every use in `uses` keeps its value in the frame. */
const allStay = (uses: readonly FactUse[]): boolean => {
solve(uses);
for (const use of uses) {
const s = subject(use);
if (s === null || solved.get(s.key) !== true) return false;
}
return true;
};
const locals = new Map<string, ReadonlySet<string>>();
return {
parameterStays(callee: string, index: number): boolean {
return allStay([{ fn: callee, index, id: "" }]);
},
locals(fn: IrFunction): ReadonlySet<string> {
let kept = locals.get(fn.name);
if (kept === undefined) {
const set = new Set<string>();
if (!suspends(fn)) {
const summary = summaryOf(fn);
for (const local of fn.locals) {
if (!refType(local.type)) continue;
const r = summary.root.get(local.id) ?? local.id;
if (!summary.escaping.has(r) && allStay(summary.uses.get(r) ?? NO_USES))
set.add(local.id);
}
}
locals.set(fn.name, (kept = set));
}
return kept;
},
};
}
packages/compiler/src/backend/llvm/lib-dispatch.ts
+50 -4
View File
@@ -24,7 +24,7 @@ import {
} from "./lib-shared.js";
import { isDynUndefined } from "../../ir/runtime-features.js";
import { DYN_KIND } from "./dyn.js";
import { caBytes } from "./lib-abi.js";
import { caBytes, callRuntime, ptr } from "./lib-abi.js";
import { emitInlineAtomicsOp } from "./expr-bytes.js";
import { regexLiteralCaptures } from "./regex-matchers.js";
import { registerReplay } from "./emit-journal.js";
@@ -1110,14 +1110,44 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue {
if (e.fn === "record.isFrozen") {
// The frozen bit of the record's rc word (scr_rc.h SCR_RC_FROZEN) is
// its second-highest bit; an immortal word sets the highest one too.
const value = host.emitReadReceiver(e.args[0]!);
// Where the record's shape may be published, an immortal word reads as
// frozen: publish() freezes what it reaches, as Node's deep freeze does.
const receiver = e.args[0]!;
const value = host.emitReadReceiver(receiver);
const S = host.sizeType;
const rc = host.B.tmp(),
top = host.B.tmp(),
t = host.B.tmp();
top = host.B.tmp();
let t = host.B.tmp();
host.B.line(`${rc} = load ${S}, ptr ${value.name}`);
host.B.line(`${top} = lshr ${S} ${rc}, ${S === "i64" ? 62 : 30}`);
host.B.line(`${t} = icmp eq ${S} ${top}, 1`);
if (
receiver.type.kind === "record" &&
host.publishedTypes?.recordGuarded(receiver.type.shapeId) === true
)
t = orPublished(host, t, rc);
return { name: t, type: e.type };
}
if (
(e.fn === "array.isFrozen" || e.fn === "collection.isExtensible") &&
host.publishedTypes?.containerPublished(e.args[0]!.type) === true
) {
// publish() freezes what it reaches, as Node's deep freeze does: a
// published array is frozen, and a published Map or Set is not extensible.
const value = host.emitReadReceiver(e.args[0]!);
const answer = callRuntime(
host,
e.fn === "array.isFrozen" ? "scr_arr_is_frozen" : "scr_map_is_extensible",
"i1",
[ptr(value.name)],
);
const rc = host.B.tmp();
host.B.line(`${rc} = load ${host.sizeType}, ptr ${value.name}`);
if (e.fn === "array.isFrozen") return { name: orPublished(host, answer, rc), type: e.type };
const mortal = host.B.tmp(),
t = host.B.tmp();
host.B.line(`${mortal} = icmp ne ${host.sizeType} ${rc}, -1`);
host.B.line(`${t} = and i1 ${answer}, ${mortal}`);
return { name: t, type: e.type };
}
const prefix = e.fn.slice(0, e.fn.indexOf(".")) as LibCallPrefix;
@@ -1241,6 +1271,11 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue {
return host.emitGenericLibCall(e);
case "nodeIsland":
return host.emitGenericLibCall(e);
case "webstream":
// A stream's start, pull and cancel steps run as promise jobs, so
// a program with the classes runs its event loop.
if (e.fn === "webstream.class") host.usesTimers = true;
return host.emitGenericLibCall(e);
case "test":
return emitTestLibCall(host, e);
case "tls":
@@ -1348,3 +1383,14 @@ function emitPromiseAllTuple(host: LlvmEmitterContext, e: LibCallExpr): LlValue
B.line(`call void @scr_promise_release(ptr ${all})`);
return host.own({ name: result, type: e.type });
}
/** `frozen` or'ed with "the object whose rc word is `rc` is published": an
* immortal word (all ones), the test of the published write guards
* (emitter.ts emitPublishedTest). */
function orPublished(host: LlvmEmitterContext, frozen: string, rc: string): string {
const immortal = host.B.tmp(),
t = host.B.tmp();
host.B.line(`${immortal} = icmp eq ${host.sizeType} ${rc}, -1`);
host.B.line(`${t} = or i1 ${frozen}, ${immortal}`);
return t;
}
packages/compiler/src/backend/llvm/lib-process.ts
+26
View File
@@ -741,6 +741,32 @@ export function emitProcessLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll
export function emitErrorsEventsLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue {
const B = host.B;
if (e.fn === "error.captureStackTrace") {
// The second argument names the function constructorOpt skips through;
// its frame text is the constant its own frame pushes (emit-function.ts),
// or null when it has no frame, which leaves no frames.
const target = host.emitExpr(e.args[0]!);
const named = e.args[1]!;
const skip = e.args[2]!.kind === "boolLit" && e.args[2]!.value;
let through = "null";
if (named.kind === "strLit" && named.value !== "") {
const fn = host.mod.functions.find((f) => f.name === named.value);
if (
fn !== undefined &&
host.stackTraces &&
!host.wasi &&
fn.sourceName !== undefined &&
!host.unframedFunctions.has(fn.name)
)
through = host.cstr(` at ${fn.sourceName} (${fn.loc.file})`);
}
host.declare(`declare void @scr_error_capture_stack_trace(ptr, ptr, i1)`);
B.line(
`call void @scr_error_capture_stack_trace(ptr ${target.name}, ptr ${through}, i1 ${skip ? "true" : "false"})`,
);
host.emitPendingCheck();
return { name: "", type: e.type };
}
if (e.fn === "error.argTypeThrow") {
// Always throws with the runtime-rendered Received tail (the
// error.nodeThrow dummy pattern). Borrows all three.
packages/compiler/src/backend/llvm/lib-shared.ts
+2 -8
View File
@@ -118,6 +118,8 @@ export const LIB_FN_SYMS: Record<string, string> = {
"util.diff": "scr_util_diff",
"util.debounce": "scr_util_debounce",
"util.throttle": "scr_util_throttle",
"webstream.class": "scr_webstream_class",
"webstream.instanceOf": "scr_webstream_instanceof",
"mime.typeNew": "scr_mime_type_new",
"mime.typeParse": "scr_mime_type_parse",
"mime.paramsNew": "scr_mime_params_new",
@@ -1030,10 +1032,6 @@ export const LIB_FN_SYMS: Record<string, string> = {
"fetch.responseJson": "scr_fetch_response_json",
"fetch.responseText": "scr_fetch_response_text",
"fetch.responseBytes": "scr_fetch_response_bytes",
"fetch.abortControllerNew": "scr_fetch_abort_controller_new",
"fetch.abortTimeout": "scr_fetch_abort_timeout",
"fetch.abortNow": "scr_fetch_abort_now",
"fetch.abortAny": "scr_fetch_abort_any",
"fetch.webIs": "scr_fetch_web_is",
"fetch.streamIs": "scr_fetch_stream_is",
"zlib.streamIs": "scr_zlib_stream_is",
@@ -1695,7 +1693,6 @@ export const LIB_FN_SYMS: Record<string, string> = {
"dyn.objectPrototype": "scr_dyn_object_prototype",
"dyn.errorPrototype": "scr_dyn_error_prototype",
"dyn.inheritsErrorPrototype": "scr_dyn_inherits_error_prototype",
"dyn.errorCaptureStackTrace": "scr_dyn_error_capture_stack_trace",
"dyn.errorCaptureStackTraceFunction": "scr_dyn_error_capture_stack_trace_function",
"dyn.functionApply": "scr_dyn_function_apply",
"dyn.builtinMethod": "scr_dyn_builtin_method",
@@ -1840,14 +1837,11 @@ export const USES_TIMERS_LIB_FNS = new Set<string>([
"fetch.input",
"fetch.function",
"fetch.requestNew",
"fetch.abortTimeout",
"abort.timeout",
// EventTarget listeners report their exceptions from a nextTick.
"event.targetNew",
"abort.controllerNew",
"abort.any",
"fetch.abortControllerNew",
"fetch.abortAny",
"fetch.streamNew",
"fetch.streamFrom",
"fetch.transformNew",
packages/compiler/src/backend/llvm/loop-entry-versions.ts
+3 -3
View File
@@ -1,4 +1,4 @@
/* Loop-entry range versions (ir/loop-entry-ranges.ts): a while loop whose
/* Loop-entry range versions (ir/loop-entry-ranges.ts): a while or for loop whose
* integer work depends on locals entering it without range facts runs as
* two copies. A test on entry checks that each such local holds an integer
* in its assumed range; the fast copy is emitted under the facts that hold
@@ -16,7 +16,7 @@ interface LoopEntryState {
fn: IrFunction | null;
facts: IntegerRangeFacts | null;
slots: IntegerSlotFacts;
/** Found on the first while loop the function emits. */
/** Found on the first while or for loop the function emits. */
versions: Map<IrStmt, LoopEntryVersion> | null;
/** Loops whose copies are being emitted: a loop is versioned once. */
readonly versioning: Set<IrStmt>;
@@ -61,7 +61,7 @@ export function beginLoopEntryVersions(
* its locals: plain double slots of the current function, outside a
* speculative copy and a coroutine. */
export function loopEntryVersion(emitter: LlEmitter, loop: IrStmt): LoopEntryVersion | null {
if (loop.kind !== "while") return null;
if (loop.kind !== "while" && loop.kind !== "for") return null;
const state = ENTRY_VERSIONS.of(emitter);
if (state.fn === null || state.facts === null || state.versioning.has(loop)) return null;
if (
packages/compiler/src/backend/llvm/nullable-unions.test.ts
+16
View File
@@ -130,3 +130,19 @@ test("function arms are nullable pointers compared by identity root", () => {
expect(isPointerIdentityArm(single)).toBe(false);
expect(unions.armTypes().get("fu")).toEqual(fn);
});
test("typed array arms are nullable pointers compared by identity", () => {
const i32: IrType = { kind: "bytes", elem: "i32" };
const unions = new NullableUnions(
module([
{ id: "iu", arms: [i32, UNDEFINED_T] },
{ id: "nin", arms: [NULL_T, i32, UNDEFINED_T] },
{ id: "if", arms: [i32, F64] },
]),
);
expect(unions.single("iu")).toMatchObject({ refTag: 0, unitTag: 1, nullTag: -1, arm: i32 });
expect(unions.single("nin")).toMatchObject({ refTag: 1, unitTag: 2, nullTag: 0, arm: i32 });
expect(unions.get("if")).toBeNull();
expect(isObjectArm(unions.single("iu")!)).toBe(true);
expect(isPointerIdentityArm(unions.single("iu")!)).toBe(true);
});
packages/compiler/src/backend/llvm/nullable-unions.ts
+13 -7
View File
@@ -88,17 +88,17 @@ export const NULLABLE_NULL = "@sc_nullable_null";
export const NULLABLE_ABSENT = "@sc_nullable_absent";
/** The reference arms whose instances are never NULL. Class and record
* entry points all tolerate NULL; strings, Stats snapshots, arrays, maps and
* sets retain through a NULL-skipping wrapper (shapes.ts), as do closures
* (function arms). Object arms are always truthy and compare by identity
* (function arms by identity root); string arms are truthy when non-empty
* and compare bytes. */
* entry points all tolerate NULL; strings, Stats snapshots, arrays, typed
* arrays, maps and sets retain through a NULL-skipping wrapper (shapes.ts),
* as do closures (function arms). Object arms are always truthy and compare
* by identity (function arms by identity root); string arms are truthy when
* non-empty and compare bytes. */
export type NullableRefArm =
| { kind: "object"; className: string }
| { kind: "record"; shapeId: string }
| { kind: "string" }
| { kind: "stats" }
| (IrType & { kind: "array" | "map" | "set" | "func" });
| (IrType & { kind: "array" | "bytes" | "map" | "set" | "func" });
/** The arm is a JS object: always truthy. */
export function isObjectArm(nullable: PointerUnion): boolean {
@@ -273,7 +273,13 @@ function nullableShape(
return { ...base, arm: { kind: "record", shapeId: arm.shapeId } };
if (arm.kind === "string") return { ...base, arm: { kind: "string" } };
if (arm.kind === "stats") return { ...base, arm: { kind: "stats" } };
if (arm.kind === "array" || arm.kind === "map" || arm.kind === "set" || arm.kind === "func")
if (
arm.kind === "array" ||
arm.kind === "bytes" ||
arm.kind === "map" ||
arm.kind === "set" ||
arm.kind === "func"
)
return { ...base, arm };
return null;
}
packages/compiler/src/backend/llvm/record-views.test.ts
+130
View File
@@ -0,0 +1,130 @@
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, test } from "vitest";
import { compile, deserializeModule } from "../../index.js";
import type { IrModule } from "../../ir/ir.js";
import { computeTraced } from "../cycle-analysis.js";
import { planRecordViews } from "./record-views.js";
/** The views planned for `source`'s module, as target field names → source
* field names (each list sorted). */
async function plannedViews(source: string): Promise<Map<string, string[]>> {
const dir = await mkdtemp(join(tmpdir(), "scriptc-record-views-"));
try {
const entry = join(dir, "main.ts"),
outPath = join(dir, "main.ir.json");
await writeFile(entry, source);
const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" });
if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n"));
const mod: IrModule = deserializeModule(await readFile(outPath, "utf8"));
const traced = computeTraced(mod).shapes;
const views = planRecordViews(mod, {
traced: (id) => traced.has(`record:${id}`),
excluded: () => false,
});
const names = new Map(
(mod.records ?? []).map((shape) => [shape.id, shape.fields.map((f) => f.name).join(",")]),
);
return new Map(
[...views.targets].map((target) => [
names.get(target)!,
views.sources(target).map((id) => names.get(id)!),
]),
);
} finally {
await rm(dir, { recursive: true, force: true });
}
}
const conversions = `
const o: unknown = wide;
const v = o as AC;
v.a = 2;
console.log(wide.a, v.a, o === v);
`;
test("a checked conversion of a live record capsule into a record type it covers plans a view", async () => {
const views = await plannedViews(`
type Wide = { a: number; b: string; c: boolean };
type AC = { a: number; c: boolean };
const wide: Wide = { a: 1, b: "x", c: true };
${conversions}`);
expect(views).toEqual(new Map([["a,c", ["a,b,c"]]]));
});
test("a source whose fields can hold the target type keeps the copy", async () => {
// A view stored in its own source would be a cycle through the view's
// capsule reference, which the cycle analysis does not see.
const views = await plannedViews(`
type AC = { a: number; c: boolean };
type Holder = { items: AC[] };
type Wide = { a: number; b: string; c: boolean; self: AC | undefined; holder: Holder };
const wide: Wide = { a: 1, b: "x", c: true, self: undefined, holder: { items: [] } };
${conversions}
wide.self = v;
wide.holder.items.push(v);`);
expect(views).toEqual(new Map());
});
test("a source that holds the target type through a class instance keeps the copy", async () => {
const views = await plannedViews(`
type AC = { a: number; c: boolean };
class Base { note = 0; }
class Keeper extends Base { kept: AC | undefined = undefined; }
type Wide = { a: number; b: string; c: boolean; keeper: Base };
const wide: Wide = { a: 1, b: "x", c: true, keeper: new Keeper() };
${conversions}`);
expect(views).toEqual(new Map());
});
test("a source that holds itself still views a type it cannot hold", async () => {
const views = await plannedViews(`
type Tree = { a: number; b: string; c: boolean; children: Tree[] };
type AC = { a: number; c: boolean };
const wide: Tree = { a: 1, b: "x", c: true, children: [] };
wide.children.push({ a: 2, b: "y", c: false, children: [] });
${conversions}`);
expect(views).toEqual(new Map([["a,c", ["a,b,c,children"]]]));
});
test("of two sources that can hold each other's views, one keeps the copy", async () => {
// Each source only holds the other's target type, but a view of that type
// names the other source, so the two views could form a cycle. Without
// Parent's view, Kid holds only a copy and its own view is safe.
const views = await plannedViews(`
type PView = { id: number; name: string };
type KView = { id: number; tag: string };
type Parent = { id: number; name: string; kids: KView[] };
type Kid = { id: number; tag: string; parent: PView | undefined };
const parent: Parent = { id: 1, name: "p", kids: [] };
const kid: Kid = { id: 2, tag: "k", parent: undefined };
const p: unknown = parent;
const k: unknown = kid;
kid.parent = p as PView;
parent.kids.push(k as KView);
console.log(parent.kids.length, kid.parent === p);`);
expect(views).toEqual(new Map([["id,tag", ["id,parent,tag"]]]));
});
test("a source with a function field keeps the copy", async () => {
// A function can capture anything, the view included.
const views = await plannedViews(`
type AC = { a: number; c: boolean };
type Wide = { a: number; b: string; c: boolean; f: () => number };
const wide: Wide = { a: 1, b: "x", c: true, f: () => 1 };
${conversions}`);
expect(views).toEqual(new Map());
});
test("a source that may be frozen keeps the copy", async () => {
// Writes through a view take the target type's stores, which have no
// frozen-write guard.
const views = await plannedViews(`
type Wide = { a: number; b: string; c: boolean };
type AC = { a: number; c: boolean };
const wide: Wide = { a: 1, b: "x", c: true };
${conversions}
Object.freeze(wide);`);
expect(views).toEqual(new Map());
});
packages/compiler/src/backend/llvm/record-views.ts
+268
View File
@@ -0,0 +1,268 @@
/* Record views: one object across record layouts (decision D3).
*
* A checked conversion of a capsule of a native record A into another
* record type B whose fields A also has (same names and types, any
* offsets) answers a view: a B object, its rc word marked with the frozen
* bit, whose record-source entry names A and A's byte offsets for B's
* fields (scr_dyn_record_view). Every field address of a B value tests
* the mark (recordViewFieldPtr) and takes A's slot for a view, so reads
* and writes through either name reach the same storage, and boxing a
* view answers A's capsule, so identity holds too. B's storage itself is
* unchanged.
*
* Only the target types of such conversions carry the test. They are
* found from the IR, before any code is emitted: B is a record type a
* checked conversion can produce (the closure of dynCheck targets), and
* some other record type whose values reach untyped code as live capsules
* (the closure of live dynFrom types) has all of B's fields. A program
* without checked conversions has none, so its code is unchanged. */
import {
isDirentShape,
KEY_ORDER_FIELD,
shapeHasAccessorSlots,
shapeKeyOrders,
typeKey,
type IrModule,
type IrRecordShape,
type IrType,
} from "../../ir/ir.js";
import { everyStmtList } from "../../ir/traverse.js";
import type { BlockBuilder } from "./blocks.js";
export interface RecordViews {
/** Shape ids whose values may be views. */
readonly targets: ReadonlySet<string>;
/** The record shapes whose capsules convert to a view of `target`. */
sources(target: string): readonly string[];
}
export const NO_RECORD_VIEWS: RecordViews = { targets: new Set(), sources: () => [] };
interface ViewOptions {
readonly traced: (shapeId: string) => boolean;
readonly excluded: (shapeId: string) => boolean;
}
/** Value kinds that hold no reference to a record. */
const LEAF_KINDS: ReadonlySet<IrType["kind"]> = new Set([
"f64",
"bool",
"string",
"bigint",
"date",
"regex",
"bytes",
"symbol",
"url",
"searchParams",
"stats",
"undefinedT",
"nullT",
"void",
]);
function plainShape(shape: IrRecordShape): boolean {
return (
!shape.tuple &&
!shape.indexValue &&
shape.fields.length > 0 &&
!isDirentShape(shape) &&
!shapeHasAccessorSlots(shape) &&
shapeKeyOrders(shape).length === 1 &&
!shape.fields.some((field) => field.name === KEY_ORDER_FIELD || field.name.startsWith("%"))
);
}
/** Whether every field of `target` is a field of `source` with the same
* type (the view reads A's field with B's representation). */
function covers(source: IrRecordShape, target: IrRecordShape): boolean {
const fields = new Map(source.fields.map((field) => [field.name, typeKey(field.type)]));
return target.fields.every((field) => fields.get(field.name) === typeKey(field.type));
}
export function planRecordViews(mod: IrModule, options: ViewOptions): RecordViews {
const shapes = new Map((mod.records ?? []).map((shape) => [shape.id, shape]));
const unions = new Map((mod.unions ?? []).map((union) => [union.id, union]));
// Targets: record types a checked conversion produces. Sources: record
// types whose values reach untyped code as live capsules. A view arises
// only where the two meet; elsewhere a conversion keeps its copy.
const closure = (roots: IrType[]): Set<string> => {
const out = new Set<string>();
const seen = new Set<string>();
const visit = (t: IrType): void => {
const key = typeKey(t);
if (seen.has(key)) return;
seen.add(key);
switch (t.kind) {
case "record": {
out.add(t.shapeId);
for (const field of shapes.get(t.shapeId)?.fields ?? []) visit(field.type);
return;
}
case "array":
visit(t.elem);
return;
case "union":
for (const arm of unions.get(t.unionId)?.arms ?? []) visit(arm);
return;
default:
return;
}
};
for (const root of roots) visit(root);
return out;
};
// The views planned so far: target shape → the sources it may view.
const sources = new Map<string, string[]>();
// Whether a value of a record can hold a value of another record type:
// through its fields, elements, map entries, union arms and overflow
// properties, and through the record a view of it names. Class instances,
// functions, untyped values, promises and the other kinds that can
// reference arbitrary objects count as holding every type.
const held = new Map<string, Set<string> | "any">();
const holds = (shapeId: string): Set<string> | "any" => {
const known = held.get(shapeId);
if (known) return known;
const out = new Set<string>();
const seen = new Set<string>();
const visit = (t: IrType): boolean => {
const key = typeKey(t);
if (seen.has(key)) return true;
seen.add(key);
switch (t.kind) {
case "record": {
out.add(t.shapeId);
const shape = shapes.get(t.shapeId);
if (shape?.indexValue !== undefined && !visit(shape.indexValue)) return false;
if (!(sources.get(t.shapeId) ?? []).every((id) => visit({ kind: "record", shapeId: id })))
return false;
return (shape?.fields ?? []).every((field) => visit(field.type));
}
case "array":
case "set":
return visit(t.elem);
case "map":
return visit(t.key) && visit(t.value);
case "union":
return (unions.get(t.unionId)?.arms ?? []).every(visit);
default:
return LEAF_KINDS.has(t.kind);
}
};
const result = visit({ kind: "record", shapeId }) ? out : "any";
held.set(shapeId, result);
return result;
};
const checked: IrType[] = [];
const boxed: IrType[] = [];
for (const fn of mod.functions)
everyStmtList(fn.body, {
stmt: () => true,
expr: (e) => {
if (e.kind === "dynCheck") checked.push(e.type);
else if (e.kind === "dynFrom" && e.liveRef === true) boxed.push(e.value.type);
return true;
},
});
if (checked.length === 0 || boxed.length === 0) return NO_RECORD_VIEWS;
const reachable = closure(checked);
const live = closure(boxed);
if (reachable.size === 0) return NO_RECORD_VIEWS;
for (const id of reachable) {
const target = shapes.get(id);
if (!target || !plainShape(target) || target.freezable === true) continue;
if (options.traced(id) || options.excluded(id)) continue;
const found: string[] = [];
for (const source of shapes.values()) {
if (source.id === id || !live.has(source.id) || !plainShape(source)) continue;
// A view writes the source's slots through the target type, which has
// no frozen-write guard, so a source that may be frozen keeps the copy.
if (source.freezable === true) continue;
if (covers(source, target)) found.push(source.id);
}
if (found.length > 0) sources.set(id, found);
}
// A view's record-source entry owns the source's capsule, an edge the
// cycle analysis does not see. A view stays off every cycle only when its
// source cannot hold a value of the target type, such as the view itself,
// including through other views. Dropping a pair removes edges, so the
// check repeats until no pair is dropped.
for (let changed = true; changed;) {
changed = false;
held.clear();
for (const [id, list] of sources) {
const kept = list.filter((source) => {
const sourceHolds = holds(source);
return sourceHolds !== "any" && !sourceHolds.has(id);
});
if (kept.length === list.length) continue;
changed = true;
if (kept.length > 0) sources.set(id, kept);
else sources.delete(id);
}
}
if (sources.size === 0) return NO_RECORD_VIEWS;
const targets = new Set(sources.keys());
return { targets, sources: (target) => sources.get(target) ?? [] };
}
/** Lines setting `dest` (i1) to whether the record `obj` of a view target
* type is a view. A view's rc word carries the frozen bit (scr_rc.h
* SCR_RC_FROZEN), which no other value of such a type has: view targets
* are never freezable, and a word with the program-lifetime bit (an
* immortal one included) is no view, so a view's word is the frozen bit
* plus a count. The
*/
export function recordViewTestLines(sizeType: "i32" | "i64", obj: string, dest: string): string[] {
// The frozen bit with a count and no program-lifetime bit: one range.
const [bit, span] =
sizeType === "i64" ? ["4611686018427387904", "4294967296"] : ["1073741824", "1073741824"];
return [
` ${dest}.w = load ${sizeType}, ptr ${obj}`,
` ${dest}.d = sub ${sizeType} ${dest}.w, ${bit}`,
` ${dest} = icmp ult ${sizeType} ${dest}.d, ${span}`,
];
}
/** The address of field `position` (declaration index) of the record
* `obj` of view-target shape `struct`: the slot itself, or the viewed
* record's slot when `obj` is a view. Optimized builds test the mark
* inline and look the slot up out of line; unoptimized builds, which give
* every value a frame slot, make the address one call. */
export function recordViewFieldPtr(
B: BlockBuilder,
sizeType: "i32" | "i64",
struct: string,
obj: string,
slotIndex: number,
position: number,
declare: (decl: string) => void,
outOfLine: boolean,
): string {
if (outOfLine) {
declare(`declare ptr @scr_record_view_field(ptr, ${sizeType}, ${sizeType})`);
const ptr = B.tmp();
B.line(
`${ptr} = call ptr @scr_record_view_field(ptr ${obj}, ${sizeType} ptrtoint (ptr getelementptr (%${struct}, ptr null, i32 0, i32 ${slotIndex}) to ${sizeType}), ${sizeType} ${position})`,
);
return ptr;
}
const own = B.tmp();
B.line(`${own} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${slotIndex}`);
const isView = B.tmp();
for (const line of recordViewTestLines(sizeType, obj, isView)) B.line(line.trimStart());
const from = B.currentLabel();
const view = B.newLabel("rv.view");
const join = B.newLabel("rv.join");
// Views are rare: the slot lookup stays out of line.
B.condBr(isView, view, join, `, !prof !{!"branch_weights", i32 1, i32 1000}`);
B.startBlock(view);
declare(`declare ptr @scr_record_view_slot(ptr, ${sizeType})`);
const viewed = B.tmp();
B.line(`${viewed} = call ptr @scr_record_view_slot(ptr ${obj}, ${sizeType} ${position})`);
B.br(join);
B.startBlock(join);
const ptr = B.tmp();
B.line(`${ptr} = phi ptr [ ${own}, %${from} ], [ ${viewed}, %${view} ]`);
return ptr;
}
packages/compiler/src/backend/llvm/repr-report.test.ts
+5 -5
View File
@@ -50,11 +50,11 @@ console.log(scale(boxes[1]!.id), scale(3, 4), half(boxes.length), boxes[2]!.item
expect(row("field", "Box", "hits")?.[4]).toBe("double");
expect(row("field", "Box", "hits")?.[5]).not.toBe("");
expect(row("field", "Box", "items")?.slice(3, 5)).toEqual(["array<f64>", "ptr"]);
expect(row("param", "scale", "value")?.[4]).toBe("i32");
// A defaulted parameter is a `number | undefined` box; the variant for
// calls that pass a number takes an i32.
expect(row("param", "scale", "by")?.slice(3, 5)).toEqual(["f64 | undefined", "box"]);
expect(row("param", "scale.nativeNumber", "by")?.[4]).toBe("i32");
// Both calls of `scale` take its variant, whose defaulted parameter is
// a number: the one that omits it passes the default.
expect(row("param", "scale", "by")).toBeUndefined();
expect(row("param", "scale.nativeNumber", "value")?.[4]).toBe("i32");
expect(row("param", "scale.nativeNumber", "by")?.slice(3, 5)).toEqual(["f64", "i32"]);
// An array length may exceed the int32 range.
expect(row("param", "half", "value")?.[4]).toBe("double");
expect(row("param", "half", "value")?.[5]).toMatch(/^argument in .* is not int32$/);
packages/compiler/src/backend/llvm/shapes.ts
+35 -7
View File
@@ -69,6 +69,7 @@ import {
disposeFieldLines,
plainRefType,
} from "./arenas.js";
import { recordViewTestLines, type RecordViews } from "./record-views.js";
import { recordMayKeepSource } from "./record-sources.js";
import { LlvmUnsupportedError } from "./unsupported.js";
@@ -151,7 +152,12 @@ export interface ShapeHost {
/** Record shapes and classes with an arena-aware constructor
* (arenas.ts). */
arenaShapes?: ReadonlySet<string>;
/** Record view targets and their sources (record-views.ts). */
recordViews?: RecordViews;
arenaClasses?: ReadonlySet<string>;
/** Every arena the program opens is closed (ArenaPlan.allClosed): the
* arena-aware constructors take no dispose list. */
arenasClosed?: boolean;
/** The target C ABI's `size_t` / refcount integer type. */
readonly sizeType: "i32" | "i64";
/** Worker executables: the inline allocator state is thread-local. */
@@ -448,7 +454,11 @@ export function emitShapeUnionHelpers(host: ShapeHost): string[] {
const traced = union.arms.map((arm) => host.tracedShapes.has(`record:${arm.shapeId}`));
// A freezable arm's count is the word without its frozen bit, as in
// releaseFastPath: a frozen record's last reference takes the slow path.
const frozen = union.arms.some((arm) => host.recordsById.get(arm.shapeId)?.freezable === true);
const frozen = union.arms.some(
(arm) =>
host.recordsById.get(arm.shapeId)?.freezable === true ||
host.recordViews?.targets.has(arm.shapeId) === true,
);
const notFrozen = ~recordFrozenBit(S);
const release = shapeReleaseFn(union.unionId);
if (traced.every((t) => t === traced[0]))
@@ -2288,7 +2298,9 @@ export function emitRecordShapes(
false,
chained,
segmented,
shape.freezable === true,
// A record view's word carries the frozen bit as its view mark
// (record-views.ts), so view targets release like freezable shapes.
shape.freezable === true || host.recordViews?.targets.has(shape.id) === true,
iterative
? iterativeTeardownDefinitions(
host,
@@ -2347,7 +2359,8 @@ export function emitRecordShapes(
defs.push(...nw);
if (host.arenaShapes?.has(shape.id) === true && !shape.indexValue) {
const newSym = mangleRecordNew(shape.id);
const dispose = refMembers.length > 0 ? arenaDisposeSym(newSym) : null;
const dispose =
refMembers.length > 0 && host.arenasClosed !== true ? arenaDisposeSym(newSym) : null;
const init = [` store ${host.sizeType} ${ARENA_RC}, ptr %o`];
if (shapeNumber !== 0)
init.push(
@@ -2386,10 +2399,25 @@ export function emitRecordShapes(
const index = i + base;
const inline = host.inlineUnions.record(field.type);
const fieldTy = llFieldType(field.type, inline);
clone.push(
` %sp${i} = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${index}`,
` %sv${i} = load ${fieldTy}, ptr %sp${i}`,
);
if (host.recordViews?.targets.has(shape.id) === true) {
// A view's fields live in the viewed record (record-views.ts).
host.declare(`declare ptr @scr_record_view_slot(ptr, ${host.sizeType})`);
clone.push(
` %so${i} = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${index}`,
...recordViewTestLines(host.sizeType, "%src", `%sr${i}v`),
` br i1 %sr${i}v, label %sv${i}.view, label %sv${i}.join`,
`sv${i}.view:`,
` %sw${i} = call ptr @scr_record_view_slot(ptr %src, ${host.sizeType} ${i})`,
` br label %sv${i}.join`,
`sv${i}.join:`,
` %sp${i} = phi ptr [ %so${i}, %${i === 0 ? "entry" : `sv${i - 1}.join`} ], [ %sw${i}, %sv${i}.view ]`,
` %sv${i} = load ${fieldTy}, ptr %sp${i}`,
);
} else
clone.push(
` %sp${i} = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${index}`,
` %sv${i} = load ${fieldTy}, ptr %sp${i}`,
);
const owned = isRefCounted(field.type) && !inline;
const stored = owned ? `%sr${i}` : `%sv${i}`;
if (owned) {
packages/compiler/src/backend/llvm/speculative-integers.ts
+24 -9
View File
@@ -773,6 +773,9 @@ export interface ProvenLoopPlan {
locals: Map<string, IntegerRange>;
/** Statement-level writes computed as integers. */
writes: Set<IrStmt>;
/** Expression-position `++`/`--` of shadowed bindings (`out[o++] = v`),
* computed on the shadows. */
increments: Set<IrExpr>;
}
/** Plan a proven integer loop. `cursors` are the loop's bounded bindings,
@@ -791,10 +794,17 @@ export function planProvenIntegerLoop(
): ProvenLoopPlan | null {
if (cursors.size === 0) return null;
const writes = new Map<string, IrStmt[]>();
const increments = new Map<string, IrExpr[]>();
const refused = new Set<string>();
let rejected = false;
const expr = (e: IrExpr): boolean => {
if (e.kind === "assignExpr" || e.kind === "incDec") refused.add(e.localId);
// Every value a cursor holds in the loop lies in its loop range, so an
// increment of one writes an integer of that range.
if (e.kind === "incDec" && cursors.has(e.localId)) {
const list = increments.get(e.localId) ?? [];
list.push(e);
increments.set(e.localId, list);
} else if (e.kind === "assignExpr" || e.kind === "incDec") refused.add(e.localId);
return everyExprChild(e, expr, nested);
};
const nested = (s: IrStmt): boolean => {
@@ -827,12 +837,13 @@ export function planProvenIntegerLoop(
for (const s of loop.body) stmt(s);
if (rejected) return null;
const locals = new Map<string, IntegerRange>();
for (const [id, list] of writes) {
for (const id of new Set([...writes.keys(), ...increments.keys()])) {
if (refused.has(id) || !eligible(id)) continue;
const exact = list.every((s) => {
const value = speculativeWriteTarget(s)!.value!;
return isSpeculableValue(value, false) && proven(value);
});
const exact =
(writes.get(id) ?? []).every((s) => {
const value = speculativeWriteTarget(s)!.value!;
return isSpeculableValue(value, false) && proven(value);
}) && (increments.get(id) ?? []).every(proven);
if (exact) locals.set(id, cursors.get(id)!);
}
// Every binding a shadowed write reads is shadowed, already an integer,
@@ -844,15 +855,19 @@ export function planProvenIntegerLoop(
for (let changed = true; changed;) {
changed = false;
for (const id of [...locals.keys()])
if (!writes.get(id)!.every((s) => grounded(speculativeWriteTarget(s)!.value!))) {
if (!(writes.get(id) ?? []).every((s) => grounded(speculativeWriteTarget(s)!.value!))) {
locals.delete(id);
changed = true;
}
}
if (locals.size === 0) return null;
const planned = new Set<IrStmt>();
for (const id of locals.keys()) for (const s of writes.get(id)!) planned.add(s);
return { locals, writes: planned };
const plannedIncrements = new Set<IrExpr>();
for (const id of locals.keys()) {
for (const s of writes.get(id) ?? []) planned.add(s);
for (const e of increments.get(id) ?? []) plannedIncrements.add(e);
}
return { locals, writes: planned, increments: plannedIncrements };
}
const SAFE = Number.MAX_SAFE_INTEGER;
packages/compiler/src/backend/llvm/split.test.ts
+27
View File
@@ -324,6 +324,33 @@ test("debug metadata stays on definitions when splitting LLVM modules", async ()
}
});
test("personality routines stay on definitions when splitting LLVM modules", async () => {
// Functions with landing pads (Linux unwinding) carry a personality; every
// other shard declares them, and a declaration must not repeat it.
const source =
SAMPLE.replaceAll(") #0 {", ") #0 personality ptr @__gcc_personality_v0 {") +
"declare i32 @__gcc_personality_v0(...)\n";
const split = splitLlvmProgram(source, { minimumBytes: 0 });
expect(split).not.toBeNull();
const dir = await mkdtemp(join(tmpdir(), "scriptc-personality-split-"));
scratch.push(dir);
for (const shard of split!.shards) {
expect(shard.source()).not.toMatch(/^declare.* personality ptr/m);
const path = join(dir, shard.name);
await writeFile(path, shard.source());
const result = spawnSync("clang", ["-Wno-override-module", "-c", path, "-o", `${path}.o`], {
encoding: "utf8",
});
expect(result.status, result.stderr).toBe(0);
expect(result.stderr).toBe("");
}
expect(
split!.shards.flatMap((shard) => [
...shard.source().matchAll(/^define .* personality ptr @__gcc_personality_v0 \{/gm),
]).length,
).toBe(3);
});
test("dev libraries split at the measured 2MB crossover while executables retain 4MB", () => {
const body = Array.from(
{ length: 700 },
packages/compiler/src/backend/llvm/split.ts
+6 -1
View File
@@ -104,7 +104,12 @@ function functionHeader(
// attributes, zeroext, and varargs stay byte-exact.
// A definition's DISubprogram cannot be attached to a declaration in a
// different shard. Its owning definition retains the original metadata.
const attrs = match[5]!.replace(/\s+!dbg !\d+/g, "");
// A personality routine belongs to a definition too: LLVM rejects it on a
// declaration, which would fail every shard that calls a function with
// landing pads (Linux unwinding) and fall back to the single-TU compile.
const attrs = match[5]!
.replace(/\s+!dbg !\d+/g, "")
.replace(/\s+personality\s+ptr\s+@[-$._A-Za-z][-$.\w]*/g, "");
const declaration = `declare ${linkage}${match[2]} ${match[3]}(${match[4]})${attrs}`.trimEnd();
return { line, declaration, symbol, promoted };
}
packages/compiler/src/backend/llvm/stack-objects.test.ts
+156
View File
@@ -337,3 +337,159 @@ test("spread updates copy into the binding's slot and release the replaced field
/(%t\d+) = load ptr, ptr %s\d+\n\s+(%t\d+) = load %sc_rs_\w+, ptr %s\d+\n\s+store %sc_rs_\w+ \2, ptr %s\d+\n(?:.*\n)*?\s+call void @\w+\(ptr \1\)/,
);
});
const COLLECTORS = `
class Tag { readonly name: string; constructor(name: string) { this.name = name; } }
class Collector {
readonly items: string[] = [];
last: string | undefined = undefined;
tag: Tag | null = null;
count = 0;
add(item: string): void { this.items.push(item); this.last = item; this.count++; }
label(name: string): void { this.tag = new Tag(name); }
}
function fill(c: Collector, n: number): void { for (let i = 0; i < n; i++) c.add("w" + i); }
const kept: Collector[] = [];
let outer: Collector | null = null;
export function local(n: number): number {
const c = new Collector();
fill(c, n);
c.label("x");
return c.count + c.items.length + (c.tag?.name.length ?? 0);
}
export function stored(n: number): number {
const c = new Collector();
fill(c, n);
kept.push(c);
return c.count;
}
export function assigned(n: number): number {
const c = new Collector();
outer = c;
fill(c, n);
return c.count;
}
export function returned(n: number): Collector {
const c = new Collector();
fill(c, n);
return c;
}
export function inCase(n: number): number {
switch (n) {
case 1:
const c = new Collector();
fill(c, n);
return c.count;
default:
return 0;
}
}
class Link { next: Link | null = null; readonly id: number; constructor(id: number) { this.id = id; } }
export function chain(n: number): number {
const head = new Link(0);
let tail = new Link(1);
head.next = tail;
for (let i = 2; i < n; i++) { const l = new Link(i); tail.next = l; tail = l; }
tail.next = head.next;
return head.next.id + (head.next.next?.id ?? 0);
}
console.log(local(2), stored(1), assigned(1), returned(1).count, inCase(1), chain(4), kept.length, outer !== null);
`;
test("class instances that own references live in frame storage while they stay in the frame", async () => {
const ir = emitLlvmModule(await moduleFor(COLLECTORS));
const local = body(ir, "local");
expect(local).toMatch(/alloca %sc_o_Collector, align 8/);
expect(local).not.toContain("@sc_new_Collector()");
expect(local).toContain("store i64 -1, ptr");
// The frame releases the reference fields when the binding's scope exits.
expect(local).toMatch(/call void @scr_arr_release\(/);
// A cycle-capable class keeps its collector header in front of the object.
const chain = body(ir, "chain");
expect(chain).toMatch(/alloca \{ \[8 x i8\], %sc_o_Link \}, align 8/);
expect(chain).toContain("store i32 -1, ptr");
// Stored, assigned to another binding, returned, or declared in a switch
// case: the instance stays on the heap.
for (const name of ["stored", "assigned", "returned", "inCase"]) {
const fn = body(ir, name);
expect(fn, name).not.toMatch(/alloca %sc_o_Collector/);
expect(fn, name).toContain("@sc_new_Collector()");
}
});
test("fresh instances passed in place and instances captured by invoked closures live in frame storage", async () => {
const ir = emitLlvmModule(
await moduleFor(`
class Checker {
readonly names: string[];
readonly seen = new Set<string>();
constructor(names: string[]) { this.names = names; }
check(): number { let d = 0; for (const n of this.names) { if (this.seen.has(n)) d++; this.seen.add(n); } return d; }
}
function each(xs: number[], f: (x: number) => void): void { for (const x of xs) f(x); }
class Totals {
readonly seen: number[] = [];
total = 0;
add(x: number): void { this.total += x; this.seen.push(x); }
addAll(xs: number[]): void { each(xs, (x) => this.add(x)); }
}
const handlers: ((x: number) => void)[] = [];
export function inPlace(names: string[]): number { return new Checker(names).check(); }
export function captured(xs: number[]): number {
const t = new Totals();
t.addAll(xs);
each(xs, (x) => t.add(x * 2));
return t.total;
}
export function kept(xs: number[]): number {
const t = new Totals();
handlers.push((x) => t.add(x));
return t.total;
}
console.log(inPlace(["a", "a"]), captured([1, 2]), kept([3]), handlers.length);
`),
);
const inPlace = body(ir, "inPlace");
expect(inPlace).toMatch(/alloca %sc_o_Checker, align 8/);
expect(inPlace).not.toContain("@sc_new_Checker()");
const captured = body(ir, "captured");
expect(captured).toMatch(/alloca %sc_o_Totals, align 8/);
expect(captured).not.toContain("@sc_new_Totals()");
// A closure that outlives the call keeps its captured instance on the heap.
const kept = body(ir, "kept");
expect(kept).not.toMatch(/alloca %sc_o_Totals/);
expect(kept).toContain("@sc_new_Totals()");
});
test("a nullable class field in frame storage is released as the class pointer it holds", async () => {
const ir = emitLlvmModule(
await moduleFor(`
class Tag { readonly name: string; constructor(name: string) { this.name = name; } }
const seen = new Set<Tag | null>();
const counts = new Map<Tag | null, number>();
class Holder {
tag: Tag | null = null;
set(name: string): void { this.tag = new Tag(name); }
}
export function label(names: string[]): string {
const h = new Holder();
for (const n of names) h.set(n);
seen.add(h.tag);
counts.set(h.tag, 1);
return h.tag?.name ?? "-";
}
console.log(label(["a"]), seen.size, counts.size);
`),
);
const label = body(ir, "label");
expect(label).toMatch(/alloca %sc_o_Holder, align 8/);
// The union is boxed where it is a Set element or Map key, but the field
// keeps the raw pointer: its release at scope exit is the class release.
const field = /(%s\d+) = getelementptr inbounds %sc_o_Holder, ptr %s\d+, i64 0, i32 1/.exec(
label,
)![1]!;
const release = new RegExp(
`(%t\\d+) = load ptr, ptr ${field}\\n\\s+call void (@\\w+)\\(ptr \\1\\)`,
).exec(label);
expect(release?.[2]).toBe("@sc_release_Tag");
});
packages/compiler/src/backend/llvm/stack-objects.ts
+112 -21
View File
@@ -1,3 +1,4 @@
import { constructorEscapePoint } from "../cycle-mutability.js";
import {
VOID,
isRefCounted,
@@ -15,8 +16,8 @@ import {
} from "./record-results.js";
import { everyStmtList } from "../../ir/traverse.js";
import { mangleClassStruct, mangleRecordStruct } from "../mangle.js";
import { undefinedArmTag } from "../../ir/analysis.js";
import { classFieldIndex } from "./classes.js";
import { classFieldIndex, classUndefinedFieldInits } from "./classes.js";
import { CYC_BUF, CYC_HDR } from "./alloc.js";
import type { CallLifetimes } from "./call-lifetimes.js";
import { emitCallExpr, emitConstruction } from "./expr-calls.js";
import type { LlValue, LlvmEmitterContext } from "./expr-context.js";
@@ -83,8 +84,39 @@ export function canStackObject(host: LlvmEmitterContext, e: IrExpr): e is StackO
);
}
/** A standalone class whose instance can live in the frame of a binding
* that frame-escapes.ts proves stays in the frame. Unlike
* scalarStandaloneClass, its fields may own references: the frame owns
* them and releases them when the binding's scope exits. Classes in a
* hierarchy keep heap storage, as do the kinds canStackObject refuses. */
export function frameClass(host: LlvmEmitterContext, className: string): boolean {
const meta = host.classMeta.get(className);
const ctor = host.fnByName.get(`%${className}.constructor`);
if (meta === undefined || ctor === undefined) return false;
const cls = meta.def;
if (
meta.hierarchy ||
meta.compact ||
meta.threadSlot === true ||
host.programLifetime.has(className) ||
host.publishedTypes?.classGuarded(className) === true ||
cls.runtime ||
cls.base !== undefined ||
cls.arrayElem !== undefined ||
cls.localCaptures !== undefined ||
cls.tracksOwnFields ||
(cls.symbolFields?.length ?? 0) > 0 ||
ctor.async ||
ctor.generator
)
return false;
const self = ctor.params[0]?.localId;
return self !== undefined && constructorEscapePoint(ctor, self, null) === ctor.body.length;
}
/** Whether a frame object of this type has fields that own references. */
function ownsReferences(host: LlvmEmitterContext, type: IrType): boolean {
if (type.kind === "object") return classReferenceFields(host, type.className).length > 0;
return (
type.kind === "record" &&
host
@@ -93,6 +125,30 @@ function ownsReferences(host: LlvmEmitterContext, type: IrType): boolean {
);
}
/** The fields of a frame class that own references: refcounted, not inline
* unions, and not compressed references to program-lifetime instances.
* Each has its storage type: a nullable class field holds the class pointer
* itself, whatever form its union takes elsewhere (nullable-fields.ts). */
function classReferenceFields(
host: LlvmEmitterContext,
className: string,
): { name: string; index: number; type: IrType }[] {
const meta = host.classMeta.get(className);
if (meta === undefined) return [];
const out: { name: string; index: number; type: IrType }[] = [];
for (const field of meta.def.fields) {
if (!isRefCounted(field.type) || host.inlineUnions.classField(className, field.name)) continue;
const { index, compressed } = classFieldIndex(meta, field.name, "compressed");
if (compressed) continue;
out.push({
name: field.name,
index,
type: host.nullableFields.storageType(className, field.name, field.type),
});
}
return out;
}
/** Locals declared directly in a switch case. The cases share one scope
* whose exit runs whichever case was entered, so a scope-owned field would
* be released without its declaration having run. */
@@ -112,7 +168,10 @@ function caseScopedLocals(fn: IrFunction): Set<string> {
/** Locals initialized once by a stack-eligible object and only projected
* afterwards (CallLifetimes.locals): field reads, and arguments to
* parameters that are themselves only projected. */
* parameters that are themselves only projected. Also locals initialized
* once by a `new` of a frame class (frameClass) whose value stays in the
* frame (frame-escapes.ts): their methods and callees may store into the
* instance. */
export function findLocalStackObjects(
host: LlvmEmitterContext,
fn: IrFunction,
@@ -120,19 +179,28 @@ export function findLocalStackObjects(
): Map<string, StackObjectSource> {
const result = new Map<string, StackObjectSource>();
const safe = lifetimes.locals.get(fn.name);
if (!safe?.size || fn.async || fn.generator) return result;
if (fn.async || fn.generator) return result;
// Frame facts are computed only for functions that construct a frame class.
let frame: ReadonlySet<string> | undefined;
const frameLocal = (localId: string): boolean => {
if (host.frameEscapes === null) return false;
frame ??= host.frameEscapes.locals(fn);
return frame.has(localId);
};
const caseScoped = caseScopedLocals(fn);
everyStmtList(fn.body, {
expr: () => true,
stmt: (stmt) => {
if (
stmt.kind === "varDecl" &&
safe.has(stmt.localId) &&
stmt.init &&
canStackObject(host, stmt.init) &&
((safe?.has(stmt.localId) === true && canStackObject(host, stmt.init)) ||
(stmt.init.kind === "new" &&
frameClass(host, stmt.init.className) &&
frameLocal(stmt.localId))) &&
!(caseScoped.has(stmt.localId) && ownsReferences(host, stmt.init.type))
)
result.set(stmt.localId, stmt.init);
result.set(stmt.localId, stmt.init as StackObjectSource);
return true;
},
});
@@ -194,17 +262,29 @@ export function stackObjectStruct(type: IrType): string {
/** The reference fields of the frame object in `slot`, addressed from the
* entry block so that every cleanup path can reach them. Inline union
* fields copy bits and own nothing; class objects in frame storage are
* scalar. */
* fields copy bits and own nothing, and so do a class's compressed
* references to program-lifetime instances. */
export function stackObjectFields(
host: LlvmEmitterContext,
type: IrType,
slot: string,
): readonly StackObjectField[] {
if (type.kind !== "record") return [];
if (type.kind !== "record" && type.kind !== "object") return [];
const cached = host.stackObjectFieldPtrs.get(slot);
if (cached) return cached;
const struct = stackObjectStruct(type);
if (type.kind === "object") {
const fields: StackObjectField[] = [];
for (const field of classReferenceFields(host, type.className)) {
const ptr = host.B.slot();
host.B.entryAllocas.push(
`${ptr} = getelementptr inbounds ${struct}, ptr ${slot}, i64 0, i32 ${field.index}`,
);
fields.push({ name: field.name, ptr, type: field.type });
}
host.stackObjectFieldPtrs.set(slot, fields);
return fields;
}
const base = host.nullableUnions.fieldBase(type.shapeId);
const fields: StackObjectField[] = [];
host.recordShape(type.shapeId).fields.forEach((field, index) => {
@@ -272,7 +352,19 @@ export function emitStackObject(
let slot = into;
if (slot === undefined) {
slot = B.slot();
B.entryAllocas.push(`${slot} = alloca ${struct}, align 8`);
if (type.kind === "object" && host.shapeHost.tracedShapes.has(`object:${type.className}`)) {
// A cycle-capable class keeps its collector header in front of the
// object, never a candidate (SCR_CYC_ACYCLIC), as arena objects do.
const frame = B.slot();
B.entryAllocas.push(
`${frame} = alloca { [${CYC_HDR} x i8], ${struct} }, align 8`,
`${slot} = getelementptr inbounds { [${CYC_HDR} x i8], ${struct} }, ptr ${frame}, i64 0, i32 1`,
);
const buf = B.tmp();
B.line(`store i16 0, ptr ${frame} ; ScrCycHdr.id`);
B.line(`${buf} = getelementptr inbounds i8, ptr ${frame}, i64 ${CYC_BUF}`);
B.line(`store i32 -1, ptr ${buf} ; ScrCycHdr.buf = SCR_CYC_ACYCLIC`);
} else B.entryAllocas.push(`${slot} = alloca ${struct}, align 8`);
}
// A literal's interned string fields are immortal and need no owner. A
// binding's own slot (`into`) keeps an owner for every field: later
@@ -293,16 +385,15 @@ export function emitStackObject(
const ctor = host.fnByName.get(`%${type.className}.constructor`)!;
if (!constructorInitializesFields(meta.def, ctor)) {
B.line(`store ${struct} zeroinitializer, ptr ${slot}`);
// Inline union fields start undefined, as the class's `new` sets them.
for (const field of meta.def.fields) {
const tag = undefinedArmTag(field.type, host.unionsById);
if (tag < 0 || !host.inlineUnions.classField(type.className, field.name)) continue;
const p = B.tmp();
B.line(
`${p} = getelementptr inbounds ${struct}, ptr ${slot}, i64 0, i32 ${classFieldIndex(meta, field.name).index}, i32 0`,
);
B.line(`store i32 ${tag}, ptr ${p}`);
}
// Union and dyn fields start undefined, as the class's `new` sets them.
for (const line of classUndefinedFieldInits(
host.shapeHost,
meta,
host.nullableFields,
slot,
`${B.tmp()}.uf`,
))
B.line(line.trim());
}
}
B.line(`store ${host.sizeType} -1, ptr ${slot}`);
packages/compiler/src/backend/llvm/walk-borrows.test.ts
+21
View File
@@ -139,3 +139,24 @@ test("returns of walks, including calls to borrowed-return callees, return borro
);
expect(analyzeWalks(owned, borrowedHost).returnsWalk).toBe(true);
});
test("a string literal return keeps the body's other walks borrowed", () => {
const ret = (value: IrExpr | null): IrStmt => ({ kind: "return", value, loc });
const STRING: IrType = { kind: "string" };
const text: IrExpr = {
kind: "fieldGet",
obj: ref("node"),
className: "Node",
field: "text",
type: STRING,
loc,
};
const empty: IrExpr = { kind: "strLit", value: "", type: STRING, loc };
const getter = fn([ret(text), ret(empty)], []);
expect(analyzeWalks(getter, host).returnsWalk).toBe(true);
// The literal needs no unwritten location; the field read still does.
expect(analyzeWalks(getter, { ...host, unwritten: () => false }).returnsWalk).toBe(false);
expect(analyzeWalks(fn([ret(empty)], []), { ...host, unwritten: () => false }).returnsWalk).toBe(
true,
);
});
packages/compiler/src/backend/llvm/walk-borrows.ts
+5 -4
View File
@@ -42,8 +42,9 @@ const WALK_KINDS: readonly IrNodeKind[] = [
export interface WalkFacts {
locals: ReadonlySet<string>;
/** Every return yields a walk (or a throw), so the body can return its
* pointer without a reference: no try statement, at least one return. */
/** Every return yields a walk, a throw or a string literal (an interned
* constant that nothing frees), so the body can return its pointer
* without a reference: no try statement, at least one return. */
returnsWalk: boolean;
}
@@ -176,6 +177,7 @@ export function analyzeWalks(fn: IrFunction, host: WalkBorrowHost): WalkFacts {
};
const isThrow = (e: IrExpr): boolean => e.kind === "libCall" && e.fn === "error.nodeThrow";
const walkOrThrow = (e: IrExpr): boolean => isThrow(e) || walk(e);
const constant = (e: IrExpr): boolean => isThrow(e) || e.kind === "strLit";
for (let changed = true; changed;) {
changed = false;
for (const id of candidates) {
@@ -199,8 +201,7 @@ export function analyzeWalks(fn: IrFunction, host: WalkBorrowHost): WalkFacts {
returns.every(
(value) =>
value !== null &&
walkOrThrow(value) &&
(isThrow(value) || host.unwritten?.(value, candidates) !== false),
(constant(value) || (walk(value) && host.unwritten?.(value, candidates) !== false)),
);
return { locals: result, returnsWalk };
}
packages/compiler/src/backend/llvm/walkers.ts
+20 -4
View File
@@ -1,4 +1,5 @@
import { emitPerKeyOrder } from "./key-orders.js";
import { recordViewFieldPtr } from "./record-views.js";
import { f64Lit } from "./common.js";
import { InternalCompilerError } from "../../errors.js";
/* Structure-walking helper EMITTERS for the LLVM backend — the .ll mirror
@@ -280,10 +281,25 @@ export class LlWalkers {
index: number,
t: IrType,
): string {
const p = B.tmp();
B.line(
`${p} = getelementptr inbounds %${mangleRecordStruct(shapeId)}, ptr ${recName}, i64 0, i32 ${index}`,
);
const base = this.host.nullableUnions.fieldBase(shapeId);
let p: string;
if (this.host.recordViews?.targets.has(shapeId) === true) {
p = recordViewFieldPtr(
B,
this.S,
mangleRecordStruct(shapeId),
recName,
index,
index - base,
(decl) => this.host.declare(decl),
this.host.rcMode === undefined,
);
} else {
p = B.tmp();
B.line(
`${p} = getelementptr inbounds %${mangleRecordStruct(shapeId)}, ptr ${recName}, i64 0, i32 ${index}`,
);
}
// An inline union field lends a private stack box (ABSENT kept).
if (this.host.inlineUnions.record(t)) return loadStructField(B, this.S, p, t, true);
const fieldTy = llFieldType(t, false);
packages/compiler/src/backend/native-codegen.test.ts
+9 -3
View File
@@ -186,13 +186,17 @@ test("profiles reach the helper and key the native cache by their bytes", async
const used = {
...base,
outputPath: join(pkg.root, "used.o"),
profile: { use: { path: "/profiles/a.profdata", digest: "a".repeat(64) } },
profile: {
use: { path: "/profiles/a.profdata", digest: "a".repeat(64), contextSensitive: false },
},
};
await emitNativeArtifact(used);
await emitNativeArtifact({
...used,
outputPath: join(pkg.root, "changed.o"),
profile: { use: { path: "/profiles/a.profdata", digest: "b".repeat(64) } },
profile: {
use: { path: "/profiles/a.profdata", digest: "b".repeat(64), contextSensitive: false },
},
});
await emitNativeArtifact({ ...used, outputPath: join(pkg.root, "again.o") });
const calls = (await readFile(pkg.log, "utf8")).trim().split("\n");
@@ -209,7 +213,9 @@ test("a cached artifact keeps the profile match of the build that produced it",
});
const used = {
...request(pkg.root, pkg.packageJson, join(pkg.root, "used.o")),
profile: { use: { path: "/profiles/a.profdata", digest: "a".repeat(64) } },
profile: {
use: { path: "/profiles/a.profdata", digest: "a".repeat(64), contextSensitive: false },
},
};
const match = { annotated: 120, mismatched: 7, missing: 40, unknownPartitions: 0 };
expect((await emitNativeArtifact(used)).profileMatch).toEqual(match);
packages/compiler/src/backend/native-link-info-core.ts
+6
View File
@@ -70,6 +70,12 @@ export interface NativeLinkFeatures {
/** `--large-pages`: scr_text_pages.c, which moves the executable's code
* onto transparent huge pages at startup. Absent (false) without it. */
largePages?: boolean;
/** Web streams in checked-dynamic code: scr_webstreams.c and
* scr_dyn_handle_class.c. */
webStreams?: boolean;
/** AbortController and AbortSignal: scr_abort.c, unless the program
* links the fetch unit, which has them. */
abortSignal?: boolean;
/** util.debounce and util.throttle: scr_util_timing.c. */
utilTiming?: boolean;
/** Buffer.prototype methods called on checked-dynamic values link
packages/compiler/src/backend/native-profile.test.ts
+54 -2
View File
@@ -5,8 +5,11 @@ import {
nativeProfileHelperArgs,
renderNativeProfileMatch,
nativeProfileKey,
nativeProfileLayout,
nativeProfileRefusal,
profileGenerateLinkerArgs,
profileHasContextCounts,
profileLayoutLinkerArgs,
} from "./native-profile.js";
import { NATIVE_TARGETS } from "./targets.js";
@@ -27,10 +30,21 @@ test("profiles are accepted for optimized executables", () => {
expect(
nativeProfileRefusal({ ...BASE, platform: "darwin", profileUse: "/p/a.profdata" }),
).toBeNull();
// The second phase of a context-sensitive profile.
expect(
nativeProfileRefusal({
...BASE,
profileGenerate: "/p/%p.profraw",
profileUse: "/p/a.profdata",
}),
).toBeNull();
});
test.each([
[{ profileGenerate: "/p/%p.profraw", profileUse: "/p/a.profdata" }, /cannot be combined/],
[
{ profileGenerate: "/p/%p.profraw", profileUse: "/p/a.profdata", platform: "darwin" },
/Linux executables/,
],
[{ profileUse: "/p/a.profdata", outputKind: "obj" }, /--profile-use is only supported/],
[{ profileGenerate: "/p/%p.profraw", sanitize: true }, /--sanitize/],
[{ profileUse: "/p/a.profdata", optimization: "dev" }, /optimized build/],
@@ -48,9 +62,47 @@ test("helper arguments and cache keys name the profile", () => {
"/p/%p.profraw",
]);
expect(nativeProfileKey({ generate: "/p/%p.profraw" })).toBe("profile-generate:/p/%p.profraw");
const use = { use: { path: "/p/a.profdata", digest: "d".repeat(64) } };
const use = { use: { path: "/p/a.profdata", digest: "d".repeat(64), contextSensitive: false } };
expect(nativeProfileHelperArgs(use)).toEqual(["--profile-use", "/p/a.profdata"]);
expect(nativeProfileKey(use)).toBe(`profile-use:${"d".repeat(64)}`);
const both = { generate: "/p/%p.profraw", ...use };
expect(nativeProfileHelperArgs(both)).toEqual([
"--profile-generate",
"/p/%p.profraw",
"--profile-use",
"/p/a.profdata",
]);
expect(nativeProfileKey(both)).toBe(
`profile-generate:/p/%p.profraw\0profile-use:${"d".repeat(64)}`,
);
});
test("a profile's context-sensitive flag is read from its header", () => {
const header = (last: number) =>
Uint8Array.from([0xff, 0x6c, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x81, 12, 0, 0, 0, 0, 0, 0, last]);
// VARIANT_MASK_IR_PROF (bit 56) alone, then with VARIANT_MASK_CSIR_PROF (bit 57).
expect(profileHasContextCounts(header(0x01))).toBe(false);
expect(profileHasContextCounts(header(0x03))).toBe(true);
expect(profileHasContextCounts(header(0x03).slice(0, 15))).toBe(false);
const raw = header(0x03);
raw[0] = 0x00;
expect(profileHasContextCounts(raw)).toBe(false);
});
test("only a context-sensitive profile lays out code and keeps its section kinds apart", () => {
const use = (contextSensitive: boolean) => ({
use: { path: "/p/a.profdata", digest: "d".repeat(64), contextSensitive },
});
expect(nativeProfileLayout(undefined)).toBe(false);
expect(nativeProfileLayout(use(false))).toBe(false);
expect(nativeProfileLayout(use(true))).toBe(true);
expect(nativeProfileLayout({ generate: "/p/%p.profraw", ...use(true) })).toBe(false);
const linux = { platform: "linux", defaultLinker: "clang" };
expect(profileLayoutLinkerArgs(linux, true)).toEqual(["-Wl,-z,keep-text-section-prefix"]);
expect(profileLayoutLinkerArgs(linux, false)).toEqual([]);
expect(profileLayoutLinkerArgs({ platform: "darwin", defaultLinker: "clang" }, true)).toEqual([]);
// Zig links cross-compiled and glibc-versioned Linux targets and rejects the flag.
expect(profileLayoutLinkerArgs({ platform: "linux", defaultLinker: "zig" }, true)).toEqual([]);
});
test("only the instrumented link adds the profile runtime", () => {
packages/compiler/src/backend/native-profile.ts
+48 -10
View File
@@ -5,14 +5,32 @@
* merge-profiles` turns those raw profiles into one indexed profile, and an
* optimized build (`--profile-use`) lays out, inlines, and splits code by it.
* The helper applies both in the pre-link pipeline of every partition.
*
* A second phase records context-sensitive counts: a build with both flags
* is optimized by the first profile and instrumented again after inlining,
* and merging its raw profiles into the first profile adds the counts of the
* inlined code. A profile with those counts also orders each function's
* blocks by them and moves cold blocks to separate sections.
*/
import type { NativeOptimization } from "./optimization.js";
export interface NativeProfileOptions {
/** Raw profile path pattern written by the instrumented executable. */
/** Raw profile path pattern written by the instrumented executable. With
* `use`, the instrumentation runs after inlining and records
* context-sensitive counts. */
generate?: string;
/** Indexed profile and the digest of its bytes, which key every cache. */
use?: { path: string; digest: string };
/** Indexed profile, the digest of its bytes (which key every cache), and
* whether it holds context-sensitive counts. */
use?: { path: string; digest: string; contextSensitive: boolean };
}
/** Whether an indexed profile holds context-sensitive counts: the header's
* version word carries LLVM's VARIANT_MASK_CSIR_PROF flag, bit 57, which is
* bit 1 of its last byte. */
export function profileHasContextCounts(bytes: Uint8Array): boolean {
const magic = [0xff, 0x6c, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x81];
if (bytes.length < 16 || magic.some((byte, index) => bytes[index] !== byte)) return false;
return (bytes[15]! & 0x02) !== 0;
}
/** Why a request cannot use profiles, or null when it can. */
@@ -28,7 +46,6 @@ export function nativeProfileRefusal(options: {
const use = options.profileUse !== undefined;
if (!generate && !use) return null;
const flag = generate ? "--profile-generate" : "--profile-use";
if (generate && use) return "--profile-generate and --profile-use cannot be combined";
if (options.outputKind !== "exe") return `${flag} is only supported for executable output`;
if (options.sanitize) return `${flag} cannot be combined with --sanitize`;
if (options.optimization === "dev")
@@ -39,9 +56,10 @@ export function nativeProfileRefusal(options: {
}
export function nativeProfileHelperArgs(profile: NativeProfileOptions | undefined): string[] {
if (profile?.generate !== undefined) return ["--profile-generate", profile.generate];
if (profile?.use !== undefined) return ["--profile-use", profile.use.path];
return [];
return [
...(profile?.generate === undefined ? [] : ["--profile-generate", profile.generate]),
...(profile?.use === undefined ? [] : ["--profile-use", profile.use.path]),
];
}
/** How an optimized build's profile matched the program's functions, as the
@@ -111,12 +129,32 @@ export function renderNativeProfileMatch(profile: string, match: NativeProfileMa
/** Cache-key part; empty without a profile so other keys are unchanged. */
export function nativeProfileKey(profile: NativeProfileOptions | undefined): string {
if (profile?.generate !== undefined) return `profile-generate:${profile.generate}`;
if (profile?.use !== undefined) return `profile-use:${profile.use.digest}`;
return "";
return [
...(profile?.generate === undefined ? [] : [`profile-generate:${profile.generate}`]),
...(profile?.use === undefined ? [] : [`profile-use:${profile.use.digest}`]),
].join("\0");
}
/** Whether a build lays out its code by context-sensitive counts: it uses a
* profile that has them, and records none. */
export function nativeProfileLayout(profile: NativeProfileOptions | undefined): boolean {
return profile?.use?.contextSensitive === true && profile.generate === undefined;
}
/** The instrumented link adds the compiler driver's profile runtime. */
export function profileGenerateLinkerArgs(generate: boolean): string[] {
return generate ? ["-fprofile-generate"] : [];
}
/** A link that lays out code by a profile keeps the hot, unlikely and split
* sections the helper writes apart, so each kind of code sits together. Zig,
* which links cross-compiled and glibc-versioned Linux targets, rejects the
* flag, so those links leave it out. */
export function profileLayoutLinkerArgs(
target: { platform: string; defaultLinker: string },
layout: boolean,
): string[] {
return layout && target.platform === "linux" && target.defaultLinker !== "zig"
? ["-Wl,-z,keep-text-section-prefix"]
: [];
}
packages/compiler/src/backend/native/contracts.ts
+6
View File
@@ -131,6 +131,12 @@ export interface CcOptions {
mime?: boolean;
/** `--large-pages`: compiles scr_text_pages.c. */
largePages?: boolean;
/** Web streams in checked-dynamic code: compiles scr_webstreams.c and
* scr_dyn_handle_class.c. */
webStreams?: boolean;
/** AbortController and AbortSignal: compiles scr_abort.c, unless the
* program compiles the fetch unit, which has them. */
abortSignal?: boolean;
/** util.debounce and util.throttle: compiles scr_util_timing.c. */
utilTiming?: boolean;
/** Buffer.prototype methods called on checked-dynamic values:
packages/compiler/src/backend/native/executable.ts
+30 -10
View File
@@ -195,7 +195,10 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [
unit((u) => u.opts.computedRequire === true, "scr_require.c"),
unit((u) => u.opts.dc, "scr_dc.c"),
unit((u) => u.opts.dynTier, "scr_js_value.c", "scr_js_object.c"),
unit((u) => u.opts.eventTarget || u.nativeFetch, "scr_event_target.c"),
unit(
(u) => u.opts.eventTarget || u.nativeFetch || u.opts.abortSignal === true,
"scr_event_target.c",
),
unit((u) => u.opts.wasm, "scr_wasm.c"),
// Module namespace objects as checked-dynamic values.
unit((u) => u.opts.moduleNamespace === true, "scr_module_namespace.c"),
@@ -248,6 +251,9 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [
unit((u) => u.opts.eventsOnceSignal === true, "scr_events_once_signal.c"),
unit((u) => u.opts.mime === true, "scr_mime.c"),
unit((u) => u.opts.largePages === true, "scr_text_pages.c"),
unit((u) => u.opts.webStreams === true, "scr_webstreams.c"),
unit((u) => u.opts.webStreams === true, "scr_dyn_handle_class.c"),
unit((u) => u.opts.abortSignal === true && !u.nativeFetch, "scr_abort.c"),
unit((u) => u.opts.utilTiming === true, "scr_util_timing.c"),
// The checked-dynamic HANDLE support unit (listener gate + runtime
// adapter closures): every referencing unit is one of the emitter or
@@ -391,6 +397,11 @@ const RUNTIME_DEFINES: readonly {
{ define: "SCR_JS_TIER", when: (opts) => opts.dynTier === true },
{ define: "SCR_WARNING_REPLACE", when: (opts) => opts.warningReplace },
{ define: "SCR_STDIO_EXTENDED", when: (opts) => opts.stdioExtended },
// scr_async.c's native promise reactions, which the web streams unit uses,
// scr_json.c's and scr_dyn_builtin_class.c's instanceof over its classes,
// and scr_fetch.c's bodies of its streams.
{ define: "SCR_NATIVE_REACTIONS", when: (opts) => opts.webStreams },
{ define: "SCR_HANDLE_CLASSES", when: (opts) => opts.webStreams },
{ define: "SCR_YIN", when: (opts) => opts.youngNursery },
{ define: "SCR_NODE_ISLANDS", when: (opts) => opts.nodeIsland },
];
@@ -1483,12 +1494,18 @@ export async function compileCInternal(
missed: false,
};
});
const shardWidth = Math.min(8, availableParallelism());
for (let i = 0; i < entries.length; i += shardWidth) {
await Promise.all(
entries.slice(i, i + shardWidth).map(async (entry) => {
// Up to eight clangs (one per available core), each taking the next
// shard as it finishes: shards differ in size, and a fixed batch
// would wait for its slowest member before starting the next. A
// failed shard stops the others from taking more work.
let nextShard = 0;
let failed = false;
await Promise.all(
Array.from({ length: Math.min(entries.length, 8, availableParallelism()) }, async () => {
while (!failed && nextShard < entries.length) {
const entry = entries[nextShard++]!;
await writeFile(entry.sourcePath, entry.source());
if (await copyValidCachedFile(entry.cachePath, entry.staged)) return;
if (await copyValidCachedFile(entry.cachePath, entry.staged)) continue;
entry.missed = true;
await runClang([
...programCompilerArgs,
@@ -1497,10 +1514,13 @@ export async function compileCInternal(
entry.sourcePath,
"-o",
entry.staged,
]);
}),
);
}
]).catch((error: unknown) => {
failed = true;
throw error;
});
}
}),
);
const arArgv = isZigDriver(driver) ? [driver.argv[0]!, "ar"] : ["ar"];
const merged = await localizeLibraryObjects(
driver,
packages/compiler/src/backend/publication.ts
+5
View File
@@ -73,6 +73,11 @@ export class PublishedTypes {
return this.containers.has(key) || this.frozenArrays.has(key);
}
/** Whether an array, Map or Set of type `t` may be published. */
containerPublished(t: IrType): boolean {
return this.containers.has(typeKey(t));
}
/** Whether an array of type `t` may be published, and whether it may be
* frozen: the two states its write guard tests. */
arrayGuards(t: IrType): { published: boolean; frozen: boolean } {
packages/compiler/src/backend/runtime-pack-core.ts
+8
View File
@@ -92,6 +92,8 @@ export interface RuntimeFeatureSet extends NativeLinkFeatures {
eventsOnceSignal: boolean;
mime: boolean;
largePages: boolean;
webStreams: boolean;
abortSignal: boolean;
utilTiming: boolean;
dynBuffer: boolean;
builtinValues: boolean;
@@ -102,6 +104,7 @@ export interface RuntimeFeatureSet extends NativeLinkFeatures {
brotli: boolean;
moduleNamespace: boolean;
nativeFetch: boolean;
abortSignalOnly: boolean;
netEffective: boolean;
httpEffective: boolean;
tlsEffective: boolean;
@@ -298,6 +301,8 @@ export function effectiveRuntimeFeatures(
eventsOnceSignal: features.eventsOnceSignal === true,
mime: features.mime === true,
largePages: features.largePages === true,
webStreams: features.webStreams === true,
abortSignal: features.abortSignal === true,
utilTiming: features.utilTiming === true,
dynBuffer: features.dynBuffer === true,
builtinValues: features.builtinValues === true,
@@ -308,6 +313,9 @@ export function effectiveRuntimeFeatures(
brotli: features.brotli === true,
moduleNamespace: features.moduleNamespace === true,
nativeFetch,
// The fetch unit has AbortController and AbortSignal; scr_abort.c is
// the same implementation without the fetch transport.
abortSignalOnly: features.abortSignal === true && !nativeFetch,
netEffective,
httpEffective,
tlsEffective,
packages/compiler/src/backend/runtime-pack.test.ts
+25 -1
View File
@@ -21,7 +21,12 @@ import {
platformLinkerSupportsPersistentCache,
resolvePlatformLinker,
} from "./linker.js";
import { MACOS_ARM64_TARGET, WINDOWS_X64_MSVC_TARGET, type NativeTargetSpec } from "./targets.js";
import {
LINUX_X64_GNU_TARGET,
MACOS_ARM64_TARGET,
WINDOWS_X64_MSVC_TARGET,
type NativeTargetSpec,
} from "./targets.js";
const VERSION = compilerReleaseVersion();
@@ -242,6 +247,25 @@ describe("runtime pack manifests", () => {
expect(guiPlan.driverFlags).toEqual([...defaultPlan.driverFlags, "-Wl,--subsystem,windows"]);
});
test("Linux runtime-pack links are static PIE only when requested", async () => {
const { packagePath, root } = await fixture(LINUX_X64_GNU_TARGET);
const options = {
target: LINUX_X64_GNU_TARGET,
programObject: join(root, "program.o"),
outPath: join(root, "program"),
features: BASE,
ffi: null,
optimization: "release" as const,
resolver: () => packagePath,
};
const defaultPlan = await createNativeLinkPlan(options);
const staticPlan = await createNativeLinkPlan({ ...options, staticLink: true });
expect(defaultPlan.driverFlags).not.toContain("-static-pie");
expect(defaultPlan.staticLink).toBeUndefined();
expect(staticPlan.driverFlags).toEqual([...defaultPlan.driverFlags, "-static-pie"]);
expect(staticPlan.staticLink).toBe(true);
});
test("selection chooses the most-specific variant and feature archive", async () => {
const { packagePath } = await fixture();
const resolver = () => packagePath;
packages/compiler/src/backend/target-diagnostics.test.ts
+1 -1
View File
@@ -117,7 +117,7 @@ test("prototype tables store optional methods without invoking their host capabi
test("local abort signals remain available without a reflection dispatcher", () => {
const mod = module();
mod.functions = [
fn("%main", [{ kind: "libCall", fn: "fetch.abortControllerNew", args: [], type: DYN, loc }]),
fn("%main", [{ kind: "libCall", fn: "abort.controllerNew", args: [], type: DYN, loc }]),
];
fenceSpeculativeWasiFunctions(mod);
expect(moduleWasiUnavailableSurface(mod)).toBeNull();
packages/compiler/src/backend/target-diagnostics.ts
+1 -19
View File
@@ -1,4 +1,4 @@
import { everyModuleNode, everyStmtList, transformStmtList } from "../ir/traverse.js";
import { everyModuleNode, everyStmtList } from "../ir/traverse.js";
import type { LlvmUnsupportedError } from "./llvm/emitter.js";
import type { ScrDiagnostic } from "../diagnostics/diagnostic.js";
import { type IrModule, type SrcLoc } from "../ir/ir.js";
@@ -14,24 +14,6 @@ import {
* calls, escaped callables, constructors, and exported roots still retain
* the ordinary target diagnostic. ABI types remain checked by the gate. */
export function fenceSpeculativeWasiFunctions(mod: IrModule): void {
for (const fn of mod.functions)
fn.body = transformStmtList(fn.body, {
stmt: (statement) => statement,
expr: (expr) => {
if (expr.kind !== "libCall") return expr;
const portable =
expr.fn === "fetch.abortControllerNew"
? "abort.controllerNew"
: expr.fn === "fetch.abortTimeout"
? "abort.timeout"
: expr.fn === "fetch.abortNow"
? "abort.now"
: expr.fn === "fetch.abortAny"
? "abort.any"
: null;
return portable ? { ...expr, fn: portable } : expr;
},
});
if (!mod.functions.some((fn) => fn.speculativeDispatch)) return;
const functions = new Map(mod.functions.map((fn) => [fn.name, fn]));
const classes = new Map((mod.classes ?? []).map((info) => [info.name, info]));
packages/compiler/src/backend/targets.test.ts
+98
View File
@@ -13,9 +13,15 @@ import {
WASM32_WASI_TARGET,
WINDOWS_X64_MSVC_TARGET,
executableOptimizationLinkerArgs,
executableStaticLinkerArgs,
ffiExtendsNarrowIntegers,
nativeCodegenTarget,
nativeCodegenTargetRefusal,
selectNativeTarget,
staticExecutableRefusal,
staticLinkFailureNote,
staticLinkMapArgs,
staticLinkRefusal,
windowsSubsystemLinkerArgs,
} from "./targets.js";
@@ -232,4 +238,96 @@ describe("native code-generation targets", () => {
"unknown Windows subsystem",
);
});
test("links host glibc executables as static PIE only for --static", () => {
expect(executableStaticLinkerArgs(LINUX_X64_GNU_TARGET, false)).toEqual([]);
expect(executableStaticLinkerArgs(LINUX_X64_GNU_TARGET, true)).toEqual(["-static-pie"]);
expect(executableStaticLinkerArgs(LINUX_ARM64_GNU_TARGET, true)).toEqual(["-static-pie"]);
// musl executables are static already.
expect(executableStaticLinkerArgs(LINUX_X64_MUSL_TARGET, true)).toEqual([]);
expect(executableStaticLinkerArgs(MACOS_ARM64_TARGET, false)).toEqual([]);
expect(() => executableStaticLinkerArgs(MACOS_ARM64_TARGET, true)).toThrow(
"only supported for Linux executables",
);
});
test("refuses --static where glibc cannot be linked statically", () => {
const request = { staticLink: true, outputKind: "exe", sanitize: false };
expect(staticLinkRefusal({ ...request, staticLink: undefined, target: null })).toBeNull();
expect(staticLinkRefusal({ ...request, target: LINUX_X64_GNU_TARGET })).toBeNull();
expect(staticLinkRefusal({ ...request, target: LINUX_X64_MUSL_TARGET })).toBeNull();
expect(staticLinkRefusal({ ...request, outputKind: "obj", target: LINUX_X64_GNU_TARGET })).toBe(
"--static is only supported for executable output",
);
expect(staticLinkRefusal({ ...request, sanitize: true, target: LINUX_X64_GNU_TARGET })).toMatch(
"cannot be combined with --sanitize",
);
for (const target of [MACOS_ARM64_TARGET, WINDOWS_X64_MSVC_TARGET, WASM32_WASI_TARGET, null])
expect(staticLinkRefusal({ ...request, target })).toBe(
"--static is only supported for Linux executables",
);
// zig links cross-compiled glibc targets, and links glibc dynamically.
const cross = selectNativeTarget("aarch64-linux-gnu", LINUX_X64_GNU_TARGET, "linux")!;
expect(staticLinkRefusal({ ...request, target: cross })).toMatch("<arch>-linux-musl");
const crossMusl = selectNativeTarget("aarch64-linux-musl", LINUX_X64_GNU_TARGET, "linux")!;
expect(staticLinkRefusal({ ...request, target: crossMusl })).toBeNull();
});
test("refuses static glibc executables whose link map keeps the NSS loader", () => {
const libc = "/lib/x86_64-linux-gnu/libc.a";
// lld lists the input sections it kept.
const lld = (member: string) =>
[
" VMA LMA Size Align Out In Symbol",
` 11e1a0 11e1a0 640 16 ${libc}(${member}):(.text)`,
` 11e7e0 11e7e0 38 1 ${libc}(${member}):(.rodata)`,
].join("\n");
const lldPlain = [
" VMA LMA Size Align Out In Symbol",
` 2c62c0 2c62c0 a10 16 ${libc}(dl-open.o):(.text)`,
].join("\n");
// GNU ld lists discarded sections first, and sections relaxation emptied with size 0.
const bfd = (text: boolean) =>
[
"Discarded input sections",
"",
` .text 0x0000000000000000 0x640 ${libc}(nss_module.o)`,
"",
"Linker script and memory map",
"",
` .eh_frame 0x00000000000db750 0x0 ${libc}(nss_module.o)`,
" 0x1b8 (size before relaxing)",
...(text ? [` .text 0x00000000000a81b0 0x640 ${libc}(nsswitch.o)`] : []),
].join("\n");
const refusal =
/^--static cannot link this program: its executable includes host, user or group name lookups/;
expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, lld("nss_module.o"))).toMatch(
refusal,
);
expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, lld("nsswitch.o"))).toMatch(refusal);
expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, bfd(true))).toMatch(refusal);
expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, bfd(false))).toBeNull();
expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, lldPlain)).toBeNull();
// A program's own string never counts.
expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, "libnss_files.so.2")).toBeNull();
expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, false, lld("nss_module.o"))).toBeNull();
expect(staticExecutableRefusal(LINUX_X64_MUSL_TARGET, true, lld("nss_module.o"))).toBeNull();
});
test("writes a link map only for --static glibc links", () => {
expect(staticLinkMapArgs(LINUX_X64_GNU_TARGET, true, "/stage/link.map")).toEqual([
"-Xlinker",
"-Map",
"-Xlinker",
"/stage/link.map",
]);
expect(staticLinkMapArgs(LINUX_X64_GNU_TARGET, false, "/stage/link.map")).toEqual([]);
expect(staticLinkMapArgs(LINUX_X64_MUSL_TARGET, true, "/stage/link.map")).toEqual([]);
});
test("notes glibc's static libraries on failed --static links", () => {
expect(staticLinkFailureNote(LINUX_X64_GNU_TARGET, true)).toMatch("libc.a");
expect(staticLinkFailureNote(LINUX_X64_GNU_TARGET, false)).toBe("");
expect(staticLinkFailureNote(LINUX_X64_MUSL_TARGET, true)).toBe("");
});
});
packages/compiler/src/backend/targets.ts
+90
View File
@@ -91,6 +91,96 @@ export function executableStripLinkerArgs(platform: string, strip: boolean): str
}
}
/** Why a `--static` request cannot be linked, or null when it can. A glibc
* executable links statically only through the host's clang driver: zig,
* which links cross-compiled glibc targets, links glibc dynamically. */
export function staticLinkRefusal(options: {
staticLink: boolean | undefined;
outputKind: string;
sanitize: boolean;
target: Pick<NativeTargetSpec, "name" | "platform" | "defaultLinker"> | null;
}): string | null {
if (options.staticLink !== true) return null;
if (options.outputKind !== "exe") return "--static is only supported for executable output";
if (options.sanitize)
return "--static cannot be combined with --sanitize; AddressSanitizer requires a dynamically linked executable";
const target = options.target;
if (target === null || target.platform !== "linux")
return "--static is only supported for Linux executables";
if (!target.name.endsWith("-musl") && target.defaultLinker === "zig")
return "--static links glibc from the build host and is only supported for the host's own glibc target; use an <arch>-linux-musl target to cross-compile a static executable";
return null;
}
/** Link the C library and the other system libraries into the executable.
* glibc executables link as static PIE, so the kernel still loads them at a
* random address; musl executables are always linked statically. */
export function executableStaticLinkerArgs(
target: Pick<NativeTargetSpec, "name" | "platform" | "defaultLinker">,
staticLink: boolean,
): string[] {
if (!staticLink) return [];
const refusal = staticLinkRefusal({ staticLink, outputKind: "exe", sanitize: false, target });
if (refusal !== null) throw new Error(refusal);
return target.name.endsWith("-musl") ? [] : ["-static-pie"];
}
/** The link map a `--static` glibc link writes to `path`, which
* staticExecutableRefusal reads. `-Xlinker` keeps the path one argument
* through the driver and into the linker's job. */
export function staticLinkMapArgs(
target: Pick<NativeTargetSpec, "name">,
staticLink: boolean,
path: string,
): string[] {
return staticLink && !target.name.endsWith("-musl") ? ["-Xlinker", "-Map", "-Xlinker", path] : [];
}
/** The libc.a members of glibc's NSS loader: nss_module.o (glibc 2.33 and
* later) and nsswitch.o. */
const GLIBC_NSS_LOADER_MEMBER = /libc\.a\((?:nss_module|nsswitch)\.o\)/;
/** Whether a link map line places code from an archive member in the
* output: lld lists the input sections it kept (`… libc.a(m.o):(.text)`),
* GNU ld each input section with its size (` .text 0x… 0x… …libc.a(m.o)`),
* and lists sections that relaxation emptied with size 0. */
function keepsText(line: string): boolean {
return /\):\(\.text\)\s*$/.test(line) || /^\s*\.text\s+0x[0-9a-f]+\s+0x0*[1-9a-f]/.test(line);
}
/** Why a linked `--static` executable is refused, or null. A static glibc
* executable answers host, user and group lookups with glibc's built-in
* `files` and `dns` sources, and loads every other module the run host's
* nsswitch.conf names from that host, which glibc supports only for the
* glibc version the executable was linked with. `linkMap` is the link's
* map (staticLinkMapArgs): the executable has the NSS loader when the map
* places the loader's code in it. GNU ld lists the input sections that
* garbage collection discarded before its memory map. */
export function staticExecutableRefusal(
target: Pick<NativeTargetSpec, "name">,
staticLink: boolean,
linkMap: string,
): string | null {
if (!staticLink || target.name.endsWith("-musl")) return null;
const memoryMap = linkMap.indexOf("Linker script and memory map");
const kept = memoryMap === -1 ? linkMap : linkMap.slice(memoryMap);
const loader = kept
.split("\n")
.some((line) => GLIBC_NSS_LOADER_MEMBER.test(line) && keepsText(line));
if (!loader) return null;
return "--static cannot link this program: its executable includes host, user or group name lookups (which dns.lookup, net and http connections, fetch and os.userInfo use), and a statically linked glibc executable loads the NSS modules for those lookups from the run host, which glibc supports only for the glibc version it was linked with. Build without --static, or use an <arch>-linux-musl target";
}
/** The note a failed `--static` glibc link adds to the linker's error. */
export function staticLinkFailureNote(
target: Pick<NativeTargetSpec, "name">,
staticLink: boolean,
): string {
return staticLink && !target.name.endsWith("-musl")
? "\n--static links glibc's static libraries (libc.a, libm.a), which the libc6-dev package installs on Debian and Ubuntu and glibc-static on Fedora"
: "";
}
export interface NativeTargetSpec {
/** Stable scriptc-facing identity used in cache keys and diagnostics. */
name: NativeTargetName;
packages/compiler/src/cli/command.ts
+9 -1
View File
@@ -115,6 +115,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
values.ffi !== undefined ||
values.profile !== undefined ||
values.strip ||
values.static ||
values["windows-subsystem"] !== undefined ||
values["detached-exit"] ||
values["large-pages"] ||
@@ -176,6 +177,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
values.ffi !== undefined ||
values.profile !== undefined ||
values.strip ||
values.static ||
values.sanitize ||
values["windows-subsystem"] !== undefined ||
values["detached-exit"] ||
@@ -269,6 +271,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
values["fail-on"] !== undefined ||
values.optimization !== undefined ||
values.strip ||
values.static ||
values.ffi !== undefined ||
values["windows-subsystem"] !== undefined ||
values["detached-exit"] ||
@@ -284,7 +287,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
hasNodeIslandFlags(values)
) {
fail(
"scriptc build --lib takes no --backend/--emit/--print/--optimization/--strip/--windows-subsystem/--detached-exit/--large-pages/--target-cpu/--cpu-dispatch/--profile-generate/--profile-use/--npm-static/--ffi/--external-types/--tier/--explain-dynamic: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, external type mappings belong to coverage, and library artifacts are static-tier only",
"scriptc build --lib takes no --backend/--emit/--print/--optimization/--strip/--static/--windows-subsystem/--detached-exit/--large-pages/--target-cpu/--cpu-dispatch/--profile-generate/--profile-use/--npm-static/--ffi/--external-types/--tier/--explain-dynamic: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, external type mappings belong to coverage, and library artifacts are static-tier only",
);
}
const profilePath = resolve(profileArg);
@@ -327,6 +330,9 @@ async function main(args: string[], host: CliHost): Promise<number> {
if (command === "coverage" && values.strip) {
fail(`--strip is a build/run option\n\n${USAGE}`);
}
if (command === "coverage" && values.static) {
fail(`--static is a build/run option\n\n${USAGE}`);
}
if (command === "coverage" && values["target-cpu"] !== undefined) {
fail(`--target-cpu is a build/run option\n\n${USAGE}`);
}
@@ -515,6 +521,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
sanitize: values.sanitize,
...(values.optimization === undefined ? {} : { optimization: values.optimization }),
strip: values.strip,
staticLink: values.static,
detachedExit: values["detached-exit"],
largePages: values["large-pages"],
...(values["cpu-dispatch"] === undefined ? {} : { cpuDispatch: values["cpu-dispatch"] }),
@@ -635,6 +642,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
...(backend !== undefined ? { backend } : {}),
...(optimization !== undefined ? { optimization } : {}),
...(values.strip ? { strip: true } : {}),
...(values.static ? { staticLink: true } : {}),
...(windowsSubsystem !== undefined ? { windowsSubsystem } : {}),
...(values["detached-exit"] ? { detachedExit: true } : {}),
...(values.libnode !== undefined
packages/compiler/src/cli/output-options.ts
+4 -3
View File
@@ -9,6 +9,7 @@ export interface OutputOptionValues {
sanitize: boolean;
optimization?: string;
strip?: boolean;
staticLink?: boolean;
windowsSubsystem?: string;
detachedExit?: boolean;
largePages?: boolean;
@@ -74,6 +75,9 @@ export function resolveOutputOptions(
if (values.strip && emit !== "exe") {
return { ok: false, message: `--strip is only supported with --emit=exe` };
}
if (values.staticLink && emit !== "exe") {
return { ok: false, message: `--static is only supported with --emit=exe` };
}
if (values.detachedExit && emit !== "exe") {
return { ok: false, message: `--detached-exit is only supported with --emit=exe` };
}
@@ -83,9 +87,6 @@ export function resolveOutputOptions(
if (values.cpuDispatch && emit !== "exe") {
return { ok: false, message: `--cpu-dispatch is only supported with --emit=exe` };
}
if (values.profileGenerate !== undefined && values.profileUse !== undefined) {
return { ok: false, message: `--profile-generate and --profile-use cannot be combined` };
}
if (values.profileGenerate !== undefined && emit !== "exe") {
return { ok: false, message: `--profile-generate is only supported with --emit=exe` };
}
packages/compiler/src/cli/usage.ts
+6
View File
@@ -56,6 +56,11 @@ Options:
counting, cache-line-aligned runtime code on x86-64)
--strip remove symbol/debug payload from the linked executable
for smaller builds (opt in; --emit=exe only)
--static Linux: link the C library into the executable, which
then starts without the dynamic loader. glibc builds
need its static libraries (libc.a) on the build host
and refuse executables that include host, user or
group name lookups (opt in; --emit=exe only)
--windows-subsystem <console|gui>
Windows executable subsystem (default: console). gui
prevents Windows from opening a console window
@@ -181,6 +186,7 @@ export const CLI_OPTIONS = {
backend: { type: "string" },
optimization: { type: "string" },
strip: { type: "boolean", default: false },
static: { type: "boolean", default: false },
"windows-subsystem": { type: "string" },
"detached-exit": { type: "boolean", default: false },
libnode: { type: "string" },
packages/compiler/src/compile-types.ts
+5
View File
@@ -31,6 +31,11 @@ export interface CompileBaseOptions {
optimization?: NativeOptimization;
/** Remove symbol/debug payload from an executable at link time. */
strip?: boolean;
/** Link the C library and the other system libraries into the executable
* (`--static`). glibc executables link as static PIE and need the C
* library's static archives on the build host; musl executables are always
* static. Linux executables only. */
staticLink?: boolean;
/** Windows PE executable subsystem. Console is the default; GUI suppresses
* automatic console-window creation. Only valid for Windows executables. */
windowsSubsystem?: WindowsSubsystem;
packages/compiler/src/executable/executable-cache.ts
+6
View File
@@ -82,6 +82,8 @@ export interface EarlyExecutableNativeFeatures {
dynForAwait?: boolean;
eventsOnceSignal?: boolean;
mime?: boolean;
webStreams?: boolean;
abortSignal?: boolean;
utilTiming?: boolean;
dynBuffer?: boolean;
builtinValues?: boolean;
@@ -152,6 +154,8 @@ export interface EarlyExecutableCacheOptions {
optimization?: "dev" | "speed";
/** Omitted retains the unstripped executable's historical cache key. */
strip?: true;
/** Omitted unless --static is set. */
staticLink?: true;
/** Omitted for the default Windows console subsystem. */
windowsSubsystem?: "gui";
/** Omitted unless --detached-exit is set. */
@@ -327,6 +331,7 @@ function cacheKey(options: EarlyExecutableCacheOptions): string {
options.backend,
...optimizationKeyParts(options.optimization),
...(options.strip ? ["strip"] : []),
...(options.staticLink ? ["static-link"] : []),
...(options.windowsSubsystem === "gui" ? ["windows-subsystem-gui"] : []),
...(options.detachedExit ? ["detached-exit"] : []),
...(options.largePages ? ["large-pages"] : []),
@@ -374,6 +379,7 @@ function routeKey(options: Omit<EarlyExecutableRouteOptions, "nativeEnvironment"
.update("\0");
for (const part of optimizationKeyParts(options.optimization)) hash.update(`${part}\0`);
if (options.strip) hash.update("strip\0");
if (options.staticLink) hash.update("static-link\0");
if (options.windowsSubsystem === "gui") hash.update("windows-subsystem-gui\0");
if (options.detachedExit) hash.update("detached-exit\0");
if (options.largePages) hash.update("large-pages\0");
packages/compiler/src/frontend/canon-map.ts
+27 -6
View File
@@ -25,8 +25,21 @@ import {
const MAX_DEPTH = 24;
/** What a mapping did besides its result. */
export interface CanonMapNotes {
/** A union in the form mapped to the checked dynamic because an arm
* did (`string | {}`): the result holds a dynamic part the form did not
* spell as `unknown`. */
collapsedToDynamic: boolean;
}
/** The IR type of a canonical node, or null when no rule applies. */
export function mapCanonical(c: Canon, ctx: TypeMapperCtx, depth = 0): IrType | null {
export function mapCanonical(
c: Canon,
ctx: TypeMapperCtx,
depth = 0,
notes?: CanonMapNotes,
): IrType | null {
if (depth > MAX_DEPTH) return null;
const viaChecker = (): IrType | null => (c.src === undefined ? null : mapType(c.src, ctx));
switch (c.k) {
@@ -55,13 +68,13 @@ export function mapCanonical(c: Canon, ctx: TypeMapperCtx, depth = 0): IrType |
case "tuple":
return viaChecker();
case "array": {
const elem = mapCanonical(c.elem, ctx, depth + 1);
const elem = mapCanonical(c.elem, ctx, depth + 1, notes);
return elem === null ? null : { kind: "array", elem };
}
case "union": {
const byKey = new Map<string, IrType>();
for (const arm of c.arms) {
const mapped = mapCanonical(arm, ctx, depth + 1);
const mapped = mapCanonical(arm, ctx, depth + 1, notes);
if (mapped === null || mapped.kind === "void") return null;
if (mapped.kind === "union") {
const def = ctx.unions.get(mapped.unionId);
@@ -71,7 +84,10 @@ export function mapCanonical(c: Canon, ctx: TypeMapperCtx, depth = 0): IrType |
}
const arms = [...byKey.values()];
if (arms.length === 1) return arms[0]!;
if (arms.some((a) => a.kind === "dyn")) return DYN;
if (arms.some((a) => a.kind === "dyn")) {
if (notes !== undefined) notes.collapsedToDynamic = true;
return DYN;
}
arms.sort((a, b) => (typeKey(a) < typeKey(b) ? -1 : 1));
return { kind: "union", unionId: ctx.unions.intern(arms) };
}
@@ -80,9 +96,13 @@ export function mapCanonical(c: Canon, ctx: TypeMapperCtx, depth = 0): IrType |
return viaChecker();
if (c.index().length > 0) return viaChecker();
const props = c.props();
// Accessor members are closure slots (`%get:x`, `%set:x`), and a
// getter and setter of different types have no single slot type:
// mapType's rules decide them, not a data field of the read type.
if (props.some((p) => p.accessor)) return viaChecker();
// `{}` holds any non-nullish value: the checked dynamic, as mapType.
if (props.length === 0) return DYN;
const structural = mapCanonicalRecord(props, ctx, depth);
const structural = mapCanonicalRecord(props, ctx, depth, notes);
// An object whose members do not map structurally (an interface over
// Map or Set, methods) keeps its per-type recognition.
return structural ?? viaChecker();
@@ -96,10 +116,11 @@ function mapCanonicalRecord(
props: readonly CanonProp[],
ctx: TypeMapperCtx,
depth: number,
notes: CanonMapNotes | undefined,
): IrType | null {
const fields: { name: string; type: IrType }[] = [];
for (const p of props) {
let t = mapCanonical(p.type(), ctx, depth + 1);
let t = mapCanonical(p.type(), ctx, depth + 1, notes);
if (t === null) return null;
if (p.optional && t.kind !== "dyn") {
const armed = withUndefinedArm(t, ctx.unions);
packages/compiler/src/frontend/canon.ts
+14 -2
View File
@@ -21,6 +21,8 @@ export interface CanonProp {
name: string;
optional: boolean;
readonly: boolean;
/** A get or set accessor, not a data property. */
accessor: boolean;
type: () => Canon;
}
@@ -131,7 +133,10 @@ function canonKeyInner(c: Canon, depth: number, budget: { left: number }): strin
}
const props = c
.props()
.map((p) => `${p.name}${p.optional ? "?" : ""}: ${canonKey(p.type(), depth + 1)}`);
.map(
(p) =>
`${p.accessor ? "get " : ""}${p.name}${p.optional ? "?" : ""}: ${canonKey(p.type(), depth + 1)}`,
);
const sig = (s: CanonSig): string =>
`(${s.params
.map(
@@ -251,7 +256,13 @@ function sameCanon(a: Canon, b: Canon, assumed: Assumptions): boolean {
if (ap.length !== op.length) return false;
for (const p of ap) {
const q = op.find((x) => x.name === p.name);
if (q === undefined || q.optional !== p.optional || !eq(p.type(), q.type())) return false;
if (
q === undefined ||
q.optional !== p.optional ||
q.accessor !== p.accessor ||
!eq(p.type(), q.type())
)
return false;
}
const ai = a.index();
const oi = o.index();
@@ -514,6 +525,7 @@ export class Normalizer {
name: ts.symbolName(p),
optional: (p.flags & ts.SymbolFlags.Optional) !== 0,
readonly: false,
accessor: (p.flags & (ts.SymbolFlags.GetAccessor | ts.SymbolFlags.SetAccessor)) !== 0,
type: () => this.canon(checker.getTypeOfSymbol(p), env),
}))),
index: () =>
packages/compiler/src/frontend/dts-paths.ts
+5
View File
@@ -220,6 +220,11 @@ export function fallbackModulesNamedIn(text: string): string[] {
return [...files];
}
/** True for the fallback declarations of a Node module (node-fallback/). */
export function isFallbackModuleDtsPath(file: string): boolean {
return tsgoPath(file).startsWith(fallbackModuleRoot() + "/");
}
/** True for the fallback declarations: the core file and the module files. */
export function isFallbackDtsPath(file: string): boolean {
const path = tsgoPath(file);
packages/compiler/src/frontend/lowering/builtins/errors.ts
+155 -56
View File
@@ -1,8 +1,8 @@
import * as ts from "../../ts7/adapter.js";
import { type Lowerer } from "../lowerer.js";
import { locOf } from "../../program.js";
import { tryLowerExpression } from "../expressions/try-lower-expression.js";
import { isJsSourceFile, locOf } from "../../program.js";
import { dynUndefinedExpr } from "../../../ir/build.js";
import { isSafeToDiscard } from "../expressions/evaluation-safety.js";
import {
BOOL,
CAUGHT,
@@ -13,6 +13,7 @@ import {
type IrStmt,
type IrType,
STRING,
VOID,
} from "../../../ir/ir.js";
const ASSERTION_ERROR_PROPERTIES: ReadonlySet<string> = new Set([
@@ -324,60 +325,6 @@ export function lowerErrorFunctionCall(lowerer: Lowerer, call: ts.CallExpression
: error;
}
/** `Error.captureStackTrace(target[, constructorOpt])` (also called as
* `Error.captureStackTrace?.(...)`, which always calls it). On a
* checked-dynamic target (`this` in a JavaScript constructor function) the
* runtime defines target's `stack`. An instance of a class that extends
* Error derives its stack from its name and message when the stack is
* read, so the call only evaluates its arguments. Both arguments are
* evaluated; the second only selects frames in Node. Null for other
* targets and callees. */
export function lowerErrorCaptureStackTraceCall(
lowerer: Lowerer,
call: ts.CallExpression,
access: ts.PropertyAccessExpression,
): IrExpr | null {
if (access.questionDotToken) return null;
if (lowerer.stdlibGlobalMember(access, "Error") !== "captureStackTrace") return null;
const args = call.arguments.nodes;
if (args.length < 1 || args.length > 2 || args.some(ts.isSpreadElement)) return null;
const probed = tryLowerExpression(lowerer, args[0]!);
const loc = locOf(call);
if (
probed?.type.kind === "object" &&
(probed.type.className === "%Error" || lowerer.isSubclassOf(probed.type.className, "%Error"))
) {
const stmts: IrStmt[] = [];
for (const arg of args) {
const value = lowerer.lowerExpr(arg);
if (value.kind !== "varRef") stmts.push({ kind: "exprStmt", expr: value, loc: value.loc });
}
const result = dynUndefinedExpr(loc);
return stmts.length > 0 ? { kind: "seqExpr", stmts, result, type: DYN, loc } : result;
}
if (probed?.type.kind !== "dyn") return null;
const target = lowerer.lowerExpr(args[0]!);
const result: IrExpr = {
kind: "libCall",
fn: "dyn.errorCaptureStackTrace",
args: [target],
type: DYN,
loc,
};
if (args.length === 1) return result;
const local = lowerer.declareHiddenLocal("%captureTarget", DYN);
return {
kind: "seqExpr",
stmts: [
{ kind: "varDecl", localId: local.id, init: target, loc },
{ kind: "exprStmt", expr: lowerer.lowerExprExpecting(args[1]!, DYN), loc },
],
result: { ...result, args: [{ kind: "varRef", localId: local.id, type: DYN, loc }] },
type: DYN,
loc,
};
}
/** `Error.isError(value)`. A catch binding tests its payload against the
* Error class, an `unknown` value tests the checked-dynamic tree's error
* encoding, and a union tests its runtime tag against the arms whose class
@@ -470,3 +417,155 @@ export function lowerErrorIsErrorCall(
loc,
};
}
/** The class whose constructor body `node` is directly in (not inside a
* nested function): while it runs, `new` of that class is on the stack. */
function enclosingConstructorClass(node: ts.Node): ts.ClassLikeDeclaration | undefined {
for (let n = node.parent; n !== undefined; n = n.parent) {
if (ts.isConstructorDeclaration(n))
return ts.isClassDeclaration(n.parent) || ts.isClassExpression(n.parent)
? n.parent
: undefined;
if (ts.isFunctionLike(n) && !ts.isArrowFunction(n)) return undefined;
}
return undefined;
}
/** Whether `access` (`this.constructor`) reads a `constructor` accessor
* declared on the class or a base class. */
function readsConstructorAccessor(lowerer: Lowerer, access: ts.PropertyAccessExpression): boolean {
const property = lowerer.checker.getPropertyOfType(
lowerer.typeOf(access.expression),
"constructor",
);
return property !== undefined && (property.flags & ts.SymbolFlags.GetAccessor) !== 0;
}
/** `Error.captureStackTrace(target, constructorOpt?)`, also through `?.`:
* target.stack becomes the target's heading and the current stack frames,
* as V8 does it (scr_error_capture_stack_trace). The target is an error, a
* checked-dynamic object, or a primitive (a TypeError at run time).
* constructorOpt leaves out the frames up to and including its topmost call:
* a function declaration skips through its own frame; `this.constructor`,
* `new.target` or the class's own name inside a class constructor leave the
* frames from the code that ran `new`, which are the current frames
* (constructors have no frames of their own). Other targets are refused,
* and so are other constructorOpt values in TypeScript; JavaScript evaluates
* them and keeps all frames. Null when the callee is not the standard
* library's Error.captureStackTrace. */
export function lowerErrorCaptureStackTraceCall(
lowerer: Lowerer,
call: ts.CallExpression,
access: ts.PropertyAccessExpression,
): IrExpr | null {
if (lowerer.stdlibGlobalMember(access, "Error") !== "captureStackTrace") return null;
const args = call.arguments.nodes;
const loc = locOf(call);
if (args.length < 1 || args.length > 2 || args.some(ts.isSpreadElement)) {
lowerer.noLowering(
"Error.captureStackTrace arguments",
call,
"pass a target and an optional constructor",
);
}
const target = lowerer.lowerExpr(args[0]!);
// Primitives (and unions of them) reach the runtime, which throws V8's
// TypeError for them; objects without a checked-dynamic view are refused,
// since a boxed copy would not show the stack.
const primitive =
["f64", "string", "bool", "nullT", "undefinedT", "bigint"].includes(target.type.kind) ||
(target.type.kind === "union" &&
lowerer.dynConvertible(target.type) &&
(lowerer.unions.get(target.type.unionId)?.arms ?? []).every((arm) =>
["f64", "string", "bool", "nullT", "undefinedT", "bigint"].includes(arm.kind),
));
if (target.type.kind !== "dyn" && !primitive && !isErrorClassType(lowerer, target.type)) {
lowerer.noLowering(
`Error.captureStackTrace on '${lowerer.fmt(target.type)}' values`,
args[0]!,
"pass an Error instance, or an untyped object in JavaScript",
);
}
let through = "";
let skip = false;
// A JavaScript constructorOpt that is not a recognized form still
// evaluates, after the target and before the capture, as V8 evaluates
// the call's arguments: its getters and calls run and its exceptions
// propagate.
let evaluated: IrExpr | null = null;
const ctor = args[1];
if (ctor !== undefined) {
let node: ts.Expression = ctor;
while (ts.isParenthesizedExpression(node)) node = node.expression;
const symbol = ts.isIdentifier(node) ? lowerer.checker.getSymbolAtLocation(node) : undefined;
const resolved =
symbol !== undefined && symbol.flags & ts.SymbolFlags.Alias
? lowerer.checker.getAliasedSymbol(symbol)
: symbol;
const cls = enclosingConstructorClass(call);
const constructing =
(ts.isPropertyAccessExpression(node) &&
node.expression.kind === ts.SyntaxKind.ThisKeyword &&
node.name.text === "constructor") ||
(ts.isMetaProperty(node) && node.name.text === "target") ||
(resolved !== undefined &&
cls?.name !== undefined &&
lowerer.checker.getSymbolAtLocation(cls.name) === resolved);
const sig = resolved !== undefined ? lowerer.fnSigsBySymbol.get(resolved) : undefined;
if (constructing && cls !== undefined) {
skip = false;
// `this.constructor` runs a `constructor` getter of the class, whose
// calls and exceptions are observable.
if (ts.isPropertyAccessExpression(node) && readsConstructorAccessor(lowerer, node)) {
const value = lowerer.lowerExpr(ctor);
if (!isSafeToDiscard(value)) evaluated = value;
}
} else if (sig !== undefined) {
lowerer.noteEdge(sig.name);
through = sig.name;
skip = true;
} else if (isJsSourceFile(ts.getSourceFileOfNode(call))) {
// JavaScript (undici's `captureStackTrace(err, module.exports.fetch)`)
// keeps every frame: the same frames V8 keeps, and the ones above them.
skip = false;
const value = lowerer.lowerExpr(ctor);
if (!isSafeToDiscard(value)) evaluated = value;
} else {
lowerer.noLowering(
"Error.captureStackTrace with this constructor argument",
ctor,
"pass a function declaration, or this.constructor or the class inside its constructor",
);
}
}
const stmts: IrStmt[] = [];
let receiver = lowerer.coerceToExpected(target, DYN);
if (evaluated !== null) {
// V8 captures onto the target value it evaluated first. The constructor
// argument may assign the variable or field the target was read from, so
// even a plain read is kept in a local before the argument runs.
const local = lowerer.declareHiddenLocal("%captureTarget", DYN);
stmts.push({ kind: "varDecl", localId: local.id, init: receiver, loc });
receiver = { kind: "varRef", localId: local.id, type: DYN, loc };
stmts.push({ kind: "exprStmt", expr: evaluated, loc });
}
const capture: IrExpr = {
kind: "libCall",
fn: "error.captureStackTrace",
args: [
receiver,
{ kind: "strLit", value: through, type: STRING, loc },
{ kind: "boolLit", value: skip, type: BOOL, loc },
],
type: VOID,
loc,
};
stmts.push({ kind: "exprStmt", expr: capture, loc });
return {
kind: "seqExpr",
stmts,
result: dynUndefinedExpr(loc),
type: DYN,
loc,
};
}
packages/compiler/src/frontend/lowering/builtins/module-bindings.ts
+7
View File
@@ -31,6 +31,13 @@ export function builtinImportOf(
return binding?.module === "stream/web" ? null : binding;
}
/** The member a node:stream/web import binding names (`import {
* WritableStream } from "node:stream/web"`, or its CommonJS twin), or null. */
export function streamWebImportOf(lowerer: Lowerer, ident: ts.Identifier): string | null {
const binding = builtinModuleBindingOf(lowerer, ident);
return binding?.module === "stream/web" ? binding.member : null;
}
function builtinModuleBindingOf(
lowerer: Lowerer,
ident: ts.Identifier,
packages/compiler/src/frontend/lowering/builtins/web-streams-dyn.ts
+187
View File
@@ -0,0 +1,187 @@
import { numLit } from "../../../ir/build.js";
import { BOOL, DYN, STRING, type IrExpr, type SrcLoc } from "../../../ir/ir.js";
import * as ts from "../../ts7/adapter.js";
import { isJsSourceFile, locOf } from "../../program.js";
import { constituentTypes } from "../../ts7/checker.js";
import type { Lowerer } from "../lowerer.js";
import { stdlibGlobalNameOf } from "../surfaces.js";
import { streamWebImportOf } from "./module-bindings.js";
/* The web stream classes in JavaScript sources: the globals and the
* node:stream/web exports, which are the same constructors. Each reference
* is the class's constructor value (scr_webstreams.c, webstream.class), so
* construction, static members, prototypes and instanceof go through the
* checked-dynamic paths over that value. TypeScript sources keep their
* static lowerings (lower-stdlib.ts). */
/** The classes in scr_webstream_class's order. */
export const WEB_STREAM_CLASSES = [
"ReadableStream",
"ReadableStreamDefaultReader",
"ReadableStreamDefaultController",
"CountQueuingStrategy",
"ByteLengthQueuingStrategy",
] as const;
const CLASS_NAMES: readonly string[] = WEB_STREAM_CLASSES;
function unparenthesized(node: ts.Expression): ts.Expression {
let current = node;
while (ts.isParenthesizedExpression(current)) current = current.expression;
return current;
}
/** The class a JavaScript expression names (an identifier bound to the
* global or to a node:stream/web import, or a member of the global
* object), as its index in WEB_STREAM_CLASSES; null otherwise. */
export function webStreamClassKindOf(lowerer: Lowerer, node: ts.Expression): number | null {
const expr = unparenthesized(node);
if (!ts.isIdentifier(expr) && !ts.isPropertyAccessExpression(expr)) return null;
if (!isJsSourceFile(ts.getSourceFileOfNode(expr))) return null;
const member = ts.isPropertyAccessExpression(expr) ? lowerer.builtinMemberOf(expr) : null;
const name =
(ts.isIdentifier(expr) ? streamWebImportOf(lowerer, expr) : null) ??
(member?.module === "stream/web" ? member.member : null) ??
stdlibGlobalNameOf(lowerer, expr);
if (name === null) return null;
const index = CLASS_NAMES.indexOf(name);
return index < 0 ? null : index;
}
export function webStreamClassValue(kind: number, loc: SrcLoc): IrExpr {
return { kind: "libCall", fn: "webstream.class", args: [numLit(kind, loc)], type: DYN, loc };
}
/** A JavaScript read of one of the classes as a value; null otherwise. */
export function lowerWebStreamClassRead(lowerer: Lowerer, node: ts.Expression): IrExpr | null {
const kind = webStreamClassKindOf(lowerer, node);
return kind === null ? null : webStreamClassValue(kind, locOf(node));
}
/** `new X(...args)` on one of the classes in JavaScript. */
export function lowerWebStreamNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr | null {
const kind = webStreamClassKindOf(lowerer, expr.expression);
if (kind === null) return null;
const nodes = expr.arguments?.nodes ?? [];
if (nodes.some(ts.isSpreadElement)) return null;
const loc = locOf(expr);
return {
kind: "libCall",
fn: "dyn.construct",
args: [
webStreamClassValue(kind, loc),
{
kind: "dynArrLit",
elems: nodes.map((arg) => lowerer.lowerExprExpecting(arg, DYN)),
type: DYN,
loc,
},
{ kind: "strLit", value: CLASS_NAMES[kind]!, type: STRING, loc },
],
type: DYN,
loc,
};
}
/** `X.from(…)` and other calls of a class's static members in JavaScript:
* a member call on the constructor value. */
export function lowerWebStreamStaticCall(lowerer: Lowerer, call: ts.CallExpression): IrExpr | null {
const access = call.expression;
if (call.questionDotToken || !ts.isPropertyAccessExpression(access) || access.questionDotToken)
return null;
const kind = webStreamClassKindOf(lowerer, access.expression);
if (kind === null) return null;
const nodes = call.arguments.nodes;
if (nodes.some(ts.isSpreadElement)) return null;
const loc = locOf(call);
return {
kind: "dynInvoke",
recv: webStreamClassValue(kind, locOf(access.expression)),
method: access.name.text,
calleeName: ts.getTextOfNode(access),
args: nodes.map((arg) => lowerer.lowerExprExpecting(arg, DYN)),
type: DYN,
loc,
};
}
/** `value instanceof X` on one of the classes in JavaScript. */
export function lowerWebStreamInstanceof(
lowerer: Lowerer,
expr: ts.BinaryExpression,
): IrExpr | null {
const kind = webStreamClassKindOf(lowerer, expr.right);
if (kind === null) return null;
const loc = locOf(expr);
return {
kind: "libCall",
fn: "webstream.instanceOf",
args: [lowerer.lowerExprExpecting(expr.left, DYN), numLit(kind, loc)],
type: BOOL,
loc,
};
}
/** The ambient web stream interfaces whose members JavaScript reads from
* checked-dynamic values. */
const WEB_STREAM_INTERFACES: ReadonlySet<string> = new Set([
...WEB_STREAM_CLASSES,
"ReadableStreamReadResult",
"ReadableStreamReadValueResult",
"ReadableStreamReadDoneResult",
"UnderlyingSource",
"QueuingStrategy",
"QueuingStrategySize",
]);
/** Whether a JavaScript member read names a member of a web stream
* interface, which the receiver's checked-dynamic value answers. */
export function isWebStreamMemberInJavaScript(
lowerer: Lowerer,
access: ts.PropertyAccessExpression,
): boolean {
if (!isJsSourceFile(ts.getSourceFileOfNode(access))) return false;
const member = lowerer.checker.getSymbolAtLocation(access.name);
const declaration = member?.declarations[0];
let owner: ts.Node | undefined = declaration?.parent;
while (
owner !== undefined &&
!ts.isInterfaceDeclaration(owner) &&
!ts.isTypeAliasDeclaration(owner)
)
owner = owner.parent;
return owner !== undefined && WEB_STREAM_INTERFACES.has(owner.name.text);
}
/** Whether a checker type is, or is a promise of, a web stream interface
* (a read result, a reader's closed promise). JavaScript keeps such values
* checked-dynamic, as their handles answered them: a typed copy would lose
* the value's identity (a chunk object, a promise). */
export function typeInvolvesWebStream(lowerer: Lowerer, type: ts.Type, depth = 0): boolean {
if (depth > 4) return false;
if ((type.flags & ts.TypeFlags.Union) !== 0)
return constituentTypes(type).some((t) => typeInvolvesWebStream(lowerer, t, depth + 1));
for (const symbol of [type.getSymbol(), type.getAliasSymbol()]) {
if (symbol === undefined) continue;
const name = ts.symbolName(symbol);
if (WEB_STREAM_INTERFACES.has(name) && lowerer.isStdlibSymbol(symbol)) return true;
if (name === "Promise") {
const args = lowerer.checker.getTypeArguments(type as ts.TypeReference);
if (args.some((t) => typeInvolvesWebStream(lowerer, t, depth + 1))) return true;
}
}
return false;
}
/** Whether a JavaScript expression produces a web stream value: a member of
* a web stream interface (`reader.closed`), a call of one (`reader.read()`),
* an await of either, or a value whose type involves one. */
export function isWebStreamValueInJavaScript(lowerer: Lowerer, node: ts.Expression): boolean {
if (!isJsSourceFile(ts.getSourceFileOfNode(node))) return false;
let expr = unparenthesized(node);
if (ts.isAwaitExpression(expr)) expr = unparenthesized(expr.expression);
if (ts.isCallExpression(expr)) expr = unparenthesized(expr.expression);
if (ts.isPropertyAccessExpression(expr) && isWebStreamMemberInJavaScript(lowerer, expr))
return true;
return typeInvolvesWebStream(lowerer, lowerer.typeOf(node));
}
packages/compiler/src/frontend/lowering/class-attributes.ts
+6 -5
View File
@@ -53,13 +53,14 @@ function attributeClassNamed(lowerer: Lowerer, className: string): ClassInfo | n
/** Adds a zero-initialized f64 to every class of the hierarchy, at the same
* offset, like the property bag. */
function addHiddenField(info: ClassInfo, name: string): void {
function addHiddenField(lowerer: Lowerer, info: ClassInfo, name: string): void {
let root = info;
while (root.base && !root.base.def.runtime) root = root.base;
if (root.fields.has(name)) return;
const index = root.def.fields.length;
const add = (current: ClassInfo): void => {
if (!current.fields.has(name)) {
lowerer.classLayoutChanges++;
current.fields.set(name, F64);
current.def.fields.splice(index, 0, { name, type: F64 });
}
@@ -78,7 +79,7 @@ export function addUntypedIntegrityWords(lowerer: Lowerer): void {
if (!lowerer.untypedIntegrity) return;
for (const [className, info] of lowerer.classes) {
if (isDynTypedRefType({ kind: "object", className }) && attributeClassNamed(lowerer, className))
addHiddenField(info, CLASS_ATTRIBUTES);
addHiddenField(lowerer, info, CLASS_ATTRIBUTES);
}
}
@@ -97,7 +98,7 @@ export function lowerClassIntegrity(
): IrExpr | null {
const info = attributeClass(lowerer, target);
if (!info) return null;
addHiddenField(info, CLASS_ATTRIBUTES);
addHiddenField(lowerer, info, CLASS_ATTRIBUTES);
const loc = locOf(call);
const type = target.type as ClassObjectType;
const bits =
@@ -204,8 +205,8 @@ export function lowerClassFieldDefine(
)
return null;
const attributesField = classFieldAttributesName(field);
addHiddenField(info, CLASS_ATTRIBUTES);
addHiddenField(info, attributesField);
addHiddenField(lowerer, info, CLASS_ATTRIBUTES);
addHiddenField(lowerer, info, attributesField);
const loc = locOf(call);
const type = target.type as ClassObjectType;
const receiverSlot = lowerer.declareHiddenLocal("%defineReceiver", type);
packages/compiler/src/frontend/lowering/coercions/function-adapters.ts
+20 -2
View File
@@ -16,6 +16,7 @@ import {
import { typeKey } from "../../type-mapper.js";
import { newFnCtx } from "../function-context.js";
import type { Lowerer } from "../lowerer.js";
import { omittedArgFor } from "../call-arguments.js";
import { maybePromiseUnionOf } from "../maybe-promise.js";
/** Wrap a zero-argument function whose array result needs an element
@@ -273,7 +274,22 @@ export function funcCoerceAdapter(
if (toT.rest === true && toT.restAbi === undefined && toT.params.length === 0)
return funcIntoDynRestAdapter(lowerer, fromT, toT, loc);
if (fromT.rest === true || toT.rest === true) return null;
if (fromT.params.length > toT.params.length) return null;
// A source with MORE parameters than the slot: the checker admits it only
// when every extra trailing parameter is optional, so a call through the
// slot hands them undefined (JavaScript ignores surplus arguments). Pass
// that absent value explicitly. A pair whose extra parameter cannot hold
// undefined stays refused.
const omittedParams = new Map<number, IrExpr>();
for (let i = toT.params.length; i < fromT.params.length; i++) {
const param = fromT.params[i]!;
const absent =
omittedArgFor(lowerer, param, loc) ??
(typeEquals(param, UNDEFINED_T)
? ({ kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc } as IrExpr)
: null);
if (!absent) return null;
omittedParams.set(i, absent);
}
// Piece dispositions beyond coercibleValue, all CHECKER-APPROVED
// function compatibilities (bivariant method params under the suite's
// non-strict settings, `() => never` throwers displayed as void by
@@ -290,7 +306,7 @@ export function funcCoerceAdapter(
let strandParams = false;
const narrowedParams = new Map<number, ReadonlySet<number>>();
const partialDynParams = new Map<number, IrType>();
for (let i = 0; i < fromT.params.length; i++) {
for (let i = 0; i < toT.params.length && i < fromT.params.length; i++) {
const actual = toT.params[i]!;
const expected = fromT.params[i]!;
if (lowerer.coercibleValue(actual, expected)) continue;
@@ -412,6 +428,8 @@ export function funcCoerceAdapter(
];
} else {
const args = fromT.params.map((pt, i) => {
const omitted = omittedParams.get(i);
if (omitted) return omitted;
const aRef: IrExpr = { kind: "varRef", localId: `a.${i}`, type: toT.params[i]!, loc };
const partial = partialDynParams.get(i);
if (partial && aRef.type.kind === "union") {
packages/compiler/src/frontend/lowering/deferred-module-initializers.ts
+7
View File
@@ -26,6 +26,13 @@ export class DeferredModuleInitializers {
/** The last scan of each function (scanOf). */
private readonly scans = new Map<IrFunction, FunctionScan>();
/** The sizes of the deferral tables (unit-journal.ts). */
journalSizes(): number[] {
let initializers = 0;
for (const list of this.pending.values()) initializers += list.length;
return [this.pending.size, initializers, this.registryGlobals.size, this.deferredKeys.size];
}
defer(lowerer: Lowerer, statement: ts.Statement): IrStmt | null {
let key: string;
if (
packages/compiler/src/frontend/lowering/expressions/binary.ts
+3 -3
View File
@@ -1599,9 +1599,9 @@ export function lowerBinary(lowerer: Lowerer, expr: ts.BinaryExpression): IrExpr
reference.kind === "map" ||
reference.kind === "set" ||
reference.kind === "union" ||
// A promise of dyn boxes its own promise, so identity holds;
// other promise types box a fresh adapter.
(reference.kind === "promise" && reference.inner.kind === "dyn") ||
// A promise of dyn boxes its own promise; other promise types
// box an adapter that shares the promise's identity.
reference.kind === "promise" ||
DYN_HANDLE_KINDS.has(reference.kind)) &&
lowerer.dynConvertible(reference)
) {
packages/compiler/src/frontend/lowering/expressions/identifiers.ts
+5
View File
@@ -3,6 +3,7 @@ import { globalMemberValue } from "../global-values.js";
import { predeclareForwardClass } from "../lower-stmts.js";
import { cjsMemberAliasValueOf } from "../cjs-exports-object.js";
import { builtinClassValue, builtinMemberClassKind } from "../builtins/builtin-class-dyn.js";
import { lowerWebStreamClassRead } from "../builtins/web-streams-dyn.js";
import * as ts from "../../ts7/adapter.js";
import { InternalCompilerError } from "../../../errors.js";
import * as posix from "node:path/posix";
@@ -784,6 +785,10 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc:
// global, one token; what a token cannot do (be called, answer
// typeof "function") meets per-site fences/divergences, and
// SEMANTICS.md documents the stance.
{
const webStream = lowerWebStreamClassRead(lowerer, expr);
if (webStream) return webStream;
}
if (stdlibGlobalNameOf(lowerer, expr) === "globalThis") {
return lowerGlobalValue(lowerer, expr);
}
packages/compiler/src/frontend/lowering/expressions/narrowing.ts
+13
View File
@@ -1,5 +1,9 @@
/* Type narrowing of lowered values, conditional and as-expressions, and runtime-optional receiver reads. */
import { declBoundaryOrigin } from "../../decl-projection.js";
import {
isWebStreamMemberInJavaScript,
typeInvolvesWebStream,
} from "../builtins/web-streams-dyn.js";
import * as ts from "../../ts7/adapter.js";
import { type Lowerer, dynFallbackType } from "../lowerer.js";
import {
@@ -427,6 +431,15 @@ export function maybeNarrow(lowerer: Lowerer, expr: IrExpr, node: ts.Node): IrEx
// A process.versions entry declared as a string can be absent here.
if (isOptionalProcessVersionsRead(lowerer, node))
return lowerer.coerceToExpected(expr, lowerer.withUndefinedArm(STRING));
// A web stream member in JavaScript (a reader's closed promise, a read
// result's value) keeps the checked-dynamic value its handle answered.
if (ts.isPropertyAccessExpression(node) && isWebStreamMemberInJavaScript(lowerer, node))
return expr;
if (
isJsSourceFile(ts.getSourceFileOfNode(node)) &&
typeInvolvesWebStream(lowerer, lowerer.typeOf(node))
)
return expr;
// An inferred JS member's storage may be deliberately wider than its
// declaration (prototype-selected buffers, null-initialized scratch
// objects). Extract at the consuming operation, not at the read.
packages/compiler/src/frontend/lowering/expressions/property-access.ts
+6
View File
@@ -11,6 +11,7 @@ import { indexedUnionArms, lowerIndexedUnionMember } from "../indexed-unions.js"
import { keyedHelperName } from "../stable-names.js";
import { builtinNamespaceValue } from "../builtin-namespaces.js";
import { staticallySelected } from "../static-typeof.js";
import { lowerWebStreamClassRead } from "../builtins/web-streams-dyn.js";
import * as ts from "../../ts7/adapter.js";
import { lowerHistogramProperty } from "../builtins/performance.js";
import { staticFunctionNameRead } from "../static-function-names.js";
@@ -595,6 +596,11 @@ export function lowerPropertyAccessExpr(
if (lowerer.stdlibGlobalMember(expr, "globalThis") === "console") {
return { kind: "libCall", fn: "console.native", args: [], type: DYN, loc };
}
{
// `globalThis.ReadableStream` in JavaScript: the class value.
const webStream = lowerWebStreamClassRead(lowerer, expr);
if (webStream) return webStream;
}
if (lowerer.stdlibGlobalMember(expr, "console") !== null) {
return {
kind: "dynKeyGet",
packages/compiler/src/frontend/lowering/expressions/relational.ts
+3
View File
@@ -2,6 +2,7 @@
import { lowerArrayClassInstanceOf } from "../builtins/array-classes.js";
import { dateRootOf } from "../builtins/date-classes.js";
import { keyedHelperName } from "../stable-names.js";
import { lowerWebStreamInstanceof } from "../builtins/web-streams-dyn.js";
import * as ts from "../../ts7/adapter.js";
import { builtinClassInstanceOfAny, builtinClassKind } from "../builtins/builtin-class-dyn.js";
import {
@@ -386,6 +387,8 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc
loc,
};
}
const webStream = lowerWebStreamInstanceof(lowerer, expr);
if (webStream) return webStream;
if (lowerer.isStdlibGlobal(expr.right, "ReadableStream")) {
const value = lowerer.lowerExprExpecting(expr.left, DYN);
return { kind: "libCall", fn: "fetch.streamIs", args: [value], type: BOOL, loc };
packages/compiler/src/frontend/lowering/key-order-witness.ts
+11
View File
@@ -140,6 +140,17 @@ export class KeyOrderNotes {
/** Shapes whose table this pass replaced. */
readonly changed = new Set<string>();
/** The sizes of the notes' tables (unit-journal.ts). */
journalSizes(): number[] {
return [
this.literals.size,
this.copies.size,
this.methodResults.size,
this.observed.size,
this.changed.size,
];
}
literal(node: IrExpr, segments: KeyOrderSegment[]): IrExpr {
this.literals.set(node, segments);
return node;
packages/compiler/src/frontend/lowering/lower-calls.ts
+26
View File
@@ -564,6 +564,14 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr {
const timeoutOptional = lowerer.lowerTimeoutMethodCall(expr, expr.expression);
if (timeoutOptional) return timeoutOptional;
}
// `Error.captureStackTrace?.(this, this.constructor)`: Node always has it.
if (
ts.isPropertyAccessExpression(expr.expression) &&
(expr.questionDotToken || expr.expression.questionDotToken)
) {
const captured = lowerErrorCaptureStackTraceCall(lowerer, expr, expr.expression);
if (captured) return captured;
}
// `req.session?.m(...)` remains absent on compatibility requests. Live
// `req.stream` calls now pass through the normal optional-chain and h2
// stream lowering using lowerServerProperty's checked getter.
@@ -6110,6 +6118,24 @@ export function lowerPromiseMethodCall(
const fulfilledAbsent = isAbsentHandler(fulfilledNode);
const rejectedAbsent = isAbsentHandler(rejectedNode);
const resultT = lowerer.mapTypeOf(lowerer.typeOf(call));
// An untyped call in JavaScript whose receiver arrived as a typed
// promise (a function instantiated with one): Promise.prototype.then
// over the promise as a checked-dynamic value, which shares its
// identity.
if (resultT?.kind !== "promise" && isJsSourceFile(ts.getSourceFileOfNode(call))) {
return {
kind: "dynInvoke",
recv: lowerer.coerceToExpected(receiver, DYN),
method: "then",
calleeName: ts.getTextOfNode(access),
args: [
lowerer.lowerExprExpecting(fulfilledNode, DYN),
lowerer.lowerExprExpecting(rejectedNode, DYN),
],
type: DYN,
loc,
};
}
if (resultT?.kind !== "promise") {
lowerer.noLowering(
"then with these handlers",
packages/compiler/src/frontend/lowering/lower-stdlib.ts
+26 -6
View File
@@ -1,6 +1,7 @@
import { dynUndefinedExpr, nodeThrowExpr } from "../../ir/build.js";
import { InternalCompilerError } from "../../errors.js";
import { webStreamClassKindOf } from "./builtins/web-streams-dyn.js";
import * as ts from "../ts7/adapter.js";
import { lowerToNumberArguments } from "./lower-exprs.js";
import type { Lowerer } from "./lowerer.js";
@@ -31,7 +32,13 @@ import {
unsupportedNativeApiDiag,
unsupportedPackageValueDiag,
} from "../../diagnostics/diagnostic.js";
import { esmNamedImportLinkCrash, isCjsJsFile, locOf, npmPackageNameOf } from "../program.js";
import {
esmNamedImportLinkCrash,
isCjsJsFile,
isJsSourceFile,
locOf,
npmPackageNameOf,
} from "../program.js";
import { foldedStringKeyOf, lowerDynObjectLiteral } from "./expressions/object-literals.js";
import { noteEventListeners } from "./lower-event-target.js";
import { PoisonError, newFnCtx, own } from "./lowerer.js";
@@ -1026,6 +1033,12 @@ export function fenceStaticAbortControllerMemberRead(
);
}
/** JavaScript sources read web stream values through their handles'
* checked-dynamic surfaces (scr_webstreams.c), not the static lowerings. */
function inJavaScript(node: ts.Node): boolean {
return isJsSourceFile(ts.getSourceFileOfNode(node));
}
function isStdlibFetchInterface(lowerer: Lowerer, node: ts.Node, owner: string): ts.Symbol | null {
const type = lowerer.checker.getBaseTypeOfLiteralType(lowerer.typeOf(node));
for (const symbol of [type.getSymbol(), type.getAliasSymbol()]) {
@@ -1377,6 +1390,7 @@ export function fenceStaticReadableStreamMember(
access: StaticResponseAccess,
use: "read" | "call",
): IrExpr | null {
if (inJavaScript(access.expression)) return null;
const sym = isStdlibFetchInterface(lowerer, access.expression, "ReadableStream");
if (!sym) return null;
const members = fetchAccessMemberNames(lowerer, access);
@@ -1481,6 +1495,9 @@ export function lowerStaticFetchCompanionCall(
return null;
}
const root = call.expression.expression;
// JavaScript reads the web stream classes as constructor values
// (web-streams-dyn.ts).
if (webStreamClassKindOf(lowerer, root) !== null) return null;
const symbol = lowerer.resolveValueSymbol(root);
if ((!symbol || !lowerer.isStdlibSymbol(symbol)) && !isStreamWebReadableStream(lowerer, root))
return null;
@@ -1500,7 +1517,7 @@ export function lowerStaticFetchCompanionCall(
: dynUndefinedExpr(loc),
(first) => ({
kind: "libCall",
fn: "fetch.abortTimeout",
fn: "abort.timeout",
args: [first],
type: DYN,
loc,
@@ -1510,7 +1527,7 @@ export function lowerStaticFetchCompanionCall(
if (call.arguments.nodes.length === 0) {
return {
kind: "libCall",
fn: "fetch.abortNow",
fn: "abort.now",
args: [dynUndefinedExpr(loc)],
type: DYN,
loc,
@@ -1523,7 +1540,7 @@ export function lowerStaticFetchCompanionCall(
lowerLiveWebValue(lowerer, call.arguments.nodes[0]!),
(first) => ({
kind: "libCall",
fn: "fetch.abortNow",
fn: "abort.now",
args: [first],
type: DYN,
loc,
@@ -1541,7 +1558,7 @@ export function lowerStaticFetchCompanionCall(
: dynUndefinedExpr(loc),
(first) => ({
kind: "libCall",
fn: "fetch.abortAny",
fn: "abort.any",
args: [first],
type: DYN,
loc,
@@ -1683,6 +1700,7 @@ export function lowerStaticReadableStreamControllerCall(
): IrExpr | null {
const access = call.expression;
if (
inJavaScript(call) ||
call.questionDotToken ||
call.arguments.nodes.length > 1 ||
(!ts.isPropertyAccessExpression(access) && !ts.isElementAccessExpression(access)) ||
@@ -1739,6 +1757,7 @@ export function lowerStaticReadableStreamCancelCall(
): IrExpr | null {
const access = call.expression;
if (
inJavaScript(call) ||
call.questionDotToken ||
call.arguments.nodes.length > 1 ||
call.arguments.nodes.some((argument) => ts.isSpreadElement(argument)) ||
@@ -1886,7 +1905,7 @@ export function lowerAbortControllerNew(lowerer: Lowerer, expr: ts.NewExpression
const result: IrExpr = {
kind: "libCall",
fn: "fetch.abortControllerNew",
fn: "abort.controllerNew",
args: [],
type: DYN,
loc,
@@ -2163,6 +2182,7 @@ export function lowerStaticReadableStreamReaderCall(
): IrExpr | null {
const access = call.expression;
if (
inJavaScript(call) ||
call.questionDotToken ||
call.arguments.nodes.length !== 0 ||
(!ts.isPropertyAccessExpression(access) && !ts.isElementAccessExpression(access)) ||
packages/compiler/src/frontend/lowering/lower-stmts.ts
+12
View File
@@ -34,6 +34,7 @@ import { InternalCompilerError } from "../../errors.js";
* declarations including destructuring patterns, scoped blocks, control
* flow (if/while/for/for-of/do, switch, try/catch, abrupt completions),
* explicit resource management, and blocked-binding poisoning. */
import { isWebStreamValueInJavaScript, typeInvolvesWebStream } from "./builtins/web-streams-dyn.js";
import * as ts from "../ts7/adapter.js";
import { lowerOptionalIndex } from "./undefined-index.js";
import { processDataMemberValue, programReadsExitCode } from "./builtins/process.js";
@@ -6110,6 +6111,17 @@ export function lowerVarDecl(
// Promise.prototype is an ordinary object the checker types Promise<any>.
if (!decl.type && !hasJsTypeAnnotation(decl) && isPromisePrototypeRead(lowerer, decl.initializer))
type = DYN;
// A web stream value in JavaScript keeps its checked-dynamic identity.
if (
init.type.kind === "dyn" &&
!decl.type &&
!hasJsTypeAnnotation(decl) &&
decl.initializer !== undefined &&
(isWebStreamValueInJavaScript(lowerer, decl.initializer) ||
(isJsSourceFile(ts.getSourceFileOfNode(decl)) &&
typeInvolvesWebStream(lowerer, lowerer.typeOf(decl.name))))
)
type = DYN;
// A const whose initializer has another record or array layout than the
// binding's type keeps the initializer's layout: converting would copy the
// value, so the binding would not be the value its initializer names.
packages/compiler/src/frontend/lowering/lower-stream.ts
+4 -4
View File
@@ -5176,7 +5176,7 @@ function lowerPipelineStages(
{
kind: "varDecl",
localId: ac.id,
init: { kind: "libCall", fn: "fetch.abortControllerNew", args: [], type: DYN, loc },
init: { kind: "libCall", fn: "abort.controllerNew", args: [], type: DYN, loc },
loc,
},
],
@@ -5502,7 +5502,7 @@ function duplexFromBody(lowerer: Lowerer, node: ts.Expression, loc: SrcLoc): IrE
{
kind: "varDecl",
localId: acLocal.id,
init: { kind: "libCall", fn: "fetch.abortControllerNew", args: [], type: DYN, loc },
init: { kind: "libCall", fn: "abort.controllerNew", args: [], type: DYN, loc },
loc,
},
];
@@ -5954,7 +5954,7 @@ export function operatorSignalOptions(lowerer: Lowerer, options: IrExpr, loc: Sr
type: DYN,
loc,
};
const signal: IrExpr = { kind: "libCall", fn: "fetch.abortAny", args: [list], type: DYN, loc };
const signal: IrExpr = { kind: "libCall", fn: "abort.any", args: [list], type: DYN, loc };
return { kind: "libCall", fn: "readable.opSignalOpts", args: [signal], type: DYN, loc };
}
@@ -5978,7 +5978,7 @@ export function reduceSignalSetup(
{
kind: "varDecl",
localId: acLocal.id,
init: { kind: "libCall", fn: "fetch.abortControllerNew", args: [], type: DYN, loc },
init: { kind: "libCall", fn: "abort.controllerNew", args: [], type: DYN, loc },
loc,
},
{
packages/compiler/src/frontend/lowering/lower-timers.ts
+8
View File
@@ -15,6 +15,7 @@ import {
typeKey,
} from "../../ir/ir.js";
import { buildFunctionAdapter } from "./coercions/builders.js";
import { lowerOptionalNumber } from "./lower-exprs.js";
import type { Lowerer } from "./lowerer.js";
/** setTimeout invokes its callback with NO arguments, but @types/node's
@@ -409,6 +410,13 @@ function lowerTimerCall(
type: F64,
loc,
};
} else if (ms.type.kind === "union") {
// A delay that may be undefined: an optional `ms?: number` passed on,
// or an element read (`delays[0]`) whose storage keeps the
// out-of-range undefined. Node converts the delay with ToNumber
// (`after *= 1`), so undefined is NaN, the omitted delay. Other
// unions stay as they are, and the lib boundary refuses them.
ms = lowerOptionalNumber(lowerer, ms, loc, expr.arguments.nodes[1]);
}
if (member === "setInterval") {
return { kind: "libCall", fn: "timers.setInterval", args: [cb, ms], type: F64, loc };
packages/compiler/src/frontend/lowering/lowerer.ts
+754 -14
View File
@@ -10,7 +10,8 @@ import {
arrayRootOf,
lowerWithArraySpecies,
} from "./builtins/array-classes.js";
import { canonShadowHook } from "../canon-map.js";
import { canonShadowHook, mapCanonical, type CanonMapNotes } from "../canon-map.js";
import { Normalizer } from "../canon.js";
import {
compactDeclOrigins,
inPackageFile,
@@ -20,6 +21,26 @@ import {
} from "./decl-origin.js";
import { isDeclBoundaryFile } from "../decl-projection.js";
import { relieveHeapPressure } from "../../heap-pressure.js";
import {
checkedTable,
countTable,
fixedScalar,
flagScalar,
functionTable,
irDataTable,
journalDump,
journalMode,
lowerWaveRecorded,
mapTable,
setTable,
waveStats,
writeRecords,
type JournalEvent,
type JournalScalar,
type JournalTable,
type UnitEvents,
type WaveHost,
} from "./unit-journal.js";
import { moduleNamespaceObject, moduleNamespaceValue } from "./module-namespace-elements.js";
import { keyedHelperName, moduleTagNumber, portableRelativePath } from "./stable-names.js";
import { positionTag, StructuralPositions } from "./structural-positions.js";
@@ -90,6 +111,7 @@ import { fenceWebCryptoMember, lowerWebCryptoCall } from "./builtins/webcrypto.j
import { valueStreamOf } from "./builtins/hash-stream.js";
import { lowerNetAddressCall, lowerNetAddressNew } from "./builtins/net-address.js";
import { lowerMimeCall, lowerMimeNew } from "./builtins/mime.js";
import { lowerWebStreamNew, lowerWebStreamStaticCall } from "./builtins/web-streams-dyn.js";
import { dynUndefinedExpr, nodeThrowExpr, numLit, varRef } from "../../ir/build.js";
import { iteratorSlotAdapter } from "./iterator-adapters.js";
import { iterableSlotValue } from "./iterable-values.js";
@@ -755,6 +777,9 @@ export interface DynamicSlot {
/** The slot keeps its type's compilable shape; only the failing
* components are checked-dynamic (push-down, stage L3). */
partial?: true;
/** The type mapped exactly from its canonical form (stage N2): a static
* slot, never reported as dynamic. */
canonical?: true;
}
/** Localized failure: rounds of the declaration flush retry. */
@@ -1012,6 +1037,9 @@ export interface DegradedType {
lenient: IrType | null;
code: string;
reason: string;
/** The type maps exactly from its canonical form (stage N2): its slots
* are static. */
canonical?: true;
}
/** The Lowerer's pass configuration (see lowerToIr). */
@@ -2214,6 +2242,7 @@ export class Lowerer {
}
noteGenericInstanceDemand(inst: GenericInstance): void {
this.unitEvents?.record("functionDemand", inst.name);
const owner = this.genericDemandOwner;
owner?.functionDemands.push(inst);
const ref = this.genericDemandPriority.get(inst) ?? { rank: [4, Number.MAX_SAFE_INTEGER] };
@@ -2229,6 +2258,7 @@ export class Lowerer {
}
noteGenericClassInstanceDemand(info: ClassInfo): void {
this.unitEvents?.record("classDemand", info.def.name);
const owner = this.genericDemandOwner;
owner?.classDemands.push(info);
const ref = this.genericClassDemandPriority.get(info) ?? { rank: [4, Number.MAX_SAFE_INTEGER] };
@@ -3780,6 +3810,14 @@ export class Lowerer {
* promise operand (returnedPromiseAwait). */
private returnAwaitedPromise = false;
private readonly virtualEdges = new Map<ClassInfo, Set<string>>();
/** The events of the unit being recorded (unit-journal.ts); null
* otherwise. */
unitEvents: UnitEvents | null = null;
/** Fields added to collected classes (class-attributes.ts) and reachable
* units registered while units lower: a unit that changes either has no
* record (journalScalars). */
classLayoutChanges = 0;
unitRegistrations = 0;
classCanBeConstructed(info: ClassInfo): boolean {
return (
@@ -3917,6 +3955,520 @@ export class Lowerer {
}
return name === undefined ? undefined : this.liftedFnsByName.get(name);
}
/** The tables lowering a unit can grow (unit-journal.ts): the
* appendable tables whose entries a unit's record carries, then the
* tables whose growth leaves the unit without a record.
* Lowerer.unjournaled lists the collections that are in neither. */
journalTables(): JournalTable[] {
const sizes = (name: string, of: () => readonly number[]): JournalTable =>
checkedTable(name, () => {
let hash = 0;
for (const size of of()) hash = (Math.imul(hash, 31) + size) | 0;
return hash;
});
return [
functionTable("implicitFns", this.implicitFns, (fn) =>
this.implicitFns.some((other) => other.name === fn),
),
mapTable("equalityHelpers", this.equalityHelpers),
mapTable("arrHofHelpers", this.arrHofHelpers),
irDataTable("arrHofInfo", this.arrHofInfo),
mapTable("unionCallHelpers", this.unionCallHelpers),
mapTable("indexedUnionNumericTwins", this.indexedUnionNumericTwins),
setTable("coercions.checkedNarrows", this.coercions.checkedNarrows),
irDataTable("coercions.checkedNarrowArms", this.coercions.checkedNarrowArms),
mapTable("coercions.retags", this.coercions.retags),
mapTable("coercions.narrows", this.coercions.narrows),
mapTable("coercions.widthNarrows", this.coercions.widthNarrows),
setTable("coercions.widenedReturns", this.coercions.widenedReturns),
mapTable("valueHelpers", this.valueHelpers),
countTable("keyedNameKeys", this.keyedNameKeys),
mapTable("assertHelpers", this.assertHelpers),
mapTable("inspectHelpers", this.inspectHelpers),
setTable("classMethodValueHelpers", this.classMethodValueHelpers),
setTable("freshClosureAdapters", this.freshClosureAdapters),
mapTable("iterDrainHelpers", this.iterDrainHelpers),
mapTable("mapHofHelpers", this.mapHofHelpers),
mapTable("setHofHelpers", this.setHofHelpers),
mapTable("spHofHelpers", this.spHofHelpers),
mapTable("primitiveCtorFns", this.primitiveCtorFns),
mapTable("builtinCallableValueFns", this.builtinCallableValueFns),
mapTable("builtinCallableValueNames", this.builtinCallableValueNames),
countTable("builtinCallableValueLengths", this.builtinCallableValueLengths),
mapTable("regionRunHelpers", this.regionRunHelpers),
mapTable("internTableHelpers", this.internTableHelpers),
functionTable(
"liftedFns",
this.liftedFns,
(fn) => this.liftedFnNamed(fn) !== undefined,
() => {
this.liftedFnsByName.clear();
this.liftedFnsIndexed = 0;
},
),
countTable("closureNameCounts", this.closureNameCounts),
setTable("sharedHelperNames", this.sharedHelperNames),
setTable("sloppyRecordWrites", this.sloppyRecordWrites),
setTable("recordFieldDeletes", this.recordFieldDeletes),
setTable("freezableArrays", this.freezableArrays),
checkedTable("moduleOrder", () => this.moduleOrder.length),
checkedTable("diags", () => this.diags.length),
checkedTable("fnSigsBySymbol", () => this.fnSigsBySymbol.size),
checkedTable("cjsExportsAssignments", () => this.cjsExportsAssignments.size),
checkedTable("inlineRequireCycleCalls", () => this.inlineRequireCycleCalls.size),
checkedTable("staticCallables", () => this.staticCallables.size),
checkedTable("genericFnsBySymbol", () => this.genericFnsBySymbol.size),
checkedTable("objLitGenericFns", () => this.objLitGenericFns.size),
checkedTable("bindingGenericFns", () => this.bindingGenericFns.size),
checkedTable("bindingIdentifiers", () => this.bindingIdentifiers.size),
checkedTable("ownToStringTargets", () => this.ownToStringTargets.size),
checkedTable("neverReassignedCache", () => this.neverReassignedCache.size),
checkedTable("jsBindingWritesByOwner", () => this.jsBindingWritesByOwner.size),
checkedTable("rebindableFunctions", () => this.rebindableFunctions.size),
checkedTable("trapBindings", () => this.trapBindings.size),
checkedTable("deadBindings", () => this.deadBindings.size),
checkedTable("instantiationQueue", () => this.instantiationQueue.length),
checkedTable("genericFunctionDemandOwner", () => this.genericFunctionDemandOwner.size),
checkedTable("genericClassDemandOwner", () => this.genericClassDemandOwner.size),
checkedTable("erasingLambdas", () => this.erasingLambdas.size),
checkedTable("typedArgumentsLambdas", () => this.typedArgumentsLambdas.size),
checkedTable("tupleAliasSymbols", () => this.tupleAliasSymbols.size),
checkedTable("nullishFuncResults", () => this.nullishFuncResults.size),
checkedTable("localClassInstantiations", () => this.localClassInstantiations.length),
checkedTable("localClassInstanceTypes", () => this.localClassInstanceTypes.size),
checkedTable("localClassValueTypes", () => this.localClassValueTypes.size),
checkedTable("dynParamLambdas", () => this.dynParamLambdas.size),
checkedTable("aliasNarrowTypes", () => this.aliasNarrowTypes.size),
checkedTable("classSymbolKeys", () => this.classSymbolKeys.size),
checkedTable("registeredClassSymbols", () => this.registeredClassSymbols.size),
checkedTable("symbolFieldIdentities", () => this.symbolFieldIdentities.size),
checkedTable("httpClientFnBindings", () => this.httpClientFnBindings.size),
checkedTable("emitterComputedPatterns", () => this.emitterComputedPatterns.length),
checkedTable("unstableSwitchBindings", () => this.unstableSwitchBindings.size),
checkedTable("earlyReadVars", () => this.earlyReadVars.size),
checkedTable("runtimeOptionalGlobals", () => this.runtimeOptionalGlobals.size),
checkedTable("cjsExportsThis", () => this.cjsExportsThis.size),
checkedTable("inlineRequireInitDone", () => this.inlineRequireInitDone.size),
checkedTable(
"runtimeOptionalArithmeticGlobals",
() => this.runtimeOptionalArithmeticGlobals.size,
),
checkedTable("runtimeOptionalBindingTypes", () => this.runtimeOptionalBindingTypes.size),
checkedTable("stageSourceTypes", () => this.stageSourceTypes.size),
checkedTable(
"runtimeOptionalAssignedSymbols",
() => this.runtimeOptionalAssignedSymbols.size,
),
checkedTable("runtimeOptionalGenericReturns", () => this.runtimeOptionalGenericReturns.size),
checkedTable(
"runtimeOptionalFunctionReturns",
() => this.runtimeOptionalFunctionReturns.size,
),
checkedTable("contextualFunctionReturns", () => this.contextualFunctionReturns.size),
checkedTable(
"runtimeOptionalArithmeticTypes",
() => this.runtimeOptionalArithmeticTypes.size,
),
checkedTable("runtimeOptionalFields", () => this.runtimeOptionalFields.size),
checkedTable("dateJsonBindings", () => this.dateJsonBindings.size),
checkedTable(
"runtimeOptionalRecordsKeptDeclared",
() => this.runtimeOptionalRecordsKeptDeclared.size,
),
checkedTable("runtimeOptionalPatternTypes", () => this.runtimeOptionalPatternTypes.size),
checkedTable("runtimeOptionalReduceTypes", () => this.runtimeOptionalReduceTypes.size),
checkedTable("runtimeOptionalAssertions", () => this.runtimeOptionalAssertions.size),
checkedTable("presentElementCalls", () => this.presentElementCalls.size),
checkedTable("presentElementLoops", () => this.presentElementLoops.size),
checkedTable("errorFirstLiterals", () => this.errorFirstLiterals.size),
checkedTable("anySymbols", () => this.anySymbols.size),
checkedTable("knownErrors", () => this.knownErrors.size),
checkedTable("shapeOrderMetadataFinalizers", () => this.shapeOrderMetadataFinalizers.length),
checkedTable("receiverTypeOverrides", () => this.receiverTypeOverrides.size),
checkedTable("shapeOrderHelperFinalizers", () => this.shapeOrderHelperFinalizers.length),
checkedTable("emitSpecDone", () => this.emitSpecDone.size),
checkedTable("emitSpecQueue", () => this.emitSpecQueue.length),
checkedTable("structuralClassPeers", () => this.structuralClassPeers.size),
checkedTable("methodLambdasInProgress", () => this.methodLambdasInProgress.size),
checkedTable("classMethodValueSelections", () => this.classMethodValueSelections.size),
checkedTable("stdlibGlobalAliases", () => this.stdlibGlobalAliases.size),
checkedTable("cjsAccessorFns", () => this.cjsAccessorFns.size),
checkedTable("cjsAccessorSnapshots", () => this.cjsAccessorSnapshots.size),
checkedTable("cjsSnapshotReady", () => this.cjsSnapshotReady.size),
checkedTable("matchAllIndexBindings", () => this.matchAllIndexBindings.size),
checkedTable("matchAllDrainIndexes", () => this.matchAllDrainIndexes.size),
checkedTable("numericIterators", () => this.numericIterators.size),
checkedTable("classes", () => this.classes.size),
checkedTable("classBySymbol", () => this.classBySymbol.size),
checkedTable("prototypeMethodAccesses", () => this.prototypeMethodAccesses.size),
checkedTable("replacedSymbolMethods", () => this.replacedSymbolMethods.size),
checkedTable("protocolDispatches", () => this.protocolDispatches.size),
checkedTable("protocolOwnedBindings", () => this.protocolOwnedBindings.size),
checkedTable("receiverFreeCallbackFields", () => this.receiverFreeCallbackFields.size),
checkedTable("virtualJsMethods", () => this.virtualJsMethods.size),
checkedTable("inheritedJsFieldWrites", () => this.inheritedJsFieldWrites.size),
checkedTable("virtualJsMethodArity", () => this.virtualJsMethodArity.size),
checkedTable("checkedVirtualJsMethods", () => this.checkedVirtualJsMethods.size),
checkedTable("unionReturnOverrides", () => this.unionReturnOverrides.size),
checkedTable("checkedVirtualJsAccessors", () => this.checkedVirtualJsAccessors.size),
checkedTable("checkedVirtualJsValueReturns", () => this.checkedVirtualJsValueReturns.size),
checkedTable("globalsBySymbol", () => this.globalsBySymbol.size),
checkedTable("expandoMembers", () => this.expandoMembers.size),
checkedTable("globalsByDeclNode", () => this.globalsByDeclNode.size),
checkedTable("globalsList", () => this.globalsList.length),
checkedTable("npmInitActions", () => this.npmInitActions.size),
checkedTable("jsonInitActions", () => this.jsonInitActions.size),
checkedTable("jsonRequireModules", () => this.jsonRequireModules.size),
checkedTable("jsonRequireCalls", () => this.jsonRequireCalls.size),
checkedTable("dynImports", () => this.dynImports.size),
checkedTable("createRequireImports", () => this.createRequireImports.size),
checkedTable("dynNsBuilders", () => this.dynNsBuilders.size),
checkedTable("checkedCallbackParams", () => this.checkedCallbackParams.size),
checkedTable("valueStreamReceivers", () => this.valueStreamReceivers.size),
checkedTable("stagedArguments", () => this.stagedArguments.size),
checkedTable("moduleNsParamOverrides", () => this.moduleNsParamOverrides.size),
checkedTable("fileTag", () => this.fileTag.size),
checkedTable("forkTargets", () => this.forkTargets.length),
checkedTable("workerTargets", () => this.workerTargets.length),
checkedTable("workerTargetIdByPath", () => this.workerTargetIdByPath.size),
checkedTable("experimentalFiles", () => this.experimentalFiles.size),
checkedTable("internTableNames", () => this.internTableNames.size),
checkedTable("threadSlotNames", () => this.threadSlotNames.size),
checkedTable("threadSlotFamilies", () => this.threadSlotFamilies.size),
checkedTable("threadSlotKeys", () => this.threadSlotKeys.size),
checkedTable("forkTargetIdByPath", () => this.forkTargetIdByPath.size),
checkedTable("nsBlocks", () => this.nsBlocks.size),
checkedTable("initNameOf", () => this.initNameOf.size),
checkedTable("cjsModuleIdOf", () => this.cjsModuleIdOf.size),
checkedTable("cjsModuleFiles", () => this.cjsModuleFiles.length),
checkedTable("moduleGuardOf", () => this.moduleGuardOf.size),
checkedTable("asyncInitFiles", () => this.asyncInitFiles.size),
checkedTable("modulePromiseOf", () => this.modulePromiseOf.size),
checkedTable("asyncCycleRepresentativeOf", () => this.asyncCycleRepresentativeOf.size),
checkedTable("asyncCyclePromiseOf", () => this.asyncCyclePromiseOf.size),
checkedTable("runtimeClassValues", () => this.runtimeClassValues.size),
checkedTable("preparedClassFactories", () => this.preparedClassFactories.size),
checkedTable("preparingClassFactories", () => this.preparingClassFactories.size),
checkedTable("varGlobalEntryInits", () => this.varGlobalEntryInits.size),
checkedTable("cycleInitFlags", () => this.cycleInitFlags.size),
checkedTable("classInitFlags", () => this.classInitFlags.size),
checkedTable("cycleInitDeclarations", () => this.cycleInitDeclarations.size),
checkedTable("uninitializedLexicalGlobals", () => this.uninitializedLexicalGlobals.size),
checkedTable("requireInitFlags", () => this.requireInitFlags.size),
checkedTable("requireEarlyReferences", () => this.requireEarlyReferences.length),
checkedTable("sharedHelpers", () => this.sharedHelpers.size),
checkedTable("dynamicFunctions", () => this.dynamicFunctions.length),
checkedTable("dynamicBodies", () => this.dynamicBodies.length),
checkedTable("deferredDiags", () => this.deferredDiags.size),
checkedTable("deferredClassByName", () => this.deferredClassByName.size),
checkedTable("flushedSymbols", () => this.flushedSymbols.size),
checkedTable("alreadyFlushed", () => this.alreadyFlushed.size),
checkedTable("pendingFlushes", () => this.pendingFlushes.size),
checkedTable("checkedJsFiles", () => this.checkedJsFiles.size),
checkedTable("ffiImports", () => this.ffiImports.length),
checkedTable("ffiImportsByName", () => this.ffiImportsByName.size),
checkedTable("externalTypes", () => this.externalTypes.size),
checkedTable("externalTypeSpecifiersByFile", () => this.externalTypeSpecifiersByFile.size),
checkedTable("blockedBindings", () => this.blockedBindings.size),
checkedTable("runtimeFences", () => this.runtimeFences.length),
checkedTable("provenanceElided", () => this.provenanceElided.length),
checkedTable("statsByFile", () => this.statsByFile.size),
checkedTable(
"reachedBlockedBodyDiagnostics",
() => this.reachedBlockedBodyDiagnostics.length,
),
checkedTable("exprClasses", () => this.exprClasses.length),
checkedTable("exprClassInfoByName", () => this.exprClassInfoByName.size),
checkedTable("collectingExprClasses", () => this.collectingExprClasses.size),
checkedTable("pendingClassExprInits", () => this.pendingClassExprInits.length),
checkedTable("computedClassBases", () => this.computedClassBases.size),
checkedTable("computedCallableBases", () => this.computedCallableBases.size),
checkedTable("mixinInstanceByCall", () => this.mixinInstanceByCall.size),
checkedTable("mixinCollectingCalls", () => this.mixinCollectingCalls.size),
checkedTable("mixinOrdinals", () => this.mixinOrdinals.size),
checkedTable("mixinInstancesByClassNode", () => this.mixinInstancesByClassNode.size),
checkedTable("genericClassByDecl", () => this.genericClassByDecl.size),
checkedTable("collectingClassDeclarations", () => this.collectingClassDeclarations.size),
checkedTable("genericClassInstances", () => this.genericClassInstances.length),
checkedTable("mainPrelude", () => this.mainPrelude.length),
checkedTable("structuralFnInfos", () => this.structuralFnInfos.size),
checkedTable("preservedLayoutTypes", () => this.preservedLayoutTypes.size),
checkedTable("layoutKeptNodes", () => this.layoutKeptNodes.size),
checkedTable("boundAliasInstances", () => this.boundAliasInstances.size),
checkedTable("castGlobalAliases", () => this.castGlobalAliases.size),
checkedTable("methodAccessSites", () => this.methodAccessSites.size),
checkedTable("checkedJsOverrideMethods", () => this.checkedJsOverrideMethods.size),
checkedTable("untypedLoops", () => this.untypedLoops.size),
checkedTable("classBagInitializers", () => this.classBagInitializers.size),
checkedTable("degradedTypes", () => this.degradedTypes.size),
checkedTable("rejectedGroups", () => this.rejectedGroups.size),
checkedTable("lenientPieces", () => this.lenientPieces.size),
checkedTable("dynamicSlots", () => this.dynamicSlots.length),
checkedTable("retriedSymbols", () => this.retriedSymbols.size),
checkedTable("skippedBodies", () => this.skippedBodies.size),
checkedTable("retriedDeclarations", () => this.retriedDeclarations.length),
checkedTable("loweredOnceDiags", () => this.loweredOnceDiags.size),
checkedTable("staticInitsPassed", () => this.staticInitsPassed.size),
checkedTable("deferredCollectors", () => this.deferredCollectors.size),
checkedTable("retriableDiags", () => this.retriableDiags.size),
checkedTable("failedTypes", () => this.failedTypes.size),
checkedTable("retryDegraded", () => this.retryDegraded.length),
checkedTable("declOrigins", () => this.declOrigins.length),
checkedTable("declOriginIds", () => this.declOriginIds.size),
sizes("shapes", () => this.shapes.journalSizes()),
this.unions.journalTable(),
sizes("unions.recIds", () => this.unions.journalSizes()),
sizes("keyOrder", () => this.keyOrder.journalSizes()),
sizes("deferredModuleInitializers", () =>
this.deferredModuleInitializers === null
? []
: this.deferredModuleInitializers.journalSizes(),
),
];
}
/** The fields a unit's record need not carry: settings, caches of
* program facts that are computed again when needed, state of the unit
* being lowered (keyed by its own IR, nodes or frames), scratch that
* unwinds before the unit ends, and state that applying the unit's events
* rebuilds. */
static readonly unjournaled: readonly string[] = [
// settings
"frontendServices",
"ambient",
"overridesAmbient",
"remainder",
"tierMode",
"generalNodeStreams",
"dynNodeStreams",
"targetPlatform",
"startupCrash",
"projectCheckJs",
"libraryCallbacks",
"ffiBindingSymbols",
"priorRetriedSymbols",
"wholeSlotDegradation",
"inferredCollectionTypes",
"foreignElementKeys",
// caches made when first needed
"nanCodedReads",
"mentionsProcessVersions",
"stringIndexBoundsCache",
"functionLocalSubclasses",
"socketEncodingReachable",
"thenablesMayBeDefined",
"unixListenReachable",
"streamPropMutatedSyms",
"readsExitCode",
"requestInitPropMutatedSyms",
"classProtocols",
"cjsSplitRequireBindings",
"cjsWholeExportSymbols",
"cjsLiveMemberExports",
"cjsNamespaces",
"createRequireBindingTargets",
"createRequireElementTargets",
"computedGlobalReads",
"timeValueDateDecisions",
"timeValueDateWrites",
"dynamicOverloadAnswers",
"argumentCountSlots",
"coercions.widenedReturnTypes",
"packageFileMemo",
"slotOriginMemo",
"recordOriginMemo",
"boundaryCastOrder",
"declBoundaryMemo",
"structuredCloneReadCounts",
// state of the unit being lowered
"assertedValues",
"timeValueDateLocals",
"speculatedClosures",
"fsPathPlans",
"fallibleFsCalls",
"failureTouches",
"partialHits",
// sized by journalTables, indexes over journaled tables, and state that
// applying a unit's events rebuilds
"deferredModuleInitializers",
"liftedFnsIndexed",
"liftedFnsByName",
"reachable",
"reachableForArtifacts",
"constructedClasses",
"virtualEdges",
"countedStatements",
"failedStatements",
"genericDemandRoots",
"genericDemandPriority",
"genericClassDemandPriority",
"unitEvents",
// scratch of the lowering in progress, and memos keyed by nodes
"nullishBindings",
"implicitLocalFns",
"builtinClassAliases",
"runtimeOptionalLocals.active",
"runtimeOptionalLocals.scopes",
"narrowing.facts",
"narrowing.verdicts",
"narrowing.effects",
"narrowing.capturedMutable",
"narrowing.staleBindings",
"narrowing.identifierWrites",
"narrowing.widened",
"narrowing.plainContainers",
"runtimeOptionalStorageLocals",
"runtimeOptionalArithmeticLocals",
"runtimeOptionalRoots",
"coercions.planning",
"coercions.copyRetags",
"assertInstances",
"unionLiteralOwners",
"strippedUndefinedArms",
"addedUndefinedArms",
"stdlibSymbols",
"chainRecvByNode",
"neverValued",
"chainNarrowedType",
"chainHandled",
"splitIdentifierUses",
"typeMemo",
"fnStack",
"positions.nodesByKey",
"positions.paths",
"activeStmtLists",
"constructorReturnPlans",
"mixinFnShapes",
];
/** The fields a unit must leave as they were, and the flags it may set
* (unit-journal.ts). */
journalScalars(): JournalScalar[] {
const flag = flagScalar;
const fixed = fixedScalar;
return [
flag(
"handleEventReader",
() => this.handleEventReader,
() => (this.handleEventReader = true),
),
flag(
"writesOntoChildren",
() => this.writesOntoChildren,
() => (this.writesOntoChildren = true),
),
flag(
"regexGroupResults",
() => this.regexGroupResults,
() => (this.regexGroupResults = true),
),
flag(
"requiresHistoricalOrderRelower",
() => this.requiresHistoricalOrderRelower,
() => (this.requiresHistoricalOrderRelower = true),
),
flag(
"requiresGeneralNodeStreams",
() => this.requiresGeneralNodeStreams,
() => (this.requiresGeneralNodeStreams = true),
),
flag(
"requiresDynNodeStreams",
() => this.requiresDynNodeStreams,
() => (this.requiresDynNodeStreams = true),
),
flag(
"errorPrototypeValues",
() => this.errorPrototypeValues,
() => (this.errorPrototypeValues = true),
),
flag(
"usesWorkers",
() => this.usesWorkers,
() => (this.usesWorkers = true),
),
flag(
"usesWebAssembly",
() => this.usesWebAssembly,
() => (this.usesWebAssembly = true),
),
flag(
"usesWasmCustomSections",
() => this.usesWasmCustomSections,
() => (this.usesWasmCustomSections = true),
),
flag(
"usesLifetimeRegions",
() => this.usesLifetimeRegions,
() => (this.usesLifetimeRegions = true),
),
flag(
"cjsModuleGraphEnabled",
() => this.cjsModuleGraphEnabled,
() => (this.cjsModuleGraphEnabled = true),
),
flag(
"globalThisComputedWrite",
() => this.globalThisComputedWrite,
() => (this.globalThisComputedWrite = true),
),
flag(
"untypedIntegrity",
() => this.untypedIntegrity,
() => (this.untypedIntegrity = true),
),
flag(
"partialDegradedTypesMapped",
() => this.partialDegradedTypesMapped,
() => (this.partialDegradedTypesMapped = true),
),
fixed("errorAccessorReflection", () => this.errorAccessorReflection),
fixed("subclassLinks", () => this.subclassLinks),
fixed("errorPathCopies", () => this.errorPathCopies),
fixed("errorMethodDispatchRevision", () => this.errorMethodDispatchRevision),
fixed("errorPropertyDispatchRevision", () => this.errorPropertyDispatchRevision),
fixed("childAdditions", () => this.childAdditions),
fixed("emitterEventTable", () => this.emitterEventTable),
fixed("globalThisWrittenNames", () => this.globalThisWrittenNames),
fixed("importRetryModules", () => this.importRetryModules),
fixed("localSubclassIndex", () => this.localSubclassIndex),
// State of the lowering in progress, which a unit restores.
fixed("genericInstantiationPath", () => this.genericInstantiationPath),
fixed("genericDemandOwner", () => this.genericDemandOwner),
fixed("typeParamBindings", () => this.typeParamBindings),
fixed("staticFnNames", () => this.staticFnNames),
fixed("erasingDepth", () => this.erasingDepth),
fixed("instanceResultClass", () => this.instanceResultClass),
fixed("typeParamTsBindings", () => this.typeParamTsBindings),
fixed("implicitParamTypes", () => this.implicitParamTypes),
fixed("fspGlobHead", () => this.fspGlobHead),
fixed("instantiationContext", () => this.instantiationContext),
fixed("suppressStats", () => this.suppressStats),
fixed("emitSpecCtx", () => this.emitSpecCtx),
fixed("assertCallOptions", () => this.assertCallOptions),
fixed("currentClass", () => this.currentClass),
fixed("closureRoot", () => this.closureRoot),
fixed("superConstructorEdge", () => this.superConstructorEdge),
fixed("returnAwaitedPromise", () => this.returnAwaitedPromise),
fixed("collecting", () => this.collecting),
fixed("diagSink", () => this.diagSink),
fixed("speculativeTypeQueryDepth", () => this.speculativeTypeQueryDepth),
fixed("mixinTypeContext", () => this.mixinTypeContext),
fixed("pendingLabels", () => this.pendingLabels),
fixed("onEdge", () => this.onEdge),
fixed("onExprClassCollected", () => this.onExprClassCollected),
fixed("onLateClassCollected", () => this.onLateClassCollected),
fixed("onDeclarationRegistered", () => this.onDeclarationRegistered),
fixed("classLayoutChanges", () => this.classLayoutChanges),
fixed("unitRegistrations", () => this.unitRegistrations),
fixed("liftDynFieldPolicy", () => this.liftDynFieldPolicy),
fixed("coercionOriginNode", () => this.coercionOriginNode),
fixed("coercionOriginUnion", () => this.coercionOriginUnion),
fixed("coercionFieldOrigin", () => this.coercionFieldOrigin),
];
}
/** Coverage remainder mode: the reachability gate inverts (see wantBody)
* and no module is built. */
readonly remainder: boolean;
@@ -4067,6 +4619,12 @@ export class Lowerer {
countStatement(node: ts.Node, file: string | undefined): void {
if (this.countedStatements.has(node)) return;
this.countedStatements.add(node);
this.unitEvents?.record(
"statement",
ts.getSourceFileOfNode(node).fileName,
file ?? null,
node.index,
);
this.stats.statementsTotal++;
if (file !== undefined) this.bumpFileStat(file, "total");
}
@@ -4076,6 +4634,12 @@ export class Lowerer {
countStatementFailed(node: ts.Node, file: string | undefined): void {
if (this.failedStatements.has(node)) return;
this.failedStatements.add(node);
this.unitEvents?.record(
"failedStatement",
ts.getSourceFileOfNode(node).fileName,
file ?? null,
node.index,
);
this.stats.statementsFailed++;
if (file !== undefined) this.bumpFileStat(file, "failed");
}
@@ -4238,9 +4802,11 @@ export class Lowerer {
// checked dynamic cuts the knot while it does.
if (degraded.lenient === null) {
degraded.lenient = DYN;
degraded.lenient = this.lenientType(type);
degraded.lenient =
(degraded.canonical === true ? this.canonicalType(type) : null) ??
this.lenientType(type);
}
if (degraded.lenient.kind !== "dyn") {
if (degraded.lenient.kind !== "dyn" && degraded.canonical !== true) {
this.partialHits[this.partialHits.length - 1]?.add(id);
this.partialDegradedTypesMapped = true;
}
@@ -9312,6 +9878,7 @@ export class Lowerer {
lower: () => IrFunction | null,
): void => {
units.set(name, { order: unitOrder++, roots, lower });
this.unitRegistrations++;
if (reachable.has(name)) queue.push(name);
metadataPriority.set(name, [3, expressionMetadataOrder++]);
};
@@ -9352,6 +9919,7 @@ export class Lowerer {
roots: functionRoots(decl),
lower: () => this.lowerFunction(decl),
});
this.unitRegistrations++;
if (reachable.has(sig.name)) queue.push(sig.name);
metadataPriority.set(sig.name, [3, expressionMetadataOrder++]);
} else if (ts.isClassDeclaration(decl)) {
@@ -9372,6 +9940,7 @@ export class Lowerer {
roots: functionRoots(entry.member),
lower: () => lowerStaticMethod(this, info, name),
});
this.unitRegistrations++;
if (reachable.has(unit)) queue.push(unit);
metadataPriority.set(unit, [3, expressionMetadataOrder++]);
}
@@ -9515,25 +10084,36 @@ export class Lowerer {
const rootEdge = this.onEdge;
for (const root of extraRoots ?? []) rootEdge?.(root);
const shuffle = lowerShuffle();
const lowerUnitBody = (name: string): IrFunction | null => {
const unit = units.get(name)!;
const priority = metadataPriority.get(name) ?? [3, expressionMetadataOrder++];
const owner = demandOwner(priority);
return this.withGenericDemandOwner(owner, () =>
this.shapes.withDeclaredOrderPriority(priority, () =>
this.withClosureRoot(name, unit.lower),
),
);
};
const recorded = journalMode() === "serial" ? waveStats() : null;
const host = recorded === null ? null : this.waveHost(loweredUnits, lowerUnitBody);
const dump = recorded === null ? null : journalDump();
const recordLines: string[] = [];
const drainUnits = (): void => {
while (queue.length > 0) {
const wave = queue.splice(0);
shuffle?.(wave);
this.checker.prefetchRoots(wave.flatMap((name) => units.get(name)!.roots));
if (host !== null && recorded !== null) {
lowerWaveRecorded(host, wave, recorded, dump === null ? null : recordLines);
continue;
}
for (const name of wave) {
// A body-level poison outside the per-statement catches (a fenced
// constructor/method parameter default lowered by declareParams):
// every edge fired before poisoning remains retained; the diagnostic
// stays recorded and the member stays omitted.
try {
const unit = units.get(name)!;
const priority = metadataPriority.get(name) ?? [3, expressionMetadataOrder++];
const owner = demandOwner(priority);
const fn = this.withGenericDemandOwner(owner, () =>
this.shapes.withDeclaredOrderPriority(priority, () =>
this.withClosureRoot(name, unit.lower),
),
);
const fn = lowerUnitBody(name);
if (fn) loweredUnits.set(name, fn);
relieveHeapPressure();
} catch (e) {
@@ -9584,6 +10164,11 @@ export class Lowerer {
this.restoreGenericClassInstanceOrder(clsInstLowered);
}
classDispatch.finalize(this);
if (dump !== null) writeRecords(dump, recordLines);
if (recorded !== null && process.env["SCRIPTC_TIMING"] === "1")
process.stderr.write(
`scriptc lowering ${JSON.stringify({ phase: "unit-records", ...recorded })}\n`,
);
const orderedUnits = unitsInOrder();
this.settleGenericDemandPriorities();
this.shapes.settleDeclaredOrderPriorities();
@@ -9628,12 +10213,101 @@ export class Lowerer {
return { reachable, result: this.finishModule(functions) };
}
/** The lowerer's side of a recorded wave (unit-journal.ts).
* `lowerUnitBody` lowers a unit as reachable emit does. */
private waveHost(
loweredUnits: Map<string, IrFunction>,
lowerUnitBody: (name: string) => IrFunction | null,
): WaveHost {
const scalars = this.journalScalars();
const flags = new Map(scalars.map((scalar) => [scalar.name, scalar]));
let instances = new Map<string, GenericInstance>();
let indexed = -1;
const instanceNamed = (name: string): GenericInstance | undefined => {
if (indexed !== this.instantiationQueue.length) {
instances = new Map(this.instantiationQueue.map(({ inst }) => [inst.name, inst]));
indexed = this.instantiationQueue.length;
}
return instances.get(name);
};
const classNamed = (name: string): ClassInfo | undefined => {
const info = this.classes.get(name);
return info !== undefined && info.def.name === name ? info : undefined;
};
const files = new Map<string, ts.SourceFile | undefined>();
const statementNode = (event: JournalEvent): ts.Node | undefined => {
let sf = files.get(event.name);
if (sf === undefined && !files.has(event.name)) {
sf = this.program.getSourceFile(event.name);
files.set(event.name, sf);
}
if (sf === undefined) return undefined;
try {
return sf.file.node(event.index);
} catch {
return undefined;
}
};
return {
tables: this.journalTables(),
scalars,
lowerRecorded: (name, events) => {
this.unitEvents = events;
try {
const fn = lowerUnitBody(name);
relieveHeapPressure();
return { fn, poisoned: false };
} catch (e) {
if (!(e instanceof PoisonError)) throw e;
return { fn: null, poisoned: true };
} finally {
this.unitEvents = null;
}
},
store: (name, fn) => {
if (fn) loweredUnits.set(name, fn);
},
statementCounts: () => ({
total: this.stats.statementsTotal,
failed: this.stats.statementsFailed,
other: this.stats.functionsSkipped + this.diags.length,
}),
resolves: (event) => {
switch (event.kind) {
case "virtualEdge":
case "classDemand":
return classNamed(event.name) !== undefined;
case "functionDemand":
return instanceNamed(event.name) !== undefined;
case "statement":
case "failedStatement":
return statementNode(event) !== undefined;
case "flag":
return flags.get(event.name)?.set !== undefined;
default:
return true;
}
},
};
}
/** Reachability hook (see emitReachable): fires when lowering resolves a
* reference to a lowerable body. */
noteEdge(name: string): void {
// Called as a plain function: the instance is not its receiver.
const onEdge = this.onEdge;
if (onEdge) onEdge(name);
const events = this.unitEvents;
if (events === null) {
if (onEdge) onEdge(name);
return;
}
events.record(this.superConstructorEdge ? "superConstructorEdge" : "edge", name);
events.depth++;
try {
if (onEdge) onEdge(name);
} finally {
events.depth--;
}
}
/** A subclass constructor's super() edge: the base constructor runs, but
@@ -9654,6 +10328,21 @@ export class Lowerer {
* them). */
noteVirtualEdge(info: ClassInfo, method: string): void {
if (!this.onEdge) return;
const events = this.unitEvents;
if (events === null) {
this.virtualEdgeOf(info, method);
return;
}
events.record("virtualEdge", info.def.name, method);
events.depth++;
try {
this.virtualEdgeOf(info, method);
} finally {
events.depth--;
}
}
private virtualEdgeOf(info: ClassInfo, method: string): void {
if (this.constructedClasses) {
const methods = this.virtualEdges.get(info) ?? new Set<string>();
methods.add(method);
@@ -9930,8 +10619,10 @@ export class Lowerer {
}
if (committed === undefined) return null;
this.degradeType(type, committed.code, committed.message);
const canonical = this.isCanonicalDegraded(id);
const partial = this.degradedTypes.get(id)?.lenient?.kind !== "dyn";
return {
...(canonical ? { canonical: true as const } : {}),
...(partial ? { partial: true as const } : {}),
loc: committed.loc,
failure,
@@ -10076,6 +10767,14 @@ export class Lowerer {
this.degradedTypes.set(id, { lenient: DYN, code, reason });
return true;
}
// Canonical normalization (stage N2): a type the per-form rules miss
// maps exactly from its canonical form when that form maps; no dynamic
// slot then. Otherwise the lenient (pushed-down) form.
const canonical = this.canonicalType(type);
if (canonical !== null) {
this.degradedTypes.set(id, { lenient: canonical, code, reason, canonical: true });
return true;
}
this.degradedTypes.set(id, {
lenient: this.wholeSlotDegradation ? DYN : this.lenientType(type),
code,
@@ -10084,13 +10783,44 @@ export class Lowerer {
return true;
}
/** The IR type of a type's canonical form (canon-map.ts), or null when
* the form is irreducible, unbound, only the checked dynamic, or holds a
* union that mapped to the checked dynamic (such a slot is dynamic in
* part, so it degrades like any other). Runs only at localized failure's
* commit points, so a program that compiles today never consults it. */
canonicalType(type: ts.Type): IrType | null {
try {
const resolve = this.typeCtx.resolveTypeParamTs;
const c = new Normalizer(this.checker).canon(type, resolve ? (t) => resolve(t) : new Map());
if (c.k === "opaque" || c.k === "param") return null;
const { typeMemo: _strictMemo, ...strict } = this.typeCtx;
const notes: CanonMapNotes = { collapsedToDynamic: false };
const mapped = mapCanonical(c, { ...strict, canMemoizeType: () => false }, 0, notes);
if (notes.collapsedToDynamic) return null;
return mapped === null || mapped.kind === "void" || mapped.kind === "dyn" ? null : mapped;
} catch {
return null;
}
}
/** The degraded type of `id` maps exactly (canonical form): its slots
* are static, not dynamic. */
isCanonicalDegraded(id: number): boolean {
return this.degradedTypes.get(id)?.canonical === true;
}
/** Some degraded type kept a pushed-down (partial) shape, or one was
* mapped before a retry widened it: the IR lowered before the retry keeps
* that shape. */
hasPartialDegradedTypes(): boolean {
if (this.partialDegradedTypesMapped) return true;
for (const degraded of this.degradedTypes.values())
if (degraded.lenient !== null && degraded.lenient.kind !== "dyn") return true;
if (
degraded.canonical !== true &&
degraded.lenient !== null &&
degraded.lenient.kind !== "dyn"
)
return true;
return false;
}
@@ -10124,6 +10854,7 @@ export class Lowerer {
/** Records a dynamic slot whose unit lowered with it (one per site). */
noteDynamicSlot(slot: DynamicSlot): void {
if (slot.canonical === true) return;
const seen = this.dynamicSlots.some(
(s) =>
s.loc.file === slot.loc.file &&
@@ -13282,15 +14013,22 @@ export class Lowerer {
prior:
prior === undefined
? undefined
: { lenient: prior.lenient, code: prior.code, reason: prior.reason },
: {
lenient: prior.lenient,
code: prior.code,
reason: prior.reason,
...(prior.canonical === true ? { canonical: true as const } : {}),
},
});
this.degradeType(type, diag.code, diag.message);
const canonical = this.isCanonicalDegraded(id);
const partial = this.degradedTypes.get(id)?.lenient?.kind !== "dyn";
slots.push({
loc: diag.loc,
failure,
code: diag.code,
reason: diag.message,
...(canonical ? { canonical: true as const } : {}),
...(partial ? { partial: true as const } : {}),
});
}
@@ -15109,6 +15847,7 @@ export class Lowerer {
lowerStaticFetchCompanionCall(this, expr) ??
lowerStaticAbortControllerCall(this, expr) ??
lowerStaticAbortSignalListenerCall(this, expr) ??
lowerWebStreamStaticCall(this, expr) ??
lowerStaticReadableStreamCancelCall(this, expr) ??
lowerStaticReadableStreamControllerCall(this, expr) ??
lowerStaticReadableStreamReaderCall(this, expr) ??
@@ -15361,6 +16100,7 @@ export class Lowerer {
lowerBlobNew(this, expr) ??
lowerNetAddressNew(this, expr) ??
lowerMimeNew(this, expr) ??
lowerWebStreamNew(this, expr) ??
lowerStaticReadableStreamNew(this, expr) ??
lowerStaticTransformStreamNew(this, expr) ??
lowerConsoleNew(this, expr);
packages/compiler/src/frontend/lowering/surfaces.ts
+4
View File
@@ -1,3 +1,4 @@
import { isWebStreamMemberInJavaScript } from "./builtins/web-streams-dyn.js";
import * as ts from "../ts7/adapter.js";
import type { Lowerer } from "./lowerer.js";
import { staticCondition, staticallySelected } from "./static-typeof.js";
@@ -3279,6 +3280,9 @@ export function stdlibMemberFence(lowerer: Lowerer, access: ts.PropertyAccessExp
// below this fence answers, member-or-refusal ladder, so no compile
// fence belongs here.
if (lowerer.mapTypeOf(lowerer.typeOf(recv))?.kind === "dyn") return;
// JavaScript reads web stream values (read results, sources, strategies)
// as checked-dynamic objects (web-streams-dyn.ts).
if (isWebStreamMemberInJavaScript(lowerer, access)) return;
// Name the container the way the source reads: the global's name when
// the receiver IS a stdlib global (Math, process), the dotted path for
// a member of one (process.stdout — its TYPE text would be the useless
packages/compiler/src/frontend/lowering/unit-journal.test.ts
+376
View File
@@ -0,0 +1,376 @@
import { expect, test } from "vitest";
import type { Program, SourceFile } from "../ts7/adapter.js";
import { Lowerer } from "./lowerer.js";
import {
checkedTable,
contentDigest,
encodeUnitFunction,
flagScalar,
functionTable,
lowerWaveRecorded,
mapTable,
recordUnit,
setTable,
waveStats,
type WaveHost,
} from "./unit-journal.js";
import type { IrFunction } from "../../ir/ir.js";
const lowerer = (): Lowerer =>
new Lowerer(
{ getTypeChecker: () => ({}), isSourceFileDefaultLibrary: () => false } as unknown as Program,
{ fileName: "/island/entry.ts" } as SourceFile,
[],
);
const isCollection = (value: unknown): boolean =>
value instanceof Map || value instanceof Set || Array.isArray(value);
/** The registries whose collections their journalSizes and journalTable
* cover, or that are memos of them, with those collections. */
const sized: Record<string, readonly string[]> = {
shapes: [
"keyedNameKeys",
"byKey",
"byId",
"declaredOrderPriority",
"declaredOrderCandidates",
"derivedDeclaredOrderFinalizers",
"shapes",
"recIds",
"pendingRec",
"extendedNames",
],
unions: ["keyedNameKeys", "byKey", "byId", "armTags", "unions", "recIds", "pendingRec"],
keyOrder: ["literals", "copies", "methodResults", "observed", "changed"],
};
/** Scratch of the registries' own mapping (it unwinds before mapType returns). */
const scratch = new Set([
"shapes.inProgress",
"shapes.currentDeclaredOrderPriority",
"unions.inProgress",
]);
/** Owned objects whose collections are listed one by one. */
const owned = new Set(["coercions", "narrowing", "runtimeOptionalLocals", "positions"]);
test("every collection of the lowerer is journaled, checked or left out by name", () => {
const subject = lowerer();
const named = new Set([
...subject.journalTables().map((table) => table.name),
...Lowerer.unjournaled,
]);
const missing: string[] = [];
const fields = subject as unknown as Record<string, unknown>;
for (const field of Object.keys(fields)) {
const value = fields[field];
if (isCollection(value)) {
if (!named.has(field)) missing.push(field);
continue;
}
if (value === null || typeof value !== "object") continue;
const inner = value as Record<string, unknown>;
const covered = sized[field];
if (covered === undefined && !owned.has(field)) continue;
for (const key of Object.keys(inner)) {
const path = `${field}.${key}`;
if (!isCollection(inner[key]) || scratch.has(path)) continue;
if (covered !== undefined ? !covered.includes(key) : !named.has(path)) missing.push(path);
}
}
expect(missing).toEqual([]);
});
test("every scalar field of the lowerer is journaled or left out by name", () => {
const subject = lowerer();
const named = new Set([
...subject.journalScalars().map((scalar) => scalar.name),
...Lowerer.unjournaled,
]);
const fields = subject as unknown as Record<string, unknown>;
const missing = Object.keys(fields).filter((field) => {
const value = fields[field];
if (isCollection(value) || typeof value === "function") return false;
const scalar = value === null || value === undefined || typeof value !== "object";
return scalar && !named.has(field);
});
expect(missing).toEqual([]);
});
test("every name the lists hold is a field of the lowerer", () => {
const subject = lowerer();
const fields = subject as unknown as Record<string, unknown>;
const names = [
...subject.journalTables().map((table) => table.name),
...subject.journalScalars().map((scalar) => scalar.name),
...Lowerer.unjournaled,
];
const missing = names.filter((name) => {
const [head, key] = name.split(".");
if (!(head! in fields)) return true;
if (key === undefined) return false;
const inner = fields[head!];
return inner === null || typeof inner !== "object" || !(key in inner);
});
expect(missing).toEqual([]);
});
const fn = (name: string, ret: "f64" | "string" = "f64"): IrFunction =>
({
name,
params: [],
returnType: { kind: ret },
locals: [],
body: [],
loc: { file: "a.ts", start: 0, end: 0 },
}) as unknown as IrFunction;
test("a map table hands over the entries added after a size", () => {
const map = new Map<string, string>([["a", "%h.1"]]);
const table = mapTable("helpers", map);
map.set("b", "%h.2");
map.set("c", "%h.3");
const entries = table.entries!(1)!;
expect(entries.length).toBe(2);
const parent = new Map<string, string>([
["a", "%h.1"],
["c", "%h.3"],
]);
mapTable("helpers", parent).apply!(entries);
expect([...parent]).toEqual([
["a", "%h.1"],
["c", "%h.3"],
["b", "%h.2"],
]);
table.truncate!(1);
expect([...map]).toEqual([["a", "%h.1"]]);
});
test("a held table notices a replaced entry", () => {
const map = new Map<string, string>([["a", "%h.1"]]);
const held = mapTable("helpers", map).hold!();
map.set("b", "%h.2");
expect(held.same()).toBe(true);
map.set("a", "%h.9");
expect(held.same()).toBe(false);
const set = new Set(["x"]);
const heldSet = setTable("names", set).hold!();
set.delete("x");
set.add("x");
expect(heldSet.same()).toBe(true);
set.delete("x");
expect(heldSet.same()).toBe(false);
});
test("a function table keeps the function an earlier unit made", () => {
const worker = [fn("%a")];
const table = functionTable("lifted", worker, () => false);
worker.push(fn("%b"), fn("%c"));
const entries = table.entries!(1)!;
const parent = [fn("%c", "string")];
const target = functionTable("lifted", parent, (name) =>
parent.some((other) => other.name === name),
);
target.apply!(entries);
expect(parent.map((f) => [f.name, f.returnType.kind])).toEqual([
["%c", "string"],
["%b", "f64"],
]);
});
test("a set table adds what the parent lacks", () => {
const set = new Set(["x"]);
const table = setTable("names", set);
set.add("y");
const parent = new Set<string>();
setTable("names", parent).apply!(table.entries!(1)!);
expect([...parent]).toEqual(["y"]);
});
/** A wave host over one helper map, one checked list and one flag, whose
* units are lowered by `lower`. */
function host(lower: (name: string, state: HostState) => IrFunction | null): {
host: WaveHost;
state: HostState;
} {
const state: HostState = { helpers: new Map(), checked: [], flag: false, stored: new Map() };
return {
state,
host: {
tables: [
mapTable("helpers", state.helpers),
checkedTable("checked", () => state.checked.length),
],
scalars: [
flagScalar(
"flag",
() => state.flag,
() => (state.flag = true),
),
],
lowerRecorded: (name, events) => {
events.record("edge", `%callee.${name}`);
return { fn: lower(name, state), poisoned: false };
},
store: (name, unitFn) => {
if (unitFn !== null) state.stored.set(name, unitFn);
},
statementCounts: () => ({ total: 0, failed: 0, other: 0 }),
resolves: () => true,
},
};
}
interface HostState {
helpers: Map<string, string>;
checked: string[];
flag: boolean;
stored: Map<string, IrFunction>;
}
test("a recorded wave keeps what the records carry, in wave order", () => {
const made = new Map<string, IrFunction>();
const { host: subject, state } = host((name, held) => {
held.helpers.set(`key.${name}`, `%h.${name}`);
if (name === "b") held.flag = true;
if (name === "c") held.checked.push(name);
const unitFn = fn(`%${name}`);
made.set(name, unitFn);
return unitFn;
});
const stats = waveStats();
lowerWaveRecorded(subject, ["a", "b", "c"], stats);
expect([...state.helpers.keys()]).toEqual(["key.a", "key.b", "key.c"]);
expect(stats).toMatchObject({
waves: 1,
units: 3,
recorded: 2,
unrecorded: 1,
reasons: { checked: 1 },
});
// Recorded units keep the function decoded from their record; the unit
// without one keeps the function lowering made.
expect(state.stored.get("a")).not.toBe(made.get("a"));
expect(state.stored.get("a")).toEqual(made.get("a"));
expect(state.stored.get("c")).toBe(made.get("c"));
});
test("a unit's record lists its events, flags and added entries", () => {
const { host: subject } = host((name, held) => {
held.helpers.set(`key.${name}`, `%h.${name}`);
held.flag = true;
return fn(`%${name}`);
});
const { record } = recordUnit(subject, "a");
expect(record).not.toBeNull();
expect(record!.events.map((event) => [event.kind, event.name])).toEqual([
["edge", "%callee.a"],
["flag", "flag"],
]);
expect(record!.tables).toEqual({ helpers: [`"key.a"\n"%h.a"`] });
expect(JSON.parse(JSON.stringify(record))).toEqual(record);
});
test("a unit that changes an entry it did not add has no record", () => {
const { host: subject, state } = host((name, held) => {
held.helpers.set("shared", `%h.${name}`);
return null;
});
state.helpers.set("shared", "%h.first");
const unit = recordUnit(subject, "a");
expect(unit.record).toBeNull();
expect(unit.reasons).toEqual(["helpers:replaced"]);
});
test("a recorded wave throws when a unit changes a held function in place", () => {
const functions = [fn("%shared")];
const subject: WaveHost = {
tables: [functionTable("lifted", functions, (name) => functions.some((f) => f.name === name))],
scalars: [],
lowerRecorded: (name) => {
if (name === "b")
functions[0]!.locals.push({ id: "x.0", name: "x", type: { kind: "f64" } } as never);
return { fn: fn(`%${name}`), poisoned: false };
},
store: () => {},
statementCounts: () => ({ total: 0, failed: 0, other: 0 }),
resolves: () => true,
};
lowerWaveRecorded(subject, ["a"], waveStats());
expect(() => lowerWaveRecorded(subject, ["a", "b"], waveStats())).toThrow(
"a unit changed lifted entry %shared in place without naming it",
);
});
/** A wave host over one function table holding `%shared` and one checked
* list, whose units change `%shared` in place and name it in their IR. */
function sharedHost(grows: (name: string) => boolean): { host: WaveHost; checked: string[] } {
const functions = [fn("%shared")];
const checked: string[] = [];
return {
checked,
host: {
tables: [
functionTable("lifted", functions, (name) => functions.some((f) => f.name === name)),
checkedTable("checked", () => checked.length),
],
scalars: [],
lowerRecorded: (name) => {
functions[0]!.body = [{ kind: "exprStmt", note: name } as never];
if (grows(name)) checked.push(name);
return {
fn: { ...fn(`%${name}`), body: [{ kind: "call", callee: "%shared" } as never] },
poisoned: false,
};
},
store: () => {},
statementCounts: () => ({ total: 0, failed: 0, other: 0 }),
resolves: () => true,
},
};
}
test("units without a record may change a function they name in place", () => {
const { host: subject } = sharedHost(() => true);
const stats = waveStats();
lowerWaveRecorded(subject, ["a", "b"], stats);
expect(stats).toMatchObject({ recorded: 0, unrecorded: 2, reasons: { checked: 2 } });
});
test("a unit that changes a function it names in place has no record", () => {
const { host: subject } = sharedHost(() => false);
const stats = waveStats();
lowerWaveRecorded(subject, ["a"], stats);
expect(stats).toMatchObject({ recorded: 0, unrecorded: 1, reasons: { "lifted:changed": 1 } });
});
test("a record lists the held functions it names, with their digests", () => {
const functions = [fn("%shared"), fn("%unused")];
const subject: WaveHost = {
tables: [functionTable("lifted", functions, (name) => functions.some((f) => f.name === name))],
scalars: [],
lowerRecorded: () => {
functions.push(fn("%own"));
return {
fn: {
...fn("%a"),
body: [
{ kind: "call", callee: "%shared" },
{ kind: "call", callee: "%own" },
] as never,
},
poisoned: false,
};
},
store: () => {},
statementCounts: () => ({ total: 0, failed: 0, other: 0 }),
resolves: () => true,
};
const { record } = recordUnit(subject, "a");
// The function the unit added is in its entries, not its reads.
expect(record!.reads).toEqual({
lifted: [["%shared", contentDigest(encodeUnitFunction(functions[0]!)!)]],
});
expect(Object.keys(record!.tables)).toEqual(["lifted"]);
});
packages/compiler/src/frontend/lowering/unit-journal.ts
+675
View File
@@ -0,0 +1,675 @@
/* Per-unit records of reachable-emit lowering (Lowerer.emitReachable).
*
* A unit's record is what another process needs to reach the state the
* unit's lowering reached: the unit's IR, the entries it added to the
* tables that collect lowering results (Lowerer.journalTables), and the
* calls it made into reachability, generic demand and statement counting
* (JournalEvent). Records hold only strings, numbers and names, so they can
* be written to a file and read by another process or another build.
*
* A unit has no record when its lowering changed state that no record
* carries: a table that is not appendable grew, a field changed, an entry
* held before the unit was replaced, or a diagnostic was emitted. Such a
* unit lowers again wherever its record would have been applied.
*
* SCRIPTC_LOWER_JOURNAL=serial records every unit while it lowers, and
* replaces the unit's results with the ones its record carries: the
* function decoded from the record and the table entries added again from
* it. The build's output is then the output of applying the records, which
* tests compare with ordinary lowering. SCRIPTC_LOWER_JOURNAL_DUMP=<dir>
* turns the serial mode on and appends each unit's record to
* <dir>/records.jsonl, one JSON line per unit in lowering order; a unit
* without a record has a line with its name and reasons. */
import { appendFileSync, mkdirSync } from "node:fs";
import { join } from "node:path";
import { InternalCompilerError } from "../../errors.js";
import type { IrFunction } from "../../ir/ir.js";
import { deserializeFunction, encodeFunction } from "../../ir/serialize.js";
/** A function as records hold it, or null when it does not encode. Every
* function a record holds goes through this pair. */
export function encodeUnitFunction(fn: IrFunction): string | null {
return encodeFunction(fn) ?? null;
}
/** A function from its record encoding (encodeUnitFunction). */
export function decodeUnitFunction(text: string): IrFunction {
return deserializeFunction(text);
}
/** A call a unit made while it lowered, which applying its record makes
* again. */
export interface JournalEvent {
readonly kind:
| "edge"
| "superConstructorEdge"
| "virtualEdge"
| "functionDemand"
| "classDemand"
| "statement"
| "failedStatement"
| "flag";
/** The edge's body name, the class of a virtual edge or class demand, the
* generic instance's name, a counted statement's source file, or the
* flag's field. */
readonly name: string;
/** The virtual edge's method, or the file a statement counts toward. */
readonly detail: string | null;
/** A counted statement's node index in its source file. */
readonly index: number;
}
/** One unit's record. */
export interface UnitRecord {
readonly name: string;
/** The unit's function in the compact IR encoding, or null when the unit
* lowered to nothing. */
readonly fn: string | null;
/** True when the unit's lowering ended in a poison: its diagnostics and
* the edges it fired before poisoning stay, and it has no function. */
readonly poisoned: boolean;
readonly events: readonly JournalEvent[];
/** The encoded entries the unit added, by table name, for the tables it
* grew. */
readonly tables: Readonly<Record<string, readonly string[]>>;
/** The entries the unit's IR and new entries name that its tables held
* before it, by table name, each with a digest of its content. Applying
* the record needs each to be held with the same content. */
readonly reads: Readonly<Record<string, readonly (readonly [string, string])[]>>;
}
/** A table a unit's lowering can grow. An appendable table only adds
* entries at the end (Map and Set insertion order, array pushes), so a
* unit's additions are the entries past the size the table had when the
* unit started. A checked table has only `size`: when it grows, the unit
* has no record. */
export interface JournalTable {
readonly name: string;
size(): number;
/** The encoded entries from position `from` on, or null when one of
* them does not encode. */
readonly entries?: (from: number) => string[] | null;
/** Adds the encoded entries whose keys this process's table lacks. */
readonly apply?: (entries: readonly string[]) => void;
/** A digest of the content of the entry under `key`, when the table
* holds one before position `before`; null otherwise. */
readonly digest?: (key: string, before: number) => string | null;
/** Removes the entries from position `from` on. */
readonly truncate?: (from: number) => void;
/** The entries the table holds now. */
readonly hold?: () => HeldEntries;
/** The content of the entries the table holds now, for tables whose
* values are objects a unit could change in place. */
readonly contents?: () => HeldContents;
}
/** The content of a table's entries at one point (JournalTable.contents). */
export interface HeldContents {
/** Adds the content of the entries past the ones this holds. */
extend(): void;
/** The key of an entry this holds whose content has changed, or null. */
changed(): string | null;
/** The key of an entry among `keys` whose content has changed, or null. */
changedAmong(keys: ReadonlySet<string>): string | null;
/** Takes the current content of the entries among `keys`. */
refresh(keys: ReadonlySet<string>): void;
}
/** A digest of `text`: its length and two 32-bit FNV-1a hashes. */
export function contentDigest(text: string): string {
let a = 0x811c9dc5;
let b = 0x01000193 ^ 0x5bd1e995;
for (let i = 0; i < text.length; i++) {
const c = text.charCodeAt(i);
a = Math.imul(a ^ c, 0x01000193);
b = Math.imul(b ^ c, 0x01000193) ^ (b >>> 13);
}
return `${text.length}:${(a >>> 0).toString(16)}${(b >>> 0).toString(16)}`;
}
/** The strings quoted in JSON `texts`, unescaped only as far as their
* delimiting quotes: the names a unit's encoded IR and entries refer to. */
export function quotedStrings(texts: readonly string[]): Set<string> {
const out = new Set<string>();
for (const text of texts) {
let start = -1;
for (let index = 0; index < text.length; index++) {
const char = text.charCodeAt(index);
if (char === 92 /* \\ */) index++;
else if (char === 34 /* " */) {
if (start < 0) start = index + 1;
else {
out.add(text.slice(start, index));
start = -1;
}
}
}
}
return out;
}
/** A table's entries at one point (JournalTable.hold). */
export interface HeldEntries {
/** Whether the table's first entries are still these keys and values. */
same(): boolean;
}
/** A field a unit's lowering must leave as it found it (fixedScalar), or a
* flag it may set, which applying its record sets too (flagScalar). */
export interface JournalScalar {
readonly name: string;
/** Takes the field's value; the result answers how the field changed. */
mark(): ScalarMark;
readonly set?: () => void;
}
/** A field's value when a unit started. */
export interface ScalarMark {
/** Not at all, set (a flag that went from false to true), or otherwise. */
change(): "same" | "set" | "changed";
}
/** A field a unit's lowering must leave as it found it. */
export function fixedScalar<T>(name: string, get: () => T): JournalScalar {
return {
name,
mark: () => {
const start = get();
return { change: () => (get() === start ? "same" : "changed") };
},
};
}
/** A flag a unit's lowering may set. */
export function flagScalar(name: string, get: () => boolean, set: () => void): JournalScalar {
return {
name,
mark: () => {
const start = get();
return {
change: () => {
const now = get();
return now === start ? "same" : now && !start ? "set" : "changed";
},
};
},
set,
};
}
/** The events of the unit being recorded (Lowerer.unitEvents). */
export class UnitEvents {
readonly events: JournalEvent[] = [];
/** Calls into reachability nest (an edge can fire virtual edges). Only
* the outermost call is recorded, because making it again makes the
* nested ones. */
depth = 0;
record(kind: JournalEvent["kind"], name: string, detail: string | null = null, index = 0): void {
if (this.depth === 0) this.events.push({ kind, name, detail, index });
}
}
/** The lowerer's side of a recorded wave (Lowerer.emitReachable). */
export interface WaveHost {
readonly tables: readonly JournalTable[];
readonly scalars: readonly JournalScalar[];
/** Lowers `name` as reachable emit does, with `events` recording: the
* function, or null, and whether the unit poisoned. Throws what lowering
* throws other than a poison. */
lowerRecorded(name: string, events: UnitEvents): { fn: IrFunction | null; poisoned: boolean };
/** Keeps a unit's function. */
store(name: string, fn: IrFunction | null): void;
/** Statement totals, and the count of diagnostics and skipped functions. */
statementCounts(): { total: number; failed: number; other: number };
/** Whether `event` names something this process holds. */
resolves(event: JournalEvent): boolean;
}
/** Whether reachable emit records its units (SCRIPTC_LOWER_JOURNAL=serial,
* or a SCRIPTC_LOWER_JOURNAL_DUMP directory). */
export function journalMode(): "serial" | "off" {
return process.env["SCRIPTC_LOWER_JOURNAL"] === "serial" || journalDump() !== null
? "serial"
: "off";
}
/** The directory recorded waves write their records to
* (SCRIPTC_LOWER_JOURNAL_DUMP), or null. */
export function journalDump(): string | null {
const dir = process.env["SCRIPTC_LOWER_JOURNAL_DUMP"];
return dir === undefined || dir === "" ? null : dir;
}
/** Appends `lines` to `dir`/records.jsonl. */
export function writeRecords(dir: string, lines: readonly string[]): void {
if (lines.length === 0) return;
mkdirSync(dir, { recursive: true });
appendFileSync(join(dir, "records.jsonl"), `${lines.join("\n")}\n`);
}
/** What a build's recorded waves did (SCRIPTC_TIMING). */
export interface WaveStats {
waves: number;
units: number;
/** Units whose record stands for them. */
recorded: number;
/** Units without a record. */
unrecorded: number;
/** Per table, field or check, the units it left without a record. */
reasons: Record<string, number>;
}
export function waveStats(): WaveStats {
return { waves: 0, units: 0, recorded: 0, unrecorded: 0, reasons: {} };
}
/** What lowering one unit with its events recording produced. */
export interface RecordedUnit {
/** The unit's record, or null when the unit has none. */
readonly record: UnitRecord | null;
/** Why the unit has no record. */
readonly reasons: readonly string[];
/** The function lowering made. */
readonly fn: IrFunction | null;
/** Each table's size before the unit. */
readonly before: readonly number[];
}
/** Lowers `name` with its events recording, and makes its record. The unit
* has a record when every table that grew is appendable and holds the
* entries it held before, every field is as it was or a flag that was set,
* statement totals grew by the recorded statements alone, no diagnostic was
* emitted, every event resolves and the function encodes. */
export function recordUnit(
host: WaveHost,
name: string,
contents: readonly (HeldContents | null)[] | null = null,
): RecordedUnit {
const tables = host.tables;
const before = tables.map((table) => table.size());
const held = tables.map((table) => (table.hold === undefined ? null : table.hold()));
const marks = host.scalars.map((scalar) => scalar.mark());
const counts = host.statementCounts();
const recorder = new UnitEvents();
const { fn, poisoned } = host.lowerRecorded(name, recorder);
const reasons: string[] = [];
const grown: number[] = [];
tables.forEach((table, index) => {
const check = held[index] ?? null;
if (check !== null && !check.same()) reasons.push(`${table.name}:replaced`);
if (table.size() === before[index]) return;
if (table.entries === undefined) reasons.push(table.name);
else grown.push(index);
});
const events = [...recorder.events];
host.scalars.forEach((scalar, index) => {
const change = marks[index]!.change();
if (change === "set") events.push({ kind: "flag", name: scalar.name, detail: null, index: 0 });
else if (change === "changed") reasons.push(scalar.name);
});
let statements = 0;
let failed = 0;
for (const event of events) {
if (event.kind === "statement") statements++;
else if (event.kind === "failedStatement") failed++;
if (!host.resolves(event)) reasons.push(`event:${event.kind}`);
}
const now = host.statementCounts();
if (now.total - counts.total !== statements || now.failed - counts.failed !== failed)
reasons.push("statements");
if (now.other !== counts.other) reasons.push("diagnostics");
const text = fn === null ? null : encodeUnitFunction(fn);
if (fn !== null && text === null) reasons.push("encoding");
const added: Record<string, readonly string[]> = {};
if (reasons.length === 0)
for (const index of grown) {
const table = tables[index]!;
const entries = table.entries!(before[index]!);
if (entries === null) reasons.push(`${table.name}:encoding`);
else added[table.name] = entries;
}
// A unit uses the entries its IR and new entries name: it reads them, and
// a unit that changes one in place changes one it names.
const texts: string[] = [];
if (text !== null) texts.push(text);
for (const index of grown) {
const table = tables[index]!;
const entries = added[table.name] ?? table.entries!(before[index]!);
if (entries !== null) for (const entry of entries) texts.push(entry);
}
const names = quotedStrings(texts);
if (contents !== null) {
// A unit with a record must leave the entries it names as they were; a
// unit without one keeps its changes.
if (reasons.length === 0)
tables.forEach((table, index) => {
if ((contents[index] ?? null)?.changedAmong(names) != null)
reasons.push(`${table.name}:changed`);
});
if (reasons.length > 0) for (const held of contents) held?.refresh(names);
for (const held of contents) held?.extend();
}
const reads: Record<string, (readonly [string, string])[]> = {};
if (reasons.length === 0)
tables.forEach((table, index) => {
if (table.digest === undefined) return;
const read: (readonly [string, string])[] = [];
for (const key of names) {
const digest = table.digest(key, before[index]!);
if (digest !== null) read.push([key, digest]);
}
if (read.length > 0) reads[table.name] = read;
});
if (reasons.length > 0) return { record: null, reasons, fn, before };
return {
record: { name, fn: text, poisoned, events, tables: added, reads },
reasons,
fn,
before,
};
}
/** Lowers `wave` with every unit recorded. A unit with a record keeps the
* results its record carries: the entries it added are taken out of their
* tables and added again from the record, and its function is the one
* decoded from the record. A unit without one keeps what lowering made.
* Throws an InternalCompilerError when a unit changed the content of an
* entry a table held when the unit started (JournalTable.contents), which
* no record carries.
* `lines`, when given, receives each unit's record as a JSON line. */
export function lowerWaveRecorded(
host: WaveHost,
wave: readonly string[],
stats: WaveStats,
lines: string[] | null = null,
): void {
stats.waves++;
const tables = host.tables;
// Units keep the content of the entries that tables held when they
// started: a record carries a unit's new entries, not changes to others.
const contents = tables.map((table) => (table.contents === undefined ? null : table.contents()));
for (const name of wave) {
const unit = recordUnit(host, name, contents);
stats.units++;
const record = unit.record;
if (record === null) {
lines?.push(JSON.stringify({ name, reasons: unit.reasons }));
stats.unrecorded++;
for (const reason of unit.reasons) stats.reasons[reason] = (stats.reasons[reason] ?? 0) + 1;
host.store(name, unit.fn);
continue;
}
lines?.push(JSON.stringify(record));
stats.recorded++;
tables.forEach((table, index) => {
const entries = record.tables[table.name];
if (entries === undefined) return;
table.truncate!(unit.before[index]!);
table.apply!(entries);
});
host.store(name, record.fn === null ? null : decodeUnitFunction(record.fn));
}
tables.forEach((table, index) => {
const key = contents[index]?.changed() ?? null;
if (key !== null)
throw new InternalCompilerError(
`a unit changed ${table.name} entry ${key} in place without naming it, which no unit record carries`,
);
});
}
/** An appendable table over a Map from strings to strings, such as helper
* names by key. */
export function mapTable(name: string, map: Map<string, string>): JournalTable {
return codecTable(
name,
map,
(value) => JSON.stringify(value),
(text) => JSON.parse(text) as string,
);
}
/** An appendable table over a Map from strings to numbers. */
export function countTable(name: string, map: Map<string, number>): JournalTable {
return codecTable(
name,
map,
(value) => String(value),
(text) => Number(text),
);
}
/** An appendable table over a Map from strings to IR data, encoded as
* JSON. JSON writes a number that is not finite as null, so a value whose
* encoding holds a null does not encode. */
export function irDataTable<V>(name: string, map: Map<string, V>): JournalTable {
const encode = (value: V): string | null => {
const text = JSON.stringify(value);
return /[:,[]null\b/.test(text) ? null : text;
};
return {
...codecTable(name, map, encode, (text) => JSON.parse(text) as V),
contents: () => {
const texts = new Map<string, string | null>();
const differs = (key: string): boolean => {
const value = map.get(key);
return value === undefined || encode(value) !== texts.get(key);
};
const held: HeldContents = {
extend: () => {
if (texts.size === map.size) return;
for (const [key, value] of map) if (!texts.has(key)) texts.set(key, encode(value));
},
changed: () => {
for (const key of texts.keys()) if (differs(key)) return key;
return null;
},
changedAmong: (keys) => {
for (const key of keys) if (texts.has(key) && differs(key)) return key;
return null;
},
refresh: (keys) => {
for (const key of keys) {
const value = map.get(key);
if (texts.has(key) && value !== undefined) texts.set(key, encode(value));
}
},
};
held.extend();
return held;
},
};
}
/** An appendable table over a Map from strings to values that `encode`
* writes, or answers null for, and `decode` reads back. An entry is its key
* as a JSON string, a line break and its value: a JSON string holds no line
* break. A value that does not encode leaves its unit without a record. */
export function codecTable<V>(
name: string,
map: Map<string, V>,
encode: (value: V) => string | null,
decode: (text: string) => V,
): JournalTable {
return {
name,
size: () => map.size,
entries: (from) => {
const out: string[] = [];
let index = 0;
for (const [key, value] of map) {
if (index++ < from) continue;
const text = encode(value);
if (text === null) return null;
out.push(`${JSON.stringify(key)}\n${text}`);
}
return out;
},
apply: (entries) => {
for (const entry of entries) {
const split = entry.indexOf("\n");
const key = JSON.parse(entry.slice(0, split)) as string;
if (!map.has(key)) map.set(key, decode(entry.slice(split + 1)));
}
},
truncate: (from) => {
const keys = [...map.keys()].slice(from);
for (const key of keys) map.delete(key);
},
hold: () => {
const keys = [...map.keys()];
const values = [...map.values()];
return {
same: () => {
let index = 0;
for (const [key, value] of map) {
if (index >= keys.length) break;
if (key !== keys[index] || value !== values[index]) return false;
index++;
}
return index === keys.length;
},
};
},
};
}
/** An appendable table over a Set of strings. */
export function setTable(name: string, set: Set<string>): JournalTable {
return {
name,
size: () => set.size,
entries: (from) => [...set].slice(from),
apply: (entries) => {
for (const entry of entries) set.add(entry);
},
truncate: (from) => {
const entries = [...set].slice(from);
for (const entry of entries) set.delete(entry);
},
hold: () => {
const entries = [...set];
return {
same: () => {
let index = 0;
for (const entry of set) {
if (index >= entries.length) break;
if (entry !== entries[index]) return false;
index++;
}
return index === entries.length;
},
};
},
};
}
/** An appendable list of functions this process holds one of per name. A
* function whose name the list already holds is the same function, made by
* an earlier unit. `changed` runs after the list changes, for indexes over
* it. */
export function functionTable(
name: string,
functions: IrFunction[],
has: (name: string) => boolean,
changed: () => void = () => {},
): JournalTable {
// The position of each function by name, for the functions indexed so far.
let indexes = new Map<string, number>();
let indexed = 0;
const lookup = (key: string): number | undefined => {
for (; indexed < functions.length; indexed++) indexes.set(functions[indexed]!.name, indexed);
const index = indexes.get(key);
return index !== undefined && functions[index]?.name === key ? index : undefined;
};
const indexOf = (key: string): number | undefined => {
if (indexed > functions.length) {
indexes = new Map();
indexed = 0;
}
const index = lookup(key);
if (index !== undefined || !has(key)) return index;
// A list that was cut and grown again holds other functions at the
// positions indexed before.
indexes = new Map();
indexed = 0;
return lookup(key);
};
return {
name,
size: () => functions.length,
digest: (key, before) => {
if (!has(key)) return null;
const index = indexOf(key);
if (index === undefined || index >= before) return null;
const text = encodeUnitFunction(functions[index]!);
return text === null ? null : contentDigest(text);
},
entries: (from) => {
const out: string[] = [];
for (const fn of functions.slice(from)) {
const text = encodeUnitFunction(fn);
if (text === null) return null;
out.push(text);
}
return out;
},
apply: (entries) => {
for (const entry of entries) {
const fn = decodeUnitFunction(entry);
if (!has(fn.name)) functions.push(fn);
}
changed();
},
truncate: (from) => {
while (functions.length > from) functions.pop();
changed();
},
hold: () => {
const held = [...functions];
return { same: () => held.every((fn, index) => functions[index] === fn) };
},
contents: () => {
const texts: (string | null)[] = [];
const indexes = new Map<string, number>();
const differs = (index: number): boolean =>
index < functions.length && encodeUnitFunction(functions[index]!) !== texts[index];
const held: HeldContents = {
extend: () => {
for (let index = texts.length; index < functions.length; index++) {
const fn = functions[index]!;
texts.push(encodeUnitFunction(fn));
indexes.set(fn.name, index);
}
},
changed: () => {
for (let index = 0; index < texts.length; index++)
if (differs(index)) return functions[index]!.name;
return null;
},
changedAmong: (keys) => {
for (const key of keys) {
const index = indexes.get(key);
if (index !== undefined && differs(index)) return key;
}
return null;
},
refresh: (keys) => {
for (const key of keys) {
const index = indexes.get(key);
if (index !== undefined && index < functions.length)
texts[index] = encodeUnitFunction(functions[index]!);
}
},
};
held.extend();
return held;
},
};
}
/** A table whose growth leaves the unit without a record. */
export function checkedTable(name: string, size: () => number): JournalTable {
return { name, size };
}
packages/compiler/src/frontend/ts7/checker-server.test.ts
+8 -3
View File
@@ -12,13 +12,18 @@ const posix = process.platform === "linux" || process.platform === "darwin";
afterAll(() => rmSync(dir, { recursive: true, force: true }));
test("builds use the shared checker server only when they opt in, on Linux and macOS, with tsc-ts", () => {
expect(checkerServerName(executable, dir, false, {})).toBeNull();
test("builds use the shared checker server by default outside CI, on Linux and macOS, with tsc-ts", () => {
expect(checkerServerName(executable, dir, false, { SCRIPTC_CHECKER_SERVER: "0" })).toBeNull();
expect(checkerServerName(executable, dir, false, { SCRIPTC_CHECKER: "tsgo" })).toBeNull();
expect(
checkerServerName(executable, dir, false, { ...enabled, SCRIPTC_CHECKER: "tsgo" }),
).toBeNull();
const name = checkerServerName(executable, dir, false, enabled);
// CI turns the default off; SCRIPTC_CHECKER_SERVER=1 turns it back on.
expect(checkerServerName(executable, dir, false, { CI: "true" })).toBeNull();
expect(checkerServerName(executable, dir, false, { CI: "1" })).toBeNull();
const name = checkerServerName(executable, dir, false, {});
for (const env of [enabled, { CI: "false" }, { CI: "0" }, { CI: "" }, { ...enabled, CI: "true" }])
expect(checkerServerName(executable, dir, false, env)).toBe(name);
if (!posix) expect(name).toBeNull();
else expect(name).toMatch(/^[0-9a-f]{32}$/);
});
packages/compiler/src/frontend/ts7/checker-server.ts
+27 -11
View File
@@ -5,23 +5,37 @@ import { TS7_FILE_SYSTEM_CALLBACKS } from "./rpc-filesystem.js";
/** How long a shared checker server waits for the next build before it
* exits, in seconds. */
export const CHECKER_SERVER_IDLE_EXIT = 300;
export const CHECKER_SERVER_IDLE_EXIT = 600;
/** The resident memory, in megabytes, above which a shared checker server
* exits after a build. tsc-ts never frees a program's syntax trees, types and
* symbols, so its memory grows with each build it serves. */
export const CHECKER_SERVER_MEMORY = 2048;
/** Whether builds use a shared checker server: by default, except in CI
* (`CI` set to anything but `0` or `false`), where a job usually builds once
* per directory and would only leave the server behind.
* `SCRIPTC_CHECKER_SERVER=0` turns it off and `SCRIPTC_CHECKER_SERVER=1` turns
* it on in CI. */
function checkerServerEnabled(env: NodeJS.ProcessEnv): boolean {
const setting = env["SCRIPTC_CHECKER_SERVER"];
if (setting === "1") return true;
if (setting === "0") return false;
const ci = env["CI"];
return ci === undefined || ci === "" || ci === "0" || ci === "false";
}
/**
* With `SCRIPTC_CHECKER_SERVER=1` on Linux and macOS, the native compiler
* checks with a tsc-ts server shared by the builds that follow
* (`tsc-ts --api --listen`): a build connects to it, starting it when none
* answers, and the server keeps the files it parsed for the next build.
* Returns the server's name, a digest of everything a server's answers
* depend on besides the files it reads through the build: the tsc-ts
* executable, the working directory, the file system callbacks, timing,
* whether it reads its library files from the disk, and the `TSC_TS_*`
* environment. Returns null when the shared server is not used.
* On Linux and macOS, the native compiler checks with a tsc-ts server shared
* by the builds that follow (`tsc-ts --api --listen`): a build connects to
* it, starting it when none answers, and the server keeps the files it
* parsed for the next build (see checkerServerEnabled for when). Returns the
* server's name, a digest of everything a server's answers depend on
* besides the files it reads through the build: the tsc-ts executable, the
* working directory, the file system callbacks, timing, whether it reads its
* library files from the disk, and the `TSC_TS_*` environment; and of the
* compiler executable, so that each scriptc build has servers of its own.
* Returns null when the shared server is not used.
*/
export function checkerServerName(
executable: string,
@@ -29,8 +43,9 @@ export function checkerServerName(
timing: boolean,
env: NodeJS.ProcessEnv = process.env,
librariesFromDisk = false,
compiler: string = process.execPath,
): string | null {
if (env["SCRIPTC_CHECKER_SERVER"] !== "1") return null;
if (!checkerServerEnabled(env)) return null;
if (process.platform !== "linux" && process.platform !== "darwin") return null;
if (checkerBackend(env) !== "tsc-ts") return null;
const tscTsEnv = Object.keys(env)
@@ -41,6 +56,7 @@ export function checkerServerName(
.update(
JSON.stringify([
checkerExecutableIdentity(executable),
checkerExecutableIdentity(compiler),
cwd,
TS7_FILE_SYSTEM_CALLBACKS,
timing,
packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts
+175 -2
View File
@@ -1,6 +1,6 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { dirname, join } from "node:path";
import { afterAll, afterEach, expect, test } from "vitest";
import type { SourceFile } from "./ast-types.js";
import {
@@ -12,7 +12,9 @@ import {
setLibDiagnosticsStore,
type LibDiagnosticsStore,
type PrebuiltLibDiagnostics,
type SuppliedKind,
} from "./lib-diagnostics.js";
import { hostTscTsExecutable } from "./checker-backend.js";
import type { Ts7DiagnosticData } from "./session-schema.generated.js";
import type { Ts7CheckedFileDiagnosticData, Ts7SessionProgram } from "./session.js";
import { closeSourceParser, parseSourceFile } from "./source-parser-node.js";
@@ -382,3 +384,174 @@ test("prebuilt answer files round-trip, and a malformed one is no answer", () =>
expect(readPrebuiltLibDiagnostics(path)).toBeNull();
expect(readPrebuiltLibDiagnostics(join(dir, "missing.json"))).toBeNull();
});
// Node module fallback files: ambient modules only, or one that also declares a global.
const moduleA = join(dir, "node-fallback-a.d.ts");
const moduleB = join(dir, "node-fallback-b.d.ts");
const moduleGlobal = join(dir, "node-fallback-global.d.ts");
const moduleTexts: Record<string, string> = {
[moduleA]: 'declare module "node:a" {\n export const a: number;\n}\n',
[moduleB]: 'declare module "node:b" {\n export const b: string;\n}\n',
[moduleGlobal]:
'declare module "node:g" {\n global {\n var g: number;\n }\n export {};\n}\n',
};
for (const [name, text] of Object.entries(moduleTexts)) writeFileSync(name, text);
const kind = (name: string): SuppliedKind =>
name === lib ? "library" : name.includes("node-fallback-") ? "module" : "declarations";
const withModules = (...modules: string[]): string[] => [lib, fallback, ...modules, main];
const suppliedWithModules = (name: string): boolean => name !== main;
const moduleSources = files({ ...moduleTexts, [main]: "export const n: number = 'x';\n" });
test("a set with module files reuses the libs' answers of the set without them", () => {
const store = memoryStore();
setLibDiagnosticsStore(store, "checker");
const semantic = new Map([
[lib, [diagnostic(lib, 1)]],
[main, [diagnostic(main, 5)]],
]);
// Without an answer for the set without module files, a set with them checks in full.
const first = sessionProgram({ semantic });
reusedLibDiagnostics(first, withModules(moduleA), suppliedWithModules, moduleSources, kind);
expect(first.asked[0]).toBe("getGlobalDiagnostics");
const base = sessionProgram({ semantic });
reusedLibDiagnostics(base, withModules(), suppliedWithModules, moduleSources, kind);
expect(base.asked[0]).toBe("getGlobalDiagnostics");
// That answer reports a diagnostic in a lib, so a set with another module file reuses the
// libs' answers and checks the program, scriptc's declarations and that file.
const second = sessionProgram({ semantic });
const composed = reusedLibDiagnostics(
second,
withModules(moduleB),
suppliedWithModules,
moduleSources,
kind,
);
expect(second.asked).toEqual([`checked ${main},${fallback},${moduleB}`]);
expect(composed).toEqual({ semantic: [diagnostic(lib, 1), diagnostic(main, 5)], global: [] });
// Its own answer is stored, so the same set then checks only the program.
const third = sessionProgram({ semantic });
reusedLibDiagnostics(third, withModules(moduleB), suppliedWithModules, moduleSources, kind);
expect(third.asked).toEqual([`checked ${main}`]);
});
test("a clean answer of the set without module files stands in for all of its files", () => {
const semantic = new Map([[main, []]]);
const store = memoryStore();
setLibDiagnosticsStore(store, "checker");
reusedLibDiagnostics(
sessionProgram({ semantic }),
withModules(),
suppliedWithModules,
moduleSources,
kind,
);
const fromBase = sessionProgram({ semantic });
reusedLibDiagnostics(fromBase, withModules(moduleA), suppliedWithModules, moduleSources, kind);
expect(fromBase.asked).toEqual([`checked ${main},${moduleA}`]);
// So does a prebuilt answer for it.
const keys = new Set<string>();
recordPrebuiltLibDiagnostics({ checker: "1.0.0", record: (key) => keys.add(key) });
setLibDiagnosticsStore(memoryStore(), "build-time checker");
reusedLibDiagnostics(
sessionProgram({ semantic }),
withModules(),
suppliedWithModules,
moduleSources,
kind,
);
recordPrebuiltLibDiagnostics(null);
expect(keys.size).toBe(1);
setLibDiagnosticsStore(memoryStore(), "installed checker", () => ({ checker: "1.0.0", keys }));
const prebuilt = sessionProgram({ semantic });
reusedLibDiagnostics(
prebuilt,
withModules(moduleA, moduleB),
suppliedWithModules,
moduleSources,
kind,
);
expect(prebuilt.asked).toEqual([`checked ${main},${moduleA},${moduleB}`]);
});
test("scriptc's declarations are checked again when they report something or declare a module file's module", () => {
const declared = join(dir, "scriptc-declared.d.ts");
const moduleX = join(dir, "node-fallback-x.d.ts");
const texts = {
[declared]: 'declare module "node:x" {\n export const x: number;\n}\n',
[moduleX]: 'declare module "node:x" {\n export const y: number;\n}\n',
};
for (const [name, text] of Object.entries(texts)) writeFileSync(name, text);
const sources = files({ ...moduleTexts, ...texts, [main]: "export const n: number = 'x';\n" });
const semantic = new Map([[main, []]]);
setLibDiagnosticsStore(memoryStore(), "checker");
const base = [lib, fallback, declared, main];
const set = [lib, fallback, declared, moduleX, main];
reusedLibDiagnostics(sessionProgram({ semantic }), base, suppliedWithModules, sources, kind);
const merged = sessionProgram({ semantic });
reusedLibDiagnostics(merged, set, suppliedWithModules, sources, kind);
expect(merged.asked).toEqual([`checked ${main},${fallback},${declared},${moduleX}`]);
// With a diagnostic in scriptc's declarations, only the libs' answers are reused.
const reporting = new Map([
[main, []],
[fallback, [diagnostic(fallback, 3)]],
]);
setLibDiagnosticsStore(memoryStore(), "checker");
reusedLibDiagnostics(
sessionProgram({ semantic: reporting }),
withModules(),
suppliedWithModules,
moduleSources,
kind,
);
const again = sessionProgram({ semantic: reporting });
const answer = reusedLibDiagnostics(
again,
withModules(moduleA),
suppliedWithModules,
moduleSources,
kind,
);
expect(again.asked).toEqual([`checked ${main},${fallback},${moduleA}`]);
expect(answer?.semantic).toEqual([diagnostic(fallback, 3)]);
});
test("module files that declare something global, and programs with scripts, check in full", () => {
const semantic = new Map([[main, []]]);
const store = memoryStore();
setLibDiagnosticsStore(store, "checker");
reusedLibDiagnostics(
sessionProgram({ semantic }),
withModules(),
suppliedWithModules,
moduleSources,
kind,
);
const global = sessionProgram({ semantic });
reusedLibDiagnostics(global, withModules(moduleGlobal), suppliedWithModules, moduleSources, kind);
expect(global.asked[0]).toBe("getGlobalDiagnostics");
const other = sessionProgram({ semantic });
reusedLibDiagnostics(
other,
withModules(moduleGlobal, moduleA),
suppliedWithModules,
moduleSources,
kind,
);
expect(other.asked[0]).toBe("getGlobalDiagnostics");
const script = sessionProgram({ semantic });
const scriptSources = files({ ...moduleTexts, [main]: "const total = 1;\n" });
reusedLibDiagnostics(script, withModules(moduleA), suppliedWithModules, scriptSources, kind);
expect(script.asked[0]).toBe("getGlobalDiagnostics");
});
test("the TypeScript libs declare no ambient module and import none, so module files cannot reach them", () => {
const libDir = dirname(hostTscTsExecutable());
const libFiles = readdirSync(libDir).filter((name) => /^lib\..*\.d\.ts$/.test(name));
expect(libFiles.length).toBeGreaterThan(50);
for (const name of libFiles) {
const text = readFileSync(join(libDir, name), "utf8");
expect(text, name).not.toMatch(
/\bmodule\s+["']|\bimport\s*\(\s*["']|\bfrom\s+["']|\brequire\s*\(/,
);
}
});
packages/compiler/src/frontend/ts7/lib-diagnostics.ts
+191 -7
View File
@@ -24,7 +24,20 @@
* the distribution was built. A build whose store has no answer uses a
* prebuilt one when the checker is the distribution's own and the files'
* content key is among them, and stores it, so builds with an empty cache
* skip the check of the supplied files too. */
* skip the check of the supplied files too.
*
* The supplied set depends on the Node modules a program imports: each one
* adds its fallback declarations. Those files declare ambient modules and
* nothing global, and the TypeScript libs declare only globals and name no
* module, so adding module files cannot change what checking the libs
* reports. Nor can it change what checking scriptc's declaration files
* reports when the set without module files reports nothing in them (every
* name they use resolves among them and the libs) and no module file
* declares a module they declare. A program whose set has no stored answer
* therefore reuses the answers of the set without module files, stored or
* prebuilt: for every file of that set when both hold, for the libs alone
* otherwise. The server then checks the program's other files and the
* supplied files not reused, and the answer for the whole set is stored. */
import { createHash } from "node:crypto";
import { readFileSync, statSync } from "node:fs";
import { basename, dirname, join } from "node:path";
@@ -48,6 +61,11 @@ export interface PrebuiltLibDiagnostics {
keys: ReadonlySet<string>;
}
/** A supplied file's part in reuse across sets: "library" for the
* TypeScript libs, "module" for a Node module's fallback declarations, and
* "declarations" for scriptc's other declaration files. */
export type SuppliedKind = "library" | "module" | "declarations";
const SCHEMA = "scriptc.lib-diagnostics.v2";
const PREBUILT_SCHEMA = "scriptc.lib-diagnostics-prebuilt.v1";
@@ -159,14 +177,16 @@ export function checkerExecutableIdentity(path: string): string {
* them, using stored diagnostics for the supplied declaration files; null
* when they cannot be reused for this program (then the caller asks for
* them as before). `supplied` tells whether a program file is one of the
* supplied declaration files. */
* supplied declaration files, and `kind` which kind of supplied file it is
* (by default scriptc's declarations, which reuse nothing across sets). */
export function reusedLibDiagnostics(
program: Ts7SessionProgram,
fileNames: readonly string[],
supplied: (fileName: string) => boolean,
sourceFile: (fileName: string) => SourceFile | undefined,
kind: (fileName: string) => SuppliedKind = () => "declarations",
): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null {
return reuse(program, fileNames, supplied, sourceFile, () => true, false);
return reuse(program, fileNames, supplied, sourceFile, () => true, false, kind);
}
/** reusedLibDiagnostics for a caller that keeps only the diagnostics of the
@@ -180,7 +200,15 @@ export function reusedKeptLibDiagnostics(
sourceFile: (fileName: string) => SourceFile | undefined,
kept: ReadonlySet<string>,
): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null {
return reuse(program, fileNames, supplied, sourceFile, (name) => kept.has(name), true);
return reuse(
program,
fileNames,
supplied,
sourceFile,
(name) => kept.has(name),
true,
() => "declarations",
);
}
function reuse(
@@ -190,9 +218,11 @@ function reuse(
sourceFile: (fileName: string) => SourceFile | undefined,
keep: (fileName: string) => boolean,
storedOnly: boolean,
kind: (fileName: string) => SuppliedKind,
): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null {
if (configured === null || process.env["SCRIPTC_LIB_DIAGNOSTICS"] === "0") return null;
const { store, checker } = configured;
identities.clear();
const libs: string[] = [];
const own: string[] = [];
for (const name of fileNames) (supplied(name) ? libs : own).push(name);
@@ -240,6 +270,31 @@ function reuse(
return { semantic: inFileOrder(byFile), global: checked.global };
}
if (storedOnly) return null;
// Another set's answers for the libs (see the header), for a program of
// modules: its scripts' globals would need every supplied file's names.
const kinds = new Map<string, SuppliedKind>();
if (stored === null && scripts.length === 0) for (const name of libs) kinds.set(name, kind(name));
const kindOf = (name: string): SuppliedKind => kinds.get(name) ?? "declarations";
const libraries = libs.filter((name) => kindOf(name) === "library");
const modules = libs.filter((name) => kindOf(name) === "module");
const shared =
modules.length > 0
? sharedAnswers(store, checker, options, libs, libraries, modules, sourceFile, kindOf)
: null;
if (shared !== null) {
const checkedNames = [...own, ...libs.filter((name) => !shared.files.has(name))];
const checked = program.checkedFileDiagnosticData(checkedNames);
if (checked !== null) {
const byFile = new Map(shared.files);
checkedNames.forEach((name, index) => byFile.set(name, checked.files[index] ?? []));
if (checked.global.length === 0) {
const files: Record<string, Ts7DiagnosticData[]> = {};
for (const name of libs) files[name] = byFile.get(name) ?? [];
writeEntry(store, key, null, files);
}
return { semantic: inFileOrder(byFile), global: checked.global };
}
}
// The full check, as before. Its answers for the supplied files are stored
// when it reports no global diagnostics and the program's scripts declare
// none of the supplied files' globals (so nothing of the program reached
@@ -424,8 +479,7 @@ function entryKey(checker: string, options: unknown, libs: readonly string[]): s
const files: unknown[] = [];
for (const name of libs) {
try {
const info = statSync(name);
files.push([name, info.size, info.mtimeMs, info.ino]);
files.push(fileIdentity(name));
} catch {
return null;
}
@@ -435,6 +489,121 @@ function entryKey(checker: string, options: unknown, libs: readonly string[]): s
.digest("hex");
}
/** The files' identities (path, size, modification time, inode), kept for
* one reuse: the keys of a set and of the set without its module files
* share most of their files. */
const identities = new Map<string, [string, number, number, number]>();
function fileIdentity(name: string): [string, number, number, number] {
let identity = identities.get(name);
if (identity === undefined) {
const info = statSync(name);
identity = [name, info.size, info.mtimeMs, info.ino];
identities.set(name, identity);
}
return identity;
}
/** Answers of the set without module files (see the header), stored or
* prebuilt, when the program's module files declare nothing global: for
* every file of that set when its answer reports no diagnostics and the
* module files declare no module its declaration files declare, otherwise
* for the libs alone; null when there is none. */
function sharedAnswers(
store: LibDiagnosticsStore,
checker: string,
options: unknown,
libs: readonly string[],
libraries: readonly string[],
modules: readonly string[],
sourceFile: (fileName: string) => SourceFile | undefined,
kind: (fileName: string) => SuppliedKind,
): { files: Map<string, Ts7DiagnosticData[]> } | null {
if (libraries.length === 0) return null;
const base = libs.filter((name) => kind(name) !== "module");
const key = entryKey(checker, options, base);
const entry = key === null ? null : readEntry(store, key, base);
const prebuilt = configured?.prebuilt?.() ?? null;
const prebuiltClean = (): boolean => {
if (prebuilt === null || prebuilt.keys.size === 0) return false;
const content = contentKey(prebuilt.checker, options, base);
return content !== null && prebuilt.keys.has(content);
};
const clean =
entry !== null ? [...entry.files.values()].every((list) => list.length === 0) : prebuiltClean();
if (clean) {
const declared = moduleNames(modules, sourceFile);
if (declared === null) return null;
const own = declaredModuleNames(base.filter((name) => kind(name) === "declarations"));
const reused = own !== null && ![...declared].some((name) => own.has(name)) ? base : libraries;
const files = new Map<string, Ts7DiagnosticData[]>();
for (const name of reused) files.set(name, []);
return { files };
}
if (entry === null || moduleNames(modules, sourceFile) === null) return null;
return {
files: new Map(
libraries.map((name): [string, Ts7DiagnosticData[]] => [name, entry.files.get(name)!]),
),
};
}
/** The ambient modules the module files declare, when each one declares
* only ambient modules (`declare module "x"`) with no `global` augmentation,
* or is a module that declares nothing global: then it adds nothing the libs
* can see. Null otherwise. */
function moduleNames(
modules: readonly string[],
sourceFile: (fileName: string) => SourceFile | undefined,
): Set<string> | null {
const names = new Set<string>();
for (const name of modules) {
const file = sourceFile(name);
if (file === undefined) return null;
if (file.externalModuleIndicator !== undefined) {
if (!declaresNothingGlobal(file)) return null;
continue;
}
for (const statement of file.statements.nodes) {
if (
statement.kind !== SyntaxKind.ModuleDeclaration ||
statement.name?.kind !== SyntaxKind.StringLiteral
)
return null;
names.add(statement.name.text ?? "");
for (const inner of statement.body?.statements?.nodes ?? [])
if (
inner.kind === SyntaxKind.ModuleDeclaration &&
inner.name?.kind === SyntaxKind.Identifier &&
inner.name.text === "global"
)
return null;
}
}
return names;
}
/** The module names scriptc's declaration files name in a module
* declaration (`module "x"` or `module 'x'`, in their text, so also in
* comments); null when one of them cannot be read or names a pattern
* (`"*"`), which a module file's module could also match. */
function declaredModuleNames(files: readonly string[]): Set<string> | null {
const names = new Set<string>();
for (const name of files) {
let text: string;
try {
text = readFileSync(name, "utf8");
} catch {
return null;
}
for (const match of text.matchAll(/\bmodule\s+(["'])(.*?)\1/g)) {
if (match[2]!.includes("*")) return null;
names.add(match[2]!);
}
}
return names;
}
/** The supplied files' answer from the distribution's prebuilt answers,
* stored under `key` for later builds: every file without diagnostics, when
* they hold the files' content key; null otherwise. */
@@ -461,7 +630,7 @@ function contentKey(checker: string, options: unknown, libs: readonly string[]):
const files: [string, string][] = [];
for (const name of libs) {
try {
files.push([basename(name), createHash("sha256").update(readFileSync(name)).digest("hex")]);
files.push([basename(name), contentDigest(name)]);
} catch {
return null;
}
@@ -471,6 +640,21 @@ function contentKey(checker: string, options: unknown, libs: readonly string[]):
.digest("hex");
}
/** The digests of files' contents by path and file identity: a build that
* misses its answer computes the content keys of its set and of the set
* without module files, which share most of their files. */
const contentDigests = new Map<string, string>();
function contentDigest(name: string): string {
const identity = JSON.stringify(fileIdentity(name));
let digest = contentDigests.get(identity);
if (digest === undefined) {
digest = createHash("sha256").update(readFileSync(name)).digest("hex");
contentDigests.set(identity, digest);
}
return digest;
}
function readEntry(
store: LibDiagnosticsStore,
key: string,
packages/compiler/src/frontend/ts7/native-api.ts
+41 -2
View File
@@ -4,7 +4,41 @@ import { checkerServerName } from "./checker-server.js";
import { attachNativeTs7Process, spawnNativeTs7Wire } from "./native-process.js";
import { Ts7RpcClient } from "./rpc-client.js";
import type { Ts7FileSystem } from "./rpc-filesystem.js";
import { Ts7Wire } from "./rpc-wire.js";
import { Ts7Wire, type Ts7WireIo } from "./rpc-wire.js";
let sharedServerLost = false;
/** Whether a connection to the shared checker server failed in this process
* after the server answered: a transfer failed or the server ended the
* connection. The build then runs again with its own checker (cli.ts). */
export function checkerServerLost(): boolean {
return sharedServerLost;
}
/** `io`, recording a failed transfer or an ended stream in sharedServerLost. */
function watched(io: Ts7WireIo): Ts7WireIo {
return {
read: (buffer, offset, length) => {
try {
const count = io.read(buffer, offset, length);
if (count === 0) sharedServerLost = true;
return count;
} catch (error) {
sharedServerLost = true;
throw error;
}
},
write: (buffer, offset, length) => {
try {
return io.write(buffer, offset, length);
} catch (error) {
sharedServerLost = true;
throw error;
}
},
close: () => io.close(),
};
}
/** The native driver supplies its installed TypeScript executable path.
* Package discovery is separate from starting and owning a server. */
@@ -32,10 +66,15 @@ export function createNativeTs7Api(options: {
try {
// A server that ended the connection leaves this build to start its own.
rpc.requestText("ping", "null");
return new Ts7Api(rpc, options.fs, timing);
} catch {
rpc.close();
return new Ts7Api(
new Ts7RpcClient(spawnNativeTs7Wire(options.executable, options.cwd, timing, libraries)),
options.fs,
timing,
);
}
return new Ts7Api(new Ts7RpcClient(new Ts7Wire(watched(shared))), options.fs, timing);
}
return new Ts7Api(
new Ts7RpcClient(spawnNativeTs7Wire(options.executable, options.cwd, timing, libraries)),
packages/compiler/src/frontend/ts7/program-host.ts
+20 -3
View File
@@ -47,8 +47,18 @@ import {
} from "../input-tracker.js";
import { packageShimDeclarationText } from "../package-shims.js";
import { typePatchedText, typePatchFileExists } from "../type-patches/layer.js";
import { ambientDtsPath, isFallbackDtsPath, overridesDtsPath } from "../dts-paths.js";
import { inFileOrder, reusedKeptLibDiagnostics, reusedLibDiagnostics } from "./lib-diagnostics.js";
import {
ambientDtsPath,
isFallbackDtsPath,
isFallbackModuleDtsPath,
overridesDtsPath,
} from "../dts-paths.js";
import {
inFileOrder,
reusedKeptLibDiagnostics,
reusedLibDiagnostics,
type SuppliedKind,
} from "./lib-diagnostics.js";
import { ts7Diagnostic } from "./diagnostic.js";
import type { Ts7DiagnosticData } from "./session-schema.generated.js";
@@ -446,8 +456,15 @@ export class Ts7Program {
isFallbackDtsPath(name) ||
program.getSourceFileMetadataByPath(name)?.isDefaultLibrary === true;
const sourceFile = (name: string): SourceFile | undefined => this.getSourceFile(name);
// By path: a supplied file that is not one of scriptc's is a default library.
const kind = (name: string): SuppliedKind =>
isFallbackModuleDtsPath(name)
? "module"
: declarations.has(tsgoPath(name)) || isFallbackDtsPath(name)
? "declarations"
: "library";
return kept === null
? reusedLibDiagnostics(program, names, supplied, sourceFile)
? reusedLibDiagnostics(program, names, supplied, sourceFile, kind)
: reusedKeptLibDiagnostics(program, names, supplied, sourceFile, kept);
}
packages/compiler/src/frontend/type-mapper.ts
+130 -1
View File
@@ -1,5 +1,6 @@
import { InternalCompilerError } from "../errors.js";
import { keyedHelperName, type KeyedNameOwner } from "./lowering/stable-names.js";
import type { HeldContents, JournalTable } from "./lowering/unit-journal.js";
import { mockCallRecordType, mockContextType } from "./mock-shapes.js";
import * as ts from "./ts7/adapter.js";
import { bodyReadsArguments } from "./arguments-usage.js";
@@ -324,6 +325,30 @@ export class ShapeRegistry {
private currentDeclaredOrderPriority: DeclaredOrderPriority = [0, 0];
/** All interned shapes in first-seen order. */
readonly shapes: IrRecordShape[] = [];
/** The sizes of the registry's tables, which only grow while shapes are
* interned (unit-journal.ts). */
journalSizes(): number[] {
let candidates = 0;
for (const list of this.declaredOrderCandidates.values()) candidates += list.length;
let extended = 0;
for (const names of this.extendedNames.values()) extended += names.length;
return [
this.revision,
this.shapes.length,
this.byKey.size,
this.byId.size,
this.keyedNameKeys.size,
this.declaredOrderPriority.size,
this.declaredOrderCandidates.size,
candidates,
this.derivedDeclaredOrderFinalizers.length,
this.recIds.size,
this.pendingRec.size,
this.extendedNames.size,
extended,
];
}
/** ts.Types currently being mapped — a BACK-REFERENCE to one of these is
* the recursive knot (`interface TreeNode { children: TreeNode[] }`):
* mapType answers a NAMED RECURSIVE SHAPE (recursiveRef) whose fields
@@ -675,6 +700,105 @@ export class UnionRegistry {
private readonly armTags = new Map<string, Map<string, number[]>>();
/** All interned unions in first-seen order. */
readonly unions: IrUnionDef[] = [];
/** The sizes of the tables recursive unions change, which no unit
* record carries (unit-journal.ts). */
journalSizes(): number[] {
return [this.revision, this.recIds.size, this.pendingRec.size];
}
/** The interned unions as a table unit records carry (unit-journal.ts).
* An entry is a union's definition; applying it interns the union under
* the key its arms give. A unit that adds a recursive union changes
* journalSizes, so the unions past a unit's start are interned ones. */
journalTable(): JournalTable {
return {
name: "unions",
size: () => this.unions.length,
entries: (from) => {
const out: string[] = [];
for (const def of this.unions.slice(from)) {
// JSON writes a number that is not finite as null.
const text = JSON.stringify(def);
if (/[:,[]null\b/.test(text)) return null;
out.push(text);
}
return out;
},
apply: (entries) => {
for (const entry of entries) {
const def = JSON.parse(entry) as IrUnionDef;
const key = unionKey(def.arms, def.discriminant);
if (this.byKey.has(key)) continue;
if (keyedHelperName(this, "u", key, "") !== def.id)
throw new InternalCompilerError(`union record ${def.id} does not match its arms`);
this.byKey.set(key, def.id);
this.byId.set(def.id, def);
this.unions.push(def);
}
},
truncate: (from) => {
while (this.unions.length > from) {
const def = this.unions.pop()!;
this.byKey.delete(unionKey(def.arms, def.discriminant));
this.byId.delete(def.id);
this.keyedNameKeys.delete(def.id);
}
},
hold: () => {
const held = [...this.unions];
return { same: () => held.every((def, index) => this.unions[index] === def) };
},
contents: () => {
// A recursive placeholder still pending fills in place when its
// outer frame completes, which changes journalSizes.
const texts: (string | null)[] = [];
const indexes = new Map<string, number>();
const differs = (index: number): boolean => {
const text = texts[index] ?? null;
return (
text !== null &&
index < this.unions.length &&
JSON.stringify(this.unions[index]!) !== text
);
};
const held: HeldContents = {
extend: () => {
for (let index = texts.length; index < this.unions.length; index++) {
const def = this.unions[index]!;
texts.push(this.pendingRec.has(def.id) ? null : JSON.stringify(def));
indexes.set(def.id, index);
}
},
changed: () => {
for (let index = 0; index < texts.length; index++)
if (differs(index)) return this.unions[index]!.id;
return null;
},
changedAmong: (keys) => {
for (const key of keys) {
const index = indexes.get(key);
if (index !== undefined && differs(index)) return key;
}
return null;
},
refresh: (keys) => {
for (const key of keys) {
const index = indexes.get(key);
if (
index !== undefined &&
index < this.unions.length &&
(texts[index] ?? null) !== null
)
texts[index] = JSON.stringify(this.unions[index]!);
}
},
};
held.extend();
return held;
},
};
}
/** ts.Types currently being mapped — a back-reference to one is the
* recursive knot passing through a union (`type Tree = Leaf | Branch`
* whose Branch arm carries `Tree[]`, the optional field `a?: A` of a
@@ -747,7 +871,7 @@ export class UnionRegistry {
/** Interns canonical arms and discriminator cases, returning their unionId. */
intern(arms: IrType[], discriminant?: IrUnionDiscriminant): string {
const key = JSON.stringify([arms.map(typeKey), discriminant]);
const key = unionKey(arms, discriminant);
let id = this.byKey.get(key);
if (id === undefined) {
id = keyedHelperName(this, "u", key, "");
@@ -789,6 +913,11 @@ export class UnionRegistry {
}
}
/** The key UnionRegistry interns a union's arms and discriminant under. */
function unionKey(arms: readonly IrType[], discriminant: IrUnionDiscriminant | undefined): string {
return JSON.stringify([arms.map(typeKey), discriminant]);
}
/** Human-readable rendering of an IrType for diagnostics (records expand to
* their canonical field list, unions to their arms; `checker.typeToString`
* can't — it never sees IR types). Bound the traversal as well as its output:

Some files were not shown because too many files have changed in this diff Show More