Merge the loader branch, key the build by the libnode pack, and pick musl packs

This commit is contained in:
Chris Tate
2026-10-23 13:09:08 -05:00
parent 9b3c7bb520
commit 1562d08609
171 changed files with 4583 additions and 311 deletions
+5 -1
View File
@@ -52,9 +52,13 @@ jobs:
const stackBase = pr?.stack?.base?.ref;
const prBase = pr?.base?.ref;
// Position 1 can already be merged; compare the current base branches.
// A PR into the default branch is the lowest unmerged layer of any
// stack, whatever the stack metadata names as its base.
const defaultBranch = event.repository?.default_branch;
const upperLayer = process.env.GITHUB_EVENT_NAME === 'pull_request' &&
sameRepository && trustedAuthor && typeof stackBase === 'string' && stackBase.length > 0 &&
typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase;
typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase &&
prBase !== defaultBranch;
const full = !upperLayer;
appendFileSync(process.env.GITHUB_OUTPUT, `full=${full}\n`);
console.log(full ? 'Run full CI' : 'Defer full CI until this PR becomes the lowest unmerged layer');
+3 -1
View File
@@ -32,7 +32,9 @@ on:
- ".github/workflows/libnode-packs.yml"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
# A fixed name: under workflow_call, github.workflow is the caller's, and
# sharing the caller's group would cancel the run as a deadlock.
group: libnode-packs-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
+1 -1
View File
@@ -53,7 +53,7 @@ Type-only imports follow Node.js's type stripping. `import type` and `export typ
scriptc compiles a package automatically when it:
- ships its own type declarations, or is typed by its DefinitelyTyped `@types/` package, or is a dependency of another package that has no declarations at all, and
- ships readable JavaScript: unminified, or minified by a bundler whose helpers keep their names.
- ships JavaScript that isn't a webpack bundle. Minified JavaScript compiles like unminified JavaScript.
The declarations check your program; the package's JavaScript compiles with the types scriptc infers from it. A program that doesn't typecheck against the inferred types, for example because an `@types/` package describes another release, fails the build at that use.
@@ -18,7 +18,7 @@
Stages may take the `{ signal }` argument.
With a function destination, `pipeline()` returns an object-mode PassThrough that receives an async generator destination's values, or an async function destination's value unless it is `null` or `undefined`.
`@types/node` types that result as `NodeJS.WritableStream`, so `for await` over it needs the `PassThrough` type (`pipeline(...) as PassThrough`, or a JSDoc `@type`).
In typed code, the callback of a pipeline with an async function destination receives `null` instead of `undefined` when there is no error.
In typed code, a pipeline callback with an async function or async generator destination receives `null` for no error when its error parameter cannot hold `undefined` (@types/node declares it `NodeJS.ErrnoException | null`), and `undefined` when it can, as in Node.
`Readable.from` accepts `objectMode` and `highWaterMark` options.
The `Readable` operators (`map`, `filter`, `forEach`, `toArray`, `some`, `find`, `every`, `flatMap`, `drop`, `take`, and `reduce`) accept `concurrency`, `highWaterMark`, and `signal` options.
Their callbacks may take the `{ signal }` argument, `map` may return any value, and `flatMap` may return any iterable, async iterable, or stream.
@@ -32,9 +32,10 @@
`readable.compose(stage, { signal })` is `stream.compose(readable, stage)` with the signal added; the options are an object literal whose `signal` is a variable or a property read.
`pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags.
Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream.
The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on Writable and Duplex; `writev` on a Transform or PassThrough, assigning `_construct` after construction, and a non-literal `decodeStrings` or `defaultEncoding` in an options object literal are unsupported.
The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on every class with a writable side.
A `decodeStrings` or `defaultEncoding` value known only at run time is unsupported after an option member whose value has effects (a call or an assignment).
In typed code, a `writev` entry's `encoding` declared as a string receives `""` where Node passes `undefined` or `null` (an object-mode write).
`setDefaultEncoding()` takes any encoding Node knows.
A `transform` or `flush` option callback whose completion callback types `data` as `any`, as @types/node's `TransformCallback` does, may pass a Buffer, a string, or nothing; any other value throws a TypeError where the callback is called, instead of Node's `ERR_INVALID_ARG_TYPE` stream error.
`node:readline` and `node:readline/promises` (also reached as `readline.promises`) interfaces, from `createInterface(options)` or `createInterface(input, output, completer, terminal)`, read `process.stdin` or a readable stream: `question`, `prompt`, `write` (with a key object literal of `name`, `ctrl`, `meta`, and `shift`), `pause`, `resume`, `'line'`/`'close'`/`'pause'`/`'resume'`/`'error'`/`'history'`/`'SIGINT'`/`'SIGTSTP'`/`'SIGCONT'` listeners, and `for await` compile.
`on`, `once`, and `addListener` return the interface, so registrations chain.
The `history`, `historySize`, `removeHistoryDuplicates`, `completer`, `tabSize`, and `escapeCodeTimeout` options are validated as in Node.
@@ -42,7 +43,7 @@
The output's `'resize'` event does not redraw the line.
The cursor never stops between the two UTF-16 units of a supplementary character: where Node's Up or Down movement through a multiline line would, the cursor moves to the character's start or end.
In JavaScript, a `'history'` listener receives a copy of the history; its changes are written back when the listener returns, but the copy is not `rl.history` itself.
`rl.history`, `historySize`, `historyIndex`, `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `cursor`, and `terminal` read as in Node, except that `rl.cursor` of an interface that is not a terminal throws instead of answering `undefined`.
`rl.history`, `historySize`, `historyIndex`, `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `cursor`, and `terminal` read as in Node; `rl.cursor` is `undefined` on an interface that is not a terminal.
`rl[Symbol.dispose]()` closes the interface, and `instanceof` with `readline.Interface` or `readlinePromises.Interface` answers whether the interface came from that module, as in Node.
In JavaScript, the input and output can be any expression that evaluates to a stream, such as a function's result or an object's member; a value that is not a stream is unsupported.
In JavaScript, `question()` and `setPrompt()` accept a value that is not a string: when such a prompt is written, an interface that is not a terminal throws `ERR_INVALID_ARG_TYPE` as Node does (the `readline/promises` question rejects with it) and the question stays pending, and a terminal interface writes `String(value)`.
+2 -3
View File
@@ -70,9 +70,6 @@ async function tryFastPath(): Promise<number | null> {
values["profile-generate"] !== undefined ||
values["profile-use"] !== undefined ||
values.sparse ||
// The full compiler keys these builds by the recorded libnode pack.
values.libnode !== undefined ||
process.env["SCRIPTC_LIBNODE"] !== undefined ||
(values["external-types"] ?? []).length > 0
)
return null;
@@ -159,6 +156,8 @@ async function tryFastPath(): Promise<number | null> {
)
: startup.executableNativeEnvironmentFingerprint(),
nodeVersion: process.version,
// An entry of a program that calls into Node.js serves only the same pack.
libnodeIdentity: () => startup.libnodeCacheKey(values.libnode, process.env, dirname(input)),
});
if (hit === null) return null;
if (!values["keep-llvm"]) await rm(hit.llvmPath, { force: true });
+2 -2
View File
@@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => {
minimum_os: "14.0",
},
program: { object, entry_symbol: "main" },
runtime_abi: { version: 36, marker: "scr_runtime_abi_v36" },
runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" },
runtime_pack: {
kind: "precompiled",
package: "@scriptc/runtime-darwin-arm64",
@@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => {
() => null,
(failure: { stderr?: string }) => failure,
);
expect(error?.stderr).toContain("scr_runtime_abi_v36");
expect(error?.stderr).toContain("scr_runtime_abi_v37");
});
});
+35 -4
View File
@@ -370,6 +370,36 @@ export class LlDyn {
B.line(`call void @scr_dyn_build_add(ptr %d, ptr @${name}, ptr ${value})`);
}
/** The order a dyn value tries a union's arms in: the canonical order,
* except that among the record arms, those with more required fields go
* first, then those with more fields. Records are width-tolerant, so an
* arm with fewer fields (`{}`, or one whose fields are all optional,
* matches every object) would otherwise take an object that a wider arm
* also matches and drop the fields only the wider arm has. */
private unionArmOrder(def: IrUnionDef): number[] {
const width = (i: number): [number, number] | null => {
const arm = def.arms[i]!;
if (arm.kind !== "record") return null;
const shape = this.host.recordsById.get(arm.shapeId);
if (!shape || shape.tuple) return null;
const required = shape.fields.filter(
(field) =>
field.type.kind !== "union" ||
!this.host.unionsById.get(field.type.unionId)?.arms.some((x) => x.kind === "undefinedT"),
).length;
return [required, shape.fields.length];
};
const order = def.arms.map((_, i) => i);
const records = order.filter((i) => width(i) !== null);
const widest = [...records].sort((x, y) => {
const [rx, fx] = width(x)!;
const [ry, fy] = width(y)!;
return ry - rx || fy - fx || x - y;
});
records.forEach((slot, k) => (order[slot] = widest[k]!));
return order;
}
/** Branch to a variant only after its literal discriminator and complete
* structure match. Use the same path for predicates and builders. */
private unionArmMatch(
@@ -2368,9 +2398,10 @@ export class LlDyn {
throw new InternalCompilerError(
`llvm emitter bug: dynCheck of unknown union ${t.unionId}`,
);
// Arms in CANONICAL order, first FULL match wins. The matched
// arm's builder can no longer fail.
def.arms.forEach((arm, i) => {
// Arms in unionArmOrder, first FULL match wins. The matched arm's
// builder can no longer fail.
for (const i of this.unionArmOrder(def)) {
const arm = def.arms[i]!;
const lHit = B.newLabel("dcu.h");
const lNext = B.newLabel("dcu.n");
this.unionArmMatch(B, def, i, lHit, lNext, preserveRefs);
@@ -2414,7 +2445,7 @@ export class LlDyn {
B.terminate(`ret ptr ${u}`);
}
B.startBlock(lNext);
});
}
if (preserveRefs) {
B.line(
`call void @scr_dyn_check_fail(ptr %path, ptr ${host.cstr("reference with its original layout")}, ptr %d)`,
@@ -17,6 +17,7 @@ import {
} from "../mangle.js";
import {
type IrGlobal,
type IrNodeIslandModules,
type IrType,
RUNTIME_ERROR_CLASSES,
RUNTIME_EVENT_CLASS,
@@ -57,7 +58,8 @@ import {
ffiNativeTypeLl,
} from "./common.js";
import { literalCapWord, literalCapWord32 } from "./string-key-hash.js";
import { hasLoneSurrogate, wtf8Bytes } from "../literals.js";
import { hasLoneSurrogate, llvmBytes, wtf8Bytes } from "../literals.js";
import { nodeIslandManifest } from "../../node-islands/modules.js";
import { undefinedArmTag } from "../../ir/analysis.js";
import { NULLABLE_ABSENT, NULLABLE_NULL } from "./nullable-unions.js";
import { literalEqualityDefinition } from "./string-literal-equality.js";
@@ -381,6 +383,11 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
emitter.cstr(emitter.mod.nodeIsland?.nodeVersion ?? ""),
]
: null;
// The island module tree: its manifest, and its files unless they ship
// beside the executable (node-islands/modules.ts).
const nodeIslandModules = usesNodeIsland ? emitter.mod.nodeIsland?.modules : undefined;
const nodeIslandModulesCall =
nodeIslandModules === undefined ? null : nodeIslandModulesInstall(emitter, nodeIslandModules);
if (emitter.ffiHasForeignCallback) emitter.declare(`declare void @scr_ffi_install()`);
if (usesStream) emitter.declare(`declare void @scr_stream_install()`);
if (usesNetLookup) emitter.declare(`declare void @scr_net_lookup_install()`);
@@ -732,6 +739,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
for (let site = 0; site < emitter.jsIcSiteCount; site++)
out.push(`@sc_js_ic_${site} = internal ${tl}global [${JS_IC_WORDS} x i64] zeroinitializer`);
}
if (nodeIslandModulesCall !== null) out.push(...nodeIslandModulesCall.globals);
for (const table of emitter.constantNumericTables.values()) {
const n = table.values.length;
let tableTmp = 0;
@@ -1027,6 +1035,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
` call void @scr_node_island_install(ptr ${nodeIslandConfig[0]}, ptr ${nodeIslandConfig[1]})`,
]
: []),
...(nodeIslandModulesCall === null ? [] : [nodeIslandModulesCall.call]),
...(emitter.ffiHasForeignCallback ? [` call void @scr_ffi_install()`] : []),
...(snapshotsTlsCa ? [` call void @scr_tls_ca_install()`] : []),
...(usesFetch ? [` call void @scr_fetch_install()`] : []),
@@ -1171,6 +1180,33 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
/** The shared abort helpers (emitted only when referenced): the OOM
* abort of untraced shape allocation and the invalid-union-tag abort —
* both print the runtime ABI’s exact message on fd 2 and abort. */
/** The island module tree's install call before %main, and the constant
* that carries its files when the executable ships them itself
* (scr_node_island.c: scr_node_island_install_modules). */
function nodeIslandModulesInstall(
emitter: LlEmitter,
modules: IrNodeIslandModules,
): { call: string; globals: string[] } {
emitter.declare(
`declare void @scr_node_island_install_modules(ptr, ${emitter.sizeType}, ptr, ${emitter.sizeType}, ptr)`,
);
const manifest = nodeIslandManifest(modules);
const manifestPtr = emitter.cstr(manifest);
const location = emitter.cstr(modules.location ?? "");
const data = modules.location === undefined ? emitter.nodeIslandModuleData : null;
const globals =
data === null || data.length === 0
? []
: [
`@sc_node_island_modules = private unnamed_addr constant [${data.length} x i8] c"${llvmBytes(data, false)}", align 16`,
];
const st = emitter.sizeType;
return {
call: ` call void @scr_node_island_install_modules(ptr ${manifestPtr}, ${st} ${Buffer.byteLength(manifest, "utf8")}, ptr ${globals.length > 0 ? "@sc_node_island_modules" : "null"}, ${st} ${globals.length > 0 ? data!.length : 0}, ptr ${location})`,
globals,
};
}
export function helperDefs(emitter: LlEmitter): string[] {
const defs: string[] = [];
const msgHelper = (fnName: string, msgSym: string, msg: string): void => {
@@ -473,6 +473,9 @@ export interface LlvmTargetOptions {
* teardown after `process.exit` to a helper process (`--detached-exit`).
* Executables only; the runtime's weak default keeps it off. */
detachedExit?: boolean;
/** The files of the program's Node island module tree, in manifest order
* (node-islands/modules.ts), when the executable carries the tree. */
nodeIslandModuleData?: Uint8Array;
/** Translated WebAssembly memories check bounds explicitly instead of
* relying on guard pages (wasm-modules.ts wasmBoundsChecks). */
wasmBoundsChecks?: boolean;
@@ -1036,6 +1039,7 @@ export class LlEmitter {
readonly emitLibraryIdentity: boolean;
readonly runtimeAbiMarker: boolean;
readonly detachedExit: boolean;
readonly nodeIslandModuleData: Uint8Array | null;
readonly wasmBoundsChecks: boolean;
/** Interned string literals: UTF-8 text → { symbol, byte length } —
* first-use order, the runtime ABI’s determinism discipline. */
@@ -1618,6 +1622,7 @@ export class LlEmitter {
this.emitLibraryIdentity = options.emitLibraryIdentity !== false;
this.runtimeAbiMarker = options.runtimeAbiMarker === true;
this.detachedExit = options.detachedExit === true;
this.nodeIslandModuleData = options.nodeIslandModuleData ?? null;
this.wasmBoundsChecks =
options.wasmBoundsChecks === true ||
wasmBoundsChecks({
@@ -727,6 +727,17 @@ export function streamCbThunkFor(
const ty = host.llType(p);
host.declare(`declare zeroext i1 @scr_exc_pending()`);
if (hasThis) host.declare(`declare void @scr_stream_release_v(ptr)`);
// An entry `encoding` declared as a string (the declarations'
// `encoding: string`) receives "" where Node passes undefined or
// null (an object-mode write), as a write's encoding parameter does.
const entry =
p.kind === "array" && p.elem.kind === "record"
? host.recordsById.get(p.elem.shapeId)
: undefined;
if (entry?.fields.some((f) => f.name === "encoding" && f.type.kind === "string")) {
host.declare(`declare void @scr_stream_writev_fill_encodings(ptr)`);
d.push(` call void @scr_stream_writev_fill_encodings(ptr %chunks)`);
}
d.push(
` %vt${i} = call ${ty === "i1" ? "zeroext i1" : ty} @${helper}(ptr %chunks, ptr null)`,
` %vtbad${i} = call zeroext i1 @scr_exc_pending()`,
@@ -1142,6 +1142,8 @@ export const LIB_FN_SYMS: Record<string, string> = {
"readable.pushEncoding": "scr_stream_set_push_encoding",
"stream.objectMode": "scr_stream_object_mode",
"stream.decodeStrings": "scr_stream_decode_strings",
"stream.optionValues": "scr_stream_option_values",
"stream.checkEncodingOption": "scr_stream_check_encoding_option",
"stream.duplexFromGen": "scr_stream_duplex_from_gen",
"stream.duplexFromDyn": "scr_stream_duplex_from_dyn",
"stream.duplexifyReadable": "scr_stream_duplexify_readable",
@@ -1776,6 +1778,7 @@ export const USES_TIMERS_LIB_FNS = new Set<string>([
"stream.setTransform",
"stream.setFlush",
"stream.setConstruct",
"stream.replaceConstruct",
"stream.duplexFromGen",
"stream.duplexFromDyn",
"stream.duplexifyReadable",
@@ -189,6 +189,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV
e.fn === "stream.setTransform" ||
e.fn === "stream.setFlush" ||
e.fn === "stream.setConstruct" ||
e.fn === "stream.replaceConstruct" ||
e.fn === "stream.setWritev" ||
e.fn === "stream.setWritevDefault"
) {
@@ -203,6 +204,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV
"stream.setTransform": "t",
"stream.setFlush": "l",
"stream.setConstruct": "c",
"stream.replaceConstruct": "c",
"stream.setWritev": "v",
"stream.setWritevDefault": "v",
};
@@ -214,6 +216,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV
"stream.setTransform": "scr_stream_set_transform",
"stream.setFlush": "scr_stream_set_flush",
"stream.setConstruct": "scr_stream_set_construct",
"stream.replaceConstruct": "scr_stream_replace_construct",
"stream.setWritev": "scr_stream_set_writev",
"stream.setWritevDefault": "scr_stream_set_writev_default",
};
+2 -2
View File
@@ -2,8 +2,8 @@
* of the link contract: a program object keeps this symbol undefined and the
* matching runtime defines it, so incompatible manual links fail before the
* program can start. */
export const RUNTIME_ABI_VERSION = 36 as const;
export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v36" as const;
export const RUNTIME_ABI_VERSION = 37 as const;
export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v37" as const;
/** Object consumption is useful today, but the complete scr_* surface may
* still change before 1.0. The versioned marker prevents accidental mixing;
@@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) {
object_format: target.objectFormat,
minimum_os: target.minimumOs,
},
runtime_abi: { version: 36, marker: "scr_runtime_abi_v36" },
runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" },
compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple },
macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" },
flavors: {
@@ -127,7 +127,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) {
object_format: target.objectFormat,
minimum_os: target.minimumOs,
},
runtime_abi: { version: 36, marker: "scr_runtime_abi_v36" },
runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" },
compiler: {
command: "clang",
identity: "fixture clang",
+3 -3
View File
@@ -82,9 +82,9 @@ Options:
--npm-static <pkg[,pkg…]|auto>
compile the named npm packages' shipped JS statically as
program modules (repeatable; "auto" opts in every eligible
direct import: own .d.ts, unminified JS, no build-transform
markers; "pkg..." also names every package pkg depends
on). Packages that fail preflight are refused with a
direct import: own .d.ts, no build-transform markers;
"pkg..." also names every package pkg depends on).
Packages that fail preflight are refused with a
diagnostic. Eligible dependencies compile automatically.
--provenance-sources
EXPERIMENTAL: compile npm dependencies from their
@@ -66,7 +66,8 @@
* likely-hot functions that use untyped values (SC6101,
* diagnostics/dynamic-tier.ts)
* SC8xxx Node islands: the program calls into Node.js, and the build
* finds no usable libnode pack (SC8005)
* finds no usable libnode pack (SC8005), or cannot collect the
* modules a call loads (SC8006)
* SC9xxx internal compiler errors (still source-anchored)
*/
import type { SrcLoc } from "../ir/ir.js";
@@ -133,6 +134,18 @@ export function nodeIslandNodeDiag(message: string, file: string): ScrDiagnostic
};
}
/** SC8006 — a Node island call whose modules the build cannot collect into
* the executable: a computed specifier, or a module of the program that
* loads others through computed specifiers. */
export function nodeIslandModuleDiag(message: string, loc: SrcLoc): ScrDiagnostic {
return {
code: "SC8006",
message,
loc,
hint: "name the module with a string literal (a relative path or a package name); the build ships the modules it can see with the executable",
};
}
/* ── SC6xxx: divergences from Node (warnings) ─────────────────────────── */
/** SC6001 — a value passed to (or declared as) a narrower record type is
@@ -121,6 +121,9 @@ interface EarlyExecutableCacheStamp {
native: EarlyExecutableNativeFeatures;
/** The build's performance warnings (SC61xx), which a cache hit repeats. */
performance?: ScrDiagnostic[];
/** Programs that call into Node.js: the libnode pack identity the build
* recorded (EarlyExecutableCacheOptions.libnodeIdentity). */
libnode?: string;
integrity: string;
}
@@ -139,9 +142,12 @@ export interface EarlyExecutableCacheOptions {
windowsSubsystem?: "gui";
/** Omitted unless --detached-exit is set. */
detachedExit?: true;
/** Omitted unless --libnode or SCRIPTC_LIBNODE is set (calls into Node.js
* record the pack). */
libnode?: string;
/** The identity of the libnode pack a program that calls into Node.js
* records (node-islands/libnode.ts libnodeCacheKey), or undefined when the
* build names none. Not part of the key: an entry of a program that calls
* into Node records it, and a hit requires the same identity, so the pack
* settings never change the key of a program that does not. */
libnodeIdentity?: () => string | undefined;
/** Omitted without --profile-generate/--profile-use; otherwise the
* pattern, or the digest of the profile's bytes. */
profile?: string;
@@ -283,7 +289,6 @@ function cacheKey(options: EarlyExecutableCacheOptions): string {
...(options.strip ? ["strip"] : []),
...(options.windowsSubsystem === "gui" ? ["windows-subsystem-gui"] : []),
...(options.detachedExit ? ["detached-exit"] : []),
...(options.libnode !== undefined ? [`libnode:${options.libnode}`] : []),
...(options.profile !== undefined ? [options.profile] : []),
...(options.checker !== undefined ? [`checker:${options.checker}`] : []),
...(options.youngNursery ? ["young-nursery"] : []),
@@ -326,7 +331,6 @@ function routeKey(options: Omit<EarlyExecutableRouteOptions, "nativeEnvironment"
if (options.strip) hash.update("strip\0");
if (options.windowsSubsystem === "gui") hash.update("windows-subsystem-gui\0");
if (options.detachedExit) hash.update("detached-exit\0");
if (options.libnode !== undefined) hash.update(`libnode:${options.libnode}\0`);
if (options.profile !== undefined) hash.update(`${options.profile}\0`);
if (options.checker !== undefined) hash.update(`checker:${options.checker}\0`);
if (options.youngNursery) hash.update("young-nursery\0");
@@ -533,6 +537,8 @@ async function readExecutableCache(
(stamp.nativeDependencies !== null && !Array.isArray(stamp.nativeDependencies)) ||
(stamp.files.ir !== null) !== options.emitIr ||
stampIntegrity(unsigned) !== integrity ||
(stamp.native.nodeIsland === true &&
(stamp.libnode ?? "") !== (options.libnodeIdentity?.() ?? "")) ||
!frontendInputsStillMatch(
validateRoute === undefined ? stamp.frontend : routedFrontendCandidate(stamp.frontend),
executableFrontendOutputExclusions(options, stamp.native.backend),
@@ -848,6 +854,7 @@ export async function publishEarlyExecutableCache(
...(result.performance === undefined || result.performance.length === 0
? {}
: { performance: result.performance }),
...(result.native.nodeIsland === true ? { libnode: options.libnodeIdentity?.() ?? "" } : {}),
};
const stamp: EarlyExecutableCacheStamp = { ...unsigned, integrity: stampIntegrity(unsigned) };
await writeFile(join(stage, "stamp.json"), `${JSON.stringify(stamp)}\n`, { mode: 0o600 });
@@ -13,6 +13,8 @@ import { observeSourceText } from "./ts7/ast-node.js";
*/
export type FrontendInputProbe =
| { op: "file"; path: string; digest: string }
/** A read of the file's raw bytes (files that need not be text). */
| { op: "bytes"; path: string; digest: string }
| { op: "read-error"; path: string }
| { op: "kind"; path: string; kind: "file" | "directory" | "other" | "missing" }
| { op: "entries"; path: string; files: string[]; directories: string[] }
@@ -208,6 +210,21 @@ export function trackedReadFile(path: string): string | null {
}
}
/** A file's raw bytes, recorded by their digest. */
export function trackedReadBytes(path: string): Uint8Array | null {
path = resolve(path);
const virtual = virtualSources.get(path);
if (virtual !== undefined) return new TextEncoder().encode(virtual);
try {
const bytes = new Uint8Array(readFileSync(path));
record({ op: "bytes", path, digest: createHash("sha256").update(bytes).digest("hex") });
return bytes;
} catch {
record({ op: "read-error", path });
return null;
}
}
export function trackedFileExists(path: string): boolean {
path = resolve(path);
if (virtualSources.has(path)) return true;
@@ -343,6 +360,15 @@ export function frontendInputsStillMatch(
return false;
}
}
case "bytes": {
try {
return (
createHash("sha256").update(readFileSync(probe.path)).digest("hex") === probe.digest
);
} catch {
return false;
}
}
case "read-error": {
try {
readFileSync(probe.path, "utf8");
@@ -488,6 +514,7 @@ export function validFrontendInputSnapshot(snapshot: unknown): snapshot is Front
if (typeof value.path !== "string" || typeof value.op !== "string") return false;
switch (value.op) {
case "file":
case "bytes":
return typeof value.digest === "string" && /^[0-9a-f]{64}$/.test(value.digest);
case "read-error":
return true;
@@ -6,9 +6,11 @@ import {
DYN,
F64,
type IrExpr,
type IrType,
type IrStmt,
type SrcLoc,
STRING,
UNDEFINED_T,
VOID,
arrayOf,
BYTES_U8,
@@ -54,6 +56,21 @@ export function isReadlineTyped(lowerer: Lowerer, node: ts.Expression): boolean
});
}
/** `rl.cursor`: @types/node declares a number, but an interface that is
* not a terminal answers undefined, so the read keeps its
* `number | undefined` union. */
export function isOptionalReadlineCursorRead(
lowerer: Lowerer,
expr: ts.PropertyAccessExpression,
): boolean {
return (
expr.name.text === "cursor" &&
!expr.questionDotToken &&
isReadlineTyped(lowerer, expr.expression) &&
lowerer.isStdlibMember(expr)
);
}
/** The runtime's listener event numbers (scr_readline.c). */
const RL_EVENTS: Record<string, number> = {
line: 0,
@@ -1341,7 +1358,7 @@ export function lowerReadlineCreateCall(
/** Property reads on an Interface: `history` (the history option itself,
* or the interface's own empty array), `historySize`, `historyIndex`,
* `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `terminal` and
* `cursor` (a non-terminal interface's undefined cursor throws SC2020).
* `cursor` (undefined on a non-terminal interface).
* Null for other receivers. */
export function lowerReadlineProperty(
lowerer: Lowerer,
@@ -1373,10 +1390,38 @@ export function lowerReadlineProperty(
loc,
};
}
const reads: Record<string, ["rl.line" | "rl.isTerminal" | "rl.cursor", IrExpr["type"]]> = {
if (name === "cursor") {
// `number | undefined`: a non-terminal interface has no cursor
const recv = lowerer.lowerExprExpecting(access.expression, F64);
const type: IrType = { kind: "union", unionId: lowerer.unions.intern([F64, UNDEFINED_T]) };
const local = recv.kind === "varRef" ? null : lowerer.declareHiddenLocal("%rlCursor", F64);
const id: IrExpr = local ? { kind: "varRef", localId: local.id, type: F64, loc } : recv;
const result: IrExpr = {
kind: "ternary",
cond: { kind: "libCall", fn: "rl.isTerminal", args: [id], type: BOOL, loc },
then: lowerer.coerceToExpected(
{ kind: "libCall", fn: "rl.cursor", args: [id], type: F64, loc },
type,
),
else_: lowerer.coerceToExpected(
{ kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc },
type,
),
type,
loc,
};
if (!local) return result;
return {
kind: "seqExpr",
stmts: [{ kind: "varDecl", localId: local.id, init: recv, loc }],
result,
type,
loc,
};
}
const reads: Record<string, ["rl.line" | "rl.isTerminal", IrExpr["type"]]> = {
line: ["rl.line", STRING],
terminal: ["rl.isTerminal", BOOL],
cursor: ["rl.cursor", F64],
};
const read = reads[name];
if (read !== undefined) {
@@ -2,6 +2,7 @@ import { keyedHelperName } from "../stable-names.js";
import * as ts from "../../ts7/adapter.js";
import {
BOOL,
DYN,
F64,
REGEX,
STRING,
@@ -14,7 +15,7 @@ import {
type IrType,
type SrcLoc,
} from "../../../ir/ir.js";
import { numLit, strLit, varRef } from "../../../ir/build.js";
import { dynUndefinedExpr, numLit, strLit, varRef } from "../../../ir/build.js";
import { InternalCompilerError } from "../../../errors.js";
import { locOf } from "../../program.js";
import type { Lowerer } from "../lowerer.js";
@@ -65,14 +66,14 @@ export function lowerStringReplacement(
const search = argument(args[0], false);
const replacement = argument(args[1], true);
if (search.type.kind === "dyn" || replacement.type.kind === "dyn") {
if (replacement.type.kind === "func") {
lowerer.noLowering(`string .${method} with checked search values and callbacks`, call);
}
// Evaluate all inputs once before choosing the search protocol. Regexes
// retain their native pattern; plain values use the string-search helper.
// A typed callback receives the regex match's arguments or the plain
// search's (match, offset, subject).
const values = [receiver, search, replacement];
const locals = values.map((value) => lowerer.declareHiddenLocal("%replaceArg", value.type));
const refs = locals.map((local) => varRef(local.id, local.type, loc));
const callback = replacement.type.kind === "func";
const checkedReplacement =
replacement.type.kind === "dyn"
? lowerer.coerceInto(args[1] ?? call, refs[2]!, STRING)
@@ -91,14 +92,27 @@ export function lowerStringReplacement(
? {
kind: "ternary",
cond: { kind: "libCall", fn: "dyn.nativeRegexIs", args: [refs[1]!], type: BOOL, loc },
then: {
kind: "regexIntrinsic",
method,
receiver: refs[0]!,
args: [lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX), checkedReplacement],
type: STRING,
loc,
},
then: callback
? {
kind: "libCall",
fn: "regex.replaceCallback",
args: [
refs[0]!,
lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX),
lowerer.coerceToExpected(refs[2]!, DYN),
{ kind: "boolLit", value: method === "replaceAll", type: BOOL, loc },
],
type: STRING,
loc,
}
: {
kind: "regexIntrinsic",
method,
receiver: refs[0]!,
args: [lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX), checkedReplacement],
type: STRING,
loc,
},
else_: plain,
type: STRING,
loc,
@@ -148,11 +162,17 @@ function lowerStringReplacementValues(
if (callback.kind !== "func") {
throw new InternalCompilerError(`string .${method} replacement of type ${callback.kind}`);
}
// A checked parameter receives the boxed argument (undefined after the
// subject), and a checked, number or boolean result converts with
// ToString.
const full = [STRING, F64, STRING];
const ret = callback.ret.kind;
if (
callback.ret.kind !== "string" ||
callback.params.length > full.length ||
!callback.params.every((type, index) => typeEquals(type, full[index]!))
(ret !== "string" && ret !== "dyn" && ret !== "f64" && ret !== "bool") ||
!callback.params.every(
(type, index) =>
type.kind === "dyn" || (index < full.length && typeEquals(type, full[index]!)),
)
) {
lowerer.noLowering(
`string .${method} callback signature`,
@@ -175,13 +195,21 @@ function lowerStringReplacementValues(
const chunksType = arrayOf(STRING);
const chunks = varRef("chunks.0", chunksType, loc);
const builder = stringBuilder(loc, chunks);
const value: IrExpr = {
const result: IrExpr = {
kind: "callValue",
callee: replace,
args: [needle, match, subject].slice(0, callback.params.length),
type: STRING,
args: callback.params.map((type, index) => {
const arg = [needle, match, subject][index];
if (arg === undefined) return dynUndefinedExpr(loc);
return type.kind === "dyn" ? lowerer.coerceToExpected(arg, DYN) : arg;
}),
type: callback.ret,
loc,
};
const value: IrExpr =
ret === "dyn"
? { kind: "libCall", fn: "dyn.toStringCoerce", args: [result], type: STRING, loc }
: lowerer.ensureString(result, call);
const stop: IrStmt = { kind: "break", loc };
const body: IrStmt[] = [
builder.declare("chunks.0", { kind: "arrayLit", elems: [], type: chunksType, loc }),
@@ -2841,6 +2841,24 @@ function lowerLogicalPair(lowerer: Lowerer, expr: ts.BinaryExpression, left: IrE
loc,
};
}
// A JavaScript default that tsc types `any` (`e.title || null` in an
// instance of a function with untyped parameters, where `e.title` is a
// string): the checked domain, as for union operands above.
if (
target === null &&
isJsSourceFile(ts.getSourceFileOfNode(expr)) &&
lowerer.dynConvertible(left.type) &&
lowerer.dynConvertible(right.type)
) {
return {
kind: "logical",
op: op === ts.SyntaxKind.AmpersandAmpersandToken ? "&&" : "||",
left: lowerer.coerceInto(expr.left, left, DYN),
right: lowerer.coerceInto(expr.right, right, DYN),
type: DYN,
loc,
};
}
lowerer.unsupported("SC1042", expr);
}
return {
@@ -33,6 +33,7 @@ import {
isOptionalProcessVersionsRead,
} from "../builtins/process.js";
import { requestGeneralStreamSurface } from "../general-streams.js";
import { isOptionalReadlineCursorRead } from "../builtins/readline.js";
import { literalValues } from "../../literal-values.js";
import { literalUnionArm } from "../../union-discriminants.js";
import { checkedArmExtraction } from "../coercions/unions.js";
@@ -617,6 +618,8 @@ export function maybeNarrow(lowerer: Lowerer, expr: IrExpr, node: ts.Node): IrEx
if (ts.isPropertyAccessExpression(node) && isOptionalProcessStreamProperty(lowerer, node))
return expr;
if (isOptionalProcessVersionsRead(lowerer, node)) return expr;
if (ts.isPropertyAccessExpression(node) && isOptionalReadlineCursorRead(lowerer, node))
return expr;
// Represented fields and index values can include undefined even when
// checker inference or lib.d.ts declares strings (RegExp captures).
@@ -308,6 +308,22 @@ function mayHoldError(lowerer: Lowerer, type: ts.Type, errorType: ts.Type | null
);
}
/** The `f` of a `(0, f)` callee (a numeric left operand, so dropping it
* changes only the call's receiver), or null. */
function receiverlessCalleeOf(callee: ts.Expression): ts.Expression | null {
let e = callee;
while (ts.isParenthesizedExpression(e)) e = e.expression;
if (
!ts.isBinaryExpression(e) ||
e.operatorToken.kind !== ts.SyntaxKind.CommaToken ||
!ts.isNumericLiteral(e.left)
)
return null;
let right = e.right;
while (ts.isParenthesizedExpression(right)) right = right.expression;
return right;
}
export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr {
const loc = locOf(expr);
if (lowerer.errorAccessorReflection === null) noteErrorAccessorReflection(lowerer, expr);
@@ -331,6 +347,17 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr {
const json = lowerJsonRequireCall(lowerer, expr);
if (json) return json;
// `(0, ns.member)(...)` over a builtin module (tsc's and esbuild's call
// without a receiver): Node's module functions do not read `this`, so it
// is the member call.
{
const member = receiverlessCalleeOf(expr.expression);
if (member !== null && ts.isPropertyAccessExpression(member)) {
const builtin = lowerer.lowerNamespaceBuiltinCall(expr, member);
if (builtin) return builtin;
}
}
// eval(...) and Function(...): Node's EvalError under
// --disallow-code-generation-from-strings (code-generation.ts).
{
@@ -2738,8 +2738,8 @@ export function collectClassShapeInner(
// state, so an override or shadowing field would split-brain.
// Underscore methods are the SUPPORTED override form — but only
// the ones the class's own base consumes (a `_read` on a
// Transform, or `_writev`/`_construct` anywhere, would be consumed
// by Node machinery that has no lowering here).
// Transform, or `_writev` on a Readable, would be consumed by Node
// machinery that has no lowering here).
if (memberName && ts.isIdentifier(memberName)) {
const streamBase = (() => {
for (let c = base; c; c = c.base) if (c.builtinStream) return c;
@@ -2758,7 +2758,7 @@ export function collectClassShapeInner(
lowerer.unsupported(
"SC1090",
memberName,
`declaring '_writev' on a ${streamBase.def.name.slice(1)} subclass (batched writes are supported on Writable and Duplex subclasses)`,
`declaring '_writev' on a ${streamBase.def.name.slice(1)} subclass (a Readable has no writable side)`,
);
}
const accepted = streamCtorShape(streamBase.def.name).accepted;
@@ -8,7 +8,11 @@
* Semantics, shared with packages/node/index.js (the Node implementation):
* the arguments are copied with JSON.stringify of the argument array; the
* module loads with require() from the calling module, so relative
* specifiers resolve from it; the result crosses as its JSON copy and is
* specifiers resolve from it. The specifier is a string literal: the build
* collects the module and what it loads into the executable
* (node-islands/modules.ts), and the call names its caller by its path in
* that tree, never by a path on the build machine. The result crosses as
* its JSON copy and is
* checked against the declared result type, like `JSON.parse(s) as T`; a
* thrown Error crosses as its nearest intrinsic class with its message and
* string `code`. */
@@ -16,7 +20,8 @@ import { realpathSync } from "node:fs";
import * as ts from "../ts7/adapter.js";
import { locOf } from "../program.js";
import type { NodeApiMember } from "../node-api-import.js";
import { type Lowerer } from "./lowerer.js";
import { PoisonError, type Lowerer } from "./lowerer.js";
import { nodeIslandModuleDiag } from "../../diagnostics/diagnostic.js";
import { DYN, STRING, type IrExpr, type IrType } from "../../ir/ir.js";
const JSON_HINT =
@@ -89,7 +94,20 @@ export function lowerNodeApiCall(
"pass the module specifier, the export name and each argument explicitly",
);
}
const specifier = lowerer.lowerExprExpecting(argNodes[0]!, STRING);
const specifierNode = argNodes[0]!;
if (!ts.isStringLiteral(specifierNode) && !ts.isNoSubstitutionTemplateLiteral(specifierNode)) {
lowerer.pushDiag(
nodeIslandModuleDiag(`${surface} with a computed module specifier`, locOf(specifierNode)),
);
throw new PoisonError();
}
const caller = callerPath(call);
const problem = lowerer.nodeIslandModules.add(caller, specifierNode.text);
if (problem !== null) {
lowerer.pushDiag(nodeIslandModuleDiag(`${surface}: ${problem}`, locOf(specifierNode)));
throw new PoisonError();
}
const specifier = lowerer.lowerExprExpecting(specifierNode, STRING);
const name = lowerer.lowerExprExpecting(argNodes[1]!, STRING);
const values = argNodes.slice(2).map((arg) => {
const value = lowerer.lowerExpr(arg);
@@ -109,15 +127,18 @@ export function lowerNodeApiCall(
lowerer.noLowering(`'${lowerer.fmt(bad)}' results of ${surface}`, call, JSON_HINT);
}
lowerer.usesNodeIsland = true;
// The caller's tree path is known once every call is collected (Lowerer).
const callerLiteral: IrExpr & { kind: "strLit" } = {
kind: "strLit",
value: caller,
type: STRING,
loc,
};
lowerer.nodeIslandCallers.push({ literal: callerLiteral, caller });
const raw: IrExpr = {
kind: "libCall",
fn: member === "call" ? "nodeIsland.call" : "nodeIsland.callSync",
args: [
{ kind: "strLit", value: callerPath(call), type: STRING, loc },
specifier,
name,
{ kind: "dynArrLit", elems: values, type: DYN, loc },
],
args: [callerLiteral, specifier, name, { kind: "dynArrLit", elems: values, type: DYN, loc }],
type: member === "call" ? { kind: "promise", inner: DYN } : DYN,
loc,
};
@@ -106,6 +106,61 @@ function pureRead(node: ts.Expression): boolean {
return false;
}
/** A literal, `undefined`, or a function expression: a value that reads
* no binding or property. */
function constantOptionValue(node: ts.Expression): boolean {
let e = node;
while (ts.isParenthesizedExpression(e) || ts.isAsExpression(e) || ts.isTypeAssertion(e))
e = e.expression;
if (ts.isIdentifier(e)) return e.text === "undefined";
if (
ts.isPrefixUnaryExpression(e) &&
(e.operator === ts.SyntaxKind.MinusToken ||
e.operator === ts.SyntaxKind.PlusToken ||
e.operator === ts.SyntaxKind.ExclamationToken)
)
return constantOptionValue(e.operand);
return (
ts.isStringLiteralLike(e) ||
ts.isNumericLiteral(e) ||
ts.isBigIntLiteral(e) ||
e.kind === ts.SyntaxKind.TrueKeyword ||
e.kind === ts.SyntaxKind.FalseKeyword ||
e.kind === ts.SyntaxKind.NullKeyword ||
ts.isArrowFunction(e) ||
ts.isFunctionExpression(e)
);
}
/** Whether a runtime decodeStrings or defaultEncoding value (`prop`'s
* `value`) may be read before the other option members. Those values are
* read into locals before the constructor evaluates the other members,
* while Node evaluates the members in order, so every earlier member must
* evaluate without effects, and when the value itself has effects, every
* earlier member must also read nothing those effects could change. */
function earlierOptionsPure(
options: ts.ObjectLiteralExpression,
prop: ts.Node,
value: ts.Expression,
): boolean {
const valueHasEffects = !sideEffectFreeOptionValue(value) && !pureRead(value);
for (const member of options.properties.nodes) {
if (member === prop) return true;
if (ts.isPropertyAssignment(member)) {
const init = member.initializer;
if (
valueHasEffects
? !constantOptionValue(init)
: !sideEffectFreeOptionValue(init) && !pureRead(init)
)
return false;
} else if (ts.isShorthandPropertyAssignment(member)) {
if (valueHasEffects) return false;
} else if (ts.isSpreadAssignment(member)) return false;
}
return true;
}
/** The stream sides of a receiver class: the nearest stream-class
* ancestor's, or null off the stream hierarchy. */
export function streamSidesOf(
@@ -179,15 +234,15 @@ export const UNDERSCORE_METHODS: ReadonlyMap<string, string> = new Map([
/** The option callbacks applied after construction rather than through
* the constructor's callback slots (option name → method name): construct
* on every stream class, and writev on the writable classes whose
* machinery hands it batches (Writable and Duplex). */
* on every stream class, and writev on every class with a writable side
* (a Transform's batches reach writev, not the transform, as in Node). */
export const STREAM_EXTRA_CALLBACKS: ReadonlyMap<string, string> = new Map([
["construct", "_construct"],
["writev", "_writev"],
]);
export function streamExtraCallbacks(cls: string): readonly string[] {
return cls === "%Writable" || cls === "%Duplex" ? ["construct", "writev"] : ["construct"];
return cls === "%Readable" ? ["construct"] : ["construct", "writev"];
}
/** The completion callback's parameter position in each underscore
@@ -442,6 +497,13 @@ interface StreamOptions {
* right after construction (Node's constructors call addAbortSignal
* once the state exists), or null when absent. */
signal: IrExpr | null;
/** Non-literal decodeStrings and defaultEncoding values: valueDecls
* evaluates them into locals before construction (the defaultEncoding
* validated there, as the state constructors throw before the stream
* exists), and stream.optionValues applies them to the fresh stream. */
valueDecls: IrStmt[];
decodeStringsValue: IrExpr | null;
defaultEncodingValue: IrExpr | null;
/** The objectMode option's sides (a Duplex's readableObjectMode and
* writableObjectMode set one side): applied with stream.objectMode
* right after construction. */
@@ -732,6 +794,12 @@ function pinAnyDataCompletion(
if (!symbol || lowerer.stageSourceTypes.has(symbol)) return null;
const declared = lowerer.paramShape(param).type;
if (declared.kind !== "func" || declared.rest || declared.params[1]?.kind !== "dyn") return null;
// Only when every use calls the callback with a Buffer, a string or
// nothing as `data`: another value (or a callback that escapes) keeps
// `data` checked-dynamic, so the runtime pushes it as push(data) does
// and reports ERR_INVALID_ARG_TYPE through 'error', as Node does.
if (fnNode.body === undefined || !completionDataIsChunk(lowerer, fnNode.body, symbol, param.name))
return null;
const data = unionOf(lowerer, [BYTES, STRING, { kind: "undefinedT" }]);
lowerer.stageSourceTypes.set(
symbol,
@@ -740,6 +808,59 @@ function pinAnyDataCompletion(
return symbol;
}
/** Whether every reference to the completion callback `symbol` in `body`
* calls it with a `data` argument that is a Buffer, a string, null or
* undefined (or with no data). */
function completionDataIsChunk(
lowerer: Lowerer,
body: ts.Node,
symbol: ts.Symbol,
declaration: ts.Node,
): boolean {
let ok = true;
const visit = (node: ts.Node): void => {
if (!ok) return;
if (
ts.isIdentifier(node) &&
node !== declaration &&
lowerer.checker.getSymbolAtLocation(node) === symbol
) {
const call = node.parent;
if (!call || !ts.isCallExpression(call) || call.expression !== node) {
ok = false;
return;
}
const data = call.arguments.nodes[1];
if (data !== undefined) {
let t: IrType | null;
try {
t = lowerer.mapTypeOf(lowerer.typeOf(data));
} catch {
t = null;
}
const arms =
t?.kind === "union" ? (lowerer.unions.get(t.unionId)?.arms ?? null) : t ? [t] : null;
if (
arms === null ||
!arms.every(
(a) =>
(a.kind === "bytes" && a.elem === "u8") ||
a.kind === "string" ||
a.kind === "undefinedT" ||
a.kind === "nullT",
)
) {
ok = false;
return;
}
}
}
ts.forEachChild(node, visit);
};
visit(body);
return ok;
}
/** Parses the options object literal of a stream constructor. `which`
* names the callbacks this class accepts, in canonical flag order. */
function parseStreamOptions(
@@ -767,6 +888,9 @@ function parseStreamOptions(
badEncoding: null,
pushEncoding: null,
signal: null,
valueDecls: [],
decodeStringsValue: null,
defaultEncodingValue: null,
readableObject: false,
writableObject: false,
decodeStringsOff: false,
@@ -802,6 +926,21 @@ function parseStreamOptions(
optsLit !== undefined &&
ts.isObjectLiteralExpression(optsLit) &&
literalWritesValues(optsLit, ctorName !== "Readable" && ctorName !== "Writable");
// In JavaScript a decodeStrings value known only at run time may let
// string writes through, so the write and transform callbacks' chunk
// parameters read as any (the declarations type it Buffer).
const runtimeDecodeStrings =
optsLit !== undefined &&
isJsSourceFile(ts.getSourceFileOfNode(optsLit)) &&
ts.isObjectLiteralExpression(optsLit) &&
(optsLit.properties?.nodes ?? []).some(
(p) =>
ts.isPropertyAssignment(p) &&
(ts.isIdentifier(p.name) || ts.isStringLiteral(p.name)) &&
p.name.text === "decodeStrings" &&
p.initializer.kind !== ts.SyntaxKind.TrueKeyword &&
p.initializer.kind !== ts.SyntaxKind.FalseKeyword,
);
if (out.readableObject) out.hwmR = numLit(16, loc);
if (out.writableObject) out.hwmW = numLit(16, loc);
const tuples = cbTuples(lowerer);
@@ -851,6 +990,7 @@ function parseStreamOptions(
}
const full = tuples[name]!.map((t) => (t === "fn" ? null : t));
const pinned = out.readableObject ? null : pinAnyDataCompletion(lowerer, name, fnNode);
let anyChunk: ts.Symbol | undefined;
try {
// The completion-callback tail: typed by the checker contextually;
// accept the user's declared func type as the tuple's entry so the
@@ -867,12 +1007,22 @@ function parseStreamOptions(
}
return declared ?? funcOf([], VOID);
});
const chunkParam =
runtimeDecodeStrings && (name === "write" || name === "transform")
? fnNode.parameters.nodes[0]
: undefined;
anyChunk =
chunkParam !== undefined && ts.isIdentifier(chunkParam.name) && !chunkParam.type
? lowerer.checker.getSymbolAtLocation(chunkParam.name)
: undefined;
if (anyChunk) lowerer.anySymbols.add(anyChunk);
present.set(
name,
lowerStreamCallback(lowerer, name, fnNode, thisType, tuple, writesValues),
);
} finally {
if (pinned) lowerer.stageSourceTypes.delete(pinned);
if (anyChunk) lowerer.anySymbols.delete(anyChunk);
}
continue;
}
@@ -925,11 +1075,25 @@ function parseStreamOptions(
if (writableClass) out.decodeStringsOff = true;
break;
}
lowerer.noLowering(
`${ctorName} with a runtime 'decodeStrings' value`,
prop,
"write 'decodeStrings: true' or 'decodeStrings: false'",
);
// a runtime value: WritableState's `options.decodeStrings !== false`
if (
optsLit !== undefined &&
ts.isObjectLiteralExpression(optsLit) &&
!earlierOptionsPure(optsLit, prop, value)
)
lowerer.noLowering(
`the ${ctorName} option 'decodeStrings' with a runtime value that an earlier option value could observe or change`,
prop,
"compute the option values into consts before the options object (the runtime value is read before the other options run)",
);
const local = lowerer.declareHiddenLocal("%decodeStrings", DYN);
out.valueDecls.push({
kind: "varDecl",
localId: local.id,
init: lowerer.lowerExprExpecting(value, DYN),
loc,
});
if (writableClass) out.decodeStringsValue = varRef(local.id, DYN, loc);
break;
}
case "encoding": {
@@ -973,11 +1137,43 @@ function parseStreamOptions(
break;
}
}
lowerer.noLowering(
`the ${ctorName} option 'defaultEncoding' with a runtime value`,
prop,
"write the encoding as a string literal",
);
{
// a runtime value: null, undefined and utf8 are the default,
// another name Buffer.isEncoding knows is kept as given, and any
// other value throws ERR_UNKNOWN_ENCODING before construction
if (
optsLit !== undefined &&
ts.isObjectLiteralExpression(optsLit) &&
!earlierOptionsPure(optsLit, prop, value)
)
lowerer.noLowering(
`the ${ctorName} option 'defaultEncoding' with a runtime value that an earlier option value could observe or change`,
prop,
"compute the option values into consts before the options object (the runtime value is read before the other options run)",
);
const local = lowerer.declareHiddenLocal("%defaultEncoding", DYN);
const ref = varRef(local.id, DYN, loc);
out.valueDecls.push(
{
kind: "varDecl",
localId: local.id,
init: lowerer.lowerExprExpecting(value, DYN),
loc,
},
{
kind: "exprStmt",
expr: {
kind: "libCall",
fn: "stream.checkEncodingOption",
args: [ref],
type: VOID,
loc,
},
loc,
},
);
out.defaultEncodingValue = ref;
}
break;
case "signal": {
// `if (options.signal) addAbortSignal(options.signal, this)`:
@@ -1041,9 +1237,7 @@ function parseStreamOptions(
lowerer.noLowering(
`the ${ctorName} option 'writev'`,
prop,
ctorName === "Readable"
? "a Readable has no writable side"
: "batched writes on a Transform would bypass its transform — writev is supported on Writable and Duplex",
"a Readable has no writable side",
);
break;
default:
@@ -1083,6 +1277,9 @@ function streamCtorArgs(
badEncoding: string | null;
pushEncoding: string | null;
signal: IrExpr | null;
valueDecls: IrStmt[];
decodeStringsValue: IrExpr | null;
defaultEncodingValue: IrExpr | null;
readableObject: boolean;
writableObject: boolean;
decodeStringsOff: boolean;
@@ -1117,6 +1314,9 @@ function streamCtorArgs(
badEncoding: o.badEncoding,
pushEncoding: o.pushEncoding,
signal: o.signal,
valueDecls: o.valueDecls,
decodeStringsValue: o.decodeStringsValue,
defaultEncodingValue: o.defaultEncodingValue,
readableObject: o.readableObject,
writableObject: o.writableObject,
decodeStringsOff: o.decodeStringsOff,
@@ -1237,6 +1437,18 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C
loc,
};
}
const values = optionValuesArgs(parsed, loc);
if (values) {
// runtime decodeStrings and defaultEncoding values (answers the
// receiver +1)
built = {
kind: "libCall",
fn: "stream.optionValues",
args: [built, ...values],
type: thisType,
loc,
};
}
if (parsed.pushEncoding !== null) {
// { defaultEncoding }: chain the push-side decode default (the
// setEncoding chaining shape — answers the receiver +1).
@@ -1269,8 +1481,11 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C
loc,
};
}
if (!parsed.decodeStringsOff && parsed.writeEncoding === null && parsed.extras.length === 0)
return built;
if (!parsed.decodeStringsOff && parsed.writeEncoding === null && parsed.extras.length === 0) {
return parsed.valueDecls.length > 0
? { kind: "seqExpr", stmts: parsed.valueDecls, result: built, type: thisType, loc }
: built;
}
const fresh = lowerer.declareHiddenLocal("%stream", thisType);
const followUps = streamOptionFollowUps(
parsed,
@@ -1280,13 +1495,30 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C
);
return {
kind: "seqExpr",
stmts: [{ kind: "varDecl", localId: fresh.id, init: built, loc }, ...followUps],
stmts: [
...parsed.valueDecls,
{ kind: "varDecl", localId: fresh.id, init: built, loc },
...followUps,
],
result: varRef(fresh.id, thisType, loc),
type: thisType,
loc,
};
}
/** stream.optionValues' (decodeStrings, defaultEncoding) arguments, or
* null when both options are literals or absent. */
function optionValuesArgs(
parsed: { decodeStringsValue: IrExpr | null; defaultEncodingValue: IrExpr | null },
loc: SrcLoc,
): IrExpr[] | null {
if (parsed.decodeStringsValue === null && parsed.defaultEncodingValue === null) return null;
return [
parsed.decodeStringsValue ?? dynUndefinedExpr(loc),
parsed.defaultEncodingValue ?? dynUndefinedExpr(loc),
];
}
/** True when a value of Node's argument type `passed` widens implicitly
* into the declared parameter type `declared` (one of its union arms, or a
* sub-union of it). */
@@ -1589,6 +1821,7 @@ export function lowerStreamSuperCall(
];
}
const out: IrStmt[] = [
...parsed.valueDecls,
{
kind: "exprStmt",
expr: {
@@ -1614,6 +1847,20 @@ export function lowerStreamSuperCall(
loc,
});
}
const values = optionValuesArgs(parsed, loc);
if (values) {
out.push({
kind: "exprStmt",
expr: {
kind: "libCall",
fn: "stream.optionValues",
args: [thisRef(), ...values],
type: thisType,
loc,
},
loc,
});
}
if (parsed.pushEncoding !== null) {
out.push({
kind: "exprStmt",
@@ -1697,16 +1944,15 @@ export function lowerStreamUnderscoreAssign(
const loc = locOf(expr);
const cls = base.def.name;
const clsName = cls.slice(1);
if (methodName === "_construct") {
// Node's constructor reads _construct once; a later assignment has no
// effect there
lowerer.noLowering(
`assigning '${methodName}' on a ${clsName}`,
expr.left,
"declare _construct on the class or pass the construct option",
);
}
const which = methodName === "_writev" ? "writev" : optionByMethod.get(methodName)!;
// `_construct`: Node's constructNT reads it on the construction tick, so
// an assignment before that tick replaces a scheduled hook
// (stream.replaceConstruct) and has no effect after it or without one.
const which =
methodName === "_writev"
? "writev"
: methodName === "_construct"
? "construct"
: optionByMethod.get(methodName)!;
const { accepted } = streamCtorShape(cls);
if (!accepted.includes(which) && !streamExtraCallbacks(cls).includes(which)) {
lowerer.noLowering(
@@ -1745,6 +1991,7 @@ export function lowerStreamUnderscoreAssign(
transform: "stream.setTransform",
flush: "stream.setFlush",
writev: "stream.setWritev",
construct: "stream.replaceConstruct",
};
return {
kind: "exprStmt",
@@ -2699,6 +2946,22 @@ class StreamArgTypeThrow extends Error {
}
}
/** Whether a checked-dynamic pipeline callback can take undefined as its
* error: a callback that is already checked-dynamic, one without
* parameters, and one whose first parameter (or rest element) type holds
* undefined. */
function callbackErrorAdmitsUndefined(lowerer: Lowerer, cb: IrExpr): boolean {
if (cb.kind !== "dynFrom" || cb.value.type.kind !== "func") return true;
const fnT = cb.value.type;
const first = fnT.params[0];
if (first === undefined) return true;
const p0 =
fnT.rest === true && fnT.params.length === 1 && first.kind === "array" ? first.elem : first;
if (p0.kind === "dyn" || p0.kind === "jsval" || p0.kind === "undefinedT") return true;
if (p0.kind !== "union") return false;
return lowerer.unions.get(p0.unionId)?.arms.some((a) => a.kind === "undefinedT") ?? false;
}
/** The finished/pipeline completion callback: an inline function lowers
* through the option-callback machinery (leading `this`, an `Error |
* null` prefix — Node binds the stream and passes the error); any other
@@ -3160,12 +3423,16 @@ export function lowerStreamModuleCall(
// an async generator) reaches the callback through the
// checked-dynamic path.
const cb = lowerer.lowerExprExpecting(args[args.length - 1]!, DYN);
// SCR_PIPE_TYPED_CB: typed code's callback takes null for no error
// SCR_PIPE_TYPED_CB: a typed callback whose error parameter cannot
// hold undefined takes null for no error; others take undefined.
const stagedArgs = [...staged.args];
const flagsArg = stagedArgs[3]!;
if (flagsArg.kind !== "numLit")
throw new InternalCompilerError("pipeline flags not a literal");
if (!isJsSourceFile(ts.getSourceFileOfNode(call)))
if (
!isJsSourceFile(ts.getSourceFileOfNode(call)) &&
!callbackErrorAdmitsUndefined(lowerer, cb)
)
stagedArgs[3] = numLit(flagsArg.value | 128, loc);
return staged.wrap({
kind: "libCall",
@@ -496,7 +496,7 @@ import { lowerBuiltinExtraProperty } from "./builtins/properties.js";
import { registerPromisifiedBuiltinDecl } from "./builtins/callable-aliases.js";
import { lowerExecFileAsyncCall } from "./builtins/child-process-async.js";
import { lowerStringDecoderMethodCall, lowerTextCodecCall } from "./builtins/text-codecs.js";
import { lowerReadlineMethodCall } from "./builtins/readline.js";
import { isOptionalReadlineCursorRead, lowerReadlineMethodCall } from "./builtins/readline.js";
import {
lowerDiagnosticsChannelMethodCall,
lowerDiagnosticsChannelProperty,
@@ -675,6 +675,7 @@ import {
maybePromiseUnionOf,
promiseArmedUnionOf,
} from "./maybe-promise.js";
import { NodeIslandModuleCollector } from "../../node-islands/modules.js";
/** Entry function name. '%' cannot appear in a TS identifier, so a user
* function can never collide with it (mangling is injective per prefix). */
@@ -3140,6 +3141,10 @@ export class Lowerer {
/** The program calls into Node.js (@scriptc/node): the module carries
* the nodeIsland record, which the executable build fills in. */
usesNodeIsland = false;
/** The modules Node island calls load (lower-node-island.ts). */
readonly nodeIslandModules = new NodeIslandModuleCollector();
/** Each call's caller literal, set to its tree path when lowering ends. */
readonly nodeIslandCallers: { literal: IrExpr & { kind: "strLit" }; caller: string }[] = [];
/** Per-directory answers of experimental-import.ts's declaration-file test. */
readonly experimentalFiles = new Map<string, boolean>();
/** Interned LifetimeRegion.run helpers, by callback type key. */
@@ -5558,6 +5563,7 @@ export class Lowerer {
if (isIndexedRead(e)) return true;
if (ts.isPropertyAccessExpression(e) && isRecordIndexPropertyRead(e)) return true;
if (ts.isPropertyAccessExpression(e) && isOptionalProcessStreamProperty(this, e)) return true;
if (ts.isPropertyAccessExpression(e) && isOptionalReadlineCursorRead(this, e)) return true;
if (isOptionalProcessVersionsRead(this, e)) return true;
if (ts.isPropertyAccessExpression(e) && isSettledIteratorValueRead(this, e)) return true;
if (ts.isPropertyAccessExpression(e) && isOsConstantsSymbol(symbolOf(e.expression), this))
@@ -7804,7 +7810,7 @@ export class Lowerer {
entry: ENTRY_NAME,
...(this.usesWorkers ? { workers: true } : {}),
...(this.usesLifetimeRegions ? { lifetimeRegions: true as const } : {}),
...(this.usesNodeIsland ? { nodeIsland: { libnode: "", nodeVersion: "" } } : {}),
...(this.usesNodeIsland ? { nodeIsland: this.nodeIslandRecord() } : {}),
...(this.declBoundary ? { declBoundary: true as const } : {}),
...(this.threadSlotKeys.size > 0
? { threadSlotKeys: [...this.threadSlotKeys].sort() }
@@ -8643,6 +8649,16 @@ export class Lowerer {
this.diags.push(d);
}
/** The module's Node island record: the collected module tree, with each
* call's caller literal set to its path in that tree. The executable
* build fills in the libnode pack (index.ts). */
private nodeIslandRecord(): NonNullable<IrModule["nodeIsland"]> {
const modules = this.nodeIslandModules.tree();
for (const { literal, caller } of this.nodeIslandCallers)
literal.value = NodeIslandModuleCollector.callerPath(modules, caller);
return { libnode: "", nodeVersion: "", ...(modules === null ? {} : { modules }) };
}
unsupported(
code: keyof typeof UNSUPPORTED & `SC${number}`,
node: ts.Node,
@@ -6,9 +6,11 @@
*
* A binding settles at the first statement of its hoisting scope (the
* function body or the source file) that stores it unconditionally: a
* direct `var x = e`, `x = e;`, or a loop declaring it (`for (var i = 0;
* ...)`, `for (var k in o)`), whose own condition, update and body run
* after the store. A read settles when it runs after that statement:
* direct `var x = e`, `x = e;`, or a `for` loop declaring it (`for (var
* i = 0; ...)`), whose own condition, update and body run after the store.
* A `for (var k in o)` or `for (var v of xs)` loop stores the binding only
* when it iterates, so it settles the reads in its body and no others. A
* read settles when it runs after the settling statement:
* - a read in the scope's own code, in a TypeScript file: tsc's
* definite-assignment analysis rejects any earlier one;
* - a read in the scope's own code, in a JavaScript file: its statement
@@ -194,6 +196,18 @@ function referencesBySymbol(
return out;
}
/** Where a binding is first stored unconditionally: the scope statement's
* index, and for a declarator of a `var` statement or a `for` initializer,
* the list and the declarator's end. */
interface Settle {
index: number;
list: ts.VariableDeclarationList | null;
end: number;
/** The first for-in/of loop declaring the binding before `index`, whose
* body runs after its store, or -1. */
loop: number;
}
class ScopeAnalysis {
private readonly js: boolean;
/** Whether a function declaration's references all settle, by scope
@@ -214,10 +228,13 @@ class ScopeAnalysis {
}
/** The index of the scope statement that first stores `symbol`
* unconditionally, or Infinity. */
settleIndex(scope: Scope, symbol: ts.Symbol): number {
* unconditionally (Infinity when none does), and, when that statement
* declares it in a `var` statement or a `for` initializer, the end of the
* storing declarator: later declarators of the list run after it. */
settleIndex(scope: Scope, symbol: ts.Symbol): Settle {
const statements = statementsOf(scope);
const checker = this.lowerer.checker;
let loopIndex = -1;
for (let i = 0; i < statements.length; i++) {
const s = statements[i]!;
const list = ts.isVariableStatement(s)
@@ -233,8 +250,10 @@ class ScopeAnalysis {
if (
(d.initializer !== undefined || loop) &&
boundIdentifiersOf(d.name).some((n) => checker.getSymbolAtLocation(n) === symbol)
)
return i;
) {
if (!loop) return { index: i, list, end: d.end, loop: loopIndex };
if (loopIndex < 0) loopIndex = i;
}
}
} else if (
ts.isExpressionStatement(s) &&
@@ -243,20 +262,21 @@ class ScopeAnalysis {
ts.isIdentifier(s.expression.left) &&
checker.getSymbolAtLocation(s.expression.left) === symbol
) {
return i;
return { index: i, list: null, end: -1, loop: loopIndex };
}
}
return Infinity;
return { index: Infinity, list: null, end: -1, loop: loopIndex };
}
/** Whether `ref` (inside `scope`, or in another module) runs only after
* the statement at index `settle`. `read`: `ref` reads the binding
* itself, so tsc vouches for one in the scope's own code. */
settles(scope: Scope, settle: number, ref: ts.Identifier, read = true): boolean {
* the store `settle` describes. `read`: `ref` reads the binding itself,
* so tsc vouches for one in the scope's own code. */
settles(scope: Scope, settle: Settle, ref: ts.Identifier, read = true): boolean {
if (ts.getSourceFileOfNode(ref) !== this.sf) return true;
const statements = statementsOf(scope);
let deferred: ts.Node | null = null;
let top: ts.Node | null = null;
const container: ts.Node | undefined = ts.isSourceFile(scope) ? scope : scope.body;
for (let n: ts.Node = ref; n.parent !== undefined && n !== scope; n = n.parent) {
if (
ts.isFunctionLike(n) ||
@@ -265,18 +285,26 @@ class ScopeAnalysis {
ts.isClassStaticBlockDeclaration(n)
)
deferred = n;
if (n.parent === scope || (!ts.isSourceFile(scope) && n.parent === scope.body)) top = n;
// The scope's own statement, not the function body that holds it.
if (n.parent === container) top = n;
}
if (top === null) return false;
if (deferred === null && read && !this.js) return true;
if (deferred !== null && ts.isFunctionDeclaration(deferred) && deferred.name !== undefined)
return this.functionSettles(scope, settle, deferred);
const index = statements.indexOf(top as ts.Statement);
return index > settle || (index === settle && settlesWithin(top, ref));
return (
index > settle.index ||
(index === settle.index &&
(settlesWithin(top, ref) ||
// A later declarator of the storing list (`var a = {}, b = { ...a }`).
(settle.list !== null && isInside(ref, settle.list) && ref.pos >= settle.end))) ||
(index === settle.loop && settlesWithin(top, ref))
);
}
private functionSettles(scope: Scope, settle: number, fn: ts.FunctionDeclaration): boolean {
const key = `${fn.pos}:${settle}`;
private functionSettles(scope: Scope, settle: Settle, fn: ts.FunctionDeclaration): boolean {
const key = `${fn.pos}:${settle.index}:${settle.end}:${settle.loop}`;
const known = this.functionMemo.get(key);
if (known !== undefined) return known;
this.functionMemo.set(key, true);
@@ -276,13 +276,14 @@ export default require_f();
).toBeNull();
});
test("an identifier-minified bundle names its lost helper names", () => {
// esbuild --minify output: the helper shapes survive, their names do not.
const result = split(
test("an identifier-minified bundle splits through its helper's shape", () => {
// esbuild --minify output: the helper shape survives, its name does not.
const answer = expectSameBehavior(
`var c=(r,e)=>()=>(e||r((e={exports:{}}).exports,e),e.exports);` +
`var x=c((t,m)=>{m.exports={v:1}});console.log(x().v);`,
`var x=c((t,m)=>{m.exports={v:1}});var y=c(t=>{t.w=x().v+1});` +
`console.log(x().v,y().w,x()===x());`,
);
expect(result).toEqual({ degrade: expect.stringContaining("identifier-minified") as string });
expect(answer.chunks.map((c) => basename(c.path))).toEqual(["index.js.x.cjs", "index.js.y.cjs"]);
});
test("an ES-module bundle of CommonJS chunks names the chunk interop", () => {
@@ -525,7 +525,7 @@ export function splitBundledChunks(
}
}
}
const helperNames = [...top.keys()].filter((n) => helperBase(n) === "__commonJS");
let helperNames = [...top.keys()].filter((n) => helperBase(n) === "__commonJS");
// Every name a recognizer relies on is written nowhere and declared
// once at the top level; `Object` itself is never rebound.
@@ -552,18 +552,14 @@ export function splitBundledChunks(
);
};
if (helperNames.length === 0) {
// The helper's shape under another name: an identifier-minified bundle,
// whose other interop helpers lost their names as well.
for (const [name, t] of top) {
// An identifier-minified bundle: the helper's shape under another name.
helperNames = [...top].flatMap(([name, t]) => {
const init = ts.isVariableDeclaration(t.node) ? t.node.initializer : undefined;
if (init !== undefined && stable(name) && helperForm(init, sf, ownNames) !== null)
return {
degrade:
"its shipped JS is an identifier-minified bundle whose bundler-interop helpers " +
"(__commonJS and the rest) have lost the names the recognizers verify",
};
}
return null;
return init !== undefined && stable(name) && helperForm(init, sf, ownNames) !== null
? [name]
: [];
});
if (helperNames.length === 0) return null;
}
const forms = new Map<string, HelperForm>();
for (const name of helperNames) {
@@ -260,11 +260,21 @@ function definePropertyExportOf(
value = getterReturnOf(p);
}
}
const esModuleStamp =
name === "__esModule" && value !== null && value.kind === ts.SyntaxKind.TrueKeyword;
const esModuleStamp = name === "__esModule" && value !== null && isTrueLiteral(value);
return { name, value, esModuleStamp };
}
/** `true`, or `!0`, its minified spelling. */
function isTrueLiteral(expr: ts.Expression): boolean {
return (
expr.kind === ts.SyntaxKind.TrueKeyword ||
(ts.isPrefixUnaryExpression(expr) &&
expr.operator === ts.SyntaxKind.ExclamationToken &&
ts.isNumericLiteral(expr.operand) &&
expr.operand.text === "0")
);
}
/** Space-pads [start, end) of `text` preserving newlines — the recognized
* statement disappears while every original offset survives. */
function padSpan(text: string, start: number, end: number): string {
+4 -21
View File
@@ -614,27 +614,12 @@ export function clearProgramBuiltinAliasCache(): void {
* but the bundler-emitted CJS export rewrite (npm-static-rewrite.ts)
* types the getter-table shapes and the offender attribution degrades a
* package whose surface still breaks its consumers — so those bundles are
* worth ATTEMPTING: eligible when the .d.ts and unminified criteria hold,
* gracefully per-package-degraded when the attempt fails. */
* worth ATTEMPTING: eligible when the .d.ts criteria hold, gracefully
* per-package-degraded when the attempt fails. Minified JS is attempted
* too: a minified dist is ordinary JavaScript, and the bundle recognizers
* know esbuild's __commonJS helper by its shape under any name. */
const TRANSFORM_MARKERS = ["__webpack_require__"];
/** Unminified-JS heuristic over an entry source: at least two lines and
* an average line length under 200 characters (minified dists are one
* enormous line; readable code averages well under 100). */
function looksUnminified(source: string): boolean {
const lines = source.split("\n");
if (lines.length < 2) return false;
return source.length / lines.length <= 200;
}
/** A whitespace/syntax-minified bundler dist whose interop helpers keep
* their names (`var __toESM=…,__toCommonJS=mod=>…`): the recognizers
* verify those shapes and refuse the package precisely when they cannot,
* so the minification heuristic need not refuse it up front. */
function namedBundlerHelpers(source: string): boolean {
return /\b__(?:toCommonJS|commonJS|toESM)\d*\s*=\s*(?:\(|[A-Za-z_$][\w$]*\s*=>)/.test(source);
}
function hasTransformMarkers(source: string): boolean {
return TRANSFORM_MARKERS.some((m) => source.includes(m));
}
@@ -676,8 +661,6 @@ export function npmStaticIneligibleReason(
if (jsEntry === null) return "no runtime JS entry resolves";
const source = trackedReadFile(jsEntry);
if (source === null) return `its runtime entry ${jsEntry} cannot be read`;
if (!looksUnminified(source) && !namedBundlerHelpers(source))
return "its shipped JS looks minified";
if (hasTransformMarkers(source)) {
return "its shipped JS carries build-transform markers (bundled/transpiled dist)";
}
+14 -2
View File
@@ -1436,8 +1436,20 @@ function requiresOf7(
return out;
}
/** `true`, or `!0`, its minified spelling. */
function isTrueLiteral(expr: ts.Expression): boolean {
return (
expr.kind === ts.SyntaxKind.TrueKeyword ||
(ts.isPrefixUnaryExpression(expr) &&
expr.operator === ts.SyntaxKind.ExclamationToken &&
ts.isNumericLiteral(expr.operand) &&
expr.operand.text === "0")
);
}
/** The exact `Object.defineProperty(exports|module.exports, "__esModule",
* { value: true })` interop stamp (shared by preflight and the no-op lowering). */
* { value: true })` interop stamp (shared by preflight and the no-op lowering);
* `{ value: !0 }` is the same stamp. */
export function isEsModuleStamp(expr: ts.Expression): boolean {
if (!ts.isCallExpression(expr) || expr.questionDotToken !== undefined) return false;
const callee = expr.expression;
@@ -1471,7 +1483,7 @@ export function isEsModuleStamp(expr: ts.Expression): boolean {
ts.isPropertyAssignment(p) &&
ts.isIdentifier(p.name) &&
p.name.text === "value" &&
p.initializer.kind === ts.SyntaxKind.TrueKeyword
isTrueLiteral(p.initializer)
);
}
+41 -16
View File
@@ -111,9 +111,11 @@ import {
iceDiag,
nativeCodegenDiag,
nodeIslandNodeDiag,
nodeIslandModuleDiag,
type ScrDiagnostic,
} from "./diagnostics/diagnostic.js";
import { libnodeCacheKey, libnodeRecord } from "./node-islands/libnode.js";
import { placeNodeIslandModules, readNodeIslandModules } from "./node-islands/modules.js";
import { loadLibraryProfile, type LibraryProfile } from "./library/library-profile.js";
import { clearFenceEvalCaches, decorateLibraryRefusals } from "./library/fence-eval.js";
import {
@@ -785,7 +787,32 @@ async function prepareExecutableInput(
const recorded = libnodeRecord(opts.libnode, opts.outPath, process.env, dirname(entryPath));
if (typeof recorded === "string")
return { ok: false, diagnostics: [nodeIslandNodeDiag(recorded, entryPath)], sourceTexts };
mod.nodeIsland = recorded;
mod.nodeIsland = { ...mod.nodeIsland, ...recorded };
}
}
// The island module tree ships with the program: beside the executable
// with --libnode=beside, else inside it (node-islands/modules.ts).
let nodeIslandModuleData: Uint8Array | undefined;
const islandModules = mod.nodeIsland?.modules;
if (islandModules !== undefined && outputKind !== "ir") {
if (outputKind === "exe" && opts.libnode === "beside") {
const problem = placeNodeIslandModules(islandModules, `${opts.outPath}.libnode/modules`);
if (problem !== null)
return {
ok: false,
diagnostics: [nodeIslandModuleDiag(problem, { file: entryPath, start: 0, end: 0 })],
sourceTexts,
};
islandModules.location = `$ORIGIN/${basename(opts.outPath)}.libnode/modules`;
} else {
const data = readNodeIslandModules(islandModules);
if (typeof data === "string")
return {
ok: false,
diagnostics: [nodeIslandModuleDiag(data, { file: entryPath, start: 0, end: 0 })],
sourceTexts,
};
nodeIslandModuleData = data;
}
}
if (warningSink !== undefined && prepared.warnings.length > 0) {
@@ -820,6 +847,7 @@ async function prepareExecutableInput(
lowRegion: lowRegionTarget(buildPlatform, opts.sanitize === true),
...(opts.sanitize === true ? { wasmBoundsChecks: true } : {}),
...inlineRcOption(opts.optimization, opts.sanitize === true),
...(nodeIslandModuleData === undefined ? {} : { nodeIslandModuleData }),
});
} catch (err) {
if (!(err instanceof LlvmUnsupportedError)) throw err;
@@ -890,6 +918,7 @@ async function prepareExecutableInput(
...(opts.sanitize === true ? { wasmBoundsChecks: true } : {}),
...inlineRcOption(opts.optimization, opts.sanitize === true),
...(opts.detachedExit === true ? { detachedExit: true } : {}),
...(nodeIslandModuleData === undefined ? {} : { nodeIslandModuleData }),
});
} catch (err) {
if (!(err instanceof LlvmUnsupportedError)) throw err;
@@ -1469,8 +1498,6 @@ async function compileTracked(
if (outputKind === "exe") {
const helperObjectRoute =
opts.nativeProgramObject === true || usesPrecompiledRuntimePack(opts, "llvm");
// Calls into Node record the libnode pack, keyed by its manifest (node-islands/libnode.ts).
const libnodeKey = libnodeCacheKey(opts.libnode);
const routeOptions: Omit<
EarlyExecutableCacheOptions,
"implementation" | "implementationDependencies" | "nativeEnvironment"
@@ -1485,7 +1512,9 @@ async function compileTracked(
...(opts.strip ? { strip: true as const } : {}),
...(opts.windowsSubsystem === "gui" ? { windowsSubsystem: "gui" as const } : {}),
...(opts.detachedExit === true ? { detachedExit: true as const } : {}),
...(libnodeKey === undefined ? {} : { libnode: libnodeKey }),
// Calls into Node record the libnode pack; their entries record its
// identity (node-islands/libnode.ts) and a hit requires it unchanged.
libnodeIdentity: () => libnodeCacheKey(opts.libnode, process.env, dirname(entryPath)),
...(profile === undefined ? {} : { profile: nativeProfileKey(profile) }),
...checkerField(),
...(youngNurseryRequested() ? { youngNursery: true as const } : {}),
@@ -1531,26 +1560,22 @@ async function compileTracked(
implementationDependencies: implementation.dependencies,
};
}
const earlyHit =
const cachedHit =
earlyCacheOptions === null
? null
: await readEarlyExecutableCache(cacheRoot, earlyCacheOptions);
// A --libnode=beside build that calls into Node places its island modules
// next to the executable from the lowered module, which a cache hit does
// not have: such builds take the full path.
const earlyHit =
cachedHit !== null && cachedHit.native.nodeIsland === true && opts.libnode === "beside"
? null
: cachedHit;
if (earlyHit !== null) {
timing("executable-cache-hit");
if (earlyCacheOptions === null) {
throw new InternalCompilerError("executable cache hit without executable cache options");
}
// The pack a --libnode=beside build places next to the executable is an output of the
// build, like the executable: a restored executable that calls into Node gets it again.
if (earlyHit.native.nodeIsland === true && opts.libnode === "beside") {
const recorded = libnodeRecord(opts.libnode, opts.outPath);
if (typeof recorded === "string")
return {
ok: false,
diagnostics: [nodeIslandNodeDiag(recorded, entryPath)],
sourceTexts: new Map(),
};
}
// A cached build repeats its performance warning (SC6101): the code it
// names is just as untyped as when it was built.
if (warningSink !== undefined && earlyHit.performance !== undefined) {
@@ -366,6 +366,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet<IrLibFn> = new Set([
"transform.newDyn",
"passthrough.newDyn",
"readable.initDyn",
"stream.checkEncodingOption",
"writable.initDyn",
"duplex.initDyn",
"transform.initDyn",
@@ -4004,6 +4004,7 @@ export const LIB_FN_SIGS = defineLibFnSignatures({
"stream.setTransform": { argTypes: [null, null], result: VOID },
"stream.setFlush": { argTypes: [null, null], result: VOID },
"stream.setConstruct": { argTypes: [null, null], result: VOID },
"stream.replaceConstruct": { argTypes: [null, null], result: VOID },
"stream.setWritev": { argTypes: [null, null], result: VOID },
"stream.setWritevDefault": { argTypes: [null, null], result: VOID },
/** The dyn-options twins (a checked-dynamic options record — the JS
@@ -4140,6 +4141,11 @@ export const LIB_FN_SIGS = defineLibFnSignatures({
* after construction; answers the receiver (+1, the chaining shape). */
"stream.objectMode": { argTypes: [null, BOOL, BOOL], result: VOID },
"stream.decodeStrings": { argTypes: [null, BOOL], result: VOID },
/** Runtime decodeStrings and defaultEncoding option values, applied
* after construction (answers the receiver +1); the encoding was
* validated by stream.checkEncodingOption before construction. */
"stream.optionValues": { argTypes: [null, DYN, DYN], result: VOID },
"stream.checkEncodingOption": { argTypes: [DYN], result: VOID },
"stream.duplexFromGen": { argTypes: [null, null, null, null, null, null], result: VOID },
"stream.duplexFromDyn": { argTypes: [DYN], result: VOID },
/** Duplex.from over a lone Readable or Writable, or a { readable,
@@ -221,6 +221,12 @@ const MODULE_FIELDS: Record<StringFields<Omit<IrModule, "functions">>, Role> = {
"threadSlotKeys[]": "other",
"nodeIsland.libnode": "other",
"nodeIsland.nodeVersion": "other",
"nodeIsland.modules.root": "other",
"nodeIsland.modules.location": "other",
"nodeIsland.modules.files[].path": "other",
"nodeIsland.modules.files[].sha256": "other",
"nodeIsland.modules.links[].path": "other",
"nodeIsland.modules.links[].target": "other",
};
const loc = { file: "references.ts", start: 0, end: 0 };
+23 -3
View File
@@ -1009,6 +1009,18 @@ export function isRefCounted(t: IrType): boolean {
/* ── module ────────────────────────────────────────────────────────────── */
/** The island module tree of a program (node-islands/modules.ts): paths
* relative to `root`, the directory on the build machine that holds them.
* Only the relative paths enter an executable. */
export interface IrNodeIslandModules {
root: string;
files: { path: string; size: number; sha256: string }[];
links: { path: string; target: string }[];
/** Where the executable finds the tree: absent when it carries the tree
* itself, else a `$ORIGIN/` path (`--libnode=beside`). */
location?: string;
}
export interface IrModule {
/** Bumped on any breaking IR change; serialize.ts refuses mismatches. */
irVersion: 17;
@@ -1067,7 +1079,13 @@ export interface IrModule {
* executable) and the exact Node version its bridge links. Set by the
* executable build (node-islands/libnode.ts); the emitted main records
* both before %main. */
nodeIsland?: { libnode: string; nodeVersion: string };
nodeIsland?: {
libnode: string;
nodeVersion: string;
/** The modules the calls load (node-islands/modules.ts), when any call
* loads a file. */
modules?: IrNodeIslandModules;
};
/** Outbound native FFI declarations used by `ffiCall` expressions.
* These are link-time C ABI imports, not runtime dynamic-library handles:
* executable builds resolve their symbols from the manifest's archive
@@ -3904,8 +3922,10 @@ export type IrExpr =
* not shape equality; missing or wrong-typed fields throw, EXCEPT that a
* missing key for an undefined-armed union field — an optional field —
* builds the interned undefined arm instead); arrays check every element;
* unions try arms in canonical order and the first FULL match wins (no
* match → throw; an undefined arm matches no dyn value); JSON null matches
* unions try arms in canonical order, except that record arms with more
* required fields and then more fields go first, and the first FULL match
* wins (no match → throw; an undefined arm matches no dyn value); JSON
* null matches
* exactly the nullT arm of a union target (bare null targets cannot
* exist). MAY THROW:
* backends' may-throw analyses must treat it like a `throw` statement.
@@ -220,6 +220,8 @@ const STREAM_LIB_FNS: ReadonlySet<IrLibFn> = new Set([
"stream.chunkDone",
"stream.objectMode",
"stream.decodeStrings",
"stream.optionValues",
"stream.checkEncodingOption",
"stream.duplexFromGen",
"stream.duplexFromDyn",
"stream.duplexFromPromise",
+3
View File
@@ -5500,6 +5500,7 @@ function validateFunction(
e.fn === "stream.destroyErr" ||
e.fn === "stream.objectMode" ||
e.fn === "stream.decodeStrings" ||
e.fn === "stream.optionValues" ||
e.fn === "stream.duplexFromGen" ||
e.fn === "stream.duplexFromDyn" ||
e.fn === "stream.duplexifyReadable" ||
@@ -5510,6 +5511,7 @@ function validateFunction(
e.fn === "stream.duplexFromGenSignal" ||
e.fn === "stream.duplexFromAsyncFnSignal" ||
e.fn === "stream.setConstruct" ||
e.fn === "stream.replaceConstruct" ||
e.fn === "stream.setWritev" ||
e.fn === "stream.setWritevDefault" ||
e.fn === "stream.iteratorClose" ||
@@ -5872,6 +5874,7 @@ function validateFunction(
e.fn === "readable.pushEncoding" ||
e.fn === "stream.objectMode" ||
e.fn === "stream.decodeStrings" ||
e.fn === "stream.optionValues" ||
e.fn === "writable.writeEncoding" ||
e.fn === "writable.setDefaultEncoding" ||
e.fn === "stream.destroy" ||
@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { afterEach, describe, expect, test } from "vitest";
import { compilerReleaseVersion } from "../library/sidecar.js";
import { NODE_BRIDGE_ABI, libnodeRecord } from "./libnode.js";
import {
LIBNODE_PACK_SCHEMA,
@@ -48,7 +49,10 @@ function fakePack(
writeFileSync(join(dir, "libnode.so.137"), libnode);
writeFileSync(join(dir, "libscr_node_bridge.so"), bridge);
writeFileSync(join(dir, "LICENSE"), "Node.js is licensed for use as follows:\n\nMIT\n");
writeFileSync(join(dir, "package.json"), JSON.stringify({ name, version: "0.4.1" }));
writeFileSync(
join(dir, "package.json"),
JSON.stringify({ name, version: compilerReleaseVersion() }),
);
const manifest: Record<string, unknown> = {
schema: LIBNODE_PACK_SCHEMA,
nodeVersion: "24.22.0",
@@ -59,7 +63,7 @@ function fakePack(
libnode: "libnode.so.137",
bridge: "libscr_node_bridge.so",
package: name,
version: "0.4.1",
version: compilerReleaseVersion(),
target: "linux-x64-gnu",
license: "LICENSE",
files: { "libnode.so.137": entry(libnode), "libscr_node_bridge.so": entry(bridge) },
@@ -180,6 +184,22 @@ describe("finding the installed pack", () => {
});
});
test("an installed pack of another scriptc version is refused", () => {
const root = temp();
const name = libnodePackPackage(hostLibnodeTarget());
if (name === null) return;
const pack = fakePack(
join(root, "node_modules", name),
(m) => {
m["version"] = "0.0.1";
},
name,
);
expect(libnodeRecord(undefined, join(root, "app"), {}, root)).toBe(
`the installed libnode pack ${name} at ${pack} is version 0.0.1; this scriptc is ${compilerReleaseVersion()}: install ${name}@${compilerReleaseVersion()}`,
);
});
describe("readLibnodePack", () => {
test("reads a pack in build-pack.mjs's format and checks its package", () => {
const pack = fakePack(join(temp(), "pack"));
@@ -233,7 +253,7 @@ test("the license notices carry Node's LICENSE", () => {
expect(path).toBe(`${out}.licenses.txt`);
expect(readFileSync(path, "utf8")).toBe(
[
"server runs part of its code in Node.js 24.22.0, loaded from libnode.so.137 (@scriptc/libnode-linux-x64-gnu@0.4.1).",
`server runs part of its code in Node.js 24.22.0, loaded from libnode.so.137 (@scriptc/libnode-linux-x64-gnu@${compilerReleaseVersion()}).`,
"Node.js and the components it bundles are licensed as follows. The scriptc bridge,",
"libscr_node_bridge.so, is licensed under Apache-2.0.",
"",
@@ -3,7 +3,8 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, test } from "vitest";
import { USES_TIMERS_LIB_FNS } from "../backend/llvm/lib-shared.js";
import { libnodeCacheKey, libnodeRecord, NODE_BRIDGE_ABI } from "./libnode.js";
import { libnodeCacheKey, libnodeRecord, libnodeTarget, NODE_BRIDGE_ABI } from "./libnode.js";
import { libnodePackPackage } from "./libnode-pack.js";
import { findInstalledLibnodePack, hostLibnodeTarget } from "./libnode-pack.js";
function pack(nodeVersion: string, bridgeAbi = NODE_BRIDGE_ABI): string {
@@ -84,6 +85,31 @@ describe("the recorded libnode pack", () => {
expect(readFileSync(join(out, "app.libnode/libscr_node_bridge.so"), "utf8")).toBe("bridge");
});
test("beside refuses to replace the pack it copies", () => {
const out = mkdtempSync(join(tmpdir(), "scriptc-libnode-out-"));
const first = pack("24.22.0");
libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: first });
// The placed copy as the source of the next beside build.
const placed = join(out, "app.libnode");
expect(libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: placed })).toMatch(
/^--libnode=beside would replace the libnode pack it copies/,
);
expect(readFileSync(join(placed, "libscr_node_bridge.so"), "utf8")).toBe("bridge");
});
test("beside refuses manifest file names outside the pack", () => {
const dir = pack("24.22.0");
writeFileSync(
join(dir, "manifest.json"),
JSON.stringify({ nodeVersion: "24.22.0", bridgeAbi: NODE_BRIDGE_ABI, files: { "../x": {} } }),
);
const out = mkdtempSync(join(tmpdir(), "scriptc-libnode-out-"));
expect(libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: dir })).toBe(
`the libnode pack at ${dir} lists a file outside the pack ('../x')`,
);
expect(existsSync(join(out, "app.libnode"))).toBe(false);
});
test("a pack the deployment places beside the executable needs no manifest at build time", () => {
expect(libnodeRecord("$ORIGIN/node", "/out/app", {})).toEqual({
libnode: "$ORIGIN/node",
@@ -92,9 +118,24 @@ describe("the recorded libnode pack", () => {
});
});
describe("the libnode pack in an executable's cache key", () => {
test("is absent without --libnode or SCRIPTC_LIBNODE", () => {
expect(libnodeCacheKey(undefined, {})).toBeUndefined();
describe("the libnode pack identity a cached build keeps", () => {
test("is absent when no pack is named or installed", () => {
expect(libnodeCacheKey(undefined, {}, "/nonexistent")).toBeUndefined();
});
test("names the installed pack, and changes with it", () => {
const project = mkdtempSync(join(tmpdir(), "scriptc-libnode-project-"));
const name = libnodePackPackage(libnodeTarget({}));
if (name === null) return;
const installed = join(project, "node_modules", name);
mkdirSync(installed, { recursive: true });
writeFileSync(join(installed, "manifest.json"), JSON.stringify({ nodeVersion: "24.22.0" }));
const before = libnodeCacheKey(undefined, {}, project);
expect(before).toContain(installed);
writeFileSync(join(installed, "manifest.json"), JSON.stringify({ nodeVersion: "24.23.0" }));
expect(libnodeCacheKey(undefined, {}, project)).not.toBe(before);
rmSync(installed, { recursive: true, force: true });
expect(libnodeCacheKey(undefined, {}, project)).toBeUndefined();
});
test("changes when the pack in the same directory is upgraded", () => {
+103 -28
View File
@@ -13,9 +13,26 @@
* executable and its Node travel together; a recorded `$ORIGIN/` path is
* also the hook for bundling Node into the executable itself later. */
import { createHash } from "node:crypto";
import { copyFileSync, linkSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
import { basename, isAbsolute, join, resolve } from "node:path";
import { findInstalledLibnodePack, hostLibnodeTarget, libnodePackPackage } from "./libnode-pack.js";
import {
copyFileSync,
linkSync,
mkdirSync,
readFileSync,
realpathSync,
rmSync,
statSync,
} from "node:fs";
import { basename, isAbsolute, join, resolve, sep } from "node:path";
import { compilerReleaseVersion } from "../library/sidecar.js";
import { nativeCodegenTarget } from "../backend/targets.js";
import {
findInstalledLibnodePack,
hostLibnodeTarget,
LibnodePackError,
libnodePackPackage,
readLibnodePack,
type LibnodePack,
} from "./libnode-pack.js";
/** The oldest Node.js release Node islands support: synchronous
* module.registerHooks, which shared modules need, is a release candidate
@@ -23,7 +40,7 @@ import { findInstalledLibnodePack, hostLibnodeTarget, libnodePackPackage } from
export const NODE_ISLAND_MIN_NODE = "24.13.1";
/** The bridge ABI this compiler's runtime speaks (scr_node_bridge.h). */
export const NODE_BRIDGE_ABI = 2;
export const NODE_BRIDGE_ABI = 3;
/** The prefix of a recorded path relative to the executable's directory. */
export const ORIGIN = "$ORIGIN/";
@@ -68,23 +85,66 @@ function readManifest(dir: string): PackManifest | string {
};
}
/** Copies the pack next to the executable (hard links where possible). */
function placeBeside(source: string, manifest: PackManifest, target: string): void {
/** Copies the pack next to the executable (hard links where possible), or
* returns why it cannot: the target would replace the pack itself, or the
* manifest names a file outside the pack. */
function placeBeside(source: string, manifest: PackManifest, target: string): string | undefined {
const names = ["manifest.json", "LICENSE", ...Object.keys(manifest.files)];
const unsafe = names.find(
(name) => name === "" || name === "." || name === ".." || /[/\\]/.test(name),
);
if (unsafe !== undefined)
return `the libnode pack at ${source} lists a file outside the pack ('${unsafe}')`;
const from = realpathOr(source);
const to = realpathOr(target);
if (from === to || from.startsWith(to + sep) || to.startsWith(from + sep))
return `--libnode=beside would replace the libnode pack it copies (${source}); point SCRIPTC_LIBNODE at a pack outside ${target}`;
rmSync(target, { recursive: true, force: true });
mkdirSync(target, { recursive: true });
for (const name of ["manifest.json", "LICENSE", ...Object.keys(manifest.files)]) {
const from = join(source, name);
for (const name of names) {
const file = join(source, name);
try {
statSync(from);
statSync(file);
} catch {
continue;
}
try {
linkSync(from, join(target, name));
linkSync(file, join(target, name));
} catch {
copyFileSync(from, join(target, name));
copyFileSync(file, join(target, name));
}
}
return undefined;
}
/** The canonical path of `path`, or `path` resolved when it does not exist. */
function realpathOr(path: string): string {
try {
return realpathSync(path);
} catch {
return resolve(path);
}
}
/** The native target whose libnode pack a build records: the codegen
* target (SCRIPTC_TARGET, and the host's libc), else the host. */
export function libnodeTarget(env: NodeJS.ProcessEnv = process.env): string {
return nativeCodegenTarget(env)?.name ?? hostLibnodeTarget();
}
/** Why the installed pack in `dir` cannot serve this compiler, or null: it
* must be the target's package at the compiler's own version. */
function installedPackProblem(dir: string, packageName: string): string | null {
let pack: LibnodePack;
try {
pack = readLibnodePack(dir, { packageName });
} catch (error) {
return error instanceof LibnodePackError ? error.message : String(error);
}
const version = compilerReleaseVersion();
if (pack.manifest.version !== version)
return `the installed libnode pack ${packageName} at ${dir} is version ${pack.manifest.version ?? "unknown"}; this scriptc is ${version}: install ${packageName}@${version}`;
return null;
}
/** What to record for a program that calls into Node, or the reason the
@@ -98,13 +158,23 @@ export function libnodeRecord(
const fromEnv = env["SCRIPTC_LIBNODE"];
const beside = option === "beside";
const explicit = option !== undefined && !beside ? option : fromEnv;
// Without an explicit pack, the installed @scriptc/libnode-<platform>.
const source =
const target = libnodeTarget(env);
// Without an explicit pack, the installed @scriptc/libnode-<target>.
const installed =
explicit !== undefined && explicit !== ""
? explicit
: findInstalledLibnodePack({ targetName: hostLibnodeTarget(), projectDir });
if (source === undefined || source === null || source === "")
return `this program calls into Node.js (@scriptc/node), but no libnode pack is installed: install ${libnodePackPackage(hostLibnodeTarget()) ?? "a libnode pack"}, set SCRIPTC_LIBNODE or pass --libnode=<pack directory>`;
? null
: findInstalledLibnodePack({ targetName: target, projectDir });
const source = installed ?? explicit;
if (source === undefined || source === null || source === "") {
const pack = libnodePackPackage(target);
return pack === null
? `this program calls into Node.js (@scriptc/node), but there is no libnode pack for ${target}: set SCRIPTC_LIBNODE or pass --libnode=<pack directory>`
: `this program calls into Node.js (@scriptc/node), but no libnode pack is installed: install ${pack}, set SCRIPTC_LIBNODE or pass --libnode=<pack directory>`;
}
if (installed !== null) {
const problem = installedPackProblem(installed, libnodePackPackage(target)!);
if (problem !== null) return problem;
}
if (source.startsWith(ORIGIN)) {
// A pack the deployment places next to the executable: nothing to read.
return { libnode: source, nodeVersion: "" };
@@ -114,25 +184,30 @@ export function libnodeRecord(
if (typeof manifest === "string") return manifest;
if (!beside) return { libnode: resolve(source), nodeVersion: manifest.nodeVersion };
const name = `${basename(outPath)}.libnode`;
placeBeside(source, manifest, `${outPath}.libnode`);
const refused = placeBeside(source, manifest, `${outPath}.libnode`);
if (refused !== undefined) return refused;
return { libnode: `${ORIGIN}${name}`, nodeVersion: manifest.nodeVersion };
}
/** The part of an executable's cache key that names its libnode pack, or
* undefined when neither --libnode nor SCRIPTC_LIBNODE is set. It holds the
* option, the pack directory and a digest of the pack's manifest.json, so
* that a pack upgraded in place (another Node version, bridge ABI or file
* list) gives cached executables a new key instead of their old record.
* An installed @scriptc/libnode-<platform> pack needs no part: it versions
* in lockstep with the compiler, whose version every key already holds, so
* the bootstrap's routed cache keeps serving builds where one is installed. */
/** The identity of the libnode pack a build that calls into Node.js
* records, which the program's executable cache entry keeps
* (EarlyExecutableCacheOptions.libnodeIdentity): the option, the pack
* directory (named by --libnode or SCRIPTC_LIBNODE, or the installed pack
* found from `projectDir`), and a digest of the pack's manifest.json. A pack
* upgraded in place, removed, or replaced by another installed pack changes
* it. Undefined when no pack is named or installed. */
export function libnodeCacheKey(
option: string | undefined,
env: NodeJS.ProcessEnv = process.env,
projectDir: string = process.cwd(),
): string | undefined {
const fromEnv = env["SCRIPTC_LIBNODE"];
if (option === undefined && fromEnv === undefined) return undefined;
const source = option !== undefined && option !== "beside" ? option : (fromEnv ?? "");
const explicit = option !== undefined && option !== "beside" ? option : fromEnv;
const source =
explicit !== undefined && explicit !== ""
? explicit
: findInstalledLibnodePack({ targetName: libnodeTarget(env), projectDir });
if (source === undefined || source === null) return option === undefined ? undefined : option;
let manifest = "none";
if (isAbsolute(source)) {
try {
@@ -0,0 +1,188 @@
import { mkdirSync, mkdtempSync, realpathSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { describe, expect, test } from "vitest";
import {
NodeIslandModuleCollector,
nodeIslandManifest,
readNodeIslandModules,
scanModuleSpecifiers,
} from "./modules.js";
function project(files: Record<string, string>): string {
const root = realpathSync(mkdtempSync(join(tmpdir(), "scriptc-island-modules-")));
for (const [path, text] of Object.entries(files)) {
mkdirSync(dirname(join(root, path)), { recursive: true });
writeFileSync(join(root, path), text);
}
return root;
}
describe("module specifiers in JavaScript", () => {
test("string-literal loads in every form", () => {
const scan = scanModuleSpecifiers(`
const a = require("./a");
const b = require.resolve('./b.json');
import c from "c";
import * as d from "d/sub";
import { e } from "./e.mjs";
import "./side";
export * from "./f";
export { g } from "./g";
const h = await import("./h.js");
import.meta.url;
obj.require("x");
const t = \`\${require("./t")}\`;
`);
expect(scan.specifiers).toEqual([
"./a",
"./b.json",
"c",
"d/sub",
"./e.mjs",
"./side",
"./f",
"./g",
"./h.js",
"./t",
]);
expect(scan.computed).toEqual([]);
});
test("comments, strings and regular expressions hide nothing and invent nothing", () => {
const scan = scanModuleSpecifiers(`
// require("./commented")
/* import x from "./block" */
const s = "require('./in-string')";
const r = /require\\("\\.\\/in-regex"\\)/g;
const q = a / b / c;
const real = require("./real");
`);
expect(scan.specifiers).toEqual(["./real"]);
expect(scan.computed).toEqual([]);
});
test("computed loads are reported", () => {
const forms = (text: string) => scanModuleSpecifiers(text).computed.map((c) => c.form);
expect(forms("require(name)")).toEqual(["require() with a computed specifier"]);
expect(forms("require('./a' + b)")).toEqual(["require() with a computed specifier"]);
expect(forms("import(`./x/${y}`)")).toEqual(["import() with a computed specifier"]);
expect(forms("const r = require; r('./a')")).toEqual(["require as a value"]);
expect(forms("createRequire(import.meta.url)('./a')")).toEqual([
"a require() from createRequire()",
]);
expect(forms("function require(x) {} typeof require")).toEqual([]);
});
});
describe("the collected module tree", () => {
test("a relative module and what it loads; missing modules are not shipped", () => {
const root = project({
"src/main.ts": "",
"src/math.cjs":
"module.exports = require('./util');\ntry { require('./optional'); } catch {}",
"src/util.js": "exports.x = require('../data.json');",
"data.json": "{}",
"src/unused.js": "",
});
const collector = new NodeIslandModuleCollector();
expect(collector.add(join(root, "src/main.ts"), "./math.cjs")).toBeNull();
expect(collector.add(join(root, "src/main.ts"), "./does-not-exist.cjs")).toBeNull();
expect(collector.add(join(root, "src/main.ts"), "node:fs")).toBeNull();
const tree = collector.tree()!;
expect(tree.root).toBe(root);
expect(tree.files.map((f) => f.path)).toEqual(["data.json", "src/math.cjs", "src/util.js"]);
expect(NodeIslandModuleCollector.callerPath(tree, join(root, "src/main.ts"))).toBe(
"src/main.ts",
);
});
test("a module of the program ships with the package.json that scopes it", () => {
const root = project({
"package.json": JSON.stringify({ type: "module" }),
"src/main.ts": "",
"src/esm.js": "export const x = 1;",
});
const collector = new NodeIslandModuleCollector();
expect(collector.add(join(root, "src/main.ts"), "./esm.js")).toBeNull();
expect(collector.tree()!.files.map((f) => f.path)).toEqual(["package.json", "src/esm.js"]);
});
test("a computed load in a module of the program is refused", () => {
const root = project({ "main.ts": "", "dyn.cjs": "module.exports = require(process.env.X);" });
const problem = new NodeIslandModuleCollector().add(join(root, "main.ts"), "./dyn.cjs");
expect(problem).toBe(
`${join(root, "dyn.cjs")}:1 loads a module through require() with a computed specifier; a module that runs in a Node island loads modules by string-literal specifiers only`,
);
});
test("a package ships whole, with its dependencies, but not its nested node_modules", () => {
const root = project({
"main.ts": "",
"node_modules/pkg/package.json": JSON.stringify({ dependencies: { dep: "1", absent: "1" } }),
"node_modules/pkg/index.js": "module.exports = require(process.env.ANYTHING_GOES);",
"node_modules/pkg/lib/extra.json": "{}",
"node_modules/pkg/node_modules/.bin/x": "",
"node_modules/dep/package.json": "{}",
"node_modules/dep/index.js": "",
"node_modules/other/index.js": "",
});
const collector = new NodeIslandModuleCollector();
expect(collector.add(join(root, "main.ts"), "pkg/lib/extra.json")).toBeNull();
expect(collector.tree()!.files.map((f) => f.path)).toEqual([
"node_modules/dep/index.js",
"node_modules/dep/package.json",
"node_modules/pkg/index.js",
"node_modules/pkg/lib/extra.json",
"node_modules/pkg/package.json",
]);
});
test("pnpm's links between package directories stay links inside the tree", () => {
const root = project({
"app/main.ts": "",
"app/node_modules/.pnpm/a@1/node_modules/a/package.json": JSON.stringify({
dependencies: { b: "1" },
}),
"app/node_modules/.pnpm/a@1/node_modules/a/index.js": "",
"app/node_modules/.pnpm/b@1/node_modules/b/index.js": "",
});
symlinkSync(".pnpm/a@1/node_modules/a", join(root, "app/node_modules/a"));
symlinkSync(
"../../b@1/node_modules/b",
join(root, "app/node_modules/.pnpm/a@1/node_modules/b"),
);
const collector = new NodeIslandModuleCollector();
expect(collector.add(join(root, "app/main.ts"), "a")).toBeNull();
const tree = collector.tree()!;
expect(tree.root).toBe(join(root, "app"));
expect(tree.links).toEqual([
{ path: "node_modules/.pnpm/a@1/node_modules/b", target: "../../b@1/node_modules/b" },
{ path: "node_modules/a", target: ".pnpm/a@1/node_modules/a" },
]);
expect(tree.files.map((f) => f.path)).toEqual([
"node_modules/.pnpm/a@1/node_modules/a/index.js",
"node_modules/.pnpm/a@1/node_modules/a/package.json",
"node_modules/.pnpm/b@1/node_modules/b/index.js",
]);
});
test("built-in modules alone ship no tree", () => {
const root = project({ "main.ts": "" });
const collector = new NodeIslandModuleCollector();
expect(collector.add(join(root, "main.ts"), "os")).toBeNull();
expect(collector.tree()).toBeNull();
expect(NodeIslandModuleCollector.callerPath(null, join(root, "main.ts"))).toBe("main.ts");
});
test("the manifest names relative paths only, and the data is checked against it", () => {
const root = project({ "main.ts": "", "m.cjs": "exports.f = () => 1;" });
const collector = new NodeIslandModuleCollector();
collector.add(join(root, "main.ts"), "./m.cjs");
const tree = collector.tree()!;
expect(nodeIslandManifest(tree)).not.toContain(root);
expect(readNodeIslandModules(tree)).toEqual(new TextEncoder().encode("exports.f = () => 1;"));
writeFileSync(join(root, "m.cjs"), "exports.f = () => 2;");
expect(readNodeIslandModules(tree)).toBe(`${join(root, "m.cjs")} changed during the build`);
});
});
@@ -0,0 +1,627 @@
/* The island module tree: the JavaScript a program's Node island calls load,
* collected at build time and shipped with the executable, so a deployed
* binary never reads modules from the machine that built it.
*
* Each `call`/`callSync` names its module with a string literal. The build
* resolves it from the calling module as Node's require() does:
* - a relative or absolute specifier names a module of the program: the
* module and every module its string-literal `require()`, `import` and
* `export ... from` reach are collected; a computed specifier in such a
* module is refused, since the build cannot know what it loads;
* - a package specifier names a package: its whole directory is collected
* (computed requires inside packages keep working), and so are its
* `dependencies`, `optionalDependencies` and `peerDependencies`, found as
* Node finds them;
* - a Node built-in module needs nothing, and neither does a module the
* build cannot find: loading it throws MODULE_NOT_FOUND when the call
* runs, as under Node.
* Paths keep their layout relative to one root directory (the deepest
* directory that holds every collected path), with symbolic links between
* package directories (pnpm's layout) kept as links. Only these relative
* paths enter the executable. Every read goes through the frontend input
* tracker, so an edit to an island module invalidates cached builds. */
import { createHash } from "node:crypto";
import { mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { builtinModules } from "node:module";
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
import {
trackedAccessibleEntries,
trackedDirectoryExists,
trackedFileExists,
trackedReadBytes,
trackedReadFile,
trackedRealpath,
} from "../frontend/input-tracker.js";
import type { IrNodeIslandModules } from "../ir/ir.js";
export type NodeIslandModuleTree = IrNodeIslandModules;
type NodeIslandModuleFile = IrNodeIslandModules["files"][number];
type NodeIslandModuleLink = IrNodeIslandModules["links"][number];
const BUILTINS: ReadonlySet<string> = new Set(builtinModules);
/** Extensions require() tries for an extensionless file, in its order. */
const FILE_EXTENSIONS = [".js", ".json", ".node"];
/** Module files whose imports are traced. */
const TRACED = /\.(?:[cm]?js|[cm]?ts)$/;
function isBuiltin(specifier: string): boolean {
return specifier.startsWith("node:") || BUILTINS.has(specifier);
}
function isPathSpecifier(specifier: string): boolean {
return (
specifier === "." ||
specifier === ".." ||
specifier.startsWith("./") ||
specifier.startsWith("../") ||
isAbsolute(specifier)
);
}
function packageName(specifier: string): string {
const parts = specifier.split("/");
return specifier.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0]!;
}
/** The file require() loads for `base` (a file, an extension away, or a
* directory with a package.json main or an index file), or null. */
function resolveFile(base: string): string | null {
if (trackedFileExists(base)) return base;
for (const extension of FILE_EXTENSIONS)
if (trackedFileExists(base + extension)) return base + extension;
if (!trackedDirectoryExists(base)) return null;
const manifest = trackedReadFile(join(base, "package.json"));
if (manifest !== null) {
let main: unknown;
try {
main = (JSON.parse(manifest) as { main?: unknown }).main;
} catch {
main = undefined;
}
if (typeof main === "string" && main !== "") {
const target = join(base, main);
const found = resolveFile(target) ?? resolveFile(join(target, "index"));
if (found !== null) return found;
}
}
for (const extension of FILE_EXTENSIONS) {
const index = join(base, `index${extension}`);
if (trackedFileExists(index)) return index;
}
return null;
}
/** The directory of package `name` as require() finds it from `fromDir`:
* `node_modules/<name>` in `fromDir` and each ancestor not itself named
* node_modules. */
function packageDirectory(fromDir: string, name: string): string | null {
for (let dir = fromDir; ;) {
if (basename(dir) !== "node_modules") {
const candidate = join(dir, "node_modules", name);
if (trackedDirectoryExists(candidate)) return candidate;
}
const parent = dirname(dir);
if (parent === dir) return null;
dir = parent;
}
}
/** The package.json nearest to `dir`, as Node finds a module's package
* scope, or null. */
function packageScope(dir: string): string | null {
for (let at = dir; ;) {
if (basename(at) === "node_modules") return null;
const candidate = join(at, "package.json");
if (trackedFileExists(candidate)) return candidate;
const parent = dirname(at);
if (parent === at) return null;
at = parent;
}
}
function sha256(bytes: Uint8Array): string {
return createHash("sha256").update(bytes).digest("hex");
}
/** The deepest directory that contains every path. */
function commonDirectory(paths: readonly string[]): string {
let common = paths[0]!.split(sep);
for (const path of paths.slice(1)) {
const parts = path.split(sep);
let n = 0;
while (n < common.length && n < parts.length && common[n] === parts[n]) n++;
common = common.slice(0, n);
}
const joined = common.join(sep);
return joined === "" ? sep : joined;
}
function portable(path: string): string {
return sep === "/" ? path : path.split(sep).join("/");
}
export class NodeIslandModuleCollector {
/** Real paths of collected files. */
private readonly files = new Set<string>();
/** Link path (its directory real) → real target. */
private readonly links = new Map<string, string>();
private readonly packages = new Set<string>();
private readonly traced = new Set<string>();
/** Real paths of the calling modules. */
private readonly callers = new Set<string>();
/** Collects the module `specifier` names from `caller` (a real path).
* Returns null, or the reason the build cannot ship it. */
add(caller: string, specifier: string): string | null {
this.callers.add(caller);
return this.resolveFrom(caller, specifier);
}
/** The collected tree, or null when no call loads a file. */
tree(): NodeIslandModuleTree | null {
if (this.files.size === 0 && this.links.size === 0) return null;
const anchors = [
...this.files,
...this.links.keys(),
...this.links.values(),
...[...this.callers].map((caller) => dirname(caller)),
];
const root = commonDirectory(anchors);
const files: NodeIslandModuleFile[] = [];
for (const file of [...this.files].sort()) {
const bytes = trackedReadBytes(file);
if (bytes === null) continue;
files.push({
path: portable(relative(root, file)),
size: bytes.length,
sha256: sha256(bytes),
});
}
const links: NodeIslandModuleLink[] = [...this.links]
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
.map(([path, target]) => ({
path: portable(relative(root, path)),
target: portable(relative(dirname(path), target)),
}));
return { root, files, links };
}
/** The tree path a caller's module has: relative to the tree root. */
static callerPath(tree: NodeIslandModuleTree | null, caller: string): string {
return tree === null ? basename(caller) : portable(relative(tree.root, caller));
}
private resolveFrom(fromFile: string, specifier: string): string | null {
if (isBuiltin(specifier)) return null;
if (specifier.startsWith("#"))
return `'${specifier}' is a package import ("imports" in package.json), which Node island modules of the program cannot use`;
if (specifier.startsWith("file:") || /^[a-z][a-z0-9+.-]*:/i.test(specifier))
return `'${specifier}' is a URL; name the module by its relative path or package name`;
// A module the build cannot find is not shipped: loading it throws
// MODULE_NOT_FOUND when the call runs, as it does under Node (an
// optional require in a try block keeps working).
if (isPathSpecifier(specifier)) {
const target = resolveFile(resolve(dirname(fromFile), specifier));
return target === null ? null : this.addModule(this.placed(target));
}
const dir = packageDirectory(dirname(fromFile), packageName(specifier));
if (dir !== null) this.addPackage(dir);
return null;
}
/** The real path of `path`, recording a link when `path` itself (not an
* ancestor) is a symbolic link. */
private placed(path: string): string {
const target = trackedRealpath(path) ?? path;
const parent = trackedRealpath(dirname(path)) ?? dirname(path);
const link = join(parent, basename(path));
if (link !== target) this.links.set(link, target);
return target;
}
/** A module of the program: collected with the package.json that scopes
* it (Node reads its "type"), and its imports traced. */
private addModule(file: string): string | null {
if (this.traced.has(file)) return null;
this.traced.add(file);
this.files.add(file);
const scope = packageScope(dirname(file));
if (scope !== null) this.files.add(scope);
if (!TRACED.test(file)) return null;
const text = trackedReadFile(file);
if (text === null) return `cannot read ${file}`;
const scan = scanModuleSpecifiers(text);
if (scan.computed.length > 0) {
const first = scan.computed[0]!;
const line = text.slice(0, first.at).split("\n").length;
return `${file}:${line} loads a module through ${first.form}; a module that runs in a Node island loads modules by string-literal specifiers only`;
}
for (const specifier of scan.specifiers) {
const problem = this.resolveFrom(file, specifier);
if (problem !== null) return problem;
}
return null;
}
/** A package: its whole directory, then its dependencies. */
private addPackage(found: string): void {
const dir = this.placed(found);
if (this.packages.has(dir)) return;
this.packages.add(dir);
this.addDirectory(dir, new Set());
const manifest = trackedReadFile(join(dir, "package.json"));
if (manifest === null) return;
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(manifest) as Record<string, unknown>;
} catch {
return;
}
const names = new Set<string>();
for (const field of ["dependencies", "optionalDependencies", "peerDependencies"]) {
const deps = parsed[field];
if (deps !== null && typeof deps === "object")
for (const name of Object.keys(deps)) names.add(name);
}
for (const name of [...names].sort()) {
// A dependency that is not installed (optional, or a peer the program
// does not use) is not loaded either.
const depDir = packageDirectory(dir, name);
if (depDir !== null) this.addPackage(depDir);
}
}
/** Every file below `dir`, except nested node_modules (dependencies come
* through package.json) and .git. */
private addDirectory(dir: string, visited: Set<string>): void {
const real = trackedRealpath(dir) ?? dir;
if (visited.has(real)) return;
visited.add(real);
const entries = trackedAccessibleEntries(dir);
if (entries === null) return;
for (const file of entries.files) this.files.add(join(dir, file));
for (const sub of entries.directories) {
if (sub === "node_modules" || sub === ".git") continue;
this.addDirectory(join(dir, sub), visited);
}
}
}
/* ── module specifiers in JavaScript source ───────────────────────────── */
interface Token {
kind: "name" | "string" | "punct" | "other";
value: string;
at: number;
}
/** Keywords after which a `/` starts a regular expression. */
const REGEX_AFTER = new Set([
"return",
"typeof",
"instanceof",
"in",
"of",
"new",
"delete",
"void",
"throw",
"case",
"do",
"else",
"yield",
"await",
]);
function isNameStart(c: string): boolean {
return /[A-Za-z_$\u0080-￿]/.test(c);
}
function isNamePart(c: string): boolean {
return /[A-Za-z0-9_$\u0080-￿]/.test(c);
}
/** The significant tokens of a JavaScript or TypeScript source: names,
* string literals (cooked) and punctuators, without comments, template
* text or regular expressions. */
export function moduleTokens(text: string): Token[] {
const tokens: Token[] = [];
/** Brace depths at which a template literal's `${` was opened. */
const templates: number[] = [];
let depth = 0;
let i = 0;
const n = text.length;
const regexAllowed = (): boolean => {
const last = tokens[tokens.length - 1];
if (last === undefined) return true;
if (last.kind === "name") return REGEX_AFTER.has(last.value);
if (last.kind === "punct")
return last.value !== ")" && last.value !== "]" && last.value !== "}";
return false;
};
const template = (start: number): void => {
// From just after a backtick or a closing `}` to the end of the
// template or the next `${`.
while (i < n) {
const c = text[i]!;
if (c === "\\") {
i += 2;
continue;
}
if (c === "`") {
i++;
tokens.push({ kind: "other", value: "`", at: start });
return;
}
if (c === "$" && text[i + 1] === "{") {
i += 2;
templates.push(depth);
depth++;
tokens.push({ kind: "punct", value: "${", at: start });
return;
}
i++;
}
};
while (i < n) {
const c = text[i]!;
if (c === " " || c === "\t" || c === "\n" || c === "\r" || c === "\f" || c === "\v") {
i++;
continue;
}
if (c === "/" && text[i + 1] === "/") {
while (i < n && text[i] !== "\n") i++;
continue;
}
if (c === "/" && text[i + 1] === "*") {
const end = text.indexOf("*/", i + 2);
i = end < 0 ? n : end + 2;
continue;
}
if (c === "#" && i === 0 && text[1] === "!") {
while (i < n && text[i] !== "\n") i++;
continue;
}
if (c === "'" || c === '"') {
const start = i;
let value = "";
i++;
while (i < n && text[i] !== c && text[i] !== "\n") {
if (text[i] === "\\") {
const e = text[i + 1] ?? "";
value += e === "n" ? "\n" : e === "t" ? "\t" : e === "r" ? "\r" : e === "\n" ? "" : e;
i += 2;
continue;
}
value += text[i];
i++;
}
i++;
tokens.push({ kind: "string", value, at: start });
continue;
}
if (c === "`") {
const start = i;
i++;
template(start);
continue;
}
if (c === "/" && regexAllowed()) {
const start = i;
i++;
let inClass = false;
while (i < n && text[i] !== "\n") {
const r = text[i]!;
if (r === "\\") {
i += 2;
continue;
}
if (r === "[") inClass = true;
else if (r === "]") inClass = false;
else if (r === "/" && !inClass) break;
i++;
}
i++;
while (i < n && isNamePart(text[i]!)) i++;
tokens.push({ kind: "other", value: "/re/", at: start });
continue;
}
if (isNameStart(c)) {
const start = i;
while (i < n && isNamePart(text[i]!)) i++;
tokens.push({ kind: "name", value: text.slice(start, i), at: start });
continue;
}
if (/[0-9]/.test(c) || (c === "." && /[0-9]/.test(text[i + 1] ?? ""))) {
const start = i;
while (i < n && /[0-9A-Za-z_.]/.test(text[i]!)) i++;
tokens.push({ kind: "other", value: text.slice(start, i), at: start });
continue;
}
if (c === "{") depth++;
if (c === "}") {
if (templates.length > 0 && templates[templates.length - 1] === depth - 1) {
templates.pop();
depth--;
const start = i;
i++;
tokens.push({ kind: "punct", value: "}", at: start });
template(start);
continue;
}
depth--;
}
if (c === "." && text[i + 1] === "." && text[i + 2] === ".") {
tokens.push({ kind: "punct", value: "...", at: i });
i += 3;
continue;
}
if (c === "?" && text[i + 1] === ".") {
tokens.push({ kind: "punct", value: "?.", at: i });
i += 2;
continue;
}
tokens.push({ kind: "punct", value: c, at: i });
i++;
}
return tokens;
}
export interface ModuleSpecifierScan {
/** String-literal specifiers of require(), require.resolve(), import()
* and import/export declarations, in source order. */
specifiers: string[];
/** Loads whose module the source does not name with a string literal. */
computed: { at: number; form: string }[];
}
export function scanModuleSpecifiers(text: string): ModuleSpecifierScan {
const tokens = moduleTokens(text);
const specifiers: string[] = [];
const computed: { at: number; form: string }[] = [];
const at = (k: number): Token | undefined => tokens[k];
const is = (k: number, kind: Token["kind"], value?: string): boolean => {
const t = tokens[k];
return t !== undefined && t.kind === kind && (value === undefined || t.value === value);
};
/** A call `(` at k whose single argument is a string literal. */
const literalCall = (k: number): string | null =>
is(k, "punct", "(") &&
is(k + 1, "string") &&
(is(k + 2, "punct", ")") || is(k + 2, "punct", ","))
? at(k + 1)!.value
: null;
for (let k = 0; k < tokens.length; k++) {
const t = tokens[k]!;
if (t.kind !== "name") continue;
const member = is(k - 1, "punct", ".") || is(k - 1, "punct", "?.");
if (t.value === "require" && !member && !is(k + 1, "punct", ":")) {
if (is(k - 1, "name", "function") || is(k - 1, "name", "typeof")) continue;
let call = k + 1;
let form = "require()";
if (is(k + 1, "punct", ".") && is(k + 2, "name", "resolve")) {
call = k + 3;
form = "require.resolve()";
} else if (is(k + 1, "punct", ".")) {
continue;
}
if (!is(call, "punct", "(")) {
computed.push({ at: t.at, form: "require as a value" });
continue;
}
const literal = literalCall(call);
if (literal === null) computed.push({ at: t.at, form: `${form} with a computed specifier` });
else specifiers.push(literal);
continue;
}
if (t.value === "createRequire" && !member && is(k + 1, "punct", "(")) {
computed.push({ at: t.at, form: "a require() from createRequire()" });
continue;
}
if (t.value === "import" && !member) {
if (is(k + 1, "punct", ".")) continue; // import.meta
if (is(k + 1, "punct", "(")) {
const literal = literalCall(k + 1);
if (literal === null)
computed.push({ at: t.at, form: "import() with a computed specifier" });
else specifiers.push(literal);
continue;
}
if (is(k + 1, "string")) {
specifiers.push(at(k + 1)!.value);
continue;
}
for (let j = k + 1; j < tokens.length && j < k + 400; j++) {
if (is(j, "punct", ";")) break;
if (is(j, "name", "from") && is(j + 1, "string")) {
specifiers.push(at(j + 1)!.value);
break;
}
}
continue;
}
if (t.value === "export" && !member) {
let j = k + 1;
if (is(j, "name", "type")) j++;
if (is(j, "punct", "*")) {
for (; j < tokens.length && j < k + 400; j++) {
if (is(j, "punct", ";")) break;
if (is(j, "name", "from") && is(j + 1, "string")) {
specifiers.push(at(j + 1)!.value);
break;
}
}
} else if (is(j, "punct", "{")) {
while (j < tokens.length && !is(j, "punct", "}")) j++;
if (is(j + 1, "name", "from") && is(j + 2, "string")) specifiers.push(at(j + 2)!.value);
}
}
}
return { specifiers, computed };
}
/* ── shipping the tree ────────────────────────────────────────────────── */
/** The manifest the executable records (scr_node_island.c reads it):
* `{"files":[[path,size,sha256],...],"links":[[path,target],...]}`. Its
* SHA-256 names the tree. */
export function nodeIslandManifest(modules: IrNodeIslandModules): string {
return JSON.stringify({
files: modules.files.map((file) => [file.path, file.size, file.sha256]),
links: modules.links.map((link) => [link.path, link.target]),
});
}
/** The tree's files concatenated in manifest order, each checked against
* the digest the build recorded, or the reason one no longer matches. */
export function readNodeIslandModules(modules: IrNodeIslandModules): Uint8Array | string {
const parts: Uint8Array[] = [];
let total = 0;
for (const file of modules.files) {
const path = join(modules.root, ...file.path.split("/"));
let bytes: Uint8Array;
try {
bytes = new Uint8Array(readFileSync(path));
} catch {
return `${path} was removed during the build`;
}
if (bytes.length !== file.size || sha256(bytes) !== file.sha256)
return `${path} changed during the build`;
parts.push(bytes);
total += bytes.length;
}
const data = new Uint8Array(total);
let offset = 0;
for (const part of parts) {
data.set(part, offset);
offset += part.length;
}
return data;
}
/** Writes the tree to `target` (replacing it), for `--libnode=beside`, or
* returns the reason it cannot. */
export function placeNodeIslandModules(
modules: IrNodeIslandModules,
target: string,
): string | null {
const data = readNodeIslandModules(modules);
if (typeof data === "string") return data;
rmSync(target, { recursive: true, force: true });
mkdirSync(target, { recursive: true });
let offset = 0;
for (const file of modules.files) {
const path = join(target, ...file.path.split("/"));
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, data.subarray(offset, offset + file.size));
offset += file.size;
}
for (const link of modules.links) {
const path = join(target, ...link.path.split("/"));
mkdirSync(dirname(path), { recursive: true });
symlinkSync(link.target, path);
}
return null;
}
+1
View File
@@ -2,6 +2,7 @@
* full lowering/emitter graph; exact builds need only these cache/native
* identity primitives before deciding whether that graph is necessary. */
export { readRoutedExecutableCache } from "./executable/executable-cache.js";
export { libnodeCacheKey } from "./node-islands/libnode.js";
export type { EarlyExecutableRouteOptions } from "./executable/executable-cache.js";
export {
executableNativeEnvironmentFingerprint,
@@ -378,7 +378,7 @@ done:
expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"]));
expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"]));
expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]);
expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v36");
expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v37");
const clangExe = join(dir, "clang-program");
const linkDriver = join(dir, "clang-driver.c");
@@ -712,7 +712,7 @@ done:
.split("\n");
expect(undefinedSymbols).toContain("_sf_scale");
expect(undefinedSymbols).toContain("_sf_callback_mix");
expect(undefinedSymbols).toContain("_scr_runtime_abi_v36");
expect(undefinedSymbols).toContain("_scr_runtime_abi_v37");
});
test("helper and clang object paths retain the same outbound FFI C ABI", async () => {
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7447-union-record-arm-width.mjs"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7448-union-record-arm-width.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7460-transform-writev.mjs"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7461-transform-writev.cjs"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7462-transform-writev.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7463-stream-option-values.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7464-stream-option-values.mjs"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7465-transform-callback-values.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7466-stream-construct-assign.mjs"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7500-pipeline-callback-undefined.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7501-pipeline-callback-undefined.mjs"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7502-readline-cursor-undefined.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7543-var-list-declarator-order.js"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7545-string-regex-checked-arguments.js"
],
"diags": []
}
@@ -0,0 +1,7 @@
{
"order": [
"<repo>/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs",
"<repo>/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs"
],
"diags": []
}
@@ -0,0 +1,8 @@
{
"order": [
"<repo>/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs",
"<repo>/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs",
"<repo>/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7553-var-for-in-of-zero-iterations.js"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7620-js-class-view-descriptors.js"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7661-node-call-modules/main.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7662-node-call-exit-events/main.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/diagnostics/stream-runtime-option-order.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts"
],
"diags": []
}
@@ -49,6 +49,7 @@ node::EmbedderSnapshotData::Pointer g_snapshot;
std::unique_ptr<node::CommonEnvironmentSetup> g_setup;
Locker *g_locker = nullptr;
Global<Function> g_call;
Global<Function> g_finish;
char *Copy(const std::string &text, size_t *len) {
char *out = static_cast<char *>(std::malloc(text.size() + 1));
@@ -59,11 +60,23 @@ char *Copy(const std::string &text, size_t *len) {
return out;
}
// The start of the runtime's stack when the caller runs on it (see the
// v8::base::Stack definitions at the end of this file), else 0.
uintptr_t HostStackStart() {
return g_host != nullptr && g_host->stack_start != nullptr ? g_host->stack_start() : 0;
}
Local<String> Utf8(Isolate *isolate, const char *text, size_t len) {
return String::NewFromUtf8(isolate, text, NewStringType::kNormal, static_cast<int>(len))
.ToLocalChecked();
}
// A string longer than V8 can hold is empty instead of a fatal error.
v8::MaybeLocal<String> MaybeUtf8(Isolate *isolate, const char *text, size_t len) {
if (len > static_cast<size_t>(String::kMaxLength)) return {};
return String::NewFromUtf8(isolate, text, NewStringType::kNormal, static_cast<int>(len));
}
Local<String> Utf8(Isolate *isolate, const char *text) {
return Utf8(isolate, text, std::strlen(text));
}
@@ -75,15 +88,20 @@ std::string ToStdString(Isolate *isolate, Local<Value> value) {
// The outcome of an exception the dispatcher itself did not catch (a stack
// overflow, for example): an Error copy with its message.
// Reading the message runs user code (a getter, a toString), which may throw
// again; the copy then has an empty message.
std::string ThrownOutcome(Isolate *isolate, Local<Context> context, Local<Value> exception) {
TryCatch inner(isolate);
Local<Value> message = exception;
if (exception->IsObject()) {
Local<Value> own;
if (exception.As<Object>()->Get(context, Utf8(isolate, "message")).ToLocal(&own))
message = own;
}
Local<String> text;
Local<String> quoted;
if (!v8::JSON::Stringify(context, message->ToString(context).ToLocalChecked()).ToLocal(&quoted))
if (!message->ToString(context).ToLocal(&text) ||
!v8::JSON::Stringify(context, text).ToLocal(&quoted))
return "T{\"error\":{\"kind\":0,\"message\":\"\"}}";
return "T{\"error\":{\"kind\":0,\"message\":" + ToStdString(isolate, quoted) + "}}";
}
@@ -106,12 +124,18 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar
if (dladdr(reinterpret_cast<void *>(&napi_create_function), &info) && info.dli_fname)
dlopen(info.dli_fname, RTLD_NOW | RTLD_NOLOAD | RTLD_GLOBAL);
std::vector<std::string> args = {argv0};
// The program is not a node process: NODE_OPTIONS from its environment
// does not apply, and Node leaves the process's resource limits and code
// pages as they are.
auto result = node::InitializeOncePerProcess(
args, {node::ProcessInitializationFlags::kNoInitializeV8,
node::ProcessInitializationFlags::kNoInitializeNodeV8Platform,
node::ProcessInitializationFlags::kNoStdioInitialization,
node::ProcessInitializationFlags::kNoDefaultSignalHandling,
node::ProcessInitializationFlags::kNoPrintHelpOrVersionOutput});
node::ProcessInitializationFlags::kNoPrintHelpOrVersionOutput,
node::ProcessInitializationFlags::kDisableNodeOptionsEnv,
node::ProcessInitializationFlags::kNoAdjustResourceLimits,
node::ProcessInitializationFlags::kNoUseLargePages});
if (result->early_return() || result->exit_code() != 0) {
std::string message = "Node initialization failed";
for (const std::string &error : result->errors()) message += ": " + error;
@@ -121,9 +145,12 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar
v8::V8::InitializePlatform(g_platform.get());
v8::V8::Initialize();
g_snapshot = node::EmbedderSnapshotData::BuiltinSnapshotData();
// Modules resolve inside the program's own module tree, never from
// NODE_PATH or the user's global module directories.
auto flags = static_cast<node::EnvironmentFlags::Flags>(
node::EnvironmentFlags::kDefaultFlags | node::EnvironmentFlags::kNoCreateInspector |
node::EnvironmentFlags::kNoStartDebugSignalHandler);
node::EnvironmentFlags::kNoStartDebugSignalHandler |
node::EnvironmentFlags::kNoGlobalSearchPaths);
std::vector<std::string> errors;
g_setup = g_snapshot ? node::CommonEnvironmentSetup::CreateFromSnapshot(
g_platform.get(), &errors, g_snapshot.get(), result->args(),
@@ -171,12 +198,16 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar
}
Local<Value> dispatcher;
Local<Value> call;
Local<Value> finish;
if (!global->Get(context, Utf8(isolate, "__scriptcNodeIsland")).ToLocal(&dispatcher) ||
!dispatcher->IsObject() ||
!dispatcher.As<Object>()->Get(context, Utf8(isolate, "call")).ToLocal(&call) ||
!call->IsFunction())
!call->IsFunction() ||
!dispatcher.As<Object>()->Get(context, Utf8(isolate, "finish")).ToLocal(&finish) ||
!finish->IsFunction())
return Copy("the Node island bootstrap left no dispatcher", nullptr);
g_call.Reset(isolate, call.As<Function>());
g_finish.Reset(isolate, finish.As<Function>());
global->Delete(context, Utf8(isolate, "__scriptcNodeIsland")).FromMaybe(false);
global->Delete(context, Utf8(isolate, "__scriptcNodeIslandHost")).FromMaybe(false);
return nullptr;
@@ -192,12 +223,23 @@ char *Call(uint32_t id, int sync, const char *file, const char *specifier, const
{
node::CallbackScope scope(isolate, context->Global(), node::async_context{0, 0});
TryCatch try_catch(isolate);
Local<String> args_text;
if (!MaybeUtf8(isolate, args, args_len).ToLocal(&args_text)) {
// The argument JSON is longer than V8's string limit: the RangeError
// V8 throws for such a string.
outcome = "T{\"error\":{\"kind\":2,\"message\":\"Invalid string length\"}}";
if (!sync) {
g_host->settle(id, outcome.data(), outcome.size());
return nullptr;
}
return Copy(outcome, len);
}
Local<Value> argv[] = {Integer::NewFromUnsigned(isolate, id),
v8::Boolean::New(isolate, sync != 0),
Utf8(isolate, file),
Utf8(isolate, specifier),
Utf8(isolate, name),
Utf8(isolate, args, args_len)};
args_text};
Local<Value> result;
if (g_call.Get(isolate)
->Call(context, v8::Undefined(isolate), sizeof argv / sizeof argv[0], argv)
@@ -232,11 +274,29 @@ void Pump() {
g_platform->DrainTasks(isolate);
}
// The program ends with `code`: Node's process ends with it too. Node's
// 'beforeExit' runs (and the work it schedules), then 'exit'; the result
// is the exit code Node's process ends with (process.exitCode).
int Finish(int code) {
if (!g_setup) return code;
Isolate *isolate = g_setup->isolate();
{
HandleScope handles(isolate);
Local<Context> context = g_setup->context();
Context::Scope context_scope(context);
TryCatch try_catch(isolate);
Local<Value> argv[] = {Integer::New(isolate, code)};
(void)g_finish.Get(isolate)->Call(context, v8::Undefined(isolate), 1, argv);
}
return node::SpinEventLoop(g_setup->env()).FromMaybe(code);
}
void Stop() {
if (!g_setup) return;
Isolate *isolate = g_setup->isolate();
node::Stop(g_setup->env());
g_call.Reset();
g_finish.Reset();
isolate->Exit();
delete g_locker;
g_locker = nullptr;
@@ -249,10 +309,37 @@ void Stop() {
const ScrNodeBridge kBridge = {
SCR_NODE_BRIDGE_ABI, NODE_VERSION_STRING, NODE_MODULE_VERSION, Start, Call, Release,
BackendFd, BackendTimeout, Alive, Pump, Stop,
Finish,
};
} // namespace
#if defined(__linux__)
// V8 asks v8::base::Stack for the start of the current thread's stack when
// it records the bound of its conservative stack scan (isolate creation,
// Locker). libnode calls these exported functions through its PLT, and this
// library precedes libnode in the lookup scope, so the definitions below
// answer for it: the runtime's stack while the caller runs on it, else
// libnode's own answer (RTLD_NEXT). Without them a garbage collection that
// scans the stack while JavaScript runs on the runtime's stack walks from
// that stack up to the main thread's and faults in the unmapped gap.
// (macOS binds libnode's calls inside libnode; its packs need another way.)
#define SCR_V8_STACK_START(fn, symbol) \
extern "C" __attribute__((visibility("default"))) void *fn() __asm__(symbol); \
extern "C" void *fn() { \
static void *(*next)() = nullptr; \
uintptr_t start = HostStackStart(); \
if (start != 0) return reinterpret_cast<void *>(start); \
if (next == nullptr) next = reinterpret_cast<void *(*)()>(dlsym(RTLD_NEXT, symbol)); \
return next != nullptr ? next() : nullptr; \
}
SCR_V8_STACK_START(ScrV8GetStackStart, "_ZN2v84base5Stack13GetStackStartEv")
SCR_V8_STACK_START(ScrV8GetStackStartUnchecked, "_ZN2v84base5Stack22GetStackStartUncheckedEv")
SCR_V8_STACK_START(ScrV8ObtainCurrentThreadStackStart,
"_ZN2v84base5Stack29ObtainCurrentThreadStackStartEv")
#undef SCR_V8_STACK_START
#endif
extern "C" __attribute__((visibility("default"))) const ScrNodeBridge *scr_node_bridge_v1() {
return &kBridge;
}
+5 -2
View File
@@ -84,10 +84,13 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
const { values } = parseArgs({
options: { pack: { type: "string" }, platform: { type: "string" } },
});
// An explicit --pack or --platform names the pack; SCRIPTC_LIBNODE only
// replaces the host's default.
const pack = resolve(
values.pack ??
process.env.SCRIPTC_LIBNODE ??
join(repoRoot, "packages", `libnode-${values.platform ?? hostPlatform()}`),
(values.platform !== undefined
? join(repoRoot, "packages", `libnode-${values.platform}`)
: (process.env.SCRIPTC_LIBNODE ?? join(repoRoot, "packages", `libnode-${hostPlatform()}`))),
);
const manifest = JSON.parse(readFileSync(join(pack, "manifest.json"), "utf8"));
const { status, stdout, stderr } = runSmoke(pack);
+1 -1
View File
@@ -45,7 +45,7 @@ exports.quit = (code) => process.exit(code);
expect(result.stderr).toBe("");
expect(result.stdout).toBe(
[
"abi 2 ok 1",
"abi 3 ok 1",
"sync add -> R5",
'sync fails -> T{"error":{"kind":2,"message":"out of range","code":"ERR_X"}}',
`sync missing -> T{"error":{"kind":1,"message":"'missing' exported by './smoke.cjs' is not a function"}}`,
@@ -16,6 +16,9 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"files": [
"libnode.so.137",
"libscr_node_bridge.so",
@@ -16,6 +16,9 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"files": [
"libnode.so.137",
"libscr_node_bridge.so",
+2
View File
@@ -10,6 +10,8 @@ const config = await call<{ regions: string[] }>("config-parser", "parse", sourc
```
- The module loads with `require()` from the calling module: relative specifiers resolve from its directory, bare ones from its `node_modules`.
- In scriptc builds the specifier is a string literal. The build collects the module into the executable: a module of the program with the modules its string-literal `require()` and `import` reach, or a whole package with its dependencies. The executable loads them from its own copy, never from the build machine's files. A module of the program that loads others through computed specifiers is refused (SC8006); a module the build cannot find throws `MODULE_NOT_FOUND` when the call runs.
- The executable extracts its modules into `$XDG_CACHE_HOME/scriptc/node-islands/` (or `~/.cache/scriptc/node-islands/`) and checks every file against the build's digests on each start. With `--libnode=beside` they are written next to the executable instead, in `<output>.libnode/modules/`.
- Arguments and results cross as JSON copies; scriptc checks the result against the declared type.
- A thrown `Error` crosses as its nearest intrinsic error class with its `message` and string `code`.
- The executable loads Node from the libnode pack recorded at build time (`SCRIPTC_LIBNODE` or `scriptc build --libnode=<pack directory>`; `--libnode=beside` copies the pack next to the executable). `SCRIPTC_LIBNODE` overrides it at run time. A program that never calls into Node never loads it.
+2
View File
@@ -9,6 +9,8 @@
* - the module loads with require() from the calling module, so a relative
* specifier resolves from the caller's directory and a bare one from its
* node_modules (ES modules load through require(esm));
* - in scriptc builds `specifier` is a string literal: the build ships the
* module, and what it loads, inside the executable;
* - the arguments cross as a copy, `JSON.parse(JSON.stringify(args))`, and
* the function runs with the module namespace as `this`;
* - the result crosses as its JSON copy (undefined stays undefined), and
@@ -340,7 +340,7 @@ async function build() {
package: packageManifest.name,
version: packageManifest.version,
target: RUNTIME_PACK_MATRIX.target,
runtime_abi: { version: 36, marker: "scr_runtime_abi_v36" },
runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" },
compiler: {
command: compiler,
identity: compilerVersion,
@@ -437,7 +437,7 @@ async function build() {
package: packageManifest.name,
version: packageManifest.version,
target: matrix.target,
runtime_abi: { version: 36, marker: "scr_runtime_abi_v36" },
runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" },
compiler: {
command: compiler,
identity: compilerVersion,
+1 -1
View File
@@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) {
}
#endif
void scr_runtime_abi_v36(void) {}
void scr_runtime_abi_v37(void) {}
void scr_init(void) {
#ifdef SCR_WORKERS
+1 -1
View File
@@ -81,7 +81,7 @@ void scr_init(void);
/* Program objects emitted by the bundled LLVM helper reference this symbol.
* Its versioned spelling makes a mismatched manual runtime link fail before
* the program can start. */
void scr_runtime_abi_v36(void);
void scr_runtime_abi_v37(void);
void scr_runtime_workers_v8(void);
/* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ──────
@@ -34,7 +34,7 @@
* receiver without a reference; the hook closures outlive the record.
*
* Not supported here (each throws the "not supported yet" Error where it
* runs): the options scr_stream.c's dyn option walk refuses, `_writev` and `_construct`, symbol event names, listeners() /
* runs): the options scr_stream.c's dyn option walk refuses, symbol event names, listeners() /
* rawListeners(), `wrap()`, and pipes into values that are neither runtime
* streams nor constructor-function streams. */
#include "scr_runtime.h"
@@ -898,20 +898,15 @@ static ScrDyn *bcs_construct(int kind, ScrDyn *self, ScrDyn *const *args, size_t
}
bool failed = false;
bool writes = kind != BCS_READABLE;
bool transformish = kind == BCS_TRANSFORM || kind == BCS_PASSTHROUGH;
ScrDyn *opts = bcs_arg(args, argc, 0);
/* Node's constructors assign options.writev and options.construct to
* the instance (this._writev = options.writev), then read _writev and
* _construct, so an option shadows the prototype's method. */
if (opts->kind == SCR_DYN_OBJ) {
if (writes && !transformish && !bcs_copy_option(self, opts, "writev", "_writev")) return NULL;
if (writes && !bcs_copy_option(self, opts, "writev", "_writev")) return NULL;
if (!bcs_copy_option(self, opts, "construct", "_construct")) return NULL;
}
bool has_writev = writes && bcs_is_function(self, "_writev", &failed);
if (has_writev && transformish) {
bcs_unsupported("_writev on a constructor-function Transform is not supported yet");
return NULL;
}
bool has_construct = !failed && bcs_is_function(self, "_construct", &failed);
if (failed) return NULL;
ScrStream *s = kind == BCS_READABLE ? scr_stream_new_readable_dyn(opts)
+4
View File
@@ -3358,6 +3358,10 @@ static bool sf_header_value_ok(const ScrStr *value) {
}
static ScrStr *sf_header_value_bytestring(ScrStr *value) {
/* An ASCII value is its own ByteString: each byte is one UTF-16 unit. */
size_t ascii = 0;
while (ascii < value->len && (unsigned char)value->data[ascii] < 0x80) ascii++;
if (ascii == value->len) return scr_str_retain(value);
size_t len = (size_t)scr_str_utf16_len(value);
char *bytes = malloc(len ? len : 1);
if (!bytes) sf_oom();
+64 -9
View File
@@ -13898,19 +13898,74 @@ ScrDyn *scr_dyn_object_rest(const ScrDyn *source, const ScrDyn *excluded) {
/* Compiler-owned native class views retain every property and its flags,
* including nonenumerable data, without invoking accessors. */
ScrDyn *scr_dyn_copy_property_descriptors(ScrDyn *target, const ScrDyn *src) {
if (!src || src->kind == SCR_DYN_UNDEF) return scr_dyn_retain(target);
ScrDyn *keys = scr_dyn_obj_own_keys(src);
if (!keys) return NULL;
for (size_t i = 0; i < keys->v.arr.len; i++) {
ScrDyn *key = keys->v.arr.items[i];
ScrDyn *descriptor = scr_dyn_own_descriptor(src, key->v.str);
static void scr_dyn_copy_property_descriptor(ScrDyn *target, const ScrDyn *src,
const ScrDynEntry *from) {
/* Exactly scr_dyn_define_property(target, key, own descriptor of key)
* when `target` has no such property and accepts new ones: the entry
* takes the source's attributes, value or accessor pair. */
ScrDynEntry *prior = scr_dyn_find_entry(target, from->key, from->key_len);
if (prior || target->non_extensible) {
ScrStr *name = scr_str_new(from->key, from->key_len);
ScrDyn *key = scr_dyn_new_str(name);
ScrDyn *descriptor = scr_dyn_own_descriptor(src, name);
ScrDyn *result = descriptor ? scr_dyn_define_property(target, key, descriptor) : NULL;
scr_dyn_release(descriptor);
scr_dyn_release(result);
if (scr_exc_pending()) break;
scr_dyn_release(key);
scr_str_release(name);
return;
}
/* The names whose definition may move into a native Error's slots
* (scr_error_sync_slots). */
static const char *const slots[] = {"cause", "stack", "name", "message", "code"};
bool slot = false;
for (size_t i = 0; i < sizeof slots / sizeof *slots && !slot; i++)
slot = from->key_len == strlen(slots[i]) && memcmp(from->key, slots[i], from->key_len) == 0;
bool accessor = from->accessor, writable = !accessor && from->writable;
bool enumerable = from->enumerable, configurable = from->configurable;
ScrDyn *value = scr_dyn_retain(accessor ? scr_dyn_undefined() : from->value);
ScrDyn *getter = accessor && from->getter ? scr_dyn_retain(from->getter) : NULL;
ScrDyn *setter = accessor && from->setter ? scr_dyn_retain(from->setter) : NULL;
ScrStr *name = slot ? scr_str_new(from->key, from->key_len) : NULL;
scr_dyn_obj_set(target, from->key, from->key_len, value);
ScrDynEntry *entry = &target->v.obj.entries[target->v.obj.len - 1];
entry->accessor = accessor;
entry->getter = getter;
entry->setter = setter;
entry->writable = writable;
entry->enumerable = enumerable;
entry->configurable = configurable;
if (name) {
scr_error_sync_cause(target, name);
scr_str_release(name);
}
}
ScrDyn *scr_dyn_copy_property_descriptors(ScrDyn *target, const ScrDyn *src) {
if (!src || src->kind == SCR_DYN_UNDEF) return scr_dyn_retain(target);
if (src->kind == SCR_DYN_OBJ && target->kind == SCR_DYN_OBJ && src != target &&
src != scr_builtin_object_prototype && target != scr_builtin_error_prototype) {
/* Plain objects on both sides: no fence, proxy trap or accessor runs,
* so each property is copied from its entry without building the key
* and descriptor objects the general path goes through. */
if (src->v.obj.len) scr_dyn_region_store(target, NULL);
size_t *order = scr_dyn_obj_key_order(src);
for (size_t i = 0; i < src->v.obj.len && !scr_exc_pending(); i++)
scr_dyn_copy_property_descriptor(target, src, &src->v.obj.entries[order ? order[i] : i]);
free(order);
} else {
ScrDyn *keys = scr_dyn_obj_own_keys(src);
if (!keys) return NULL;
for (size_t i = 0; i < keys->v.arr.len; i++) {
ScrDyn *key = keys->v.arr.items[i];
ScrDyn *descriptor = scr_dyn_own_descriptor(src, key->v.str);
ScrDyn *result = descriptor ? scr_dyn_define_property(target, key, descriptor) : NULL;
scr_dyn_release(descriptor);
scr_dyn_release(result);
if (scr_exc_pending()) break;
}
scr_dyn_release(keys);
}
scr_dyn_release(keys);
if (!scr_exc_pending()) {
ScrDyn *symbols = scr_dyn_get_own_property_symbols(src);
if (!symbols) return NULL;
+11 -1
View File
@@ -16,7 +16,7 @@
#include <stddef.h>
#include <stdint.h>
#define SCR_NODE_BRIDGE_ABI 2u
#define SCR_NODE_BRIDGE_ABI 3u
/* The symbol a bridge library exports: `const ScrNodeBridge *(void)`. */
#define SCR_NODE_BRIDGE_ENTRY "scr_node_bridge_v1"
@@ -27,6 +27,12 @@ typedef struct ScrNodeBridgeHost {
/* Node ends the process (process.exit, an uncaught exception): the
* runtime runs its exit sequence with this code and never returns. */
void (*exit)(int code);
/* The start (highest address) of the stack the caller runs on when the
* runtime provides it (Node's own stack), 0 for a thread's own stack.
* V8 scans the stack conservatively from the current position up to the
* start it records for the thread, so it must not record the thread's
* when JavaScript runs on the runtime's stack. */
uintptr_t (*stack_start)(void);
} ScrNodeBridgeHost;
typedef struct ScrNodeBridge {
@@ -58,6 +64,10 @@ typedef struct ScrNodeBridge {
void (*pump)(void);
/* Tears Node down at process exit. */
void (*stop)(void);
/* The program ends with `code`, before stop: Node's 'beforeExit' and
* 'exit' run, and the result is the code Node's process ends with
* (process.exitCode in Node, else `code`). */
int (*finish)(int code);
} ScrNodeBridge;
#endif /* SCR_NODE_BRIDGE_H */
+679 -23
View File
@@ -6,13 +6,15 @@
* dlopens the bridge library of a libnode pack (scr_node_bridge.h), which
* links libnode, so a run that never calls into Node never maps it.
*
* Locating Node. The pack directory is SCRIPTC_LIBNODE when set, else the
* path recorded at build time; a recorded path that starts with `$ORIGIN/`
* resolves against the directory of the running executable (the `beside`
* packaging, and the hook for shipping Node with the program). No search
* path is used. The bridge must speak SCR_NODE_BRIDGE_ABI and link the
* Node version recorded at build time; anything else is a fatal error at
* first use, never a silent use of another Node.
* Locating Node. The pack directory is SCRIPTC_LIBNODE when set (an
* absolute path), else the path recorded at build time; a recorded path
* that starts with `$ORIGIN/` resolves against the directory of the
* running executable (the `beside` packaging, and the hook for shipping
* Node with the program). A process with elevated privileges ignores
* SCRIPTC_LIBNODE and refuses `$ORIGIN/`. No search path is used. The
* bridge must speak SCR_NODE_BRIDGE_ABI and link the Node version recorded
* at build time; anything else is a fatal error at first use, never a
* silent use of another Node.
*
* Calls. A call serializes its arguments as JSON, enters JavaScript through
* the bridge (a callback scope, so ticks and microtasks drain when it
@@ -43,9 +45,17 @@
#if !defined(_WIN32) && !defined(__wasi__)
#include "scr_node_island_host.h"
#include <dirent.h>
#include <dlfcn.h>
#include <errno.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <ucontext.h>
#include <unistd.h>
#if defined(__linux__)
#include <sys/auxv.h>
#endif
#ifdef __APPLE__
#define NI_BRIDGE_FILE "libscr_node_bridge.dylib"
@@ -86,6 +96,15 @@ void scr_node_island_install(const char *libnode, const char *node_version) {
SCR_COLD static void ni_oom(void) { scr_trap("scriptc: out of memory\n"); }
/* `len` bytes as a NUL-terminated malloc'd copy. */
static char *ni_cstr_of(const char *bytes, size_t len) {
char *copy = malloc(len + 1);
if (copy == NULL) ni_oom();
memcpy(copy, bytes, len);
copy[len] = '\0';
return copy;
}
SCR_COLD static void ni_fatal(const char *message) {
fflush(stdout);
fputs(message, stderr);
@@ -93,27 +112,600 @@ SCR_COLD static void ni_fatal(const char *message) {
_Exit(1);
}
/* True when the process runs with elevated privileges (setuid, setgid or
* file capabilities). The pack directory then comes only from an absolute
* path recorded at build time, as the dynamic linker ignores
* LD_LIBRARY_PATH and $ORIGIN for such processes. */
static bool ni_secure(void) {
#if defined(__APPLE__)
return issetugid() != 0;
#elif defined(__linux__)
return getauxval(AT_SECURE) != 0;
#else
return false;
#endif
}
/* The pack directory: SCRIPTC_LIBNODE, else the recorded path with a
* leading `$ORIGIN/` resolved against the executable's own directory. */
* leading `$ORIGIN/` resolved against the executable's own directory.
* Both must resolve to absolute paths. */
static const char *ni_pack_dir(void) {
const char *override = getenv("SCRIPTC_LIBNODE");
bool secure = ni_secure();
const char *override = secure ? NULL : getenv("SCRIPTC_LIBNODE");
if (override != NULL && override[0] != '\0' && override[0] != '/')
ni_fatal("scriptc: SCRIPTC_LIBNODE must be an absolute path to a libnode pack directory\n");
const char *dir = override != NULL && override[0] != '\0' ? override : ni_libnode;
if (dir == NULL || dir[0] == '\0')
ni_fatal("scriptc: this program calls into Node.js (@scriptc/node), but no libnode pack was "
"recorded at build time; set SCRIPTC_LIBNODE to a pack directory\n");
static const char origin[] = "$ORIGIN/";
int written;
if (dir != override && strncmp(dir, origin, sizeof origin - 1) == 0) {
if (secure)
ni_fatal("scriptc: a program running with elevated privileges cannot load a libnode pack "
"beside its executable; build it with an absolute SCRIPTC_LIBNODE\n");
ScrStr *self = scr_process_exec_path();
size_t len = self->len;
while (len > 0 && self->data[len - 1] != '/') len--;
snprintf(ni_dir, sizeof ni_dir, "%.*s%s", (int)len, self->data, dir + sizeof origin - 1);
if (len == 0 || self->data[0] != '/') {
scr_str_release(self);
ni_fatal("scriptc: cannot locate this program's executable to find its libnode pack\n");
}
written = snprintf(ni_dir, sizeof ni_dir, "%.*s%s", (int)len, self->data,
dir + sizeof origin - 1);
scr_str_release(self);
} else {
snprintf(ni_dir, sizeof ni_dir, "%s", dir);
written = snprintf(ni_dir, sizeof ni_dir, "%s", dir);
}
if (written < 0 || (size_t)written >= sizeof ni_dir)
ni_fatal("scriptc: the libnode pack path is too long\n");
return ni_dir;
}
/* ── the island module tree ───────────────────────────────────────────── */
/* The JavaScript the program's calls load (node-islands/modules.ts), named
* by a manifest of relative paths with their sizes and SHA-256 digests, plus
* the links between package directories. Before Node loads anything, the
* tree is made available at one directory, and every file is checked against
* the manifest:
* - beside the executable (`--libnode=beside`), at the recorded `$ORIGIN/`
* path, checked on every start;
* - otherwise from the executable itself, extracted once into
* $XDG_CACHE_HOME/scriptc/node-islands/<manifest digest>/ (or
* ~/.cache/...), checked on every start; when that directory cannot be
* used, into a private directory under $TMPDIR removed at exit.
* Directories are created 0700 and owned by the process's user; no link is
* followed below the cache directory. A process with elevated privileges
* refuses the tree, as it refuses SCRIPTC_LIBNODE. A call names its module
* by its path relative to the tree. */
static const char *ni_manifest = NULL;
static size_t ni_manifest_len = 0;
static const unsigned char *ni_modules = NULL;
static size_t ni_modules_len = 0;
static const char *ni_modules_location = NULL;
/* The tree's directory at run time; "/" when the calls load only Node's
* built-in modules. */
static char ni_root[4096] = "/";
void scr_node_island_install_modules(const char *manifest, size_t manifest_len,
const unsigned char *data, size_t data_len,
const char *location) {
ni_manifest = manifest;
ni_manifest_len = manifest_len;
ni_modules = data;
ni_modules_len = data_len;
ni_modules_location = location;
}
/* SHA-256 (FIPS 180-4), for the manifest's digests. */
typedef struct {
uint32_t h[8];
uint64_t bytes;
unsigned char tail[64];
size_t tail_len;
} NiSha256;
static const uint32_t ni_sha256_k[64] = {
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};
static uint32_t ni_rotr(uint32_t x, unsigned n) { return (x >> n) | (x << (32 - n)); }
static void ni_sha256_block(uint32_t h[8], const unsigned char *p) {
uint32_t w[64];
for (int i = 0; i < 16; i++)
w[i] = ((uint32_t)p[i * 4] << 24) | ((uint32_t)p[i * 4 + 1] << 16) |
((uint32_t)p[i * 4 + 2] << 8) | (uint32_t)p[i * 4 + 3];
for (int i = 16; i < 64; i++) {
uint32_t s0 = ni_rotr(w[i - 15], 7) ^ ni_rotr(w[i - 15], 18) ^ (w[i - 15] >> 3);
uint32_t s1 = ni_rotr(w[i - 2], 17) ^ ni_rotr(w[i - 2], 19) ^ (w[i - 2] >> 10);
w[i] = w[i - 16] + s0 + w[i - 7] + s1;
}
uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], k = h[7];
for (int i = 0; i < 64; i++) {
uint32_t t1 = k + (ni_rotr(e, 6) ^ ni_rotr(e, 11) ^ ni_rotr(e, 25)) + ((e & f) ^ (~e & g)) +
ni_sha256_k[i] + w[i];
uint32_t t2 = (ni_rotr(a, 2) ^ ni_rotr(a, 13) ^ ni_rotr(a, 22)) + ((a & b) ^ (a & c) ^ (b & c));
k = g;
g = f;
f = e;
e = d + t1;
d = c;
c = b;
b = a;
a = t1 + t2;
}
h[0] += a;
h[1] += b;
h[2] += c;
h[3] += d;
h[4] += e;
h[5] += f;
h[6] += g;
h[7] += k;
}
static void ni_sha256_init(NiSha256 *s) {
static const uint32_t initial[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19};
memcpy(s->h, initial, sizeof initial);
s->bytes = 0;
s->tail_len = 0;
}
static void ni_sha256_update(NiSha256 *s, const unsigned char *data, size_t len) {
s->bytes += len;
while (len > 0) {
if (s->tail_len == 0 && len >= 64) {
ni_sha256_block(s->h, data);
data += 64;
len -= 64;
continue;
}
size_t take = 64 - s->tail_len < len ? 64 - s->tail_len : len;
memcpy(s->tail + s->tail_len, data, take);
s->tail_len += take;
data += take;
len -= take;
if (s->tail_len == 64) {
ni_sha256_block(s->h, s->tail);
s->tail_len = 0;
}
}
}
static void ni_sha256_final(NiSha256 *s, unsigned char out[32]) {
uint64_t bits = s->bytes * 8;
unsigned char pad[72] = {0x80};
size_t padding = (s->tail_len < 56 ? 56 : 120) - s->tail_len;
for (int i = 0; i < 8; i++) pad[padding + i] = (unsigned char)(bits >> (56 - 8 * i));
ni_sha256_update(s, pad, padding + 8);
for (int i = 0; i < 8; i++)
for (int b = 0; b < 4; b++) out[i * 4 + b] = (unsigned char)(s->h[i] >> (24 - 8 * b));
}
typedef struct {
char *path;
size_t size;
unsigned char sha256[32];
size_t offset; /* into ni_modules */
} NiFile;
typedef struct {
char *path, *target;
} NiLink;
static NiFile *ni_files = NULL;
static size_t ni_nfiles = 0;
static NiLink *ni_links = NULL;
static size_t ni_nlinks = 0;
SCR_COLD static void ni_bad_manifest(void) {
ni_fatal("scriptc: this program's Node island module manifest is malformed\n");
}
static char *ni_dup_str(const ScrDyn *d) {
if (d == NULL || d->kind != SCR_DYN_STR) ni_bad_manifest();
if (memchr(d->v.str->data, '\0', d->v.str->len) != NULL) ni_bad_manifest();
return ni_cstr_of(d->v.str->data, d->v.str->len);
}
/* A relative path inside the tree: no leading `/`, no empty, `.` or `..`
* component. */
static bool ni_relative_inside(const char *path) {
if (path[0] == '\0' || path[0] == '/') return false;
for (const char *p = path;;) {
const char *slash = strchr(p, '/');
size_t n = slash != NULL ? (size_t)(slash - p) : strlen(p);
if (n == 0 || (n == 1 && p[0] == '.') || (n == 2 && p[0] == '.' && p[1] == '.')) return false;
if (slash == NULL) return true;
p = slash + 1;
}
}
/* A link target that, read from the link's directory, stays inside the
* tree. */
static bool ni_link_inside(const char *link, const char *target) {
if (target[0] == '\0' || target[0] == '/') return false;
long depth = 0;
for (const char *p = link; (p = strchr(p, '/')) != NULL; p++) depth++;
for (const char *p = target;;) {
const char *slash = strchr(p, '/');
size_t n = slash != NULL ? (size_t)(slash - p) : strlen(p);
if (n == 2 && p[0] == '.' && p[1] == '.') {
if (--depth < 0) return false;
} else if (n == 0 || !(n == 1 && p[0] == '.')) {
if (n == 0) return false;
depth++;
}
if (slash == NULL) return true;
p = slash + 1;
}
}
static int ni_hex(char c) {
return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : -1;
}
static void ni_parse_manifest(void) {
ScrStr *text = scr_str_new(ni_manifest, ni_manifest_len);
ScrDyn *body = scr_json_parse(text);
scr_str_release(text);
if (body == NULL || body->kind != SCR_DYN_OBJ) ni_bad_manifest();
ScrDyn *files = scr_dyn_obj_get(body, "files", 5);
ScrDyn *links = scr_dyn_obj_get(body, "links", 5);
if (files == NULL || files->kind != SCR_DYN_ARR || links == NULL || links->kind != SCR_DYN_ARR)
ni_bad_manifest();
ni_nfiles = (size_t)scr_dyn_arr_len(files);
ni_nlinks = (size_t)scr_dyn_arr_len(links);
ni_files = calloc(ni_nfiles + 1, sizeof *ni_files);
ni_links = calloc(ni_nlinks + 1, sizeof *ni_links);
if (ni_files == NULL || ni_links == NULL) ni_oom();
size_t offset = 0;
for (size_t i = 0; i < ni_nfiles; i++) {
ScrDyn *entry = scr_dyn_arr_at(files, (double)i);
if (entry->kind != SCR_DYN_ARR || scr_dyn_arr_len(entry) != 3) ni_bad_manifest();
ScrDyn *path = scr_dyn_arr_at(entry, 0), *size = scr_dyn_arr_at(entry, 1),
*digest = scr_dyn_arr_at(entry, 2);
NiFile *f = &ni_files[i];
f->path = ni_dup_str(path);
if (!ni_relative_inside(f->path) || size->kind != SCR_DYN_NUM || size->v.num < 0 ||
digest->kind != SCR_DYN_STR || digest->v.str->len != 64)
ni_bad_manifest();
f->size = (size_t)size->v.num;
for (int b = 0; b < 32; b++) {
int hi = ni_hex(digest->v.str->data[b * 2]), lo = ni_hex(digest->v.str->data[b * 2 + 1]);
if (hi < 0 || lo < 0) ni_bad_manifest();
f->sha256[b] = (unsigned char)(hi * 16 + lo);
}
f->offset = offset;
offset += f->size;
scr_dyn_release(path);
scr_dyn_release(size);
scr_dyn_release(digest);
scr_dyn_release(entry);
}
if (ni_modules != NULL && offset != ni_modules_len) ni_bad_manifest();
for (size_t i = 0; i < ni_nlinks; i++) {
ScrDyn *entry = scr_dyn_arr_at(links, (double)i);
if (entry->kind != SCR_DYN_ARR || scr_dyn_arr_len(entry) != 2) ni_bad_manifest();
ScrDyn *path = scr_dyn_arr_at(entry, 0), *target = scr_dyn_arr_at(entry, 1);
ni_links[i].path = ni_dup_str(path);
ni_links[i].target = ni_dup_str(target);
if (!ni_relative_inside(ni_links[i].path) ||
!ni_link_inside(ni_links[i].path, ni_links[i].target))
ni_bad_manifest();
scr_dyn_release(path);
scr_dyn_release(target);
scr_dyn_release(entry);
}
scr_dyn_release(body);
}
/* The directory `name` in `dirfd`, never through a link; created 0700 when
* `create` and absent. -1 when it cannot be opened. */
static int ni_dir_at(int dirfd, const char *name, bool create) {
if (create && mkdirat(dirfd, name, 0700) != 0 && errno != EEXIST) return -1;
return openat(dirfd, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
}
/* The parent directory of the relative `path` below `rootfd` (its last
* component is left in *base), opened component by component without
* following links; created when `create`. -1 when it cannot be opened. */
static int ni_parent_at(int rootfd, const char *path, bool create, const char **base) {
int fd = dup(rootfd);
const char *p = path;
for (const char *slash; fd >= 0 && (slash = strchr(p, '/')) != NULL; p = slash + 1) {
char component[256];
size_t n = (size_t)(slash - p);
if (n >= sizeof component) {
close(fd);
return -1;
}
memcpy(component, p, n);
component[n] = '\0';
int next = ni_dir_at(fd, component, create);
close(fd);
fd = next;
}
*base = p;
return fd;
}
/* A directory this process may trust: owned by its user, writable by no
* one else. */
static bool ni_private_dir(int fd) {
struct stat st;
return fd >= 0 && fstat(fd, &st) == 0 && S_ISDIR(st.st_mode) && st.st_uid == geteuid() &&
(st.st_mode & 022) == 0;
}
static bool ni_digest_matches(const unsigned char *data, size_t len,
const unsigned char expected[32]) {
NiSha256 s;
unsigned char got[32];
ni_sha256_init(&s);
ni_sha256_update(&s, data, len);
ni_sha256_final(&s, got);
return memcmp(got, expected, 32) == 0;
}
/* Whether the tree below `rootfd` holds exactly the manifest's files (by
* size and digest) and links. */
static bool ni_tree_matches(int rootfd) {
for (size_t i = 0; i < ni_nfiles; i++) {
const char *base;
int dirfd = ni_parent_at(rootfd, ni_files[i].path, false, &base);
if (dirfd < 0) return false;
int fd = openat(dirfd, base, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
close(dirfd);
struct stat st;
if (fd < 0 || fstat(fd, &st) != 0 || !S_ISREG(st.st_mode) ||
(size_t)st.st_size != ni_files[i].size) {
if (fd >= 0) close(fd);
return false;
}
NiSha256 s;
ni_sha256_init(&s);
unsigned char buffer[65536];
size_t total = 0;
for (;;) {
ssize_t n = read(fd, buffer, sizeof buffer);
if (n < 0 && errno == EINTR) continue;
if (n <= 0) break;
ni_sha256_update(&s, buffer, (size_t)n);
total += (size_t)n;
}
close(fd);
unsigned char got[32];
ni_sha256_final(&s, got);
if (total != ni_files[i].size || memcmp(got, ni_files[i].sha256, 32) != 0) return false;
}
for (size_t i = 0; i < ni_nlinks; i++) {
const char *base;
int dirfd = ni_parent_at(rootfd, ni_links[i].path, false, &base);
if (dirfd < 0) return false;
char target[4096];
ssize_t n = readlinkat(dirfd, base, target, sizeof target - 1);
close(dirfd);
if (n < 0) return false;
target[n] = '\0';
if (strcmp(target, ni_links[i].target) != 0) return false;
}
return true;
}
/* Writes the executable's tree below `rootfd` (an empty directory). */
static bool ni_extract(int rootfd) {
for (size_t i = 0; i < ni_nfiles; i++) {
const unsigned char *data = ni_modules + ni_files[i].offset;
if (!ni_digest_matches(data, ni_files[i].size, ni_files[i].sha256)) return false;
const char *base;
int dirfd = ni_parent_at(rootfd, ni_files[i].path, true, &base);
if (dirfd < 0) return false;
int fd = openat(dirfd, base, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600);
close(dirfd);
if (fd < 0) return false;
size_t done = 0;
while (done < ni_files[i].size) {
ssize_t n = write(fd, data + done, ni_files[i].size - done);
if (n < 0 && errno == EINTR) continue;
if (n <= 0) break;
done += (size_t)n;
}
if (close(fd) != 0 || done != ni_files[i].size) return false;
}
for (size_t i = 0; i < ni_nlinks; i++) {
const char *base;
int dirfd = ni_parent_at(rootfd, ni_links[i].path, true, &base);
if (dirfd < 0) return false;
int status = symlinkat(ni_links[i].target, dirfd, base);
close(dirfd);
if (status != 0) return false;
}
return true;
}
/* Removes everything below `dirfd` and closes it, never following links. */
static void ni_remove_below(int dirfd) {
DIR *dir = fdopendir(dirfd);
if (dir == NULL) {
close(dirfd);
return;
}
for (struct dirent *e; (e = readdir(dir)) != NULL;) {
if (strcmp(e->d_name, ".") == 0 || strcmp(e->d_name, "..") == 0) continue;
int sub = openat(dirfd, e->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (sub >= 0) {
ni_remove_below(sub);
(void)unlinkat(dirfd, e->d_name, AT_REMOVEDIR);
} else {
(void)unlinkat(dirfd, e->d_name, 0);
}
}
closedir(dir);
}
static char ni_temp_root[4096] = "";
static void ni_remove_temp_root(void) {
int fd = open(ni_temp_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (fd >= 0) ni_remove_below(fd);
(void)rmdir(ni_temp_root);
}
static bool ni_set_root(const char *a, const char *b, const char *c) {
int n = snprintf(ni_root, sizeof ni_root, "%s%s%s", a, b, c);
return n > 0 && (size_t)n < sizeof ni_root;
}
/* The tree in the cache directory, extracted there when absent. */
static bool ni_cached_root(const char *digest) {
const char *xdg = getenv("XDG_CACHE_HOME");
const char *home = getenv("HOME");
char base[4096];
int n;
if (xdg != NULL && xdg[0] == '/')
n = snprintf(base, sizeof base, "%s", xdg);
else if (home != NULL && home[0] == '/')
n = snprintf(base, sizeof base, "%s/.cache", home);
else
return false;
if (n <= 0 || (size_t)n >= sizeof base) return false;
(void)mkdir(base, 0700);
int basefd = open(base, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
if (basefd < 0) return false;
int scriptc = ni_dir_at(basefd, "scriptc", true);
close(basefd);
int islands = ni_private_dir(scriptc) ? ni_dir_at(scriptc, "node-islands", true) : -1;
if (scriptc >= 0) close(scriptc);
if (!ni_private_dir(islands)) {
if (islands >= 0) close(islands);
return false;
}
bool ok = false;
int tree = ni_dir_at(islands, digest, false);
if (tree < 0 && errno == ENOENT) {
/* Extract into a fresh sibling, then move it into place. */
char staging[128];
snprintf(staging, sizeof staging, "%s.%ld.tmp", digest, (long)getpid());
int fd = ni_dir_at(islands, staging, true);
if (ni_private_dir(fd) && ni_extract(fd) && ni_tree_matches(fd) &&
renameat(islands, staging, islands, digest) == 0) {
close(fd);
tree = ni_dir_at(islands, digest, false);
} else {
if (fd >= 0) ni_remove_below(fd);
(void)unlinkat(islands, staging, AT_REMOVEDIR);
tree = ni_dir_at(islands, digest, false); /* another process may have placed it */
}
}
if (ni_private_dir(tree) && ni_tree_matches(tree))
ok = ni_set_root(base, "/scriptc/node-islands/", digest);
if (tree >= 0) close(tree);
close(islands);
return ok;
}
/* The tree in a private directory of this run, removed at exit. */
static bool ni_temporary_root(void) {
const char *tmp = getenv("TMPDIR");
if (tmp == NULL || tmp[0] != '/') tmp = "/tmp";
int n = snprintf(ni_temp_root, sizeof ni_temp_root, "%s/scriptc-node-islands-XXXXXX", tmp);
if (n <= 0 || (size_t)n >= sizeof ni_temp_root || mkdtemp(ni_temp_root) == NULL) return false;
scr_atexit(ni_remove_temp_root);
int fd = open(ni_temp_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
bool ok = ni_private_dir(fd) && ni_extract(fd) && ni_tree_matches(fd);
if (fd >= 0) close(fd);
return ok && ni_set_root(ni_temp_root, "", "");
}
/* Makes the tree available and sets ni_root; fatal when it cannot. */
static void ni_open_modules(void) {
if (ni_manifest == NULL) return; /* only built-in modules */
if (ni_secure())
ni_fatal("scriptc: a program running with elevated privileges cannot load Node island "
"modules\n");
ni_parse_manifest();
if (ni_modules_location != NULL && ni_modules_location[0] != '\0') {
static const char origin[] = "$ORIGIN/";
if (strncmp(ni_modules_location, origin, sizeof origin - 1) != 0) ni_bad_manifest();
ScrStr *self = scr_process_exec_path();
size_t len = self->len;
while (len > 0 && self->data[len - 1] != '/') len--;
bool absolute = len > 0 && self->data[0] == '/';
char dir[4096];
int n = snprintf(dir, sizeof dir, "%.*s", (int)len, self->data);
scr_str_release(self);
if (!absolute || n <= 0 || (size_t)n >= sizeof dir ||
!ni_set_root(dir, ni_modules_location + sizeof origin - 1, ""))
ni_fatal("scriptc: cannot locate this program's Node island modules\n");
int fd = open(ni_root, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
bool ok = fd >= 0 && ni_tree_matches(fd);
if (fd >= 0) close(fd);
if (!ok) {
char message[8192];
snprintf(message, sizeof message,
"scriptc: the Node island modules at %s do not match this program\n", ni_root);
ni_fatal(message);
}
return;
}
NiSha256 s;
unsigned char digest[32];
ni_sha256_init(&s);
ni_sha256_update(&s, (const unsigned char *)ni_manifest, ni_manifest_len);
ni_sha256_final(&s, digest);
char hex[65];
for (int i = 0; i < 32; i++) snprintf(hex + i * 2, 3, "%02x", digest[i]);
if (ni_modules == NULL && ni_nfiles > 0) ni_bad_manifest();
if (!ni_cached_root(hex) && !ni_temporary_root())
ni_fatal("scriptc: cannot write this program's Node island modules to a private directory\n");
}
/* The calling module's path inside the tree: `<root>/<relative path>`. */
static char *ni_caller_path(const ScrStr *relative) {
size_t root = strlen(ni_root);
bool slash = root > 0 && ni_root[root - 1] == '/';
char *path = malloc(root + 1 + relative->len + 1);
if (path == NULL) ni_oom();
memcpy(path, ni_root, root);
if (!slash) path[root++] = '/';
memcpy(path + root, relative->data, relative->len);
path[root + relative->len] = '\0';
return path;
}
/* The bootstrap: the tree root for the host's resolution guard, then the
* host script. +1 malloc'd. */
static char *ni_bootstrap(void) {
size_t host = strlen(scr_node_island_host_js);
size_t cap = 64 + strlen(ni_root) * 6 + host;
char *out = malloc(cap);
if (out == NULL) ni_oom();
size_t n = (size_t)snprintf(out, cap, "globalThis.__scriptcNodeIslandRoot = \"");
for (const char *p = ni_root; *p != '\0'; p++) {
unsigned char c = (unsigned char)*p;
if (c == '"' || c == '\\')
n += (size_t)snprintf(out + n, cap - n, "\\%c", c);
else if (c < 0x20)
n += (size_t)snprintf(out + n, cap - n, "\\u%04x", c);
else
out[n++] = (char)c;
}
n += (size_t)snprintf(out + n, cap - n, "\";\n");
memcpy(out + n, scr_node_island_host_js, host + 1);
return out;
}
/* ── outcomes ─────────────────────────────────────────────────────────── */
/* The result of an "R" outcome: its JSON copy, or undefined. +1, or NULL
@@ -180,7 +772,10 @@ static void ni_exit(int code) {
ni_fatal("");
}
static const ScrNodeBridgeHost ni_host = {&ni_settle, &ni_exit};
/* The start of Node's stack for callers on it (V8's stack-scan bound). */
static uintptr_t ni_stack_start(void);
static const ScrNodeBridgeHost ni_host = {&ni_settle, &ni_exit, &ni_stack_start};
/* ── the loop slot ────────────────────────────────────────────────────── */
@@ -191,7 +786,27 @@ static void ni_on_node_stack(void (*job)(void *), void *arg);
static void ni_pump(void) { ni_on_node_stack(ni_pump_job, NULL); }
static int ni_backend_fd(void) { return ni_bridge->backend_fd(); }
static int ni_backend_timeout(void) { return ni_bridge->backend_timeout(); }
static void ni_stop_atexit(void) { ni_on_node_stack(ni_stop_job, NULL); }
typedef struct {
int code;
} NiFinish;
static void ni_finish_job(void *p);
/* At exit, after the program's own exit work: Node's process ends with
* the program's code ('beforeExit', then 'exit'), and Node stops. A code
* Node's process changed (process.exitCode in Node) is the program's. */
static void ni_stop_atexit(void) {
int code = (int)scr_process_exit_code_or_zero();
NiFinish finish = {code};
/* Node writes to the same stdout. */
fflush(stdout);
ni_on_node_stack(ni_finish_job, &finish);
ni_on_node_stack(ni_stop_job, NULL);
if (finish.code != code) {
fflush(NULL);
if (ni_temp_root[0] != '\0') ni_remove_temp_root();
_Exit(finish.code & 0xff);
}
}
/* ── Node's stack ─────────────────────────────────────────────────────── */
@@ -219,6 +834,13 @@ static void ni_trampoline(void) {
}
}
static uintptr_t ni_stack_start(void) {
uintptr_t sp = (uintptr_t)__builtin_frame_address(0);
return ni_stack != NULL && sp >= (uintptr_t)ni_stack && sp < (uintptr_t)ni_stack + NI_STACK
? (uintptr_t)ni_stack + NI_STACK
: 0;
}
/* Runs job(arg) on Node's stack (directly when already there). */
static void ni_on_node_stack(void (*job)(void *), void *arg) {
if (ni_on_stack) {
@@ -294,13 +916,21 @@ static void ni_stop_job(void *p) {
ni_bridge->stop();
}
static void ni_finish_job(void *p) {
NiFinish *a = p;
a->code = ni_bridge->finish(a->code);
}
/* ── loading Node ─────────────────────────────────────────────────────── */
static void ni_start(void) {
if (ni_bridge != NULL) return;
const char *dir = ni_pack_dir();
ni_open_modules();
char path[4352];
snprintf(path, sizeof path, "%s/%s", dir, NI_BRIDGE_FILE);
int written = snprintf(path, sizeof path, "%s/%s", dir, NI_BRIDGE_FILE);
if (written < 0 || (size_t)written >= sizeof path)
ni_fatal("scriptc: the libnode pack path is too long\n");
/* RTLD_GLOBAL: native addons resolve napi_* and node::* from the global
* namespace. The executable exports no symbols libnode could bind to. */
void *library = dlopen(path, RTLD_NOW | RTLD_GLOBAL);
@@ -337,8 +967,10 @@ static void ni_start(void) {
/* Flush first: Node writes to the same stdout. */
fflush(stdout);
ni_bridge = bridge;
NiStart start = {scr_node_island_host_js, argv0, NULL};
char *bootstrap = ni_bootstrap();
NiStart start = {bootstrap, argv0, NULL};
ni_on_node_stack(ni_start_job, &start);
free(bootstrap);
char *error = start.error;
if (error != NULL) {
char message[8192];
@@ -364,17 +996,11 @@ static ScrDyn *ni_args_json(ScrDyn *args) {
}
/* A borrowed ScrStr as a NUL-terminated copy for the bridge. */
static char *ni_cstr(const ScrStr *s) {
char *copy = malloc(s->len + 1);
if (copy == NULL) ni_oom();
memcpy(copy, s->data, s->len);
copy[s->len] = '\0';
return copy;
}
static char *ni_cstr(const ScrStr *s) { return ni_cstr_of(s->data, s->len); }
static char *ni_call(uint32_t id, bool sync, ScrStr *file, ScrStr *specifier, ScrStr *name,
ScrDyn *json, size_t *len) {
char *f = ni_cstr(file), *s = ni_cstr(specifier), *n = ni_cstr(name);
char *f = ni_caller_path(file), *s = ni_cstr(specifier), *n = ni_cstr(name);
NiCall call = {id, sync ? 1 : 0, f, s, n, json->v.str->data, json->v.str->len, 0, NULL};
ni_on_node_stack(ni_call_job, &call);
char *outcome = call.outcome;
@@ -385,7 +1011,23 @@ static char *ni_call(uint32_t id, bool sync, ScrStr *file, ScrStr *specifier, Sc
return outcome;
}
/* Node runs on the main thread only: its isolate is entered under the
* Locker the starting thread holds, and the loader's state and Node's
* stack are process-wide. A call from a worker thread throws an Error. */
static bool ni_main_thread(void) {
if (scr_context_is_main()) return true;
ScrStr *code = scr_str_new("", 0);
static const char text[] =
"Node islands run on the main thread only; this call came from a worker thread";
ScrStr *message = scr_str_new(text, sizeof text - 1);
scr_throw_node_coded(0, code, message);
scr_str_release(message);
scr_str_release(code);
return false;
}
ScrDyn *scr_node_island_call_sync(ScrStr *file, ScrStr *specifier, ScrStr *name, ScrDyn *args) {
if (!ni_main_thread()) return NULL;
ScrDyn *json = ni_args_json(args);
if (json == NULL) return NULL;
ni_start();
@@ -400,6 +1042,10 @@ ScrDyn *scr_node_island_call_sync(ScrStr *file, ScrStr *specifier, ScrStr *name,
ScrPromise *scr_node_island_call(ScrStr *file, ScrStr *specifier, ScrStr *name, ScrDyn *args) {
ScrPromise *p = scr_promise_new();
if (!ni_main_thread()) {
scr_promise_reject_pending(p);
return p;
}
ScrDyn *json = ni_args_json(args);
if (json == NULL) {
scr_promise_reject_pending(p);
@@ -434,6 +1080,16 @@ void scr_node_island_install(const char *libnode, const char *node_version) {
(void)node_version;
}
void scr_node_island_install_modules(const char *manifest, size_t manifest_len,
const unsigned char *data, size_t data_len,
const char *location) {
(void)manifest;
(void)manifest_len;
(void)data;
(void)data_len;
(void)location;
}
SCR_COLD static void ni_unsupported(void) {
fputs("scriptc: Node islands are not supported on this target\n", stderr);
fflush(stderr);
+9
View File
@@ -14,6 +14,15 @@
* before %main. */
void scr_node_island_install(const char *libnode, const char *node_version);
/* Records the island module tree (node-islands/modules.ts): its manifest,
* and its files concatenated in manifest order when the executable carries
* them (NULL when they ship beside it at `location`, a `$ORIGIN/` path; ""
* otherwise). All static. The emitted main calls it after
* scr_node_island_install. */
void scr_node_island_install_modules(const char *manifest, size_t manifest_len,
const unsigned char *data, size_t data_len,
const char *location);
/* @scriptc/node callSync(specifier, name, ...args): `file` is the calling
* module's path (relative specifiers resolve from it), `args` the boxed
* argument array, copied as JSON. Returns the JSON copy of the result, +1,
+63 -3
View File
@@ -13,10 +13,36 @@
* them in-process under Node: both must produce the same results, errors
* and messages. Outcomes are "R<json>" (returned; "R" alone is undefined)
* or "T<json>" (threw), the encoding of scr_node_bridge.h. */
const { createRequire } = require("node:module");
const { createRequire, registerHooks } = require("node:module");
const nativeHost = globalThis.__scriptcNodeIslandHost;
// The program's island module tree (scr_node_island.c prefixes this script
// with its root, then the bridge removes the global): modules resolve only
// to files inside it, never to a file elsewhere on the machine.
const islandRoot = globalThis.__scriptcNodeIslandRoot;
delete globalThis.__scriptcNodeIslandRoot;
if (typeof islandRoot === "string" && islandRoot !== "/") {
const { realpathSync } = require("node:fs");
const { sep } = require("node:path");
const { fileURLToPath } = require("node:url");
const root = realpathSync(islandRoot);
const inside = root.endsWith(sep) ? root : root + sep;
registerHooks({
resolve(specifier, context, nextResolve) {
const result = nextResolve(specifier, context);
if (result.url.startsWith("file:") && !fileURLToPath(result.url).startsWith(inside)) {
const error = new Error(
`Cannot find module '${specifier}': it is not one of the modules this program ships`,
);
error.code = "MODULE_NOT_FOUND";
throw error;
}
return result;
},
});
}
// Error classes that cross as copies, at their scr_error.h kind index.
const ERROR_KINDS = [
[TypeError, 1],
@@ -74,8 +100,18 @@ function errorKind(error) {
}
// A thrown value crosses as a copy: an Error as its nearest intrinsic
// class with its message and string code, anything else as JSON data.
// class with its message and string code, anything else as JSON data. An
// error whose class, message or code cannot be read crosses as an Error
// with an empty message.
function thrown(error) {
try {
return thrownCopy(error);
} catch {
return 'T{"error":{"kind":0,"message":""}}';
}
}
function thrownCopy(error) {
if (error instanceof Error) {
const copy = { kind: errorKind(error), message: String(error.message) };
if (typeof error.code === "string") copy.code = error.code;
@@ -125,4 +161,28 @@ for (const stream of [process.stdout, process.stderr]) {
stream._handle.setBlocking(true);
}
globalThis.__scriptcNodeIsland = { call };
// The program ends with `code` (the bridge then runs Node's loop to its
// end, which emits 'beforeExit' and 'exit'): Node's process shares it, as
// it shares one process under Node.
function finish(code) {
if (code !== 0) process.exitCode = code;
}
// process.execPath is this program, not a node executable: a fork without
// an execPath of its own would start the program again.
const childProcess = require("node:child_process");
const nodeFork = childProcess.fork;
childProcess.fork = function fork(modulePath, args, options) {
const forkOptions = Array.isArray(args) ? options : args;
const execPath =
forkOptions === null || typeof forkOptions !== "object" ? undefined : forkOptions.execPath;
if (execPath === undefined || execPath === process.execPath) {
throw new Error(
"child_process.fork() in a Node island needs options.execPath (a node executable): process.execPath is this program",
);
}
return Reflect.apply(nodeFork, this, arguments);
};
require("node:module").syncBuiltinESMExports();
globalThis.__scriptcNodeIsland = { call, finish };
+63 -3
View File
@@ -19,10 +19,36 @@ static const char scr_node_island_host_js[] =
" * them in-process under Node: both must produce the same results, errors\012"
" * and messages. Outcomes are \"R<json>\" (returned; \"R\" alone is undefined)\012"
" * or \"T<json>\" (threw), the encoding of scr_node_bridge.h. */\012"
"const { createRequire } = require(\"node:module\");\012"
"const { createRequire, registerHooks } = require(\"node:module\");\012"
"\012"
"const nativeHost = globalThis.__scriptcNodeIslandHost;\012"
"\012"
"// The program's island module tree (scr_node_island.c prefixes this script\012"
"// with its root, then the bridge removes the global): modules resolve only\012"
"// to files inside it, never to a file elsewhere on the machine.\012"
"const islandRoot = globalThis.__scriptcNodeIslandRoot;\012"
"delete globalThis.__scriptcNodeIslandRoot;\012"
"if (typeof islandRoot === \"string\" && islandRoot !== \"/\") {\012"
" const { realpathSync } = require(\"node:fs\");\012"
" const { sep } = require(\"node:path\");\012"
" const { fileURLToPath } = require(\"node:url\");\012"
" const root = realpathSync(islandRoot);\012"
" const inside = root.endsWith(sep) \077 root : root + sep;\012"
" registerHooks({\012"
" resolve(specifier, context, nextResolve) {\012"
" const result = nextResolve(specifier, context);\012"
" if (result.url.startsWith(\"file:\") && !fileURLToPath(result.url).startsWith(inside)) {\012"
" const error = new Error(\012"
" `Cannot find module '${specifier}': it is not one of the modules this program ships`,\012"
" );\012"
" error.code = \"MODULE_NOT_FOUND\";\012"
" throw error;\012"
" }\012"
" return result;\012"
" },\012"
" });\012"
"}\012"
"\012"
"// Error classes that cross as copies, at their scr_error.h kind index.\012"
"const ERROR_KINDS = [\012"
" [TypeError, 1],\012"
@@ -80,8 +106,18 @@ static const char scr_node_island_host_js[] =
"}\012"
"\012"
"// A thrown value crosses as a copy: an Error as its nearest intrinsic\012"
"// class with its message and string code, anything else as JSON data.\012"
"// class with its message and string code, anything else as JSON data. An\012"
"// error whose class, message or code cannot be read crosses as an Error\012"
"// with an empty message.\012"
"function thrown(error) {\012"
" try {\012"
" return thrownCopy(error);\012"
" } catch {\012"
" return 'T{\"error\":{\"kind\":0,\"message\":\"\"}}';\012"
" }\012"
"}\012"
"\012"
"function thrownCopy(error) {\012"
" if (error instanceof Error) {\012"
" const copy = { kind: errorKind(error), message: String(error.message) };\012"
" if (typeof error.code === \"string\") copy.code = error.code;\012"
@@ -131,6 +167,30 @@ static const char scr_node_island_host_js[] =
" stream._handle.setBlocking(true);\012"
"}\012"
"\012"
"globalThis.__scriptcNodeIsland = { call };\012";
"// The program ends with `code` (the bridge then runs Node's loop to its\012"
"// end, which emits 'beforeExit' and 'exit'): Node's process shares it, as\012"
"// it shares one process under Node.\012"
"function finish(code) {\012"
" if (code !== 0) process.exitCode = code;\012"
"}\012"
"\012"
"// process.execPath is this program, not a node executable: a fork without\012"
"// an execPath of its own would start the program again.\012"
"const childProcess = require(\"node:child_process\");\012"
"const nodeFork = childProcess.fork;\012"
"childProcess.fork = function fork(modulePath, args, options) {\012"
" const forkOptions = Array.isArray(args) \077 options : args;\012"
" const execPath =\012"
" forkOptions === null || typeof forkOptions !== \"object\" \077 undefined : forkOptions.execPath;\012"
" if (execPath === undefined || execPath === process.execPath) {\012"
" throw new Error(\012"
" \"child_process.fork() in a Node island needs options.execPath (a node executable): process.execPath is this program\",\012"
" );\012"
" }\012"
" return Reflect.apply(nodeFork, this, arguments);\012"
"};\012"
"require(\"node:module\").syncBuiltinESMExports();\012"
"\012"
"globalThis.__scriptcNodeIsland = { call, finish };\012";
#endif /* SCR_NODE_ISLAND_HOST_H */
+83 -7
View File
@@ -269,6 +269,7 @@ struct ScrStreamState {
* destroyed and runs its _destroy once construction completes. */
bool constructing, destroy_deferred;
bool construct_finished; /* onConstruct ran: a second callback is ERR_MULTIPLE_CALLBACK */
bool construct_called; /* the construct tick called the hook */
ScrError *destroy_err; /* owned: the deferred destroy's error */
ScrClosure *construct_cb;
ScrStreamPlainInv construct_inv;
@@ -2157,6 +2158,24 @@ static ScrDyn *scr_stream_wentry_encoding(const ScrStreamState *st, ScrDyn *enc)
return scr_dyn_retain(enc ? enc : scr_dyn_undefined());
}
/* A compiled writev whose entries declare `encoding` a string: an entry
* without one (an object-mode write passes undefined or null) gets "".
* The batch array is the one this call receives. */
void scr_stream_writev_fill_encodings(ScrDyn *chunks) {
if (!chunks || chunks->kind != SCR_DYN_ARR) return;
double n = scr_dyn_arr_len(chunks);
for (double i = 0; i < n; i++) {
ScrDyn *entry = scr_dyn_arr_at(chunks, i); /* +1 */
ScrDyn *enc = entry->kind == SCR_DYN_OBJ ? scr_dyn_obj_get(entry, "encoding", 8) : NULL;
if (entry->kind == SCR_DYN_OBJ && (!enc || enc->kind == SCR_DYN_UNDEF || enc->kind == SCR_DYN_NULL)) {
ScrStr *empty = scr_str_new("", 0);
scr_dyn_obj_set(entry, "encoding", 8, scr_dyn_new_str(empty));
scr_str_release(empty);
}
scr_dyn_release(entry);
}
}
/* A batch entry's callback: the write's callback, or Node's nop. */
static ScrDyn *scr_stream_writev_entry_cb_call(ScrClosure *clo, ScrDyn *const *args, size_t argc) {
(void)args;
@@ -2675,9 +2694,8 @@ void *scr_stream_native(ScrStream *s, void (*release)(void *)) {
* way the compiler-emitted dyn thunks do (chunk as Buffer-flavored bytes,
* encoding 'buffer', the error via the boundary encoding, the completion
* callback as a callable dyn over the *_done glue). Options that Node's
* base WOULD consume but have no lowering here (objectMode truthy,
* decodeStrings:false, non-utf8 defaultEncoding, signal, construct,
* writev, and read/write/final on the transformish bases) throw the
* base WOULD consume but have no lowering here (signal, and
* read/write/final on the transformish bases) throw the
* "not supported yet" Error — the compile-time fence's runtime twin. */
/* The boxed dyn callable (+1 — scr_box_get_ref retains; callers release
@@ -2903,11 +2921,8 @@ static bool scr_stream_parse_dyn_opts(const ScrDyn *opts, char base, ScrStreamDy
}
}
out->construct = scr_stream_dynopt_fn(opts, "construct", 9);
/* a Transform's batches reach writev, not the transform, as in Node */
if (base != 'r') out->writev = scr_stream_dynopt_fn(opts, "writev", 6);
if (transformish && out->writev != NULL) {
static const char m[] = "the stream option 'writev' on a Transform is not supported yet (batches would bypass the transform)";
return scr_stream_dynopt_unsupported(m, sizeof m - 1);
}
{
/* a known encoding or ERR_UNKNOWN_ENCODING (the states' constructors) */
ScrDyn *v = scr_dyn_obj_get((ScrDyn *)opts, "defaultEncoding", 15);
@@ -6296,6 +6311,55 @@ ScrStream *scr_stream_set_default_encoding(ScrStream *s, ScrDyn *enc) {
return scr_stream_retain(s);
}
/* Whether a defaultEncoding option value is the default: null, undefined,
* 'utf8' or 'utf-8' (the state constructors' kDefaultUTF8Encoding). */
static bool scr_stream_default_encoding_value(const ScrDyn *v) {
if (!v || v->kind == SCR_DYN_UNDEF || v->kind == SCR_DYN_NULL) return true;
return v->kind == SCR_DYN_STR && ((v->v.str->len == 4 && memcmp(v->v.str->data, "utf8", 4) == 0) ||
(v->v.str->len == 5 && memcmp(v->v.str->data, "utf-8", 5) == 0));
}
/* A runtime defaultEncoding option value, checked before construction as
* the state constructors check it: the default, a name Buffer.isEncoding
* knows, or ERR_UNKNOWN_ENCODING (thrown). Borrows v. */
void scr_stream_check_encoding_option(ScrDyn *v) {
if (scr_stream_default_encoding_value(v)) return;
if (v->kind == SCR_DYN_STR) {
ScrStr *canonical = scr_bytes_normalize_encoding(v->v.str);
if (canonical) scr_str_release(canonical);
return;
}
ScrStr *shown = scr_dyn_string_coerce_js(v);
if (!shown) return;
char msg[128];
int len = snprintf(msg, sizeof msg, "Unknown encoding: %.*s", (int)(shown->len < 64 ? shown->len : 64),
shown->data);
scr_str_release(shown);
scr_throw_error_msg_code(SCR_ERR_TYPE, msg, (size_t)(len < 0 ? 0 : len), "ERR_UNKNOWN_ENCODING");
}
/* Runtime decodeStrings and defaultEncoding option values over a fresh
* stream (after its objectMode): decodeStrings === false keeps string
* writes, and a known defaultEncoding is the write side's default as
* given and how a byte-mode readable side decodes push(string). Borrows
* both; answers the receiver (+1). */
ScrStream *scr_stream_option_values(ScrStream *s, ScrDyn *decode_strings, ScrDyn *default_encoding) {
ScrStreamState *st = s->st;
if (st->has_w && decode_strings && decode_strings->kind == SCR_DYN_BOOL && !decode_strings->v.b)
scr_stream_set_decode_strings(s, false);
if (!scr_stream_default_encoding_value(default_encoding) && default_encoding->kind == SCR_DYN_STR) {
ScrStr *canonical = scr_bytes_normalize_encoding(default_encoding->v.str);
if (canonical) {
scr_stream_set_write_encoding(s, default_encoding->v.str);
if (st->has_r && !st->r.object_entries) scr_stream_release(scr_stream_set_push_encoding(s, canonical));
scr_str_release(canonical);
} else if (scr_exc_pending()) {
scr_exc_clear(); /* checked before construction */
}
}
return scr_stream_retain(s);
}
/* stream.objectMode: scr_stream_set_object_mode, answering the receiver
* (+1). */
ScrStream *scr_stream_object_mode(ScrStream *s, bool readable, bool writable) {
@@ -7460,6 +7524,17 @@ void scr_stream_set_construct(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamP
scr_stream_construct(s, cb, inv);
}
void scr_stream_replace_construct(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamPlainInv inv) {
ScrStreamState *st = s->st;
if (!st->constructing || st->construct_called || !st->construct_cb) {
scr_closure_release(cb);
return;
}
scr_closure_release(st->construct_cb);
st->construct_cb = cb;
st->construct_inv = inv;
}
/* A class's _writev under options that may carry their own writev (the
* option, an instance property in Node, wins). */
void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamWritevInv inv) {
@@ -7728,6 +7803,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) {
scr_stream_error_buffer(s);
break;
case SCR_ST_CONSTRUCT: {
st->construct_called = true;
if (st->construct_cb) {
st->construct_inv(st->construct_cb, s);
/* a throw inside _construct reaches onConstruct on a tick */
+12
View File
@@ -561,8 +561,15 @@ void scr_stream_set_writev_concat(ScrStream *s);
/* The user construct option / _construct method (scheduled now; cb
* moves), and the writev option / _writev method (cb moves). */
void scr_stream_set_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv);
/* `s._construct = fn` after construction: replaces a construct hook whose
* tick has not run yet (Node's constructNT reads _construct then), and has
* no effect otherwise (cb moves). */
void scr_stream_replace_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv);
void scr_stream_set_writev(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv);
void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv);
/* A compiled writev whose entries declare `encoding` a string: entries
* without an encoding (object mode) get "" before the conversion. */
void scr_stream_writev_fill_encodings(struct ScrDyn *chunks);
/* Duplex.from (duplexify.js): over an async generator function (the
* compiled stage takes the source generator that the compiled source
* makes from next() and ack(); all borrowed, helpers too), and over a
@@ -605,6 +612,11 @@ ScrStream *scr_stream_duplex_from_promise(struct ScrPromise *p, ScrClosure *box)
* forms answer the stream (+1). */
void scr_stream_set_decode_strings(ScrStream *s, bool decode);
ScrStream *scr_stream_decode_strings(ScrStream *s, bool decode);
/* Runtime decodeStrings/defaultEncoding option values: checked before
* construction (ERR_UNKNOWN_ENCODING thrown), then applied to the fresh
* stream, answering it (+1). Values borrowed. */
void scr_stream_check_encoding_option(struct ScrDyn *v);
ScrStream *scr_stream_option_values(ScrStream *s, struct ScrDyn *decode_strings, struct ScrDyn *default_encoding);
void scr_stream_set_write_encoding(ScrStream *s, ScrStr *enc);
ScrStream *scr_stream_write_encoding(ScrStream *s, ScrStr *enc);
ScrStream *scr_stream_set_default_encoding(ScrStream *s, struct ScrDyn *enc);
+86
View File
@@ -551,6 +551,91 @@ static void test_class_values(void) {
scr_str_release(name);
}
/* The class-view copy (scr_dyn_copy_property_descriptors) gives each
* property of the bag the attributes, value or accessors, and own-key
* position that defining it from its own descriptor would. */
static ScrDyn *define_from(ScrDyn *target, const char *key, ScrDyn *value, bool writable,
bool enumerable, bool configurable) {
ScrDyn *descriptor = scr_dyn_new_obj();
scr_dyn_obj_set(descriptor, "value", 5, value);
scr_dyn_obj_set(descriptor, "writable", 8, scr_dyn_new_bool(writable));
scr_dyn_obj_set(descriptor, "enumerable", 10, scr_dyn_new_bool(enumerable));
scr_dyn_obj_set(descriptor, "configurable", 12, scr_dyn_new_bool(configurable));
ScrStr *name = S(key);
ScrDyn *k = scr_dyn_new_str(name);
ScrDyn *result = scr_dyn_define_property(target, k, descriptor);
scr_dyn_release(k);
scr_str_release(name);
scr_dyn_release(descriptor);
return result;
}
static const ScrDynEntry *entry_of(const ScrDyn *obj, const char *key) {
for (size_t i = 0; i < obj->v.obj.len; i++)
if (obj->v.obj.entries[i].key_len == strlen(key) &&
memcmp(obj->v.obj.entries[i].key, key, strlen(key)) == 0)
return &obj->v.obj.entries[i];
return NULL;
}
static void test_copy_property_descriptors(void) {
ScrDyn *bag = scr_dyn_new_obj();
ScrDyn *getter = scr_dyn_new_obj();
scr_dyn_release(define_from(bag, "label", scr_dyn_new_num(1), true, true, true));
scr_dyn_release(define_from(bag, "hidden", scr_dyn_new_num(2), false, false, true));
scr_dyn_obj_set(bag, "7", 1, scr_dyn_new_num(7));
scr_dyn_obj_set(bag, "3", 1, scr_dyn_new_num(3));
/* An accessor entry (the getter object stands in for its function). */
scr_dyn_obj_set(bag, "computed", 8, scr_dyn_retain(scr_dyn_undefined()));
ScrDynEntry *accessor = (ScrDynEntry *)entry_of(bag, "computed");
accessor->accessor = true;
accessor->getter = scr_dyn_retain(getter);
accessor->writable = false;
accessor->enumerable = false;
accessor->configurable = false;
ScrDyn *view = scr_dyn_new_obj();
scr_dyn_release(scr_dyn_copy_property_descriptors(view, bag));
check(!scr_exc_pending() && view->v.obj.len == 5, "copy keeps every bag property");
ScrDyn *keys = scr_dyn_obj_own_keys(view);
const char *order[] = {"3", "7", "label", "hidden", "computed"};
bool ordered = keys->v.arr.len == 5;
for (size_t i = 0; ordered && i < 5; i++) ordered = str_is(keys->v.arr.items[i]->v.str, order[i]);
check(ordered, "copy keeps the bag's own-key order");
scr_dyn_release(keys);
const ScrDynEntry *hidden = entry_of(view, "hidden");
check(hidden && !hidden->accessor && hidden->value->v.num == 2 && !hidden->writable &&
!hidden->enumerable && hidden->configurable,
"copy keeps data attributes");
const ScrDynEntry *copied = entry_of(view, "computed");
check(copied && copied->accessor && copied->getter == getter && !copied->setter &&
!copied->writable && !copied->enumerable && !copied->configurable &&
copied->value->kind == SCR_DYN_UNDEF,
"copy keeps an accessor without calling it");
/* A property the target already has is redefined like defineProperty. */
ScrDyn *existing = scr_dyn_new_obj();
scr_dyn_obj_set(existing, "label", 5, scr_dyn_new_num(9));
scr_dyn_release(scr_dyn_copy_property_descriptors(existing, bag));
const ScrDynEntry *label = entry_of(existing, "label");
check(!scr_exc_pending() && existing->v.obj.len == 5 && existing->v.obj.entries[0].key_len == 5 &&
label->value->v.num == 1,
"copy redefines an existing property in place");
/* A target that takes no new properties throws as defineProperty does. */
ScrDyn *closed = scr_dyn_new_obj();
scr_dyn_release(scr_dyn_prevent_extensions(closed));
ScrDyn *refused = scr_dyn_copy_property_descriptors(closed, bag);
check(!refused && scr_exc_pending() && closed->v.obj.len == 0, "copy onto a non-extensible object throws");
scr_exc_clear();
scr_dyn_release(closed);
scr_dyn_release(existing);
scr_dyn_release(view);
scr_dyn_release(bag);
scr_dyn_release(getter);
}
static void test_primitive_owners(void) {
ScrDyn *values[] = {scr_dyn_new_null(), scr_dyn_new_bool(false), scr_dyn_new_bool(true)};
const char *texts[] = {"null", "false", "true"};
@@ -579,6 +664,7 @@ int main(void) {
test_class_values();
scr_init();
test_primitive_owners();
test_copy_property_descriptors();
/* ── primitives ─────────────────────────────────────────────────── */
ScrDyn *d = parse_ok("null", "parse null");
@@ -0,0 +1,37 @@
// Values read back from arrays whose elements mix object literals of
// different widths, an empty object among them, keep every field: the
// record arm with more fields is tried before `{}`, which matches every
// object.
const pairs = [
["a", { publicKeyEncoding: 5 }],
["b", {}],
["c", { publicKeyEncoding: 6, extra: true }],
];
for (const [label, value] of pairs) console.log(label, Object.keys(value), JSON.stringify(value));
const options = [
["rsa", { modulusLength: 1024 }],
["ec", { namedCurve: "P-256", paramEncoding: "named" }],
["none", {}],
];
for (const [type, value] of options) {
const { modulusLength, namedCurve, paramEncoding } = value;
console.log(type, modulusLength, namedCurve, paramEncoding);
}
const mixed = [{ a: 1 }, {}, "s", { a: 2, b: "two" }];
for (const value of mixed) {
console.log(typeof value, JSON.stringify(value), typeof value === "object" ? Object.keys(value).length : -1);
}
function describe(value) {
return Object.entries(value)
.map(([key, field]) => `${key}=${field}`)
.join(";");
}
const rows = [
[1, { x: 1, y: 2 }],
[2, {}],
[3, { x: 3 }],
];
for (const [id, value] of rows) console.log(id, describe(value), value.x, value.y);
@@ -0,0 +1,21 @@
// A checked cast to a union of record types keeps every field of the
// value: arms with more required fields, then more fields, are tried
// before an arm whose fields are all optional, which matches every object.
type Full = { p: number; q: string };
type Bare = { p?: number };
type Other = { r: boolean };
const values = JSON.parse('[{"p":5,"q":"x"},{},{"r":true},{"p":7}]') as (Full | Bare | Other)[];
for (const value of values) {
const q = "q" in value ? value.q : "-";
const r = "r" in value ? value.r : "-";
console.log(JSON.stringify(value), "q" in value, "r" in value, q, r);
}
const one = JSON.parse('{"p":1,"q":"y"}') as Bare | Full;
console.log(JSON.stringify(one), "q" in one ? one.q : "no q");
type Point = { x: number; y?: number };
type Labeled = { x: number; label: string };
const points = JSON.parse('[{"x":1,"label":"a"},{"x":2,"y":3},{"x":4}]') as (Point | Labeled)[];
for (const point of points) console.log(point.x, "label" in point ? point.label : "none", "y" in point ? point.y : "none");
+33
View File
@@ -0,0 +1,33 @@
// writev on a Transform and a PassThrough (the writev option): a batch of
// buffered writes reaches writev instead of the transform, a lone write
// still reaches the transform, and the readable side gets what writev
// pushes.
import { Transform, PassThrough } from "node:stream";
const t = new Transform({
transform(chunk, enc, cb) { console.log("transform", String(chunk), enc); cb(null, chunk); },
writev(chunks, cb) {
console.log("writev", chunks.map((c) => `${c.chunk}/${c.encoding}`).join(","));
for (const { chunk } of chunks) this.push(String(chunk).toUpperCase());
cb();
},
});
t.on("data", (d) => console.log("data", String(d)));
t.on("finish", () => console.log("finish"));
t.on("end", () => console.log("end"));
t.write("a");
t.cork();
t.write("b");
t.write("c", "latin1");
process.nextTick(() => { t.uncork(); t.end("d"); });
const p = new PassThrough({
objectMode: true,
writev(chunks, cb) { console.log("pt writev", JSON.stringify(chunks.map((c) => c.chunk))); cb(); },
});
p.on("data", (d) => console.log("pt data", JSON.stringify(d)));
p.cork();
p.write({ x: 1 });
p.write([2]);
p.uncork();
p.write("solo");
p.end();
+33
View File
@@ -0,0 +1,33 @@
// Constructor-function Transform and PassThrough streams with _writev:
// the prototype's _writev takes the batches, and a writev option shadows
// it with the stream as this.
'use strict';
const { Transform, PassThrough } = require('node:stream');
function T(opts) { Transform.call(this, opts); }
Object.setPrototypeOf(T.prototype, Transform.prototype);
Object.setPrototypeOf(T, Transform);
T.prototype._transform = function (chunk, enc, cb) { cb(null, 'T(' + chunk + ')'); };
T.prototype._writev = function (chunks, cb) {
this.push('V(' + chunks.map((c) => String(c.chunk)).join('+') + ')');
cb();
};
const t = Object.create(T.prototype);
T.call(t, {});
t.on('data', (d) => console.log('t data', String(d)));
t.on('end', () => console.log('t end'));
t.write('a');
t.cork();
t.write('b');
t.write('c');
process.nextTick(() => { t.uncork(); t.end(); });
function P(opts) { PassThrough.call(this, opts); }
Object.setPrototypeOf(P.prototype, PassThrough.prototype);
Object.setPrototypeOf(P, PassThrough);
const p = Object.create(P.prototype);
P.call(p, { writev(chunks, cb) { console.log('p writev', chunks.length, this === p); cb(); } });
p.on('data', (d) => console.log('p data', String(d)));
p.cork();
p.write('x');
p.write('y');
p.uncork();
p.write('z');
+109
View File
@@ -0,0 +1,109 @@
// Typed writev on Transform and PassThrough: an options writev with
// decodeStrings: false and flush, a Transform subclass's _writev, a
// batch completing with an error, and object-mode entries whose
// encoding is declared as a string or as Node's string | null |
// undefined.
import { Transform, PassThrough, Writable } from "node:stream";
type WritevChunks = Array<{ chunk: any; encoding: BufferEncoding }>;
type Cb = (error?: Error | null) => void;
type TransformCallback = (error?: Error | null, data?: any) => void;
const log: string[] = [];
const t = new Transform({
decodeStrings: false,
transform(chunk: string, enc: BufferEncoding, cb: TransformCallback): void {
log.push(`transform ${chunk} ${enc}`);
cb(null, chunk);
},
flush(cb: TransformCallback): void {
log.push("flush");
cb(null, "!");
},
writev(chunks: WritevChunks, cb: Cb): void {
log.push(`writev ${chunks.map((c) => `${c.chunk}/${c.encoding}`).join(",")}`);
this.push(chunks.map((c) => String(c.chunk)).join(""));
cb();
},
});
t.on("data", (d: Buffer) => log.push(`data ${d.toString()}`));
t.on("end", () => log.push("end"));
t.on("finish", () => log.push("finish"));
t.on("close", () => {
console.log(log.join("\n"));
next();
});
t.write("one");
t.cork();
t.write("two", "latin1");
t.write("three");
t.uncork();
t.end("four");
class Batcher extends Transform {
_transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void {
console.log("batcher transform", chunk.toString());
cb(null, chunk);
}
_writev(chunks: Array<{ chunk: any; encoding: BufferEncoding }>, cb: Cb): void {
console.log("batcher writev", chunks.length);
this.push(Buffer.concat(chunks.map((c) => c.chunk as Buffer)));
cb();
}
}
function next(): void {
const b = new Batcher();
b.on("data", (d: Buffer) => console.log("batcher data", d.toString()));
b.write("x");
b.cork();
b.write("y");
b.write("z");
process.nextTick(() => {
b.uncork();
b.end();
});
b.on("finish", () => {
const p = new PassThrough({
writev(chunks: WritevChunks, cb: Cb): void {
console.log("pt writev", chunks.length, String(chunks[0]!.chunk));
cb(new Error("batch failed"));
},
});
p.on("error", (e: Error) => {
console.log("pt error", e.message);
objectEntries();
});
p.cork();
p.write("1");
p.write("2");
p.uncork();
});
}
function objectEntries(): void {
const declared = new Writable({
objectMode: true,
write(_chunk: any, _enc: BufferEncoding, cb: Cb): void { cb(); },
writev(chunks: WritevChunks, cb: Cb): void {
console.log("string encodings", JSON.stringify(chunks.map((c) => c.chunk)));
cb();
},
});
declared.write({ n: 0 });
declared.cork();
declared.write({ n: 1 });
declared.write({ n: 2 }, () => {});
declared.uncork();
const nodeTyped = new Writable({
objectMode: true,
write(_chunk: any, _enc: BufferEncoding, cb: Cb): void { cb(); },
writev(chunks: Array<{ chunk: any; encoding: string | null | undefined }>, cb: Cb): void {
console.log("node encodings", JSON.stringify(chunks.map((c) => [c.chunk, c.encoding === undefined ? "undefined" : c.encoding])));
cb();
},
});
nodeTyped.write({ n: 0 });
nodeTyped.cork();
nodeTyped.write({ n: 1 });
nodeTyped.write({ n: 2 }, () => {});
nodeTyped.uncork();
}
+58
View File
@@ -0,0 +1,58 @@
// Typed decodeStrings and defaultEncoding options whose values are known
// only at run time: decodeStrings === false keeps string writes, a known
// defaultEncoding is the write default as given and how push(string)
// decodes, null and utf8 are the default, an unknown name throws
// ERR_UNKNOWN_ENCODING before construction, and a subclass's super()
// options take them the same way.
import { Writable, Readable, Duplex } from "node:stream";
type Cb = (error?: Error | null) => void;
const flags: boolean[] = JSON.parse("[false, true]");
const encodings: Array<string | null> = JSON.parse('["hex", "latin1", null, "UTF8"]');
for (const decode of flags) {
const w = new Writable({
decodeStrings: decode,
defaultEncoding: encodings[0] as BufferEncoding,
write(chunk: any, enc: string, cb: Cb): void {
console.log("write", decode, typeof chunk, enc, String(chunk));
cb();
},
});
w.write("6869");
}
const r = new Readable({ defaultEncoding: encodings[1] as BufferEncoding, read(): void {} });
r.push("é");
r.push(null);
r.on("data", (d: Buffer) => console.log("readable", d.toString("hex")));
const dflt = new Writable({
defaultEncoding: encodings[2] as any,
write(chunk: Buffer, enc: string, cb: Cb): void {
console.log("null default", enc, chunk.toString());
cb();
},
});
dflt.write("ok");
const upper = new Duplex({
defaultEncoding: encodings[3] as BufferEncoding,
decodeStrings: flags[0],
read(): void {},
write(chunk: any, enc: string, cb: Cb): void {
console.log("duplex", typeof chunk, enc);
cb();
},
});
upper.write("x");
try {
new Writable({ defaultEncoding: JSON.parse('"nope"') });
} catch (e) {
console.log("bad", (e as Error & { code?: string }).code, (e as Error).message);
}
class Sink extends Writable {
constructor(decode: boolean, encoding: BufferEncoding) {
super({ decodeStrings: decode, defaultEncoding: encoding });
}
_write(chunk: any, enc: string, cb: Cb): void {
console.log("sink", typeof chunk, enc);
cb();
}
}
new Sink(flags[0]!, encodings[1] as BufferEncoding).write("y");
@@ -0,0 +1,28 @@
// Option values known only at run time in JavaScript: decodeStrings and
// defaultEncoding read from an object, a write callback whose chunk is
// then a string, decodeStrings: 0 (only false turns decoding off), a
// defaultEncoding that is not a string, and a _construct assigned after
// construction, which Node's constructor never reads.
import { Writable, Readable } from "node:stream";
const opts = { decode: false, enc: "base64" };
const w = new Writable({
decodeStrings: opts.decode,
defaultEncoding: opts.enc,
write(c, e, cb) { console.log("js", typeof c, e, String(c)); cb(); },
});
w.write("aGk=");
try { new Writable({ defaultEncoding: 5 }); } catch (e) { console.log("bad", e.code, e.message); }
const zero = new Writable({ decodeStrings: 0, write(c, e, cb) { console.log("zero", Buffer.isBuffer(c), e); cb(); } });
zero.write("z");
const late = new Writable({ write(chunk, enc, cb) { console.log("write", chunk.toString()); cb(); } });
let ran = false;
late._construct = (cb) => { ran = true; cb(); };
late.write("a");
late.end(() => console.log("ended, construct ran:", ran));
const r = new Readable({ read() {} });
const order = [];
(order.push("receiver"), r)._construct = (order.push("value"), function (cb) { console.log("never"); cb(); });
console.log(order.join(","));
r.push("x");
r.push(null);
r.on("data", (d) => console.log("data", d.toString()));

Some files were not shown because too many files have changed in this diff Show More