Merge main

This commit is contained in:
Chris Tate
2026-10-26 12:38:20 -05:00
125 changed files with 6643 additions and 416 deletions
+2
View File
@@ -122,6 +122,8 @@ jobs:
SCRIPTC_TEST_WORKERS: "1"
- name: Verify mixed-source generated AST getters
run: pnpm generate:ts7-ast --check
- name: Verify the generated IR field readers
run: pnpm generate:ir-fields --check
native_macos:
needs: [validation_scope, lanes]
+1 -1
View File
@@ -33,7 +33,7 @@ lowering, verifies before and after optimization, and publishes through a
private sibling file so a failed or interrupted request cannot truncate the
requested output.
Repeating `--output` divides the program into that many partitions, which are compiled concurrently and linked together. The helper splits the emitter's LLVM assembly text directly, so partitions also parse concurrently; input outside the emitter's narrow grammar is parsed whole and split with LLVM instead. Internal definitions become hidden so partitions can reference each other. At opt-level 0 each partition is compiled independently. Optimized partitions follow the ThinLTO model: each is simplified and summarized on its own thread, a thin link decides which small functions every partition imports for inlining and which hidden definitions become internal again, and each partition is then optimized and compiled on its own thread. Partition placement depends only on the module and the partition count, so output never depends on the host.
Repeating `--output` divides the program into that many partitions, which are compiled concurrently and linked together. The helper splits the emitter's LLVM assembly text directly, so partitions also parse concurrently; input outside the emitter's narrow grammar is parsed whole and split with LLVM instead. Internal definitions that no other definition reaches are left out of every partition, and the others become hidden so partitions can reference each other. At opt-level 0 each partition is compiled independently. Optimized partitions follow the ThinLTO model: each is simplified and summarized on its own thread, a thin link decides which small functions every partition imports for inlining and which hidden definitions become internal again, and each partition is then optimized and compiled on its own thread. Partition placement depends only on the module and the partition count, so output never depends on the host.
WebAssembly code generation differs from LLVM's default pipeline in two ways that do not change its output. The pass that gives multiple-entry loops a single entry runs only on functions whose control flow is irreducible; its reachability analysis grows with the square of a function's block count even when it has nothing to change. Object emission also leaves out the module pass that collects external symbols for assembly declarations, so each function's machine code is freed once it is printed instead of when the whole module is compiled.
+117 -61
View File
@@ -98,7 +98,7 @@ void placeGroups(std::vector<Symbol> &Symbols, ArrayRef<unsigned> Roots,
StringMap<uint64_t> GroupBytes;
for (unsigned I = 0; I < Symbols.size(); ++I) {
const Symbol &S = Symbols[I];
if (S.Kind != SymbolKind::Function)
if (S.Kind != SymbolKind::Function || S.Partition == NoPartition)
continue;
Total += Bytes(S);
StringRef Group = Symbols[Roots[I]].Group;
@@ -119,8 +119,10 @@ void placeGroups(std::vector<Symbol> &Symbols, ArrayRef<unsigned> Roots,
StringMap<uint64_t> KeyBytes;
for (unsigned I = 0; I < Symbols.size(); ++I) {
const Symbol &S = Symbols[I];
if (S.Kind != SymbolKind::Function || S.Partition == NoPartition)
continue;
StringRef Group = Symbols[Roots[I]].Group;
if (S.Kind != SymbolKind::Function || Group.empty())
if (Group.empty())
continue;
uint64_t Chunks = 1;
while (Chunks * Chunk < GroupBytes[Group])
@@ -534,64 +536,6 @@ std::optional<ProgramPartitions> partitionProgramText(StringRef Text,
}
}
// An internal function referenced by exactly one other function, and by
// no global, is a whole-program inlining candidate; keep it with that
// function so its partition can inline it without importing. A function
// body larger than this is past LLVM's inline cost limit even for the only
// call to a local function, so it is placed by its own name. Otherwise a
// program's module initializers, which only the entry point calls, would
// all share the entry point's partition.
constexpr size_t MaxInlinedBodyBytes = 256 * 1024;
constexpr unsigned Shared = ~1u;
std::vector<unsigned> Referrer(Symbols.size(), NoPartition);
for (unsigned I = 0; I < Symbols.size(); ++I) {
const Symbol &S = Symbols[I];
if (S.Kind == SymbolKind::Declaration)
continue;
for (unsigned Ref : S.Refs) {
if (Ref == I)
continue;
if (S.Kind == SymbolKind::Variable || Referrer[Ref] != NoPartition)
Referrer[Ref] = Shared;
else
Referrer[Ref] = I;
}
}
std::vector<unsigned> Roots(Symbols.size());
for (unsigned I = 0; I < Symbols.size(); ++I) {
if (Symbols[I].Kind != SymbolKind::Function)
continue;
unsigned Root = I;
for (unsigned Steps = 0; Steps < 1024; ++Steps) {
const Symbol &S = Symbols[Root];
unsigned Parent = Referrer[Root];
if (!S.Local || S.Body.size() > MaxInlinedBodyBytes ||
Parent == NoPartition || Parent == Shared ||
Symbols[Parent].Kind != SymbolKind::Function || Parent == I)
break;
Root = Parent;
}
Roots[I] = Root;
Symbols[I].Partition = stableHash(Symbols[Root].Name) % Count;
}
placeGroups(Symbols, Roots, Count);
for (const Symbol &S : Symbols) {
if (S.Kind != SymbolKind::Function)
continue;
for (unsigned Ref : S.Refs)
if (Symbols[Ref].Kind == SymbolKind::Variable &&
Symbols[Ref].Partition == NoPartition)
Symbols[Ref].Partition = S.Partition;
}
for (Symbol &S : Symbols)
if (S.Kind == SymbolKind::Variable && S.Partition == NoPartition)
S.Partition = stableHash(S.Name) % Count;
ProgramPartitions Result;
for (const Symbol &S : Symbols)
if (S.Kind != SymbolKind::Declaration && !S.Local)
Result.PublicSymbols.insert(S.Name);
// Metadata nodes are numbered in the emitter's global order, so one added
// node would renumber the references in every partition. Each partition
// instead numbers the nodes it uses in order of first use, which keeps an
@@ -606,10 +550,122 @@ std::optional<ProgramPartitions> partitionProgramText(StringRef Text,
for (StringRef Line : NamedMetadata)
scanReferences(Line, Index, NamedSymbols, NamedRoots);
// An internal definition that no other definition reaches is left out of
// every partition: the optimizer would delete it, and the emitter writes
// variants of helpers that the program never calls. Leaving them out
// shortens every partition's parse, summary and optimization, and since
// placement ignores them too, an edit that adds one changes no partition.
std::vector<char> Live(Symbols.size(), 0);
{
std::vector<unsigned> Work;
auto Reach = [&](unsigned I) {
if (!Live[I]) {
Live[I] = 1;
Work.push_back(I);
}
};
for (unsigned I = 0; I < Symbols.size(); ++I)
if (Symbols[I].Kind != SymbolKind::Declaration && !Symbols[I].Local)
Reach(I);
for (unsigned I : NamedSymbols)
Reach(I);
for (auto &Entry : Metadata)
for (unsigned I : Entry.second.Symbols)
Reach(I);
while (!Work.empty()) {
unsigned I = Work.back();
Work.pop_back();
for (unsigned Ref : Symbols[I].Refs)
Reach(Ref);
}
}
// An internal definition reached from exactly one other definition is an
// inlining candidate of it, or data only it uses; keep it with that
// definition so its partition can inline it without importing, and so a
// helper an edit adds lands in the partition of the code that uses it. A
// chain through a global continues to the function that uses the global.
// A function body larger than this is past LLVM's inline cost limit even
// for the only call to a local function, so it is placed by its own name.
// A chain stops where it would leave a function's source-module group:
// module initializers, which only the entry point calls, keep their own
// partitions instead of all sharing the entry point's.
constexpr size_t MaxInlinedBodyBytes = 256 * 1024;
constexpr unsigned Shared = ~1u;
std::vector<unsigned> Referrer(Symbols.size(), NoPartition);
for (unsigned I = 0; I < Symbols.size(); ++I) {
const Symbol &S = Symbols[I];
if (S.Kind == SymbolKind::Declaration || !Live[I])
continue;
for (unsigned Ref : S.Refs) {
if (Ref == I)
continue;
Referrer[Ref] = Referrer[Ref] == NoPartition ? I : Shared;
}
}
auto Follows = [&](unsigned Child, unsigned Parent) {
const Symbol &S = Symbols[Child];
if (!S.Local || Parent == NoPartition || Parent == Shared ||
Symbols[Parent].Kind == SymbolKind::Declaration)
return false;
if (S.Kind == SymbolKind::Function &&
(S.Body.size() > MaxInlinedBodyBytes ||
(Symbols[Parent].Kind == SymbolKind::Function && !S.Group.empty() &&
S.Group != Symbols[Parent].Group)))
return false;
return true;
};
std::vector<unsigned> Roots(Symbols.size());
for (unsigned I = 0; I < Symbols.size(); ++I) {
if (Symbols[I].Kind == SymbolKind::Declaration || !Live[I])
continue;
unsigned Root = I;
for (unsigned Steps = 0; Steps < 1024; ++Steps) {
unsigned Parent = Referrer[Root];
if (!Follows(Root, Parent) || Parent == I)
break;
Root = Parent;
}
// A function whose chain ends at a global that is not itself placed
// with a function (a table many functions use) is placed by its own
// name, so the functions of one table spread over the partitions.
if (Symbols[I].Kind == SymbolKind::Function &&
Symbols[Root].Kind != SymbolKind::Function)
Root = I;
Roots[I] = Root;
if (Symbols[I].Kind == SymbolKind::Function)
Symbols[I].Partition = stableHash(Symbols[Root].Name) % Count;
}
placeGroups(Symbols, Roots, Count);
// A global follows the function its chain ends at, then the first
// function that references it, then its own name.
for (unsigned I = 0; I < Symbols.size(); ++I)
if (Symbols[I].Kind == SymbolKind::Variable && Live[I] &&
Symbols[Roots[I]].Kind == SymbolKind::Function)
Symbols[I].Partition = Symbols[Roots[I]].Partition;
for (unsigned I = 0; I < Symbols.size(); ++I) {
const Symbol &S = Symbols[I];
if (S.Kind != SymbolKind::Function || !Live[I])
continue;
for (unsigned Ref : S.Refs)
if (Symbols[Ref].Kind == SymbolKind::Variable &&
Symbols[Ref].Partition == NoPartition)
Symbols[Ref].Partition = S.Partition;
}
for (unsigned I = 0; I < Symbols.size(); ++I)
if (Symbols[I].Kind == SymbolKind::Variable && Live[I] &&
Symbols[I].Partition == NoPartition)
Symbols[I].Partition = stableHash(Symbols[I].Name) % Count;
ProgramPartitions Result;
for (const Symbol &S : Symbols)
if (S.Kind != SymbolKind::Declaration && !S.Local)
Result.PublicSymbols.insert(S.Name);
Result.Sources.resize(Count);
std::vector<std::vector<unsigned>> Owned(Count);
for (unsigned I = 0; I < Symbols.size(); ++I)
if (Symbols[I].Kind != SymbolKind::Declaration)
if (Symbols[I].Kind != SymbolKind::Declaration && Live[I])
Owned[Symbols[I].Partition].push_back(I);
auto Assemble = [&](unsigned P) {
+9 -5
View File
@@ -24,11 +24,15 @@ struct ProgramPartitions {
// more partitions, functions under a "; scriptc-group <key>" line are placed
// with the other functions of that source-module group, which is divided
// into chunks when it is larger than the mean partition or 1 MiB; other
// functions are placed by a stable hash of their name. An internal function referenced by exactly one other function stays
// with that function unless its body is too large to inline, and a global
// stays with the first function that references it. Returns nullopt for any
// construct outside the emitter's narrow top-level grammar; callers then
// split a parsed module.
// functions are placed by a stable hash of their name. An internal
// definition referenced by exactly one other definition stays with it, also
// through a global, unless it is a function whose body is too large to
// inline or that would join a function of another source-module group; a
// global otherwise stays with the first function that references it.
// Internal definitions that no non-internal definition, named metadata or
// metadata node reaches are left out of every partition. Returns nullopt
// for any construct outside the emitter's narrow top-level grammar; callers
// then split a parsed module.
std::optional<ProgramPartitions> partitionProgramText(llvm::StringRef Text,
unsigned Count);
+1
View File
@@ -16,6 +16,7 @@
"test:ts7": "node scripts/test-ts7.mjs",
"test:checker-identity": "node scripts/checker-identity.mjs",
"generate:ts7-ast": "node scripts/generate-ts7-ast-schema.mjs",
"generate:ir-fields": "node scripts/generate-ir-fields.mjs",
"test:fetch-conformance": "vitest run tests/harness/fetch-conformance.test.ts",
"test:test262": "tsx tests/test262/run.ts",
"test:test262:fetch": "node tests/test262/fetch.mjs",
+6
View File
@@ -144,6 +144,11 @@ async function tryFastPath(): Promise<number | null> {
...(values["cpu-dispatch"] === true ? { cpuDispatch: true as const } : {}),
...(checker === undefined ? {} : { checker }),
npmStatic,
// Mirrors the full compiler's key: shape inference's overlay is opt-in
// (SCRIPTC_SHAPE_OVERLAY=1) and --no-shape-overlay turns it off.
...(process.env["SCRIPTC_SHAPE_OVERLAY"] === "1" && values["shape-overlay"] !== false
? { shapeOverlay: true as const }
: {}),
ffiProfile: ffiPath === null ? null : { path: ffiPath, bytes: ffiBytes! },
target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${process.arch}:${
helperObjectRoute ? "runtime-pack" : "driver-tu"
@@ -231,6 +236,7 @@ async function coverageFastPath(
values["provenance-sources"] ||
values["emit-ir"] ||
(values["npm-static"] ?? []).length > 0 ||
values["shape-overlay"] === false ||
(values["external-types"] ?? []).length > 0 ||
// The replay covers the default Node island policy only.
hasNodeIslandFlags(values)
+2 -2
View File
@@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => {
minimum_os: "14.0",
},
program: { object, entry_symbol: "main" },
runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" },
runtime_abi: { version: 47, marker: "scr_runtime_abi_v47" },
runtime_pack: {
kind: "precompiled",
package: "@scriptc/runtime-darwin-arm64",
@@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => {
() => null,
(failure: { stderr?: string }) => failure,
);
expect(error?.stderr).toContain("scr_runtime_abi_v46");
expect(error?.stderr).toContain("scr_runtime_abi_v47");
});
});
+6
View File
@@ -356,6 +356,12 @@ declare module "net" {
},
connectListener?: () => void,
): Socket;
/* A Unix domain socket client (Node's IpcNetConnectOpts and path
* overloads). */
export function connect(path: string, connectListener?: () => void): Socket;
export function connect(options: { path: string }, connectListener?: () => void): Socket;
export function createConnection(path: string, connectListener?: () => void): Socket;
export function createConnection(options: { path: string }, connectListener?: () => void): Socket;
/* The happy-eyeballs attempt budget (Node's process-wide knob; the
* autoSelectFamily dial consults it): one runtime int, default 250ms
* like Node's. */
+2 -2
View File
@@ -103,8 +103,8 @@ export class BlockBuilder {
}
}
br(label: string): void {
this.terminate(`br label %${label}`);
br(label: string, attachment = ""): void {
this.terminate(`br label %${label}${attachment}`);
}
condBr(v: string, t: string, f: string, attachment = ""): void {
@@ -87,9 +87,13 @@ test("teardowns of types with children of their own type free their blocks as on
classes: [
{
name: "Linked",
// The array field keeps the recursive teardown: a type whose every
// reference field holds its own type tears down in one loop
// (iterative-teardown.test.ts).
fields: [
{ name: "next", type: linked },
{ name: "other", type: linked },
{ name: "tags", type: { kind: "array", elem: { kind: "string" } } },
],
methods: [],
loc,
@@ -45,6 +45,7 @@ import {
directDestroyTarget,
segmentPrologueLines,
segmentTeardownLines,
iterativeTeardownDefinitions,
segmentedTeardown,
usedDirectTargets,
type DirectDestroy,
@@ -1334,6 +1335,15 @@ export function emitClassShapes(
} else {
const freeBody: string[] = [];
teardown(freeBody);
// A segmented class whose every reference field holds its own type and
// that has no collector header tears down in one loop (shapes.ts
// iterativeTeardownDefinitions).
const iterative =
segmented &&
!traced &&
refFields.every(
(f) => directDestroyTarget(host, f.type, directTargets)?.destroy === destroy,
);
defs.push(
...releaseBody(
host,
@@ -1345,6 +1355,18 @@ export function emitClassShapes(
lifetime,
chained,
segmented,
false,
iterative
? iterativeTeardownDefinitions(
host,
destroy,
struct,
refFields.map((f) => f.index),
`@${mangleClassRelease(cls.name)}`,
"weak",
count,
)
: null,
),
``,
);
@@ -1,4 +1,5 @@
/* LlEmitter's module-level output: module parts, helper definitions, library declarations and FFI callback definitions. */
import { loopUnrollMetadata } from "./periodic-guards.js";
import { relieveHeapPressure } from "../../heap-pressure.js";
import { emitWasmModules } from "./wasm-modules.js";
import { type LlEmitter, ffiCallbackDummyLl, llStrBytes } from "./emitter.js";
@@ -1099,6 +1100,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
out.push(...functionAttributes(emitter, hasNoInlineRecordClone));
out.push(``);
out.push(...emitter.fieldAliasMetadata());
out.push(...loopUnrollMetadata(out));
if (emitter.debug !== null) out.push(emitter.debug.render());
return out;
}
@@ -1274,6 +1276,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
);
}
out.push(...emitter.fieldAliasMetadata());
out.push(...loopUnrollMetadata(out));
if (emitter.debug !== null) out.push(emitter.debug.render());
return out;
}
@@ -95,6 +95,7 @@ import {
} from "./string-construction.js";
import { emitStringInputs } from "./string-lifetimes.js";
import { LOOPS } from "./loop-state.js";
import { loopUnrollAttachment, periodicGuardPeriod } from "./periodic-guards.js";
import { loopAccessesTypedArrays } from "../loop-polls.js";
import { emitConstantChoice, matchConstantChoice } from "./constant-choices.js";
@@ -1315,6 +1316,13 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
const lb = B.newLabel("loop.b");
const lu =
s.update || capturedLets.length > 0 || workerBudget !== null ? B.newLabel("loop.u") : lc;
// A periodic guard on the counter (periodic-guards.ts): the back
// edge asks LLVM to unroll the loop by the guard's period.
const period =
countedLoop && integerSlot
? periodicGuardPeriod(s, countedLoop.localId, countedLoop.start, countedLoop.step)
: null;
const backEdge = period === null ? "" : loopUnrollAttachment(period);
B.br(lc);
B.startBlock(lc);
enterLoopHeader(emitter, lc);
@@ -1375,7 +1383,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
});
emitter.emitBlock(s.body);
emitter.jumpTargets.pop();
B.br(lu);
B.br(lu, lu === lc ? backEdge : "");
if (lu !== lc) {
B.startBlock(lu);
emitter.emitWorkerLoopCheck(workerBudget);
@@ -1408,7 +1416,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
B.line(`store i64 ${next}, ptr ${pairedSlots[i]}`);
});
} else if (s.update) emitter.emitStmt(s.update);
B.br(lc);
B.br(lc, backEdge);
}
});
B.startBlock(le);
@@ -409,12 +409,24 @@ export function emitSerializationExpr(
}
const dynV = emitBorrowedInput(host, e.value);
const helper = host.dyn.dynCheckHelper(e.type, e.preserveRefs);
if (isDynTypedRefType(e.type)) return emitClassCheck(host, e.type, dynV.name, helper, note);
// A check of an analysis-proven fact (shape inference's overlay)
// names its site, so a failure reports a compiler bug.
const site = e.inferredSite !== undefined ? host.cstr(e.inferredSite) : null;
if (site !== null) {
host.declare(`declare void @scr_dyn_inferred_site(ptr)`);
B.line(`call void @scr_dyn_inferred_site(ptr ${site})`);
}
if (isDynTypedRefType(e.type)) {
const checked = emitClassCheck(host, e.type, dynV.name, helper, note);
if (site !== null) B.line(`call void @scr_dyn_inferred_site(ptr null)`);
return checked;
}
const ty = host.llType(e.type);
const t = B.tmp();
B.line(
`${t} = call ${ty === "i1" ? "zeroext i1" : ty} @${helper}(ptr ${dynV.name}, ptr null)`,
);
if (site !== null) B.line(`call void @scr_dyn_inferred_site(ptr null)`);
const out = host.own({ name: t, type: e.type });
host.emitPendingCheck(undefined, false, undefined, note);
return out;
@@ -195,9 +195,31 @@ function emitReplaceWith(
return host.own({ name: t, type: e.type });
}
/** Substitutions whose result owns its text: once the runtime call returns,
* no operand is read again. */
function releasesInputsOnReturn(method: string): boolean {
return method === "replace" || method === "replaceAll";
}
export function emitRegexIntrinsic(
host: LlvmEmitterContext,
e: IrExpr & { kind: "regexIntrinsic" },
): LlValue {
if (!releasesInputsOnReturn(e.method)) return emitRegexOperation(host, e);
// The operands' temps (the subject of a chained substitution, a computed
// replacement) are released when the call returns rather than at the
// end of the statement, so a chain of substitutions holds at most its
// current subject and result.
host.frames.push([]);
const out = emitRegexOperation(host, e);
host.toFrame(out);
host.releaseTemps(host.frames.pop()!);
return host.own(out);
}
function emitRegexOperation(
host: LlvmEmitterContext,
e: IrExpr & { kind: "regexIntrinsic" },
): LlValue {
const B = host.B;
const method = e.method;
@@ -684,22 +684,15 @@ export function streamCbThunkFor(
passed.push(`ptr %sr`);
}
// The stream-owning completion closure (a 1-cap closure boxing the
// retained stream) — shared by the typed and dyn done shapes.
// retained stream) — shared by the typed and dyn done shapes. Every
// write and writev call receives the same one, as Node passes one
// onwrite function; other calls receive a new one, which the stream
// reuses while it holds the only reference.
const doneEntry =
kind === "w" || kind === "v" ? "scr_stream_write_closure" : "scr_stream_done_closure";
const doneClosure = (fnRef: string, tag: string): string => {
host.declare(`declare ptr @scr_closure_new(ptr, ${host.sizeType})`);
host.declare(`declare ptr @scr_box_new_obj(ptr, ptr, ptr)`);
host.declare(`declare void @scr_box_set_ref(ptr, ptr)`);
host.declare(`declare ptr @scr_stream_retain_v(ptr)`);
host.declare(`declare void @scr_stream_release_v(ptr)`);
host.declare(`declare void @scr_stream_trace(ptr, ptr, ptr)`);
d.push(
` %${tag}clo = call ptr @scr_closure_new(ptr ${fnRef}, ${host.sizeType} 1)`,
` %${tag}bx = call ptr @scr_box_new_obj(ptr @scr_stream_retain_v, ptr @scr_stream_release_v, ptr @scr_stream_trace)`,
` %${tag}cp = getelementptr inbounds %ScrClosure, ptr %${tag}clo, i64 1`,
` store ptr %${tag}bx, ptr %${tag}cp`,
` %${tag}sr = call ptr @scr_stream_retain_v(ptr %s)`,
` call void @scr_box_set_ref(ptr %${tag}bx, ptr %${tag}sr)`,
);
host.declare(`declare ptr @${doneEntry}(ptr, ptr)`);
d.push(` %${tag}clo = call ptr @${doneEntry}(ptr %s, ptr ${fnRef})`);
return `%${tag}clo`;
};
for (let i = off; i < declared.length; i++) {
@@ -0,0 +1,68 @@
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, test } from "vitest";
import { compile, deserializeModule } from "../../index.js";
import { emitLlvmModule } from "./emitter.js";
async function moduleFor(source: string) {
const dir = await mkdtemp(join(tmpdir(), "scriptc-iterative-teardown-"));
try {
const entry = join(dir, "main.ts"),
output = join(dir, "main.ir.json");
await writeFile(entry, source);
const result = await compile(entry, { outDir: dir, outPath: output, outputKind: "ir" });
if (!result.ok)
throw new Error(result.diagnostics.map((item) => `${item.code}: ${item.message}`).join("\n"));
return deserializeModule(await readFile(output, "utf8"));
} finally {
await rm(dir, { recursive: true, force: true });
}
}
const SOURCE = `
interface TreeNode { left: TreeNode | null; right: TreeNode | null }
class ListNode {
next: ListNode | null;
value: number;
constructor(next: ListNode | null, value: number) { this.next = next; this.value = value; }
}
interface Labeled { child: Labeled | null; label: string[] }
function tree(d: number): TreeNode { return d > 0 ? { left: tree(d - 1), right: tree(d - 1) } : { left: null, right: null }; }
function size(n: TreeNode | null): number { return n === null ? 0 : 1 + size(n.left) + size(n.right); }
let list: ListNode | null = null;
for (let i = 0; i < 10; i++) list = new ListNode(list, i);
let labeled: Labeled | null = { child: { child: null, label: ["b"] }, label: ["a"] };
console.log(size(tree(4)), list!.value, labeled!.label.length);
list = null;
labeled = null;
`;
/** The definitions whose symbols end in `suffix`. */
function definitions(ir: string, suffix: string): string[] {
return [
...ir.matchAll(new RegExp(`^define internal [^\\n]*@([\\w.$%-]+${suffix})\\(`, "gm")),
].map((m) => m[1]!);
}
test("self-recursive records and classes tear down in one loop", async () => {
const ir = emitLlvmModule(await moduleFor(SOURCE), { targetTriple: "x86_64-unknown-linux-gnu" });
const loops = definitions(ir, "_destroy_it");
// The tree record and the list class; the labeled record also holds an
// array, so it keeps the recursive teardown.
expect(loops.length).toBe(2);
expect(loops.some((name) => name.includes("ListNode"))).toBe(true);
for (const name of loops) {
const body = new RegExp(
`^define internal void @${name.replace(/[$.]/g, "\\$&")}\\(ptr %root\\)[^]*?^}`,
"m",
).exec(ir)![0];
// No call to a teardown: dead children go onto the pending list.
expect(body).not.toMatch(/call [^\n]*_destroy/);
expect(body).toContain("inttoptr");
const base = name.slice(0, -"_it".length);
expect(ir).toContain(`define internal void @${base}(ptr %o)`);
expect(ir).not.toContain(`@${base}_s(`);
}
expect(definitions(ir, "_destroy_s").length).toBe(1);
});
@@ -401,6 +401,22 @@ export function emitNetworkHttpLibCall(host: LlvmEmitterContext, e: LibCallExpr)
host.emitPendingCheck();
return out;
}
if (e.fn === "net.connectPath" || e.fn === "net.connectPathCb") {
const args = e.args.map((a) => host.emitExpr(a));
let cb = "null";
if (e.fn === "net.connectPathCb") {
host.toHeap(args[2]!);
cb = args[2]!.name;
}
host.declare(`declare ptr @scr_net_connect_path_checked(ptr, i1 zeroext, ptr)`);
const t = B.tmp();
B.line(
`${t} = call ptr @scr_net_connect_path_checked(ptr ${args[0]!.name}, i1 ${args[1]!.name}, ptr ${cb})`,
);
const out = host.own({ name: t, type: e.type });
host.emitPendingCheck();
return out;
}
if (e.fn === "net.sockOnData" || e.fn === "http.reqOnData") {
const cbT = e.args[1]!.type;
if (cbT.kind !== "func")
@@ -1980,6 +1980,8 @@ export const USES_TIMERS_LIB_FNS = new Set<string>([
"net.listenOptsReusePortCb",
"net.connect",
"net.connectCb",
"net.connectPath",
"net.connectPathCb",
"net.connectLookup",
"net.connectLookupDyn",
"net.connectAttempt",
@@ -0,0 +1,80 @@
import { expect, test } from "vitest";
import { countedFor } from "../../ir/build.js";
import { BOOL, F64, type IrExpr, type IrStmt } from "../../ir/ir.js";
import {
loopUnrollAttachment,
loopUnrollMetadata,
periodicGuardPeriod,
} from "./periodic-guards.js";
const loc = { file: "guards.ts", start: 0, end: 0 };
const ref = (localId: string): IrExpr => ({ kind: "varRef", localId, type: F64, loc });
const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc });
const bin = (op: string, left: IrExpr, right: IrExpr, type = F64): IrExpr =>
({ kind: "bin", op, left, right, type, loc }) as IrExpr;
const and = (left: IrExpr, right: IrExpr): IrExpr => ({
kind: "logical",
op: "&&",
left,
right,
type: BOOL,
loc,
});
const guard = (cond: IrExpr): IrStmt => ({
kind: "if",
cond,
then: [{ kind: "break", loc } as IrStmt],
else_: null,
loc,
});
/** `for (i = 0; i < 50; i++) { ...body }` */
const loop = (body: IrStmt[]) => countedFor(loc, num(50), () => body) as IrStmt & { kind: "for" };
const period = (body: IrStmt[], start = 0, step = 1) =>
periodicGuardPeriod(loop(body), "i.0", num(start), step);
test("a guard on the counter's low bits gives the loop its period", () => {
// if ((i & 3) === 3 && x > 4) break;
const masked = bin("===", bin("&", ref("i.0"), num(3)), num(3), BOOL);
expect(period([guard(and(masked, bin(">", ref("x"), num(4), BOOL)))])).toBe(4);
// if (0 == (7 & i)) ...
expect(period([guard(bin("==", num(0), bin("&", num(7), ref("i.0")), BOOL))])).toBe(8);
expect(period([guard(bin("!==", bin("&", ref("i.0"), num(1)), num(0), BOOL))])).toBe(2);
});
test("other guards and loops get no period", () => {
const masked = (mask: number, value: number, counter = "i.0") =>
guard(bin("===", bin("&", ref(counter), num(mask)), num(value), BOOL));
// Masks that are not 1, 3 or 7, values outside the mask, another variable.
expect(period([masked(5, 1)])).toBeNull();
expect(period([masked(15, 15)])).toBeNull();
expect(period([masked(3, 4)])).toBeNull();
expect(period([masked(3, 3, "j")])).toBeNull();
// The masked test is not the condition's first conjunct.
const later = and(
bin(">", ref("x"), num(4), BOOL),
bin("===", bin("&", ref("i.0"), num(3)), num(3), BOOL),
);
expect(period([guard(later)])).toBeNull();
// A start that is not a multiple of the period, a step other than 1.
expect(period([masked(3, 3)], 1)).toBeNull();
expect(period([masked(3, 3)], 0, 2)).toBeNull();
// A nested loop.
expect(period([masked(3, 3), loop([])])).toBeNull();
});
test("a large body is not unrolled", () => {
const work: IrStmt[] = [];
for (let k = 0; k < 400; k++)
work.push({ kind: "assign", localId: "x", value: bin("+", ref("x"), num(k)), loc } as IrStmt);
expect(
period([guard(bin("===", bin("&", ref("i.0"), num(3)), num(3), BOOL)), ...work]),
).toBeNull();
});
test("the module carries the loop ID nodes its back edges refer to", () => {
const lines = [` br label %loop.c1${loopUnrollAttachment(4)}`, " ret void"];
expect(loopUnrollMetadata(lines)).toEqual([
`!1000004 = distinct !{!1000004, !{!"llvm.loop.unroll.count", i32 4}}`,
]);
expect(loopUnrollMetadata([" ret void"])).toEqual([]);
});
@@ -0,0 +1,97 @@
/* Counted loops with a periodic guard: a branch on the counter's low bits,
* such as `if ((j & 3) === 3 && ...)` (a check every fourth iteration).
* LLVM checks such a guard on every iteration, and when the guarded
* condition is cheap it speculates the condition into the loop. The loop's
* back edge asks LLVM to unroll it by the guard's period
* (`llvm.loop.unroll.count`): in each unrolled copy the counter's low bits
* are known, so the guard folds to a constant, and only one copy in each
* period keeps the check. The metadata changes how LLVM compiles the loop,
* not what it computes. */
import type { IrExpr, IrStmt } from "../../ir/ir.js";
import { everyExprChild, everyStmtChild } from "../../ir/traverse.js";
/** The most IR nodes a loop body may have, times the period, for the loop
* to be unrolled: a larger body gains less than its unrolled copies cost. */
const UNROLLED_BODY_NODES = 4096;
/** The loop IDs, one per period (2, 4 and 8). A loop ID is a distinct
* node whose first operand is itself; loops with the same period share
* one, as the copies of an inlined loop do. */
const loopIds = new Map<number, number>([
[2, 1_000_002],
[4, 1_000_004],
[8, 1_000_008],
]);
/** The guard period of `loop`, a counted loop on `counter` that starts at
* `start` and steps by `step`, or null. The period is M + 1 where the body
* has an `if` whose condition is, or starts with an `&&` of,
* `(counter & M) === C` (or `==`, `!==`, `!=`, either operand order) with
* M one of 1, 3 and 7 and C an integer from 0 to M. The loop steps by 1
* from a multiple of the period, so that the counter's low bits are known
* in each unrolled copy, and its body has no nested loop. */
export function periodicGuardPeriod(
loop: IrStmt & { kind: "for" },
counter: string,
start: IrExpr,
step: number,
): number | null {
if (step !== 1 || start.kind !== "numLit") return null;
let period: number | null = null;
let nodes = 0;
let plain = true;
const expr = (e: IrExpr): boolean => {
nodes++;
return everyExprChild(e, expr, stmt);
};
const stmt = (s: IrStmt): boolean => {
nodes++;
if (s.kind === "for" || s.kind === "while" || s.kind === "doWhile" || s.kind === "forOf") {
plain = false;
return false;
}
if (s.kind === "if" && period === null) period = guardPeriod(s.cond, counter);
return everyStmtChild(s, expr, stmt);
};
for (const s of loop.body) if (!stmt(s)) break;
if (!plain || period === null) return null;
if (start.value % period !== 0 || nodes * period > UNROLLED_BODY_NODES) return null;
return period;
}
function guardPeriod(cond: IrExpr, counter: string): number | null {
let test = cond;
while (test.kind === "logical" && test.op === "&&") test = test.left;
if (test.kind !== "bin" || !["===", "==", "!==", "!="].includes(test.op)) return null;
const masked = (e: IrExpr, other: IrExpr): number | null => {
if (e.kind !== "bin" || e.op !== "&" || other.kind !== "numLit") return null;
const [ref, mask] = e.left.kind === "varRef" ? [e.left, e.right] : [e.right, e.left];
if (ref.kind !== "varRef" || ref.localId !== counter || mask.kind !== "numLit") return null;
const m = mask.value;
if (m !== 1 && m !== 3 && m !== 7) return null;
const c = other.value;
return Number.isInteger(c) && c >= 0 && c <= m ? m + 1 : null;
};
return masked(test.left, test.right) ?? masked(test.right, test.left);
}
/** The attachment that asks LLVM to unroll a loop by `period`, for its
* back edge. */
export function loopUnrollAttachment(period: number): string {
return `, !llvm.loop !${loopIds.get(period)!}`;
}
/** The loop ID nodes that the module's text `lines` refer to. */
export function loopUnrollMetadata(lines: readonly string[]): string[] {
const used = new Set<number>();
for (const line of lines) {
if (!line.includes("!llvm.loop !")) continue;
for (const [period, id] of loopIds) if (line.includes(`!llvm.loop !${id}`)) used.add(period);
}
return [...used]
.sort((a, b) => a - b)
.map((period) => {
const id = loopIds.get(period)!;
return `!${id} = distinct !{!${id}, !{!"llvm.loop.unroll.count", i32 ${period}}}`;
});
}
+129 -1
View File
@@ -35,6 +35,7 @@ import {
emitObjectAlloc,
emitObjectFree,
emitSegmentFree,
type FreeObservers,
CYC_APPEND,
cycleAppendDecls,
cycleAppendHelper,
@@ -1820,6 +1821,115 @@ function sizeLiteral(sizeType: RcCount, value: bigint): string {
return BigInt.asIntN(sizeType === "i64" ? 64 : 32, value).toString();
}
/** The teardown of a type whose reference fields all hold its own type
* (`fields`: their struct indices; `release`: the type's inlinable release;
* no collector header): one loop instead of a call per node. A child whose
* last reference the dead node held joins a pending list linked through its
* own count word, which it no longer needs; the loop takes the list's first
* node next, so nodes are visited parent first, fields in order. Each node
* releases its block as emitObjectFree does, keeping a block of the calling
* thread's arena in a segment linked through its first word, and the segment
* goes onto the free list in one step at the end (emitSegmentFree): the next
* structure built parent first takes the blocks in the order it allocates
* them, as segmentTeardownLines arranges for the recursive form. The loop
* needs no depth budget. `_d` (the direct-chain entry other teardowns call)
* and the plain entry both run it. A class that keeps its id in the count
* word counts in the low half (`count`); a dead node's whole word is free. */
export function iterativeTeardownDefinitions(
host: ShapeHost,
destroy: string,
struct: string,
fields: readonly number[],
release: string,
observers: FreeObservers,
count: RcCount = host.sizeType,
): string[] {
const S = host.sizeType;
const lines = [
`define internal void @${destroy}_it(ptr %root) ${FN_ATTRS} {`,
`entry:`,
` %sg.own = alloca ptr`,
` br label %it.node`,
`it.node:`,
` %o = phi ptr [ %root, %entry ], [ %it.top, %it.next ]`,
` %it.pend = phi ptr [ null, %entry ], [ %it.rest, %it.next ]`,
` %it.first = phi ptr [ null, %entry ], [ %it.fa, %it.next ]`,
` %it.last = phi ptr [ null, %entry ], [ %it.la, %it.next ]`,
...fields.flatMap((index, i) => [
` %it.f${i} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${index}`,
` %it.v${i} = load ptr, ptr %it.f${i}`,
]),
` store ptr null, ptr %sg.own`,
...emitObjectFree(host, false, observers, "%sg.own"),
` br label %it.freed`,
`it.freed:`,
` %it.k = load ptr, ptr %sg.own`,
` %it.kept = icmp ne ptr %it.k, null`,
` br i1 %it.kept, label %it.app, label %it.kids`,
`it.app:`,
` %it.none = icmp eq ptr %it.first, null`,
` br i1 %it.none, label %it.appd, label %it.link`,
`it.link:`,
` store ptr %it.k, ptr %it.last`,
` br label %it.appd`,
`it.appd:`,
` %it.nf = select i1 %it.none, ptr %it.k, ptr %it.first`,
` br label %it.kids`,
`it.kids:`,
` %it.fa = phi ptr [ %it.nf, %it.appd ], [ %it.first, %it.freed ]`,
` %it.la = phi ptr [ %it.k, %it.appd ], [ %it.last, %it.freed ]`,
];
// The fields go onto the list last first, so the first field is next.
let pend = "%it.pend";
let from = "it.kids";
for (let i = fields.length - 1; i >= 0; i--) {
const v = `%it.v${i}`;
lines.push(
` %it.nn${i} = icmp ne ptr ${v}, null`,
` br i1 %it.nn${i}, label %it.chk${i}, label %it.j${i}`,
`it.chk${i}:`,
` %it.rc${i} = load ${count}, ptr ${v}`,
` %it.last${i} = icmp eq ${count} %it.rc${i}, 1`,
` br i1 %it.last${i}, label %it.push${i}, label %it.sh${i}`,
`it.push${i}:`,
` %it.pl${i} = ptrtoint ptr ${pend} to ${S}`,
` store ${S} %it.pl${i}, ptr ${v}`,
` br label %it.j${i}`,
`it.sh${i}:`,
` call void ${release}(ptr ${v}) ; a shared or immortal child`,
` br label %it.j${i}`,
`it.j${i}:`,
` %it.p${i} = phi ptr [ ${pend}, %${from} ], [ ${v}, %it.push${i} ], [ ${pend}, %it.sh${i} ]`,
);
pend = `%it.p${i}`;
from = `it.j${i}`;
}
lines.push(
` %it.empty = icmp eq ptr ${pend}, null`,
` br i1 %it.empty, label %it.done, label %it.next`,
`it.next:`,
` %it.top = getelementptr i8, ptr ${pend}, i64 0`,
` %it.lw = load ${S}, ptr %it.top`,
` %it.rest = inttoptr ${S} %it.lw to ptr`,
` br label %it.node`,
`it.done:`,
...emitSegmentFree(host, "sgf", "%it.fa", "%it.la"),
` ret void`,
`}`,
`define internal void @${destroy}_d(ptr %o, i32 %d) ${FN_ATTRS} {`,
`entry:`,
` call void @${destroy}_it(ptr %o)`,
` ret void`,
`}`,
`define internal void @${destroy}(ptr %o) ${FN_ATTRS} {`,
`entry:`,
` call void @${destroy}_it(ptr %o)`,
` ret void`,
`}`,
);
return lines;
}
/** Common NULL/immortal/decrement skeleton for ordinary object releases.
* `freeBody` owns the zero-ref teardown and must leave the current block at
* its end; traced objects also get the possible-cycle-root branch.
@@ -1838,6 +1948,7 @@ export function releaseBody(
chained = false,
segmented = false,
frozen = false,
destroyDefs: readonly string[] | null = null,
): string[] {
const S = host.sizeType;
if (bounded) host.declare(`declare void @scr_rc_destroy(ptr, ptr)`);
@@ -1845,7 +1956,9 @@ export function releaseBody(
// A freezable record's count is the word without its frozen bit.
const notFrozen = sizeLiteral(S, ~recordFrozenBit(S));
const lines = [
...(bounded ? destroyDefinitions(destroy, freeBody, chained, segmented ? host : null) : []),
...(bounded
? (destroyDefs ?? destroyDefinitions(destroy, freeBody, chained, segmented ? host : null))
: []),
...releaseFastPath(host, fnName, `${fnName}_slow`, traced, comment, immortal, S, frozen),
`define internal void @${fnName}_slow(ptr %o) noinline ${FN_ATTRS} {${comment ? ` ; ${comment}` : ""}`,
`entry:`,
@@ -2141,6 +2254,11 @@ export function emitRecordShapes(
),
);
else freeBody.push(...emitObjectFree(host, traced, observers));
// A segmented shape whose every reference field holds its own type, with
// no collector header and no frozen bit in its count, tears down in one
// loop (iterativeTeardownDefinitions).
const iterative =
segmented && !traced && shape.freezable !== true && ownType.length === refMembers.length;
defs.push(
...releaseBody(
host,
@@ -2153,6 +2271,16 @@ export function emitRecordShapes(
chained,
segmented,
shape.freezable === true,
iterative
? iterativeTeardownDefinitions(
host,
destroy,
struct,
refMembers.map((m) => m.index),
`@${mangleRecordRelease(shape.id)}`,
observers,
)
: null,
),
``,
);
@@ -2,6 +2,7 @@
* pending flag (see unwindAbiFunctions). */
import type { IrExpr, IrModule } from "../../ir/ir.js";
import { everyStmtList } from "../../ir/traverse.js";
import { eachStringField } from "../../ir/fields.generated.js";
/** How many calls above a `throw` functions keep the pending flag. */
const THROW_LEVELS = 3;
@@ -129,13 +130,14 @@ export function unwindAbiFunctions(
}
// Any function a node names other than as a call's callee may be reached
// through a pointer.
// Every string field counts (eachStringField), as a key walk of the node
// would find them; a call's callee does not.
const referenced = new Set<string>();
const noteNames = (node: object, skip: string | undefined): void => {
for (const key in node) {
if (key === skip) continue;
const value = (node as Record<string, unknown>)[key];
if (typeof value === "string" && names.has(value)) referenced.add(value);
}
const noteName = (value: string): void => {
if (names.has(value)) referenced.add(value);
};
const noteCallName = (value: string, field: string): void => {
if (field !== "callee") noteName(value);
};
// The call graph, for the functions near a throw and the recursive ones.
// Calls through values (closures, class values, dynamic calls, callbacks
@@ -171,10 +173,10 @@ export function unwindAbiFunctions(
// flag they keep landing pads out of the top-level code inlined
// into the entry.
if (coroutine || fn.name === mod.entry) blocked.add(e.callee);
noteNames(e, "callee");
eachStringField(e, noteCallName);
return true;
}
noteNames(e, undefined);
eachStringField(e, noteName);
if (e.kind === "virtualCall")
for (const target of e.targets ?? methodsNamed.get(e.method) ?? []) edge(fn.name, target);
else if (e.kind === "new") {
@@ -193,7 +195,7 @@ export function unwindAbiFunctions(
stmt: (s) => {
const kind = s.kind as string;
if (kind === "throw" || kind === "rethrow") throws.add(fn.name);
noteNames(s, undefined);
eachStringField(s, noteName);
return true;
},
});
+2 -2
View File
@@ -2,8 +2,8 @@
* of the link contract: a program object keeps this symbol undefined and the
* matching runtime defines it, so incompatible manual links fail before the
* program can start. */
export const RUNTIME_ABI_VERSION = 46 as const;
export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v46" as const;
export const RUNTIME_ABI_VERSION = 47 as const;
export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v47" as const;
/** Object consumption is useful today, but the complete scr_* surface may
* still change before 1.0. The versioned marker prevents accidental mixing;
@@ -399,6 +399,19 @@ export function runtimePackFlavorKey(
return optimized;
}
/** The units a flavor links for `features`, each with its selected
* variant. Variants of one unit can share an artifact file (the pack stores
* identical objects once), so a variant is identified by its unit and id,
* not by its path. */
export function selectedRuntimeUnits(
flavor: RuntimePackFlavorManifest,
features: RuntimeFeatureSet,
): { source: string; variant: RuntimePackVariant }[] {
return flavor.runtime_units
.filter((unit) => evaluateRuntimePredicate(unit.predicate, features))
.map((unit) => ({ source: unit.source, variant: selectVariant(unit, features) }));
}
/** Selection is deterministic and independent of installation paths. Both
* hosts verify and privately stage exactly these artifacts before linking. */
export function selectRuntimePackArtifacts(
@@ -420,9 +433,7 @@ export function selectRuntimePackArtifacts(
);
return {
features,
runtime: selectedFlavor.runtime_units
.filter((unit) => evaluateRuntimePredicate(unit.predicate, features))
.map((unit) => selectVariant(unit, features)),
runtime: selectedRuntimeUnits(selectedFlavor, features).map(({ variant }) => variant),
archives: manifest.archives.filter((archive) =>
evaluateRuntimePredicate(archive.predicate, features),
),
@@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) {
object_format: target.objectFormat,
minimum_os: target.minimumOs,
},
runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" },
runtime_abi: { version: 47, marker: "scr_runtime_abi_v47" },
compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple },
macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" },
flavors: {
@@ -129,7 +129,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) {
object_format: target.objectFormat,
minimum_os: target.minimumOs,
},
runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" },
runtime_abi: { version: 47, marker: "scr_runtime_abi_v47" },
compiler: {
command: "clang",
identity: "fixture clang",
+1
View File
@@ -641,6 +641,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
? {}
: { profileUse: resolve(values["profile-use"]) }),
...(npmStatic !== undefined ? { npmStatic } : {}),
...(values["shape-overlay"] === false ? { shapeOverlay: false } : {}),
...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}),
...(printNativeLinkInfo ? { nativeLinkInfo: true } : {}),
...(tier !== undefined ? { tier } : {}),
+5
View File
@@ -104,6 +104,10 @@ Options:
"pkg..." also names every package pkg depends on).
Packages that fail preflight are refused with a
diagnostic. Eligible dependencies compile automatically.
--no-shape-overlay
compile npm packages' JavaScript without the types shape
inference proves (per package: "scriptc": {"packages":
{"<name>": {"shapes": false}}} in package.json)
--provenance-sources
EXPERIMENTAL: compile npm dependencies from their
provenance-attested SOURCE (fetched at the attested
@@ -180,6 +184,7 @@ export const CLI_OPTIONS = {
sanitize: { type: "boolean", default: false },
ffi: { type: "string" },
"npm-static": { type: "string", multiple: true },
"shape-overlay": { type: "boolean" },
"provenance-sources": { type: "boolean", default: false },
"external-types": { type: "string", multiple: true },
tier: { type: "string" },
+4
View File
@@ -67,6 +67,10 @@ export interface CompileBaseOptions {
/** Eligible packages compile automatically. Explicit names override automatic selection heuristics. */
npmStatic?: readonly string[] | "auto";
/** Shape inference's typed overlay for opted-in packages' JavaScript.
* False disables it (`--no-shape-overlay`); a package opts out with
* `"scriptc": { "packages": { "<name>": { "shapes": false } } }`. */
shapeOverlay?: boolean;
/** Outbound native FFI manifest. Its signature-only TypeScript bindings
* lower to direct C ABI calls. Source outputs retain those declarations;
* archive/system-library inputs join only an executable link. */
@@ -64,7 +64,10 @@
* parses date strings with V8's grammar. SC61xx are performance
* notes, also warnings (coverage --fail-on=dynamic opts in):
* likely-hot functions that use untyped values (SC6101,
* diagnostics/dynamic-tier.ts)
* diagnostics/dynamic-tier.ts), package files whose projected
* declaration types were dropped (SC6102), and shape inference's
* overlay report: what it typed and how many inferred facts the
* build checks (SC6103, frontend/shape-inference)
* SC8xxx Node island policy: packages and modules outside the native
* subset that run (or, in report-only mode, would run) in a
* Node island — unknown project configuration (SC8004), a
@@ -165,6 +165,9 @@ export interface EarlyExecutableCacheOptions {
/** Omitted unless SCRIPTC_YOUNG_NURSERY=1 (backend/young-nursery.ts). */
youngNursery?: true;
npmStatic: readonly string[] | "auto" | null;
/** Omitted unless shape inference's overlay compiles the packages
* (frontend/shape-inference). */
shapeOverlay?: true;
/** Raw manifest identity: path and bytes. Native archives remain under the
* stricter native cache's independent dependency validation. */
ffiProfile: { path: string; bytes: Uint8Array } | null;
+156
View File
@@ -0,0 +1,156 @@
/* Canonical mapping (localized-failure design §4.3), stage N1: the IR type
* of a canonical node, built from about ten structural rules plus the
* nominal table. Nominal objects (classes, standard-library interfaces,
* handles), callables, index-signature shapes, tuples, bigint and symbol
* keep their per-type recognitions: they map through mapType on the
* checker type the node came from. Everything structural (primitives,
* literals, units, arrays, flat unions, plain records) maps here, from the
* canonical form alone.
*
* Shadow mode (SCRIPTC_CANON_SHADOW=1) compares this mapping with mapType
* for every successful top-level mapping and appends one JSON line per
* comparison to SCRIPTC_CANON_SHADOW_OUT. It maps into throwaway
* registries, so the program's own IR is untouched. */
import { appendFileSync } from "node:fs";
import { BOOL, DYN, F64, NULL_T, STRING, UNDEFINED_T, typeKey, type IrType } from "../ir/ir.js";
import { canonKey, Normalizer, type Canon } from "./canon.js";
import * as ts from "./ts7/adapter.js";
import {
mapType,
ShapeRegistry,
UnionRegistry,
withUndefinedArm,
type TypeMapperCtx,
} from "./type-mapper.js";
const MAX_DEPTH = 24;
/** The IR type of a canonical node, or null when no rule applies. */
export function mapCanonical(c: Canon, ctx: TypeMapperCtx, depth = 0): IrType | null {
if (depth > MAX_DEPTH) return null;
const viaChecker = (): IrType | null => (c.src === undefined ? null : mapType(c.src, ctx));
switch (c.k) {
case "prim":
return c.p === "number"
? F64
: c.p === "string"
? STRING
: c.p === "boolean"
? BOOL
: viaChecker();
case "lit":
return typeof c.value === "number" ? F64 : typeof c.value === "string" ? STRING : BOOL;
case "unit":
return c.u === "undefined" ? UNDEFINED_T : c.u === "null" ? NULL_T : viaChecker();
case "top":
return DYN;
case "param":
return null;
case "opaque":
case "tuple":
return viaChecker();
case "array": {
const elem = mapCanonical(c.elem, ctx, depth + 1);
return elem === null ? null : { kind: "array", elem };
}
case "union": {
const byKey = new Map<string, IrType>();
for (const arm of c.arms) {
const mapped = mapCanonical(arm, ctx, depth + 1);
if (mapped === null || mapped.kind === "void") return null;
if (mapped.kind === "union") {
const def = ctx.unions.get(mapped.unionId);
if (def === undefined) return null;
for (const a of def.arms) byKey.set(typeKey(a), a);
} else byKey.set(typeKey(mapped), mapped);
}
const arms = [...byKey.values()];
if (arms.length === 1) return arms[0]!;
if (arms.some((a) => a.kind === "dyn")) return DYN;
arms.sort((a, b) => (typeKey(a) < typeKey(b) ? -1 : 1));
return { kind: "union", unionId: ctx.unions.intern(arms) };
}
case "obj": {
if (c.nominal !== undefined || c.call().length > 0 || c.construct().length > 0)
return viaChecker();
if (c.index().length > 0) return viaChecker();
const props = c.props();
// `{}` holds any non-nullish value: the checked dynamic, as mapType.
if (props.length === 0) return DYN;
const fields: { name: string; type: IrType }[] = [];
for (const p of props) {
let t = mapCanonical(p.type(), ctx, depth + 1);
if (t === null) return null;
if (p.optional && t.kind !== "dyn") {
const armed = withUndefinedArm(t, ctx.unions);
if (armed === null) return null;
t = armed;
}
fields.push({ name: p.name, type: t });
}
const declaredOrder = fields.map((f) => f.name);
fields.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
return {
kind: "record",
shapeId: ctx.shapes.intern(fields, false, undefined, declaredOrder),
};
}
}
}
export type ShadowStatus = "equal" | "diverge" | "canon-fail";
/** Installs nothing unless SCRIPTC_CANON_SHADOW=1: the hook mapType calls
* after each successful top-level mapping. */
export function canonShadowHook(
checker: ts.TypeChecker,
): ((type: ts.Type, mapped: IrType, ctx: TypeMapperCtx) => void) | undefined {
if (process.env["SCRIPTC_CANON_SHADOW"] !== "1") return undefined;
const out = process.env["SCRIPTC_CANON_SHADOW_OUT"];
let busy = false;
return (type, mapped, ctx) => {
if (busy) return;
busy = true;
// The shadow records and must never break a compile.
try {
const resolve = ctx.resolveTypeParamTs;
const normalizer = new Normalizer(checker);
const c = normalizer.canon(type, resolve ? (t) => resolve(t) : new Map());
const { typeMemo: _memo, canonShadow: _hook, ...rest } = ctx;
const shadowCtx: TypeMapperCtx = {
...rest,
shapes: new ShapeRegistry(),
unions: new UnionRegistry(),
canMemoizeType: () => false,
};
let canonical: IrType | null = null;
try {
canonical = mapCanonical(c, shadowCtx);
} catch {
canonical = null;
}
const status: ShadowStatus =
canonical === null
? "canon-fail"
: typeKey(canonical) === typeKey(mapped)
? "equal"
: "diverge";
if (out !== undefined)
appendFileSync(
out,
`${JSON.stringify({
status,
kind: c.k,
type: checker.typeToString(type).slice(0, 160),
canon: canonKey(c).slice(0, 160),
strict: typeKey(mapped),
...(canonical !== null ? { canonical: typeKey(canonical) } : {}),
})}\n`,
);
} catch {
// A census line lost, nothing else.
} finally {
busy = false;
}
};
}
+48 -8
View File
@@ -44,7 +44,13 @@ export interface AliasRef {
tsType: ts.Type;
}
export type Canon =
export type Canon = CanonNode & {
/** The checker type the node was normalized from (a type parameter's
* binding for substituted parameters). */
src?: ts.Type;
};
type CanonNode =
| { k: "prim"; p: CanonPrim }
| { k: "lit"; value: string | number | boolean }
| { k: "unit"; u: "undefined" | "null" | "void" | "never" }
@@ -67,8 +73,10 @@ export type Canon =
| { k: "param"; name: string }
| { k: "opaque"; ts: ts.Type; reason: string };
/** Type-parameter bindings: the checker type a parameter is bound to. */
export type CanonEnv = ReadonlyMap<ts.Symbol, ts.Type>;
/** Type-parameter bindings: the checker type a parameter is bound to, by
* the parameter's symbol, or a resolver over the parameter type (the
* lowerer's resolveTypeParamTs). */
export type CanonEnv = ReadonlyMap<ts.Symbol, ts.Type> | ((param: ts.Type) => ts.Type | null);
const PRIM_FLAGS: readonly [number, CanonPrim][] = [
[ts.TypeFlags.Number, "number"],
@@ -78,9 +86,23 @@ const PRIM_FLAGS: readonly [number, CanonPrim][] = [
[ts.TypeFlags.ESSymbol | ts.TypeFlags.UniqueESSymbol, "symbol"],
];
/** A stable key for a canonical node, for deduplication and tests. */
export function canonKey(c: Canon, depth = 0): string {
if (depth > 6) return "…";
/** A stable key for a canonical node, for deduplication and tests. Deep
* and wide types are cut off (depth 6, about 4,000 characters), so equal
* keys mean equal up to that horizon. */
export function canonKey(c: Canon): string {
const budget = { left: 4000 };
return canonKeyAt(c, 0, budget);
}
function canonKeyAt(c: Canon, depth: number, budget: { left: number }): string {
const key = canonKeyInner(c, depth, budget);
budget.left -= key.length;
return key;
}
function canonKeyInner(c: Canon, depth: number, budget: { left: number }): string {
if (depth > 6 || budget.left <= 0) return "…";
const canonKey = (n: Canon, d: number) => canonKeyAt(n, d, budget);
switch (c.k) {
case "prim":
return c.p;
@@ -259,11 +281,15 @@ export class Normalizer {
if (known !== undefined) return known;
}
const node = this.build(type, env, key);
if (node.src === undefined && node.k !== "param") node.src = type;
if (key !== null && !this.interned.has(key)) this.interned.set(key, node);
return node;
}
private envKey(env: CanonEnv): string {
// A resolver's bindings are the caller's current instantiation: a
// normalizer is used for one instantiation at a time.
if (typeof env === "function") return "fn";
if (env.size === 0) return "";
const parts: string[] = [];
for (const [param, bound] of env) {
@@ -294,7 +320,12 @@ export class Normalizer {
for (const [flag, p] of PRIM_FLAGS) if (flags & flag) return { k: "prim", p };
if (flags & ts.TypeFlags.TypeParameter) {
const symbol = type.getSymbol();
const bound = symbol === undefined ? undefined : env.get(symbol);
const bound =
typeof env === "function"
? (env(type) ?? undefined)
: symbol === undefined
? undefined
: env.get(symbol);
if (bound !== undefined && bound !== type) return this.canon(bound, env);
return { k: "param", name: symbol === undefined ? "?" : ts.symbolName(symbol) };
}
@@ -361,7 +392,8 @@ export class Normalizer {
return arms.length === 1 ? arms[0]! : { k: "union", arms };
}
const prims = parts.filter((p) => p.k === "prim" || p.k === "lit");
if (prims.length === 1 && parts.every((p) => p === prims[0] || p.k === "obj")) return prims[0]!;
if (prims.length === 1 && parts.every((p) => p.k === "prim" || p.k === "lit" || p.k === "obj"))
return prims[0]!;
if (parts.every((p) => p.k === "obj" || (p.k === "top" && p.t === "object")))
return this.object(type, env, key);
return { k: "opaque", ts: type, reason: "intersection" };
@@ -406,6 +438,14 @@ export class Normalizer {
});
return node;
}
// A mapped type the checker could not resolve (over an unbound or
// symbolically bound parameter) shows no members: irreducible here.
if (
((type as ts.ObjectType).objectFlags & ts.ObjectFlags.Mapped) !== 0 &&
checker.getPropertiesOfType(type).length === 0 &&
checker.getIndexInfosOfType(type).length === 0
)
return { k: "opaque", ts: type, reason: "unresolved mapped type" };
const symbol = type.getSymbol();
const decl = symbol?.declarations[0];
const nominal =
@@ -7,13 +7,27 @@ export { tsgoPath } from "./ts7/session-path.js";
const require = createRequire(import.meta.url);
let installedDeclarationRoot: string | null = null;
/** The paths `declarationPath` answered, by name, for the current root.
* Lowering asks for them once per mapped type (isFallbackDtsPath), and each
* answer without an installed root costs a package resolution. */
const declarationPaths = new Map<string, string>();
/** A native distribution supplies its relocatable declaration directory. */
export function setDeclarationRoot(root: string): void {
installedDeclarationRoot = root;
declarationPaths.clear();
}
function declarationPath(name: string): string {
let path = declarationPaths.get(name);
if (path === undefined) {
path = resolveDeclarationPath(name);
declarationPaths.set(name, path);
}
return path;
}
function resolveDeclarationPath(name: string): string {
return tsgoPath(
installedDeclarationRoot === null
? name === "scriptc.d.ts"
@@ -316,3 +316,41 @@ test("a file changed shortly before the read is hashed on every check", async ()
expect(frontendInputsStillMatch(snapshot)).toBe(true);
expect(reads.mock.calls.filter(([path]) => path === source)).toHaveLength(1);
});
test("a read before and a read after a file settles record one probe", async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-"));
scratch.push(dir);
const source = join(dir, "entry.ts");
await writeFile(source, "export const answer = 1;\n");
const tracker = new FrontendInputTracker();
// The first read comes right after the write, the second once the file
// has been unchanged long enough for its identity to stand for it.
tracker.run(() => trackedReadFile(source));
vi.useFakeTimers({ toFake: ["Date"] });
vi.setSystemTime(Date.now() + 5000);
try {
tracker.run(() => trackedReadFile(source));
} finally {
vi.useRealTimers();
}
const snapshot = tracker.snapshot();
expect(snapshot.stable).toBe(true);
const probe = snapshot.probes.find((entry) => entry.op === "file");
expect(probe?.op === "file" && probe.identity !== undefined).toBe(true);
expect(frontendInputsStillMatch(snapshot)).toBe(true);
// The identity still invalidates the snapshot on any write.
await writeFile(source, "export const answer = 2;\n");
expect(frontendInputsStillMatch(snapshot)).toBe(false);
});
test("two reads of a file that saw different content prevent cache publication", async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-"));
scratch.push(dir);
const source = join(dir, "entry.ts");
await writeFile(source, "export const answer = 1;\n");
const tracker = new FrontendInputTracker();
tracker.run(() => trackedReadFile(source));
await writeFile(source, "export const answer = 2;\n");
tracker.run(() => trackedReadFile(source));
expect(tracker.snapshot().stable).toBe(false);
});
@@ -84,6 +84,22 @@ function readWithIdentity(path: string): { text: string; identity: string | null
return { text, identity: settled && after === identity ? identity : null };
}
/** Two reads of one file that saw the same content, as one probe: a read
* before the file had been unchanged for IDENTITY_SETTLE_MS records no
* identity, and a later read of the same bytes does. The probe keeps the
* identity, which the settled read proved for that content. Null when the
* reads differ in anything else. */
function sameFileContent(
previous: FrontendInputProbe,
probe: FrontendInputProbe,
): FrontendInputProbe | null {
if (previous.op !== "file" || probe.op !== "file" || previous.digest !== probe.digest)
return null;
if (previous.identity === undefined) return probe;
if (probe.identity === undefined || probe.identity === previous.identity) return previous;
return null;
}
function pathKind(path: string): Extract<FrontendInputProbe, { op: "kind" }>["kind"] {
try {
const info = statSync(path);
@@ -142,12 +158,17 @@ export class FrontendInputTracker {
const key = `${probe.op}\0${probe.path}`;
if (probe.op === "text" && this.probes.has(key)) return;
const previous = this.probes.get(key);
if (previous !== undefined && JSON.stringify(previous) !== JSON.stringify(probe)) {
const merged = previous === undefined ? null : sameFileContent(previous, probe);
if (
merged === null &&
previous !== undefined &&
JSON.stringify(previous) !== JSON.stringify(probe)
) {
// An input changed while this frontend was running. The build may
// still finish normally, but it is not safe to publish as a cache key.
this.stable = false;
}
this.probes.set(key, probe);
this.probes.set(key, merged ?? probe);
if (probe.op === "file") {
// A successful content read supersedes earlier existence probes for the
// same path only when they also observed a file. A missing/non-file
@@ -1,3 +1,4 @@
import { shapeOverlayAnnotated } from "../shape-inference/index.js";
import { layoutReturnType } from "./structural-instances.js";
import {
typedArgumentsCandidate,
@@ -182,6 +183,12 @@ export function checkedJsDocumentedParameter(
lowerer: Lowerer,
param: ts.ParameterDeclaration,
): boolean {
if (
param.parent !== undefined &&
shapeOverlayAnnotated(param.parent) &&
documentedParameter(param)
)
return true;
return lowerer.checkedJsFile(ts.getSourceFileOfNode(param)) && documentedParameter(param);
}
@@ -45,6 +45,7 @@ const COMPLETION_CALLBACKS: ReadonlyMap<IrLibFn, CallbackPosition> = new Map<
["http.requestDynCb", "last"],
["http.requestUrlCb", "last"],
["net.connectCb", 2],
["net.connectPathCb", 2],
["net.listenCb", 2],
["net.listenOptsCb", 4],
["net.listenOptsReusePortCb", 5],
@@ -56,6 +56,7 @@ import { varRef } from "../../../ir/build.js";
import {
ambientUndefReadType,
ambientUndefinedFnSymbolOf,
ambientUndefinedVarSymbolOf,
contextualUndefReadType,
fenceEarlyAliasUse,
isTemporalGlobalRoot,
@@ -829,14 +830,7 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc:
// Stdlib/@types/node ambients never reach here (their chokepoint
// is above); only user-file declares do.
{
const sym = lowerer.resolveValueSymbol(expr);
const decl = sym ? sym.declarations[0] : undefined;
if (
decl !== undefined &&
ts.isVariableDeclaration(decl) &&
decl.initializer === undefined &&
ts.getCombinedModifierFlags(decl) & ts.ModifierFlags.Ambient
) {
if (ambientUndefinedVarSymbolOf(lowerer, expr)) {
const declared = lowerer.mapTypeOf(lowerer.typeOf(expr));
if (declared && declared.kind !== "void") {
return {
@@ -436,6 +436,22 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc
loc,
};
}
// Plain data (records, arrays, strings, numbers, booleans, and unions
// of them) is never a promise: false after the operand's evaluation.
const neverPromise = (t: IrType): boolean =>
NEVER_PROMISE_KINDS.has(t.kind) ||
(t.kind === "union" && (lowerer.unions.get(t.unionId)?.arms ?? [t]).every(neverPromise));
if (neverPromise(left.type)) {
const no: IrExpr = { kind: "boolLit", value: false, type: BOOL, loc };
if (isSafeToDiscard(left)) return no;
return {
kind: "seqExpr",
stmts: [{ kind: "exprStmt", expr: left, loc }],
result: no,
type: BOOL,
loc,
};
}
lowerer.unsupported(
"SC1090",
expr,
@@ -2275,3 +2291,20 @@ function nodeDeclaredMember(lowerer: Lowerer, t: ts.Type, key: string): boolean
);
});
}
const NEVER_PROMISE_KINDS: ReadonlySet<IrType["kind"]> = new Set([
"record",
"array",
"string",
"f64",
"bool",
"bigint",
"date",
"map",
"set",
"regex",
"bytes",
"symbol",
"nullT",
"undefinedT",
]);
@@ -34,7 +34,11 @@ import { varRef } from "../../../ir/build.js";
import { isExecExceptionType, refuseExecExceptionCode } from "../builtins/errors.js";
import { InternalCompilerError } from "../../../errors.js";
import { TEMPORAL_GLOBAL } from "../../builtin-modules.js";
import { isTemporalGlobalRoot } from "../lower-namespaces.js";
import {
ambientUndefinedFnSymbolOf,
ambientUndefinedVarSymbolOf,
isTemporalGlobalRoot,
} from "../lower-namespaces.js";
import { isF64HandleTyped } from "../f64-handles.js";
export function lowerTypeofExpr(lowerer: Lowerer, expr: ts.TypeOfExpression, loc: SrcLoc): IrExpr {
@@ -220,6 +224,16 @@ export function lowerTypeofExpr(lowerer: Lowerer, expr: ts.TypeOfExpression, loc
if (isTemporalGlobalRoot(lowerer, tested, sym)) {
return { kind: "strLit", value: "undefined", type: STRING, loc };
}
// A name the program declares ambiently but nothing defines
// (`declare const Bun`, `declare function f(): void`): Node erases the
// declaration, so typeof answers "undefined" without reading it, where
// a read throws ReferenceError.
if (
ambientUndefinedVarSymbolOf(lowerer, tested) ||
ambientUndefinedFnSymbolOf(lowerer, tested)
) {
return { kind: "strLit", value: "undefined", type: STRING, loc };
}
// `const { inspect } = require("util")` / `import { inspect } from
// "node:util"`: a binding of a builtin module's exported function,
// class or object. JavaScript sources represent such a value without
@@ -1,3 +1,5 @@
import { shapeOverlayAnnotated } from "../shape-inference/index.js";
import { documentedParameter } from "./call-signatures.js";
import { staticNameBindings } from "./static-function-names.js";
import { positionTag } from "./structural-positions.js";
import { keyedHelperName } from "./stable-names.js";
@@ -1863,10 +1865,14 @@ export function implicitAnyParamSymbolsOf(
// The variadic-`arguments` form keeps its dynRest story whole.
if (bodyReadsArguments(decl)) return null;
let hasImplicitParams = false;
const overlayTyped = shapeOverlayAnnotated(decl);
const out = decl.parameters.nodes.map((param): ts.Symbol | null => {
if (!ts.isIdentifier(param.name)) return null;
if (param.dotDotDotToken || param.questionToken || param.initializer) return null;
if (param.name.text === "this") return null;
// A parameter typed by shape inference's overlay is a proven type,
// not an implicit type parameter.
if (overlayTyped && documentedParameter(param)) return null;
const t = lowerer.typeOf(param.name);
const broadFunction = lowerer.checker.typeToString(t) === "Function";
const broadArray = lowerer.checkerAnyArrayType(t);
@@ -1,4 +1,5 @@
import { lowerAsyncLocalStorageOptions } from "./builtins/async-context.js";
import { shapeOverlayAnnotated } from "../shape-inference/index.js";
import {
DATE_TIME_FIELD,
dateRootOf,
@@ -1181,7 +1182,14 @@ function jsFieldInitializerType(lowerer: Lowerer, type: IrType, expression: ts.E
)
return DYN;
}
if (type.kind === "array" || type.kind === "record") return DYN;
// Shape inference's overlay types a field as a record or array only when
// the analysis proves its values never reach code that could observe a
// copy, so those fields keep their native layout.
if (
(type.kind === "array" || type.kind === "record") &&
!shapeOverlayAnnotated(expression.parent?.parent ?? expression)
)
return DYN;
// JavaScript documentation cannot guarantee that a call or an indexed
// read returns a present number. Numeric fields receiving those values
// must retain undefined, so later arithmetic performs JS coercion (NaN).
@@ -3374,6 +3382,7 @@ export function collectClassShapeInner(
!functionLocalClass(decl) &&
ts.isIdentifier(member.name) &&
!lowerer.virtualJsMethods.has(member) &&
!lowerer.checkedJsOverrideMethods.has(member) &&
!lowerer.prototypeMethodAccesses.has(member.name.text) &&
inst === undefined &&
decl.typeParameters === undefined &&
@@ -7692,6 +7701,16 @@ export function collectVirtualJsMethods(lowerer: Lowerer, files: readonly ts.Sou
if (!ts.isIdentifier(member.name)) continue;
const property = lowerer.checker.getPropertyOfType(base, member.name.text);
for (const declaration of property ? property.declarations : []) {
if (
ts.isMethodDeclaration(declaration) &&
isJsSourceFile(ts.getSourceFileOfNode(declaration)) &&
checkedJsOverride(lowerer, declaration, member)
) {
// A checked override pair dispatches through the vtable,
// so neither side may become an implicit generic.
lowerer.checkedJsOverrideMethods.add(declaration);
lowerer.checkedJsOverrideMethods.add(member);
}
if (
ts.isMethodDeclaration(declaration) &&
isJsSourceFile(ts.getSourceFileOfNode(declaration)) &&
@@ -7778,7 +7797,7 @@ function checkedJsOverride(
): boolean {
return [base, member].every(
(method) =>
lowerer.checkedJsFile(ts.getSourceFileOfNode(method)) &&
(lowerer.checkedJsFile(ts.getSourceFileOfNode(method)) || shapeOverlayAnnotated(method)) &&
method.parameters.nodes.every(documentedParameter),
);
}
@@ -676,6 +676,30 @@ export function ambientUndefReadType(lowerer: Lowerer, node: ts.Node): IrType |
return null;
}
/** The symbol of a variable a program file declares ambiently without
* an initializer (`declare const Bun: ...`), which nothing defines: Node
* erases the declaration, so a read throws the catchable ReferenceError
* "<name> is not defined" and typeof answers "undefined". Null for other
* identifiers, including the standard library's declarations, which
* describe real runtime values. */
export function ambientUndefinedVarSymbolOf(
lowerer: Lowerer,
ident: ts.Identifier,
): ts.Symbol | null {
const sym = lowerer.resolveValueSymbol(ident);
const decl = sym ? sym.declarations[0] : undefined;
if (
sym === null ||
decl === undefined ||
lowerer.isStdlibSymbol(sym) ||
!ts.isVariableDeclaration(decl) ||
decl.initializer !== undefined ||
!(ts.getCombinedModifierFlags(decl) & ts.ModifierFlags.Ambient)
)
return null;
return sym;
}
/** The symbol behind `ident` when it names an ambient `declare function`
* NOTHING defines: every function declaration on the symbol is body-less
* and ambient in a USER program file (merge partners — the fundule's
@@ -804,6 +804,19 @@ export function lowerNetModuleCall(
`the supported form is ${bi.member}(port, host?, connectListener?)`,
);
}
// connect(path[, connectListener]): a string first argument names a
// Unix socket, or a localhost port when it reads as a non-negative
// number (Node's isPipeName, decided at run time).
if (lowerer.mapTypeOf(lowerer.typeOf(args[0]!))?.kind === "string") {
return lowerNetConnectPath(
lowerer,
expr,
bi.member,
lowerer.lowerExpr(args[0]!),
true,
args[1],
);
}
const port = lowerer.lowerExprExpecting(args[0]!, F64);
// The optional middle host: a 2-arg call's second argument is the
// host when string-typed, the connect listener when func-typed.
@@ -841,6 +854,47 @@ export function lowerNetModuleCall(
);
}
/** net.connect / net.createConnection of a Unix socket path (a string
* argument or the `path` option) with an optional connect listener. Node
* ignores any other argument between them for a path, so only a listener
* may follow. */
function lowerNetConnectPath(
lowerer: Lowerer,
expr: ts.CallExpression,
member: string,
path: IrExpr,
pipeNameRule: boolean,
listener: ts.Expression | undefined,
): IrExpr {
const loc = locOf(expr);
const rule: IrExpr = { kind: "boolLit", value: pipeNameRule, type: BOOL, loc };
if (expr.arguments.nodes.length > 2) {
lowerer.noLowering(
`${member} of a path with ${expr.arguments.nodes.length} arguments`,
expr,
`the supported form is ${member}(path[, connectListener])`,
);
}
if (listener === undefined)
return { kind: "libCall", fn: "net.connectPath", args: [path, rule], type: NETSOCKET_T, loc };
const { cb } = lowerCallbackArg(
lowerer,
listener,
"connect listeners",
0,
() => false,
"use ()",
[],
);
return {
kind: "libCall",
fn: "net.connectPathCb",
args: [path, rule, cb],
type: NETSOCKET_T,
loc,
};
}
/** True when `t` is the lookup-function shape the connect lowering
* accepts: (hostname: string, options: unknown, callback) => void where
* callback is (err: <union with a null arm>, addresses: { address:
@@ -929,6 +983,29 @@ function lowerNetConnectOptions(
}
}
}
// connect({ path }[, connectListener]): the Unix-socket form.
{
const keys = optsNode.properties.nodes.map((p) =>
(ts.isPropertyAssignment(p) || ts.isShorthandPropertyAssignment(p)) &&
(ts.isIdentifier(p.name) || ts.isStringLiteral(p.name))
? p.name.text
: null,
);
if (keys.length === 1 && keys[0] === "path") {
const prop = optsNode.properties.nodes[0]!;
const path = ts.isPropertyAssignment(prop)
? lowerer.lowerExpr(prop.initializer)
: lowerer.lowerShorthandValue(prop as ts.ShorthandPropertyAssignment);
if (path.type.kind !== "string") {
lowerer.noLowering(
`a ${member} 'path' option of '${lowerer.fmt(path.type)}' values`,
prop,
"the path is a string here",
);
}
return lowerNetConnectPath(lowerer, expr, member, path, false, args[1]);
}
}
let port: IrExpr | null = null;
let host: IrExpr | null = null;
let lookup: IrExpr | null = null;
@@ -1,3 +1,5 @@
import { markInferredFactChecks, shapeOverlayFileNames } from "../shape-inference/index.js";
import { npmStaticPackageOfPath } from "../npm-static.js";
import { globalMemberValue } from "./global-values.js";
import { unionNarrowFailureMessage } from "./union-narrow.js";
import {
@@ -7,6 +9,7 @@ import {
arrayRootOf,
lowerWithArraySpecies,
} from "./builtins/array-classes.js";
import { canonShadowHook } from "../canon-map.js";
import {
compactDeclOrigins,
inPackageFile,
@@ -3309,6 +3312,8 @@ export class Lowerer {
/** Inferred JS methods participating in an override chain keep a vtable
* ABI instead of call-site specialization. Filled before class collection. */
readonly virtualJsMethods = new Set<ts.MethodDeclaration>();
/** Methods of checked-JavaScript override pairs (vtable dispatch). */
readonly checkedJsOverrideMethods = new Set<ts.MethodDeclaration>();
/** Descendant writes must widen the original JS field before collecting
* any class layout; base and derived instances share its storage ABI. */
readonly inheritedJsFieldWrites = new Map<
@@ -4097,6 +4102,7 @@ export class Lowerer {
this.externalTypeSpecifiersByFile =
mode.externalTypeSpecifiersByFile ?? directExternalTypeSpecifiersByFile(this.externalTypes);
this.checker = program.getTypeChecker();
const shadow = canonShadowHook(this.checker);
this.typeCtx = {
checker: this.checker,
positionOf: (node) => this.positions.of(node),
@@ -4110,6 +4116,7 @@ export class Lowerer {
const id = (type as { id?: number }).id;
return id === undefined ? null : (this.degradedTypes.get(id)?.lenient ?? null);
},
...(shadow !== undefined ? { canonShadow: shadow } : {}),
isErasing: () => this.erasingDepth > 0,
genericClassInstance: (decl, ref, mapArgument) =>
this.genericClassInstanceType(decl, ref, mapArgument),
@@ -8714,6 +8721,7 @@ export class Lowerer {
// reference are untouched — byte-stability holds.
checkRequireBindingAccesses(this, functions);
refuseX509DynValues(this, functions);
markInferredFactChecks(functions, this.program);
const module: IrModule | null =
this.diags.length > 0
? null
@@ -8729,6 +8737,13 @@ export class Lowerer {
...(this.usesWorkers ? { workers: true } : {}),
...(this.usesLifetimeRegions ? { lifetimeRegions: true as const } : {}),
...(this.declBoundary ? { declBoundary: true as const } : {}),
...(shapeOverlayFileNames().length > 0
? {
inferredFactFiles: shapeOverlayFileNames()
.map((file) => ({ file, pkg: npmStaticPackageOfPath(file) ?? "" }))
.sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0)),
}
: {}),
...(this.foreignElementKeys.size > 0
? { foreignElementKeys: [...this.foreignElementKeys].sort() }
: {}),
@@ -1,12 +1,14 @@
import * as ts from "../ts7/adapter.js";
import type { Lowerer } from "./lowerer.js";
import { ambientUndefinedFnSymbolOf, ambientUndefinedVarSymbolOf } from "./lower-namespaces.js";
/** The answer `typeof <operand>` gives without evaluating anything, for
* the global-detection idioms (`typeof window < "u"`, `typeof global !==
* "undefined"`): a name nothing declares answers "undefined" (Node's
* answer for an undeclared global such as window or self, and typeof never
* throws for one), and the Node global object and process answer
* "object". Null for every other operand. */
* "undefined"`): a name nothing declares, or one the program declares
* ambiently and nothing defines, answers "undefined" (Node's answer for an
* undeclared global such as window or self, and typeof never throws for
* one), and the Node global object and process answer "object". Null for
* every other operand. */
export function staticTypeofAnswer(lowerer: Lowerer, operand: ts.Expression): string | null {
let e = operand;
while (ts.isParenthesizedExpression(e)) e = e.expression;
@@ -14,6 +16,10 @@ export function staticTypeofAnswer(lowerer: Lowerer, operand: ts.Expression): st
if (lowerer.checker.getSymbolAtLocation(e) === undefined) return "undefined";
for (const name of ["global", "globalThis", "process"])
if (lowerer.isStdlibGlobal(e, name)) return "object";
// A name the program declares ambiently but nothing defines (`declare
// const Bun`): undeclared at run time.
if (ambientUndefinedVarSymbolOf(lowerer, e) || ambientUndefinedFnSymbolOf(lowerer, e))
return "undefined";
return null;
}
@@ -205,14 +205,26 @@ export function packageShimEntries(): string[] {
let unloadedPackages: ReadonlySet<string> = new Set();
/** Reads the project's package-shim configuration: the `scriptc.packages`
* entries marked "unloaded" in the entry's nearest package.json
* (project-config.ts validates the rest). Called once per program load. */
* entries marked "unloaded", and those with `"shapes": false`, in the
* entry's nearest package.json (project-config.ts validates the rest).
* Called once per program load. */
export function configurePackageShims(entryPath: string): void {
const packages = new Set<string>();
const noShapes = new Set<string>();
for (const [name, setting] of readProjectConfig(entryPath).packages) {
if (setting.unloaded) packages.add(name);
if (!setting.shapes) noShapes.add(name);
}
unloadedPackages = packages;
shapelessPackages = noShapes;
}
let shapelessPackages: ReadonlySet<string> = new Set();
/** Packages the project compiles without shape inference's overlay
* (`"scriptc": { "packages": { "<name>": { "shapes": false } } }`). */
export function isShapelessPackage(name: string): boolean {
return shapelessPackages.has(name);
}
/** True when `--npm-static` leaves the package `name` out of its package
+6 -2
View File
@@ -1,3 +1,4 @@
import { shapeOverlayReport } from "./shape-inference/index.js";
/** Shared frontend orchestration. Hosts supply a program loader so Node
* and native callers use the same npm selection, preflight and lowering. */
import type { ScrDiagnostic } from "../diagnostics/diagnostic.js";
@@ -801,8 +802,11 @@ function loadFrontend(
});
// The declaration-typed boundary's notes (SC6102) join the
// performance warnings; they never fail a build.
const notes = finalLoad.declBoundaryNotes;
if (notes === undefined) return lowered;
const notes = [
...(finalLoad.declBoundaryNotes ?? []),
...shapeOverlayReport(finalLoad.entry.fileName),
];
if (notes.length === 0) return lowered;
return { ...lowered, performance: [...notes, ...(lowered.performance ?? [])] };
},
npmStatic: statuses,
+30 -1
View File
@@ -148,7 +148,12 @@ import {
} from "./decl-projection.js";
import { configureTypePatches, packageSourceText } from "./type-patches/layer.js";
import { resetDeclOriginVersions } from "./lowering/decl-origin.js";
import { reportShapes } from "./shape-inference/index.js";
import {
reportShapes,
resetShapeOverlay,
shapeOverlayEnabled,
shapeOverlayer,
} from "./shape-inference/index.js";
import { inferredJsDiagnosticSuppressed } from "./inferred-js-diagnostics.js";
import {
containsJsx,
@@ -884,6 +889,9 @@ function loadProgram7(
/** The declaration-typed boundary's projection (decl-projection.ts):
* answers rewritten user files for the lowering world, or null. */
projector?: (program: ts.Program) => DeclProjection | null,
/** Shape inference's typed overlay (shape-inference/overlay.ts):
* rewritten package JavaScript for the lowering world, or null. */
overlayer?: (program: ts.Program) => ReadonlyMap<string, string> | null,
): LoadResult & { disposeAll: () => void } {
const config = adoptProjectConfig7(host, entryPath);
const configuredPaths = config.options["paths"];
@@ -1036,6 +1044,26 @@ function loadProgram7(
}
}
}
// Shape inference's overlay: package files typed from proven facts. The
// tsc gate keeps reading the files as written.
// A load without the overlay must not see an earlier load's overlay
// files (the registry is per process).
resetShapeOverlay();
if (overlayer !== undefined && jsx.refusal === null) {
const overlay = overlayer(program);
if (overlay !== null && overlay.size > 0) {
const before = program;
for (const [file, text] of overlay) host.addVirtualFile(file, text);
program = host.createProgram([...programRoots, overridesDtsPath()], options);
if (tscWorld === undefined)
tscWorld = {
program: before,
diagnostics: ts.getPreEmitDiagnostics(before),
projectWorld: null,
};
else if (tscWorld.program !== before) before.dispose();
}
}
const entry = program.getSourceFile(entryPath);
if (!entry) throw new Error(`could not load ${entryPath}`);
const externalTypeSpecifiersByFile = externalTypeFileClosure7(program, externalTypes);
@@ -1303,6 +1331,7 @@ export function loadProgram(
externalTypes,
npmStaticPackages.length > 0,
projector,
npmStaticPackages.length > 0 && shapeOverlayEnabled() ? shapeOverlayer : undefined,
);
// Shape inference, stage S1: a report only (SCRIPTC_DEBUG_SHAPES).
const shapes = process.env["SCRIPTC_DEBUG_SHAPES"];
@@ -70,6 +70,8 @@ export interface LiteralSite {
readonly kind: "object" | "array";
/** Static keys in creation order; null when a spread or computed key makes them unknown. */
readonly keys: string[] | null;
/** The literal's own static keys (without spread contributions). */
readonly ownKeys: string[];
readonly hasAccessor: boolean;
readonly hasProto: boolean;
readonly hasComputedKey: boolean;
@@ -129,6 +131,17 @@ export interface ShapeAnalysis {
readonly escapedFunctions: ReadonlySet<FunctionInfo>;
/** ES5 constructors: each function's default prototype object. */
readonly protoOf: ReadonlyMap<Atom, Atom>;
/** ES5 constructor instances: the function atom whose `new` made each. */
readonly fnOfInstance: ReadonlyMap<Atom, Atom>;
/** Every value a declaration (parameter, variable, binding element) holds,
* merged over the analysis contexts it was analyzed in. */
declValues(decl: Node): Set<Atom>;
/** The classes a class declaration's chain contains, base last. */
classChain(decl: Node): Node[];
/** `if` statements in walked code whose condition is known: the branch
* whose condition values never allowed it is absent from `walkedBranches`. */
readonly branchIfs: ReadonlySet<ts.IfStatement>;
readonly walkedBranches: ReadonlySet<Node>;
}
/** Array.prototype members: other names on an array are own properties. */
@@ -238,6 +251,47 @@ function isLoop(node: Node): boolean {
);
}
/** Whether control can reach the end of `stmts`. False only when every
* path ends in a return or throw, or in a `while (true)`, `do … while
* (true)` or `for (;;)` loop with no break inside; anything else is
* assumed to complete. */
function canComplete(stmts: readonly Node[]): boolean {
return stmts.every(canCompleteStatement);
}
function canCompleteStatement(s: Node): boolean {
if (ts.isReturnStatement(s) || ts.isThrowStatement(s)) return false;
if (ts.isBlock(s)) return canComplete(list(s.statements));
if (ts.isIfStatement(s))
return (
s.elseStatement === undefined ||
canCompleteStatement(s.thenStatement) ||
canCompleteStatement(s.elseStatement)
);
if (ts.isTryStatement(s)) {
if (s.finallyBlock !== undefined && !canCompleteStatement(s.finallyBlock)) return false;
return (
canCompleteStatement(s.tryBlock) ||
(s.catchClause !== undefined && canCompleteStatement(s.catchClause.block))
);
}
const endless =
((ts.isWhileStatement(s) || ts.isDoStatement(s)) &&
s.expression.kind === ts.SyntaxKind.TrueKeyword) ||
(ts.isForStatement(s) && s.condition === undefined);
return !endless || containsBreak(s);
}
/** Whether a break statement occurs in `node`'s body, outside nested
* functions. */
function containsBreak(node: Node): boolean {
return (
ts.forEachChild(node, (c) =>
ts.isBreakStatement(c) || (!ts.isFunctionLike(c) && containsBreak(c)) ? true : undefined,
) === true
);
}
const LOOP_CALLBACK_METHODS: ReadonlySet<string> = new Set([
"map",
"forEach",
@@ -357,6 +411,11 @@ class Analyzer {
nodeLabel: this.nodeLabel,
escapedFunctions: this.escapedFunctionCalls,
protoOf: this.protoOf,
fnOfInstance: this.fnOfInstance,
declValues: (decl) => this.declValues(decl),
classChain: (decl) => this.classChain(decl),
branchIfs: this.branchIfs,
walkedBranches: this.walkedBranches,
};
}
@@ -449,6 +508,78 @@ class Analyzer {
return p;
}
/** A boolean literal: labeled `boolean` like any boolean, but its
* truthiness is known, so `if` statements on it walk one branch. */
boolLiteral(value: boolean): Atom {
return this.atoms.intern(`prim:boolean:${value}`, () => ({
kind: "prim",
label: "boolean",
literal: value,
}));
}
/** The analysis environment a deferred walk restores. */
private env(): WalkEnv {
return {
ctx: this.ctx,
ctxRoot: this.ctxRoot,
currentFn: this.currentFn,
loopDepth: this.loopDepth,
};
}
private inEnv(env: WalkEnv, walk: () => void): void {
const saved = this.env();
this.ctx = env.ctx;
this.ctxRoot = env.ctxRoot;
this.currentFn = env.currentFn;
this.loopDepth = env.loopDepth;
try {
walk();
} finally {
this.ctx = saved.ctx;
this.ctxRoot = saved.ctxRoot;
this.currentFn = saved.currentFn;
this.loopDepth = saved.loopDepth;
}
}
readonly branchIfs = new Set<ts.IfStatement>();
readonly walkedBranches = new Set<Node>();
/** Ifs whose branches wait for condition values; at the fixpoint, an
* `if` whose condition node stayed empty walks every waiting branch. */
private readonly pendingIfs: PendingIf[] = [];
/** An `if` walks a branch once a condition value allows it: the then
* branch on a value that may be truthy, the else branch on one that may
* be falsy. A branch no value allows is never walked, so its effects
* (returns, calls, writes) stay out of the facts; the overlay turns it
* into a checked internal error. */
private ifStatement(s: ts.IfStatement): void {
const cond = this.expr(s.expression);
this.branchIfs.add(s);
const pending: PendingIf = {
s,
cond,
env: this.env(),
thenDone: false,
elseDone: s.elseStatement === undefined,
};
this.pendingIfs.push(pending);
this.on(cond, (a) => {
if (!pending.thenDone && mayBeTruthy(a)) this.walkBranch(pending, "then");
if (!pending.elseDone && mayBeFalsy(a)) this.walkBranch(pending, "else");
});
}
private walkBranch(pending: PendingIf, which: "then" | "else"): void {
const branch = which === "then" ? pending.s.thenStatement : pending.s.elseStatement!;
if (which === "then") pending.thenDone = true;
else pending.elseDone = true;
this.walkedBranches.add(branch);
this.inEnv(pending.env, () => this.stmt(branch));
}
builtin(path: string, receiver?: Atom): Atom {
const key = receiver === undefined ? `builtin:${path}` : `builtin:${path}@${receiver.id}`;
return this.atoms.intern(key, () => ({ kind: "builtin", label: path, path, receiver }));
@@ -717,6 +848,24 @@ class Analyzer {
return `${inside ? this.ctx : ""}|${this.nodeKey(node)}`;
}
private declIndex: Map<number, number[]> | null = null;
declValues(decl: Node): Set<Atom> {
if (this.declIndex === null) {
this.declIndex = new Map();
for (const [key, id] of this.bindings) {
const k = Number(key.slice(key.lastIndexOf("|") + 1));
let l = this.declIndex.get(k);
if (l === undefined) this.declIndex.set(k, (l = []));
l.push(id);
}
}
const out = new Set<Atom>();
const k = this.nodeKeys.get(decl);
if (k === undefined) return out;
for (const id of this.declIndex.get(k) ?? []) for (const a of this.nodes[id]!.atoms) out.add(a);
return out;
}
receiverCtx(atom: Atom): string {
return atom.kind === "instance" || atom.kind === "class" ? `C${atom.id}` : "";
}
@@ -812,8 +961,11 @@ class Analyzer {
}
const body = fn.body;
if (body !== undefined) {
if (ts.isBlock(body)) this.stmts(list(body.statements));
else this.edge(this.expr(body as ts.Expression), info.ret);
if (ts.isBlock(body)) {
this.stmts(list(body.statements));
// A call that runs off the end of the body returns undefined.
if (canComplete(list(body.statements))) this.add(info.ret, this.prim("undefined"));
} else this.edge(this.expr(body as ts.Expression), info.ret);
}
this.currentFn = saved;
this.loopDepth = savedLoop;
@@ -1147,8 +1299,9 @@ class Analyzer {
return;
}
if (ts.isReturnStatement(s)) {
if (s.expression !== undefined && this.currentFn !== null)
this.edge(this.expr(s.expression), this.currentFn.ret);
if (this.currentFn === null) return;
if (s.expression !== undefined) this.edge(this.expr(s.expression), this.currentFn.ret);
else this.add(this.currentFn.ret, this.prim("undefined"));
return;
}
if (ts.isFunctionDeclaration(s)) {
@@ -1222,6 +1375,10 @@ class Analyzer {
this.expr(s.expression);
return;
}
if (ts.isIfStatement(s) && this.currentFn !== null) {
this.ifStatement(s);
return;
}
// Blocks, ifs, switches, labels: walk children, evaluating expressions.
ts.forEachChild(s, (child) => {
if (ts.isExpression(child)) this.expr(child);
@@ -1316,8 +1473,9 @@ class Analyzer {
case ts.SyntaxKind.BigIntLiteral:
return this.constNode(this.prim("bigint"));
case ts.SyntaxKind.TrueKeyword:
return this.constNode(this.boolLiteral(true));
case ts.SyntaxKind.FalseKeyword:
return this.constNode(this.prim("boolean"));
return this.constNode(this.boolLiteral(false));
case ts.SyntaxKind.NullKeyword:
return this.constNode(this.prim("null"));
case ts.SyntaxKind.RegularExpressionLiteral:
@@ -1749,6 +1907,7 @@ class Analyzer {
node: e,
kind: "object",
keys: keysKnown ? keys : null,
ownKeys: keys,
hasAccessor,
hasProto,
hasComputedKey,
@@ -1915,6 +2074,7 @@ class Analyzer {
node: e,
kind: "array",
keys: null,
ownKeys: [],
hasAccessor: false,
hasProto: hasHole,
hasComputedKey: false,
@@ -2558,7 +2718,11 @@ class Analyzer {
this.activate(info);
this.add(info.thisNode, inst);
this.bindArgs(info, args, spread);
this.edge(info.ret, dst);
// `new` ignores a primitive result (and the undefined of a
// body that runs off its end) and yields the instance.
this.on(info.ret, (r) => {
if (r.kind !== "prim") this.add(dst, r);
});
this.edgeFor(e, info);
}
return;
@@ -3205,6 +3369,21 @@ class Analyzer {
const before = this.steps;
for (const c of this.lateCopies) c();
this.solve();
// A condition with no values at the fixpoint is one the analysis
// could not evaluate: walk both of its branches.
let walked = false;
for (const p of this.pendingIfs) {
if (this.nodes[p.cond]!.atoms.size > 0) continue;
if (!p.thenDone) {
this.walkBranch(p, "then");
walked = true;
}
if (!p.elseDone) {
this.walkBranch(p, "else");
walked = true;
}
}
if (walked) continue;
if (this.steps === before || this.budgetExceeded) return;
}
}
@@ -3227,3 +3406,34 @@ function cjsExportName(decl: Node): string | null {
return decl.left.name.text;
return null;
}
interface WalkEnv {
readonly ctx: string;
readonly ctxRoot: Node | null;
readonly currentFn: FunctionInfo | null;
readonly loopDepth: number;
}
interface PendingIf {
readonly s: ts.IfStatement;
readonly cond: number;
readonly env: WalkEnv;
thenDone: boolean;
elseDone: boolean;
}
/** Whether a value of the atom may be truthy (only `null`, `undefined` and
* the literal `false` never are; ⊤ may be anything). */
function mayBeTruthy(a: Atom): boolean {
if (a.kind !== "prim") return true;
if (a.label === "null" || a.label === "undefined") return false;
return a.literal !== false;
}
/** Whether a value of the atom may be falsy (objects, arrays, functions,
* classes, promises and collections never are). */
function mayBeFalsy(a: Atom): boolean {
if (a.kind === "top") return true;
if (a.kind !== "prim") return false;
return a.literal !== true;
}
@@ -38,6 +38,8 @@ export interface Atom {
/** function atoms: the analysis context of the function's clone (a
* method's receiver class), "" for the context-insensitive default. */
readonly ctx?: string | undefined;
/** prim `boolean` atoms of a literal: its value. */
readonly literal?: boolean | undefined;
}
export class AtomTable {
@@ -1,7 +1,13 @@
/* Shape inference for untyped npm JavaScript
* (results/design-shape-inference/DESIGN.md). Stage S1: report only. */
* (results/design-shape-inference/DESIGN.md): the report (S1) and the typed
* overlay (overlay.ts). */
import type * as ts from "../ts7/adapter.js";
import { analyzeShapes } from "./analyze.js";
import type { IrFunction } from "../../ir/ir.js";
import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js";
import { npmStaticPackageOfPath } from "../npm-static.js";
import { isShapelessPackage } from "../package-shims.js";
import { analyzeShapes, isPackageJsFile } from "./analyze.js";
import { OVERLAY_MARK, planShapeOverlay } from "./overlay.js";
import { buildShapeReport } from "./report.js";
import { computeConstantGraphs, constantGraphReport } from "./constant-graphs.js";
@@ -10,7 +16,8 @@ export { buildShapeReport } from "./report.js";
/** SCRIPTC_DEBUG_SHAPES: `1` reports package code in hot functions,
* `verbose` every package site, `all` user files too, any other value only
* files whose path contains it. Writes to stderr; changes nothing else. */
* files whose path contains it. Writes to stderr; changes nothing else. The
* build cache does not key on the variable: use SCRIPTC_NO_CACHE=1. */
export function reportShapes(program: ts.Program, filter: string): void {
const start = performance.now();
const analysis = analyzeShapes(program, program.getTypeChecker());
@@ -22,3 +29,134 @@ export function reportShapes(program: ts.Program, filter: string): void {
const ms = (performance.now() - start).toFixed(0);
process.stderr.write(`[shapes] ${ms} ms\n${lines.map((l) => `[shapes] ${l}`).join("\n")}\n`);
}
/* The overlay's files for the current load: the lowering treats them as
* checked JavaScript (their JSDoc types are the analysis's proven facts). */
let overlayFiles: ReadonlySet<string> = new Set();
let overlayAnnotations = 0;
let overlayDeadBranches = 0;
export function isShapeOverlayFile(fileName: string): boolean {
return overlayFiles.has(fileName);
}
/** Whether a statement or declaration carries an overlay annotation. */
export function shapeOverlayAnnotated(node: ts.Node): boolean {
const sf = node.getSourceFile();
if (!overlayFiles.has(sf.fileName)) return false;
return sf.text.slice(node.pos, node.getStart(sf)).includes(OVERLAY_MARK);
}
export function shapeOverlayFileNames(): readonly string[] {
return [...overlayFiles];
}
export function shapeOverlayAnnotationCount(): number {
return overlayAnnotations;
}
export function resetShapeOverlay(): void {
overlayFiles = new Set();
overlayAnnotations = 0;
overlayDeadBranches = 0;
inferredFactChecks = 0;
}
/** Whether the overlay runs. While it is new it is opt-in
* (SCRIPTC_SHAPE_OVERLAY=1); `--no-shape-overlay` (the `shapeOverlay:
* false` build option) sets the variable to 0 for the build. */
export function shapeOverlayEnabled(): boolean {
return process.env["SCRIPTC_SHAPE_OVERLAY"] === "1";
}
/** loadProgram7's overlayer: analyze, plan, and register the overlay files. */
export function shapeOverlayer(program: ts.Program): ReadonlyMap<string, string> | null {
resetShapeOverlay();
const analysis = analyzeShapes(program, program.getTypeChecker());
if (analysis.budgetExceeded) return null;
const report = buildShapeReport(analysis, "1");
const packageFiles = analysis.files.filter((sf) => {
if (!isPackageJsFile(sf)) return false;
const pkg = npmStaticPackageOfPath(sf.fileName);
return pkg === null || !isShapelessPackage(pkg);
});
const overlay = planShapeOverlay(analysis, report.sites, packageFiles);
overlayFiles = new Set(overlay.files.keys());
overlayAnnotations = overlay.annotations;
overlayDeadBranches = overlay.deadBranches;
if (process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY"] !== undefined) {
for (const [file, text] of overlay.files)
process.stderr.write(`[shape-overlay] ${file}\n${text}\n`);
}
return overlay.files;
}
/** Marks the checked conversions lowered in overlay files: each checks an
* analysis-proven fact, so its failure names the site as an analysis bug. */
export function markInferredFactChecks(
functions: readonly IrFunction[],
program: ts.Program,
): number {
if (overlayFiles.size === 0) return 0;
let count = 0;
const lines = new Map<string, (pos: number) => number>();
const lineOf = (file: string, pos: number): number => {
let f = lines.get(file);
if (f === undefined) {
const sf = program.getSourceFile(file);
f = sf === undefined ? () => 0 : (p: number) => sf.getLineAndCharacterOfPosition(p).line + 1;
lines.set(file, f);
}
return f(pos);
};
const visit = (x: unknown): void => {
if (Array.isArray(x)) {
for (const v of x) visit(v);
return;
}
if (x === null || typeof x !== "object") return;
const node = x as {
kind?: unknown;
loc?: { file: string; start: number };
inferredSite?: string;
};
if (node.kind === "dynCheck" && node.loc !== undefined && overlayFiles.has(node.loc.file)) {
const pkg = npmStaticPackageOfPath(node.loc.file) ?? "";
const rel = node.loc.file.split("/node_modules/").pop() ?? node.loc.file;
node.inferredSite = `${pkg} ${rel}:${lineOf(node.loc.file, node.loc.start)}`;
count++;
}
for (const v of Object.values(x)) if (v !== null && typeof v === "object") visit(v);
};
for (const fn of functions) visit(fn.body);
inferredFactChecks = count;
return count;
}
let inferredFactChecks = 0;
/** Inferred-fact checks inserted by the last build (the build report). */
export function shapeInferredFactCheckCount(): number {
return inferredFactChecks;
}
/** SC6103, the overlay's build report: what the overlay typed and how many
* checks of inferred facts the build inserted. A note, never a failure. */
export function shapeOverlayReport(entryFile: string): ScrDiagnostic[] {
if (overlayFiles.size === 0) return [];
const packages = new Set([...overlayFiles].map((f) => npmStaticPackageOfPath(f) ?? f));
return [
{
code: "SC6103",
message:
`shape inference typed ${overlayAnnotations} declarations in ${overlayFiles.size} files of ${packages.size} ` +
`package${packages.size === 1 ? "" : "s"} (${[...packages].sort().join(", ")}); the build checks ` +
`${inferredFactChecks} inferred fact${inferredFactChecks === 1 ? "" : "s"} at run time` +
(overlayDeadBranches > 0
? ` and replaces ${overlayDeadBranches} branch${overlayDeadBranches === 1 ? "" : "es"} it proved unreachable with checks`
: ""),
loc: { file: entryFile, start: 0, end: 0 },
hint: "a failing check reports a scriptc analysis bug; --no-shape-overlay compiles without the overlay",
},
];
}
@@ -0,0 +1,32 @@
import { afterAll, expect, test } from "vitest";
import { closeSourceParser, parseSourceFile } from "../ts7/source-parser-node.js";
import { neutralizeDocTags, OVERLAY_MARK } from "./overlay.js";
afterAll(closeSourceParser);
test("neutralizeDocTags rewrites JSDoc comments and keeps literals", () => {
const source = [
"/** @param {number} a */",
"function f(a) {",
' const s = "/** @type {number} */";',
" const t = `/** @returns {string} ${a} */`;",
" const r = /\\/\\*\\* @param \\*\\//;",
" // /** @type {string} */",
` /** @type {string} ${OVERLAY_MARK} */ const u = s + t;`,
" return a / 2 /** @returns {number} */;",
"}",
].join("\n");
const out = neutralizeDocTags(parseSourceFile("overlay.js", source, "js"));
expect(out.length).toBe(source.length);
expect(out.split("\n")).toEqual([
"/** -param {number} a */",
"function f(a) {",
' const s = "/** @type {number} */";',
" const t = `/** @returns {string} ${a} */`;",
" const r = /\\/\\*\\* @param \\*\\//;",
" // /** @type {string} */",
` /** @type {string} ${OVERLAY_MARK} */ const u = s + t;`,
" return a / 2 /** -returns {number} */;",
"}",
]);
});
@@ -0,0 +1,774 @@
/* Shape inference: the typed overlay (results/design-shape-inference,
* maintainer decision 2026-10-23). The analysis's proven facts become JSDoc
* types in a rewritten copy of each package JavaScript file, and the
* lowering treats the rewritten file like a checked JavaScript file, so the
* static tier compiles the package's hot code from those types.
*
* Only proven facts are written: a position whose value set contains ⊤
* (anything from islands, JSON.parse, the network or the file system,
* reflection, globals, `process`, caught exceptions, or code the analysis
* cannot see) stays untyped. A check the lowering inserts where a value
* meets an inferred type therefore fails only when the analysis is wrong;
* such checks carry the inferred-fact failure message.
*
* The package's own JSDoc is a hint, not a declaration: its tags are
* neutralized in the rewritten file, so only the analysis's types remain.
*
* Every edit is inline: no line breaks are added or removed, so positions
* keep their lines. */
import * as ts from "../ts7/adapter.js";
import type { Atom } from "./domain.js";
import type { ShapeAnalysis } from "./analyze.js";
import type { SiteVerdict } from "./report.js";
export interface ShapeOverlay {
/** Rewritten text per package file. */
readonly files: Map<string, string>;
/** Annotations written, for the build report. */
readonly annotations: number;
/** Branches the analysis proved unreachable, replaced by checks. */
readonly deadBranches: number;
}
interface Edit {
readonly pos: number;
readonly text: string;
/** Replaces pos..end instead of inserting at pos. */
readonly end?: number;
}
/** Marks the overlay's own comments (an unknown JSDoc tag, ignored). */
export const OVERLAY_MARK = "@scriptcShape";
const MARK = OVERLAY_MARK;
/** Above 1, values of several record key sets type as one merged record
* with optional fields (SCRIPTC_SHAPE_RECORD_ARMS; experimental). */
const MAX_RECORD_ARMS = Number(process.env.SCRIPTC_SHAPE_RECORD_ARMS ?? 1);
/** The most class or record arms an inferred union may have. */
const MAX_ARMS = 6;
function list<T extends ts.Node>(arr: { readonly nodes: readonly T[] } | undefined): readonly T[] {
return arr?.nodes ?? [];
}
let debugIndex = 0;
export function planShapeOverlay(
analysis: ShapeAnalysis,
verdicts: readonly SiteVerdict[],
packageFiles: readonly ts.SourceFile[],
): ShapeOverlay {
const pSites = new Map<Atom, SiteVerdict>();
const siteVerdicts = new Set<Atom>(verdicts.map((v) => v.site.atom));
// Records and arrays print only for sites whose objects never escape: a
// native record that reached untyped code would be seen as a copy.
const siteKeys = new Map<Atom, { keys: string[]; optional: Set<string> }>();
for (const v of verdicts) {
if (v.verdict !== "P-local") continue;
if (v.site.kind === "array") {
pSites.set(v.site.atom, v);
continue;
}
const keys = keysOfSite(analysis, v);
if (keys === null) continue;
siteKeys.set(v.site.atom, keys);
pSites.set(v.site.atom, v);
}
const files = new Map<string, string>();
let annotations = 0;
let deadBranches = 0;
debugIndex = 0;
// `if` branches no condition value allowed, per file: the overlay
// replaces each with a throwing check, so the checker and the lowering
// see the code the analysis saw.
const deadByFile = new Map<ts.SourceFile, ts.Statement[]>();
if (process.env["SCRIPTC_SHAPE_DEAD_BRANCHES"] !== "0")
for (const s of analysis.branchIfs) {
for (const branch of [s.thenStatement, s.elseStatement]) {
if (branch === undefined || analysis.walkedBranches.has(branch)) continue;
const sf = s.getSourceFile();
let list = deadByFile.get(sf);
if (list === undefined) deadByFile.set(sf, (list = []));
list.push(branch);
}
}
for (const sf of packageFiles) {
const typedefs = new Map<string, string>();
const typedefText: string[] = [];
const edits: Edit[] = [];
const className = (decl: ts.Node): string | null => {
if (decl.getSourceFile() !== sf) return null;
const own = (decl as ts.ClassLikeDeclaration).name;
if (own !== undefined && ts.isClassDeclaration(decl) && decl.parent === sf) return own.text;
const p = decl.parent;
if (
p !== undefined &&
ts.isVariableDeclaration(p) &&
ts.isIdentifier(p.name) &&
p.parent?.parent?.parent === sf
)
return p.name.text;
return null;
};
// `inner`: inside a record field, where ⊤ and unprintable values are `any`.
const typeOf = (atoms: ReadonlySet<Atom>, inner: boolean, depth: number): string | null => {
if (atoms.size === 0) return inner ? "any" : null;
const arms = new Set<string>();
let objects = 0;
// Every array arm becomes one Array<…> of the joined element values.
const elements = new Set<Atom>();
let hasArray = false;
const promised = new Set<Atom>();
let hasPromise = false;
const recordSites: Atom[] = [];
for (const a of atoms) {
let t: string | null = null;
switch (a.kind) {
case "prim":
t =
a.label === "null" ||
a.label === "undefined" ||
a.label === "string" ||
a.label === "number" ||
a.label === "boolean" ||
a.label === "bigint"
? a.label
: null;
break;
case "instance":
t = a.node !== undefined ? className(a.node) : null;
objects++;
break;
case "object": {
objects++;
const v = pSites.get(a);
if (v === undefined || depth >= 4 || !siteKeys.has(a)) return inner ? "any" : null;
recordSites.push(a);
continue;
}
case "array": {
objects++;
if (analysis.escaped.has(a) || (siteVerdicts.has(a) && !pSites.has(a)))
return inner ? "any" : null;
const el = analysis.elements.get(a);
if (el !== undefined) for (const x of analysis.nodes[el]!.atoms) elements.add(x);
hasArray = true;
continue;
}
case "promise": {
// One promise arm: the values it resolves to, joined.
const value = analysis.fields.get(a)?.get("value");
if (value !== undefined) for (const x of analysis.nodes[value]!.atoms) promised.add(x);
hasPromise = true;
continue;
}
default:
t = null;
}
if (t === null) return inner ? "any" : null;
arms.add(t);
}
// Records of one key set print as their typedef; records of several
// key sets merge into one record whose fields missing from some sites
// are optional (unions of overlapping records would need the lowering
// to pick an arm per value).
if (recordSites.length > 0) {
const keySets = new Set(recordSites.map(keySetOf));
if (keySets.size === 1) arms.add(recordType(recordSites[0]!, depth));
else if (MAX_RECORD_ARMS > 1) arms.add(mergedRecordType(recordSites, depth));
else return inner ? "any" : null;
}
if (hasArray) {
const inside = depth < 4 ? typeOf(elements, true, depth + 1) : "any";
arms.add(`Array<${inside ?? "any"}>`);
}
// A promise beside other values cannot cross into untyped code
// (maybe-promise unions stay typed-only), so it stays untyped here.
if (hasPromise) return inner ? "any" : null;
if (objects > MAX_ARMS) return inner ? "any" : null;
if (arms.has("any")) return inner ? "any" : null;
return [...arms].join(" | ");
};
// One typedef per key set: sites with the same keys intern to the same
// structural record, so their field types are joined.
const keySetOf = (site: Atom): string => {
const k = siteKeys.get(site)!;
return k.keys
.map((key) => (k.optional.has(key) ? `${key}?` : key))
.sort()
.join(",");
};
const sitesByKeys = new Map<string, Atom[]>();
for (const site of pSites.keys()) {
if (pSites.get(site)!.site.kind !== "object") continue;
const k = keySetOf(site);
let l = sitesByKeys.get(k);
if (l === undefined) sitesByKeys.set(k, (l = []));
l.push(site);
}
const recordType = (site: Atom, depth: number): string => {
const keySet = keySetOf(site);
const existing = typedefs.get(keySet);
if (existing !== undefined) return existing;
const name = `__shape_${typedefs.size}`;
typedefs.set(keySet, name);
const { keys, optional } = siteKeys.get(site)!;
const parts = keys.map((k) => {
const atoms = new Set<Atom>();
for (const other of sitesByKeys.get(keySet) ?? [site]) {
const id = analysis.fields.get(other)?.get(k);
if (id !== undefined) for (const a of analysis.nodes[id]!.atoms) atoms.add(a);
}
const t = typeOf(atoms, true, depth + 1);
return `${/^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k)}${optional.has(k) ? "?" : ""}: ${t ?? "any"}`;
});
typedefText.push(`/** @typedef {{${parts.join(", ")}}} ${name} ${MARK} */`);
return name;
};
const mergedRecordType = (sites: readonly Atom[], depth: number): string => {
const all = new Set<Atom>();
for (const site of sites)
for (const other of sitesByKeys.get(keySetOf(site)) ?? [site]) all.add(other);
const keys: string[] = [];
const optional = new Set<string>();
for (const site of all) {
const k = siteKeys.get(site)!;
for (const key of k.keys) {
if (!keys.includes(key)) keys.push(key);
if (k.optional.has(key)) optional.add(key);
}
}
for (const site of all) {
const own = siteKeys.get(site)!.keys;
for (const key of keys) if (!own.includes(key)) optional.add(key);
}
const keySet = `merged:${keys
.map((k) => (optional.has(k) ? `${k}?` : k))
.sort()
.join(",")}`;
const existing = typedefs.get(keySet);
if (existing !== undefined) return existing;
const name = `__shape_${typedefs.size}`;
typedefs.set(keySet, name);
const parts = keys.map((k) => {
const atoms = new Set<Atom>();
for (const site of all) {
const id = analysis.fields.get(site)?.get(k);
if (id !== undefined) for (const a of analysis.nodes[id]!.atoms) atoms.add(a);
}
const t = typeOf(atoms, true, depth + 1);
return `${/^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k)}${optional.has(k) ? "?" : ""}: ${t ?? "any"}`;
});
typedefText.push(`/** @typedef {{${parts.join(", ")}}} ${name} ${MARK} */`);
return name;
};
// Functions called with `new` yield their instance, whatever they return.
const constructed = new Set<ts.Node | undefined>(
[...analysis.fnOfInstance.values()].map((a) => a.node),
);
// Return values merged over every analysis context of each function.
const returns = new Map<ts.Node, Set<Atom>>();
for (const info of analysis.functions.values()) {
if (info.node.getSourceFile() !== sf || info.isAsync || info.isGenerator) continue;
let set = returns.get(info.node);
if (set === undefined) returns.set(info.node, (set = new Set()));
for (const a of analysis.nodes[info.ret]!.atoms) set.add(a);
}
// Receivers each method runs on, merged over its analysis contexts.
const thisAtoms = new Map<ts.Node, Set<Atom>>();
for (const info of analysis.functions.values()) {
if (info.node.getSourceFile() !== sf || !ts.isMethodDeclaration(info.node)) continue;
let set = thisAtoms.get(info.node);
if (set === undefined) thisAtoms.set(info.node, (set = new Set()));
for (const a of analysis.nodes[info.thisNode]!.atoms) set.add(a);
}
const declaresMember = (cls: ts.Node, name: string): boolean =>
analysis.classChain(cls).some((c) =>
(c as ts.ClassLikeDeclaration).members.nodes.some((m) => {
const n = (m as { name?: ts.Node }).name;
return (
n !== undefined && (ts.isIdentifier(n) || ts.isPrivateIdentifier(n)) && n.text === name
);
}),
);
// `this.m(…)` in an inherited method whose class does not declare `m`
// (zod's ZodType._parseSync calls the subclasses' _parse): the receiver
// is cast to the classes the method runs on, so the call dispatches over
// them instead of through an untyped call.
const receiverCasts = (method: ts.MethodDeclaration): void => {
const cls = method.parent;
const receivers = thisAtoms.get(method);
if (cls === undefined || receivers === undefined || receivers.size === 0) return;
if (![...receivers].every((a) => a.kind === "instance")) return;
const t = typeOf(receivers, false, 0);
if (t === null) return;
const scan = (node: ts.Node): void => {
if (ts.isFunctionLike(node) && !ts.isArrowFunction(node)) return;
if (
ts.isCallExpression(node) &&
ts.isPropertyAccessExpression(node.expression) &&
node.expression.expression.kind === ts.SyntaxKind.ThisKeyword &&
ts.isIdentifier(node.expression.name) &&
!declaresMember(cls, node.expression.name.text)
) {
const self = node.expression.expression;
edits.push({ pos: ts.tokenStartOfNode(self), text: `/** @type {${t}} ${MARK} */ (` });
edits.push({ pos: self.end, text: ")" });
annotations++;
}
ts.forEachChild(node, (c) => {
scan(c);
return undefined;
});
};
if (method.body !== undefined) for (const st of method.body.statements.nodes) scan(st);
};
// Same-named instance methods of the file's classes form a family: a
// call on a union of those classes needs one signature, so each member's
// parameters and result join the whole family's values.
const families = new Map<string, ts.MethodDeclaration[]>();
const familyVisit = (node: ts.Node): void => {
if (
ts.isMethodDeclaration(node) &&
ts.isIdentifier(node.name) &&
!list((node as { modifiers?: { nodes: readonly ts.Node[] } }).modifiers).some(
(m) => m.kind === ts.SyntaxKind.StaticKeyword,
)
) {
let l = families.get(node.name.text);
if (l === undefined) families.set(node.name.text, (l = []));
l.push(node);
}
ts.forEachChild(node, (c) => {
familyVisit(c);
return undefined;
});
};
familyVisit(sf);
// Members the program runs (they have analysis contexts); members it
// never reaches would only widen the family's types.
const reached = new Set<ts.Node>();
for (const info of analysis.functions.values())
if (info.node.getSourceFile() === sf) reached.add(info.node);
const familyOf = (fn: ts.FunctionLikeDeclaration): readonly ts.MethodDeclaration[] => {
if (!ts.isMethodDeclaration(fn) || !ts.isIdentifier(fn.name))
return [fn as ts.MethodDeclaration];
const members = (families.get(fn.name.text) ?? [fn]).filter((m) => reached.has(m));
return members.length > 0 ? members : [fn];
};
const paramValues = (fn: ts.FunctionLikeDeclaration, index: number): Set<Atom> => {
const out = new Set<Atom>();
for (const member of familyOf(fn)) {
const p = list(member.parameters)[index];
if (p === undefined) continue;
for (const a of analysis.declValues(p)) out.add(a);
}
return out;
};
const returnValues = (fn: ts.FunctionLikeDeclaration): Set<Atom> | undefined => {
let out: Set<Atom> | undefined;
for (const member of familyOf(fn)) {
const r = returns.get(member);
if (r === undefined) continue;
out ??= new Set();
for (const a of r) out.add(a);
}
return out;
};
const paramTags = (fn: ts.FunctionLikeDeclaration): string | null => {
const tags: string[] = [];
if (!ts.isConstructorDeclaration(fn) && !constructed.has(fn)) {
const ret = returnValues(fn);
const t = ret !== undefined && ret.size > 0 ? typeOf(ret, false, 0) : null;
const why = process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY_WHY"];
if (
why !== undefined &&
t === null &&
ret !== undefined &&
ts.isMethodDeclaration(fn) &&
ts.isIdentifier(fn.name) &&
fn.name.text === why
)
process.stderr.write(
`[shape-overlay-why] ${fn.name.text}: ${[...ret].map((a) => `${a.kind}:${a.label}${analysis.escaped.has(a) ? "(escaped)" : ""}`).join(" | ")}\n`,
);
if (t !== null) tags.push(`@returns {${t}}`);
}
list(fn.parameters).forEach((p, index) => {
if (
!ts.isIdentifier(p.name) ||
p.dotDotDotToken !== undefined ||
p.initializer !== undefined ||
p.questionToken !== undefined
)
return;
const t = typeOf(paramValues(fn, index), false, 0);
if (t !== null) tags.push(`@param {${t}} ${p.name.text}`);
});
return tags.length > 0 ? `${tags.join(" ")} ${MARK}` : null;
};
const visit = (node: ts.Node): void => {
if (ts.isMethodDeclaration(node)) receiverCasts(node);
if (
ts.isMethodDeclaration(node) ||
ts.isConstructorDeclaration(node) ||
ts.isFunctionDeclaration(node)
) {
if (node.body !== undefined) {
const tags = paramTags(node);
if (tags !== null) {
edits.push({ pos: ts.tokenStartOfNode(node), text: `/** ${tags} */ ` });
annotations++;
}
}
// `this.f = …` fields of a constructor's top-level statements.
if (ts.isConstructorDeclaration(node) && node.body !== undefined) {
const cls = node.parent;
const seen = new Set<string>();
for (const st of list(node.body.statements)) {
if (!ts.isExpressionStatement(st) || !ts.isBinaryExpression(st.expression)) continue;
const b = st.expression;
if (
b.operatorToken.kind !== ts.SyntaxKind.EqualsToken ||
!ts.isPropertyAccessExpression(b.left)
)
continue;
if (
b.left.expression.kind !== ts.SyntaxKind.ThisKeyword ||
!ts.isIdentifier(b.left.name)
)
continue;
const name = b.left.name.text;
if (seen.has(name)) continue;
seen.add(name);
const t = cls !== undefined ? fieldType(cls, name) : null;
if (t !== null) {
edits.push({ pos: ts.tokenStartOfNode(st), text: `/** @type {${t}} ${MARK} */ ` });
annotations++;
}
}
}
}
// Locals only: module-level bindings are initialized once (and a
// record-typed module constant can turn a return type into a union of
// overlapping records, which the lowering width-coerces).
if (
ts.isVariableStatement(node) &&
(node.parent !== sf || process.env["SCRIPTC_SHAPE_OVERLAY_MODULE_VARS"] === "1") &&
node.declarationList.declarations.nodes.length === 1
) {
const d = node.declarationList.declarations.nodes[0]!;
const init = d.initializer;
if (ts.isIdentifier(d.name) && init !== undefined && !literalInitializer(init)) {
const t = typeOf(analysis.declValues(d), false, 0);
if (t !== null) {
edits.push({ pos: ts.tokenStartOfNode(node), text: `/** @type {${t}} ${MARK} */ ` });
annotations++;
}
}
}
ts.forEachChild(node, (c) => {
visit(c);
return undefined;
});
};
const fieldType = (cls: ts.Node, name: string): string | null => {
const atoms = new Set<Atom>();
for (const a of analysis.atoms.atoms) {
if (
a.kind !== "instance" ||
a.node === undefined ||
!analysis.classChain(a.node).includes(cls)
)
continue;
const id = analysis.fields.get(a)?.get(name);
if (id !== undefined) for (const x of analysis.nodes[id]!.atoms) atoms.add(x);
}
return typeOf(atoms, false, 0);
};
visit(sf);
// Dead branches: outermost first; edits inside a replaced range drop.
const dead = (deadByFile.get(sf) ?? []).sort((a, b) => a.pos - b.pos);
const ranges: [number, number][] = [];
const removed = new Set<ts.Node>();
for (const branch of dead) {
const start = ts.tokenStartOfNode(branch);
if (ranges.some(([lo, hi]) => start >= lo && start < hi)) continue;
if (hoistedNames(branch, removed) === null) continue;
removed.add(branch);
ranges.push([start, branch.end]);
}
// A var that only removed branches declare stays declared in each.
const hoisted = new Map<number, readonly string[]>();
for (const branch of removed)
hoisted.set(ts.tokenStartOfNode(branch), hoistedNames(branch, removed)!);
if (ranges.length > 0) {
const kept = edits.filter((e) => !ranges.some(([lo, hi]) => e.pos > lo && e.pos < hi));
edits.length = 0;
edits.push(...kept);
const pkg = packageNameOf(sf.fileName);
const rel = sf.fileName.split("/node_modules/").pop() ?? sf.fileName;
for (const [lo, hi] of ranges) {
const line = sf.getLineAndCharacterOfPosition(lo).line + 1;
const message =
`scriptc internal error: shape inference proved the branch at ${rel}:${line} unreachable, but it ran. ` +
`This is a bug in scriptc's shape analysis, not in the program. Rebuild with --no-shape-overlay, ` +
`or set "scriptc": {"packages": {"${pkg}": {"shapes": false}}} in package.json.`;
// The branch's own var names stay declared in its scope, undefined.
const names = hoisted.get(lo)!;
const vars = names.length > 0 ? `var ${names.join(", ")}; ` : "";
edits.push({
pos: lo,
end: hi,
text: `{ ${vars}throw new Error(${JSON.stringify(message)}); }`,
});
deadBranches++;
}
}
// Debugging: SCRIPTC_SHAPE_OVERLAY_ONLY=a-b keeps the annotations
// numbered a..b (program order across files), to bisect a miscompile.
const only = process.env["SCRIPTC_SHAPE_OVERLAY_ONLY"];
if (only !== undefined) {
const [lo, hi] = only.split("-").map(Number) as [number, number];
edits.sort((a, b) => a.pos - b.pos);
const kept = edits.filter(() => {
const n = debugIndex++;
return n >= lo && n <= hi;
});
edits.length = 0;
edits.push(...kept);
}
if (edits.length === 0) continue;
files.set(sf.fileName, applyEdits(neutralizeDocTags(sf), edits) + typedefText.join(""));
}
return { files, annotations, deadBranches };
}
function packageNameOf(fileName: string): string {
const parts = (fileName.split("/node_modules/").pop() ?? "").split("/");
return parts[0]!.startsWith("@") ? `${parts[0]}/${parts[1]}` : parts[0]!;
}
function literalInitializer(e: ts.Expression): boolean {
return (
ts.isObjectLiteralExpression(e) ||
(ts.isArrayLiteralExpression(e) && e.elements.nodes.length > 0) ||
ts.isStringLiteral(e) ||
ts.isNumericLiteral(e) ||
ts.isNoSubstitutionTemplateLiteral(e) ||
ts.isArrowFunction(e) ||
ts.isFunctionExpression(e) ||
e.kind === ts.SyntaxKind.TrueKeyword ||
e.kind === ts.SyntaxKind.FalseKeyword
);
}
function applyEdits(text: string, edits: readonly Edit[]): string {
const sorted = [...edits].sort((a, b) => a.pos - b.pos);
let out = "";
let at = 0;
for (const e of sorted) {
out += text.slice(at, e.pos) + e.text;
at = e.end ?? e.pos;
}
return out + text.slice(at);
}
/** The file's text with the JSDoc type tags of its comments made plain
* text: package JSDoc is a hint, not a declaration. String, template and
* regular expression literals and JSX text are copied unchanged, so
* comment-like text in them keeps its runtime value. Lengths are unchanged,
* so edit positions in the source text stay valid. */
export function neutralizeDocTags(sf: ts.SourceFile): string {
const text = sf.text;
// Literal token ranges, in source order.
const literals: [number, number][] = [];
const visit = (node: ts.Node): void => {
if (ts.isJsxText(node)) {
literals.push([node.pos, node.end]);
return;
}
if (
ts.isStringLiteral(node) ||
ts.isNoSubstitutionTemplateLiteral(node) ||
ts.isTemplateHead(node) ||
ts.isTemplateMiddle(node) ||
ts.isTemplateTail(node) ||
ts.isRegularExpressionLiteral(node)
) {
literals.push([ts.tokenStartOfNode(node), node.end]);
return;
}
ts.forEachChild(node, (c) => {
visit(c);
return undefined;
});
};
visit(sf);
literals.sort((a, b) => a[0] - b[0]);
let out = "";
let copied = 0;
let next = 0;
let pos = 0;
// Outside literals, `//` and `/*` only start comments.
while (pos < text.length) {
while (next < literals.length && literals[next]![1] <= pos) next++;
const literal = literals[next];
if (literal !== undefined && literal[0] <= pos) {
pos = literal[1];
continue;
}
if (text.charCodeAt(pos) !== 47) {
pos++;
continue;
}
const second = text.charCodeAt(pos + 1);
if (second === 47) {
while (pos < text.length && !lineBreak(text.charCodeAt(pos))) pos++;
continue;
}
if (second !== 42) {
pos++;
continue;
}
const close = text.indexOf("*/", pos + 2);
const end = close < 0 ? text.length : close + 2;
const comment = text.slice(pos, end);
if (comment.startsWith("/**") && !comment.includes(MARK)) {
out += text.slice(copied, pos) + neutralizeComment(comment);
copied = end;
}
pos = end;
}
return out + text.slice(copied);
}
function lineBreak(code: number): boolean {
return code === 10 || code === 13 || code === 0x2028 || code === 0x2029;
}
function neutralizeComment(comment: string): string {
return comment.replace(
/@(param|type|typedef|returns?|template|this|callback|property|prop|arg|argument|enum|implements|extends|augments|satisfies|overload|import)\b/g,
"-$1",
);
}
/** The `var` names a dead branch declares that its function (or file)
* declares nowhere else outside the `removed` branches: replacing the
* branch must keep them bound, as undefined. Null when the branch declares
* a function in a block, whose binding outside the block depends on the
* file's strictness; such a branch is kept. */
function hoistedNames(branch: ts.Node, removed: ReadonlySet<ts.Node>): readonly string[] | null {
let scope: ts.Node | undefined = branch.parent;
while (scope !== undefined && !ts.isSourceFile(scope) && !functionScope(scope))
scope = scope.parent;
if (scope === undefined) return null;
const own = new Set<string>();
let blockFunction = false;
ts.walkPreorder(branch, (n) => {
if (n !== branch && ts.isFunctionDeclaration(n)) {
blockFunction = true;
return "skip";
}
if (n !== branch && (functionScope(n) || ts.isClassDeclaration(n) || ts.isClassExpression(n)))
return "skip";
if (ts.isVariableDeclarationList(n) && (n.flags & ts.NodeFlags.BlockScoped) === 0)
for (const d of n.declarations.nodes) addBindingNames(d.name, own);
return undefined;
});
if (blockFunction) return null;
if (own.size === 0) return [];
// Names the scope binds outside the branch: parameters, a function
// expression's name, other var declarations and function declarations.
const elsewhere = new Set<string>();
if (!ts.isSourceFile(scope)) {
for (const p of list(
(scope as { parameters?: { readonly nodes: readonly ts.ParameterDeclaration[] } }).parameters,
))
addBindingNames(p.name, elsewhere);
const name = (scope as { name?: ts.Node }).name;
if (name !== undefined && ts.isIdentifier(name)) elsewhere.add(name.text);
}
const body = ts.isSourceFile(scope) ? scope : (scope as { body?: ts.Node }).body;
if (body !== undefined)
ts.walkPreorder(body, (n) => {
if (n === branch || removed.has(n)) return "skip";
if (n !== body && ts.isFunctionDeclaration(n) && n.name !== undefined)
elsewhere.add(n.name.text);
if (n !== body && (functionScope(n) || ts.isClassDeclaration(n) || ts.isClassExpression(n)))
return "skip";
if (ts.isVariableDeclarationList(n) && (n.flags & ts.NodeFlags.BlockScoped) === 0)
for (const d of n.declarations.nodes) addBindingNames(d.name, elsewhere);
return undefined;
});
return [...own].filter((name) => !elsewhere.has(name));
}
function functionScope(n: ts.Node): boolean {
return (
ts.isFunctionDeclaration(n) ||
ts.isFunctionExpression(n) ||
ts.isArrowFunction(n) ||
ts.isMethodDeclaration(n) ||
ts.isGetAccessorDeclaration(n) ||
ts.isSetAccessorDeclaration(n) ||
ts.isConstructorDeclaration(n) ||
ts.isClassStaticBlockDeclaration(n)
);
}
function addBindingNames(name: ts.BindingName, out: Set<string>): void {
if (ts.isIdentifier(name)) {
out.add(name.text);
return;
}
for (const element of name.elements.nodes)
if (!ts.isOmittedExpression(element) && element.name !== undefined)
addBindingNames(element.name, out);
}
/** A record site's keys: its static keys, or for a literal with spreads its
* own keys plus every key a spread source contributes (optional unless
* every value of that source has it). Null when a computed key or an
* untracked spread source makes the key set unknown. */
function keysOfSite(
analysis: ShapeAnalysis,
v: SiteVerdict,
): { keys: string[]; optional: Set<string> } | null {
const site = v.site;
if (site.keys !== null) return { keys: site.keys, optional: new Set() };
if (site.hasComputedKey) return null;
const fields = analysis.fields.get(site.atom);
const any = fields?.get("*");
if (any !== undefined && analysis.nodes[any]!.atoms.size > 0) return null;
const required = new Set(site.ownKeys);
for (const spread of site.spreads) {
const sets: Set<string>[] = [];
for (const a of analysis.nodes[spread]!.atoms) {
if (a.kind === "prim" && (a.label === "undefined" || a.label === "null")) continue;
if (a.kind !== "object") return null;
const own = new Set<string>();
for (const k of analysis.fields.get(a)?.keys() ?? []) if (k !== "*") own.add(k);
sets.push(own);
}
const first = sets[0];
if (first === undefined) continue;
for (const k of first) if (sets.every((set) => set.has(k))) required.add(k);
}
const all = [...site.ownKeys];
for (const k of fields?.keys() ?? []) if (k !== "*" && !all.includes(k)) all.push(k);
return { keys: all, optional: new Set(all.filter((k) => !required.has(k))) };
}
@@ -1287,6 +1287,10 @@ export interface TypeMapperCtx {
* only after the strict mapping (and its bound-alias retry) returns null,
* at every depth, so every occurrence of a degraded type maps alike. */
degradedType?: (type: ts.Type) => IrType | null;
/** Canonical normalization's shadow mode (canon-map.ts): called after
* each successful top-level mapping; installed only under
* SCRIPTC_CANON_SHADOW=1. It records, and never changes a result. */
canonShadow?: (type: ts.Type, mapped: IrType, ctx: TypeMapperCtx) => void;
/** GENERIC program classes (monomorphization by flow): the mapping of a
* concrete instantiation reference (`Box<number>`) — the Lowerer
* registers/reuses the instantiation (`Box%0`) and answers its object
@@ -1841,6 +1845,17 @@ function arrayClassBaseType(
}
export function mapType(type: ts.Type, ctx: TypeMapperCtx): IrType | null {
const shadow = ctx.canonShadow;
if (shadow === undefined || mapTypeDepth !== 0) return mapTypeEntry(type, ctx);
// Shadow mode observes every successful top-level mapping, including the
// intrinsic primitives, local class values and memo hits that return
// before the mapping proper.
const mapped = mapTypeEntry(type, ctx);
if (mapped !== null) shadow(type, mapped, ctx);
return mapped;
}
function mapTypeEntry(type: ts.Type, ctx: TypeMapperCtx): IrType | null {
if (mapTypeDepth >= MAP_TYPE_MAX_DEPTH) return null;
// These intrinsic domains cannot depend on generic bindings, declaration
// provenance, or recursive shape state. Enum literals retain the ordinary
+12
View File
@@ -1,3 +1,4 @@
import { shapeOverlayEnabled } from "./frontend/shape-inference/index.js";
import { nodeFrontendServices } from "./frontend/services-node.js";
import {
runTypePatchCommand,
@@ -481,6 +482,10 @@ export async function compile(
opts: CompileRequestOptions,
): Promise<CompileRequestResult> {
clearCompileSessionCaches();
// `shapeOverlay: false` reaches the frontend (and its child process)
// through the environment the overlay reads.
const overlayEnv = process.env["SCRIPTC_SHAPE_OVERLAY"];
if (opts.shapeOverlay === false) process.env["SCRIPTC_SHAPE_OVERLAY"] = "0";
const frontendInputs = new FrontendInputTracker();
const warnings: CompileWarningSink = { warnings: [], sourceTexts: null };
// compileTracked starts it for a build with no earlier build of the same invocation.
@@ -493,6 +498,10 @@ export async function compile(
} finally {
setLibDiagnosticsStore(null);
frontendProcess?.release();
if (opts.shapeOverlay === false) {
if (overlayEnv === undefined) delete process.env["SCRIPTC_SHAPE_OVERLAY"];
else process.env["SCRIPTC_SHAPE_OVERLAY"] = overlayEnv;
}
}
}
@@ -1303,6 +1312,8 @@ function configureLibDiagnostics(root: string | null): void {
export async function runFrontendProcess(requestPath: string, resultPath: string): Promise<void> {
const request = JSON.parse(await readFile(requestPath, "utf8")) as FrontendProcessRequest;
// The process started before the build's options were known.
if (request.opts.shapeOverlay === false) process.env["SCRIPTC_SHAPE_OVERLAY"] = "0";
clearCompileSessionCaches();
setProvenanceSources(request.provenance);
// The compile that started this process prepared the cache root.
@@ -1650,6 +1661,7 @@ async function compileTracked(
...checkerField(),
...(youngNurseryRequested() ? { youngNursery: true as const } : {}),
npmStatic: opts.npmStatic ?? null,
...(shapeOverlayEnabled() ? { shapeOverlay: true as const } : {}),
ffiProfile:
opts.ffiProfilePath === undefined || ffiProfileBytes === null
? null
@@ -944,6 +944,8 @@ export const MAY_THROW_LIB_FNS: ReadonlySet<IrLibFn> = new Set([
"dns.setDefaultResultOrder",
"net.connect",
"net.connectCb",
"net.connectPath",
"net.connectPathCb",
// resetAndDestroy on a TLS socket throws ERR_INVALID_HANDLE_TYPE.
"net.sockResetAndDestroy",
// SocketAddress's option checks and parse()'s input check.
@@ -1554,6 +1554,15 @@ function buildLibFnSignatures() {
argTypes: [F64, STRING, { kind: "func", params: [], ret: VOID }],
result: NETSOCKET_T,
},
/** net.connect/createConnection(path[, connectListener]) and the
* `path` option: a client of the Unix socket at `path`. With the flag
* (a positional string), Node's isPipeName applies first: a string that
* reads as a non-negative number is a localhost port. */
"net.connectPath": { argTypes: [STRING, BOOL], result: NETSOCKET_T },
"net.connectPathCb": {
argTypes: [STRING, BOOL, { kind: "func", params: [], ret: VOID }],
result: NETSOCKET_T,
},
// The lookup's exact func shape is program data (its answer callback's
// union/record types) — checked specially in the libCall case.
/** connect({ port, host, autoSelectFamily: true, lookup }) — the
+27 -33
View File
@@ -8,6 +8,7 @@ import {
type IrStmt,
} from "./ir.js";
import { mapExprChildren, mapStmtChildren } from "./traverse.js";
import { type IrFieldWalker, walkStmtFields } from "./fields.generated.js";
type CharAt = IrExpr & { kind: "strIntrinsic" };
type UnitUse = IrExpr & { kind: "strEq" | "strCmp" };
@@ -302,26 +303,24 @@ function rewriteFunction(fn: IrFunction): IrFunction {
const copyOf = new Map<string, string>();
// Whether a candidate string is declared from a charAt.
let charAtDeclared = false;
const collect = (value: unknown): void => {
if (Array.isArray(value)) for (const item of value) collect(item);
else if (value !== null && typeof value === "object") {
const node = value as Record<string, unknown>;
if (node.kind === "varDecl" && candidates.has(node.localId as string) && node.init) {
const id = node.localId as string;
const init = node.init as IrExpr;
if (locals.get(id)?.type.kind === "string" && isCharAt(init)) charAtDeclared = true;
const read = optionalCharRead(init);
if (read) reads.set(id, read);
else if (init.kind === "varRef" && reads.has(init.localId)) {
reads.set(id, reads.get(init.localId)!);
copyOf.set(id, init.localId);
}
// The declarations, in the order a walk of every field finds them.
const collector: IrFieldWalker = {
node: (node) => {
if (node.kind !== "varDecl" || !candidates.has(node.localId) || !node.init) return;
const id = node.localId;
const init = node.init;
if (locals.get(id)?.type.kind === "string" && isCharAt(init)) charAtDeclared = true;
const read = optionalCharRead(init);
if (read) reads.set(id, read);
else if (init.kind === "varRef" && reads.has(init.localId)) {
reads.set(id, reads.get(init.localId)!);
copyOf.set(id, init.localId);
}
for (const key in node) if (key !== "loc") collect(node[key]);
}
},
string: () => {},
};
// Both walks only record candidates: a function without one holds none.
if (candidates.size > 0) collect(fn.body);
if (candidates.size > 0) for (const stmt of fn.body) walkStmtFields(stmt, collector, false);
const localTags = (operand: IrExpr): OptionalCharRead | null =>
operand.kind === "varRef" ? (reads.get(operand.localId) ?? null) : null;
@@ -333,24 +332,22 @@ function rewriteFunction(fn: IrFunction): IrFunction {
const note = (map: Map<string, number>, id: string): void => {
map.set(id, (map.get(id) ?? 0) + 1);
};
const count = (value: unknown): void => {
if (typeof value === "string") {
if (candidates.has(value)) note(mentions, value);
} else if (Array.isArray(value)) {
for (const item of value) count(item);
} else if (value !== null && typeof value === "object") {
const node = value as Record<string, unknown>;
const counter: IrFieldWalker = {
node: (node) => {
const id = understoodMention(node, locals, reads, localTags);
if (id !== null && candidates.has(id)) {
note(understood, id);
if (node.kind === "varDecl") note(declarations, id);
}
for (const key in node) if (key !== "loc") count(node[key]);
}
},
string: (value) => {
if (candidates.has(value)) note(mentions, value);
},
};
// Only candidates declared from a charAt or an optional read can be held
// (understoodMention), and count() only decides which are.
if (reads.size > 0 || charAtDeclared) count(fn.body);
// (understoodMention), and the count only decides which are.
if (reads.size > 0 || charAtDeclared)
for (const stmt of fn.body) walkStmtFields(stmt, counter, false);
// A copy is held only with its source, and the source's mention in the
// copy's initializer is understood only while the copy is held.
const held = new Set<string>();
@@ -492,14 +489,12 @@ function unitLocal(local: IrLocal): IrLocal {
* over one-unit cases; a union local's declaration from an optional read,
* or its test against a one-unit literal or for undefined. */
function understoodMention(
node: Record<string, unknown>,
n: IrExpr | IrStmt,
locals: ReadonlyMap<string, IrLocal>,
reads: ReadonlyMap<string, OptionalCharRead>,
localTags: (operand: IrExpr) => OptionalCharRead | null,
): string | null {
// count() visits every object in the body (types, locations, cases), so
// only the node kinds handled here are read as IR nodes
const kind = node.kind;
const kind = n.kind;
if (
kind !== "varDecl" &&
kind !== "strEq" &&
@@ -510,7 +505,6 @@ function understoodMention(
kind !== "unionIsTag"
)
return null;
const n = node as unknown as IrExpr | IrStmt;
if (n.kind === "varDecl") {
if (!n.init) return null;
const local = locals.get(n.localId);
File diff suppressed because it is too large Load Diff
+111
View File
@@ -0,0 +1,111 @@
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, test } from "vitest";
import { compile, deserializeModule } from "../index.js";
import type { IrExpr, IrStmt } from "./ir.js";
import { everyStmtList } from "./traverse.js";
import { eachStringField, walkStmtFields } from "./fields.generated.js";
// Programs with many node kinds: classes, closures, async, generators,
// dynamic values, streams and records.
const PROGRAMS = [
"1003-json-parse-unions.ts",
"114-live-record-delete.ts",
"1200-regex-test-basics.ts",
"1631-inspect-arrays.ts",
"4900-console-instances.ts",
"5612-readable-from-iterables.cjs",
"7110-als-completion-callbacks.ts",
"7321-speculative-loop-bail-sites.ts",
];
/** The string fields a key walk of `node` finds, as `field=value`. */
function keyWalk(node: IrExpr | IrStmt): string[] {
const found: string[] = [];
for (const key in node) {
const value = (node as unknown as Record<string, unknown>)[key];
if (typeof value === "string") found.push(`${key}=${value}`);
}
return found.sort();
}
/** The events of a recursive key walk of `value` that skips `loc` (and
* `type` with `skipType`): `n:<kind>` for an IR node (an object with a kind
* and a location; `n:varDecl:<local>` for a declaration), `s:<string>` for
* each string. */
function keyWalkDeep(value: unknown, events: string[], skipType: boolean): void {
if (typeof value === "string") events.push(`s:${value}`);
else if (Array.isArray(value)) for (const item of value) keyWalkDeep(item, events, skipType);
else if (value !== null && typeof value === "object") {
const record = value as Record<string, unknown>;
if ("kind" in record && "loc" in record) events.push(nodeEvent(record as unknown as IrExpr));
for (const key in record)
if (key !== "loc" && !(skipType && key === "type"))
keyWalkDeep(record[key], events, skipType);
}
}
function nodeEvent(node: IrExpr | IrStmt): string {
return node.kind === "varDecl" ? `n:varDecl:${node.localId}` : `n:${node.kind}`;
}
async function lowered(program: string, dir: string) {
const outPath = join(dir, "main.ir.json");
const entry = join(import.meta.dirname, "../../../../tests/corpus", program);
const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" });
if (!result.ok)
throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"));
return deserializeModule(await readFile(outPath, "utf8"));
}
// A key walk visits a node's fields in the order the lowering created them,
// which can differ from ir.ts's order between sibling fields (a dynKeyGet is
// built key first). The walk's callers count what they find, except that
// declarations must come in the same order.
test("walkStmtFields reports what a recursive key walk finds", async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-ir-walk-"));
try {
for (const program of PROGRAMS) {
const mod = await lowered(program, dir);
for (const skipType of [false, true])
for (const fn of mod.functions) {
const expected: string[] = [];
keyWalkDeep(fn.body, expected, skipType);
const events: string[] = [];
const walker = {
node: (node: IrExpr | IrStmt) => events.push(nodeEvent(node)),
string: (value: string) => events.push(`s:${value}`),
};
for (const stmt of fn.body) walkStmtFields(stmt, walker, skipType);
const where = `${program}: ${fn.name}, skipType ${skipType}`;
expect([...events].sort(), where).toEqual([...expected].sort());
const declarations = (list: string[]) => list.filter((e) => e.startsWith("n:varDecl:"));
expect(declarations(events), where).toEqual(declarations(expected));
}
}
} finally {
await rm(dir, { recursive: true, force: true });
}
}, 300_000);
test("eachStringField reads every string field a key walk of a node finds", async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-ir-fields-"));
try {
for (const program of PROGRAMS) {
const mod = await lowered(program, dir);
let nodes = 0;
const check = (node: IrExpr | IrStmt): boolean => {
const read: string[] = [];
eachStringField(node, (value, field) => read.push(`${field}=${value}`));
expect(read.sort(), `${program}: ${node.kind}`).toEqual(keyWalk(node));
nodes++;
return true;
};
for (const fn of mod.functions) everyStmtList(fn.body, { expr: check, stmt: check });
expect(nodes, program).toBeGreaterThan(0);
}
} finally {
await rm(dir, { recursive: true, force: true });
}
}, 300_000);
@@ -100,6 +100,7 @@ const EXPR_FIELDS: Classified<IrExpr> = {
unionIsTag: { unionId: "other" },
unionEq: { unionId: "other" },
jsonStringify: { indent: "other" },
dynCheck: { inferredSite: "other" },
};
const STMT_FIELDS: Classified<IrStmt> = {
@@ -225,6 +226,8 @@ const MODULE_FIELDS: Record<StringFields<Omit<IrModule, "functions">>, Role> = {
"threadSlotKeys[]": "other",
"foreignElementKeys[]": "other",
"freezableArrays[]": "other",
"inferredFactFiles[].file": "other",
"inferredFactFiles[].pkg": "other",
"declOrigins[]": "other",
};
+9
View File
@@ -1063,6 +1063,11 @@ export interface IrModule {
* closures boxed for package code return records and arrays as live
* capsules, so a value that comes back keeps its identity. */
declBoundary?: true;
/** Package files compiled from shape inference's typed overlay
* (frontend/shape-inference/overlay.ts), each with its package name: a
* checked conversion in them checks an analysis-proven fact, so its
* failure reports a compiler analysis bug instead of a type error. */
inferredFactFiles?: { file: string; pkg: string }[];
/** Origins of package-declared types checked by `dynCheck.origin`
* (frontend/lowering/decl-origin.ts): fields separated by \x1f —
* package, version, suggested range, selector, where it is declared,
@@ -4066,6 +4071,10 @@ export type IrExpr =
preserveRefs?: true;
adopt?: true;
exact?: true;
/** Shape inference's overlay: this conversion checks an
* analysis-proven fact ("<package> <file>:<line>"); a failure is a
* compiler analysis bug and reports itself as one. */
inferredSite?: string;
origin?: number;
type: IrType;
loc: SrcLoc;
+19 -14
View File
@@ -6,8 +6,10 @@ import {
type IrFunction,
type IrLocal,
type IrModule,
type IrParam,
type IrStmt,
} from "./ir.js";
import { type IrFieldWalker, walkStmtFields } from "./fields.generated.js";
import { everyExprChild, everyStmtChild, mapExprChildren, mapStmtChildren } from "./traverse.js";
type WhileLoop = IrStmt & { kind: "while" };
@@ -99,22 +101,20 @@ function continuesLoop(loop: WhileLoop): boolean {
return found;
}
/** Mentions of each id in `node`, in any field (a string literal equal to
* an id counts too, which only refuses more loops). */
/** Mentions of each id in `stmts`, in any field but types (a string
* literal equal to an id counts too, which only refuses more loops). */
function countMentions(
node: unknown,
stmts: readonly IrStmt[],
ids: ReadonlySet<string>,
mentions: Map<string, number>,
): void {
if (typeof node === "string") {
if (ids.has(node)) mentions.set(node, (mentions.get(node) ?? 0) + 1);
} else if (Array.isArray(node)) {
for (const item of node) countMentions(item, ids, mentions);
} else if (node !== null && typeof node === "object") {
const record = node as Record<string, unknown>;
for (const key in record)
if (key !== "loc" && key !== "type") countMentions(record[key], ids, mentions);
}
const walker: IrFieldWalker = {
node: () => {},
string: (value) => {
if (ids.has(value)) mentions.set(value, (mentions.get(value) ?? 0) + 1);
},
};
for (const stmt of stmts) walkStmtFields(stmt, walker, true);
}
/** A loop with the qualifying shape, the statements before it in its
@@ -184,7 +184,12 @@ function rewriteFunction(fn: IrFunction): IrFunction {
const params = new Set(fn.params.map((p) => p.localId));
// A counter this function reaches through its environment.
const outer = new Map<string, number>();
countMentions([fn.captures ?? [], fn.classCaptures ?? []], ids, outer);
const noteOuter = (param: IrParam): void => {
for (const value of [param.localId, param.name])
if (ids.has(value)) outer.set(value, (outer.get(value) ?? 0) + 1);
};
for (const param of fn.captures ?? []) noteOuter(param);
for (const param of fn.classCaptures ?? []) noteOuter(param);
let ranges: ReadonlyMap<IrExpr, IntegerRange | null> | null = null;
const used = new Set(fn.locals.map((l) => l.id));
@@ -192,7 +197,7 @@ function rewriteFunction(fn: IrFunction): IrFunction {
const rewritten = new Map<WhileLoop, Map<string, string>>();
for (const { loop, counters, before, nested } of candidates) {
const inside = new Map<string, number>();
countMentions(loop, new Set(counters), inside);
countMentions([loop], new Set(counters), inside);
// Each counter's start: its declaration's initializer, or null for a
// parameter.
const starts = new Map<string, IrExpr | null>();
+159 -70
View File
@@ -8,6 +8,78 @@ import type { RuntimePackManifest } from "../backend/runtime-pack-core.js";
import { NATIVE_TARGETS } from "../backend/targets.js";
import { buildSanitizedRuntime } from "./sanitizer.js";
/** Every runtime feature off except regex. */
const features = {
workers: false,
youngNursery: false,
eventTarget: false,
wasm: false,
dynTier: false,
streamDyn: false,
handleProps: false,
declAdopt: false,
typeOrigins: false,
dynForAwait: false,
eventsOnceSignal: false,
mime: false,
utilTiming: false,
dynBuffer: false,
builtinValues: false,
computedRequire: false,
warningReplace: false,
stdioExtended: false,
brotli: false,
moduleNamespace: false,
regex: true,
copying: false,
textDecoderLegacy: false,
fileHandle: false,
fetch: false,
zlib: false,
assert: false,
inspect: false,
dynInvoke: false,
dc: false,
dynAsync: false,
events: false,
emitter: false,
symbol: false,
bigint: false,
searchParams: false,
qs: false,
utilPromisify: false,
dynBuiltinClass: false,
dynBuiltinStream: false,
parseArgs: false,
stream: false,
net: false,
http: false,
http2: false,
dgram: false,
dnsPromises: false,
dgramDyn: false,
textCodecDyn: false,
watch: false,
foreignFfi: false,
nodeTest: false,
tls: false,
tlsCa: false,
cipher: false,
ecdh: false,
dh: false,
pki: false,
ed25519: false,
x509: false,
webcrypto: false,
perfLoop: false,
nativeFetch: false,
netEffective: false,
httpEffective: false,
tlsEffective: false,
tlsCaEffective: false,
zlibEffective: false,
};
test.skipIf(process.platform !== "linux" && process.platform !== "darwin")(
"sanitized vendor archives compile with host libc declarations",
() => {
@@ -72,76 +144,7 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")(
runtime: [],
archives: [artifact],
systemLibraries: [],
features: {
workers: false,
youngNursery: false,
eventTarget: false,
wasm: false,
dynTier: false,
streamDyn: false,
handleProps: false,
declAdopt: false,
typeOrigins: false,
dynForAwait: false,
eventsOnceSignal: false,
mime: false,
utilTiming: false,
dynBuffer: false,
builtinValues: false,
computedRequire: false,
warningReplace: false,
stdioExtended: false,
brotli: false,
moduleNamespace: false,
regex: true,
copying: false,
textDecoderLegacy: false,
fileHandle: false,
fetch: false,
zlib: false,
assert: false,
inspect: false,
dynInvoke: false,
dc: false,
dynAsync: false,
events: false,
emitter: false,
symbol: false,
bigint: false,
searchParams: false,
qs: false,
utilPromisify: false,
dynBuiltinClass: false,
dynBuiltinStream: false,
parseArgs: false,
stream: false,
net: false,
http: false,
http2: false,
dgram: false,
dnsPromises: false,
dgramDyn: false,
textCodecDyn: false,
watch: false,
foreignFfi: false,
nodeTest: false,
tls: false,
tlsCa: false,
cipher: false,
ecdh: false,
dh: false,
pki: false,
ed25519: false,
x509: false,
webcrypto: false,
perfLoop: false,
nativeFetch: false,
netEffective: false,
httpEffective: false,
tlsEffective: false,
tlsCaEffective: false,
zlibEffective: false,
},
features,
},
},
directory,
@@ -160,3 +163,89 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")(
}
},
);
test.skipIf(process.platform !== "linux" && process.platform !== "darwin")(
"variants that share an artifact file compile once, as the selected variant",
() => {
const target = NATIVE_TARGETS.find(
(item) =>
item.platform === process.platform &&
item.architecture === process.arch &&
!item.name.endsWith("-musl"),
);
if (target === undefined) throw new Error("unsupported sanitizer test host");
const runtimeSourceRoot = fileURLToPath(new URL("../../../runtime", import.meta.url));
const { version } = JSON.parse(
readFileSync(join(runtimeSourceRoot, "package.json"), "utf8"),
) as { version: string };
const directory = mkdtempSync("/tmp/scriptc-sanitized-shared-");
// The pack stores the identical objects of both variants once.
const shared = {
path: "artifacts/dev/runtime/default/scr_hash.o",
sha256: "0".repeat(64),
size: 0,
};
const variants = [
{ ...shared, id: "default", when: { workers: false }, defines: [] },
{ ...shared, id: "default-workers", when: { workers: true }, defines: ["SCR_WORKERS"] },
];
const manifest: RuntimePackManifest = {
schema: "scriptc.runtime-pack.v1",
format: 1,
package: target.runtimePackPackage,
version,
target: {
name: target.name,
llvm_triple: target.llvmTriple,
architecture: target.architecture,
object_format: target.objectFormat,
minimum_os: target.minimumOs,
},
runtime_abi: { version: RUNTIME_ABI_VERSION, marker: RUNTIME_ABI_MARKER },
compiler: { command: "clang", identity: "test", target: target.llvmTriple },
macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" },
flavors: {
dev: {
optimization: "-O0",
runtime_units: [{ source: "scr_hash.c", predicate: true, variants }],
},
},
archives: [],
system_libraries: [],
licenses: [],
};
try {
const built = buildSanitizedRuntime(
{
compilerVersion: version,
target,
helper: target.helper,
runtimeSourceRoot,
ts7Executable: "unused",
helperExecutable: "unused",
helperPackageRoot: "unused",
runtimePackRoot: "unused",
linker: "unused",
linkerArgs: [],
dsymutil: "unused",
},
{
root: directory,
manifest,
licenses: manifest.licenses,
packageText: "",
manifestText: "",
flavor: "dev",
selected: { runtime: [variants[0]!], archives: [], systemLibraries: [], features },
},
directory,
"executable",
null,
);
expect(built.runtimeObjects).toEqual([join(directory, "scr_hash.o")]);
expect(statSync(join(directory, "scr_hash.o")).size).toBeGreaterThan(0);
} finally {
rmSync(directory, { recursive: true, force: true });
}
},
);
+13 -14
View File
@@ -9,7 +9,11 @@ import {
type NativeRuntimePack,
type NativeRuntimeSelection,
} from "../backend/runtime-pack-native.js";
import { runtimePackFlavorKey, type RuntimePackMode } from "../backend/runtime-pack-core.js";
import {
runtimePackFlavorKey,
selectedRuntimeUnits,
type RuntimePackMode,
} from "../backend/runtime-pack-core.js";
import { LRE_SOURCES, ZLIB_SOURCES } from "../backend/vendor-inputs.js";
import { contentDigest, type NativeCache } from "./cache.js";
import type { NativeToolchain } from "./toolchain.js";
@@ -127,7 +131,6 @@ export function buildSanitizedRuntime(
const key = runtimePackFlavorKey(manifest, selection.flavor, mode);
const flavor = manifest.flavors[key];
if (flavor === undefined) throw new NativeCodegenError("SC3003", `runtime pack lacks ${key}`);
const wanted = new Set(selection.selected.runtime.map((artifact) => artifact.path));
const common = [
"-std=c11",
"-pthread",
@@ -147,18 +150,14 @@ export function buildSanitizedRuntime(
join(brotli, "include"),
...(toolchain.target.platform === "linux" ? ["-ffunction-sections", "-fdata-sections"] : []),
];
const runtimeObjects: string[] = [];
for (const unit of flavor.runtime_units) {
for (const variant of unit.variants) {
if (wanted.has(variant.path))
runtimeObjects.push(
compile(join(src, unit.source), unit.source.replace(/\.c$/, ".o"), [
...common,
...variant.defines.map((define) => "-D" + define),
]),
);
}
}
// The units and variants the pack selection chose, compiled from source.
const runtimeObjects = selectedRuntimeUnits(flavor, selection.selected.features).map(
({ source, variant }) =>
compile(join(src, source), source.replace(/\.c$/, ".o"), [
...common,
...variant.defines.map((define) => "-D" + define),
]),
);
const archives: string[] = [];
for (const archive of selection.selected.archives) {
const name = basename(archive.path)
@@ -70,6 +70,7 @@ test("package settings: strings and objects", () => {
expect(c.packages.get("semver")?.run).toBe("auto");
expect(c.packages.get("proper-lockfile")).toEqual({
unloaded: false,
shapes: true,
run: "node",
evaluate: "lazy",
share: "duplicate",
@@ -78,6 +79,24 @@ test("package settings: strings and objects", () => {
});
});
test("shapes: false turns off the shape overlay for one package", () => {
const c = config({
packages: {
zod: { shapes: false },
effect: { shapes: true, run: "native" },
ms: { shapes: "no" },
},
});
expect(c.packages.get("zod")?.shapes).toBe(false);
expect(c.packages.get("zod")?.run).toBe("auto");
expect(c.packages.get("effect")?.shapes).toBe(true);
expect(c.packages.get("effect")?.run).toBe("native");
expect(c.packages.has("ms")).toBe(false);
expect(c.diagnostics.map((d) => d.message)).toEqual([
`package.json 'scriptc.packages.ms.shapes' must be true or false, not "no"`,
]);
});
test("nodeIslands settings", () => {
const c = config({
nodeIslands: {
+25 -5
View File
@@ -10,7 +10,8 @@
* "encoding": "unloaded", // never loaded (package-shims.ts)
* "dd-trace": "node", // run in a Node island
* "hot-shots": "native", // must compile natively (SC8003)
* "proper-lockfile": { "run": "node", "evaluate": "lazy" }
* "proper-lockfile": { "run": "node", "evaluate": "lazy" },
* "zod": { "shapes": false } // no shape-inference overlay
* },
* "nodeIslands": { "auto": true, "modules": ["src/plugins/*.ts"] },
* "wasm": ["build/module.wasm"],
@@ -40,9 +41,12 @@ export type PackageMarshal = "copy" | "handles";
export type PackageSchedule = "lockstep" | "free";
/** One `scriptc.packages.<name>` entry. `unloaded` packages never load;
* the other settings choose where the package runs. */
* `shapes: false` compiles the package without shape inference's typed
* overlay (shape-inference/overlay.ts); the other settings choose where the
* package runs. */
export interface PackageSetting {
unloaded: boolean;
shapes: boolean;
run: PackageRun;
evaluate: PackageEvaluate;
share: PackageShare;
@@ -117,6 +121,9 @@ const PACKAGE_OBJECT_KEYS: Record<string, string[]> = {
marshal: ["copy", "handles"],
schedule: ["lockstep", "free"],
};
/** Every key of a package setting object: the string-valued ones above and
* the boolean `shapes`. */
const PACKAGE_SETTING_KEYS = [...Object.keys(PACKAGE_OBJECT_KEYS), "shapes"];
const NODE_ISLANDS_KEYS = [
"auto",
"modules",
@@ -129,6 +136,7 @@ const NODE_ISLANDS_KEYS = [
function defaultPackageSetting(): PackageSetting {
return {
unloaded: false,
shapes: true,
run: "auto",
evaluate: "auto",
share: "auto",
@@ -223,7 +231,7 @@ export function parseProjectConfig(manifest: string, text: string): ProjectConfi
problem(
["packages", name],
`has the unknown value ${shown(value)}`,
`use one of ${quoted(PACKAGE_STRINGS)}, or an object with ${Object.keys(PACKAGE_OBJECT_KEYS).join(", ")}`,
`use one of ${quoted(PACKAGE_STRINGS)}, or an object with ${PACKAGE_SETTING_KEYS.join(", ")}`,
);
continue;
}
@@ -236,12 +244,24 @@ export function parseProjectConfig(manifest: string, text: string): ProjectConfi
problem(
["packages", name],
`must be a string or an object, not ${shown(value)}`,
`use one of ${quoted(PACKAGE_STRINGS)}, or an object with ${Object.keys(PACKAGE_OBJECT_KEYS).join(", ")}`,
`use one of ${quoted(PACKAGE_STRINGS)}, or an object with ${PACKAGE_SETTING_KEYS.join(", ")}`,
);
continue;
}
let valid = true;
for (const key of Object.keys(value)) {
if (key === "shapes") {
const v = value[key];
if (typeof v !== "boolean") {
problem(
["packages", name, key],
`must be true or false, not ${shown(v)}`,
"use a boolean",
);
valid = false;
} else setting.shapes = v;
continue;
}
const allowed = Object.hasOwn(PACKAGE_OBJECT_KEYS, key)
? PACKAGE_OBJECT_KEYS[key]
: undefined;
@@ -249,7 +269,7 @@ export function parseProjectConfig(manifest: string, text: string): ProjectConfi
problem(
["packages", name, key],
"is not a known package setting",
`the settings are ${Object.keys(PACKAGE_OBJECT_KEYS).join(", ")}`,
`the settings are ${PACKAGE_SETTING_KEYS.join(", ")}`,
);
valid = false;
continue;
@@ -3,6 +3,14 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, beforeAll, describe, expect, test } from "vitest";
import { canonEqual, canonKey, Normalizer, type Canon } from "../src/frontend/canon.js";
import { mapCanonical } from "../src/frontend/canon-map.js";
import {
mapType,
ShapeRegistry,
UnionRegistry,
type TypeMapperCtx,
} from "../src/frontend/type-mapper.js";
import { typeKey } from "../src/ir/ir.js";
import { loadProgram } from "../src/frontend/program-node.js";
import * as ts from "../src/frontend/ts7/adapter.js";
@@ -271,3 +279,54 @@ describe("canon", () => {
expect(key(n.canon(aliases.get("Pair")!))).toBe("[string, number]");
});
});
describe("mapCanonical", () => {
const ctx = (): TypeMapperCtx => ({
checker,
shapes: new ShapeRegistry(),
unions: new UnionRegistry(),
classNamer: (decl) => decl.name?.text ?? "anonymous",
isStdlibFile: (sf) => load.program.isSourceFileDefaultLibrary(sf),
isNpmFile: () => false,
isExternalTypeFile: () => false,
isProgramFile: (sf) => load.moduleOrder.includes(sf),
});
test.for(["Normalized", "Same", "Flat", "Id", "Pair"])("%s maps like mapType", (name) => {
const strict = mapType(aliases.get(name)!, ctx());
const canonical = mapCanonical(new Normalizer(checker).canon(aliases.get(name)!), ctx());
expect(strict).not.toBeNull();
expect(canonical && typeKey(canonical)).toBe(typeKey(strict!));
});
test("shadow mode observes every successful top-level mapping", () => {
const seen: string[] = [];
const point = { kind: "object", className: "Point" } as const;
const observed: TypeMapperCtx = {
...ctx(),
typeMemo: new Map(),
localClassValue: (type) => (type === aliases.get("Points") ? point : null),
canonShadow: (type, mapped) => seen.push(`${checker.typeToString(type)} ${typeKey(mapped)}`),
};
// An intrinsic primitive, a mapping and its memo hit, and a local class
// value: one observation each; the nested mappings of `Same` are not
// observed.
const number = mapType(valueTypes.get("sampleNumber")!, observed);
const same = mapType(aliases.get("Same")!, observed);
expect(mapType(aliases.get("Same")!, observed)).toBe(same);
expect(mapType(aliases.get("Points")!, observed)).toBe(point);
expect(seen).toEqual([
`number ${typeKey(number!)}`,
`Same ${typeKey(same!)}`,
`Same ${typeKey(same!)}`,
`Points ${typeKey(point)}`,
]);
});
test("a generic alias maps under bindings, where mapType needs the instance table", () => {
const { type, params } = asTypes.get("Catalog<G, P>")!;
const env = new Map([
[params[0]!, valueTypes.get("sampleNumber")!],
[params[1]!, valueTypes.get("sampleText")!],
]);
const canonical = mapCanonical(new Normalizer(checker).canon(type, env), ctx());
expect(canonical?.kind).toBe("record");
});
});
@@ -378,7 +378,7 @@ done:
expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"]));
expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"]));
expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]);
expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v46");
expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v47");
const clangExe = join(dir, "clang-program");
const linkDriver = join(dir, "clang-driver.c");
@@ -712,7 +712,7 @@ done:
.split("\n");
expect(undefinedSymbols).toContain("_sf_scale");
expect(undefinedSymbols).toContain("_sf_callback_mix");
expect(undefinedSymbols).toContain("_scr_runtime_abi_v46");
expect(undefinedSymbols).toContain("_scr_runtime_abi_v47");
});
test("helper and clang object paths retain the same outbound FFI C ABI", async () => {
@@ -204,12 +204,111 @@ describe.runIf(helper !== null)("LLVM program partitions", () => {
expect([...after].filter((entry) => !sources.has(entry))).toHaveLength(1);
expect(await exitCode(dir, edited, "edited")).toBe(expectedExit(7, steps));
// With fewer partitions, functions keep their placement by name.
const contents = (paths: string[]) => Promise.all(paths.map((path) => readFile(path)));
// With fewer partitions, functions keep their placement by name, and a
// group line only stops a single-caller chain: step0 joins main without
// groups, and keeps its own placement in group 0.
const few = { outputs: 4, optLevel: "0" } as const;
expect(
await contents(await emit(dir, grouped(0, 24), { ...few, name: "few-grouped" })),
).toEqual(await contents(await emit(dir, program(), { ...few, name: "few-plain" })));
const placed = async (objects: string[]) => {
const where = new Map<string, number>();
for (const [index, object] of objects.entries()) {
const { stdout } = await execFileAsync("nm", [object]);
for (const line of stdout.split("\n")) {
const defined = /^[0-9a-fA-F]+ [A-Za-z] _?(step\d+|main)$/.exec(line.trim());
if (defined !== null) where.set(defined[1]!, index);
}
}
return where;
};
const fewGrouped = await placed(
await emit(dir, grouped(0, 24), { ...few, name: "few-grouped" }),
);
const fewPlain = await placed(await emit(dir, program(), { ...few, name: "few-plain" }));
expect(fewPlain.get("step0")).toBe(fewPlain.get("main"));
for (let step = 1; step < 24; step++)
expect(fewGrouped.get(`step${step}`)).toBe(fewPlain.get(`step${step}`));
expect(fewGrouped.get("main")).toBe(fewPlain.get("main"));
});
test("internal definitions nothing reaches are left out of every partition", async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-partitions-unreachable-"));
dirs.push(dir);
// An unreferenced internal function that uses live code and its own
// state, which only it references.
const unused = (name: string) =>
[
`@${name}State = internal global i64 7`,
`define internal void @${name}(i64 %value) {`,
` %state = load i64, ptr @${name}State`,
" call void @bump(i64 %state)",
" ret void",
"}",
].join("\n");
const cache = join(dir, "cache");
const options = { outputs: 4, optLevel: "0", cache } as const;
const objects = await emit(dir, program(0, unused("unusedOne")), { ...options, name: "first" });
expect(await exitCode(dir, objects, "first")).toBe(expectedExit());
for (const object of objects) {
const { stdout } = await execFileAsync("nm", [object]);
expect(stdout).not.toMatch(/unusedOne/);
}
// Adding another one changes no partition.
const before = await cacheEntries(cache, ".o");
const more = await emit(dir, program(0, `${unused("unusedOne")}\n${unused("unusedTwo")}`), {
...options,
name: "more",
});
expect(await cacheEntries(cache, ".o")).toEqual(before);
expect(await exitCode(dir, more, "more")).toBe(expectedExit());
});
test("a helper that one function uses goes to that function's partition", async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-partitions-helpers-"));
dirs.push(dir);
// step7 calls @helper directly, and step9 calls @viaTable through an
// internal global only step9 reads.
const helpers = (withHelpers: boolean) =>
program()
.split("\n")
.flatMap((line) => {
if (!withHelpers) return [line];
if (line === "define internal void @step7(i64 %value) noinline {")
return [
"define internal void @helper(i64 %value) noinline {",
" call void @bump(i64 %value)",
" ret void",
"}",
"@table = internal constant ptr @viaTable",
"define internal void @viaTable(i64 %value) noinline {",
" call void @bump(i64 %value)",
" ret void",
"}",
line,
" call void @helper(i64 0)",
];
if (line === "define internal void @step9(i64 %value) noinline {")
return [line, " %fn = load ptr, ptr @table", " call void %fn(i64 0)"];
return [line];
})
.join("\n");
const cache = join(dir, "cache");
const options = { outputs: 4, optLevel: "0", cache } as const;
await emit(dir, helpers(false), { ...options, name: "before" });
const before = await cacheEntries(cache, ".o");
const objects = await emit(dir, helpers(true), { ...options, name: "after" });
expect(await exitCode(dir, objects, "after")).toBe(expectedExit());
const where = new Map<string, number>();
for (const [index, object] of objects.entries()) {
const { stdout } = await execFileAsync("nm", [object]);
for (const line of stdout.split("\n")) {
const defined = /^[0-9a-fA-F]+ [A-Za-z] _?(step7|step9|helper|viaTable)$/.exec(line.trim());
if (defined !== null) where.set(defined[1]!, index);
}
}
expect(where.get("helper")).toBe(where.get("step7"));
expect(where.get("viaTable")).toBe(where.get("step9"));
// Only the partitions of step7 and step9 change.
const changed = [...(await cacheEntries(cache, ".o"))].filter((entry) => !before.has(entry));
expect(changed).toHaveLength(new Set([where.get("step7"), where.get("step9")]).size);
});
test("a table written one element per line keeps the text partitioner and its reuse", async () => {
@@ -0,0 +1,53 @@
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, test } from "vitest";
import { compile } from "../src/index.js";
async function emitLlvm(source: string): Promise<string> {
const dir = await mkdtemp(join(tmpdir(), "scriptc-periodic-guards-"));
try {
const entry = join(dir, "main.ts");
const outPath = join(dir, "main.ll");
await writeFile(entry, source);
const result = await compile(entry, { outDir: dir, outPath, outputKind: "llvm" });
if (!result.ok)
throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"));
return await readFile(outPath, "utf8");
} finally {
await rm(dir, { recursive: true, force: true });
}
}
test("a counted loop with a check every fourth iteration asks LLVM to unroll it by 4", async () => {
const llvm = await emitLlvm(`
function escape(c: number): number {
let z = 0;
let m = 0;
for (let j = 0; j < 50; j++) {
z = z * z + c;
m = z * z;
if ((j & 3) === 3 && !(m <= 4)) break;
}
return m;
}
console.log(escape(Number(process.argv[2] ?? "0.25")));
`);
expect(llvm).toMatch(/br label %loop\.c\d+, !llvm\.loop !1000004/);
expect(llvm).toContain(`!1000004 = distinct !{!1000004, !{!"llvm.loop.unroll.count", i32 4}}`);
});
test("a loop without a periodic guard has no loop metadata", async () => {
const llvm = await emitLlvm(`
function sum(n: number): number {
let s = 0;
for (let j = 0; j < n; j++) {
s += j;
if (s > 1000) break;
}
return s;
}
console.log(sum(Number(process.argv[2] ?? "10")));
`);
expect(llvm).not.toContain("!llvm.loop");
});
@@ -0,0 +1,57 @@
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, test } from "vitest";
import { compile } from "../src/index.js";
async function emitLlvm(source: string): Promise<string> {
const dir = await mkdtemp(join(tmpdir(), "scriptc-regex-replace-"));
try {
const entry = join(dir, "main.ts");
const outPath = join(dir, "main.ll");
await writeFile(entry, source);
const result = await compile(entry, { outDir: dir, outPath, outputKind: "llvm" });
expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true);
return await readFile(outPath, "utf8");
} finally {
await rm(dir, { recursive: true, force: true });
}
}
function functionBody(llvm: string, name: string): string[] {
const start = llvm.split("\n").findIndex((l) => l.startsWith("define") && l.includes(name));
expect(start).toBeGreaterThanOrEqual(0);
const lines = llvm.split("\n").slice(start);
return lines.slice(0, lines.indexOf("}"));
}
test("a chain of substitutions releases each intermediate result when the next one returns", async () => {
const llvm = await emitLlvm(`
function chain(s: string): number {
return s
.replace(/a[bc]/g, "<1>")
.replace(/b[cd]/g, "<2>")
.replace(/c[de]/g, "<3>")
.replaceAll(/<[^>]*>/g, "|").length;
}
console.log(chain(process.argv[2] ?? "abcabc"));
`);
const body = functionBody(llvm, "chain");
const calls = body
.map((line, index) => ({ line, index }))
.filter(({ line }) => /call ptr @scr_regex_replace(_all|_with)?\(/.test(line));
expect(calls).toHaveLength(4);
const results = calls.map(({ line }) => /^\s*(%[\w.]+) = call/.exec(line)![1]!);
// Each intermediate result is released after the call that consumes it
// and before the next substitution starts.
for (let k = 0; k < 3; k++) {
const release = body.findIndex(
(line, index) =>
index > calls[k + 1]!.index &&
line.includes("release") &&
line.includes(`(ptr ${results[k]})`),
);
expect(release, `release of ${results[k]}`).toBeGreaterThan(calls[k + 1]!.index);
if (k + 2 < calls.length) expect(release).toBeLessThan(calls[k + 2]!.index);
}
});
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7517-stream-kept-callbacks.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7518-stream-dropped-callbacks.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/8326-never-promise-instanceof.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/8400-iterative-teardown.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/array-prototype-call-native-arrays.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/net-unix-socket-connect-default-import.mjs"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/net-unix-socket-connect.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/typeof-ambient-undefined-names.ts"
],
"diags": []
}
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/fixtures/npm/cases/shape-overlay-facts/main.ts"
],
"diags": []
}
@@ -15,6 +15,7 @@ import {
} from "../../runtime-pack-common/scripts/runtime-unit.mjs";
import { createDeterministicArchive } from "./archive.mjs";
import { installRuntimePack, withBuildLock } from "./build-state.mjs";
import { shareIdenticalArtifacts } from "../../runtime-pack-common/scripts/share-artifacts.mjs";
const run = promisify(execFile);
const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
@@ -365,13 +366,14 @@ async function build() {
await copyFile(source, output);
}),
);
await shareIdenticalArtifacts(flavors, buildRoot);
const manifest = {
schema: "scriptc.runtime-pack.v1",
format: 1,
package: packageManifest.name,
version: packageManifest.version,
target: RUNTIME_PACK_MATRIX.target,
runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" },
runtime_abi: { version: 47, marker: "scr_runtime_abi_v47" },
compiler: {
command: compiler,
identity: compilerVersion,
@@ -15,6 +15,7 @@ import {
import { createDeterministicArchive } from "./archive.mjs";
import { buildImportBitcode, buildRuntimeUnit, resolveRuntimeUnitHelper } from "./runtime-unit.mjs";
import { installRuntimePack, withBuildLock } from "./build-state.mjs";
import { shareIdenticalArtifacts } from "./share-artifacts.mjs";
import { parallelMap } from "./parallel-map.mjs";
const run = promisify(execFile);
@@ -468,13 +469,14 @@ async function build() {
await copyFile(source, output);
}),
);
await shareIdenticalArtifacts(flavors, buildRoot);
const manifest = {
schema: "scriptc.runtime-pack.v1",
format: 1,
package: packageManifest.name,
version: packageManifest.version,
target: matrix.target,
runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" },
runtime_abi: { version: 47, marker: "scr_runtime_abi_v47" },
compiler: {
command: compiler,
identity: compilerVersion,
@@ -0,0 +1,48 @@
import { readdir, rm, rmdir } from "node:fs/promises";
import { join } from "node:path";
/** Points the artifacts of one runtime unit that hold the same bytes at one
* file and deletes the copies. Variants whose defines a unit does not read,
* and flavors that compile a unit the same way, produce identical objects
* and bitcode. Each link selects one flavor and one variant of each unit, so
* no link names a shared file twice. `root` is the directory artifact paths
* are relative to; flavors keep the order they were built in, so the first
* flavor's path holds the shared file. */
export async function shareIdenticalArtifacts(flavors, root) {
const firstPaths = new Map();
for (const flavor of Object.values(flavors)) {
for (const unit of flavor.runtime_units) {
let paths = firstPaths.get(unit.source);
if (paths === undefined) {
paths = new Map();
firstPaths.set(unit.source, paths);
}
for (const variant of unit.variants) {
for (const artifact of variant.bitcode === undefined
? [variant]
: [variant, variant.bitcode]) {
const first = paths.get(artifact.sha256);
if (first === undefined) {
paths.set(artifact.sha256, artifact.path);
} else if (first !== artifact.path) {
await rm(join(root, artifact.path));
artifact.path = first;
}
}
}
}
}
await removeEmptyDirectories(join(root, "artifacts"));
}
async function removeEmptyDirectories(directory) {
const entries = await readdir(directory, { withFileTypes: true });
let remaining = entries.length;
for (const entry of entries) {
if (entry.isDirectory() && (await removeEmptyDirectories(join(directory, entry.name))))
remaining--;
}
if (remaining > 0) return false;
await rmdir(directory);
return true;
}
@@ -0,0 +1,79 @@
import { existsSync } from "node:fs";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { expect, test } from "vitest";
import { shareIdenticalArtifacts } from "../scripts/share-artifacts.mjs";
type Artifact = { path: string; sha256: string; size: number; bitcode?: Artifact };
async function artifact(root: string, path: string, sha256: string): Promise<Artifact> {
await mkdir(dirname(join(root, path)), { recursive: true });
await writeFile(join(root, path), sha256);
return { path, sha256, size: sha256.length };
}
test("identical artifacts of one unit share the first flavor's file", async () => {
const root = await mkdtemp(join(tmpdir(), "scriptc-share-artifacts-"));
try {
const unit = async (flavor: string, source: string, ids: Record<string, string>) => ({
source,
variants: await Promise.all(
Object.entries(ids).map(async ([id, sha]) => ({
id,
...(await artifact(root, `artifacts/${flavor}/runtime/${id}/${source}.o`, sha)),
})),
),
});
const release = {
runtime_units: [
await unit("release", "scr_a", { default: "a1", "default-workers": "a1" }),
await unit("release", "scr_b", { default: "b1" }),
],
};
release.runtime_units[1]!.variants[0]!.bitcode = await artifact(
root,
"artifacts/release/runtime/default/scr_b.bc",
"b-bitcode",
);
const speed = {
runtime_units: [
await unit("speed", "scr_a", { default: "a2" }),
// The same bytes as another unit's object stay a separate file.
await unit("speed", "scr_b", { default: "b1", other: "a1" }),
],
};
speed.runtime_units[1]!.variants[0]!.bitcode = await artifact(
root,
"artifacts/speed/runtime/default/scr_b.bc",
"b-bitcode",
);
const flavors = { release, speed };
await shareIdenticalArtifacts(flavors, root);
const paths = (flavor: typeof release) =>
flavor.runtime_units.map((unit) =>
unit.variants.map((variant: Artifact) => [variant.path, variant.bitcode?.path]),
);
expect(paths(release)).toEqual([
[
["artifacts/release/runtime/default/scr_a.o", undefined],
["artifacts/release/runtime/default/scr_a.o", undefined],
],
[["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"]],
]);
expect(paths(speed)).toEqual([
[["artifacts/speed/runtime/default/scr_a.o", undefined]],
[
["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"],
["artifacts/speed/runtime/other/scr_b.o", undefined],
],
]);
expect(existsSync(join(root, "artifacts/release/runtime/default-workers"))).toBe(false);
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.o"))).toBe(false);
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.bc"))).toBe(false);
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_a.o"))).toBe(true);
expect(existsSync(join(root, "artifacts/speed/runtime/other/scr_b.o"))).toBe(true);
} finally {
await rm(root, { recursive: true, force: true });
}
});
+1 -1
View File
@@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) {
}
#endif
void scr_runtime_abi_v46(void) {}
void scr_runtime_abi_v47(void) {}
void scr_init(void) {
#ifdef SCR_WORKERS
+1 -1
View File
@@ -81,7 +81,7 @@ void scr_init(void);
/* Program objects emitted by the bundled LLVM helper reference this symbol.
* Its versioned spelling makes a mismatched manual runtime link fail before
* the program can start. */
void scr_runtime_abi_v46(void);
void scr_runtime_abi_v47(void);
void scr_runtime_workers_v8(void);
/* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ──────
+4
View File
@@ -1621,6 +1621,10 @@ void scr_dyn_check_fail(const ScrDynPath *path, const char *want, const ScrDyn *
/* The Date object a checked-dynamic Date handle holds, +1, or NULL with
* the check failure pending. */
ScrDate *scr_dyn_native_date_check(const ScrDyn *value, const ScrDynPath *path);
/* Names the analysis-proven fact the next checked conversion checks
* (shape inference's overlay), or NULL: its failure then reports a compiler
* analysis bug ("<package> <file>:<line>"). */
void scr_dyn_inferred_site(const char *site);
/* Matchers and builders of recursive shapes (records and arrays whose type
* reaches itself) track the dyn values they visit, so cyclic values match
+125 -102
View File
@@ -916,6 +916,117 @@ void scr_dyn_invoke_discard_id(ScrDyn *recv, ScrDyn *callee, const char *method,
scr_dyn_release(scr_dyn_invoke_prepared_id(recv, callee, method, mid_slot, args, argc, what));
}
/* The callback methods of a dense dyn array that take a thisArg:
* forEach, map, filter, some, every, find, findIndex, findLast,
* findLastIndex and flatMap. The caller binds `this` for the callbacks.
* `len` is the length before the first callback. */
static ScrDyn *dyn_arr_callback_method(ScrDyn *recv, int mid, ScrDyn *const *args, size_t argc,
size_t len, ScrDyn *callback_recv) {
if ((mid == DYN_M_findLast) || (mid == DYN_M_findLastIndex)) {
if (!dyn_cb_check(args, argc)) return NULL;
ScrDyn *visible_recv = callback_recv ? callback_recv : recv;
for (size_t i = len; i > 0; i--) {
size_t index = i - 1;
ScrDyn *item = index < recv->v.arr.len
? scr_dyn_retain(recv->v.arr.items[index])
: scr_dyn_retain(scr_dyn_undefined());
ScrDyn *result = dyn_call_cb(args[0], item, index, visible_recv);
if (!result) { scr_dyn_release(item); return NULL; }
bool matched = scr_dyn_truthy(result);
scr_dyn_release(result);
if (matched) {
if ((mid == DYN_M_findLast)) return item;
scr_dyn_release(item);
return scr_dyn_new_num((double)index);
}
scr_dyn_release(item);
}
return (mid == DYN_M_findLast)
? scr_dyn_retain(scr_dyn_undefined()) : scr_dyn_new_num(-1);
}
if ((mid == DYN_M_forEach) || (mid == DYN_M_map) ||
(mid == DYN_M_filter) || (mid == DYN_M_some) ||
(mid == DYN_M_every) || (mid == DYN_M_find) ||
(mid == DYN_M_findIndex)) {
if (!dyn_cb_check(args, argc)) return NULL;
ScrDyn *cb = args[0];
ScrDyn *out = ((mid == DYN_M_map) || (mid == DYN_M_filter)) ? scr_dyn_new_arr() : NULL;
/* Array iteration methods snapshot length before the first callback.
* A shrinking callback can remove a later dense entry; an expanding
* callback must not make the new tail observable to this pass. Live
* Web-boundary capsules are the externally visible receiver, so pass
* that capsule as callback arg 3: mutations through the callback's
* array argument then commit directly to the original static array. */
size_t n = len;
ScrDyn *visible_recv = callback_recv ? callback_recv : recv;
for (size_t step = 0; step < n; step++) {
size_t i = step;
if (i >= recv->v.arr.len) continue;
ScrDyn *item = scr_dyn_retain(recv->v.arr.items[i]);
ScrDyn *r = dyn_call_cb(cb, item, i, visible_recv);
if (!r) { scr_dyn_release(item); scr_dyn_release(out); return NULL; }
if ((mid == DYN_M_map)) {
scr_dyn_arr_push(out, r); /* ownership moves in */
r = NULL;
} else {
bool truthy = scr_dyn_truthy(r);
scr_dyn_release(r);
if ((mid == DYN_M_filter) && truthy) scr_dyn_arr_push(out, scr_dyn_retain(item));
if ((mid == DYN_M_some) && truthy) { scr_dyn_release(item); return scr_dyn_new_bool(true); }
if ((mid == DYN_M_every) && !truthy) { scr_dyn_release(item); return scr_dyn_new_bool(false); }
if ((mid == DYN_M_find) && truthy) return item; /* +1 moves out */
if ((mid == DYN_M_findIndex) && truthy) { scr_dyn_release(item); return scr_dyn_new_num((double)i); }
}
scr_dyn_release(item);
}
if (out) return out;
if ((mid == DYN_M_some)) return scr_dyn_new_bool(false);
if ((mid == DYN_M_every)) return scr_dyn_new_bool(true);
if ((mid == DYN_M_find)) return scr_dyn_retain(scr_dyn_undefined());
if ((mid == DYN_M_findIndex)) return scr_dyn_new_num(-1);
return scr_dyn_retain(scr_dyn_undefined()); /* forEach */
}
if ((mid == DYN_M_flatMap)) {
/* JS Array.prototype.flatMap over a dyn array: map + a depth-1
* flatten. Native dyn-array results flatten element-by-element; a
* WRAPPED engine array flattens through the routed keyed reads
* (elements wrap back scalar-normalized); everything else pushes
* as a single element (JS keeps non-array callback results whole).
* JS's spec snapshots the length up front (elements appended by
* the callback are not visited). */
if (!dyn_cb_check(args, argc)) return NULL;
ScrDyn *cb = args[0];
ScrDyn *out = scr_dyn_new_arr();
size_t n = recv->v.arr.len;
for (size_t i = 0; i < n && i < recv->v.arr.len; i++) {
if (!dyn_borrow_present(recv, i)) { if (scr_exc_pending()) { scr_dyn_release(out); return NULL; } continue; }
ScrDyn *item = dyn_borrow_item(recv, i);
if (!item) { scr_dyn_release(out); return NULL; }
ScrDyn *r = dyn_call_cb(
cb, item, i, callback_recv ? callback_recv : recv);
scr_dyn_release(item);
if (!r) { scr_dyn_release(out); return NULL; }
if (r->kind == SCR_DYN_TYPED_REF && scr_dyn_ref_is_array(r)) {
ScrDyn *view = scr_dyn_typed_ref_materialize(r);
scr_dyn_release(r);
r = view;
if (!r) { scr_dyn_release(out); return NULL; }
}
if (r->kind == SCR_DYN_ARR) {
for (size_t j = 0; j < r->v.arr.len; j++) {
if (dyn_borrow_present(r, j)) scr_dyn_arr_push(out, dyn_borrow_item(r, j));
if (scr_exc_pending()) { scr_dyn_release(r); scr_dyn_release(out); return NULL; }
}
scr_dyn_release(r);
} else {
scr_dyn_arr_push(out, r); /* ownership moves in */
}
}
return out;
}
return NULL;
}
static ScrDyn *scr_dyn_invoke_impl(
ScrDyn *recv, const char *method, int mid, ScrDyn *const *args, size_t argc,
const char *what, ScrDyn *callback_recv) {
@@ -1367,27 +1478,16 @@ static ScrDyn *scr_dyn_invoke_impl(
}
return (mid == DYN_M_includes) ? scr_dyn_new_bool(false) : scr_dyn_new_num(-1);
}
if ((mid == DYN_M_findLast) || (mid == DYN_M_findLastIndex)) {
if (!dyn_cb_check(args, argc)) return NULL;
ScrDyn *visible_recv = callback_recv ? callback_recv : recv;
for (size_t i = len; i > 0; i--) {
size_t index = i - 1;
ScrDyn *item = index < recv->v.arr.len
? scr_dyn_retain(recv->v.arr.items[index])
: scr_dyn_retain(scr_dyn_undefined());
ScrDyn *result = dyn_call_cb(args[0], item, index, visible_recv);
if (!result) { scr_dyn_release(item); return NULL; }
bool matched = scr_dyn_truthy(result);
scr_dyn_release(result);
if (matched) {
if ((mid == DYN_M_findLast)) return item;
scr_dyn_release(item);
return scr_dyn_new_num((double)index);
}
scr_dyn_release(item);
}
return (mid == DYN_M_findLast)
? scr_dyn_retain(scr_dyn_undefined()) : scr_dyn_new_num(-1);
if ((mid == DYN_M_forEach) || (mid == DYN_M_map) || (mid == DYN_M_filter) ||
(mid == DYN_M_some) || (mid == DYN_M_every) || (mid == DYN_M_find) ||
(mid == DYN_M_findIndex) || (mid == DYN_M_findLast) || (mid == DYN_M_findLastIndex) ||
(mid == DYN_M_flatMap)) {
/* Every callback runs with `this` bound to the thisArg, or undefined
* when there is none. */
scr_dyn_this_push_dyn(argc > 1 ? args[1] : scr_dyn_undefined());
ScrDyn *result = dyn_arr_callback_method(recv, mid, args, argc, len, callback_recv);
scr_dyn_this_pop();
return result;
}
if ((mid == DYN_M_join)) {
ScrJsonBuf b;
@@ -1550,48 +1650,6 @@ static ScrDyn *scr_dyn_invoke_impl(
}
return scr_dyn_retain(recv);
}
if ((mid == DYN_M_forEach) || (mid == DYN_M_map) ||
(mid == DYN_M_filter) || (mid == DYN_M_some) ||
(mid == DYN_M_every) || (mid == DYN_M_find) ||
(mid == DYN_M_findIndex)) {
if (!dyn_cb_check(args, argc)) return NULL;
ScrDyn *cb = args[0];
ScrDyn *out = ((mid == DYN_M_map) || (mid == DYN_M_filter)) ? scr_dyn_new_arr() : NULL;
/* Array iteration methods snapshot length before the first callback.
* A shrinking callback can remove a later dense entry; an expanding
* callback must not make the new tail observable to this pass. Live
* Web-boundary capsules are the externally visible receiver, so pass
* that capsule as callback arg 3: mutations through the callback's
* array argument then commit directly to the original static array. */
size_t n = len;
ScrDyn *visible_recv = callback_recv ? callback_recv : recv;
for (size_t step = 0; step < n; step++) {
size_t i = step;
if (i >= recv->v.arr.len) continue;
ScrDyn *item = scr_dyn_retain(recv->v.arr.items[i]);
ScrDyn *r = dyn_call_cb(cb, item, i, visible_recv);
if (!r) { scr_dyn_release(item); scr_dyn_release(out); return NULL; }
if ((mid == DYN_M_map)) {
scr_dyn_arr_push(out, r); /* ownership moves in */
r = NULL;
} else {
bool truthy = scr_dyn_truthy(r);
scr_dyn_release(r);
if ((mid == DYN_M_filter) && truthy) scr_dyn_arr_push(out, scr_dyn_retain(item));
if ((mid == DYN_M_some) && truthy) { scr_dyn_release(item); return scr_dyn_new_bool(true); }
if ((mid == DYN_M_every) && !truthy) { scr_dyn_release(item); return scr_dyn_new_bool(false); }
if ((mid == DYN_M_find) && truthy) return item; /* +1 moves out */
if ((mid == DYN_M_findIndex) && truthy) { scr_dyn_release(item); return scr_dyn_new_num((double)i); }
}
scr_dyn_release(item);
}
if (out) return out;
if ((mid == DYN_M_some)) return scr_dyn_new_bool(false);
if ((mid == DYN_M_every)) return scr_dyn_new_bool(true);
if ((mid == DYN_M_find)) return scr_dyn_retain(scr_dyn_undefined());
if ((mid == DYN_M_findIndex)) return scr_dyn_new_num(-1);
return scr_dyn_retain(scr_dyn_undefined()); /* forEach */
}
if ((mid == DYN_M_reduce) || (mid == DYN_M_reduceRight)) {
if (!dyn_cb_check(args, argc)) return NULL;
bool backward = (mid == DYN_M_reduceRight);
@@ -1616,44 +1674,6 @@ static ScrDyn *scr_dyn_invoke_impl(
scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1);
return NULL;
}
if ((mid == DYN_M_flatMap)) {
/* JS Array.prototype.flatMap over a dyn array: map + a depth-1
* flatten. Native dyn-array results flatten element-by-element; a
* WRAPPED engine array flattens through the routed keyed reads
* (elements wrap back scalar-normalized); everything else pushes
* as a single element (JS keeps non-array callback results whole).
* JS's spec snapshots the length up front (elements appended by
* the callback are not visited). */
if (!dyn_cb_check(args, argc)) return NULL;
ScrDyn *cb = args[0];
ScrDyn *out = scr_dyn_new_arr();
size_t n = recv->v.arr.len;
for (size_t i = 0; i < n && i < recv->v.arr.len; i++) {
if (!dyn_borrow_present(recv, i)) { if (scr_exc_pending()) { scr_dyn_release(out); return NULL; } continue; }
ScrDyn *item = dyn_borrow_item(recv, i);
if (!item) { scr_dyn_release(out); return NULL; }
ScrDyn *r = dyn_call_cb(
cb, item, i, callback_recv ? callback_recv : recv);
scr_dyn_release(item);
if (!r) { scr_dyn_release(out); return NULL; }
if (r->kind == SCR_DYN_TYPED_REF && scr_dyn_ref_is_array(r)) {
ScrDyn *view = scr_dyn_typed_ref_materialize(r);
scr_dyn_release(r);
r = view;
if (!r) { scr_dyn_release(out); return NULL; }
}
if (r->kind == SCR_DYN_ARR) {
for (size_t j = 0; j < r->v.arr.len; j++) {
if (dyn_borrow_present(r, j)) scr_dyn_arr_push(out, dyn_borrow_item(r, j));
if (scr_exc_pending()) { scr_dyn_release(r); scr_dyn_release(out); return NULL; }
}
scr_dyn_release(r);
} else {
scr_dyn_arr_push(out, r); /* ownership moves in */
}
}
return out;
}
if ((mid == DYN_M_sort)) {
ScrDyn *cmp = argc > 0 ? args[0] : scr_dyn_undefined();
if (cmp->kind != SCR_DYN_UNDEF && cmp->kind != SCR_DYN_FUNC) {
@@ -1801,7 +1821,10 @@ static ScrDyn *dyn_array_method_value_call(ScrClosure *closure, ScrDyn *const *a
ScrStr *method = scr_box_get_ref(closure->caps[0]);
ScrDyn *receiver = scr_dyn_this_get();
ScrDyn *result;
if (receiver->kind == SCR_DYN_ARR) {
/* An array, or a live view of a native array, runs the method its own
* dispatch has; other receivers take the generic array-like algorithms. */
if (receiver->kind == SCR_DYN_ARR ||
(receiver->kind == SCR_DYN_TYPED_REF && scr_dyn_typed_ref_array_ops(receiver))) {
result = scr_dyn_invoke(receiver, method->data, args, argc, method->data);
} else {
ScrDyn *pack = scr_dyn_new_arr();
+46
View File
@@ -11694,7 +11694,53 @@ static const char *scr_dyn_kind_name(const ScrDyn *d) {
return "unknown";
}
/* Shape inference's overlay (compiler frontend/shape-inference): a checked
* conversion of an analysis-proven fact names its site ("<package>
* <file>:<line>") while it runs. A failure there is a compiler analysis
* bug, so it reports itself as one instead of as a type error. */
static SCR_TL const char *scr_dyn_inferred_site_name;
void scr_dyn_inferred_site(const char *site) { scr_dyn_inferred_site_name = site; }
static void scr_dyn_inferred_fact_fail(const char *site, const ScrDynPath *path, const char *want,
const ScrDyn *got) {
const char *space = strchr(site, ' ');
size_t pkg_len = space ? (size_t)(space - site) : 0;
const char *where = space ? space + 1 : site;
ScrJsonBuf b;
scr_jb_init(&b);
scr_jb_puts(&b, "scriptc internal error: an inferred type was contradicted at ");
scr_jb_puts(&b, where);
scr_jb_puts(&b, " (expected ");
scr_jb_puts(&b, want);
scr_jb_puts(&b, " at ");
scr_dyn_path_render(&b, path);
scr_jb_puts(&b, ", got ");
scr_jb_puts(&b, scr_dyn_kind_name(got));
scr_jb_puts(&b, "). This is a bug in scriptc's shape analysis, not in the program's data. "
"Rebuild with --no-shape-overlay, or set \"scriptc\": {\"packages\": {\"");
for (size_t i = 0; i < pkg_len; i++) scr_jb_putc(&b, site[i]);
scr_jb_puts(&b, "\": {\"shapes\": false}}} in package.json, to compile the package without it.");
/* Validation mode: every hit is logged, also when the program catches it. */
const char *log = getenv("SCRIPTC_SHAPE_CHECK_LOG");
if (log && *log) {
FILE *f = fopen(log, "a");
if (f) {
fwrite(b.data, 1, b.len, f);
fputc('\n', f);
fclose(f);
}
}
scr_throw_error(SCR_ERR_ERROR, scr_jb_finish(&b));
}
void scr_dyn_check_fail(const ScrDynPath *path, const char *want, const ScrDyn *got) {
const char *site = scr_dyn_inferred_site_name;
if (site) {
scr_dyn_inferred_site_name = NULL;
scr_dyn_inferred_fact_fail(site, path, want, got);
return;
}
ScrJsonBuf b;
scr_jb_init(&b);
scr_jb_puts(&b, "expected ");
+18
View File
@@ -2741,6 +2741,24 @@ ScrNetSocket *scr_net_connect_checked(double port, ScrStr *host /*borrowed, null
return scr_net_connect(port, host, cb);
}
/* net.connect/createConnection(path[, connectListener]) and the `path`
* option from compiled code: a client of the Unix socket at `path`
* (borrowed). For a positional string (`pipe_name_rule`), Node's isPipeName
* applies first: a string that reads as a non-negative number is a port on
* localhost. `cb` (moves, nullable) is the connect listener. */
ScrNetSocket *scr_net_connect_path_checked(ScrStr *path, bool pipe_name_rule, ScrClosure *cb) {
double port = pipe_name_rule ? scr_string_to_number(path) : -1;
if (port >= 0) {
ScrStr *host = scr_str_new("localhost", 9);
ScrNetSocket *s = scr_net_connect_checked(port, host, cb);
scr_str_release(host);
return s;
}
ScrNetSocket *s = scr_net_connect_path(path, false);
if (cb) scr_net_ls_add(&s->conn_ls, cb, NULL, true);
return s;
}
static bool scr_net_dyn_truthy(const ScrDyn *v) {
switch (v->kind) {
case SCR_DYN_UNDEF:
+4
View File
@@ -265,6 +265,10 @@ ScrNetSocket *scr_net_connect_lookup_dyn(double port, ScrStr *host /*borrowed*/,
/* A client socket for the Unix socket at `path`, dialed now or (deferred)
* by scr_net_sock_dial_start. A failure is the deferred 'error'. */
ScrNetSocket *scr_net_connect_path(ScrStr *path /*borrowed*/, bool deferred); /* +1 */
/* net.connect(path[, cb]) and connect({ path }): with `pipe_name_rule`, a
* numeric string is a localhost port, as in Node. */
ScrNetSocket *scr_net_connect_path_checked(ScrStr *path /*borrowed*/, bool pipe_name_rule,
ScrClosure *cb /*moves, nullable*/); /* +1 */
/* server.listen(path[, cb]): a Unix-socket server, its file removed at close. */
void scr_net_listen_path(ScrNetServer *s, ScrStr *path /*borrowed*/, ScrClosure *cb /*moves, nullable*/);
void scr_net_sock_dial_start(ScrNetSocket *s);
+99 -6
View File
@@ -119,6 +119,10 @@ struct ScrStreamState {
bool reading, in_read_sync, ended, end_emitted, end_scheduled;
bool in_read_call; /* inside the user's _read (in_read_sync also starts true) */
bool need_readable, emitted_readable, readable_listening;
/* the stream has 'data' listeners: kept by scr_stream_listener_observe
* from the registry's empty/nonempty transitions, so the per-chunk
* paths read it instead of looking the name up */
bool data_listeners;
bool resume_scheduled, maybe_more_scheduled;
/* The buffer's representation: encoded means every entry is a ScrStr,
* otherwise every entry is a ScrBytes (scr_stream_rbuf_push checks
@@ -246,6 +250,16 @@ struct ScrStreamState {
ScrClosure *destroy_cb;
ScrStreamErrInv destroy_inv;
/* The completion closures (`callback`) the emitted option-callback
* thunks pass, owned. write_done is the one every write and writev call
* receives (scr_stream_write_closure); done_cache is the last one a
* transform, flush, final, construct or destroy call received, handed
* out again while this is its only reference (scr_stream_done_closure).
* Both hold the stream, so they are dropped at 'close' and traced for a
* stream dropped before it closes. */
ScrClosure *write_done;
ScrClosure *done_cache;
/* finished()/pipeline() watchers: fired once at the terminal point
* (the 'close' tick — or a registration-time tick when already closed)
* with the stream's finish status (NULL / the error / premature close).
@@ -417,6 +431,8 @@ static void scr_stream_state_drop(ScrStreamState *st, bool gc) {
if (st->transform_cb) scr_closure_release(st->transform_cb);
if (st->flush_cb) scr_closure_release(st->flush_cb);
if (st->destroy_cb) scr_closure_release(st->destroy_cb);
if (st->write_done) scr_closure_release(st->write_done);
if (st->done_cache) scr_closure_release(st->done_cache);
for (size_t i = 0; i < st->w.end_cbs_n; i++) scr_closure_release(st->w.end_cbs[i]);
for (size_t i = 0; i < st->fin.n; i++) scr_closure_release(st->fin.cb[i]);
for (size_t i = 0; i < st->pipes.n; i++) scr_stream_release(st->pipes.dst[i]);
@@ -465,6 +481,8 @@ void scr_stream_st_trace(ScrStreamState *st, ScrTraceVisit visit, void *ctx) {
if (st->transform_cb) visit(st->transform_cb, ctx);
if (st->flush_cb) visit(st->flush_cb, ctx);
if (st->destroy_cb) visit(st->destroy_cb, ctx);
if (st->write_done) visit(st->write_done, ctx);
if (st->done_cache) visit(st->done_cache, ctx);
for (size_t i = 0; i < st->w.end_cbs_n; i++) visit(st->w.end_cbs[i], ctx);
for (size_t i = 0; i < st->fin.n; i++) visit(st->fin.cb[i], ctx);
for (size_t i = st->w.head; i < st->w.n; i++) {
@@ -519,6 +537,56 @@ void scr_stream_release(ScrStream *s) {
void *scr_stream_retain_v(void *s) { return scr_stream_retain(s); }
void scr_stream_release_v(void *s) { scr_stream_release(s); }
/* A completion closure: a 1-cap closure calling fn over a box that holds
* the stream. Returns +1. */
static ScrClosure *scr_stream_done_new(ScrStream *s, void *fn) {
ScrClosure *c = scr_closure_new(fn, 1);
ScrBox *bx = scr_box_new_obj(scr_stream_retain_v, scr_stream_release_v, scr_stream_trace);
c->caps[0] = bx;
scr_box_set_ref(bx, scr_stream_retain(s));
return c;
}
/* Keeps c in *slot, replacing the closure there. Nothing is kept after
* 'close'. Borrows c. */
static void scr_stream_done_keep(ScrStreamState *st, ScrClosure **slot, ScrClosure *c) {
if (st->close_emitted) return;
ScrClosure *old = *slot;
*slot = scr_closure_retain(c);
if (old) scr_closure_release(old);
}
/* The `callback` a write or writev option callback receives. Node passes
* the stream's one onwrite function to every _write and _writev call, so
* every call receives the same closure, kept or not. Returns +1. */
ScrClosure *scr_stream_write_closure(ScrStream *s, void *fn) {
ScrStreamState *st = s->st;
ScrClosure *c = st->write_done;
if (c != NULL && c->fn == fn) return scr_closure_retain(c);
c = scr_stream_done_new(s, fn);
scr_stream_done_keep(st, &st->write_done, c);
return c;
}
/* The `callback` a transform, flush, final, construct or destroy option
* callback receives. Node passes a new function to each call. The stream
* keeps the last closure and hands it out again while it holds the only
* reference and the closure has no own properties; weak collections then
* forget it first, as they would at its free, so it is a new key to them.
* Otherwise a new closure replaces it, and one the user kept keeps its own
* stream reference. Returns +1. */
ScrClosure *scr_stream_done_closure(ScrStream *s, void *fn) {
ScrStreamState *st = s->st;
ScrClosure *c = st->done_cache;
if (c != NULL && c->rc == 1 && c->fn == fn && c->props == NULL) {
scr_weak_dispose_inline(c);
return scr_closure_retain(c);
}
c = scr_stream_done_new(s, fn);
scr_stream_done_keep(st, &st->done_cache, c);
return c;
}
/* ── the tick queue (the nextTick stand-in) ───────────────────────────── */
typedef enum {
@@ -817,10 +885,14 @@ static void scr_stream_emit_data(ScrStream *s, void *chunk, ScrEntryKind kind) {
} else {
b = chunk;
}
ScrStr *n = (ScrStr *)&scr_stream_lit_data;
scr_emitter_emit((ScrEmitter *)s, n, b, str, value);
scr_str_release(n);
if (scr_exc_pending()) return;
/* emit('data') with no listeners does nothing; a piped stream delivers
* through st->pipes below */
if (st->r.data_listeners) {
ScrStr *n = (ScrStr *)&scr_stream_lit_data;
scr_emitter_emit((ScrEmitter *)s, n, b, str, value);
scr_str_release(n);
if (scr_exc_pending()) return;
}
/* pipe delivery, with backpressure: a full destination pauses us until
* its 'drain'. Encoded sources write the string (utf8 bytes at the
* destination — Node writes the decoded string through). */
@@ -969,7 +1041,7 @@ static bool scr_stream_add_chunk(ScrStream *s, ScrBytes *chunk, bool front) {
}
if (entry_len > 0 &&
st->r.flowing == 1 && st->r.length == 0 && !st->r.in_read_sync && !front &&
(scr_emitter_has((ScrEmitter *)s, "data") || st->pipes.n > 0)) {
(st->r.data_listeners || st->pipes.n > 0)) {
/* the direct-emit fast path: flowing, empty buffer, async push */
if (st->r.emitted_readable) st->r.emitted_readable = false;
ScrEntryKind kind = scr_stream_rkind(st);
@@ -1938,7 +2010,7 @@ static int scr_stream_add_entry(ScrStream *s, void *entry /*moves*/, ScrEntryKin
}
st->r.reading = false;
if (st->r.flowing == 1 && st->r.length == 0 && !st->r.in_read_sync &&
(scr_emitter_has((ScrEmitter *)s, "data") || st->pipes.n > 0)) {
(st->r.data_listeners || st->pipes.n > 0)) {
if (st->r.emitted_readable) st->r.emitted_readable = false;
scr_stream_emit_data(s, entry, kind);
scr_stream_entry_release_kind(entry, kind);
@@ -2437,6 +2509,20 @@ static ScrStreamState *scr_stream_state_new(bool has_r, bool has_w,
return st;
}
/* The registry's empty/nonempty transitions for the names the per-chunk
* paths test (scr_emitter_observe): no script runs here. */
static void scr_stream_listener_observe(void *context, const ScrStr *name, bool present) {
ScrStream *s = context;
if (s->st != NULL && name->len == 4 && memcmp(name->data, "data", 4) == 0)
s->st->r.data_listeners = present;
}
/* Starts tracking the stream's 'data' listeners, once its state exists. */
static void scr_stream_watch_listeners(ScrStream *s) {
s->st->r.data_listeners = scr_emitter_has((ScrEmitter *)s, "data");
scr_emitter_observe((ScrEmitter *)s, &scr_stream_listener_observe, s);
}
static ScrStream *scr_stream_alloc(const ScrVt *vt, const char *cls, bool has_r, bool has_w,
double rhwm, double whwm, bool auto_destroy,
bool emit_close, bool allow_half_open) {
@@ -2465,6 +2551,7 @@ static ScrStream *scr_stream_alloc(const ScrVt *vt, const char *cls, bool has_r,
scr_emitter_reserve(em, "end");
scr_emitter_reserve(em, "readable");
}
scr_stream_watch_listeners(s);
scr_obj_alloc_note();
return s;
}
@@ -2587,6 +2674,7 @@ void scr_stream_init_readable(ScrStream *s, double hwm, bool auto_destroy, bool
ScrClosure *destroy, ScrStreamErrInv destroy_inv) {
ScrStreamState *st = scr_stream_state_new(true, false, hwm, -1, auto_destroy, emit_close, true);
s->st = st;
scr_stream_watch_listeners(s);
st->r.read_cb = read;
st->r.read_inv = read_inv;
st->destroy_cb = destroy;
@@ -2599,6 +2687,7 @@ void scr_stream_init_writable(ScrStream *s, double hwm, bool auto_destroy, bool
ScrClosure *destroy, ScrStreamErrInv destroy_inv) {
ScrStreamState *st = scr_stream_state_new(false, true, -1, hwm, auto_destroy, emit_close, true);
s->st = st;
scr_stream_watch_listeners(s);
st->w.write_cb = write;
st->w.write_inv = write_inv;
st->w.final_cb = final_cb;
@@ -2616,6 +2705,7 @@ void scr_stream_init_duplex(ScrStream *s, double rhwm, double whwm, bool auto_de
ScrStreamState *st = scr_stream_state_new(readable_side, writable_side, rhwm, whwm,
auto_destroy, emit_close, allow_half_open);
s->st = st;
scr_stream_watch_listeners(s);
if (!readable_side) {
st->r.ended = true;
st->r.end_emitted = true;
@@ -2642,6 +2732,7 @@ static void scr_stream_init_transformish(ScrStream *s, bool passthrough,
ScrStreamState *st = scr_stream_state_new(true, true, rhwm, whwm,
auto_destroy, emit_close, allow_half_open);
s->st = st;
scr_stream_watch_listeners(s);
st->is_transform = !passthrough;
st->passthrough = passthrough;
/* the Transform constructor's readable-half stance (see new_transformish) */
@@ -7960,6 +8051,8 @@ static void scr_stream_run_tick(ScrStreamTick *t) {
if (st->transform_cb) { scr_closure_release(st->transform_cb); st->transform_cb = NULL; }
if (st->flush_cb) { scr_closure_release(st->flush_cb); st->flush_cb = NULL; }
if (st->destroy_cb) { scr_closure_release(st->destroy_cb); st->destroy_cb = NULL; }
if (st->write_done) { scr_closure_release(st->write_done); st->write_done = NULL; }
if (st->done_cache) { scr_closure_release(st->done_cache); st->done_cache = NULL; }
}
break;
}
+6
View File
@@ -681,6 +681,12 @@ void scr_stream_release(ScrStream *s);
void *scr_stream_retain_v(void *s);
void scr_stream_release_v(void *s);
void scr_stream_trace(void *obj, ScrTraceVisit visit, void *ctx);
/* The `callback` a write or writev option callback receives (+1): the
* same closure for every call, as Node passes one onwrite function. */
ScrClosure *scr_stream_write_closure(ScrStream *s, void *fn);
/* The `callback` any other option callback receives (+1), reused while the
* stream holds its only reference. */
ScrClosure *scr_stream_done_closure(ScrStream *s, void *fn);
void scr_stream_install(void);
/* The loop's stream hook (scr_async.c): `pending` keeps the loop alive
+12 -2
View File
@@ -6,7 +6,11 @@ import { basename, dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { gunzipSync } from "node:zlib";
import { compile } from "../packages/compiler/src/index.js";
import { NATIVE_TARGETS, nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js";
import {
NATIVE_TARGETS,
executableStripLinkerArgs,
nativeCodegenTarget,
} from "../packages/compiler/src/backend/targets.js";
import { tsgoExecutable } from "../packages/compiler/src/frontend/ts7/rpc-api.js";
import { prebuiltLibDiagnosticsText } from "../packages/compiler/src/frontend/ts7/lib-diagnostics.js";
import { stageNativeCliAssets } from "./native-cli-assets.mjs";
@@ -103,12 +107,14 @@ const ffi = join(seed, "compiler.ffi.json");
writeFileSync(ffi, JSON.stringify({ ffi_format: 6, functions, libraries }, null, 2) + "\n");
const executable = join(bin, "scriptc" + target.outputSuffixes.exe);
const comptime = join(lib, "scriptc-comptime" + target.outputSuffixes.exe);
const strip = process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1";
execFileSync(
compiler,
[
...compilerArgs,
...cFlags,
"-O2",
...(strip ? ["-g0", ...executableStripLinkerArgs(target.platform, true)] : []),
"-I",
join(root, "packages/runtime/vendor/quickjs-ng"),
join(nativeSources, "comptime.c"),
@@ -125,12 +131,15 @@ execFileSync(
],
{ stdio: "inherit" },
);
// The Windows link writes the debug database beside the executable.
if (strip && target.platform === "win32")
rmSync(comptime.replace(/\.exe$/, ".pdb"), { force: true });
const result = await compile(join(root, "packages/compiler/src/native/cli.ts"), {
outDir: seed,
outPath: executable,
backend: "llvm",
optimization,
strip: process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1",
strip,
ffiProfilePath: ffi,
sanitize,
...(profileGenerate === undefined ? {} : { profileGenerate }),
@@ -177,6 +186,7 @@ const assets = stageNativeCliAssets({
target,
ts7: tsgoExecutable(),
compilerVersion,
strip,
});
// A cross-libc seed may use Zig on the build host. The installed compiler
// selects the destination platform's ordinary linker and SDK.
+215
View File
@@ -0,0 +1,215 @@
import { readFileSync, writeFileSync } from "node:fs";
import { createRequire } from "node:module";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
// Generates packages/compiler/src/ir/fields.generated.ts from the node types
// in ir.ts: for each IrExpr and IrStmt kind, the fields that can hold a
// string, read by name. Code that would enumerate a node's keys reads them
// through this switch instead: the native compiler enumerates and reads an
// untyped object's keys through its dynamic representation, one conversion
// per node. `--check` fails when the file is stale.
const root = fileURLToPath(new URL("..", import.meta.url));
const require = createRequire(join(root, "package.json"));
const ts = require("typescript");
const irPath = join(root, "packages/compiler/src/ir/ir.ts");
const target = join(root, "packages/compiler/src/ir/fields.generated.ts");
const program = ts.createProgram([irPath], { strict: true, target: ts.ScriptTarget.ES2022 });
const checker = program.getTypeChecker();
const source = program.getSourceFile(irPath);
const aliases = new Map();
ts.forEachChild(source, (node) => {
if (ts.isTypeAliasDeclaration(node)) aliases.set(node.name.text, node);
});
/** Whether a value of `type` can be a string. */
function holdsString(type) {
if (type.isUnion()) return type.types.some(holdsString);
return (type.flags & (ts.TypeFlags.StringLike | ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0;
}
/** Whether every value of `type` is a string. */
function onlyString(type) {
if (type.isUnion()) return type.types.every(onlyString);
return (type.flags & ts.TypeFlags.StringLike) !== 0;
}
/** Each kind's string fields, in declaration order, merged over the unions:
* a kind both unions declare lists the fields of both. */
const kinds = new Map();
for (const name of ["IrExpr", "IrStmt"]) {
const alias = aliases.get(name);
const union = checker.getTypeAtLocation(alias.name);
for (const member of union.isUnion() ? union.types : [union]) {
const kindSymbol = member.getProperty("kind");
const kindType = checker.getTypeOfSymbolAtLocation(kindSymbol, alias);
if (!kindType.isStringLiteral()) throw new Error(`${name} member without a literal kind`);
const fields = kinds.get(kindType.value) ?? [];
for (const property of member.getProperties()) {
const type = checker.getTypeOfSymbolAtLocation(property, alias);
if (!holdsString(type)) continue;
const known = fields.find((field) => field.name === property.name);
const always = onlyString(type) && (property.flags & ts.SymbolFlags.Optional) === 0;
if (known === undefined) fields.push({ name: property.name, always });
else known.always &&= always;
}
kinds.set(kindType.value, fields);
}
}
const lines = [
"// Generated from packages/compiler/src/ir/ir.ts by scripts/generate-ir-fields.mjs.",
"// Run `pnpm generate:ir-fields` after changing the fields of an IR node.",
'import type { IrExpr, IrStmt, IrType } from "./ir.js";',
"",
"/** Calls `visit` with each string field of `node` (including `kind`) and its",
" * name: the string values `for (const key in node)` finds on a node, read",
" * by name in the order ir.ts declares them. */",
"export function eachStringField(",
" node: IrExpr | IrStmt,",
" visit: (value: string, field: string) => void,",
"): void {",
" switch (node.kind) {",
];
for (const [kind, fields] of [...kinds].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) {
lines.push(` case ${JSON.stringify(kind)}: {`);
for (const { name, always } of fields) {
const read = `node.${name}`;
if (always) lines.push(` visit(${read}, ${JSON.stringify(name)});`);
else
lines.push(` if (typeof ${read} === "string") visit(${read}, ${JSON.stringify(name)});`);
}
lines.push(" return;", " }");
}
lines.push(" }", "}", "");
// The deep walkers: every field of every object under an IR node, as a
// recursive key walk finds them, read by name.
const members = new Map();
for (const name of ["IrExpr", "IrStmt", "IrType"]) {
const union = checker.getTypeAtLocation(aliases.get(name).name);
members.set(name, new Set(union.isUnion() ? union.types : [union]));
}
const nullish = ts.TypeFlags.Null | ts.TypeFlags.Undefined | ts.TypeFlags.Void;
const scalar =
ts.TypeFlags.NumberLike |
ts.TypeFlags.BooleanLike |
ts.TypeFlags.BigIntLike |
ts.TypeFlags.ESSymbolLike;
/** The walker an IR union member belongs to, or null. */
function irKind(type) {
for (const [name, set] of members) if (set.has(type)) return name;
return null;
}
let temp = 0;
/** Lines that walk `expr` of `type`: strings to `walker.string`, IR nodes to
* their walkers, arrays by element and other objects by field. */
function walkLines(expr, type, indent, depth) {
if (depth > 12) throw new Error(`field type nests too deep at ${expr}`);
const parts = type.isUnion() ? type.types : [type];
const present = parts.filter((part) => (part.flags & nullish) === 0);
const optional = present.length !== parts.length;
const strings = present.filter((part) => (part.flags & ts.TypeFlags.StringLike) !== 0);
const objects = present.filter((part) => (part.flags & (ts.TypeFlags.StringLike | scalar)) === 0);
for (const part of present)
if ((part.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0)
throw new Error(`untyped field ${expr}`);
const body = [];
if (objects.length > 0) {
const kinds = new Set(objects.map(irKind));
if (kinds.size === 1 && !kinds.has(null)) {
const [kind] = kinds;
body.push(`walk${kind.slice(2)}Fields(${expr}, walker, skipType);`);
} else if (objects.every((part) => checker.isArrayType(part))) {
if (objects.length !== 1) throw new Error(`several array types at ${expr}`);
const item = `item${temp++}`;
body.push(`for (const ${item} of ${expr}) {`);
body.push(...walkLines(item, checker.getTypeArguments(objects[0])[0], " ", depth + 1));
body.push("}");
} else if (kinds.has(null) && objects.length === 1) {
for (const property of checker.getPropertiesOfType(objects[0])) {
if (property.name === "loc") continue;
const fieldType = checker.getTypeOfSymbol(property);
const field = `${expr}.${property.name}`;
const lines = walkLines(field, fieldType, "", depth + 1);
if (lines.length === 0) continue;
if (property.name === "type")
body.push("if (!skipType) {", ...lines.map((l) => ` ${l}`), "}");
else body.push(...lines);
}
} else throw new Error(`mixed object types at ${expr}: ${checker.typeToString(type)}`);
}
let lines = body;
if (strings.length > 0 && objects.length > 0)
lines = [
`if (typeof ${expr} === "string") walker.string(${expr});`,
...(body.length > 0 ? [`else {`, ...body.map((l) => ` ${l}`), "}"] : []),
];
else if (strings.length > 0) lines = [`walker.string(${expr});`];
if (lines.length > 0 && optional && !(strings.length > 0 && objects.length > 0))
lines = [
`if (${expr} !== undefined && ${expr} !== null) {`,
...lines.map((l) => ` ${l}`),
"}",
];
else if (lines.length > 0 && optional)
lines = [
`if (${expr} !== undefined && ${expr} !== null) {`,
...lines.map((l) => ` ${l}`),
"}",
];
return lines.map((line) => indent + line);
}
lines.push(
"/** What a field walk reports (walkExprFields): each expression and",
" * statement node before its fields, and each string a field holds. */",
"export interface IrFieldWalker {",
" node(node: IrExpr | IrStmt): void;",
" string(value: string): void;",
"}",
"",
);
for (const name of ["IrExpr", "IrStmt", "IrType"]) {
const short = name.slice(2);
const nodeCall = name === "IrType" ? [] : [" walker.node(node);"];
lines.push(
`/** Walks every field of \`node\` but \`loc\` (and \`type\` with \`skipType\`), and`,
" * every field of each object under it: the strings and nodes a recursive",
" * `for (const key in value)` walk with the same skips finds, read by name in",
" * the order ir.ts declares the fields. */",
`export function walk${short}Fields(node: ${name}, walker: IrFieldWalker, skipType: boolean): void {`,
...nodeCall,
" switch (node.kind) {",
);
const byKind = new Map();
for (const member of members.get(name)) {
const kind = checker.getTypeOfSymbol(member.getProperty("kind")).value;
if (byKind.has(kind)) throw new Error(`${name} kind ${kind} declared twice`);
byKind.set(kind, member);
}
for (const [kind, member] of [...byKind].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) {
lines.push(` case ${JSON.stringify(kind)}: {`);
for (const property of member.getProperties()) {
if (property.name === "loc") continue;
const fieldType = checker.getTypeOfSymbol(property);
const field = `node.${property.name}`;
const walk = walkLines(field, fieldType, "", 0);
if (walk.length === 0) continue;
if (property.name === "type")
lines.push(" if (!skipType) {", ...walk.map((l) => ` ${l}`), " }");
else lines.push(...walk.map((l) => ` ${l}`));
}
lines.push(" return;", " }");
}
lines.push(" }", "}", "");
}
const text = lines.join("\n");
if (process.argv.includes("--check")) {
if (readFileSync(target, "utf8") !== text)
throw new Error("fields.generated.ts is stale; run node scripts/generate-ir-fields.mjs");
} else writeFileSync(target, text);
+41
View File
@@ -1,3 +1,4 @@
import { spawnSync } from "node:child_process";
import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { basename, dirname, join, relative } from "node:path";
import type { NativeTargetSpec } from "../packages/compiler/src/backend/targets.js";
@@ -23,12 +24,49 @@ function copyPackage(source: string, destination: string, members: readonly stri
}
}
/** Removes the symbol table and debug sections of a copied executable.
* llvm-strip rewrites the linker-signed ad hoc code signature of a Mach-O
* executable, which arm64 macOS requires to run it. Windows executables keep
* their debug information in separate .pdb files, which are not copied.
* Without llvm-strip on PATH the executable keeps its symbols; the release
* hosts install it with LLVM. */
function stripExecutable(path: string, platform: string): void {
if (platform === "win32") return;
const result = spawnSync("llvm-strip", [path], { stdio: "inherit" });
if ((result.error as NodeJS.ErrnoException | undefined)?.code === "ENOENT") {
console.warn(`llvm-strip is not on PATH; ${basename(path)} keeps its symbols`);
return;
}
if (result.error !== undefined) throw result.error;
if (result.status !== 0) throw new Error(`llvm-strip ${path} exited with ${result.status}`);
}
/** Rewrites CRLF line endings to LF in the text files under `directory`. A
* Windows checkout converts the repository's LF text files to CRLF; this
* restores the files the other hosts ship. Text is what Git's conversion
* treats as text: no NUL byte in the first 8000 bytes. */
function normalizeLineEndings(directory: string): void {
for (const entry of readdirSync(directory, { withFileTypes: true })) {
const path = join(directory, entry.name);
if (entry.isDirectory()) {
normalizeLineEndings(path);
continue;
}
if (!entry.isFile()) continue;
const bytes = readFileSync(path);
if (bytes.subarray(0, 8000).includes(0) || !bytes.includes("\r\n")) continue;
writeFileSync(path, bytes.toString("latin1").replaceAll("\r\n", "\n"), "latin1");
}
}
export function stageNativeCliAssets(options: {
repository: string;
output: string;
target: NativeTargetSpec;
ts7: string;
compilerVersion: string;
/** Strip the copied tsc-ts executable. */
strip: boolean;
}): Pick<
NativeToolchainManifest,
| "ts7"
@@ -67,8 +105,10 @@ export function stageNativeCliAssets(options: {
);
const tscTs = join(lib, "tsc-ts");
copyPackage(dirname(dirname(tscTsSource)), tscTs, ["lib"]);
if (options.strip) stripExecutable(join(tscTs, "lib", basename(tscTsSource)), target.platform);
const sources = join(lib, "runtime-sources");
copyPackage(join(repository, "packages/runtime"), sources, ["src", "vendor"]);
normalizeLineEndings(sources);
const declarations = join(lib, "declarations");
mkdirSync(declarations, { recursive: true });
for (const name of ["scriptc.d.ts", "scriptc-overrides.d.ts", "scriptc-node-fallback.d.ts"]) {
@@ -83,6 +123,7 @@ export function stageNativeCliAssets(options: {
cpSync(join(repository, "packages/compiler/shims"), join(declarations, "shims"), {
recursive: true,
});
normalizeLineEndings(declarations);
// This optional Node host runs the built WASI module, never the compiler.
const wasi = join(lib, "wasi");
mkdirSync(join(wasi, "cli"), { recursive: true });

Some files were not shown because too many files have changed in this diff Show More