diff --git a/docs/content/docs/contracts.mdx b/docs/content/docs/contracts.mdx index 11737da59..8f17f6a98 100644 --- a/docs/content/docs/contracts.mdx +++ b/docs/content/docs/contracts.mdx @@ -331,7 +331,7 @@ $ scriptc build graph.ts ## ProgramLifetime -`ProgramLifetime` marks a class whose instances a program keeps until it exits, such as the nodes of a syntax tree, the symbols of a symbol table or interned types. scriptc then never reference counts, collects or frees instances of the class and of its subclasses: they are never freed, every retain and release on them compiles to nothing, and the cycle collector neither walks them nor needs to trace the objects that point to them. Node.js behavior is unchanged; only memory use differs, because an instance the program drops is never reclaimed. +`ProgramLifetime` marks a class whose instances a program keeps until it exits, such as the nodes of a syntax tree, the symbols of a symbol table or interned types. scriptc then never reference counts, collects or frees instances of the class and of its subclasses: they are never freed, every retain and release on them compiles to nothing, and the cycle collector neither walks them nor needs to trace the objects that point to them. Node.js behavior is unchanged; only memory use differs, because an instance the program drops is never reclaimed. On Linux, the instances a worker thread creates are returned with the rest of its memory when it exits, unless the worker shared objects with other threads (see [cost](/docs/threads#cost)). ```ts:tree.ts import type { ProgramLifetime } from "@scriptc/types"; diff --git a/docs/content/docs/threads.mdx b/docs/content/docs/threads.mdx index 7b394e09f..15a3eb102 100644 --- a/docs/content/docs/threads.mdx +++ b/docs/content/docs/threads.mdx @@ -234,6 +234,8 @@ Without `@scriptc/threads`, a scriptc executable also shares a message of untype Starting a worker thread takes tens of microseconds plus the worker's own module initialization, since every worker initializes the program's modules again, as in Node.js. A worker that exits frees its heap first, which takes time proportional to what it still holds. `SCRIPTC_WORKER_STATS=1` prints, as each worker exits, how long it took to start, to run its script and to exit. +On Linux, a worker that exits without having shared an object it allocated with another thread returns all of its memory, including objects it never freed, such as `ProgramLifetime` instances. A worker shares objects when, for example, it sends a published value or frozen plain data, which other threads read by reference, or stores a value in an `InternTable`. The memory of a worker that shared stays allocated, as on other platforms. Set `SCR_SA_DISCARD=0` in the environment to keep the memory of every worker. + `publish` visits each object of the graph once and marks it immortal, so publication takes time proportional to the graph. Published objects then cost nothing on later sends, and reference counting and the cycle collector skip them. Stores into the types a program publishes carry one extra load and branch. Threads that add to one shared collection of published values while they run can use an [`InternTable`](/docs/experimental#interntable) from `@scriptc/experimental`. diff --git a/packages/compiler/src/backend/llvm/lib-filesystem.ts b/packages/compiler/src/backend/llvm/lib-filesystem.ts index 1083c69d5..b1d8869fd 100644 --- a/packages/compiler/src/backend/llvm/lib-filesystem.ts +++ b/packages/compiler/src/backend/llvm/lib-filesystem.ts @@ -746,6 +746,8 @@ export function emitPathUrlLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll const dict = B.tmp(); B.line(`${dict} = call ptr @${mangleRecordNew(e.type.shapeId)}()`); const out = host.own({ name: dict, type: e.type }); + // Node builds the result with a null prototype. + B.line(`store i8 1, ptr ${host.recordMapFlagsPtr(dict, e.type.shapeId)}`); const ovf = host.recordOvfPtr(dict, e.type.shapeId); B.line( `call void @scr_qs_parse_into(ptr ${ovf}, ptr ${args[0]!.name}, ptr ${args[1]!.name}, ptr ${args[2]!.name}, double ${args[3]!.name}, i32 ${strTag}, i32 ${arrTag})`, diff --git a/packages/compiler/src/diagnostics/render.test.ts b/packages/compiler/src/diagnostics/render.test.ts new file mode 100644 index 000000000..c5078dd0e --- /dev/null +++ b/packages/compiler/src/diagnostics/render.test.ts @@ -0,0 +1,56 @@ +import { expect, test } from "vitest"; +import type { ScrDiagnostic } from "./diagnostic.js"; +import { renderDiagnostic, renderDiagnostics } from "./render.js"; + +function diag(file: string, start: number, end: number): ScrDiagnostic { + return { code: "SC2020", message: "not supported", loc: { file, start, end } }; +} + +test("a short line renders whole", () => { + const text = "const a = 1;\nconst b = a.c;\nconst d = 2;"; + const start = text.indexOf("a.c"); + expect(renderDiagnostic(diag("m.ts", start, start + 3), { text })).toBe( + [ + "m.ts:2:11 - error SC2020: not supported", + "", + " 1 | const a = 1;", + " 2 | const b = a.c;", + " | ^~~", + " 3 | const d = 2;", + ].join("\n"), + ); +}); + +test("a long line renders a window around the column", () => { + const line = "x".repeat(1000) + "bad" + "y".repeat(1000); + const text = "first\n" + line + "\nlast"; + const start = text.indexOf("bad"); + const out = renderDiagnostic(diag("min.js", start, start + 3), { text }); + const lines = out.split("\n"); + expect(lines[0]).toBe("min.js:2:1001 - error SC2020: not supported"); + // the window starts 120 characters before the column + expect(lines[3]).toBe(` 2 | …${"x".repeat(120)}bad${"y".repeat(277)}…`); + expect(lines[4]).toBe(` | ${" ".repeat(121)}^~~`); + // context lines share the window: "first" and "last" start before it + expect(lines[2]).toBe(" 1 | …"); + expect(lines[5]).toBe(" 3 | …"); +}); + +test("a window at the line's end keeps the last characters", () => { + const line = "a".repeat(600) + "end"; + const start = line.indexOf("end"); + const out = renderDiagnostic(diag("e.js", start, start + 3), { text: line }); + const lines = out.split("\n"); + expect(lines[2]).toBe(` 1 | …${"a".repeat(397)}end`); + expect(lines[3]).toBe(` | ${" ".repeat(398)}^~~`); +}); + +test("thousands of diagnostics on a minified line render without exhausting string length", () => { + // one 2 MB line with 20,000 diagnostics: rendering every frame line whole + // would need tens of gigabytes + const text = "a;".repeat(1_000_000); + const diags = Array.from({ length: 20_000 }, (_, i) => diag("bundle.js", i * 100, i * 100 + 1)); + const out = renderDiagnostics(diags, new Map([["bundle.js", text]])); + expect(out.split(" - error SC2020: ").length - 1).toBe(20_000); + expect(out.length).toBeLessThan(20_000 * 1_000); +}); diff --git a/packages/compiler/src/diagnostics/render.ts b/packages/compiler/src/diagnostics/render.ts index 7fd68910f..e5a75c6bc 100644 --- a/packages/compiler/src/diagnostics/render.ts +++ b/packages/compiler/src/diagnostics/render.ts @@ -1,6 +1,8 @@ /* Terminal rendering: file:line:col, a hand-rolled code frame with ±1 line * of context and a ^~~~ underline sized to the span, optional hint line, - * and the optional attributed profile-note line after it. + * and the optional attributed profile-note line after it. A source line + * longer than FRAME_MAX_WIDTH (minified sources) shows a window of that + * width around the diagnostic's column, with "…" where text is cut. */ import type { ScrDiagnostic } from "./diagnostic.js"; @@ -11,6 +13,27 @@ export interface RenderOptions { interface SourceLookup { /** Full text of the file the diagnostic points into. */ text: string; + /** The text's lines and their start offsets; computed from `text` when + * absent. renderDiagnostics computes them once per file. */ + index?: LineIndex; +} + +interface LineIndex { + lines: string[]; + lineStarts: number[]; +} + +/** Frame lines longer than this many characters show a window of it. */ +const FRAME_MAX_WIDTH = 400; +/** Characters a windowed frame keeps before the diagnostic's column. */ +const FRAME_LEAD = 120; + +function lineIndex(text: string): LineIndex { + const lineStarts = [0]; + for (let i = 0; i < text.length; i++) { + if (text[i] === "\n") lineStarts.push(i + 1); + } + return { lines: text.split("\n"), lineStarts }; } const RED = "\x1b[31m"; @@ -33,32 +56,52 @@ export function renderDiagnostic( const frame: string[] = []; if (source) { - const lineStarts = [0]; - for (let i = 0; i < source.text.length; i++) { - if (source.text[i] === "\n") lineStarts.push(i + 1); - } + const { lines, lineStarts } = source.index ?? lineIndex(source.text); + // the last line starting at or before the diagnostic let lo = 0; - while (lo + 1 < lineStarts.length && lineStarts[lo + 1]! <= diag.loc.start) lo++; + let hi = lineStarts.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (lineStarts[mid]! <= diag.loc.start) lo = mid; + else hi = mid - 1; + } lineNum = lo + 1; colNum = diag.loc.start - lineStarts[lo]! + 1; - const lines = source.text.split("\n"); + const lineText = lines[lineNum - 1] ?? ""; + // The window every frame line shows: the whole line, or for a long + // line FRAME_MAX_WIDTH characters starting FRAME_LEAD before the column. + const windowStart = + lineText.length > FRAME_MAX_WIDTH + ? Math.max(0, Math.min(colNum - 1 - FRAME_LEAD, lineText.length - FRAME_MAX_WIDTH)) + : 0; + const shown = (text: string): string => { + if (windowStart === 0 && text.length <= FRAME_MAX_WIDTH) return text; + const end = windowStart + FRAME_MAX_WIDTH; + return ( + (windowStart > 0 ? "…" : "") + text.slice(windowStart, end) + (text.length > end ? "…" : "") + ); + }; const gutterWidth = String(Math.min(lineNum + 1, lines.length)).length; const emitLine = (n: number) => { const text = lines[n - 1]; if (text === undefined) return; - frame.push(c(DIM, ` ${String(n).padStart(gutterWidth)} | `) + text); + frame.push(c(DIM, ` ${String(n).padStart(gutterWidth)} | `) + shown(text)); }; emitLine(lineNum - 1); emitLine(lineNum); - const lineText = lines[lineNum - 1] ?? ""; + const caretCol = colNum - 1 - windowStart + (windowStart > 0 ? 1 : 0); + const visibleEnd = + windowStart === 0 && lineText.length <= FRAME_MAX_WIDTH + ? lineText.length + : Math.min(lineText.length, windowStart + FRAME_MAX_WIDTH); const spanOnLine = Math.max( 1, - Math.min(diag.loc.end - diag.loc.start, lineText.length - (colNum - 1)), + Math.min(diag.loc.end - diag.loc.start, visibleEnd - (colNum - 1)), ); frame.push( c(DIM, ` ${" ".repeat(gutterWidth)} | `) + - " ".repeat(colNum - 1) + + " ".repeat(caretCol) + c(/^SC6\d{3}$/.test(diag.code) ? YELLOW : RED, "^" + "~".repeat(spanOnLine - 1)), ); emitLine(lineNum + 1); @@ -94,10 +137,17 @@ export function renderDiagnostics( const sorted = [...diags].sort( (a, b) => a.loc.file.localeCompare(b.loc.file) || a.loc.start - b.loc.start, ); + const indexes = new Map(); return sorted .map((d) => { const text = sourceTextByFile.get(d.loc.file); - return renderDiagnostic(d, text === undefined ? undefined : { text }, opts); + if (text === undefined) return renderDiagnostic(d, undefined, opts); + let index = indexes.get(d.loc.file); + if (index === undefined) { + index = lineIndex(text); + indexes.set(d.loc.file, index); + } + return renderDiagnostic(d, { text, index }, opts); }) .join("\n\n"); } diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index 4f7344bd1..d14aaf924 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -756,6 +756,7 @@ export class ClassDynamicDispatch { const seenTypes = new Set(); const rewrite = new Set(); const toPrimitiveSites = new Set(); + const hasKeySites = new Set(); const discover = (type: IrType): void => { const key = typeKey(type); if (seenTypes.has(key)) return; @@ -818,6 +819,9 @@ export class ClassDynamicDispatch { ) rewrite.add(fn); break; + case "dynHasKey": + hasKeySites.add(fn); + break; case "dynKeyGet": case "dynInvoke": rewrite.add(fn); @@ -854,6 +858,8 @@ export class ClassDynamicDispatch { const converting = this.convertingHints(lowerer); const calling = this.calledConversions(lowerer); if (converting.size || calling.size) for (const fn of toPrimitiveSites) rewrite.add(fn); + const prototypeMembers = this.prototypeMemberNames(lowerer); + if (prototypeMembers.size) for (const fn of hasKeySites) rewrite.add(fn); for (const fn of functions) { if ( !fn.name.startsWith("%dyn.class.instanceof:") && @@ -1294,6 +1300,38 @@ export class ClassDynamicDispatch { fn.body = transformStmtList(fn.body, { stmt: (stmt) => stmt, expr: (expr) => { + if (expr.kind === "dynHasKey" && prototypeMembers.has(expr.key)) { + // A class instance's snapshot can carry its class's prototype + // data, which holds only the methods the program reads through + // the prototype. The computed-key form answers methods and + // accessors from the class itself. + const loc = expr.loc; + const test: IrExpr = { + kind: "libCall", + fn: "dyn.hasKeyComputed", + args: [ + expr.value, + { + kind: "dynFrom", + value: { kind: "strLit", value: expr.key, type: STRING, loc }, + type: DYN, + loc, + }, + ], + type: BOOL, + loc, + }; + changed = true; + return { + kind: "seqExpr", + stmts: [], + result: expr.negated + ? { kind: "unary", op: "!", operand: test, type: BOOL, loc } + : test, + type: BOOL, + loc, + }; + } const called = calling.size ? this.convertMethodCall(lowerer, fn, expr, calling) : null; if (called) { changed = true; @@ -3435,6 +3473,19 @@ export class ClassDynamicDispatch { }; } + /** The method and accessor names of boxed classes whose instances can + * carry their class's prototype data. */ + private prototypeMemberNames(lowerer: Lowerer): Set { + const names = new Set(); + for (const { info } of this.propertyReceivers(lowerer)) { + if (!hasClassPrototypeData(info)) continue; + for (let c: ClassInfo | null = info; c; c = c.base) + for (const name of c.methods.keys()) + names.add(name.startsWith("get:") || name.startsWith("set:") ? name.slice(4) : name); + } + return names; + } + /** The methods some boxed class declares among valueOf and toString. */ private calledConversions(lowerer: Lowerer): Set<"valueOf" | "toString"> { const methods = new Set<"valueOf" | "toString">(); diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index e78ae59cc..a4412f9d7 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1442,6 +1442,18 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { contextResolutions++; return viaIndexed; } + // A conditional type over a bound parameter (`Infer`, a + // mapped type's member): evaluate it against the bindings. + if (type.isConditionalType()) { + const evaluated = evalMappedTemplate(type, new Map(), new Map(), ctx); + if (evaluated && evaluated !== type) { + const viaConditional = mapType(evaluated, ctx); + if (viaConditional !== null) { + contextResolutions++; + return viaConditional; + } + } + } } const widened = checker.getBaseTypeOfLiteralType(type); const flags = widened.flags; @@ -4090,6 +4102,8 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { if (childShaped && core >= 2) return CHILD_T; } if (flags & (ts.TypeFlags.Object | ts.TypeFlags.Intersection) && callSigs.length === 0) { + const viaMapped = mapBoundMappedType(widened, ctx); + if (viaMapped) return viaMapped; const viaAlias = mapGenericUtilityAlias(widened, ctx); if (viaAlias) return viaAlias; const record = mapRecordType(widened, ctx); @@ -4729,6 +4743,187 @@ function mapGenericIndexedAccess(type: ts.Type, ctx: TypeMapperCtx): IrType | nu return covered.every((t) => typeEquals(t, first)) ? first : null; } +/** A mapped type over a type parameter of the instance being lowered (`{ + * [K in keyof S]: S[K] }`, `{ [K in keyof S]: Infer }`): the checker + * keeps it symbolic inside the generic body, but the instance binds `S`, + * so each property of the bound type maps through the template evaluated + * with `K` set to that property's key. Optionality follows the `?` + * modifier, or the bound property's own for a homomorphic mapping. Null + * when the shape is anything else or a template form does not evaluate. */ +function mapBoundMappedType(widened: ts.Type, ctx: TypeMapperCtx): IrType | null { + const { checker, resolveTypeParamTs, shapes, unions } = ctx; + if (!resolveTypeParamTs || !isMappedShape(widened)) return null; + const symbol = widened.getSymbol(); + const node = symbol + ? checker.declarationsOf(symbol).find((d): d is ts.MappedTypeNode => ts.isMappedTypeNode(d)) + : undefined; + const constraint = node?.typeParameter.constraint; + if ( + !node?.type || + !constraint || + !ts.isTypeOperatorNode(constraint) || + constraint.operator !== ts.SyntaxKind.KeyOfKeyword + ) + return null; + const source = checker.getTypeFromTypeNode(constraint.type); + if (!(source.flags & ts.TypeFlags.TypeParameter)) return null; + const bound = resolveTypeParamTs(source); + const key = checker.getSymbolAtLocation(node.typeParameter.name); + if (!bound || !key) return null; + const template = checker.getTypeFromTypeNode(node.type); + const modifier = node.questionToken?.kind; + const fields: { name: string; type: IrType }[] = []; + const order: string[] = []; + for (const property of checker.getPropertiesOfType(bound)) { + const name = property.name; + const evaluated = evalMappedTemplate(template, new Map([[key, name]]), new Map(), ctx); + let type = evaluated ? mapType(evaluated, ctx) : null; + if (!type) return null; + const optional = + modifier === ts.SyntaxKind.MinusToken + ? false + : modifier !== undefined || (property.flags & ts.SymbolFlags.Optional) !== 0; + if (optional) { + const armed = withUndefinedArm(type, unions); + if (!armed) return null; + type = armed; + } + fields.push({ name, type }); + order.push(name); + } + fields.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + return { kind: "record", shapeId: shapes.intern(fields, false, undefined, order) }; +} + +/** The checker type a mapped type's template stands for at one key: + * `keys` binds the mapped type parameter to a property name, `aliases` + * binds type-alias and `infer` parameters met on the way. Handles type + * parameters (through the instance's bindings), indexed accesses + * (`S[K]`, `S[K]["_output"]`) and conditional types whose `extends` side + * is a reference to the same generic type as the checked type + * (`S extends Schema ? T : never`); answers the type itself + * when it mentions none of the bound parameters, null otherwise. */ +function evalMappedTemplate( + type: ts.Type, + keys: Map, + aliases: Map, + ctx: TypeMapperCtx, +): ts.Type | null { + const { checker, resolveTypeParamTs } = ctx; + if (type.flags & ts.TypeFlags.TypeParameter) { + const symbol = type.getSymbol(); + if (symbol && keys.has(symbol)) return null; + if (symbol && aliases.has(symbol)) return aliases.get(symbol)!; + return resolveTypeParamTs?.(type) ?? null; + } + if (type.isIndexedAccessType()) { + const object = type.getObjectType(); + const index = type.getIndexType(); + if (!object || !index) return null; + const receiver = evalMappedTemplate(object, keys, aliases, ctx); + const indexSymbol = index.flags & ts.TypeFlags.TypeParameter ? index.getSymbol() : undefined; + const name = + indexSymbol && keys.has(indexSymbol) + ? keys.get(indexSymbol)! + : index.isStringLiteralType() + ? index.value + : index.isNumberLiteralType() + ? String(index.value) + : null; + if (!receiver || name === null) return null; + const property = checker.getPropertyOfType(receiver, name); + return property ? checker.getTypeOfSymbol(property) : null; + } + if (type.isConditionalType()) { + // A conditional reached through an alias (`Infer`) checks the + // alias's own parameters: bind them to the evaluated arguments. + let scope = aliases; + const alias = type.getAliasSymbol(); + const aliasArgs = type.getAliasTypeArguments(); + const aliasDecl = alias + ? checker + .declarationsOf(alias) + .find((d): d is ts.TypeAliasDeclaration => ts.isTypeAliasDeclaration(d)) + : undefined; + if (aliasDecl?.typeParameters && aliasArgs.length === aliasDecl.typeParameters.length) { + scope = new Map(aliases); + for (let i = 0; i < aliasArgs.length; i++) { + const param = checker.getSymbolAtLocation(aliasDecl.typeParameters[i]!.name); + const arg = evalMappedTemplate(aliasArgs[i]!, keys, aliases, ctx); + if (!param || !arg) return null; + scope.set(param, arg); + } + } + const checkType = type.getCheckType(); + const extendsType = type.getExtendsType(); + if (!checkType || !extendsType) return null; + const checked = evalMappedTemplate(checkType, keys, scope, ctx); + if (!checked) return null; + const inferred = matchInferReference(checked, extendsType, ctx); + if (inferred === undefined) return null; + const branch = inferred ? type.getTrueType() : type.getFalseType(); + const branchScope = inferred ? new Map([...scope, ...inferred]) : scope; + return evalMappedTemplate(branch, keys, branchScope, ctx); + } + return mentionsMappedParams(type, keys, aliases) ? null : type; +} + +/** `checked extends G<..., infer T, ...>`: the infer bindings when + * `checked` is a reference to the same generic `G` (its arguments bind the + * infer positions; other positions must be the same types), false when it + * is a reference to another type or a primitive, undefined when the match + * cannot be decided this way. */ +function matchInferReference( + checked: ts.Type, + extendsType: ts.Type, + ctx: TypeMapperCtx, +): Map | false | undefined { + const { checker } = ctx; + const target = (t: ts.Type): ts.Type | undefined => + t.isTypeReference?.() ? (t as ts.TypeReference).getTarget?.() : undefined; + const wantTarget = target(extendsType) ?? extendsType.getAliasSymbol(); + if (!wantTarget) return undefined; + const gotTarget = target(checked) ?? checked.getAliasSymbol(); + if (!gotTarget) return checked.flags & ts.TypeFlags.Object ? undefined : false; + if (gotTarget !== wantTarget) return undefined; + const wantArgs = target(extendsType) + ? checker.getTypeArguments(extendsType as ts.TypeReference) + : extendsType.getAliasTypeArguments(); + const gotArgs = target(checked) + ? checker.getTypeArguments(checked as ts.TypeReference) + : checked.getAliasTypeArguments(); + if (wantArgs.length !== gotArgs.length) return undefined; + const inferred = new Map(); + for (let i = 0; i < wantArgs.length; i++) { + const want = wantArgs[i]!; + const symbol = want.flags & ts.TypeFlags.TypeParameter ? want.getSymbol() : undefined; + const isInfer = symbol + ? checker + .declarationsOf(symbol) + .some((d) => d.parent !== undefined && ts.isInferTypeNode(d.parent)) + : false; + if (isInfer && symbol) inferred.set(symbol, gotArgs[i]!); + else if (want !== gotArgs[i]) return undefined; + } + return inferred; +} + +/** The type mentions a mapped key or bound alias parameter (a template + * shape evalMappedTemplate does not reduce). */ +function mentionsMappedParams( + type: ts.Type, + keys: Map, + aliases: Map, +): boolean { + if (type.flags & ts.TypeFlags.TypeParameter) { + const symbol = type.getSymbol(); + return symbol !== undefined && (keys.has(symbol) || aliases.has(symbol)); + } + if (type.isUnionType() || type.isIntersectionType()) + return ts.constituentTypes(type).some((part) => mentionsMappedParams(part, keys, aliases)); + return false; +} + function mapGenericUtilityAlias(widened: ts.Type, ctx: TypeMapperCtx): IrType | null { const { resolveTypeParam, shapes, unions } = ctx; if (!resolveTypeParam) return null; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5842-qs-parse-null-prototype.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5842-qs-parse-null-prototype.ts.json new file mode 100644 index 000000000..a5b4a3e21 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5842-qs-parse-null-prototype.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5842-qs-parse-null-prototype.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5843-qs-parse-null-prototype-js.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5843-qs-parse-null-prototype-js.mjs.json new file mode 100644 index 000000000..228f0a4c0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5843-qs-parse-null-prototype-js.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5843-qs-parse-null-prototype-js.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5914-dyn-in-class-prototype-members.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5914-dyn-in-class-prototype-members.ts.json new file mode 100644 index 000000000..4df0c23de --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5914-dyn-in-class-prototype-members.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5914-dyn-in-class-prototype-members.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5960-generic-mapped-types.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5960-generic-mapped-types.ts.json new file mode 100644 index 000000000..a95837f80 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5960-generic-mapped-types.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5960-generic-mapped-types.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/native-worker-discarded-heaps.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/native-worker-discarded-heaps.ts.json new file mode 100644 index 000000000..b7dd8fd17 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/native-worker-discarded-heaps.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/native-worker-discarded-heaps.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_alloc.c b/packages/runtime/src/scr_alloc.c index eaf391a9e..2959e642d 100644 --- a/packages/runtime/src/scr_alloc.c +++ b/packages/runtime/src/scr_alloc.c @@ -630,6 +630,13 @@ static bool scr_sa_reserve_region(void) { * scr_sa_trim). */ static size_t scr_sa_claim_carved[SCR_SA_ARENAS]; static size_t scr_sa_carved(void); +/* A worker thread's memory, discarded when it parks (see "Discarding a + * worker's memory" below). */ +typedef struct ScrSaKeep ScrSaKeep; +static _Thread_local bool scr_sa_worker; /* set by scr_sa_thread_starting */ +static _Thread_local ScrSaKeep *scr_sa_keep; /* while the worker's memory is kept apart */ +static void scr_sa_keep_begin(unsigned i); +static void scr_sa_keep_drained(ScrSaKeep *k, ScrSaBlock *b, unsigned c); #endif /* Moves every block other threads freed into this arena's free lists. */ @@ -638,6 +645,13 @@ static void scr_sa_drain_remote(void) { while (b) { ScrSaBlock *next = b->next; unsigned c = (unsigned)(((uintptr_t)b - scr_sa.base) >> scr_sa.shift); +#ifdef SCR_SA_TRIM + if (scr_sa_keep) { + scr_sa_keep_drained(scr_sa_keep, b, c); + b = next; + continue; + } +#endif b->next = scr_sa.free[c]; scr_sa.free[c] = b; b = next; @@ -672,6 +686,7 @@ static bool scr_sa_reserve(void) { scr_sa_drain_remote(); #ifdef SCR_SA_TRIM scr_sa_claim_carved[i] = scr_sa_carved(); + if (scr_sa_worker) scr_sa_keep_begin(i); #endif return true; } @@ -884,6 +899,303 @@ static void scr_sa_trim(void) { scr_sa_arena, (int)trim, scr_sa_trim_stats[0] >> 20, scr_sa_trim_stats[1] >> 20, scr_sa_trim_stats[2] >> 20); } + +/* Discarding a worker's memory (Linux). Script objects stay on the thread + * that allocated them: other threads reach one only through a message or an + * intern table that shares a published (immortal) graph by pointer, or a + * process-wide immortal cache, and each of those calls + * scr_sa_thread_shares. Runtime state that outlives a thread (messages, + * mailboxes, shared bytes, intern table entries) comes from the system + * allocator directly. So when a worker thread parks without having shared, + * nothing it allocated through scr_mem_* can be reachable from another + * thread, and what it never freed (ProgramLifetime instances, published + * module constants) is garbage. Such a thread's memory goes back whole: + * - when a worker claims an arena, the arena's free lists and returned + * regions are set aside, and the worker carves only past each slice's + * frontier, so its blocks of class c are [start[c], bump[c]), plus the + * blocks it takes from the free lists set aside once a slice has no bump + * room left, which it records; + * - the system blocks it allocates (large blocks, an exhausted slice) are + * recorded until it frees them; + * - a remote free into its blocks means another thread held one: the same + * as sharing; + * - when it parks without having shared, the trim helper returns the pages + * it carved (MADV_DONTNEED; the start of the first page, which it may + * share with older blocks, is cleared), frees the recorded system blocks, + * and parks the arena as the worker found it, with the blocks it took + * free again. A worker that shared parks + * its arena with the blocks set aside added to its free lists, and the + * ordinary trim applies. + * SCR_SA_DISCARD=0 keeps every worker's memory. SCR_SA_DISCARD=poison makes + * the carved pages inaccessible instead, and keeps the system blocks, so + * that a reference to discarded memory faults (a check for tests). */ +#define SCR_SA_SYS_GONE ((void *)1) /* a removed entry of ScrSaKeep.sys */ + +struct ScrSaKeep { + /* The arena as the worker claimed it: free lists (to which remote frees + * of older blocks are added), carve cursors and returned regions. */ + ScrSaBlock *free[SCR_SA_NCLASS]; + char *bump[SCR_SA_NCLASS], *lim[SCR_SA_NCLASS]; + ScrSaHoles holes[SCR_SA_NCLASS]; + /* Each slice's frontier then (bump and limit, NULL limits for a slice not + * opened yet), and the first address the worker carves in it. */ + char *front_bump[SCR_SA_NCLASS], *front_lim[SCR_SA_NCLASS]; + char *start[SCR_SA_NCLASS]; + /* The worker's system blocks: an open-addressing set of `cap` slots (a + * power of two, or 0), NULL for an empty slot and SCR_SA_SYS_GONE for a + * removed entry; `used` counts both kinds of filled slots. */ + void **sys; + size_t cap, used, live; + /* The blocks of the free lists set aside that the worker took once a slice + * had no bump room left, so that older free blocks are used before the + * slice grows; they go back to those lists with the worker's memory. */ + ScrSaBlock **taken; + size_t ntaken, cap_taken; + bool shared; + bool poison; +}; + +static _Thread_local bool scr_sa_discarded; /* this thread's memory went back */ +static ScrSaKeep *scr_sa_parked_keep[SCR_SA_ARENAS]; /* handed to the helper */ + +/* SCR_SA_DISCARD: 0 off, 1 on (the default), 2 poison. */ +static int scr_sa_discard_mode(void) { + const char *mode = getenv("SCR_SA_DISCARD"); + if (!mode) return 1; + if (strcmp(mode, "0") == 0) return 0; + return strcmp(mode, "poison") == 0 ? 2 : 1; +} + +static void scr_sa_keep_begin(unsigned i) { + int mode = scr_sa_discard_mode(); + if (!mode) return; + ScrSaKeep *k = calloc(1, sizeof *k); + if (!k) return; + k->poison = mode == 2; + for (unsigned c = 0; c < SCR_SA_NCLASS; c++) { + ScrSaHoles *h = &scr_sa_holes[i][c]; + k->free[c] = scr_sa.free[c]; + k->bump[c] = scr_sa.bump[c]; + k->lim[c] = scr_sa.lim[c]; + k->holes[c] = *h; + memset(h, 0, sizeof *h); + scr_sa.free[c] = NULL; + /* While the slice carves a returned region, its frontier is kept apart. */ + if (k->holes[c].lim) { + scr_sa.bump[c] = k->holes[c].bump; + scr_sa.lim[c] = k->holes[c].lim; + } + k->front_bump[c] = scr_sa.bump[c]; + k->front_lim[c] = scr_sa.lim[c]; + k->start[c] = scr_sa.lim[c] ? scr_sa.bump[c] : (char *)scr_sa.base + ((size_t)c << scr_sa.shift); + } + scr_sa_keep = k; +} + +/* A remote free of class c, drained while the worker keeps its memory + * apart: one of its own blocks makes it share; an older block goes to the + * free lists set aside. Once the worker has taken blocks from those lists, + * any older block might be one of them, so it shares as well. */ +static void scr_sa_keep_drained(ScrSaKeep *k, ScrSaBlock *b, unsigned c) { + if ((char *)b >= k->start[c] && (char *)b < scr_sa.bump[c]) { + k->shared = true; + b->next = scr_sa.free[c]; + scr_sa.free[c] = b; + } else { + if (k->ntaken) k->shared = true; + b->next = k->free[c]; + k->free[c] = b; + } +} + +/* A block of class c from the free lists set aside, for a worker whose + * slice has no bump room left; NULL when there is none or it cannot be + * recorded. */ +static ScrSaBlock *scr_sa_keep_take(ScrSaKeep *k, unsigned c) { + ScrSaBlock *b = k->free[c]; + if (!b) return NULL; + if (k->ntaken == k->cap_taken) { + size_t cap = k->cap_taken ? 2 * k->cap_taken : 256; + ScrSaBlock **taken = realloc(k->taken, cap * sizeof *taken); + if (!taken) return NULL; + k->taken = taken; + k->cap_taken = cap; + } + k->free[c] = b->next; + k->taken[k->ntaken++] = b; + return b; +} + +static size_t scr_sa_sys_slot(const void *p, size_t cap) { + return (size_t)((((uint64_t)(uintptr_t)p >> 4) * UINT64_C(0x9E3779B97F4A7C15)) >> 32) & (cap - 1); +} + +/* Records system block p. A worker whose set cannot grow keeps its memory. */ +static void scr_sa_keep_add(ScrSaKeep *k, void *p) { + if ((k->used + 1) * 2 > k->cap) { + size_t cap = 64; + while (cap < (k->live + 1) * 4) cap *= 2; + void **sys = calloc(cap, sizeof *sys); + if (!sys) { + k->shared = true; + return; + } + for (size_t s = 0; s < k->cap; s++) { + void *q = k->sys[s]; + if (!q || q == SCR_SA_SYS_GONE) continue; + size_t j = scr_sa_sys_slot(q, cap); + while (sys[j]) j = (j + 1) & (cap - 1); + sys[j] = q; + } + free(k->sys); + k->sys = sys; + k->cap = cap; + k->used = k->live; + } + size_t j = scr_sa_sys_slot(p, k->cap); + while (k->sys[j] && k->sys[j] != SCR_SA_SYS_GONE) j = (j + 1) & (k->cap - 1); + if (!k->sys[j]) k->used++; + k->sys[j] = p; + k->live++; +} + +/* Forgets system block p; whether it was recorded. */ +static bool scr_sa_keep_remove(ScrSaKeep *k, const void *p) { + if (!k->live) return false; + for (size_t j = scr_sa_sys_slot(p, k->cap); k->sys[j]; j = (j + 1) & (k->cap - 1)) { + if (k->sys[j] != p) continue; + k->sys[j] = SCR_SA_SYS_GONE; + k->live--; + return true; + } + return false; +} + +/* The calling thread's arena keeps the worker's memory (it shared): the + * free lists set aside go back, followed by the worker's own free blocks, + * and so do the returned regions, past which the worker carved. */ +static void scr_sa_keep_merge(ScrSaKeep *k) { + for (unsigned c = 0; c < SCR_SA_NCLASS; c++) { + ScrSaBlock *kept = k->free[c]; + if (kept) { + ScrSaBlock *tail = kept; + while (tail->next) tail = tail->next; + tail->next = scr_sa.free[c]; + scr_sa.free[c] = kept; + } + ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c]; + *h = k->holes[c]; + if (h->lim) { + h->bump = scr_sa.bump[c]; + h->lim = scr_sa.lim[c]; + scr_sa.bump[c] = k->bump[c]; + scr_sa.lim[c] = k->lim[c]; + } + } + free(k->sys); + free(k->taken); + free(k); +} + +/* Returns the memory of the worker whose arena the calling thread (the + * trim helper) holds, and restores the arena to its state at the worker's + * claim; with poison, makes the pages inaccessible and moves the frontier + * past them instead. */ +static void scr_sa_discard(ScrSaKeep *k) { + /* Remote frees since the worker's last drain. */ + scr_sa_keep = k; + scr_sa_drain_remote(); + scr_sa_keep = NULL; + if (k->shared) { + scr_sa_keep_merge(k); + scr_sa_trim(); + return; + } + uintptr_t page = (uintptr_t)sysconf(_SC_PAGESIZE); + if (page == 0 || page == (uintptr_t)-1) page = 4096; + size_t returned = 0, sys_bytes = 0, sys_blocks = 0; + for (unsigned c = 0; c < SCR_SA_NCLASS; c++) { + char *lo = k->start[c], *top = scr_sa.bump[c], *lim = scr_sa.lim[c]; + char *bump = k->front_bump[c], *limit = k->front_lim[c]; + if (lim && top > lo) { + char *first = (char *)(((uintptr_t)lo + page - 1) & ~(page - 1)); + char *used = (char *)(((uintptr_t)top + page - 1) & ~(page - 1)); + /* The part of a page that may also hold older blocks. */ + if (first > lo) memset(lo, 0, (size_t)((first < top ? first : top) - lo)); + if (k->poison) { + if (used > first) (void)mprotect(first, (size_t)(used - first), PROT_NONE); + bump = used > first ? used : top; + limit = lim > bump ? lim : bump; + } else { + char *end = (char *)(((uintptr_t)lim + page - 1) & ~(page - 1)); + if (end > first) { + (void)madvise(first, (size_t)(end - first), MADV_DONTNEED); + returned += (size_t)(end - first); + } + } + } else { + bump = top; + limit = lim; + } + scr_sa.free[c] = k->free[c]; + ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c]; + *h = k->holes[c]; + if (h->lim) { + h->bump = bump; + h->lim = limit; + scr_sa.bump[c] = k->bump[c]; + scr_sa.lim[c] = k->lim[c]; + } else { + scr_sa.bump[c] = bump; + scr_sa.lim[c] = limit; + } + } + /* The older free blocks the worker took are free again. */ + if (!k->poison) + for (size_t t = 0; t < k->ntaken; t++) { + ScrSaBlock *b = k->taken[t]; + unsigned c = (unsigned)(((uintptr_t)b - scr_sa.base) >> scr_sa.shift); + b->next = scr_sa.free[c]; + scr_sa.free[c] = b; + } + if (!k->poison) + for (size_t s = 0; s < k->cap; s++) { + void *p = k->sys[s]; + if (!p || p == SCR_SA_SYS_GONE) continue; +#if defined(__GLIBC__) + sys_bytes += malloc_usable_size(p); +#endif + sys_blocks++; + free(p); + } +#if defined(__GLIBC__) + if (sys_blocks) (void)malloc_trim(0); +#endif + const char *stats = getenv("SCR_SA_STATS"); + if (stats && stats[0] == '1') + fprintf(stderr, "scr_sa: discard arena %d returned %zu MB, %zu system blocks (%zu MB)\n", scr_sa_arena, + returned >> 20, sys_blocks, sys_bytes >> 20); + free(k->sys); + free(k->taken); + free(k); +} + +/* Hooks of the allocator's system paths and of sharing (scr_rc.h). */ +static void scr_sa_keep_note_system(void *p) { + if (p && scr_sa_keep) scr_sa_keep_add(scr_sa_keep, p); +} + +void scr_sa_thread_shares(void) { + if (scr_sa_keep) scr_sa_keep->shared = true; +} + +/* Library archives run on their embedder's threads and keep their memory. */ +static void scr_sa_worker_starting(void) { +#ifndef SCR_LIB + scr_sa_worker = true; +#endif +} +#define SCR_SA_WORKER_STARTING() scr_sa_worker_starting() #endif #ifdef SCR_SA_TRIM @@ -906,12 +1218,36 @@ static void *scr_sa_trim_main(void *arg) { unsigned i = (unsigned)(uintptr_t)arg; scr_sa_arena = (int)i; scr_sa = scr_sa_parked[i]; - scr_sa_trim(); + ScrSaKeep *k = scr_sa_parked_keep[i]; + scr_sa_parked_keep[i] = NULL; + if (k) scr_sa_discard(k); + else scr_sa_trim(); scr_sa_release(i); atomic_fetch_sub_explicit(&scr_sa_trimming, 1, memory_order_release); return NULL; } +/* Hands the calling thread's arena to a trim helper (with `k`, to discard + * the worker's memory); false if no helper could start. */ +static bool scr_sa_start_helper(unsigned i, ScrSaKeep *k) { + scr_sa_parked[i] = scr_sa; + scr_sa_parked_keep[i] = k; + atomic_fetch_add_explicit(&scr_sa_trimming, 1, memory_order_relaxed); + pthread_attr_t attr; + pthread_t helper; + bool started = pthread_attr_init(&attr) == 0; + if (started) { + started = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) == 0 && + pthread_create(&helper, &attr, scr_sa_trim_main, (void *)(uintptr_t)i) == 0; + pthread_attr_destroy(&attr); + } + if (!started) { + scr_sa_parked_keep[i] = NULL; + atomic_fetch_sub_explicit(&scr_sa_trimming, 1, memory_order_relaxed); + } + return started; +} + void scr_sa_wait_trims(void) { while (atomic_load_explicit(&scr_sa_trimming, memory_order_acquire)) { struct timespec pause = {0, 1000000}; @@ -920,6 +1256,7 @@ void scr_sa_wait_trims(void) { } #else void scr_sa_wait_trims(void) {} +void scr_sa_thread_shares(void) {} #endif void scr_sa_thread_park(void) { @@ -929,18 +1266,25 @@ void scr_sa_thread_park(void) { scr_sa_pf_park(); #endif #ifdef SCR_SA_TRIM - if (scr_sa_trim_worth()) { - scr_sa_parked[i] = scr_sa; - atomic_fetch_add_explicit(&scr_sa_trimming, 1, memory_order_relaxed); - pthread_attr_t attr; - pthread_t helper; - bool started = pthread_attr_init(&attr) == 0; - if (started) { - started = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) == 0 && - pthread_create(&helper, &attr, scr_sa_trim_main, (void *)(uintptr_t)i) == 0; - pthread_attr_destroy(&attr); + ScrSaKeep *k = scr_sa_keep; + scr_sa_keep = NULL; + if (k && !k->shared) { + /* Nothing this thread allocated is reachable from another one: its + * memory goes back, and later frees on this thread are dropped. */ + scr_sa_discarded = true; + if (!scr_sa_start_helper(i, k)) { + scr_sa_discard(k); + scr_sa_release(i); + return; } - if (started) { + memset(&scr_sa, 0, sizeof scr_sa); + scr_sa_arena = -1; + scr_sa_refused = true; + return; + } + if (k) scr_sa_keep_merge(k); + if (scr_sa_trim_worth()) { + if (scr_sa_start_helper(i, NULL)) { /* Later frees on this thread take the foreign path into the arena's * remote list; the helper releases the claim. */ memset(&scr_sa, 0, sizeof scr_sa); @@ -948,7 +1292,6 @@ void scr_sa_thread_park(void) { scr_sa_refused = true; return; } - atomic_fetch_sub_explicit(&scr_sa_trimming, 1, memory_order_relaxed); scr_sa_trim(); } #endif @@ -971,6 +1314,9 @@ static size_t scr_sa_foreign_size(const void *p) { } void scr_sa_free_foreign(void *p) { +#ifdef SCR_SA_TRIM + if (scr_sa_discarded) return; /* the block went back with the thread's memory */ +#endif uintptr_t region = atomic_load_explicit(&scr_sa_region, memory_order_acquire); uintptr_t off = (uintptr_t)p - region; if (region && off < scr_sa_arena_span * SCR_SA_ARENAS) { @@ -982,6 +1328,9 @@ void scr_sa_free_foreign(void *p) { memory_order_relaxed)); return; } +#ifdef SCR_SA_TRIM + if (scr_sa_keep) (void)scr_sa_keep_remove(scr_sa_keep, p); +#endif free(p); } #else @@ -1020,6 +1369,15 @@ __attribute__((cold, noinline)) void *scr_sa_slow(size_t n, bool zero) { if (zero) memset(b, 0, n); return b; } +#ifdef SCR_SA_TRIM + /* A worker that keeps its memory apart uses the arena's older free + * blocks before its slice grows. */ + if (scr_sa_keep && (size_t)(scr_sa.lim[c] - scr_sa.bump[c]) < scr_sa_class_size(c) && + (b = scr_sa_keep_take(scr_sa_keep, c)) != NULL) { + if (zero) memset(b, 0, n); + return b; + } +#endif #endif /* Carve from the slice, opening it or raising its limit as needed; * fresh pages are zero. */ @@ -1051,6 +1409,9 @@ __attribute__((cold, noinline)) void *scr_sa_slow(size_t n, bool zero) { SCR_SA_LARGE_NOTE(q, n); SCR_SA_THREAD_LARGE_NOTE(); } +#ifdef SCR_SA_TRIM + scr_sa_keep_note_system(q); +#endif return q; } @@ -1065,8 +1426,14 @@ void *scr_mem_realloc(void *p, size_t n) { } #endif else { +#ifdef SCR_SA_TRIM + bool kept = scr_sa_keep && scr_sa_keep_remove(scr_sa_keep, p); +#endif void *q = realloc(p, n); if (n > SCR_SA_MAX) SCR_SA_LARGE_NOTE(q, n); +#ifdef SCR_SA_TRIM + if (kept) scr_sa_keep_note_system(q ? q : p); /* a failed realloc leaves p */ +#endif return q; } /* Still fits its block, which is aligned for n (see the contract). */ @@ -1082,10 +1449,15 @@ void *scr_mem_realloc(void *p, size_t n) { return realloc(p, n); } #ifdef SCR_WORKERS void scr_sa_thread_park(void) {} void scr_sa_wait_trims(void) {} +void scr_sa_thread_shares(void) {} void scr_sa_free_foreign(void *p) { free(p); } #endif #endif +#ifndef SCR_SA_WORKER_STARTING +#define SCR_SA_WORKER_STARTING() ((void)0) +#endif + #ifdef SCR_WORKERS /* Blocks above SCR_SA_MAX come from glibc, which gives each thread that * allocates its own arena: a heap reserved up front and made usable with @@ -1116,7 +1488,10 @@ void scr_sa_free_foreign(void *p) { free(p); } * that are not workers (they never reserve). */ static _Thread_local unsigned scr_sa_thread_large; -void scr_sa_thread_starting(void) { scr_sa_thread_large = 1; } +void scr_sa_thread_starting(void) { + scr_sa_thread_large = 1; + SCR_SA_WORKER_STARTING(); +} static void scr_sa_thread_reserve(void) { void *blocks[SCR_SA_THREAD_HEAP_RESERVE / SCR_SA_THREAD_HEAP_STEP + 1]; @@ -1136,7 +1511,7 @@ static void scr_sa_thread_large_note(void) { if (count == SCR_SA_THREAD_RESERVE_AFTER) scr_sa_thread_reserve(); } #else -void scr_sa_thread_starting(void) {} +void scr_sa_thread_starting(void) { SCR_SA_WORKER_STARTING(); } #endif #endif diff --git a/packages/runtime/src/scr_alloc.test.c b/packages/runtime/src/scr_alloc.test.c index 26300296e..c367c9a61 100644 --- a/packages/runtime/src/scr_alloc.test.c +++ b/packages/runtime/src/scr_alloc.test.c @@ -373,6 +373,273 @@ static void check_trim(void) { } #endif +#if defined(SCR_WORKERS) && SCR_SMALL_ALLOC && defined(__linux__) +#include +#include +#include +#include +#include + +/* A worker thread (scr_sa_thread_starting) that exits without sharing + * gives its memory back: the pages it carved, and the system blocks it + * still holds. Older blocks of the arena, and the memory of a worker that + * shared (or got a remote free of its own blocks), stay. */ +/* DISCARD_SIZE and POISON_SIZE are classes the earlier checks leave unused + * in the workers' arena, so their state there is known. */ +enum { DISCARD_BLOCKS = 1 << 20, DISCARD_SIZE = 40, POISON_SIZE = 56, DISCARD_OLD = 64, DISCARD_LARGE = 64 }; +enum { DISCARD_LARGE_SIZE = 64 << 10 }; +static unsigned char *old_blocks[DISCARD_OLD]; +static unsigned char **discard_blocks; +static unsigned char *discard_large[DISCARD_LARGE]; +static uintptr_t discard_base, discard_lo, discard_hi; +static size_t block_size = DISCARD_SIZE; /* of the workers' and reusing threads' blocks */ +/* The end of the highest block of class DISCARD_SIZE so far, and how many blocks of a run lay past the mark + * at its start: those the slice grew for. */ +static uintptr_t class_hi; +static size_t discard_beyond, reuse_beyond; +static bool discard_shares; +static atomic_int discard_step; + +static void wait_step(int step) { + while (atomic_load(&discard_step) < step) { + struct timespec pause = {0, 100000}; + nanosleep(&pause, NULL); + } +} + +/* A plain script thread whose blocks stay live in its parked arena. */ +static void *discard_old_owner(void *arg) { + (void)arg; + for (unsigned i = 0; i < DISCARD_OLD; i++) { + old_blocks[i] = scr_mem_alloc(DISCARD_SIZE); + assert(owned(old_blocks[i])); + fill(old_blocks[i], DISCARD_SIZE, (unsigned char)i); + } + discard_base = scr_sa.base; + scr_sa_thread_park(); + return NULL; +} + +static void allocate_discardable(void) { + discard_lo = UINTPTR_MAX; + discard_hi = 0; + discard_beyond = 0; + for (size_t i = 0; i < DISCARD_BLOCKS; i++) { + unsigned char *p = scr_mem_alloc(block_size); + assert(p && owned(p)); + fill(p, block_size, (unsigned char)i); + discard_blocks[i] = p; + if ((uintptr_t)p >= class_hi) discard_beyond++; + if ((uintptr_t)p < discard_lo) discard_lo = (uintptr_t)p; + if ((uintptr_t)p + block_size > discard_hi) discard_hi = (uintptr_t)p + block_size; + } + if (block_size == DISCARD_SIZE && discard_hi > class_hi) class_hi = discard_hi; + /* System blocks: some freed, some moved by realloc, the rest kept. */ + for (unsigned i = 0; i < DISCARD_LARGE; i++) { + discard_large[i] = scr_mem_alloc(DISCARD_LARGE_SIZE); + assert(discard_large[i] && !owned(discard_large[i])); + fill(discard_large[i], 64, (unsigned char)i); + } + for (unsigned i = 0; i < DISCARD_LARGE; i += 4) { + scr_mem_free(discard_large[i]); + discard_large[i] = NULL; + } + for (unsigned i = 1; i < DISCARD_LARGE; i += 4) discard_large[i] = scr_mem_realloc(discard_large[i], 2 * DISCARD_LARGE_SIZE); +} + +static void *discard_worker(void *arg) { + (void)arg; + scr_sa_thread_starting(); + allocate_discardable(); + assert(scr_sa.base == discard_base); + /* None of the arena's older blocks or free blocks is reused. */ + for (unsigned i = 0; i < DISCARD_OLD; i++) assert(discard_lo > (uintptr_t)old_blocks[i]); + if (discard_shares) scr_sa_thread_shares(); + scr_sa_thread_park(); + return NULL; +} + +/* A worker one of whose blocks the main thread frees while it runs. */ +static void *remote_freed_worker(void *arg) { + (void)arg; + scr_sa_thread_starting(); + allocate_discardable(); + atomic_store(&discard_step, 1); + wait_step(2); + /* A class this thread has not used: its first block takes the slow path, + * which drains the remote free. */ + void *p = scr_mem_alloc(200); + assert(p); + scr_sa_thread_park(); + return NULL; +} + +/* A plain thread that takes as many blocks again from the arena: carved + * ones must be zero (the fast path does not clear them). */ +static size_t reuse_outside; +static uintptr_t reuse_hi; +static void *discard_reuse(void *arg) { + (void)arg; + reuse_outside = 0; + reuse_hi = 0; + reuse_beyond = 0; + for (size_t i = 0; i < DISCARD_BLOCKS; i++) { + unsigned char *p = scr_mem_calloc(block_size); + assert(p && owned(p)); + assert(scr_sa.base == discard_base); + for (size_t k = 0; k < block_size; k++) assert(p[k] == 0); + for (unsigned j = 0; j < DISCARD_OLD; j++) assert(p != old_blocks[j]); + if ((uintptr_t)p < discard_lo || (uintptr_t)p >= discard_hi) reuse_outside++; + if ((uintptr_t)p + block_size > reuse_hi) reuse_hi = (uintptr_t)p + block_size; + if ((uintptr_t)p >= class_hi) reuse_beyond++; + discard_blocks[i] = p; + } + if (block_size == DISCARD_SIZE && reuse_hi > class_hi) class_hi = reuse_hi; + for (size_t i = 0; i < DISCARD_BLOCKS; i++) scr_mem_free(discard_blocks[i]); + scr_sa_thread_park(); + return NULL; +} + +static void run(void *(*fn)(void *)) { + pthread_t t; + assert(pthread_create(&t, NULL, fn, NULL) == 0); + assert(pthread_join(t, NULL) == 0); + scr_sa_wait_trims(); +} + +static void verify_old_blocks(void) { + for (unsigned i = 0; i < DISCARD_OLD; i++) verify(old_blocks[i], DISCARD_SIZE, (unsigned char)i); +} + +/* The worker's blocks stayed as it left them, and the next thread to claim + * the arena carves past them; then they are freed (remote frees). */ +static void verify_kept(void) { + for (size_t i = 0; i < DISCARD_BLOCKS; i++) verify(discard_blocks[i], block_size, (unsigned char)i); + for (unsigned i = 0; i < DISCARD_LARGE; i++) { + if (!discard_large[i]) continue; + verify(discard_large[i], 64, (unsigned char)i); + free(discard_large[i]); + } + for (size_t i = 0; i < DISCARD_BLOCKS; i++) scr_mem_free(discard_blocks[i]); +} + +/* The protection of the mapping that holds address a, from /proc/self/maps. */ +static bool readable(uintptr_t a) { + FILE *maps = fopen("/proc/self/maps", "r"); + assert(maps); + char line[512]; + bool found = false, result = false; + while (!found && fgets(line, sizeof line, maps)) { + unsigned long lo, hi; + char perms[8]; + if (sscanf(line, "%lx-%lx %7s", &lo, &hi, perms) != 3) continue; + if (a >= lo && a < hi) { + found = true; + result = perms[0] == 'r'; + } + } + fclose(maps); + assert(found); + return result; +} + +static void check_discard(void) { + discard_blocks = malloc(DISCARD_BLOCKS * sizeof *discard_blocks); + assert(discard_blocks); + run(discard_old_owner); + size_t page = (size_t)sysconf(_SC_PAGESIZE); + + /* Discarded: its pages are returned, and carved again (zeroed) by the next + * thread, which does not grow the slice. The arena's older blocks stay. */ + discard_shares = false; + run(discard_worker); + verify_old_blocks(); + uintptr_t first = (discard_lo + page - 1) & ~(uintptr_t)(page - 1); + /* Pre-fault requests still in flight may fault in a few megabytes again. */ + size_t total = (discard_hi - first) / page; + assert(resident_pages(first, discard_hi) < total / 2); + run(discard_reuse); + assert(reuse_outside < 256); + verify_old_blocks(); + + /* SCR_SA_DISCARD=poison makes the pages inaccessible; the next thread + * carves past them. */ + block_size = POISON_SIZE; + setenv("SCR_SA_DISCARD", "poison", 1); + run(discard_worker); + unsetenv("SCR_SA_DISCARD"); + verify_old_blocks(); + first = (discard_lo + page - 1) & ~(uintptr_t)(page - 1); + assert(!readable(first)); + assert(!readable((discard_hi - 1) & ~(uintptr_t)(page - 1))); + run(discard_reuse); + assert(reuse_outside == DISCARD_BLOCKS); + verify_old_blocks(); + block_size = DISCARD_SIZE; + + /* A worker that shared keeps its memory. */ + discard_shares = true; + run(discard_worker); + verify_old_blocks(); + verify_kept(); + + /* So does one that got a remote free of one of its blocks. */ + atomic_store(&discard_step, 0); + pthread_t t; + assert(pthread_create(&t, NULL, remote_freed_worker, NULL) == 0); + wait_step(1); + scr_mem_free(discard_blocks[0]); + discard_blocks[0] = scr_mem_alloc(DISCARD_SIZE); /* main's own block, for verify_kept */ + fill(discard_blocks[0], DISCARD_SIZE, 0); + atomic_store(&discard_step, 2); + assert(pthread_join(t, NULL) == 0); + scr_sa_wait_trims(); + verify_old_blocks(); + verify_kept(); + + /* SCR_SA_DISCARD=0 keeps every worker's memory. */ + discard_shares = false; + setenv("SCR_SA_DISCARD", "0", 1); + run(discard_worker); + unsetenv("SCR_SA_DISCARD"); + verify_old_blocks(); + verify_kept(); + + /* The kept workers' blocks are free now (remote frees): a worker takes + * them once its slice has no bump room left, and they are free again + * after its memory goes back, so the next thread does not grow the slice + * either. Past the highest block so far, each carves at most the bump room + * the slice had left (a few megabytes). */ + size_t room = ((size_t)4 << 20) / DISCARD_SIZE; + run(discard_worker); + verify_old_blocks(); + assert(discard_beyond < room); + run(discard_reuse); + assert(reuse_beyond < room); + verify_old_blocks(); + + /* Returned regions (holes) of an earlier trim are kept apart while a + * worker runs, and carved again afterwards. */ + trim_blocks = malloc(TRIM_BLOCKS * sizeof *trim_blocks); /* check_trim freed its own */ + assert(trim_blocks); + run(trim_allocate); + uintptr_t hole_lo = trim_lo, hole_hi = trim_hi; + discard_shares = false; + run(discard_worker); + verify_old_blocks(); + for (size_t k = 0; k < TRIM_BLOCKS / TRIM_KEEP; k++) + verify(trim_kept[k], TRIM_SIZE, (unsigned char)(k * TRIM_KEEP)); + trim_lo = hole_lo; + trim_hi = hole_hi; + run(trim_reuse); + assert(trim_outside < 4 * TRIM_BLOCKS / TRIM_KEEP + 256); + for (size_t k = 0; k < TRIM_BLOCKS / TRIM_KEEP; k++) scr_mem_free(trim_kept[k]); + free(trim_blocks); + for (unsigned i = 0; i < DISCARD_OLD; i++) scr_mem_free(old_blocks[i]); + free(discard_blocks); +} +#endif + int main(void) { #if SCR_SMALL_ALLOC check_slice_growth(); /* first: its class must be untouched */ @@ -389,6 +656,7 @@ int main(void) { check_threads(); #ifdef __linux__ check_trim(); + check_discard(); #endif printf("allocator checks passed: size-class workers\n"); #else diff --git a/packages/runtime/src/scr_itable.c b/packages/runtime/src/scr_itable.c index 08db948b5..4842bf79c 100644 --- a/packages/runtime/src/scr_itable.c +++ b/packages/runtime/src/scr_itable.c @@ -428,11 +428,13 @@ static const ScrITableEntry *scr_itable_insert_bytes(ScrITable *t, uint64_t hash const ScrITableEntry *scr_itable_insert_str(ScrITable *t, const ScrStr *key, uint64_t value) { scr_itable_enter(); + scr_sa_thread_shares(); /* the stored value graph is read by every thread */ return scr_itable_insert_bytes(t, scr_itable_str_hash(key), key->data, key->len, value); } const ScrITableEntry *scr_itable_insert_f64(ScrITable *t, double key, uint64_t value) { scr_itable_enter(); + scr_sa_thread_shares(); /* the stored value graph is read by every thread */ uint64_t stored = scr_itable_f64_bits(key) ^ SCR_ITABLE_F64_FLIP; uint64_t hash = scr_itable_f64_hash(stored); ScrITableSegment *seg = scr_itable_segment(t, hash); @@ -588,6 +590,7 @@ const ScrITableEntry *scr_itable_find_arr(ScrITable *t, const ScrArr *key) { const ScrITableEntry *scr_itable_insert_arr(ScrITable *t, const ScrArr *key, uint64_t value) { scr_itable_enter(); + scr_sa_thread_shares(); /* the stored value graph is read by every thread */ char stack[SCR_ITABLE_STACK_KEY]; size_t len; char *bytes = scr_itable_encoded(t, key, stack, sizeof stack, &len); diff --git a/packages/runtime/src/scr_itable.test.c b/packages/runtime/src/scr_itable.test.c index 0f218806b..5e5f22bc9 100644 --- a/packages/runtime/src/scr_itable.test.c +++ b/packages/runtime/src/scr_itable.test.c @@ -8,13 +8,18 @@ #include #include -/* The table unit needs only scr_trap and scr_throw_error_msg from the - * rest of the runtime; refusals are recorded here. */ +/* The table unit needs only scr_trap, scr_throw_error_msg and + * scr_sa_thread_shares from the rest of the runtime; refusals and the + * inserting threads' shares are recorded here. */ _Noreturn void scr_trap(const char *msg) { fputs(msg, stderr); abort(); } +/* Inserts share the stored values with every thread. */ +static atomic_int shares; +void scr_sa_thread_shares(void) { atomic_fetch_add(&shares, 1); } + static char refused[128]; void scr_throw_error_msg(int kind, const char *msg, size_t len) { assert(kind == SCR_ERR_TYPE && len < sizeof refused); @@ -222,6 +227,7 @@ int main(void) { for (unsigned i = 1; i < THREADS; i++) assert(seen_str[i][k] == seen_str[0][k]); assert(scr_itable_size(strings) == KEYS); assert(scr_itable_size(numbers) == KEYS); + assert(atomic_load(&shares) >= 2 * KEYS); printf("intern tables share one entry per key across racing inserts and lookups\n"); return 0; } diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index bbd43253f..d4290e0df 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -10556,6 +10556,7 @@ static ScrArr *scr_record_shared_keys(const ScrDynTypedRecordOps *record) { #ifdef SCR_RC_AUDIT scr_arr_live_forget(); /* immortal: outside the exit-time RC audit */ #endif + scr_sa_thread_shares(); /* every thread reads the array */ return built; } built->rc = 1; diff --git a/packages/runtime/src/scr_message.c b/packages/runtime/src/scr_message.c index a0b3b8d4e..8ce162a3f 100644 --- a/packages/runtime/src/scr_message.c +++ b/packages/runtime/src/scr_message.c @@ -234,6 +234,7 @@ static size_t scr_message_value(ScrMessageEncoder *encoder, const ScrDyn *input) * capsule over the same object, a dyn graph as itself. */ if (input->kind == SCR_DYN_TYPED_REF && input->v.typed_ref.ptr && *(const size_t *)input->v.typed_ref.ptr == SIZE_MAX) { + scr_sa_thread_shares(); encoder->visits[id].view = (ScrDyn *)input; ScrMessageNode *node = &encoder->message->nodes[id]; node->kind = SCR_MESSAGE_TYPED_REF; @@ -251,6 +252,7 @@ static size_t scr_message_value(ScrMessageEncoder *encoder, const ScrDyn *input) return id; } if ((input->kind == SCR_DYN_OBJ || input->kind == SCR_DYN_ARR) && input->rc == SIZE_MAX) { + scr_sa_thread_shares(); encoder->visits[id].view = (ScrDyn *)input; ScrMessageNode *node = &encoder->message->nodes[id]; node->kind = SCR_MESSAGE_PUBLISHED; @@ -504,6 +506,7 @@ static ScrMessage *scr_message_try_publish(const ScrDyn *value) { scr_lsan_immortal(d); } } + scr_sa_thread_shares(); message = scr_message_alloc(1, sizeof(*message)); message->nodes = scr_message_alloc(1, sizeof(*message->nodes)); message->count = message->capacity = 1; diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 7069839c7..7b6f05f3a 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -336,11 +336,18 @@ void scr_sa_thread_park(void); /* Waits until the arenas of exited threads are trimmed (tests). */ void scr_sa_wait_trims(void); /* Called first on a new worker thread: prepares the thread's system- - * allocator heap (scr_alloc.c). */ + * allocator heap, and lets the thread's memory go back when it exits + * (scr_alloc.c). */ void scr_sa_thread_starting(void); +/* The calling thread made objects it allocated reachable from other + * threads: a message or an intern table shares a published graph by + * pointer, or a process-wide cache keeps an immortal object. Its memory + * then stays when it exits (scr_alloc.c). */ +void scr_sa_thread_shares(void); #define SCR_SA_FREE_FOREIGN(p) scr_sa_free_foreign(p) #else #define SCR_SA_FREE_FOREIGN(p) free(p) +#define scr_sa_thread_shares() ((void)0) #endif #if SCR_SMALL_ALLOC diff --git a/tests/corpus/5842-qs-parse-null-prototype.ts b/tests/corpus/5842-qs-parse-null-prototype.ts new file mode 100644 index 000000000..02d5cc134 --- /dev/null +++ b/tests/corpus/5842-qs-parse-null-prototype.ts @@ -0,0 +1,34 @@ +// querystring.parse and querystring.decode return a null-prototype +// object: inspection prints the "[Object: null prototype]" base, +// getPrototypeOf answers null, and ToPrimitive and Object.prototype +// methods throw TypeErrors. Spread copies keep Object.prototype. +import * as qs from "node:querystring"; +import { inspect } from "node:util"; + +const parsed = qs.parse("a=1&b=2&b=3&c"); +console.log(parsed); +console.log(inspect(parsed), inspect({ nested: parsed }), inspect([qs.parse("")])); +console.log(qs.decode("x=1;y=2", ";")); +console.log(Object.getPrototypeOf(parsed) === null, Object.keys(parsed), JSON.stringify(parsed)); + +const attempt = (label: string, f: () => unknown): void => { + try { + console.log(label, f()); + } catch (e) { + console.log(label, (e as Error).name, (e as Error).message); + } +}; +attempt("String", () => String(parsed)); +attempt("template", () => `${parsed}`); +attempt("concat", () => "q=" + parsed); +attempt("hasOwnProperty", () => parsed.hasOwnProperty("a")); +attempt("toString", () => parsed.toString()); +console.log(Object.prototype.toString.call(parsed), Object.hasOwn(parsed, "b")); +console.log(Object.prototype.hasOwnProperty.call(parsed, "c")); + +const copy = { ...parsed }; +console.log(copy, Object.getPrototypeOf(copy) === Object.prototype, String(copy)); + +parsed.added = "yes"; +delete parsed.a; +console.log(parsed, parsed.b, parsed.added); diff --git a/tests/corpus/5843-qs-parse-null-prototype-js.mjs b/tests/corpus/5843-qs-parse-null-prototype-js.mjs new file mode 100644 index 000000000..676d188c4 --- /dev/null +++ b/tests/corpus/5843-qs-parse-null-prototype-js.mjs @@ -0,0 +1,12 @@ +// querystring.parse results keep their null prototype when they cross into +// checked-dynamic values in a JavaScript module: inspection, getPrototypeOf, +// ToPrimitive and the `in` operator. +import qs from "node:querystring"; +const parsed = qs.parse("a=1&b=2&b=3"); +const holder = { parsed, list: [parsed] }; +console.log(holder); +const dyn = JSON.parse("{}"); +dyn.p = parsed; +console.log(dyn, Object.getPrototypeOf(dyn.p) === null); +try { console.log(String(dyn.p)); } catch (e) { console.log(e.name, e.message); } +console.log("toString" in parsed, "a" in parsed); diff --git a/tests/corpus/5914-dyn-in-class-prototype-members.ts b/tests/corpus/5914-dyn-in-class-prototype-members.ts new file mode 100644 index 000000000..dd4f85f75 --- /dev/null +++ b/tests/corpus/5914-dyn-in-class-prototype-members.ts @@ -0,0 +1,52 @@ +// The `in` operator with literal keys on class instances held as `any`, in +// a program whose instanceof and Object.getPrototypeOf checks give the +// classes runtime prototypes: methods, getters, setters and inherited +// members answer true, as do own fields and Object.prototype members; +// static methods and missing names answer false. + +class Shape { + name = "shape"; + static create(): Shape { + return new Shape(); + } + area(): number { + return 0; + } + get label(): string { + return this.name; + } + set size(value: number) { + this.name = `size ${value}`; + } +} + +class Square extends Shape { + side = 2; + override area(): number { + return this.side * this.side; + } + perimeter(): number { + return this.side * 4; + } +} + +const shape: any = new Shape(); +const square: any = new Square(); +console.log("instanceof", square instanceof Shape, shape instanceof Square); +console.log("same prototype", Object.getPrototypeOf(shape) === Object.getPrototypeOf(square)); + +console.log("method", "area" in shape, "area" in square); +console.log("subclass method", "perimeter" in shape, "perimeter" in square); +console.log("getter", "label" in shape, "label" in square); +console.log("setter", "size" in shape, "size" in square); +console.log("own field", "name" in shape, "side" in square, "side" in shape); +console.log("object prototype", "toString" in shape, "hasOwnProperty" in square); +console.log("static", "create" in shape, "create" in square); +console.log("missing", "volume" in shape, "volume" in square); +console.log("negated", !("area" in shape), !("volume" in square)); + +const names = ["area", "perimeter", "label", "size", "name", "create", "volume"]; +for (const name of names) { + const key: any = JSON.parse(JSON.stringify(name)); + console.log("computed", name, key in shape, key in square); +} diff --git a/tests/corpus/5960-generic-mapped-types.ts b/tests/corpus/5960-generic-mapped-types.ts new file mode 100644 index 000000000..229a97d88 --- /dev/null +++ b/tests/corpus/5960-generic-mapped-types.ts @@ -0,0 +1,64 @@ +// Mapped types over a generic function's own type parameters lower in its +// monomorphized instances: homomorphic copies, modifiers, conditional +// templates with infer, and indexed templates, including over an +// index-signature constraint whose keys only the instance knows. +type Schema = { parse(input: unknown): T }; +type Infer = S extends Schema ? T : never; +interface Typed { + readonly _output: Output; +} +type Output> = T["_output"]; + +const str: Schema = { parse: (input) => String(input) }; +const num: Schema = { parse: (input) => Number(input) }; + +function same(value: S): { [K in keyof S]: S[K] } { + return value; +} +function frozen(value: S): { readonly [K in keyof S]: S[K] } { + return { ...value }; +} +function loose(value: S): { [K in keyof S]?: S[K] } { + return value; +} +function pair; right: Schema }>( + shape: S, + input: { left: unknown; right: unknown }, +): { [K in keyof S]: Infer } { + return { + left: shape.left.parse(input.left), + right: shape.right.parse(input.right), + } as { [K in keyof S]: Infer }; +} +function objectWith>>( + shape: S, + build: (shape: S) => { [K in keyof S]: Infer }, +): { [K in keyof S]: Infer } { + const parsed: { [K in keyof S]: Infer } = build(shape); + return parsed; +} +function outputs>>( + values: S, + read: (values: S) => { [K in keyof S]: Output }, +): { [K in keyof S]: Output } { + return read(values); +} + +const s = same({ a: 1, b: "two" }); +console.log(s.a + 1, s.b.length); +const f = frozen({ x: true, y: [1, 2] }); +console.log(f.x, f.y.length); +const l = loose({ n: 3 }); +console.log(l.n ?? 0); +const p = pair({ left: str, right: num }, { left: 12, right: "34" }); +console.log(p.left.length, p.right + 1); +const person = objectWith({ name: str, age: num }, (shape) => ({ + name: shape.name.parse("Ada"), + age: shape.age.parse("36"), +})); +console.log(person.name, person.age + 1); +const out = outputs({ id: { _output: 7 }, tag: { _output: "t" } }, (values) => ({ + id: values.id._output, + tag: values.tag._output, +})); +console.log(out.id * 2, out.tag.toUpperCase()); diff --git a/tests/corpus/native-worker-discarded-heaps.ts b/tests/corpus/native-worker-discarded-heaps.ts new file mode 100644 index 000000000..bf7a76449 --- /dev/null +++ b/tests/corpus/native-worker-discarded-heaps.ts @@ -0,0 +1,102 @@ +import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads"; +import { publish } from "@scriptc/threads"; +import type { ProgramLifetime } from "@scriptc/types"; + +// Workers whose memory goes back when they exit, and workers whose memory +// stays because other threads hold what they allocated. Builder workers +// create ProgramLifetime instances (which scriptc never frees), ordinary +// objects, strings and large typed arrays, post a checksum and exit. A +// publisher worker posts published graphs (a class graph and a plain +// object) and exits. The main thread reads those graphs again after each +// wave of builders, which in scriptc on Linux reuse the memory of the +// builders before them. + +class Node implements ProgramLifetime { + readonly kind: number; + readonly label: string; + readonly children: Node[] = []; + parent: Node | undefined = undefined; + constructor(kind: number, label: string) { + this.kind = kind; + this.label = label; + } +} + +type Summary = { readonly name: string; readonly values: readonly number[]; readonly nested: { readonly total: number } }; + +type Job = { readonly role: string; readonly seed: number }; + +function build(seed: number, count: number): Node { + const root = new Node(seed, "root" + seed); + let parent = root; + for (let i = 0; i < count; i++) { + const node = new Node((i * 7 + seed) % 31, "n" + (i % 101)); + node.parent = parent; + parent.children.push(node); + if (i % 64 === 63) parent = node; + } + return root; +} + +function checksum(root: Node): number { + let sum = 0; + const stack: Node[] = [root]; + while (stack.length > 0) { + const node = stack.pop()!; + sum = (sum * 31 + node.kind + node.label.length + node.children.length) % 1_000_003; + for (const child of node.children) stack.push(child); + } + return sum; +} + +function startWorker(job: Job, onMessage: (value: unknown) => void): Promise { + return new Promise((resolve, reject) => { + const worker = new Worker(new URL(import.meta.url), { workerData: job }); + worker.on("message", onMessage); + worker.once("messageerror", reject); + worker.once("error", reject); + worker.once("exit", () => resolve()); + }); +} + +if (isMainThread) { + let tree: Node | undefined = undefined; + let summary: Summary | undefined = undefined; + // The publisher posts the tree, then the summary. + await startWorker({ role: "publisher", seed: 3 }, (value) => { + if (tree === undefined) tree = value as Node; + else summary = value as Summary; + }); + const describe = (): string => + `tree ${checksum(tree!)} summary ${summary!.name} ${summary!.values.join(",")} ${summary!.nested.total}`; + console.log("published", describe()); + for (let wave = 0; wave < 3; wave++) { + const sums: number[] = []; + await Promise.all( + [1, 2, 3, 4].map((k) => startWorker({ role: "builder", seed: wave * 10 + k }, (value) => sums.push(value as number))), + ); + sums.sort((a, b) => a - b); + console.log("wave", wave, sums.join(" "), describe()); + } +} else { + const job = workerData as Job; + if (job.role === "publisher") { + parentPort!.postMessage(publish(build(job.seed, 20_000))); + const values: number[] = []; + for (let i = 0; i < 50; i++) values.push(i * job.seed); + parentPort!.postMessage(publish({ name: "summary" + job.seed, values, nested: { total: values.length } })); + } else { + const root = build(job.seed, 150_000); + const samples: Float64Array[] = []; + for (let i = 0; i < 40; i++) { + const sample = new Float64Array(8192); + for (let j = 0; j < sample.length; j += 97) sample[j] = i + j + job.seed; + samples.push(sample); + } + let extra = 0; + for (const sample of samples) extra = (extra + sample[97]! + sample[8148]!) % 1_000_003; + const words: string[] = []; + for (let i = 0; i < 20_000; i++) words.push(`word-${i % 977}-${job.seed}`); + parentPort!.postMessage((checksum(root) + extra + words.join("").length) % 1_000_003); + } +} diff --git a/tests/harness/__snapshots__/deep-binary-nesting.ts.txt b/tests/harness/__snapshots__/deep-binary-nesting.ts.txt index 02cce3cdb..9e645656f 100644 --- a/tests/harness/__snapshots__/deep-binary-nesting.ts.txt +++ b/tests/harness/__snapshots__/deep-binary-nesting.ts.txt @@ -1,6 +1,6 @@ deep-binary-nesting.ts:2:9 - error SC1090: expressions nested deeper than 200 levels are not supported yet 1 | // A ~300-term left-nested binary chain crosses the expression-lowering depth floor (LOWER_EXPR_MAX_DEPTH): the named fence fires with ample stack headroom, where ~3000 terms used to overflow two recursive walks as an ICE (invariant signature 11). - 2 | let x = 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1; - | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + 2 | let x = 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + … + | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 3 | console.log(x); \ No newline at end of file diff --git a/tests/harness/__snapshots__/deep-nesting-preflight.ts.txt b/tests/harness/__snapshots__/deep-nesting-preflight.ts.txt index 655f881cc..c0eb7534f 100644 --- a/tests/harness/__snapshots__/deep-nesting-preflight.ts.txt +++ b/tests/harness/__snapshots__/deep-nesting-preflight.ts.txt @@ -1,6 +1,6 @@ deep-nesting-preflight.ts:2:1 - error SC1090: expressions nested deeper than 200 levels are not supported yet - 1 | // A ~7000-term left-nested binary chain, the binderBinaryExpressionStress shape: deep enough that the whole-file walks (the checker facade prefetch sweep, the require scans, the nested-var collection) used to overflow the JS stack as a RangeError ICE before the expression-lowering fence could fire — the corpus pair crashed at ~6500 terms, and the bare statement sat within ~50 terms of the nested-var walk's limit. Every such walk is iterative now (walkPreorder), so the program survives to the same named SC1090 fence the ~300-term deep-binary-nesting fixture pins. A BARE expression statement (no initializer position) keeps tsgo's per-node query cost — and this suite — fast; one term per line keeps the snapshot code frame small. 5.9.3's parser (the resolver-parity island sweeps this directory) trampolines binary chains, so the shape parses in both worlds. + 1 | // A ~7000-term left-nested binary chain, the binderBinaryExpressionStress shape: deep enough that the whole-file walks (the checker facade prefetch sweep, the require scans, the nested-var collection) used to overflow the JS stack as a RangeError ICE before the expression-lowering fence could fire — the corpus pair crashed at ~6500 terms, and the bare statement sat within ~50 terms of the nested-… 2 | 1 + | ^~~ 3 | 1 + \ No newline at end of file diff --git a/tests/harness/corpus-costs.json b/tests/harness/corpus-costs.json index 8b545a5f1..e67c9b12a 100644 --- a/tests/harness/corpus-costs.json +++ b/tests/harness/corpus-costs.json @@ -1849,6 +1849,8 @@ "5836-url-search-params-tuple-pairs.ts": 2118, "5837-util-format-json-circular.ts": 3117, "5838-qs-stringify-bigint.ts": 2207, + "5842-qs-parse-null-prototype.ts": 1774, + "5843-qs-parse-null-prototype-js.mjs": 1559, "5858-disposable-stack.ts": 3050, "5859-disposable-stack-js.js": 3100, "5881-inspect-weak-collections.ts": 2615, @@ -1863,7 +1865,9 @@ "5906-array-from-async-generator-js.js": 2950, "5910-set-prototype-of-primitive-object-kinds.ts": 2719, "5911-dyn-class-to-primitive.ts": 2850, + "5912-dyn-class-conversion-methods.ts": 3251, "5913-prototype-type-error-messages.ts": 2484, + "5914-dyn-in-class-prototype-members.ts": 3169, "5939-js-let-object-reassigned-wider.js": 1945, "5942-local-class-observed-methods.ts": 5063, "5950-tuple-predicate-methods.ts": 5990, @@ -1872,6 +1876,7 @@ "5956-erased-generic-members.ts": 6730, "5958-asserted-union-receiver.ts": 2430, "5959-date-to-json.ts": 3330, + "5960-generic-mapped-types.ts": 2540, "5970-net-connect-host-name.ts": 2400, "5971-buffer-chunks-to-string.ts": 2700, "5972-net-set-encoding-listeners.ts": 2550, @@ -2387,6 +2392,7 @@ "native-worker-clone-errors.ts": 2347, "native-worker-commonjs.cjs": 2188, "native-worker-delayed-messages.ts": 3114, + "native-worker-discarded-heaps.ts": 4430, "native-worker-environment.ts": 3538, "native-worker-error.ts": 2810, "native-worker-exception-state.ts": 3938, @@ -4507,6 +4513,8 @@ "5836-url-search-params-tuple-pairs.ts": 3193, "5837-util-format-json-circular.ts": 3408, "5838-qs-stringify-bigint.ts": 3519, + "5842-qs-parse-null-prototype.ts": 7077, + "5843-qs-parse-null-prototype-js.mjs": 5395, "5858-disposable-stack.ts": 3750, "5859-disposable-stack-js.js": 3600, "5881-inspect-weak-collections.ts": 2915, @@ -4521,7 +4529,9 @@ "5906-array-from-async-generator-js.js": 3750, "5910-set-prototype-of-primitive-object-kinds.ts": 5033, "5911-dyn-class-to-primitive.ts": 4732, + "5912-dyn-class-conversion-methods.ts": 4424, "5913-prototype-type-error-messages.ts": 4068, + "5914-dyn-in-class-prototype-members.ts": 4327, "5939-js-let-object-reassigned-wider.js": 4436, "5942-local-class-observed-methods.ts": 6212, "5950-tuple-predicate-methods.ts": 7790, @@ -4530,6 +4540,7 @@ "5956-erased-generic-members.ts": 8750, "5958-asserted-union-receiver.ts": 3160, "5959-date-to-json.ts": 4330, + "5960-generic-mapped-types.ts": 3300, "5970-net-connect-host-name.ts": 2800, "5971-buffer-chunks-to-string.ts": 3100, "5972-net-set-encoding-listeners.ts": 3000, @@ -5045,6 +5056,7 @@ "native-worker-clone-errors.ts": 3629, "native-worker-commonjs.cjs": 3085, "native-worker-delayed-messages.ts": 28752, + "native-worker-discarded-heaps.ts": 39440, "native-worker-environment.ts": 27133, "native-worker-error.ts": 2958, "native-worker-exception-state.ts": 3945,