diff --git a/Dockerfile.sandbox b/Dockerfile.sandbox index f0db1c9e5..597b33445 100644 --- a/Dockerfile.sandbox +++ b/Dockerfile.sandbox @@ -24,6 +24,7 @@ RUN apt-get update \ curl \ git \ libclang-rt-22-dev \ + libpolly-22-dev \ libzstd-dev \ llvm-22-dev \ ninja-build \ diff --git a/packages/compiler/src/backend/native-codegen.ts b/packages/compiler/src/backend/native-codegen.ts index 6e91560dd..6aab5880a 100644 --- a/packages/compiler/src/backend/native-codegen.ts +++ b/packages/compiler/src/backend/native-codegen.ts @@ -2,7 +2,7 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { constants } from "node:fs"; import { createRequire } from "node:module"; -import { access, chmod, mkdir, open, readFile, rename, rm, stat } from "node:fs/promises"; +import { access, chmod, mkdir, readFile, rename, rm, stat } from "node:fs/promises"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; import { @@ -27,6 +27,7 @@ import { type NativeTargetSpec, } from "./targets.js"; import { compilerReleaseVersion } from "../library/sidecar.js"; +import { writeTextParts } from "../text-parts.js"; import { NativeCodegenError, @@ -87,39 +88,13 @@ export interface NativeCodegenOptions { importBitcode?: { paths: readonly string[]; digests: readonly string[] }; } -/** Writes LLVM text in large batches. Programs past the engine's string - * limit arrive as many parts, and writing each part separately is several - * times slower than writing the same bytes in a few large chunks. */ +/** Writes LLVM text in large batches; see `writeTextParts`. */ export async function writeLlvmSource( path: string, llvm: string | readonly string[], mode = 0o666, ): Promise { - const file = await open(path, "w", mode); - try { - const write = async (text: string): Promise => { - const bytes = Buffer.from(text, "utf8"); - for (let offset = 0; offset < bytes.length;) { - const { bytesWritten } = await file.write(bytes, offset, bytes.length - offset); - offset += bytesWritten; - } - }; - if (typeof llvm === "string") return await write(llvm); - let batch: string[] = []; - let length = 0; - for (const part of llvm) { - batch.push(part); - length += part.length; - if (length >= 16 * 1024 * 1024) { - await write(batch.join("")); - batch = []; - length = 0; - } - } - if (batch.length > 0) await write(batch.join("")); - } finally { - await file.close(); - } + await writeTextParts(path, llvm, mode); } function parseJsonObject(text: string): Record | null { diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 0b8d48430..0759a01cb 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -103,7 +103,8 @@ import { } from "./library/sidecar.js"; import { type IrModule, type SrcLoc } from "./ir/ir.js"; import { moduleRuntimeFeatures } from "./ir/runtime-features.js"; -import { serializeModule } from "./ir/serialize.js"; +import { serializeModuleParts } from "./ir/serialize.js"; +import { writeTextParts } from "./text-parts.js"; import { validateModule } from "./ir/validate.js"; import { loadProgram } from "./frontend/program-node.js"; import { runFrontend, type FrontendFactory } from "./frontend/pipeline.js"; @@ -206,7 +207,13 @@ export { } from "./compat/fetch-profile.js"; export { LIB_FN_SIGS } from "./ir/builtin-signatures.js"; export { validateModule } from "./ir/validate.js"; -export { deserializeModule, IR_VERSION, serializeModule } from "./ir/serialize.js"; +export { + deserializeModule, + deserializeModuleBytes, + IR_VERSION, + serializeModule, + serializeModuleParts, +} from "./ir/serialize.js"; export { resolveLibraryFences, type LibraryFenceDecl, @@ -670,7 +677,7 @@ async function prepareExecutableInput( if (outputKind === "ir") { await mkdir(dirname(opts.outPath), { recursive: true }); - await writeFile(opts.outPath, serializeModule(mod, true)); + await writeTextParts(opts.outPath, serializeModuleParts(mod)); return { ok: true, artifact: { kind: "ir", path: opts.outPath } }; } @@ -692,7 +699,7 @@ async function prepareExecutableInput( } if (outputKind === "llvm") { await mkdir(dirname(opts.outPath), { recursive: true }); - await writeFile(opts.outPath, llvm); + await writeTextParts(opts.outPath, llvm); } else { try { await emitNativeArtifact({ @@ -766,7 +773,7 @@ async function prepareExecutableInput( let irPath: string | undefined; if (opts.emitIr) { irPath = defaultSourcePaths.ir; - await writeFile(irPath, serializeModule(mod, true)); + await writeTextParts(irPath, serializeModuleParts(mod)); } const nativeFeatures = executableNativeFeatures( @@ -1448,7 +1455,7 @@ async function emitSemanticLibraryHit( let irPath: string | undefined; if (opts.emitIr) { irPath = join(opts.outDir, `${stem}.lib.ir.json`); - await writeFile(irPath, serializeModule(mod, true)); + await writeTextParts(irPath, serializeModuleParts(mod)); } await compileLibraryNative(profile, llvmPath, archivePath, opts.sanitize ?? false, hit.native); timing("native-archive"); @@ -1724,7 +1731,7 @@ async function compileLibraryTracked( ? Buffer.byteLength(ll) : ll.reduce((bytes, part) => bytes + Buffer.byteLength(part), 0), }); - await writeFile(llvmPath, ll); + await writeTextParts(llvmPath, ll); timing("llvm-write"); } catch (err) { if (!(err instanceof LlvmUnsupportedError)) throw err; @@ -1734,7 +1741,7 @@ async function compileLibraryTracked( let irPath: string | undefined; if (opts.emitIr) { irPath = join(opts.outDir, `${stem}.lib.ir.json`); - await writeFile(irPath, serializeModule(mod, true)); + await writeTextParts(irPath, serializeModuleParts(mod)); } const nativeFeatures = libraryNativeFeatures(mod, profile.emission); diff --git a/packages/compiler/src/ir/serialize.test.ts b/packages/compiler/src/ir/serialize.test.ts index be933514e..8345ab10a 100644 --- a/packages/compiler/src/ir/serialize.test.ts +++ b/packages/compiler/src/ir/serialize.test.ts @@ -1,6 +1,12 @@ import { expect, test } from "vitest"; import { F64, VOID, type IrStmt, type IrModule } from "./ir.js"; -import { IR_VERSION, deserializeModule, serializeModule } from "./serialize.js"; +import { + IR_VERSION, + deserializeModule, + deserializeModuleBytes, + serializeModule, + serializeModuleParts, +} from "./serialize.js"; function numbers(values: number[]): IrModule { const loc = { file: "numbers.ts", start: 0, end: 0 }; @@ -124,3 +130,71 @@ test("compact artifacts keep JSON's circular-data refusal", () => { expect(() => serializeModule(mod)).toThrow(TypeError); expect(() => serializeModule(mod, true)).toThrow(TypeError); }); + +/** A module whose strings contain every byte the function splitter must not + * mistake for structure. */ +function structuralStrings(): IrModule { + const mod = numbers([NaN, -0, 1.5, Infinity]); + const tricky = 'q"\\"],[{"functions":[]}\\, \n\tšŸ˜€Ć©\\'; + mod.sourceFile = tricky + '"functions":['; + const base = mod.functions[0]!; + mod.functions = [ + base, + { ...base, name: tricky, loc: { file: tricky, start: -0, end: 2 } }, + { ...base, name: "]", body: [] }, + { ...base, name: "\\" }, + ]; + mod.globals = [{ id: "%g.functions", name: "functions", type: F64, mutable: false }]; + return mod; +} + +test("compact parts are exactly the single-root JSON encoding", () => { + const mod = structuralStrings(); + const parts = serializeModuleParts(mod); + expect(parts.length).toBe(2 + mod.functions.length * 2 - 1); + const sentinels = (_key: string, value: unknown): unknown => + typeof value !== "number" + ? value + : Number.isNaN(value) + ? { $nonfinite: "nan" } + : value === Infinity + ? { $nonfinite: "inf" } + : value === -Infinity + ? { $nonfinite: "-inf" } + : Object.is(value, -0) + ? { $nonfinite: "-0" } + : value; + expect(parts.join("")).toBe(JSON.stringify(mod, sentinels)); + expect(serializeModule(mod, true)).toBe(parts.join("")); +}); + +test("byte decoding splits functions without being misled by string contents", () => { + const mod = structuralStrings(); + for (const text of [serializeModule(mod, true), serializeModule(mod)]) { + const decoded = deserializeModuleBytes(Buffer.from(text, "utf8")); + expect(decoded).toEqual(mod); + expect(decoded).toEqual(deserializeModule(text)); + expect(Object.is(decoded.functions[1]!.loc.start, -0)).toBe(true); + } + mod.functions = []; + expect(deserializeModuleBytes(Buffer.from(serializeModule(mod, true)))).toEqual(mod); + expect(deserializeModuleBytes(Buffer.from(serializeModule(mod)))).toEqual(mod); +}); + +test("byte decoding keeps the whole-document errors and version refusal", () => { + const text = serializeModule(structuralStrings(), true); + for (const broken of [text.slice(0, -1), text.slice(0, text.length - 40), text + "]", "[]", ""]) { + let expected: unknown; + try { + deserializeModule(broken); + } catch (error) { + expected = error; + } + expect(expected).toBeInstanceOf(Error); + expect(() => deserializeModuleBytes(Buffer.from(broken))).toThrow(); + } + const older = text.replace(`"irVersion":${IR_VERSION}`, `"irVersion":${IR_VERSION - 1}`); + expect(() => deserializeModuleBytes(Buffer.from(older))).toThrow("IR version mismatch"); + const sentinel = text.replace('"nan"', '"future"'); + expect(() => deserializeModuleBytes(Buffer.from(sentinel))).toThrow("Invalid IR number sentinel"); +}); diff --git a/packages/compiler/src/ir/serialize.ts b/packages/compiler/src/ir/serialize.ts index b156300ea..9455191d4 100644 --- a/packages/compiler/src/ir/serialize.ts +++ b/packages/compiler/src/ir/serialize.ts @@ -1,6 +1,12 @@ /* IR ↔ JSON. Numeric literals preserve JavaScript's complete number domain: - * JSON's null/zero spellings must not erase NaN, infinities or negative zero. */ -import type { IrModule } from "./ir.js"; + * JSON's null/zero spellings must not erase NaN, infinities or negative zero. + * + * Compiler-scale artifacts are larger than one host string can hold (V8 caps + * strings at 2^29 - 24 UTF-16 units, and the self-hosted compiler's IR is + * already close to it). The compact encoding is therefore produced as ordered + * text parts and decoded from bytes one function at a time; neither direction + * materializes the complete document as a single string. */ +import type { IrFunction, IrModule } from "./ir.js"; // Version 15 expands the native Error prefix with filesystem metadata. // Older modules embed incompatible subclass offsets and must be rejected. @@ -22,52 +28,65 @@ function ordinaryNumbers(root: unknown): boolean { return visit(root); } -/** Compiler artifacts use compact JSON to keep large graphs below the host's - * string size limit. API consumers can retain the readable default. */ -export function serializeModule(mod: IrModule, compact = false): string { - const replacer = (_key: string, value: unknown): unknown => { - if (typeof value !== "number") return value; - if (!Number.isFinite(value)) { - return { $nonfinite: Number.isNaN(value) ? "nan" : value > 0 ? "inf" : "-inf" }; - } - // JSON.stringify(-0) prints "0", silently losing the sign a numLit's - // f64 semantics depend on (String(-0) is "0" but 1/-0 is -Infinity) — - // the same sentinel mechanism carries it. - if (Object.is(value, -0)) { - return { $nonfinite: "-0" }; - } - return value; - }; - if (!compact) return JSON.stringify(mod, replacer, 2); +function encodeNumber(_key: string, value: unknown): unknown { + if (typeof value !== "number") return value; + if (!Number.isFinite(value)) { + return { $nonfinite: Number.isNaN(value) ? "nan" : value > 0 ? "inf" : "-inf" }; + } + // JSON.stringify(-0) prints "0", silently losing the sign a numLit's + // f64 semantics depend on (String(-0) is "0" but 1/-0 is -Infinity) — + // the same sentinel mechanism carries it. + if (Object.is(value, -0)) { + return { $nonfinite: "-0" }; + } + return value; +} + +/** The compact encoding as ordered parts: the module header up to the + * function array, one part per function (with separate comma parts), and the + * header's remainder. Concatenating the parts yields exactly + * `serializeModule(mod, true)`; writers stream them to disk in batches. */ +export function serializeModuleParts(mod: IrModule): string[] { // A replacer reads through live native views. Serializing the complete // function array through one view refreshes every function capsule for // each element, making large compiler artifacts quadratic. The sentinel // replacer is key-independent, so encode each function as its own root. - const header = JSON.stringify({ ...mod, functions: [] }, replacer); + const header = JSON.stringify({ ...mod, functions: [] }, encodeNumber); const slot = '"functions":[]'; const offset = header.indexOf(slot); - const functions = mod.functions - .map( - (fn) => (ordinaryNumbers(fn) ? JSON.stringify(fn) : JSON.stringify(fn, replacer)) ?? "null", - ) - .join(","); - return ( - header.slice(0, offset) + '"functions":[' + functions + "]" + header.slice(offset + slot.length) - ); + const parts: string[] = [header.slice(0, offset) + '"functions":[']; + for (let index = 0; index < mod.functions.length; index++) { + const fn = mod.functions[index]!; + if (index > 0) parts.push(","); + parts.push( + (ordinaryNumbers(fn) ? JSON.stringify(fn) : JSON.stringify(fn, encodeNumber)) ?? "null", + ); + } + parts.push("]" + header.slice(offset + slot.length)); + return parts; } -export function deserializeModule(json: string): IrModule { - const mod = JSON.parse(json, (_key, value: unknown) => { - if (typeof value === "object" && value !== null && "$nonfinite" in value) { - const tag = (value as { $nonfinite: string }).$nonfinite; - if (tag === "inf") return Infinity; - if (tag === "-inf") return -Infinity; - if (tag === "-0") return -0; - if (tag === "nan") return NaN; - throw new Error(`Invalid IR number sentinel: ${String(tag)}`); - } - return value; - }) as IrModule; +/** Compiler artifacts use compact JSON; API consumers can retain the + * readable default. Both return one string, so modules whose encoding + * exceeds the host's string limit must use `serializeModuleParts`. */ +export function serializeModule(mod: IrModule, compact = false): string { + if (!compact) return JSON.stringify(mod, encodeNumber, 2); + return serializeModuleParts(mod).join(""); +} + +function decodeNumber(_key: string, value: unknown): unknown { + if (typeof value === "object" && value !== null && "$nonfinite" in value) { + const tag = (value as { $nonfinite: string }).$nonfinite; + if (tag === "inf") return Infinity; + if (tag === "-inf") return -Infinity; + if (tag === "-0") return -0; + if (tag === "nan") return NaN; + throw new Error(`Invalid IR number sentinel: ${String(tag)}`); + } + return value; +} + +function checkedVersion(mod: IrModule): IrModule { if (mod.irVersion !== IR_VERSION) { throw new Error( `IR version mismatch: file has ${String(mod.irVersion)}, compiler expects ${IR_VERSION}`, @@ -75,3 +94,150 @@ export function deserializeModule(json: string): IrModule { } return mod; } + +export function deserializeModule(json: string): IrModule { + return checkedVersion(JSON.parse(json, decodeNumber) as IrModule); +} + +const QUOTE = 0x22; +const BACKSLASH = 0x5c; +const COMMA = 0x2c; +const COLON = 0x3a; +const OPEN_BRACKET = 0x5b; +const CLOSE_BRACKET = 0x5d; +const OPEN_BRACE = 0x7b; +const CLOSE_BRACE = 0x7d; +const FUNCTIONS_KEY = '"functions"'; + +function isJsonSpace(byte: number): boolean { + return byte === 0x20 || byte === 0x0a || byte === 0x0d || byte === 0x09; +} + +function skipSpace(bytes: Buffer, index: number): number { + while (index < bytes.length && isJsonSpace(bytes[index]!)) index++; + return index; +} + +/** `index` is an opening quote; returns the index after its closing quote. */ +function skipString(bytes: Buffer, index: number): number { + const length = bytes.length; + index++; + while (index < length) { + const byte = bytes[index]!; + if (byte === BACKSLASH) index += 2; + else if (byte === QUOTE) return index + 1; + else index++; + } + return length; +} + +interface FunctionSpans { + /** Offsets of the function array's brackets. */ + open: number; + close: number; + /** Byte range of each element. */ + starts: number[]; + ends: number[]; +} + +/** Element ranges of the array starting at `open`, or null when the bytes + * are not well-formed enough to split (the whole-document parse then reports + * the exact syntax error). */ +function elementSpans(bytes: Buffer, open: number): FunctionSpans | null { + const length = bytes.length; + const starts: number[] = []; + const ends: number[] = []; + let index = skipSpace(bytes, open + 1); + if (index < length && bytes[index] === CLOSE_BRACKET) return { open, close: index, starts, ends }; + while (index < length) { + const start = index; + let depth = 0; + while (index < length) { + const byte = bytes[index]!; + if (byte === QUOTE) { + index = skipString(bytes, index); + if (depth === 0) break; + continue; + } + if (byte === OPEN_BRACE || byte === OPEN_BRACKET) depth++; + else if (byte === CLOSE_BRACE || byte === CLOSE_BRACKET) { + if (depth === 0) break; + depth--; + if (depth === 0) { + index++; + break; + } + } else if (depth === 0 && (byte === COMMA || isJsonSpace(byte))) break; + index++; + } + if (index === start || index >= length) return null; + starts.push(start); + ends.push(index); + index = skipSpace(bytes, index); + if (index >= length) return null; + if (bytes[index] === CLOSE_BRACKET) return { open, close: index, starts, ends }; + if (bytes[index] !== COMMA) return null; + index = skipSpace(bytes, index + 1); + } + return null; +} + +/** Locate the root object's `"functions"` array. */ +function functionSpans(bytes: Buffer): FunctionSpans | null { + const length = bytes.length; + let depth = 0; + let index = 0; + while (index < length) { + const byte = bytes[index]!; + if (byte === QUOTE) { + const start = index; + index = skipString(bytes, index); + if (depth === 1 && index - start === FUNCTIONS_KEY.length) { + let key = true; + for (let offset = 1; offset < FUNCTIONS_KEY.length - 1; offset++) { + if (bytes[start + offset] !== FUNCTIONS_KEY.charCodeAt(offset)) { + key = false; + break; + } + } + if (key) { + let next = skipSpace(bytes, index); + if (next < length && bytes[next] === COLON) { + next = skipSpace(bytes, next + 1); + if (next < length && bytes[next] === OPEN_BRACKET) return elementSpans(bytes, next); + } + } + } + continue; + } + if (byte === OPEN_BRACE || byte === OPEN_BRACKET) depth++; + else if (byte === CLOSE_BRACE || byte === CLOSE_BRACKET) depth--; + index++; + } + return null; +} + +/** Decode a UTF-8 IR artifact (compact or readable) without materializing + * it as one string: the header and each function are parsed separately. + * Split points are ASCII structural bytes, so every range is valid UTF-8. */ +export function deserializeModuleBytes(bytes: Buffer): IrModule { + const spans = functionSpans(bytes); + if (spans === null) return deserializeModule(bytes.toString("utf8")); + const mod = checkedVersion( + JSON.parse( + bytes.toString("utf8", 0, spans.open + 1) + bytes.toString("utf8", spans.close), + decodeNumber, + ) as IrModule, + ); + const functions: IrFunction[] = []; + for (let index = 0; index < spans.starts.length; index++) { + functions.push( + JSON.parse( + bytes.toString("utf8", spans.starts[index]!, spans.ends[index]!), + decodeNumber, + ) as IrFunction, + ); + } + mod.functions = functions; + return mod; +} diff --git a/packages/compiler/src/native/cache.test.ts b/packages/compiler/src/native/cache.test.ts index d5ac6bc6a..3ffdf2335 100644 --- a/packages/compiler/src/native/cache.test.ts +++ b/packages/compiler/src/native/cache.test.ts @@ -38,6 +38,20 @@ test("cache hits require an intact payload and digest", () => { expect(cache.read("object", key)).toBeNull(); }); +test("text parts publish one digest-checked entry and oversized reads are misses", () => { + const cache = new NativeCache(directory()); + const key = contentDigest("parts identity"); + const parts = ['{"a":', '"\u00e9Ć©šŸ˜€"', ",", "", '"b":1}']; + cache.writeParts("frontend", key, parts); + const bytes = cache.read("frontend", key); + expect(bytes?.toString("utf8")).toBe(parts.join("")); + expect(readFileSync(join(cache.root, "frontend", key + ".sha256"), "utf8").trim()).toBe( + contentDigest(parts.join("")), + ); + expect(cache.read("frontend", key, bytes!.length)).toEqual(bytes); + expect(cache.read("frontend", key, bytes!.length - 1)).toBeNull(); +}); + test("a failed cache publication does not fail the build or replace unrelated files", () => { const cache = new NativeCache(directory()); const occupied = join(cache.root, "object"); diff --git a/packages/compiler/src/native/cache.ts b/packages/compiler/src/native/cache.ts index 22741a0c4..f81311f52 100644 --- a/packages/compiler/src/native/cache.ts +++ b/packages/compiler/src/native/cache.ts @@ -11,6 +11,12 @@ import { } from "node:fs"; import { join } from "node:path"; import { resolveBuildCacheRoot } from "../backend/cache-root.js"; +import { writeTextPartsSync } from "./text-file.js"; + +/** UTF-8 bytes one native compiler string can hold (the runtime's + * SCR_STR_MAX_CAP). Exceeding it traps rather than throwing, so payloads + * decoded as one string are bounded by it up front. */ +export const NATIVE_STRING_BYTES = 0x7fffffff; export function contentDigest(value: string | Uint8Array): string { const hash = createHash("sha256"); @@ -29,9 +35,11 @@ export class NativeCache { mkdirSync(this.root, { recursive: true, mode: 0o700 }); } - read(family: string, key: string): Buffer | null { + /** Entries larger than `maxBytes` are misses. */ + read(family: string, key: string, maxBytes = Infinity): Buffer | null { try { const path = join(this.root, family, key); + if (maxBytes < Infinity && statSync(path).size > maxBytes) return null; const bytes = readFileSync(path); if (contentDigest(bytes) !== readFileSync(path + ".sha256", "utf8").trim()) return null; return bytes; @@ -77,6 +85,31 @@ export class NativeCache { } } + /** Publish text parts without joining them into one string. */ + writeParts(family: string, key: string, parts: readonly string[]): void { + let stage: string | null = null; + try { + const directory = join(this.root, family); + mkdirSync(directory, { recursive: true, mode: 0o700 }); + stage = mkdtempSync(join(directory, ".write-")); + writeTextPartsSync(join(stage, "payload"), parts); + const hash = createHash("sha256"); + for (const part of parts) hash.update(part); + writeFileSync(join(stage, "digest"), hash.digest("hex") + "\n"); + const path = join(directory, key); + renameSync(join(stage, "payload"), path); + renameSync(join(stage, "digest"), path + ".sha256"); + } catch { + /* Cache failures never change a build's result. */ + } finally { + try { + if (stage !== null) rmSync(stage, { recursive: true, force: true }); + } catch { + /* Cleanup is best effort too. */ + } + } + } + prune(): void { try { const configured = Number(process.env["SCRIPTC_CACHE_MAX_MB"] ?? "4096"); diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index 984eb4339..57d2ab4d6 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -10,6 +10,7 @@ import { writeFileSync, } from "node:fs"; import { basename, dirname, join, resolve } from "node:path"; +import { writeTextPartsSync } from "./text-file.js"; import type { AnalyzeOptions, AnalyzeResult, @@ -320,7 +321,7 @@ export class NativeCompiler { const stem = basename(entry).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); const llvmPath = join(options.outDir, `${stem}.ll`); if (outputKind === "ir") { - writeFileSync(stagedOutput, prepared.ir!); + writeTextPartsSync(stagedOutput, prepared.ir!); renameSync(stagedOutput, output); return { ok: true, artifact: { kind: "ir", path: output } }; } @@ -464,7 +465,7 @@ export class NativeCompiler { let irPath: string | undefined; if (options.emitIr) { irPath = join(options.outDir, `${stem}.ir.json`); - writeFileSync(irPath, prepared.ir!); + writeTextPartsSync(irPath, prepared.ir!); } renameSync(stagedOutput, output); if (target.platform === "darwin" && (optimization !== "dev" || options.strip)) @@ -614,7 +615,7 @@ export class NativeCompiler { let irPath: string | undefined; if (options.emitIr) { irPath = join(options.outDir, `${stem}.lib.ir.json`); - writeFileSync(irPath, prepared.ir!); + writeTextPartsSync(irPath, prepared.ir!); } let sidecarPath: string | undefined; if (prepared.sidecarJson !== null) { diff --git a/packages/compiler/src/native/prepare.test.ts b/packages/compiler/src/native/prepare.test.ts index 4295e88ef..7e22450da 100644 --- a/packages/compiler/src/native/prepare.test.ts +++ b/packages/compiler/src/native/prepare.test.ts @@ -6,6 +6,7 @@ import { nativeCodegenTarget } from "../backend/targets.js"; import { runFrontend, type FrontendFactory } from "../frontend/pipeline.js"; import { loadProgram } from "../frontend/program-node.js"; import { ts7Executable } from "../frontend/ts7/rpc-api.js"; +import { deserializeModule } from "../ir/serialize.js"; import { NativeCache } from "./cache.js"; import { prepareNativeExecutable } from "./prepare.js"; import type { NativeToolchain } from "./toolchain.js"; @@ -55,7 +56,8 @@ function fixture() { loads++; return runFrontend(path, loadProgram, npmStatic, externalTypes); }; - const prepare = () => prepareNativeExecutable(entry, options, null, toolchain, frontend, cache); + const prepare = (extra: { emitIr?: boolean } = {}) => + prepareNativeExecutable(entry, { ...options, ...extra }, null, toolchain, frontend, cache); return { root, entry, dependency, cache, options, prepare, loads: () => loads }; } @@ -88,3 +90,24 @@ test("structurally invalid cached features cause a rebuild even with a valid dig expect(f.prepare().ok).toBe(true); expect(f.loads()).toBe(2); }); + +test("serialized IR travels as bounded parts through the frontend cache", () => { + const f = fixture(); + const first = f.prepare({ emitIr: true }); + expect(first.ok, JSON.stringify(first)).toBe(true); + if (!first.ok) return; + expect(Array.isArray(first.ir)).toBe(true); + const mod = deserializeModule(first.ir!.join("")); + expect(mod.functions.length).toBeGreaterThan(0); + const warm = f.prepare({ emitIr: true }); + expect(f.loads()).toBe(1); + expect(warm).toEqual(first); + const family = join(f.cache.root, "frontend"); + const key = readdirSync(family).find((name) => /^[0-9a-f]{64}$/.test(name))!; + const payload = JSON.parse(readFileSync(join(family, key), "utf8")); + expect(payload.schema).toBe("scriptc.native-frontend.v3"); + expect(payload.input.llvm).toBe(""); + expect(payload.input.ir).toBeNull(); + expect(payload.llvm).toBe(first.llvm); + expect(payload.ir).toEqual(first.ir); +}); diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index 24f339dae..69f317f28 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -25,15 +25,18 @@ import { provenanceSources } from "../frontend/provenance-registry.js"; import { prepareLibrary } from "../library/prepare.js"; import type { LibraryProfile } from "../library/library-profile.js"; import type { IrModule } from "../ir/ir.js"; -import { deserializeModule, serializeModule } from "../ir/serialize.js"; -import { contentDigest, type NativeCache } from "./cache.js"; +import { deserializeModuleBytes, serializeModuleParts } from "../ir/serialize.js"; +import { contentDigest, NATIVE_STRING_BYTES, type NativeCache } from "./cache.js"; +import { batchTextParts } from "./text-file.js"; import type { NativeToolchain } from "./toolchain.js"; import { nativeFileIdentity } from "./file-identity.js"; export interface NativeExecutableInput { ok: true; llvm: string; - ir: string | null; + /** Compact serialized IR as bounded parts (`batchTextParts`): compiler-scale + * modules exceed one native string. */ + ir: string[] | null; sidecarJson: string | null; features: NativeLinkFeatures; sources: [string, string][]; @@ -42,10 +45,62 @@ export interface NativeExecutableInput { warnings?: ScrDiagnostic[]; } +/** The two large artifacts follow the input metadata as separate members so + * an entry can be written as parts instead of one JSON string; the stored + * input carries empty placeholders (`""`, `null`) for them. */ interface FrontendCacheEntry { - schema: "scriptc.native-frontend.v2"; + schema: "scriptc.native-frontend.v3"; probes: FrontendInputSnapshot; input: NativeExecutableInput; + llvm: string; + ir: string[] | null; +} + +function frontendCacheParts(probes: FrontendInputSnapshot, input: NativeExecutableInput): string[] { + const metadata = JSON.stringify({ + schema: "scriptc.native-frontend.v3", + probes, + input: { ...input, llvm: "", ir: null }, + }); + const parts = [metadata.slice(0, metadata.length - 1) + ',"llvm":', JSON.stringify(input.llvm)]; + if (input.ir === null) parts.push(',"ir":null}'); + else { + parts.push(',"ir":['); + for (let index = 0; index < input.ir.length; index++) { + if (index > 0) parts.push(","); + parts.push(JSON.stringify(input.ir[index]!)); + } + parts.push("]}"); + } + return parts; +} + +/** Publish an entry only when a native reader can decode it as one string; + * UTF-16 length never exceeds the UTF-8 byte length. */ +function publishFrontendCache( + cache: NativeCache, + key: string, + probes: FrontendInputSnapshot, + input: NativeExecutableInput, +): void { + const parts = frontendCacheParts(probes, input); + let length = 0; + for (const part of parts) length += part.length; + if (length <= NATIVE_STRING_BYTES) cache.writeParts("frontend", key, parts); +} + +function readFrontendCache( + cache: NativeCache, + key: string, + exclusions: FrontendInputExclusions, +): NativeExecutableInput | null { + // The entry is decoded as one string; larger entries are misses. + const bytes = cache.read("frontend", key, NATIVE_STRING_BYTES); + if (bytes === null) return null; + const cached: unknown = JSON.parse(bytes.toString("utf8")); + if (!validCachedInput(cached) || !frontendInputsStillMatch(cached.probes, exclusions)) + return null; + return { ...cached.input, llvm: cached.llvm, ir: cached.ir }; } function implementationIdentity(toolchain: NativeToolchain): string { @@ -106,7 +161,7 @@ function validCachedInput(value: unknown): value is FrontendCacheEntry { const entry = value as Partial; const input = entry.input; if ( - entry.schema !== "scriptc.native-frontend.v2" || + entry.schema !== "scriptc.native-frontend.v3" || !validFrontendInputSnapshot(entry.probes) || input === undefined || input === null || @@ -115,8 +170,9 @@ function validCachedInput(value: unknown): value is FrontendCacheEntry { return false; return ( input.ok === true && - typeof input.llvm === "string" && - (input.ir === null || typeof input.ir === "string") && + typeof entry.llvm === "string" && + (entry.ir === null || + (Array.isArray(entry.ir) && entry.ir.every((part) => typeof part === "string"))) && (input.sidecarJson === null || typeof input.sidecarJson === "string") && validFeatures(input.features) && Array.isArray(input.sources) && @@ -144,7 +200,7 @@ function preparedFromIr( ): { ok: true; mod: IrModule; sourceTexts: Map; warnings: ScrDiagnostic[] } { if ((options.outputKind ?? "exe") !== "exe" || options.strip !== true) throw new Error(`${BOOTSTRAP_IR_INPUT} requires a stripped executable build`); - const mod = deserializeModule(readFileSync(path, "utf8")); + const mod = deserializeModuleBytes(readFileSync(path)); if (resolve(mod.sourceFile) !== resolve(entry)) throw new Error(`${BOOTSTRAP_IR_INPUT} was lowered from a different entry`); return { ok: true, mod, sourceTexts: new Map(), warnings: [] }; @@ -194,14 +250,10 @@ export function prepareNativeExecutable( .sort(([a], [b]) => a.localeCompare(b)), }), ); - const bytes = cache.read("frontend", key); - if (bytes !== null) { - const cached: unknown = JSON.parse(bytes.toString("utf8")); - if (validCachedInput(cached) && frontendInputsStillMatch(cached.probes, exclusions)) { - const input: NativeExecutableInput = cached.input; - timing("frontend-cache-hit"); - return input; - } + const input = readFrontendCache(cache, key, exclusions); + if (input !== null) { + timing("frontend-cache-hit"); + return input; } } catch { key = null; @@ -217,7 +269,10 @@ export function prepareNativeExecutable( ); if (!prepared.ok) return prepared; if (irInput !== "") timing("ir-input"); - const ir = outputKind === "ir" || options.emitIr ? serializeModule(prepared.mod, true) : null; + const ir = + outputKind === "ir" || options.emitIr + ? batchTextParts(serializeModuleParts(prepared.mod)) + : null; timing("ir-serialize"); const llvm = outputKind === "ir" @@ -251,8 +306,7 @@ export function prepareNativeExecutable( probes.stable && frontendInputsStillMatch(probes, exclusions) ) { - const saved: FrontendCacheEntry = { schema: "scriptc.native-frontend.v2", probes, input }; - cache.write("frontend", key, JSON.stringify(saved)); + publishFrontendCache(cache, key, probes, input); } timing("frontend-cache-publish"); return input; @@ -307,14 +361,10 @@ export function prepareNativeLibrary( .sort(([a], [b]) => a.localeCompare(b)), }), ); - const bytes = cache.read("frontend", key); - if (bytes !== null) { - const cached: unknown = JSON.parse(bytes.toString("utf8")); - if (validCachedInput(cached) && frontendInputsStillMatch(cached.probes, exclusions)) { - const input: NativeExecutableInput = cached.input; - timing("frontend-cache-hit"); - return input; - } + const input = readFrontendCache(cache, key, exclusions); + if (input !== null) { + timing("frontend-cache-hit"); + return input; } } catch { key = null; @@ -350,7 +400,7 @@ export function prepareNativeLibrary( const input: NativeExecutableInput = { ok: true, llvm, - ir: options.emitIr ? serializeModule(prepared.mod, true) : null, + ir: options.emitIr ? batchTextParts(serializeModuleParts(prepared.mod)) : null, sidecarJson: prepared.sidecarJson, features: executableLinkFeatures(prepared.mod, false), sources: [...prepared.sourceTexts], @@ -363,8 +413,7 @@ export function prepareNativeLibrary( probes.stable && frontendInputsStillMatch(probes, exclusions) ) { - const saved: FrontendCacheEntry = { schema: "scriptc.native-frontend.v2", probes, input }; - cache.write("frontend", key, JSON.stringify(saved)); + publishFrontendCache(cache, key, probes, input); } timing("frontend-cache-publish"); return input; diff --git a/packages/compiler/src/native/text-file.ts b/packages/compiler/src/native/text-file.ts new file mode 100644 index 000000000..6f8990339 --- /dev/null +++ b/packages/compiler/src/native/text-file.ts @@ -0,0 +1,41 @@ +import { closeSync, openSync, writeSync } from "node:fs"; + +/** Native compiler strings stop at 2^31 - 1 UTF-8 bytes, and exceeding that + * traps instead of throwing. Large artifacts therefore travel as bounded + * parts; this is the part size that batching aims for. */ +export const TEXT_BATCH_LENGTH = 16 * 1024 * 1024; + +/** Join consecutive small parts into batches of about TEXT_BATCH_LENGTH + * UTF-16 units, so callers hold and write few, still-bounded strings. */ +export function batchTextParts(parts: readonly string[]): string[] { + const batches: string[] = []; + let batch: string[] = []; + let length = 0; + for (const part of parts) { + batch.push(part); + length += part.length; + if (length >= TEXT_BATCH_LENGTH) { + batches.push(batch.join("")); + batch = []; + length = 0; + } + } + if (batch.length > 0) batches.push(batch.join("")); + return batches; +} + +/** Write text parts to `path` in order without joining them first. Each part + * is one write; pass batched parts (`batchTextParts`) for many small ones. */ +export function writeTextPartsSync(path: string, parts: readonly string[]): void { + const fd = openSync(path, "w"); + try { + for (const part of parts) { + const bytes = Buffer.from(part, "utf8"); + for (let offset = 0; offset < bytes.length;) { + offset += writeSync(fd, bytes, offset, bytes.length - offset); + } + } + } finally { + closeSync(fd); + } +} diff --git a/packages/compiler/src/text-parts.ts b/packages/compiler/src/text-parts.ts new file mode 100644 index 000000000..a4519b0ba --- /dev/null +++ b/packages/compiler/src/text-parts.ts @@ -0,0 +1,37 @@ +import { open } from "node:fs/promises"; + +/** Writes compiler text artifacts (LLVM, serialized IR) in large batches. + * Programs past the engine's string limit arrive as many parts, and writing + * each part separately is several times slower than writing the same bytes in + * a few large chunks. Batches stay far below the limit. */ +export async function writeTextParts( + path: string, + text: string | readonly string[], + mode = 0o666, +): Promise { + const file = await open(path, "w", mode); + try { + const write = async (chunk: string): Promise => { + const bytes = Buffer.from(chunk, "utf8"); + for (let offset = 0; offset < bytes.length;) { + const { bytesWritten } = await file.write(bytes, offset, bytes.length - offset); + offset += bytesWritten; + } + }; + if (typeof text === "string") return await write(text); + let batch: string[] = []; + let length = 0; + for (const part of text) { + batch.push(part); + length += part.length; + if (length >= 16 * 1024 * 1024) { + await write(batch.join("")); + batch = []; + length = 0; + } + } + if (batch.length > 0) await write(batch.join("")); + } finally { + await file.close(); + } +} diff --git a/packages/compiler/test/ts7/baselines/order-parity.json b/packages/compiler/test/ts7/baselines/order-parity.json index f0c431d54..69dae17d5 100644 --- a/packages/compiler/test/ts7/baselines/order-parity.json +++ b/packages/compiler/test/ts7/baselines/order-parity.json @@ -10493,24 +10493,6 @@ ], "diags": [] }, - "/tests/corpus/missing-read-lambda-results.ts": { - "order": [ - "/tests/corpus/missing-read-lambda-results.ts" - ], - "diags": [] - }, - "/tests/corpus/missing-read-narrowed-storage.ts": { - "order": [ - "/tests/corpus/missing-read-narrowed-storage.ts" - ], - "diags": [] - }, - "/tests/corpus/missing-read-number-conversions.ts": { - "order": [ - "/tests/corpus/missing-read-number-conversions.ts" - ], - "diags": [] - }, "/tests/corpus/module-fallback-functions/main.js": { "order": [ "/tests/corpus/module-fallback-functions/platform.js", @@ -11427,18 +11409,6 @@ ], "diags": [] }, - "/tests/corpus/typed-array-nan-coded-reads.ts": { - "order": [ - "/tests/corpus/typed-array-nan-coded-reads.ts" - ], - "diags": [] - }, - "/tests/corpus/typed-array-present-reads.ts": { - "order": [ - "/tests/corpus/typed-array-present-reads.ts" - ], - "diags": [] - }, "/tests/corpus/typed-array-unknown.ts": { "order": [ "/tests/corpus/typed-array-unknown.ts" @@ -14560,6 +14530,12 @@ ], "diags": [] }, + "/tests/corpus/native-worker-arena-reuse.ts": { + "order": [ + "/tests/corpus/native-worker-arena-reuse.ts" + ], + "diags": [] + }, "/tests/corpus/native-worker-buffer-consumers.ts": { "order": [ "/tests/corpus/native-worker-buffer-consumers.ts" @@ -15773,6 +15749,42 @@ ], "diags": [] }, + "/tests/corpus/4598-map-set-clear-reuse.ts": { + "order": [ + "/tests/corpus/4598-map-set-clear-reuse.ts" + ], + "diags": [] + }, + "/tests/corpus/missing-read-lambda-results.ts": { + "order": [ + "/tests/corpus/missing-read-lambda-results.ts" + ], + "diags": [] + }, + "/tests/corpus/missing-read-narrowed-storage.ts": { + "order": [ + "/tests/corpus/missing-read-narrowed-storage.ts" + ], + "diags": [] + }, + "/tests/corpus/missing-read-number-conversions.ts": { + "order": [ + "/tests/corpus/missing-read-number-conversions.ts" + ], + "diags": [] + }, + "/tests/corpus/typed-array-nan-coded-reads.ts": { + "order": [ + "/tests/corpus/typed-array-nan-coded-reads.ts" + ], + "diags": [] + }, + "/tests/corpus/typed-array-present-reads.ts": { + "order": [ + "/tests/corpus/typed-array-present-reads.ts" + ], + "diags": [] + }, "/tests/corpus/4601-cycle-holders.ts": { "order": [ "/tests/corpus/4601-cycle-holders.ts" diff --git a/packages/runtime/src/scr_alloc.c b/packages/runtime/src/scr_alloc.c index 9e08c9594..32352d9dd 100644 --- a/packages/runtime/src/scr_alloc.c +++ b/packages/runtime/src/scr_alloc.c @@ -6,9 +6,10 @@ * and the out-of-line entry points the LLVM backend calls for acyclic * records and class instances. * - * Freed blocks are recycled within their class and never returned to the - * operating system; the reservation itself costs address space only, and - * pages are committed as the bump pointers first touch them. */ + * Freed blocks are recycled within their class; the reservation itself + * costs address space only, and pages are committed as the bump pointers + * first touch them. Only an exiting worker thread returns pages, those + * holding nothing but free blocks (scr_sa_trim, Linux). */ #include "scr_runtime.h" /* Defined in every build: the emitted fast paths read it (see the layout @@ -84,6 +85,15 @@ static char *scr_sa_extend(char *start, char *lim, char *end) { #ifdef SCR_WORKERS #include #include +#if defined(__linux__) && defined(MADV_DONTNEED) +#define SCR_SA_TRIM 1 +#include +#include +#include +#if defined(__GLIBC__) +#include +#endif +#endif /* Per-thread arenas (contract in scr_runtime.h). The region is reserved * once; arena i spans [region + i * arena_span, + arena_span) and is split @@ -123,6 +133,13 @@ static bool scr_sa_reserve_region(void) { return atomic_load_explicit(&scr_sa_region, memory_order_acquire) != 0; } +#ifdef SCR_SA_TRIM +/* The arena's carved bytes when the current thread claimed it (see + * scr_sa_trim). */ +static size_t scr_sa_claim_carved[SCR_SA_ARENAS]; +static size_t scr_sa_carved(void); +#endif + /* Moves every block other threads freed into this arena's free lists. */ static void scr_sa_drain_remote(void) { ScrSaBlock *b = atomic_exchange_explicit(&scr_sa_remote[scr_sa_arena], NULL, memory_order_acquire); @@ -161,14 +178,287 @@ static bool scr_sa_reserve(void) { scr_sa.shift = scr_sa_region_shift; } scr_sa_drain_remote(); +#ifdef SCR_SA_TRIM + scr_sa_claim_carved[i] = scr_sa_carved(); +#endif return true; } } } +#ifdef SCR_SA_TRIM +/* Returning an exited thread's free memory (Linux). When a script thread + * exits, every page of its arena that holds only free blocks goes back to + * the system (MADV_DONTNEED, after which it reads as zero), so other + * threads can use that memory while the arena is parked. The free blocks on + * the kept pages are relinked in address order; the returned runs of pages + * become zeroed carve regions (`ScrSaHoles`), which the next thread to claim + * the arena carves before the slice's own frontier: the bump fast paths + * already rely on carved memory being zero. A thread that carved less than + * SCR_SA_TRIM_MIN, or whose arena holds less than that in free blocks, + * leaves the arena as it is. */ +#define SCR_SA_TRIM_MIN ((size_t)16 << 20) +typedef struct { + char *lo, *hi; +} ScrSaRun; +typedef struct { + ScrSaRun *runs; /* zeroed regions left to carve, carved from the end */ + size_t n, cap; + char *bump, *lim; /* the frontier while a region is carved (lim != NULL) */ +} ScrSaHoles; +static ScrSaHoles scr_sa_holes[SCR_SA_ARENAS][SCR_SA_NCLASS]; + +/* Bytes carved from the calling thread's slices: up to each frontier. */ +static size_t scr_sa_carved(void) { + size_t carved = 0; + for (unsigned c = 0; c < SCR_SA_NCLASS; c++) { + if (!scr_sa.lim[c]) continue; + const ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c]; + char *top = h->lim ? h->bump : scr_sa.bump[c]; + carved += (size_t)(top - ((char *)scr_sa.base + ((size_t)c << scr_sa.shift))); + } + return carved; +} + +/* For the slow path of class c, whose bump room is below sz: moves the + * bump pointer to the next zeroed region, or back to the frontier. */ +static void scr_sa_next_hole(unsigned c, size_t sz) { + ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c]; + if ((size_t)(scr_sa.lim[c] - scr_sa.bump[c]) >= sz || (!h->n && !h->lim)) return; + if (h->n) { + if (!h->lim) { + h->bump = scr_sa.bump[c]; + h->lim = scr_sa.lim[c]; + } + ScrSaRun r = h->runs[--h->n]; + scr_sa.bump[c] = r.lo; + scr_sa.lim[c] = r.hi; + } else { + scr_sa.bump[c] = h->bump; + scr_sa.lim[c] = h->lim; + h->bump = h->lim = NULL; + } +} + +static bool scr_sa_push_run(ScrSaHoles *h, char *lo, char *hi) { + if (h->n == h->cap) { + size_t cap = h->cap ? 2 * h->cap : 64; + ScrSaRun *runs = realloc(h->runs, cap * sizeof *runs); + if (!runs) return false; + h->runs = runs; + h->cap = cap; + } + h->runs[h->n++] = (ScrSaRun){lo, hi}; + return true; +} + +/* With SCR_SA_STATS=1: carved, free and returned bytes of the parking thread. */ +static _Thread_local size_t scr_sa_trim_stats[3]; + +/* Returns the pages of class c that hold only free blocks (see above). */ +static void scr_sa_trim_class(unsigned c, size_t page) { + ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c]; + size_t sz = (size_t)(c + 1) * SCR_SA_STEP; + char *start = (char *)scr_sa.base + ((size_t)c << scr_sa.shift); + /* Close the region being carved: its rest is still zero. */ + if (h->lim) { + char *lo = scr_sa.bump[c], *hi = scr_sa.lim[c]; + scr_sa.bump[c] = h->bump; + scr_sa.lim[c] = h->lim; + h->bump = h->lim = NULL; + if ((size_t)(hi - lo) >= sz && !scr_sa_push_run(h, lo, hi)) return; + } + char *top = scr_sa.bump[c]; + size_t nblocks = (size_t)(top - start) / sz; + if (!nblocks) return; + size_t words = (nblocks + 63) / 64; + uint64_t *bits = mmap(NULL, words * 8, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + if (bits == MAP_FAILED) return; + size_t nfree = 0; + for (ScrSaBlock *b = scr_sa.free[c]; b; b = b->next) { + size_t k = (size_t)((char *)b - start) / sz; + bits[k / 64] |= (uint64_t)1 << (k % 64); + nfree++; + } + /* Uncarved zeroed regions count as free; they are rebuilt below. */ + for (size_t r = 0; r < h->n; r++) { + for (char *q = h->runs[r].lo; q + sz <= h->runs[r].hi; q += sz) { + size_t k = (size_t)(q - start) / sz; + bits[k / 64] |= (uint64_t)1 << (k % 64); + } + } + scr_sa_trim_stats[0] += nblocks * sz; + scr_sa_trim_stats[1] += nfree * sz; +#define SCR_SA_BIT(k) ((bits[(k) / 64] >> ((k) % 64)) & 1) + /* Whole pages below the frontier whose blocks are all free. */ + uintptr_t first = ((uintptr_t)start + page - 1) & ~(uintptr_t)(page - 1); + uintptr_t last = (uintptr_t)top & ~(uintptr_t)(page - 1); + h->n = 0; + char *run = NULL; + for (uintptr_t pg = first; pg <= last; pg += page) { + bool empty = false; + if (pg < last) { + size_t k0 = (size_t)((char *)pg - start) / sz; + size_t k1 = (size_t)((char *)pg + page - 1 - start) / sz; + if (k1 >= nblocks) k1 = nblocks - 1; + empty = true; + for (size_t k = k0; k <= k1 && empty; k++) empty = SCR_SA_BIT(k); + } + if (empty && !run) run = (char *)pg; + if (!empty && run) { + (void)madvise(run, (size_t)((char *)pg - run), MADV_DONTNEED); + scr_sa_trim_stats[2] += (size_t)((char *)pg - run); + /* Blocks wholly inside the returned pages carve from it later. */ + char *lo = start + ((size_t)(run - start) + sz - 1) / sz * sz; + char *hi = start + ((size_t)((char *)pg - start) / sz) * sz; + if (hi > lo) (void)scr_sa_push_run(h, lo, hi); + for (char *q = lo; q < hi; q += sz) { + size_t k = (size_t)(q - start) / sz; + bits[k / 64] &= ~((uint64_t)1 << (k % 64)); + } + /* A block starting in the returned pages but ending on a kept one is + * dropped: relinking it would fault its page back in. */ + if (hi < (char *)pg && hi >= run) { + size_t k = (size_t)(hi - start) / sz; + bits[k / 64] &= ~((uint64_t)1 << (k % 64)); + } + run = NULL; + } + } +#undef SCR_SA_BIT + /* The remaining free blocks, in address order. */ + ScrSaBlock *head = NULL, **tail = &head; + for (size_t w = 0; w < words; w++) { + for (uint64_t m = bits[w]; m; m &= m - 1) { + size_t k = w * 64 + (size_t)__builtin_ctzll(m); + ScrSaBlock *b = (ScrSaBlock *)(start + k * sz); + *tail = b; + tail = &b->next; + } + } + *tail = NULL; + scr_sa.free[c] = head; + munmap(bits, words * 8); + /* The next owner carves the returned regions before the frontier. */ + if (h->n) { + h->bump = scr_sa.bump[c]; + h->lim = scr_sa.lim[c]; + ScrSaRun r = h->runs[--h->n]; + scr_sa.bump[c] = r.lo; + scr_sa.lim[c] = r.hi; + } +} + +/* Whether the calling thread carved at least SCR_SA_TRIM_MIN since it + * claimed its arena: a short-lived thread reusing the arena's free blocks + * leaves them for the next one. */ +static bool scr_sa_trim_worth(void) { + return scr_sa_carved() - scr_sa_claim_carved[scr_sa_arena] >= SCR_SA_TRIM_MIN; +} + +/* Returns the free pages of the calling thread's arena (see above). */ +static void scr_sa_trim(void) { + scr_sa_drain_remote(); + size_t free_bytes = 0; + for (unsigned c = 0; c < SCR_SA_NCLASS; c++) { + size_t n = 0; + for (ScrSaBlock *b = scr_sa.free[c]; b && n * (c + 1) * SCR_SA_STEP < SCR_SA_TRIM_MIN; + b = b->next) + n++; + free_bytes += n * (c + 1) * SCR_SA_STEP; + if (free_bytes >= SCR_SA_TRIM_MIN) break; + } + const char *stats = getenv("SCR_SA_STATS"); + const char *off = getenv("SCR_SA_TRIM"); + bool trim = free_bytes >= SCR_SA_TRIM_MIN && !(off && off[0] == '0'); + memset(scr_sa_trim_stats, 0, sizeof scr_sa_trim_stats); + if (!trim && stats && stats[0] == '1') + for (unsigned c = 0; c < SCR_SA_NCLASS; c++) { + if (!scr_sa.lim[c]) continue; + size_t sz = (size_t)(c + 1) * SCR_SA_STEP; + char *start = (char *)scr_sa.base + ((size_t)c << scr_sa.shift); + scr_sa_trim_stats[0] += (size_t)(scr_sa.bump[c] - start); + for (ScrSaBlock *b = scr_sa.free[c]; b; b = b->next) scr_sa_trim_stats[1] += sz; + } + if (trim) { + long page = sysconf(_SC_PAGESIZE); + for (unsigned c = 0; c < SCR_SA_NCLASS; c++) + if (scr_sa.lim[c]) scr_sa_trim_class(c, page > 0 ? (size_t)page : 4096); + } +#if defined(__GLIBC__) + /* Large blocks: the system allocator's free pages too. */ + if (trim) (void)malloc_trim(0); +#endif + if (stats && stats[0] == '1') + fprintf(stderr, "scr_sa: park arena %d trim %d carved %zu MB free %zu MB returned %zu MB\n", + scr_sa_arena, (int)trim, scr_sa_trim_stats[0] >> 20, scr_sa_trim_stats[1] >> 20, + scr_sa_trim_stats[2] >> 20); +} +#endif + +#ifdef SCR_SA_TRIM +/* Trims run on a detached helper thread: the parent of a worker joins it + * when it handles the worker's exit, which must not wait for the trim (at + * the end of a program every worker exits at once, and the process ends + * right after). The helper takes over the exiting thread's claim on the + * arena, so no thread claims it during the trim. */ +static _Atomic(unsigned) scr_sa_trimming; /* helpers running */ + +static void scr_sa_release(unsigned i) { + scr_sa_parked[i] = scr_sa; + memset(&scr_sa, 0, sizeof scr_sa); + scr_sa_arena = -1; + scr_sa_refused = true; + atomic_fetch_and_explicit(&scr_sa_claimed, ~(1u << i), memory_order_release); +} + +static void *scr_sa_trim_main(void *arg) { + unsigned i = (unsigned)(uintptr_t)arg; + scr_sa_arena = (int)i; + scr_sa = scr_sa_parked[i]; + scr_sa_trim(); + scr_sa_release(i); + atomic_fetch_sub_explicit(&scr_sa_trimming, 1, memory_order_release); + return NULL; +} + +void scr_sa_wait_trims(void) { + while (atomic_load_explicit(&scr_sa_trimming, memory_order_acquire)) { + struct timespec pause = {0, 1000000}; + nanosleep(&pause, NULL); + } +} +#else +void scr_sa_wait_trims(void) {} +#endif + void scr_sa_thread_park(void) { if (scr_sa_arena < 0) return; unsigned i = (unsigned)scr_sa_arena; +#ifdef SCR_SA_TRIM + if (scr_sa_trim_worth()) { + scr_sa_parked[i] = scr_sa; + atomic_fetch_add_explicit(&scr_sa_trimming, 1, memory_order_relaxed); + pthread_attr_t attr; + pthread_t helper; + bool started = pthread_attr_init(&attr) == 0; + if (started) { + started = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) == 0 && + pthread_create(&helper, &attr, scr_sa_trim_main, (void *)(uintptr_t)i) == 0; + pthread_attr_destroy(&attr); + } + if (started) { + /* Later frees on this thread take the foreign path into the arena's + * remote list; the helper releases the claim. */ + memset(&scr_sa, 0, sizeof scr_sa); + scr_sa_arena = -1; + scr_sa_refused = true; + return; + } + atomic_fetch_sub_explicit(&scr_sa_trimming, 1, memory_order_relaxed); + scr_sa_trim(); + } +#endif scr_sa_parked[i] = scr_sa; /* Later frees on this thread (thread-exit destructors) take the foreign * path into the parked arena's remote list. */ @@ -241,6 +531,9 @@ __attribute__((cold, noinline)) void *scr_sa_slow(size_t n, bool zero) { /* Carve from the slice, opening it or raising its limit as needed; * fresh pages are zero. */ size_t sz = (size_t)(c + 1) * SCR_SA_STEP; +#ifdef SCR_SA_TRIM + if (scr_sa.lim[c]) scr_sa_next_hole(c, sz); +#endif char *start = (char *)scr_sa.base + ((size_t)c << scr_sa.shift); char *end = start + ((size_t)1 << scr_sa.shift); char *p = scr_sa.lim[c] ? scr_sa.bump[c] : start; @@ -279,6 +572,7 @@ void *scr_mem_realloc(void *p, size_t n) { void *scr_mem_realloc(void *p, size_t n) { return realloc(p, n); } #ifdef SCR_WORKERS void scr_sa_thread_park(void) {} +void scr_sa_wait_trims(void) {} void scr_sa_free_foreign(void *p) { free(p); } #endif #endif diff --git a/packages/runtime/src/scr_alloc.test.c b/packages/runtime/src/scr_alloc.test.c index ae0c92370..9978c819e 100644 --- a/packages/runtime/src/scr_alloc.test.c +++ b/packages/runtime/src/scr_alloc.test.c @@ -217,6 +217,91 @@ static void check_threads(void) { } #endif +#if defined(SCR_WORKERS) && SCR_SMALL_ALLOC && defined(__linux__) +#include +#include + +/* An exiting thread returns the pages that hold only free blocks; the next + * thread to claim the arena carves the returned range again, zeroed, before + * growing the slice. */ +enum { TRIM_BLOCKS = 1 << 20, TRIM_KEEP = 4096, TRIM_SIZE = 48 }; +static unsigned char **trim_blocks; +static unsigned char *trim_kept[TRIM_BLOCKS / TRIM_KEEP]; +static uintptr_t trim_lo, trim_hi; +static size_t trim_outside; + +static size_t resident_pages(uintptr_t lo, uintptr_t hi) { + size_t page = (size_t)sysconf(_SC_PAGESIZE); + lo &= ~(uintptr_t)(page - 1); + size_t n = (hi - lo + page - 1) / page, count = 0; + unsigned char *vec = malloc(n); + assert(vec && mincore((void *)lo, n * page, vec) == 0); + for (size_t i = 0; i < n; i++) count += vec[i] & 1; + free(vec); + return count; +} + +static void *trim_allocate(void *arg) { + (void)arg; + trim_lo = UINTPTR_MAX; + trim_hi = 0; + for (size_t i = 0; i < TRIM_BLOCKS; i++) { + trim_blocks[i] = scr_mem_alloc(TRIM_SIZE); + assert(trim_blocks[i] && owned(trim_blocks[i])); + fill(trim_blocks[i], TRIM_SIZE, (unsigned char)i); + if ((uintptr_t)trim_blocks[i] < trim_lo) trim_lo = (uintptr_t)trim_blocks[i]; + if ((uintptr_t)trim_blocks[i] + TRIM_SIZE > trim_hi) trim_hi = (uintptr_t)trim_blocks[i] + TRIM_SIZE; + } + for (size_t i = 0; i < TRIM_BLOCKS; i++) { + if (i % TRIM_KEEP == 0) trim_kept[i / TRIM_KEEP] = trim_blocks[i]; + else scr_mem_free(trim_blocks[i]); + } + scr_sa_thread_park(); + return NULL; +} + +static void *trim_reuse(void *arg) { + (void)arg; + trim_outside = 0; + for (size_t i = 0; i < TRIM_BLOCKS; i++) { + unsigned char *p = scr_mem_calloc(TRIM_SIZE); + assert(p && owned(p)); + for (size_t k = 0; k < TRIM_SIZE; k++) assert(p[k] == 0); + fill(p, TRIM_SIZE, (unsigned char)i); + if ((uintptr_t)p < trim_lo || (uintptr_t)p >= trim_hi) trim_outside++; + trim_blocks[i] = p; + } + for (size_t i = 0; i < TRIM_BLOCKS; i++) scr_mem_free(trim_blocks[i]); + scr_sa_thread_park(); + return NULL; +} + +static void check_trim(void) { + trim_blocks = malloc(TRIM_BLOCKS * sizeof *trim_blocks); + assert(trim_blocks); + pthread_t t; + assert(pthread_create(&t, NULL, trim_allocate, NULL) == 0); + assert(pthread_join(t, NULL) == 0); + scr_sa_wait_trims(); /* a helper thread returns the pages */ + /* One page per kept block stays, give or take the pages at run ends. */ + size_t total = (trim_hi - trim_lo) / (size_t)sysconf(_SC_PAGESIZE); + size_t resident = resident_pages(trim_lo, trim_hi); + assert(resident < total / 8); + for (size_t k = 0; k < TRIM_BLOCKS / TRIM_KEEP; k++) + verify(trim_kept[k], TRIM_SIZE, (unsigned char)(k * TRIM_KEEP)); + assert(pthread_create(&t, NULL, trim_reuse, NULL) == 0); + assert(pthread_join(t, NULL) == 0); + scr_sa_wait_trims(); + /* Nearly every block came from the returned range: the slice did not grow. */ + assert(trim_outside < 4 * TRIM_BLOCKS / TRIM_KEEP + 256); + for (size_t k = 0; k < TRIM_BLOCKS / TRIM_KEEP; k++) { + verify(trim_kept[k], TRIM_SIZE, (unsigned char)(k * TRIM_KEEP)); + scr_mem_free(trim_kept[k]); /* remote free into the parked arena */ + } + free(trim_blocks); +} +#endif + int main(void) { #if SCR_SMALL_ALLOC check_slice_growth(); /* first: its class must be untouched */ @@ -227,6 +312,9 @@ int main(void) { check_churn(); #if defined(SCR_WORKERS) && SCR_SMALL_ALLOC check_threads(); +#ifdef __linux__ + check_trim(); +#endif printf("allocator checks passed: size-class workers\n"); #else printf("allocator checks passed: %s\n", SCR_SMALL_ALLOC ? "size-class" : "system"); diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 064204975..804396c3f 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -518,15 +518,51 @@ static __attribute__((noinline)) void scr_map_leave_dense(ScrMap *m) { } /* Drop tombstones, preserving insertion order. Only legal when no iteration - * is active (the forEach desugar's indices would shift). */ -static void scr_map_compact(ScrMap *m) { - size_t w = 0; + * is active (the forEach desugar's indices would shift). + * + * Index tables are sized by the most entries a map has held, so a map that + * shrank keeps rebuilding (and clearing) a table sized for its peak. A + * bucket table more than eight times what the survivors need shrinks to + * twice that, leaving room to double before it grows again. With `reindex` + * the direct index also follows the surviving keys: it shrinks to cover the + * largest one, and keys that became sparse move back to hashing (at twice + * the spread that admits them, so churn cannot flip modes). The append path + * passes false: it holds a slot of the current direct index. */ +static void scr_map_compact(ScrMap *m, bool reindex) { + size_t w = 0, top = 0; + bool dense = m->ndense != 0; for (size_t r = 0; r < m->nentries; r++) { - if (m->entries[r].hash) m->entries[w++] = m->entries[r]; + if (!m->entries[r].hash) continue; + if (dense) { + size_t k = (size_t)scr_map_slot_to_f64(m->entries[r].key); + if (k > top) top = k; + } + m->entries[w++] = m->entries[r]; } m->nentries = w; - if (m->nbuckets > 0) scr_map_rebuild_buckets(m, m->nbuckets); - else if (m->ndense) scr_map_fill_dense(m); + if (m->nbuckets > 0) { + size_t want = 16; + while (want < 2 * (w + 1)) want *= 2; + scr_map_rebuild_buckets(m, m->nbuckets / 8 > want ? 2 * want : m->nbuckets); + return; + } + if (!dense) return; + if (reindex) { + if (w == 0) { + free(m->dense); + m->dense = NULL; + m->ndense = 0; + return; + } + size_t limit = scr_map_dense_limit(w); + if (top / 2 >= limit) { + scr_map_leave_dense(m); + return; + } + size_t need = scr_map_dense_size(top); + if (need < m->ndense) scr_map_dense_alloc(m, need); + } + scr_map_fill_dense(m); } static void scr_map_grow_entries(ScrMap *m) { @@ -548,7 +584,7 @@ static void scr_map_reserve_append(ScrMap *m) { if (m->ndense) { /* Directly indexed: the table is sized by keys, not entry count. */ if (m->nentries < m->ecap) return; - if (m->iter_depth == 0 && m->nlive <= m->nentries / 2) scr_map_compact(m); + if (m->iter_depth == 0 && m->nentries > 0 && m->nlive <= m->nentries / 2) scr_map_compact(m, false); if (m->nentries == m->ecap) scr_map_grow_entries(m); return; } @@ -557,7 +593,7 @@ static void scr_map_reserve_append(ScrMap *m) { m->nbuckets >= 2 * (m->nentries + 1))) return; if (m->iter_depth == 0 && m->nentries > 0 && (m->nbuckets == 0 ? m->nlive < m->nentries : m->nlive <= m->nentries / 2)) { - scr_map_compact(m); + scr_map_compact(m, false); } if (m->nentries == m->ecap) scr_map_grow_entries(m); if (m->nbuckets == 0 && m->nentries < SCR_MAP_LINEAR_LIMIT) return; @@ -701,10 +737,29 @@ void scr_map_trace_v(void *m, ScrTraceVisit visit, void *ctx) { double scr_map_size(const ScrMap *m) { return (double)m->nlive; } +/* Storage clear() keeps regardless of how many entries it removed. */ +#define SCR_MAP_CLEAR_KEEP 256 + +/* clear() costs time proportional to the entries it walks (live entries + * and tombstones appended since the last reset), never to the peak size of + * the index tables: + * - the direct index is zero exactly at the absent keys (deletes clear + * their slots), so only the live entries' slots are reset; + * - the bucket table is reset only while it is proportional to the + * entries that pointed into it; + * - storage far larger than the entries just removed is freed, returning + * the map to its initial empty state the way V8 reallocates a cleared + * table. Regrowth is amortized against the insertions that need it. + * A map without entries has empty tables already: clearing it does no + * work and keeps its storage for the next fill. */ void scr_map_clear(ScrMap *m) { - for (size_t e = 0; e < m->nentries; e++) { + size_t walked = m->nentries; + if (walked == 0) return; + uint32_t *dense = m->ndense ? m->dense : NULL; + for (size_t e = 0; e < walked; e++) { if (!m->entries[e].hash) continue; m->entries[e].hash = 0; + if (dense) dense[(size_t)scr_map_slot_to_f64(m->entries[e].key)] = 0; scr_map_release_key(m, m->entries[e].key); scr_map_release_val(m, m->entries[e].val); } @@ -714,12 +769,29 @@ void scr_map_clear(ScrMap *m) { * entries so its indices stay stable — entries added by the callback * after the clear append past them and ARE visited (Node-exact). */ m->nentries = 0; + if (m->ecap > SCR_MAP_CLEAR_KEEP && m->ecap / 8 > walked) { + free(m->entries); + m->entries = NULL; + m->ecap = 0; + } + } + /* Without either index, a map is a linear scan; dead entries kept for an + * active iteration are skipped by every later rebuild. */ + if (m->ndense > SCR_MAP_CLEAR_KEEP && m->ndense / 32 > walked) { + free(m->dense); + m->dense = NULL; + m->ndense = 0; } - if (m->ndense) memset(m->dense, 0, m->ndense * sizeof *m->dense); if (m->nbuckets) { - bool compact = scr_map_compact_buckets(m->nbuckets); - memset(m->buckets, compact ? 0 : 0xff, - m->nbuckets * (compact ? sizeof(uint32_t) : sizeof(size_t))); + if (m->nbuckets > SCR_MAP_CLEAR_KEEP && m->nbuckets / 8 > walked) { + free(m->buckets); + m->buckets = NULL; + m->nbuckets = 0; + } else { + bool compact = scr_map_compact_buckets(m->nbuckets); + memset(m->buckets, compact ? 0 : 0xff, + m->nbuckets * (compact ? sizeof(uint32_t) : sizeof(size_t))); + } } } @@ -733,13 +805,26 @@ bool scr_map_has_str(const ScrMap *m, const ScrStr *key) { return scr_map_find_str(m, key) != SCR_MAP_EMPTY; } +/* Tombstones beyond this share of the entries are compacted on delete, so + * iteration and later rebuilds cost O(live) rather than O(peak). V8 shrinks + * an ordered hash table at the same occupancy. Small maps keep theirs. */ +#define SCR_MAP_SHRINK_MIN 64 + static bool scr_map_delete_found(ScrMap *m, size_t e) { if (e == SCR_MAP_EMPTY) return false; + uint64_t key = m->entries[e].key, val = m->entries[e].val; m->entries[e].hash = 0; /* bucket slot stays: probe chains intact */ m->nlive--; - if (m->ndense) m->dense[(size_t)scr_map_slot_to_f64(m->entries[e].key)] = 0; - scr_map_release_key(m, m->entries[e].key); - scr_map_release_val(m, m->entries[e].val); + if (m->ndense) m->dense[(size_t)scr_map_slot_to_f64(key)] = 0; + /* Each such compaction follows deletes of three quarters of the entries, + * so it is amortized constant per delete. It runs before the releases: + * they may drop the last reference to this map. */ + if (__builtin_expect(m->iter_depth == 0 && m->nentries >= SCR_MAP_SHRINK_MIN && + m->nlive < m->nentries / 4, 0)) { + scr_map_compact(m, true); + } + scr_map_release_key(m, key); + scr_map_release_val(m, val); return true; } @@ -1166,9 +1251,14 @@ ScrMap *scr_map_clone(const ScrMap *source, bool keys_only) { out->entries = malloc(count * sizeof(ScrMapEntry)); if (!out->entries) scr_map_oom(); out->ecap = count; + size_t top = 0; for (size_t i = 0; i < source->nentries; i++) { const ScrMapEntry *entry = &source->entries[i]; if (!entry->hash) continue; + if (source->ndense) { + size_t k = (size_t)scr_map_slot_to_f64(entry->key); + if (k > top) top = k; + } ScrMapEntry *copy = &out->entries[out->nentries++]; *copy = *entry; if (out->key_kind == SCR_MAP_KEY_STR) scr_str_retain(scr_map_slot_to_ptr(copy->key)); @@ -1177,9 +1267,10 @@ ScrMap *scr_map_clone(const ScrMap *source, bool keys_only) { else if (out->val_kind == SCR_MAP_VAL_REF) out->val_retain(scr_map_slot_to_ptr(copy->val)); } out->nlive = count; - if (source->ndense) { - /* Same keys, so the same table size; entry indices are compacted. */ - scr_map_dense_alloc(out, source->ndense); + /* Size the copy's direct index for its own keys, not for the source's + * peak; keys that are sparse for their count hash instead. */ + if (source->ndense && top / 2 < scr_map_dense_limit(count)) { + scr_map_dense_alloc(out, scr_map_dense_size(top)); scr_map_fill_dense(out); return out; } @@ -1245,7 +1336,7 @@ void scr_map_iter_exit(ScrMap *m) { * active they are safe to drop now (bounds memory under forEach-heavy * add/delete workloads). */ if (m->iter_depth == 0 && m->nlive <= m->nentries / 2 && m->nentries >= 16) { - scr_map_compact(m); + scr_map_compact(m, true); } } diff --git a/packages/runtime/src/scr_map.test.c b/packages/runtime/src/scr_map.test.c index d947515bb..6c6e22e01 100644 --- a/packages/runtime/src/scr_map.test.c +++ b/packages/runtime/src/scr_map.test.c @@ -9,6 +9,7 @@ #include #include #include +#include #define MODEL_CAP 20000 @@ -227,10 +228,253 @@ static void modes(void) { scr_map_release(set); } +/* clear() and delete must cost time proportional to the entries they + * touch, not to the largest size the map ever had. Each case grows the + * index tables to about a million slots, then times many small rounds + * against the same rounds on a fresh map: a clear that resets the whole + * table (megabytes per call), or iteration over the tombstones of the old + * peak, is hundreds of times slower and exceeds the bound early. The bound + * is relative, so machine speed and sanitizers do not matter. */ +#define BIG_KEYS (1 << 20) +#define SMALL_ROUNDS 50000 + +typedef void (*Round)(ScrMap *map, size_t round, void *ctx); + +static double cpu_seconds(void) { + return (double)clock() / CLOCKS_PER_SEC; +} + +/* CPU seconds for SMALL_ROUNDS rounds; stops early once over budget. */ +static double timed_rounds(ScrMap *map, Round round, void *ctx, double budget) { + double start = cpu_seconds(); + for (size_t r = 0; r < SMALL_ROUNDS; r++) { + round(map, r, ctx); + if (r % 1024 == 1023 && cpu_seconds() - start > budget) break; + } + return cpu_seconds() - start; +} + +static void expect_proportional(const char *what, ScrMap *fresh, ScrMap *grown, Round round, void *ctx) { + double base = timed_rounds(fresh, round, ctx, INFINITY); + double budget = 20 * base + 0.05; + double spent = timed_rounds(grown, round, ctx, budget); + if (spent > budget) { + fprintf(stderr, "%s: cost follows the peak size (%.3f s, fresh map %.3f s)\n", what, spent, base); + exit(1); + } +} + +/* keys: 0 directly indexed integers, 1 sparse integers (hashed). */ +static double cost_key(unsigned keys, size_t i) { + return keys == 0 ? (double)i : (double)i * 1000003.0; +} + +/* clear() an empty map, add up to six keys, clear() again. */ +static void number_round(ScrMap *map, size_t round, void *ctx) { + unsigned keys = *(unsigned *)ctx; + scr_map_clear(map); + size_t n = round % 7; + for (size_t i = 0; i < n; i++) scr_map_set_f64_f64(map, cost_key(keys, round + i), (double)i); + assert(scr_map_size(map) == (double)n); + scr_map_clear(map); +} + +static void clear_cost_numbers(unsigned keys) { + ScrMap *map = scr_map_new(SCR_MAP_KEY_F64, SCR_MAP_VAL_F64, NULL, NULL, NULL); + for (size_t i = 0; i < BIG_KEYS; i++) scr_map_set_f64_f64(map, cost_key(keys, i), (double)i); + assert(keys == 0 ? map->ndense >= BIG_KEYS : map->nbuckets >= 2 * BIG_KEYS); + scr_map_clear(map); + assert(scr_map_size(map) == 0 && scr_map_iter_count(map) == 0 && !scr_map_has_f64(map, cost_key(keys, 1))); + ScrMap *fresh = scr_map_new(SCR_MAP_KEY_F64, SCR_MAP_VAL_F64, NULL, NULL, NULL); + expect_proportional(keys == 0 ? "dense clear" : "hashed clear", fresh, map, number_round, &keys); + scr_map_release(fresh); + /* Storage sized for the old peak is gone; the map is reusable. */ + assert(map->ndense <= 256 && map->nbuckets <= 256 && map->ecap <= 256); + for (size_t i = 0; i < 100; i++) scr_map_set_f64_f64(map, cost_key(keys, 99 - i), (double)i); + size_t j = 0; + for (size_t i = 0; i < (size_t)scr_map_iter_count(map); i++) { + if (!scr_map_iter_live(map, (double)i)) continue; + assert(scr_map_iter_key_f64(map, (double)i) == cost_key(keys, 99 - j)); + assert(scr_map_iter_val_f64(map, (double)i) == (double)j); + j++; + } + assert(j == 100 && !scr_map_has_f64(map, cost_key(keys, 100))); + scr_map_release(map); +} + +static ScrStr *key_str(size_t i) { + char buf[32]; + int len = snprintf(buf, sizeof buf, "k%zu", i); + return scr_str_new(buf, (size_t)len); +} + +enum { STRING_KEYS = 1 << 18 }; + +static void string_round(ScrMap *map, size_t round, void *ctx) { + ScrStr **keys = ctx; + scr_map_clear(map); + size_t n = round % 7; + for (size_t i = 0; i < n; i++) scr_map_set_str_f64(map, keys[(round + i) % STRING_KEYS], (double)i); + assert(scr_map_size(map) == (double)n); + scr_map_clear(map); +} + +static void clear_cost_strings(void) { + ScrStr **keys = malloc(STRING_KEYS * sizeof *keys); + assert(keys); + for (size_t i = 0; i < STRING_KEYS; i++) keys[i] = key_str(i); + ScrMap *map = scr_map_new(SCR_MAP_KEY_STR, SCR_MAP_VAL_F64, NULL, NULL, NULL); + for (size_t i = 0; i < STRING_KEYS; i++) scr_map_set_str_f64(map, keys[i], (double)i); + assert(map->nbuckets >= 2 * STRING_KEYS); + scr_map_clear(map); + ScrMap *fresh = scr_map_new(SCR_MAP_KEY_STR, SCR_MAP_VAL_F64, NULL, NULL, NULL); + expect_proportional("string clear", fresh, map, string_round, keys); + scr_map_release(fresh); + assert(map->nbuckets <= 256 && map->ecap <= 256); + for (size_t i = 0; i < 10; i++) scr_map_set_str_f64(map, keys[i], (double)i); + for (size_t i = 0; i < 20; i++) assert(scr_map_has_str(map, keys[i]) == (i < 10)); + scr_map_release(map); + for (size_t i = 0; i < STRING_KEYS; i++) scr_str_release(keys[i]); + free(keys); +} + +/* Walk the entries the way the forEach desugar does. */ +static void iterate_round(ScrMap *map, size_t round, void *ctx) { + (void)round; + size_t count = 0; + for (size_t i = 0; i < (size_t)scr_map_iter_count(map); i++) count += scr_map_iter_live(map, (double)i); + assert(count == *(size_t *)ctx); +} + +/* Deleting almost everything compacts the entries and index tables, so + * iteration costs O(live), and keys left far apart move back to hashing + * instead of keeping a huge direct index. */ +static void delete_shrinks(void) { + ScrMap *map = scr_map_new(SCR_MAP_KEY_F64, SCR_MAP_VAL_F64, NULL, NULL, NULL); + ScrMap *fresh = scr_map_new(SCR_MAP_KEY_F64, SCR_MAP_VAL_F64, NULL, NULL, NULL); + for (size_t i = 0; i < BIG_KEYS; i++) scr_map_set_f64_f64(map, (double)i, (double)i); + for (size_t i = 0; i + 10 < BIG_KEYS; i++) assert(scr_map_delete_f64(map, (double)i)); + for (size_t i = BIG_KEYS - 10; i < BIG_KEYS; i++) scr_map_set_f64_f64(fresh, (double)i, (double)i); + assert(scr_map_size(map) == 10); + size_t live = 10; + expect_proportional("iteration after deletes", fresh, map, iterate_round, &live); + assert(scr_map_iter_count(map) <= 4 * 10 + 64); + assert(!map->ndense); /* ten keys near 2^20: hashed */ + for (size_t i = BIG_KEYS - 10; i < BIG_KEYS; i++) assert(scr_map_has_f64(map, (double)i)); + assert(!scr_map_has_f64(map, 0) && !scr_map_has_f64(map, BIG_KEYS - 11)); + scr_map_release(fresh); + scr_map_release(map); + + /* Small survivors keep a direct index sized for them. */ + map = scr_map_new(SCR_MAP_KEY_F64, SCR_MAP_VAL_F64, NULL, NULL, NULL); + for (size_t i = 0; i < BIG_KEYS; i++) scr_map_set_f64_f64(map, (double)i, (double)i); + for (size_t i = BIG_KEYS; i-- > 20;) assert(scr_map_delete_f64(map, (double)i)); + assert(scr_map_size(map) == 20 && map->ndense && map->ndense <= 64); + for (size_t i = 0; i < 40; i++) assert(scr_map_has_f64(map, (double)i) == (i < 20)); + scr_map_release(map); + + /* A copy indexes its own keys, not the source's peak (here a quarter of + * the entries survive, too many to compact the source yet). */ + map = scr_map_new(SCR_MAP_KEY_F64, SCR_MAP_VAL_F64, NULL, NULL, NULL); + for (size_t i = 0; i < 4096; i++) scr_map_set_f64_f64(map, (double)i, (double)i); + for (size_t i = 1024; i < 4096; i++) assert(scr_map_delete_f64(map, (double)i)); + assert(map->ndense == 4096 && scr_map_iter_count(map) == 4096); + ScrMap *copy = scr_map_clone(map, false); + assert(copy->ndense == 1024 && scr_map_size(copy) == 1024); + double out; + for (size_t i = 0; i < 1100; i++) { + assert(scr_map_get_f64_f64(copy, (double)i, &out) == (i < 1024)); + if (i < 1024) assert(out == (double)i); + } + scr_map_release(copy); + scr_map_release(map); +} + +/* Grow-and-drain cycles against the ordered model: the map repeatedly + * grows large and is drained by random deletes, deletes in insertion + * order, clear, or deletes during an iteration that also appends, so + * compaction on delete, index shrinking, the move back to hashing and + * clear's storage release all run under the Map contract. */ +static double drain_key(unsigned flavor, size_t cycle, size_t i) { + switch (flavor) { + case 0: return (double)(next() % 4000); /* dense */ + case 1: return (double)(cycle * 3000 + i); /* a moving dense window */ + default: return (double)(next() % 64) * 1000003.0 + (double)(next() % 3); /* hashed */ + } +} + +static void fuzz_drain(unsigned flavor) { + static Model model; + model.n = 0; + ScrMap *map = scr_map_new(SCR_MAP_KEY_F64, SCR_MAP_VAL_F64, NULL, NULL, NULL); + for (size_t cycle = 0; cycle < 16; cycle++) { + size_t grow = 1 + next() % 3000; + for (size_t i = 0; i < grow; i++) { + double k = drain_key(flavor, cycle, i), v = (double)(next() % 1000); + scr_map_set_f64_f64(map, k, v); + model_set(&model, k, v); + } + check_same(map, &model); + size_t keep = next() % 40; + switch (next() % 4) { + case 0: + while (model.n > keep) { + double k = model.key[next() % model.n]; + assert(scr_map_delete_f64(map, k) && model_delete(&model, k)); + } + break; + case 1: + while (model.n > keep) { + double k = model.key[0]; + assert(scr_map_delete_f64(map, k) && model_delete(&model, k)); + } + break; + case 2: + scr_map_clear(map); + model.n = 0; + if (next() % 2) scr_map_clear(map); + break; + default: { + /* Delete the visited entry with probability 3/4 and sometimes append + * a fresh key, which the same iteration must still visit. */ + size_t cursor = 0, added = 0; + scr_map_iter_enter(map); + for (size_t i = 0; i < (size_t)scr_map_iter_count(map); i++) { + if (!scr_map_iter_live(map, (double)i)) continue; + double k = scr_map_iter_key_f64(map, (double)i); + assert(cursor < model.n && same_value_zero(k, model.key[cursor])); + if (added < 200 && next() % 8 == 0) { + double fresh = 1e7 + (double)(cycle * 1000 + added++); + scr_map_set_f64_f64(map, fresh, 1); + model_set(&model, fresh, 1); + } + if (next() % 4) { + assert(scr_map_delete_f64(map, k) && model_delete(&model, k)); + } else { + cursor++; + } + } + assert(cursor == model.n); + scr_map_iter_exit(map); + break; + } + } + check_same(map, &model); + for (size_t i = 0; i < 64; i++) probe(map, &model, drain_key(flavor, cycle, i)); + for (size_t i = 0; i < model.n; i++) probe(map, &model, model.key[i]); + } + scr_map_release(map); +} + int main(void) { modes(); iterate_while_mutating(); for (unsigned flavor = 0; flavor < 5; flavor++) fuzz(flavor); + clear_cost_numbers(0); + clear_cost_numbers(1); + clear_cost_strings(); + delete_shrinks(); + for (unsigned flavor = 0; flavor < 3; flavor++) fuzz_drain(flavor); puts("number-key maps passed"); return 0; } diff --git a/packages/runtime/src/scr_message.c b/packages/runtime/src/scr_message.c index 655d97219..5b27bfc4e 100644 --- a/packages/runtime/src/scr_message.c +++ b/packages/runtime/src/scr_message.c @@ -465,12 +465,14 @@ static ScrMessage *scr_message_try_publish(const ScrDyn *value) { #ifdef SCR_RC_AUDIT scr_str_live_forget(); #endif + scr_lsan_immortal(text); } else if (d->rc != SIZE_MAX) { scr_cyc_on_dead(d); /* leave the candidate and tenured buffers */ d->rc = SIZE_MAX; #ifdef SCR_RC_AUDIT scr_dyn_live_forget(); #endif + scr_lsan_immortal(d); } } message = scr_message_alloc(1, sizeof(*message)); diff --git a/packages/runtime/src/scr_publish.c b/packages/runtime/src/scr_publish.c index 819ba3f68..7ef5e50e9 100644 --- a/packages/runtime/src/scr_publish.c +++ b/packages/runtime/src/scr_publish.c @@ -282,6 +282,7 @@ void scr_publish(void *root, ScrPubFn fn) { #ifdef SCR_RC_AUDIT scr_pub_audit_forget(ctx.log[i].kind); #endif + scr_lsan_immortal(ctx.log[i].obj); } if (trace) { uint64_t t2 = scr_hrtime_ns(); diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index a83dffdae..4cff50b52 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -476,8 +476,9 @@ void scr_cyc_free(void *obj); /* frees the block, header included */ * of SCR_SA_ARENAS equal slices of a single process-wide reservation, so * the owning arena of any block is address arithmetic. A thread claims an * arena on its first allocation and parks it (free lists and bump pointers - * intact, memory never unmapped) when it exits; the next thread to claim - * it continues from there. Script objects stay on their thread, so a free + * intact) when it exits; the next thread to claim it continues from there. + * On Linux an exiting thread that carved much first returns the pages of + * its arena that hold only free blocks (scr_alloc.c). Script objects stay on their thread, so a free * normally hits the caller's own arena inline. A block freed by another * thread goes to the owning arena's lock-free remote list, which the owner * drains on its slow path; published (immortal) objects are never freed, @@ -577,6 +578,8 @@ _Static_assert(offsetof(ScrSaState, span) == 8 && offsetof(ScrSaState, shift) == void scr_sa_free_foreign(void *p); /* The calling script thread is done with its arena (worker exit). */ void scr_sa_thread_park(void); +/* Waits until the arenas of exited threads are trimmed (tests). */ +void scr_sa_wait_trims(void); #define SCR_SA_FREE_FOREIGN(p) scr_sa_free_foreign(p) #else #define SCR_SA_FREE_FOREIGN(p) free(p) @@ -2094,6 +2097,26 @@ void scr_map_live_forget(void); void scr_union_live_forget(void); #endif +/* For the same reason LeakSanitizer must not report a published object + * whose last reference is dropped (scr_message.c, scr_publish.c). Ignored + * chunks are leak-scan roots, so everything an immortal object reaches is + * covered and the verdict on the rest of the heap is unchanged. Weak: ASan + * runtimes without a leak checker leave it NULL; other builds compile the + * hook away. */ +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define SCR_LSAN_IMMORTAL 1 +void __lsan_ignore_object(const void *p) __attribute__((weak)); +#endif +#endif +static inline void scr_lsan_immortal(const void *obj) { +#ifdef SCR_LSAN_IMMORTAL + if (__lsan_ignore_object) __lsan_ignore_object(obj); +#else + (void)obj; +#endif +} + /* ── closures ─────────────────────────────────────────────────────── * A closure is a function pointer plus the boxes of its captured bindings. * A box is one shared, refcounted variable cell: every function that diff --git a/scripts/sandbox-bootstrap.sh b/scripts/sandbox-bootstrap.sh index 54c06c8b1..ac0a4558a 100644 --- a/scripts/sandbox-bootstrap.sh +++ b/scripts/sandbox-bootstrap.sh @@ -30,6 +30,7 @@ sudo env DEBIAN_FRONTEND=noninteractive apt-get install --quiet=2 --yes --no-ins clang-22 \ cmake \ libclang-rt-22-dev \ + libpolly-22-dev \ libzstd-dev \ llvm-22-dev \ ninja-build \ diff --git a/tests/corpus/4598-map-set-clear-reuse.ts b/tests/corpus/4598-map-set-clear-reuse.ts new file mode 100644 index 000000000..0a4095139 --- /dev/null +++ b/tests/corpus/4598-map-set-clear-reuse.ts @@ -0,0 +1,147 @@ +// Map and Set clear(), reuse and iteration after clear and after mass +// deletion, for number, string and object keys. The sizes cross the +// runtime's storage thresholds: maps grow far beyond their later size, so +// clear() and delete release or shrink their index tables, number keys move +// between the direct index and hashing, and every later operation must +// still follow insertion order exactly. + +class Point { + readonly x: number; + readonly y: number; + constructor(x: number, y: number) { + this.x = x; + this.y = y; + } +} + +function summarize(label: string, all: K[], show: (k: K) => string): void { + const head = all.slice(0, 4).map(show).join(","); + const tail = all.slice(-3).map(show).join(","); + console.log(`${label}: size=${all.length} head=[${head}] tail=[${tail}]`); +} + +// ── number keys: dense table grown large, then cleared and reused ────────── +const nums = new Map(); +for (let i = 0; i < 70000; i++) nums.set(i, i * 2); +console.log("nums grown", nums.size, nums.get(69999), nums.has(70000)); +nums.clear(); +console.log("nums cleared", nums.size, nums.has(0), nums.get(5), [...nums.keys()].length); +nums.clear(); // empty clear +nums.clear(); +for (const k of [5, 3, 9, 3, 1000000, 0.5, -0, 7]) nums.set(k, k + 1); +summarize("nums reuse", [...nums.keys()], (k) => String(Object.is(k, -0) ? "-0" : k)); +console.log("nums -0 stored as", Object.is([...nums.keys()][5], -0) ? "-0" : "+0"); +console.log("nums values", [...nums.values()].join(",")); +// many small fill/clear rounds after the large one +let roundSum = 0; +for (let round = 0; round < 2000; round++) { + nums.clear(); + for (let j = 0; j < round % 9; j++) nums.set(round * 10 + j, j); + for (const [k, v] of nums) roundSum += k * 3 + v; + roundSum += nums.size; +} +console.log("nums rounds", roundSum, nums.size); + +// ── number keys: mass deletion, then iteration and lookups ──────────────── +const ids = new Set(); +for (let i = 0; i < 50000; i++) ids.add(i); +for (let i = 0; i < 49990; i++) ids.delete(i); // survivors are far from zero +summarize("ids after deletes", [...ids], (k) => String(k)); +console.log("ids lookups", ids.has(49989), ids.has(49990), ids.has(49999), ids.has(0)); +ids.add(3); +ids.add(49989); +summarize("ids re-added", [...ids], (k) => String(k)); +for (let i = 49999; i >= 20; i--) ids.delete(i); // now small keys only +summarize("ids small", [...ids], (k) => String(k)); +for (let i = 0; i < 100; i++) ids.add(i); +summarize("ids regrown", [...ids], (k) => String(k)); + +// ── stack-like add/delete like a recursion guard, cleared on every reset ── +const guard = new Set(); +let maybe = 0; +function visit(depth: number, id: number): void { + if (guard.has(id)) { + maybe++; + return; + } + guard.add(id); + if (depth > 0) { + visit(depth - 1, id * 3 + 1); + visit(depth - 1, (id * 7) % 1000); + } + guard.delete(id); +} +for (let check = 0; check < 300; check++) { + guard.clear(); + visit(check % 50 === 0 ? 12 : 3, check * 100003); +} +console.log("guard", maybe, guard.size, [...guard].length); + +// ── string keys ────────────────────────────────────────────────────────── +const words = new Map(); +for (let i = 0; i < 20000; i++) words.set(`w${i}`, i); +words.clear(); +console.log("words cleared", words.size, words.has("w1"), words.get("w2")); +for (const w of ["delta", "alpha", "charlie", "alpha", "bravo"]) words.set(w, (words.get(w) ?? 0) + 1); +summarize("words reuse", [...words.entries()], ([k, v]) => `${k}=${v}`); +for (let i = 0; i < 5000; i++) words.set(`x${i}`, i); +for (let i = 0; i < 4990; i++) words.delete(`x${i}`); +summarize("words after deletes", [...words.keys()], (k) => k); +const tags = new Set(["a", "b", "c"]); +tags.clear(); +tags.add("z"); +tags.add("a"); +console.log("tags", [...tags].join(","), tags.has("b")); + +// ── object keys ────────────────────────────────────────────────────────── +const points: Point[] = []; +for (let i = 0; i < 3000; i++) points.push(new Point(i, -i)); +const owners = new Map(); +for (const p of points) owners.set(p, `p${p.x}`); +owners.clear(); +console.log("owners cleared", owners.size, owners.has(points[0]!)); +owners.set(points[10]!, "ten"); +owners.set(points[2]!, "two"); +owners.set(points[10]!, "TEN"); +summarize("owners reuse", [...owners.entries()], ([p, s]) => `${p.x}:${s}`); +const seen = new Set(points); +for (let i = 0; i < 2995; i++) seen.delete(points[i]!); +summarize("seen after deletes", [...seen], (p) => `${p.x}/${p.y}`); + +// ── clear and delete during iteration (Node's live iteration) ──────────── +const live = new Map(); +for (let i = 0; i < 400; i++) live.set(i, `v${i}`); +const visited: number[] = []; +for (const [k] of live) { + visited.push(k); + if (k === 2) { + live.clear(); + live.set(1000, "after"); + live.set(1001, "after"); + } + if (k === 1000) live.set(1002, "late"); +} +console.log("live visited", visited.join(","), "size", live.size); + +const churn = new Set(); +for (let i = 0; i < 300; i++) churn.add(i); +let walked = 0; +churn.forEach((k) => { + walked++; + churn.delete(k); + if (k < 100) churn.add(k + 1000); +}); +summarize("churn after forEach", [...churn], (k) => String(k)); +console.log("churn walked", walked); + +const strs = new Map(); +for (let i = 0; i < 500; i++) strs.set(`s${i}`, i); +const order: string[] = []; +strs.forEach((_v, k) => { + order.push(k); + if (k === "s1") { + strs.clear(); + strs.set("fresh", 1); + } +}); +console.log("strs visited", order.join(","), [...strs.keys()].join(",")); diff --git a/tests/corpus/native-worker-arena-reuse.ts b/tests/corpus/native-worker-arena-reuse.ts new file mode 100644 index 000000000..7a4f22e8f --- /dev/null +++ b/tests/corpus/native-worker-arena-reuse.ts @@ -0,0 +1,47 @@ +import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads"; + +// Workers that hold and then drop tens of megabytes of small objects, and later workers that +// allocate as much again once the first ones have exited (in scriptc on Linux, the first ones +// return those pages when they exit and the later ones reuse them). +type Item = { value: number; next: Item | undefined; tags: number[]; label: string }; + +if (isMainThread) { + for (let wave = 0; wave < 2; wave++) { + const results = await Promise.all( + [1, 2, 3].map( + (k) => + new Promise((resolve, reject) => { + let result = -1; + const worker = new Worker(new URL(import.meta.url), { workerData: wave * 10 + k }); + worker.once("message", (value: unknown) => { + result = value as number; + }); + worker.once("error", reject); + worker.once("exit", () => resolve(result)); + }), + ), + ); + console.log("wave", wave, results.join(" ")); + } +} else { + const round = workerData as number; + let items: Item[] = []; + for (let i = 0; i < 300_000; i++) + items.push({ value: i * round, next: undefined, tags: [i & 7, round, i % 13], label: "item" + (i % 97) }); + // A few items survive the rest, linked together. + const kept: Item[] = []; + for (let i = 0; i < items.length; i += 4096) kept.push(items[i]!); + for (let i = 1; i < kept.length; i++) kept[i - 1]!.next = kept[i]; + let sum = 0; + for (const item of items) sum = (sum + item.value + item.tags[2]! + item.label.length) % 1_000_003; + items = []; + let walk = 0; + for (let item: Item | undefined = kept[0]; item !== undefined; item = item.next) + walk = (walk + item.tags[0]! + item.tags[1]! + item.label.length) % 1_000_003; + // Fresh objects after the drop. + const fresh: Item[] = []; + for (let i = 0; i < 100_000; i++) fresh.push({ value: i, next: undefined, tags: [], label: "" }); + let empty = 0; + for (const item of fresh) if (item.next === undefined && item.tags.length === 0 && item.label === "") empty++; + parentPort!.postMessage((sum + walk + empty) % 1_000_003); +} diff --git a/tests/fixtures/publish-leaks/dropped.ts b/tests/fixtures/publish-leaks/dropped.ts new file mode 100644 index 000000000..51178b8ed --- /dev/null +++ b/tests/fixtures/publish-leaks/dropped.ts @@ -0,0 +1,41 @@ +import { publish } from "@scriptc/threads"; + +// Published objects are immortal: retains and releases skip them and +// nothing frees them, so dropping the last reference to one is not a leak. +// Every graph below becomes unreachable once its function returns. + +class Item { + value: number; + label: string; + next: Item | null; + self: Item | undefined; + constructor(value: number, label: string, next: Item | null) { + this.value = value; + this.label = label; + this.next = next; + } +} + +// `this` is published before its constructor returns. +class SelfPublishing { + a: number; + constructor(a: number) { + this.a = a; + publish(this); + } +} + +function dropTyped(): number { + let head: Item | null = null; + for (let i = 0; i < 100; i++) head = new Item(i, `item-${i}`, head); + head!.self = head!; + publish(head); + const labels = publish([`first-${head!.value}`, `second-${head!.next!.value}`]); + const index = publish(new Map([[`key-${head!.value}`, head!]])); + const own = new SelfPublishing(labels.length); + return head!.value + labels.length + index.size + own.a; +} + +let total = 0; +for (let round = 0; round < 3; round++) total += dropTyped(); +console.log(total); diff --git a/tests/fixtures/self-hosting/serialize.ts b/tests/fixtures/self-hosting/serialize.ts index 0146da1a4..4654ea415 100644 --- a/tests/fixtures/self-hosting/serialize.ts +++ b/tests/fixtures/self-hosting/serialize.ts @@ -1,9 +1,19 @@ import { readFileSync } from "node:fs"; -import { deserializeModule, serializeModule } from "../../../packages/compiler/src/ir/serialize.js"; +import { + deserializeModule, + deserializeModuleBytes, + serializeModule, + serializeModuleParts, +} from "../../../packages/compiler/src/ir/serialize.js"; import { validateModule } from "../../../packages/compiler/src/ir/validate.js"; try { - const mod = deserializeModule(readFileSync(process.argv[2]!, "utf8")); + // "bytes" exercises the compiler-scale path: per-function decoding and + // compact parts, as the native compiler uses for its own IR. + const bytes = process.argv[3] === "bytes"; + const mod = bytes + ? deserializeModuleBytes(readFileSync(process.argv[2]!)) + : deserializeModule(readFileSync(process.argv[2]!, "utf8")); const errors = validateModule(mod); if (errors.length > 0) { console.log(JSON.stringify(errors)); @@ -15,7 +25,10 @@ try { const statement = mod.functions[0]!.body[0]; if (statement?.kind === "exprStmt" && statement.expr.kind === "numLit") statement.expr.value = NaN; } - console.log(serializeModule(mod, process.argv[3] === "compact")); + if (bytes) { + for (const part of serializeModuleParts(mod)) process.stdout.write(part); + process.stdout.write("\n"); + } else console.log(serializeModule(mod, process.argv[3] === "compact")); } } catch (error) { if (error instanceof Error) console.log(error.name + ": " + error.message); diff --git a/tests/harness/large-artifacts.test.ts b/tests/harness/large-artifacts.test.ts new file mode 100644 index 000000000..207ce80a2 --- /dev/null +++ b/tests/harness/large-artifacts.test.ts @@ -0,0 +1,30 @@ +import { expect, test } from "vitest"; +import { occurrences, withoutFirstLine, withoutRange } from "./large-artifacts.js"; + +test("byte-level artifact edits match their string equivalents", () => { + const text = + "x declare void @scr_runtime_abi_v3()\ndeclare void @scr_runtime_abi_vX()\n" + + "declare void @scr_runtime_abi_v12()\n call void @scr_runtime_abi_v12()\n" + + "Ć©\n call void @scr_runtime_abi_v12()\n"; + const bytes = Buffer.from(text); + const declaration = /^declare void @scr_runtime_abi_v\d+\(\)\n/m; + const call = /^ {2}call void @scr_runtime_abi_v\d+\(\)\n/m; + const edited = withoutFirstLine( + withoutFirstLine( + bytes, + "declare void @scr_runtime_abi_v", + /^declare void @scr_runtime_abi_v\d+\(\)\n$/, + ), + " call void @scr_runtime_abi_v", + /^ {2}call void @scr_runtime_abi_v\d+\(\)\n$/, + ); + expect(edited.toString()).toBe(text.replace(declaration, "").replace(call, "")); + expect(withoutFirstLine(bytes, "missing", /^missing\n$/)).toBe(bytes); + + const line = " call void @scr_runtime_abi_v12()\n"; + const found = occurrences(bytes, line); + expect(found).toHaveLength(text.split(line).length - 1); + expect(withoutRange(bytes, found[0]!, Buffer.byteLength(line)).toString()).toBe( + text.replace(line, ""), + ); +}); diff --git a/tests/harness/large-artifacts.ts b/tests/harness/large-artifacts.ts new file mode 100644 index 000000000..c21dc11ae --- /dev/null +++ b/tests/harness/large-artifacts.ts @@ -0,0 +1,31 @@ +/** Byte-level helpers for compiler-scale text artifacts (the self-hosted + * compiler's LLVM and IR). They avoid decoding a complete artifact into one + * JavaScript string, which V8 caps at 2^29 - 24 UTF-16 units. */ + +/** Offsets of every occurrence of `needle`. */ +export function occurrences(bytes: Buffer, needle: string): number[] { + const found: number[] = []; + const width = Buffer.byteLength(needle); + for (let at = bytes.indexOf(needle); at >= 0; at = bytes.indexOf(needle, at + width)) + found.push(at); + return found; +} + +/** `bytes` without `length` bytes at `start`. */ +export function withoutRange(bytes: Buffer, start: number, length: number): Buffer { + return Buffer.concat([bytes.subarray(0, start), bytes.subarray(start + length)]); +} + +/** Remove the first complete line that starts with `prefix` and, including + * its newline, matches `line` (anchored), like `text.replace(/^...\n/m, "")`. */ +export function withoutFirstLine(bytes: Buffer, prefix: string, line: RegExp): Buffer { + for (let from = 0; ;) { + const at = bytes.indexOf(prefix, from); + if (at < 0) return bytes; + const newline = bytes.indexOf(10, at); + const stop = newline < 0 ? bytes.length : newline + 1; + if ((at === 0 || bytes[at - 1] === 10) && line.test(bytes.toString("utf8", at, stop))) + return withoutRange(bytes, at, stop - at); + from = at + 1; + } +} diff --git a/tests/harness/publish-leaks.test.ts b/tests/harness/publish-leaks.test.ts new file mode 100644 index 000000000..4ebfbfd28 --- /dev/null +++ b/tests/harness/publish-leaks.test.ts @@ -0,0 +1,68 @@ +/* Published objects (@scriptc/threads publish() and the zero-copy delivery + * of deeply frozen plain-data graphs) are immortal by design: nothing frees + * them, and the RC audit stops counting them. LeakSanitizer must agree, so + * the runtime marks every published object as an intentional leak-scan + * root. The sanitized lanes otherwise run with detect_leaks=0, so this pins + * the contract with LeakSanitizer explicitly on: programs that drop their + * last reference to a published graph exit cleanly. Linux only (Apple's + * ASan has no LeakSanitizer); runs in the SCRIPTC_SAN=1 lane. */ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +const programs = [ + "tests/fixtures/publish-leaks/dropped.ts", + // Zero-copy delivery of deeply frozen plain-data graphs (scr_message.c). + "tests/corpus/native-worker-published-graphs.ts", + // Typed graphs published on both threads, including a refused publish. + "tests/corpus/native-worker-publish-classes.ts", +]; + +describe.skipIf(!sanitize || process.platform !== "linux")( + "published objects under LeakSanitizer", + () => { + test.each(programs)("%s reports no leaks", { timeout: 300_000 }, async (program) => { + const file = join(repoRoot, program); + const key = createHash("sha256") + .update(file) + .update(readFileSync(file)) + .update("san") + .digest("hex") + .slice(0, 16); + const outDir = join(cacheDir, `publish-leaks-${key}`); + mkdirSync(outDir, { recursive: true }); + const result = await compile(file, { + outPath: join(outDir, "program"), + outDir, + sanitize: true, + backend: "llvm", + }); + if (!result.ok) { + throw new Error( + "program failed to compile:\n" + + result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"), + ); + } + const run = await execFileAsync(result.binaryPath, [], { + env: { ...process.env, ASAN_OPTIONS: "detect_leaks=1" }, + maxBuffer: 16 * 1024 * 1024, + }).catch((error: { code?: number; stdout?: string; stderr?: string }) => error); + expect(run.stderr).toBe(""); + expect("code" in run ? run.code : 0).toBe(0); + const expected = await execFileAsync(process.execPath, [file], { + env: { ...process.env, SCRIPTC_TEST_ENV: "from-harness" }, + maxBuffer: 16 * 1024 * 1024, + }); + expect(run.stdout).toBe(expected.stdout); + }); + }, +); diff --git a/tests/harness/self-hosting-native-driver.test.ts b/tests/harness/self-hosting-native-driver.test.ts index 87ba5a4c8..b57d0b6cf 100644 --- a/tests/harness/self-hosting-native-driver.test.ts +++ b/tests/harness/self-hosting-native-driver.test.ts @@ -24,6 +24,7 @@ import { nativeBootstrapPlan, runNativeBootstrapChecks, } from "../../scripts/ci-native-bootstrap.mjs"; +import { withoutFirstLine } from "./large-artifacts.js"; import { bootstrapStep } from "./self-hosting-timing.js"; const root = join(import.meta.dirname, "../.."); @@ -435,17 +436,20 @@ test( import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { isDeepStrictEqual } from 'node:util'; - import { deserializeModule, validateModule } from ${JSON.stringify(pathToFileURL(join(root, "packages/compiler/src/index.ts")).href)}; + import { deserializeModuleBytes, validateModule } from ${JSON.stringify(pathToFileURL(join(root, "packages/compiler/src/index.ts")).href)}; + import { occurrences, withoutRange } from ${JSON.stringify(pathToFileURL(join(root, "tests/harness/large-artifacts.ts")).href)}; + // Both artifacts are compared as bytes and per-function documents: + // either can exceed V8's maximum string length. if (${ir}) { - const expected = deserializeModule(readFileSync(process.argv[1], 'utf8')); - const actual = deserializeModule(readFileSync(process.argv[2], 'utf8')); + const expected = deserializeModuleBytes(readFileSync(process.argv[1])); + const actual = deserializeModuleBytes(readFileSync(process.argv[2])); assert.ok(actual.functions.length > 1000); assert.deepEqual(validateModule(actual), []); assert.ok(isDeepStrictEqual(actual, expected), 'native self-lowering must match the Node seed'); } if (${llvm}) { - const seedLlvm = readFileSync(process.argv[3], 'utf8'); - let nativeLlvm = readFileSync(process.argv[4], 'utf8'); + const seedLlvm = readFileSync(process.argv[3]); + let nativeLlvm = readFileSync(process.argv[4]); if (${sanitize}) { // The Node sanitizer builds runtime sources directly; the native // executable path also emits the runtime ABI check. Pin that @@ -453,12 +457,13 @@ test( const declaration = 'declare void @' + ${JSON.stringify(RUNTIME_ABI_MARKER)} + '()\\n'; const call = ' call void @' + ${JSON.stringify(RUNTIME_ABI_MARKER)} + '()\\n'; for (const line of [declaration, call]) { - assert.equal(seedLlvm.split(line).length, 1); - assert.equal(nativeLlvm.split(line).length, 2); - nativeLlvm = nativeLlvm.replace(line, ''); + assert.equal(occurrences(seedLlvm, line).length, 0); + const found = occurrences(nativeLlvm, line); + assert.equal(found.length, 1); + nativeLlvm = withoutRange(nativeLlvm, found[0], Buffer.byteLength(line)); } } - assert.ok(seedLlvm === nativeLlvm, 'native LLVM emission must match the Node seed'); + assert.ok(seedLlvm.equals(nativeLlvm), 'native LLVM emission must match the Node seed'); } `, seedIr, @@ -506,12 +511,18 @@ test( ); // Executable output adds a runtime ABI check; textual LLVM emission // omits it. Compare all other output without changing either mode. - const withoutAbiCheck = (path: string): string => - readFileSync(path, "utf8") - .replace(/^declare void @scr_runtime_abi_v\d+\(\)\n/m, "") - .replace(/^ call void @scr_runtime_abi_v\d+\(\)\n/m, ""); + const withoutAbiCheck = (path: string): Buffer => + withoutFirstLine( + withoutFirstLine( + readFileSync(path), + "declare void @scr_runtime_abi_v", + /^declare void @scr_runtime_abi_v\d+\(\)\n$/, + ), + " call void @scr_runtime_abi_v", + /^ call void @scr_runtime_abi_v\d+\(\)\n$/, + ); expect( - withoutAbiCheck(seedLlvm) === withoutAbiCheck(rebuiltLlvm), + withoutAbiCheck(seedLlvm).equals(withoutAbiCheck(rebuiltLlvm)), "native compiler generations must emit identical LLVM", ).toBe(true); }; diff --git a/tests/harness/self-hosting-serialization.test.ts b/tests/harness/self-hosting-serialization.test.ts index 3051c86f1..4139cd648 100644 --- a/tests/harness/self-hosting-serialization.test.ts +++ b/tests/harness/self-hosting-serialization.test.ts @@ -178,6 +178,26 @@ for (const backend of ["llvm"] as const) { expect(deserializeModule(many.native.stdout.toString())).toEqual( deserializeModule(many.oracle.stdout.toString()), ); + // Byte decoding and compact parts, from both encodings. + for (const input of [serializeModule(manyFunctions), serializeModule(manyFunctions, true)]) { + const parts = run(input, ["bytes"]); + expect(parts.native.status, parts.native.stdout.toString()).toBe(0); + expect(parts.native.stdout.toString().trimEnd()).not.toContain("\n"); + expect(deserializeModule(parts.native.stdout.toString())).toEqual(manyFunctions); + expect(deserializeModule(parts.native.stdout.toString())).toEqual( + deserializeModule(parts.oracle.stdout.toString()), + ); + } + const truncated = run(serializeModule(manyFunctions, true).slice(0, -1), ["bytes"]); + expect(truncated.native.status).toBe(1); + expect(truncated.oracle.status).toBe(1); + const olderBytes = run( + serializeModule({ ...manyFunctions, irVersion: IR_VERSION - 1 }, true), + ["bytes"], + ); + expect(olderBytes.native.status).toBe(1); + expect(olderBytes.native.stdout).toEqual(olderBytes.oracle.stdout); + expect(olderBytes.native.stdout.toString()).toContain("IR version mismatch"); const print = numeric.functions[0]!.body[1]!; expect(print.kind).toBe("exprStmt"); if (print.kind !== "exprStmt" || print.expr.kind !== "intrinsic")