diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16e5e78dd..23936e46a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,11 @@ on: type: choice options: [full, darwin] default: full + scope: + description: Every lane, or only the shape overlay validation (shape_overlay) + type: choice + options: [all, shape_overlay] + default: all concurrency: # A new push to a pull request cancels its previous run. Every push to main runs to completion in a group of @@ -59,7 +64,10 @@ jobs: sameRepository && trustedAuthor && typeof stackBase === 'string' && stackBase.length > 0 && typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase && prBase !== defaultBranch; - const full = !upperLayer; + // A manual run of the shape overlay validation alone skips the other lanes. + const shapeOnly = process.env.GITHUB_EVENT_NAME === 'workflow_dispatch' && + event.inputs?.scope === 'shape_overlay'; + const full = !upperLayer && !shapeOnly; appendFileSync(process.env.GITHUB_OUTPUT, `full=${full}\n`); console.log(full ? 'Run full CI' : 'Defer full CI until this PR becomes the lowest unmerged layer'); NODE @@ -390,6 +398,81 @@ jobs: - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" - run: pnpm test tests/harness/three.test.ts -t "'nodes' runs statically through LLVM on 'native'" + # Shape inference's overlay (opt-in, SCRIPTC_SHAPE_OVERLAY=1) over the + # suites of real packages: nightly and on manual runs only (scope + # shape_overlay runs it alone), outside the `test` roll-up. Every case + # must build and run as in the default build. It builds its own native + # packages, so it does not depend on the other lanes. + shape_overlay_native: + if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + name: build (Linux x64 native packages, shape overlay) + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - uses: ./.github/actions/llvm-linux + - run: pnpm install --frozen-lockfile + - uses: ./.github/actions/zig + - uses: ./.github/actions/native-packages + with: + target: linux-x64-gnu + - name: Preserve executable permissions in shared packages + run: >- + tar -czf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" + packages/llvm-linux-x64-gnu/bin + packages/runtime-linux-x64-gnu/artifacts + packages/runtime-linux-x64-gnu/runtime-pack.json + - uses: actions/upload-artifact@v4 + with: + name: native-linux-x64-gnu-shape-overlay + path: ${{ runner.temp }}/native-linux-x64-gnu.tar.gz + retention-days: 1 + + shape_overlay: + needs: [shape_overlay_native] + if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + name: test (Linux, shape overlay, ${{ matrix.suite }} ${{ matrix.shard }}/8) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + suite: [npm, npm-static, three, effect] + shard: [1, 2, 3, 4, 5, 6, 7, 8] + env: + SCRIPTC_SHAPE_OVERLAY: "1" + SCRIPTC_TEST_WORKERS: "1" + SCRIPTC_EFFECT_TEST_CONCURRENCY: "1" + SCRIPTC_TEST_SHARD: ${{ matrix.shard }}/8 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - uses: actions/download-artifact@v4 + with: + name: native-linux-x64-gnu-shape-overlay + path: ${{ runner.temp }} + - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" + - name: Run the suite with the overlay on + run: | + case "${{ matrix.suite }}" in + npm) pnpm test tests/harness/npm-shape-overlay.test.ts ;; + npm-static) pnpm test tests/harness/npm-static.test.ts ;; + three) pnpm test tests/harness/three.test.ts ;; + effect) pnpm test tests/harness/effect.test.ts tests/harness/effect4.test.ts ;; + esac + # Build each instrumented seed once. Downstream contracts retain the same # compiler and source artifacts without repeating seed construction. bootstrap_seed: diff --git a/.gitignore b/.gitignore index abb3f2558..04c889158 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ node_modules/ +# A symlinked node_modules is a file to git, which node_modules/ does not match. +/packages/*/node_modules .pnpm-store/ # test-fixture packages are TEST DATA, committed on purpose !tests/diagnostics/npm-imports/node_modules/ diff --git a/RELEASING.md b/RELEASING.md index a32971f46..e3e77d512 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -10,7 +10,8 @@ To prepare a release: 4. Wrap the new entry in `` and `` markers; this marked block is also the GitHub release body 5. Remove the `` and `` markers from the previous release entry; only the latest release should have markers 6. With Zig on `PATH`, run `SCRIPTC_CROSS=1 pnpm exec vitest run tests/harness/library-cross.test.ts` and require the cross-target library conformance lane to pass -7. Commit to `main` +7. Refresh the runtime profile on a quiet Linux x64 host: `node scripts/refresh-runtime-profile.mjs --tsc-ts --project :tsconfig.bench.json --project :tsconfig.bench.json --project /src:tsconfig.bench.json`, and include the new `packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz` (see the runtime profile paragraph below) +8. Commit to `main` CI (`.github/workflows/release.yml`) compares the version in `packages/cli/package.json` to what `scriptc` has on npm. If it differs, it builds platform packages on matching macOS, Linux, and Windows runners, verifies the version spine, and publishes the runtime, helper, compiler, and CLI packages in dependency order. After the publish succeeds, a separate job creates the git tag `v` and the GitHub release with the marked changelog entry as its body, and attaches `surface-manifest.json` — the machine-readable listing of the surface the static tier compiles at that version (stable per-entry ids, so two releases diff mechanically; see `packages/compiler/src/coverage/surface-manifest.ts` for the schema). The job regenerates the manifest from the tree and fails on any byte difference from the committed file before attaching, so the asset is always the manifest of the code being released. The same file ships inside the `@scriptc/compiler` package as `@scriptc/compiler/surface-manifest.json`. @@ -20,6 +21,8 @@ Release builds of the native CLI optimize the compiler with a profile collected Builds of the native compiler also order its functions by the committed layout `packages/compiler/native/cli.layout.gz`: the functions that `--version` and rebuilds of unchanged programs run come first, then the ones other builds run, ordered by the share of the layout's training builds that run each, so a run maps fewer pages of the executable. Functions the layout does not list keep their usual order. Function names of closures and generated helpers change with nearby edits, so the `cli-profile` job also collects a layout on the release commit (`scripts/refresh-cli-layout.mjs`), and the CLI builds use it (`SCRIPTC_NATIVE_LAYOUT`). Instrumented builds use the same layout as optimized builds, because a function laid out in a different partition than in the instrumented build no longer matches its profile record. The job therefore collects the layout first and then builds the instrumented compiler for the profile with that layout. Refresh the committed layout the same way, with `node scripts/refresh-cli-layout.mjs`, before the profile, and commit each on its own. +The Linux x64 runtime pack optimizes its speed-flavor units with the committed runtime profile `packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz`. Runtime functions whose code changed since the profile was collected are optimized as without a profile. The pack manifest records how the profile matched (`flavors.speed.runtime_profile`): the matched and changed functions, and the share of the profile's counts on changed functions. `packages/runtime-pack-common/test/runtime-profile.test.ts` fails when changed functions hold more than 5% of the counts, and names the functions with the most counts. Refresh the profile in each release commit (step 7), and between releases when that test fails. + Publishing uses npm trusted publishing (OIDC) — there is no npm token secret. Each published package must have a GitHub Actions trusted publisher for `release.yml` and the `Release` environment. A missing configuration fails diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index f3a444ea9..149c348d0 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -21,6 +21,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | | `json-unions` | cpu | `JSON.parse` into discriminated unions: an event log with eight kinds and nested expression trees with nine, evaluated after the cast | | `keyed-option-writes` | cpu | class methods whose names are also keys of option records written through element accesses: `enabled[name] = true` over a union of literal names and `totals["count"] = n` | +| `layered-errors` | cpu | validation errors thrown below a stack of small functions that hold strings and arrays, and caught per entry in a loop: about half of 640,000 entries fail | | `line-reader` | cpu | `readline` over a file stream: 'line' events for a 67 MB access log (every 50th line not ASCII), with a status count, a path tally and byte totals | | `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe | | `log-summary`, `inventory-report` | cpu | the multi-module applications from `benchmarks/builds`, on 300k-line inputs | diff --git a/benchmarks/runtime/workloads/layered-errors.json b/benchmarks/runtime/workloads/layered-errors.json new file mode 100644 index 000000000..70971552c --- /dev/null +++ b/benchmarks/runtime/workloads/layered-errors.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/layered-errors.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/layered-errors.ts b/benchmarks/runtime/workloads/layered-errors.ts new file mode 100644 index 000000000..bf2ff581c --- /dev/null +++ b/benchmarks/runtime/workloads/layered-errors.ts @@ -0,0 +1,95 @@ +// A configuration loader whose validation errors are thrown at the bottom +// of a stack of small layers and caught per entry at the top: each layer +// holds strings and arrays while the error passes through it. +class ConfigError extends Error { + readonly key: string; + constructor(key: string, message: string) { + super(message); + this.key = key; + } +} + +let seed = 4242; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} + +function fail(key: string, text: string): never { + throw new ConfigError(key, `invalid value "${text}" for ${key}`); +} +function digits(key: string, text: string): number { + let value = 0; + for (let i = 0; i < text.length; i++) { + const c = text.charCodeAt(i); + if (c < 48 || c > 57) fail(key, text); + value = value * 10 + (c - 48); + } + return value; +} +function parseNumber(key: string, text: string): number { + return text.length === 0 ? fail(key, text) : digits(key, text); +} +function checkRange(key: string, text: string, max: number): number { + const value = parseNumber(key, text); + return value > max ? fail(key, text) : value; +} +function coerce(key: string, raw: string, max: number): number { + const trimmed = raw.trim(); + const parts = [key, trimmed]; + const value = checkRange(key, trimmed, max); + return value + parts.length; +} +function resolveValue(key: string, raw: string, defaults: Map): number { + const text = raw === "default" ? (defaults.get(key) ?? "") : raw; + const label = "value:" + key; + return coerce(key, text, 100000) + label.length; +} +function loadEntry(section: string, entry: string, defaults: Map): number { + const eq = entry.indexOf("="); + const key = section + "." + entry.slice(0, eq); + const path = [section, key]; + return resolveValue(key, entry.slice(eq + 1), defaults) + path.length; +} +function loadEntryWithContext(section: string, entry: string, defaults: Map): number { + const context = `${section}/${entry.length}`; + return loadEntry(section, entry, defaults) + context.length; +} + +function makeEntry(i: number): string { + const r = random(); + const name = "k" + (i % 97); + if (r < 0.3) return `${name}=x${i}`; + if (r < 0.4) return `${name}=`; + if (r < 0.5) return `${name}=${200000 + i}`; + if (r < 0.55) return `${name}=default`; + return `${name}= ${Math.floor(random() * 90000)} `; +} + +const rounds = Number(process.argv[2] ?? "1"); +const defaults = new Map(); +for (let i = 0; i < 97; i += 2) defaults.set(`net.k${i}`, String(i * 10)); +const sections = ["net", "db", "cache", "auth"]; +const entries: string[] = []; +for (let i = 0; i < 4000; i++) entries.push(makeEntry(i)); + +let loaded = 0; +let sum = 0; +let failed = 0; +const failedKeys = new Map(); +for (let round = 0; round < rounds * 40; round++) { + for (const section of sections) { + for (const entry of entries) { + try { + sum += loadEntryWithContext(section, entry, defaults); + loaded++; + } catch (error) { + if (!(error instanceof ConfigError)) throw error; + failed++; + failedKeys.set(error.key, (failedKeys.get(error.key) ?? 0) + 1); + } + } + } +} +console.log(`loaded ${loaded} failed ${failed} sum ${sum} distinct failing keys ${failedKeys.size}`); +console.log(`net.k0 failures ${failedKeys.get("net.k0") ?? 0}`); diff --git a/docs/content/docs/cli.mdx b/docs/content/docs/cli.mdx index f2d91f1bd..9efd25342 100644 --- a/docs/content/docs/cli.mdx +++ b/docs/content/docs/cli.mdx @@ -176,7 +176,7 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
On Linux, build an instrumented executable for profile-guided optimization. Each run writes a raw profile to the path pattern, where %p expands to the process ID; the LLVM_PROFILE_FILE environment variable overrides the pattern at run time. A relative pattern resolves from the current working directory. Instrumented programs run slower. With --profile-use, the build is optimized by that profile and instrumented after inlining, the second phase of a context-sensitive profile. Valid only for executable builds with --optimization=release or speed.
--profile-use <file.profdata>
-
Optimize the executable with a profile from scriptc merge-profiles: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain. A new or changed profile compiles every partition of the program again. scriptc build prints how the profile matched the program's functions on stderr; see Profile-guided optimization. A profile with context-sensitive counts also orders each function's blocks by them on Linux. Valid only for executable builds with --optimization=release or speed.
+
Optimize the executable with a profile from scriptc merge-profiles: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain; a block-count file beside the profile gives changed functions their counts again. A new or changed profile compiles every partition of the program again. scriptc build prints how the profile matched the program's functions on stderr; see Profile-guided optimization. A profile with context-sensitive counts also orders each function's blocks by them on Linux. Valid only for executable builds with --optimization=release or speed.
--npm-static <pkg[,pkg…]|auto>
Name npm packages to compile even if they don't ship their own type declarations (repeatable). A trailing ..., as in fs-extra..., also names every package that package depends on. auto uses the default automatic selection. Packages that can't compile produce diagnostics. See npm Dependencies.
@@ -266,7 +266,7 @@ $ scriptc merge-profiles -o app.profdata profiles/*.profraw $ scriptc build app.ts --optimization=speed --profile-use app.profdata ``` -Train on workloads like the ones the program will run: code the training never reached is optimized as without a profile. The profile can be committed and reused as the program changes. Functions whose control flow changed since the profile was collected lose their profile, and the rest keep theirs, so regenerate it when much of the program has changed. [Build parallelism](/docs/how-it-works#build-parallelism) describes what a profile adds to build time. +Train on workloads like the ones the program will run: code the training never reached is optimized as without a profile. The profile can be committed and reused as the program changes, and in other checkouts and on other machines: it names functions by the entry's file name, not its directory. Functions whose control flow changed since the profile was collected lose their profile, and the rest keep theirs, so regenerate it when much of the program has changed. [Build parallelism](/docs/how-it-works#build-parallelism) describes what a profile adds to build time. `scriptc build --profile-use` prints one line on stderr with how the profile matched the program's functions: @@ -304,6 +304,25 @@ $ scriptc build app.ts --optimization=speed --profile-use app-cs.profdata A profile that already holds context-sensitive counts cannot start a second phase. Merge the second phase's raw profiles into the first-phase profile instead. Both phases run the same workloads. +### Block counts + +A profile matches a function only while the function's control flow hashes as it did when the profile was collected, so a change to the program or to scriptc leaves the changed functions without counts. A block-count file beside the profile, with its name and a `.blocks` extension in place of `.profdata` (`app.blocks` beside `app.profdata`), describes each function by the labels of its blocks instead. `--profile-use` reads it when it exists and annotates the functions whose control flow changed (the mismatched ones; a function the profile has no record of keeps no counts): blocks that keep their place in the function take their recorded counts, and the counts of the other blocks follow from the counts of the blocks and edges around them. The report then adds a line, and the warning counts only the functions that remain without counts: + +``` +scriptc: profile app-cs.profdata: 0 functions annotated, 1 mismatched, 0 without a record +scriptc: profile app-cs.profdata: 1 function without profile counts annotated from app-cs.blocks +``` + +A `--profile-use` build writes the block counts of the functions its profile annotates when the `SCRIPTC_RECORD_PROFILE_BLOCKS` environment variable names a path; such a build reuses no cached artifacts. For a context-sensitive profile, record them in the second phase, which the first-phase profile annotates: + +```console +$ SCRIPTC_RECORD_PROFILE_BLOCKS=app-cs.blocks scriptc build app.ts --optimization=speed --profile-use app.profdata --profile-generate profiles-cs/%p.profraw -o app-instrumented-cs +``` + +A new or changed block-count file compiles every partition of the program again. Delete it when the profile it was recorded with is replaced. + +The file holds the counts before inlining. With a context-sensitive profile, a function whose control flow changed, and a function it is inlined into, also lose their context-sensitive counts and keep the counts from before inlining. + ## Target CPU By default, x86-64 executables run on any x86-64 CPU, so the compiler uses only the baseline instruction set (SSE2). `--target-cpu` compiles the program's code for a higher x86-64 microarchitecture level. Each level includes the ones below it: diff --git a/docs/content/docs/contracts.mdx b/docs/content/docs/contracts.mdx index ad5953fa6..1c612c0d9 100644 --- a/docs/content/docs/contracts.mdx +++ b/docs/content/docs/contracts.mdx @@ -366,6 +366,10 @@ Static code has no `WeakRef`, a `FinalizationRegistry` never runs its cleanup ca A long-running program, such as a compiler in watch mode, can free program-lifetime instances one generation at a time with `LifetimeRegion` from [`@scriptc/experimental`](/docs/experimental). Instances created while `region.run(fn)` runs belong to the region. After `region.release()`, scriptc frees them together once nothing outside the region refers to any of them, and removes the `WeakMap` and `WeakSet` entries they are keys of. Instances created outside every region are never freed, as described above. +### Inline array fields + +An array field of a program-lifetime class whose elements are program-lifetime instances can keep its elements in the instance itself, after the other fields, with `InlineArray` from [`@scriptc/experimental`](/docs/experimental). The constructor writes the field once from one of its parameters; reads that index the field, read its `length` or iterate it load the elements from the instance, and other reads get an array of them. The array passed to the constructor is one that nothing else keeps, and the arrays fields hold are never written. scriptc reports a field that does not keep these rules (SC7009). + ## Diagnostics Each failed contract names the declaration and the reason: @@ -468,5 +472,9 @@ $ scriptc build reject.ts SC7008 A ProgramLifetime class is not the root of its hierarchy, or is part of a library build. + + SC7009 + An InlineArray field is not on a ProgramLifetime class, is written other than once from a constructor parameter, or is read in a way that could tell its reads apart. + diff --git a/docs/content/docs/experimental.mdx b/docs/content/docs/experimental.mdx index 97e048fc4..6221111ca 100644 --- a/docs/content/docs/experimental.mdx +++ b/docs/content/docs/experimental.mdx @@ -334,3 +334,73 @@ scriptc requires: Each instance of a key class, and of every class in its hierarchy, carries a 4-byte slot index, 0 until a value is set for it. The first `set` for a key assigns the index; indices are never reused. `get` and `has` read the index, a page of the thread's family and the entry, with no lock, hash or atomic update. A family stores values in pages of 1024 entries, allocated as the thread sets values in them. A family holds a reference to each of the thread's values. A value is released when it is replaced or deleted, when its key is freed, and when the thread exits. Keys that are freed with a [`LifetimeRegion`](#lifetimeregion) keep their values until the thread exits, a value that belongs to a region keeps the region allocated while the family holds it, and a value that refers to its own key keeps the key alive. Programs that never construct a `ThreadSlots` do not run any of its code, and their classes have no slot index. + +## InlineArray + +`InlineArray` marks an array field of a [`ProgramLifetime`](/docs/contracts#programlifetime) class whose elements scriptc keeps in the instance itself, after its other fields, instead of in an array object of their own. It suits a program with millions of short arrays that never change once built, such as the children of each node of a syntax tree: an ordinary array costs a separate allocation with a header, and a reference to it in the instance. + +```ts:lists.ts +import type { ProgramLifetime } from "@scriptc/types"; +import type { InlineArray } from "@scriptc/experimental"; + +class Token implements ProgramLifetime { + readonly text: string; + constructor(text: string) { + this.text = text; + } +} + +class TokenList implements ProgramLifetime { + readonly pos: number; + readonly tokens: InlineArray; + constructor(pos: number, tokens: readonly Token[]) { + this.pos = pos; + this.tokens = tokens; + } +} + +function width(list: TokenList): number { + let total = 0; + for (const token of list.tokens) total += token.text.length; + return total; +} + +const list = new TokenList(0, ["let", "x", "=", "1"].map((text) => new Token(text))); +console.log(list.tokens.length, width(list), list.tokens[0]!.text); +console.log(list.tokens.map((token) => token.text).join(" ")); +``` + +Both runtimes print the same: + +```console +$ node lists.ts +4 6 let +let x = 1 +$ scriptc build lists.ts -o lists >/dev/null +$ ./lists +4 6 let +let x = 1 +``` + +### Field rules + +scriptc requires: + +- `T` to be a `ProgramLifetime` class, and the field to be an instance field of a `ProgramLifetime` class; +- the constructor of the class that declares the field to write it once, from one of its parameters. A subclass constructor passes one of its own parameters there through `super(...)`. Nothing else writes the field; +- the array passed to the constructor to be one that nothing else keeps: a new array (an array literal, a copy such as `slice()`, `[...items]` or `Array.from(items)`, or the result of a function that returns only such arrays or its own argument), a local array at its last use (calls that only read it or add to it may come first, and a loop may pass it and then assign a new array to the local), a parameter whose callers each pass such an array, an array that an InlineArray field already holds, or a `const` array that is never written, such as a shared empty array; +- the arrays fields hold never to be written, through a read of the field (which the `readonly` type allows only through a cast) or through any other reference. scriptc follows these arrays through locals, parameters, return values, fields, the elements of arrays, Maps and Sets, and closures, and reports one that reaches a write, a dynamic value (also inside the fields of an object that becomes one), a call through a function value (also inside an array, Map or Set passed there), or a library call other than printing and `JSON.stringify` without a replacer. The report names the field read the array may come from; +- the arrays fields hold, and their reads, not to be compared by identity: with `===` or `!==`, as an `indexOf` or `includes` argument, or as a Map key or Set element. Two reads of the field need not give the same array. Compare the elements instead, or keep a copy made with `slice()` where identity matters; +- instances of the class not to be constructed through a class value, and not to become dynamic values (for example through a conversion to `unknown`), directly or inside another value that can hold one: an instance of a class with such a field, a record, array, Map or Set, or a value typed as a base class. A dynamic view reads the field as a new array each time, so a class whose instances can become dynamic values cannot hold an InlineArray field. Printing, `JSON.stringify` without a replacer, `publish` and a worker's `workerData` only read or copy the value and are allowed. The report names the conversion. + +scriptc reports a field that does not keep these rules (SC7009). + +### Reads + +Indexing the field, reading its `length` and iterating it with `for...of` read the elements from the instance, directly or through a `const` local used only in those ways. Any other use, such as passing the field to a function or calling one of its methods, gets an array of the elements, which is frozen in scriptc builds. Under Node.js the field holds the array the constructor stored. The two agree because the field rules keep that array from being written. + +### Inline array cost + +Each element takes 4 bytes in the instance, and the field's own slot holds the count. The constructor copies the elements from the array it is passed, so that array can be released. The elements start at the same offset in every class of the hierarchy, after the largest set of fixed fields. A read that gets an array allocates a small header that refers to the instance's elements, and frees it when the read's value is released. + +The elements stay in an ordinary array object, and the field behaves like any other array field, in builds with sanitizers, in programs that use [`LifetimeRegion`](#lifetimeregion), and on platforms other than 64-bit Linux. diff --git a/docs/content/docs/how-it-works.mdx b/docs/content/docs/how-it-works.mdx index 8196460cf..13b8b5a90 100644 --- a/docs/content/docs/how-it-works.mdx +++ b/docs/content/docs/how-it-works.mdx @@ -34,7 +34,7 @@ Runtime pack and linker -> executable ## Build parallelism -Most of a build runs on one CPU core. The LLVM helper uses more cores when it compiles an executable in partitions, and when it generates code for an optimized executable of one partition. +Most of a build runs on one CPU core. The LLVM helper uses more cores when it compiles an executable in partitions, and when it generates code for an optimized executable that it optimizes whole. ### Frontend @@ -42,16 +42,15 @@ Parsing, type checking, lowering, and LLVM IR generation run one after another o ### Partitions -The LLVM helper divides an executable's LLVM IR into partitions by its size. Up to 1 MiB of LLVM text is one partition, and up to 2 MiB is four. Larger programs get one partition per 4 MiB of LLVM text, rounded up to a power of two, and at least 8 and at most 64. `--emit=llvm` writes this text, so its file size shows how many partitions a program gets. `dev` executables, whose partitions are compiled on their own with nothing imported, are divided from 64 KiB: up to 128 KiB is four partitions, and larger `dev` programs get at least 8. +The LLVM helper divides an executable's LLVM IR into partitions by its size. Up to 128 KiB of LLVM text is one partition. Larger programs get one partition per 4 MiB of LLVM text, rounded up to a power of two, and at least 8 and at most 64. `--emit=llvm` writes this text, so its file size shows how many partitions a program gets. `dev` executables, whose partitions are compiled on their own with nothing imported, are divided from 64 KiB: up to 128 KiB is four partitions. An edit compiles again only the partitions whose code it changes, or that inline a function it changes. - - - + + @@ -60,6 +59,8 @@ The LLVM helper divides an executable's LLVM IR into partitions by its size. Up The count depends only on the size of the LLVM text, so a program has the same partitions, and builds the same executable, on every host. In programs with 32 partitions or more, functions from the same source file are placed in the same partition, so that their calls to each other stay inside it; a file larger than 1 MiB of LLVM text or than an average partition is divided into parts by function name. Other functions, and the functions of smaller programs, are placed by their name, and a small internal function goes with the one function that calls it. An edit leaves almost every other function in its partition. +An optimized partition is optimized twice: on its own, and again after the small functions of other partitions it calls are imported for inlining. So in optimized builds, a program whose largest partition would hold more than two thirds of its function text is optimized whole instead, as one partition, and its code is generated in parts on several threads. + These builds are not divided into partitions and compile on one thread: `--emit=asm`, `--emit=obj`, and `--print=native-link-info` output, library builds (`--lib`; `dev` library builds are compiled in pieces, one after another), and `--sanitize` builds, which compile the LLVM IR with the C compiler. The helper compiles partitions on a pool with one thread per hardware thread of the host, so an executable with four partitions uses at most four cores in the backend. @@ -71,26 +72,15 @@ How partitions are compiled depends on `--optimization`: Reading the LLVM text and dividing it into partitions, and the import step, run on one thread. -### Code generation of one partition +### Code generation of a program optimized whole -A `release` or `speed` executable of one partition is optimized as one module on one thread. When its LLVM text is larger than 128 KiB, the helper then divides the optimized module for code generation and generates machine code for the parts in parallel, on the same pool. Each part holds a run of consecutive functions with about the same amount of code, so the executable keeps the functions in the same order. The module is optimized before it is divided, so each function's machine code is the same as from one thread, except that a jump to a function that another part also uses takes the longer form. When one function holds most of the code, the first part holds the whole module and the others are empty. +A `release` or `speed` executable whose largest partition would hold more than two thirds of its function text is optimized as one module on one thread. The helper then divides the optimized module for code generation into as many parts as the program has partitions and generates machine code for the parts in parallel, on the same pool. Each part holds a run of consecutive functions with about the same amount of code, so the executable keeps the functions in the same order. The module is optimized before it is divided, so each function's machine code is the same as from one thread, except that a jump to a function that another part also uses takes the longer form. When one function holds most of the code, the first part holds the whole module and the others are empty. -
LLVM textPartitions
Up to 1 MiB1
Over 1 MiB, up to 2 MiB4
Over 2 MiB, up to 32 MiB8
Up to 128 KiB1
Over 128 KiB, up to 32 MiB8
Over 32 MiB, up to 64 MiB16
Over 64 MiB, up to 128 MiB32
Over 128 MiB64
- - - - - - - - -
LLVM textCode generation parts
Up to 128 KiB1
Over 128 KiB, up to 256 KiB4
Over 256 KiB, up to 1 MiB8
- -Like partitions, the count depends only on the size of the LLVM text. A `dev` executable of one partition is compiled on one thread. +An executable of one partition (up to 128 KiB of LLVM text, or 64 KiB in `dev` builds) is compiled on one thread. ### Rebuilds -With two or more partitions, the helper keeps each partition's summary and object in the build cache (see `SCRIPTC_CACHE_DIR` in the [CLI reference](/docs/cli#environment-variables)). A rebuild after an edit compiles again only the partitions whose functions changed and, in `release` and `speed` builds, the partitions that import a changed function; the others come from the cache. Dividing the program, the import step, and linking run on every build. A cold build, with nothing cached, compiles every partition. +With two or more partitions, the helper keeps each partition's summary and object in the build cache (see `SCRIPTC_CACHE_DIR` in the [CLI reference](/docs/cli#environment-variables)). A rebuild after an edit compiles again only the partitions whose functions changed and, in `release` and `speed` builds, the partitions that import a changed function; the others come from the cache. Dividing the program, the import step, and linking run on every build. A cold build, with nothing cached, compiles every partition. A program optimized whole is compiled again whole after any edit. A program of one partition has no partition cache: a rebuild after an edit compiles the whole program again, and a rebuild that leaves the LLVM text unchanged takes the compiled code from the build cache. diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index f3883fe8d..52bf534e8 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -65,4 +65,7 @@ Their underlying sources, sinks, and transformers, queuing strategies with a `size` function, readers and writers, `tee`, `pipeTo` and `pipeThrough` with their options and an abort signal, `ReadableStream.from`, and async iteration settle promises in Node's order, and the streams take own properties. `TextDecoderStream` decodes UTF-8; its other encodings are unsupported. A JavaScript `ReadableStream` given as a `Response` or `Request` body, or as `fetch`'s body, is read through a stream of its own: `response.body` is that stream rather than the one passed in, and the stream passed in is locked from the start. + In JavaScript, a `Response` or `Request` body has the members of a `ReadableStream`, including `tee`, `pipeTo`, `pipeThrough`, `values`, and `for await`, and its lock and `bodyUsed` follow them. + After one of those members is used, a body given to a new `Response` or `Request` is locked from the start, as a JavaScript `ReadableStream` is. + In a program that names none of the web stream classes, those members are available when the program's JavaScript names one of them or iterates a body with `for await`; reached only through a computed key, they are not. In JavaScript, a byte stream (`type: 'bytes'`), a BYOB reader, a class that extends one of these classes, `Object.getPrototypeOf` on these values, their use as `WeakMap` and `WeakSet` keys, and `ReadableStream.from` of a function that is not iterable are unsupported. diff --git a/docs/test/fixtures/migration-baseline.json b/docs/test/fixtures/migration-baseline.json index bd9749038..ea0ce3dc4 100644 --- a/docs/test/fixtures/migration-baseline.json +++ b/docs/test/fixtures/migration-baseline.json @@ -316,6 +316,22 @@ { "text": "Slot cost", "id": "slot-cost" + }, + { + "text": "InlineArray", + "id": "inlinearray" + }, + { + "text": "Field rules", + "id": "field-rules" + }, + { + "text": "Reads", + "id": "reads" + }, + { + "text": "Inline array cost", + "id": "inline-array-cost" } ] }, diff --git a/native/llvm-codegen/CMakeLists.txt b/native/llvm-codegen/CMakeLists.txt index 7ce20086f..fc4c07210 100644 --- a/native/llvm-codegen/CMakeLists.txt +++ b/native/llvm-codegen/CMakeLists.txt @@ -88,6 +88,7 @@ add_executable(scriptc-llvm-codegen src/emit.cpp src/main.cpp src/partition.cpp + src/profile-blocks.cpp src/profile.cpp src/runtime-import.cpp src/runtime-unit.cpp diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index 98e07e9ce..17d69be72 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -3,6 +3,7 @@ #include "cache.h" #include "diagnostics.h" #include "partition.h" +#include "profile-blocks.h" #include "runtime-import.h" #include "target.h" @@ -13,6 +14,7 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/RuntimeLibcallInfo.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Bitcode/BitcodeReader.h" @@ -34,6 +36,7 @@ #include "llvm/PassInfo.h" #include "llvm/PassRegistry.h" #include "llvm/Passes/PassBuilder.h" +#include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/PGOOptions.h" @@ -41,6 +44,7 @@ #include "llvm/Support/Caching.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" #include "llvm/Support/SHA256.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/ThreadPool.h" @@ -104,6 +108,10 @@ std::optional parseEmitOptions(int Argc, char **Argv) { Options.ProfileGenerate = Value.str(); else if (Arg == "--profile-use") Options.ProfileUse = Value.str(); + else if (Arg == "--profile-blocks") + Options.ProfileBlocks = Value.str(); + else if (Arg == "--record-profile-blocks") + Options.RecordProfileBlocks = Value.str(); else if (Arg == "--split" && Value == "codegen") Options.SplitCodegen = true; else @@ -245,37 +253,46 @@ struct UnnameStructTypes : PassInfoMixin { // control flow changed since it was collected (hash mismatch), and functions // it has no record of (new since it was collected, or never run in training // when the profile was merged with --sparse). A partition reused from the cache without its counts is counted -// in Unknown. +// in Unknown. Recovered counts the mismatched functions that --profile-blocks +// annotated, so it never exceeds Mismatched. struct ProfileMatch { std::atomic Annotated{0}; std::atomic Mismatched{0}; std::atomic Missing{0}; + std::atomic Recovered{0}; std::atomic Unknown{0}; + // Set once the annotation before inlining has run. The context-sensitive + // annotation after inlining, which a single-module pipeline also runs, + // reports its mismatched and missing functions again; they are not + // counted twice. + std::atomic Counted{false}; void add(const ProfileMatch &Other) { Annotated += Other.Annotated; Mismatched += Other.Mismatched; Missing += Other.Missing; + Recovered += Other.Recovered; } std::string serialize() const { - return formatv("{0} {1} {2}", Annotated.load(), Mismatched.load(), - Missing.load()) + return formatv("{0} {1} {2} {3}", Annotated.load(), Mismatched.load(), + Missing.load(), Recovered.load()) .str(); } bool deserialize(StringRef Text) { - SmallVector Parts; + SmallVector Parts; Text.split(Parts, ' '); - uint64_t Values[3]; - if (Parts.size() != 3) + uint64_t Values[4]; + if (Parts.size() != 4) return false; - for (size_t I = 0; I < 3; ++I) + for (size_t I = 0; I < 4; ++I) if (Parts[I].getAsInteger(10, Values[I])) return false; Annotated += Values[0]; Mismatched += Values[1]; Missing += Values[2]; + Recovered += Values[3]; return true; } }; @@ -285,7 +302,9 @@ struct ProfileMatch { // context's default handling. class ProfileMatchHandler : public DiagnosticHandler { public: - explicit ProfileMatchHandler(ProfileMatch &Match) : Match(Match) {} + explicit ProfileMatchHandler( + ProfileMatch &Match, std::vector *MismatchedNames = nullptr) + : Match(Match), MismatchedNames(MismatchedNames) {} bool handleDiagnostics(const DiagnosticInfo &Info) override { if (Info.getKind() != DK_PGOProfile) @@ -294,17 +313,28 @@ public: raw_string_ostream Stream(Message); DiagnosticPrinterRawOStream Printer(Stream); Info.print(Printer); - if (StringRef(Message).contains("hash mismatch")) - ++Match.Mismatched; - else if (StringRef(Message).contains_insensitive("no profile data")) - ++Match.Missing; - else + bool Mismatched = StringRef(Message).contains("hash mismatch"); + if (!Mismatched && + !StringRef(Message).contains_insensitive("no profile data")) return false; + if (Match.Counted) + return true; + if (!Mismatched) { + ++Match.Missing; + return true; + } + ++Match.Mismatched; + // "...(hash mismatch) Hash = " + if (MismatchedNames != nullptr) { + StringRef Name = StringRef(Message).split("(hash mismatch) ").second; + MismatchedNames->push_back(Name.split(" Hash = ").first.str()); + } return true; } private: ProfileMatch &Match; + std::vector *MismatchedNames; }; // The annotation warns about functions the profile does not describe only @@ -319,17 +349,61 @@ void reportMissingProfiles() { (void)Reported; } -// Functions of `M` that the profile annotated with counts. -uint64_t annotatedFunctions(const Module &M) { - uint64_t Count = 0; - for (const Function &F : M) { +// The --profile-blocks counts, read once before any partition needs them +// (inspectProfile), or null without them. +std::unique_ptr FallbackCounts; +// The block counts a --record-profile-blocks build collects, or null. +std::unique_ptr Recorder; + +// Whether the profile annotation found F's control flow changed: it tags +// such a function with this annotation. +bool hasHashMismatchTag(const Function &F) { + if (const MDNode *Annotation = F.getMetadata(LLVMContext::MD_annotation)) + for (const MDOperand &Operand : Annotation->operands()) + if (const auto *Name = dyn_cast(Operand)) + if (Name->getString() == "instr_prof_hash_mismatch") + return true; + return false; +} + +// Whether F has counts from a profile. +bool hasProfileCounts(const Function &F) { + std::optional Entry = F.getEntryCount(); + return Entry && !Entry->isSynthetic(); +} + +// Runs right after the profile annotation, before inlining and dead code +// elimination change the functions it annotated: counts them and records +// their block counts, then annotates the functions whose control flow +// changed from FallbackCounts. A function the profile has no record of keeps +// no counts, as without block counts. Returns the analyses that remain valid. +PreservedAnalyses afterProfileAnnotation(Module &M, ProfileMatch &Match) { + std::vector Mismatched; + for (Function &F : M) { if (F.isDeclaration()) continue; - std::optional Entry = F.getEntryCount(); - if (Entry && !Entry->isSynthetic()) - ++Count; + if (!hasProfileCounts(F)) { + if (hasHashMismatchTag(F)) + Mismatched.push_back(&F); + continue; + } + ++Match.Annotated; + if (Recorder) + Recorder->add(F); } - return Count; + if (!FallbackCounts) + return PreservedAnalyses::all(); + ProfileSummaryInfo Summary(M); + PreservedAnalyses Preserved = PreservedAnalyses::all(); + for (Function *F : Mismatched) { + auto Record = FallbackCounts->find(F->getName()); + if (Record == FallbackCounts->end()) + continue; + annotateFromBlocks(*F, Record->second, Summary); + ++Match.Recovered; + Preserved = PreservedAnalyses::none(); + } + return Preserved; } // Prints the profile match as the successful emit's one line of JSON. @@ -337,6 +411,7 @@ void reportProfileMatch(const ProfileMatch &Match) { json::Object Profile{{"annotated", Match.Annotated.load()}, {"mismatched", Match.Mismatched.load()}, {"missing", Match.Missing.load()}, + {"recovered", Match.Recovered.load()}, {"unknown_partitions", Match.Unknown.load()}}; json::Object Response{{"ok", true}, {"profile", std::move(Profile)}}; outs() << formatv("{0}\n", json::Value(std::move(Response))); @@ -369,9 +444,36 @@ std::optional inspectProfile(const EmitOptions &Options) { enableProfileLayout(); enableExtTspPlacement(); } + if (!Options.ProfileBlocks.empty()) { + std::string Error; + FallbackCounts = readProfileBlocks(Options.ProfileBlocks, Error); + if (!FallbackCounts) + return EmitFailure{"profile_blocks_read_failed", Error}; + } + if (!Options.RecordProfileBlocks.empty()) + Recorder = std::make_unique(); return std::nullopt; } +// Writes the block counts a --record-profile-blocks build collected. +std::optional writeRecordedBlocks(const EmitOptions &Options) { + if (!Recorder) + return std::nullopt; + if (std::optional Error = + Recorder->write(Options.RecordProfileBlocks)) + return EmitFailure{"profile_blocks_write_failed", + Options.RecordProfileBlocks + ": " + *Error}; + return std::nullopt; +} + +// The source file name a program's modules carry: the entry's file name, +// without its directory. Profiles name a module's local functions after it +// (";"), so a profile collected in one checkout matches the +// builds of every other one. +std::string moduleSourceName(StringRef SourcePath) { + return sys::path::filename(SourcePath).str(); +} + // Owns the analysis managers a pipeline needs for one module. struct Pipeline { PassInstrumentationCallbacks Callbacks; @@ -391,16 +493,41 @@ struct Pipeline { PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); } - // Adds to Count the functions the profile annotation gave counts to, as - // soon as that pass has run: inlining and dead code elimination later in - // the pipeline delete annotated functions. - void countAnnotated(std::atomic &Count) { + // Runs afterProfileAnnotation as soon as the profile annotation has run. + // The context-sensitive annotation after inlining runs the same pass + // again, on functions the first one already counted. + void afterProfileAnnotation(ProfileMatch &Match) { Callbacks.registerAfterPassCallback( - [&Count](StringRef PassID, Any IR, const PreservedAnalyses &) { + [this, &Match, Done = false](StringRef PassID, Any IR, + const PreservedAnalyses &) mutable { + if (Done || PassID != "PGOInstrumentationUse") + return; + const auto *M = llvm::any_cast(&IR); + if (M == nullptr) + return; + Done = true; + // The callback receives the module the pipeline runs on; only the + // signature is const. + Module &Annotated = const_cast(**M); + MAM.invalidate(Annotated, + scriptc::afterProfileAnnotation(Annotated, Match)); + Match.Counted = true; + }); + } + + // Adds to Names the profile names of the functions the annotation gave + // counts to, as soon as that pass has run. + void collectAnnotated(std::vector &Names) { + Callbacks.registerAfterPassCallback( + [&Names](StringRef PassID, Any IR, const PreservedAnalyses &) { if (PassID != "PGOInstrumentationUse") return; if (const auto *M = llvm::any_cast(&IR)) - Count += annotatedFunctions(**M); + for (const Function &F : **M) { + std::optional Entry = F.getEntryCount(); + if (!F.isDeclaration() && Entry && !Entry->isSynthetic()) + Names.push_back(getIRPGOFuncName(F)); + } }); } }; @@ -592,7 +719,7 @@ emitModule(Module &M, TargetMachine &Machine, OptimizationLevel Level, std::make_unique(Match)); Pipeline P(Machine, Level, std::move(Profile)); if (Annotating) - P.countAnnotated(Match.Annotated); + P.afterProfileAnnotation(Match); P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM); if (std::optional Failure = verify(M, "post_optimization_verification_failed")) @@ -678,7 +805,7 @@ Expected> loadPartition(const EmitOptions &Options, return createStringError(inconvertibleErrorCode(), Stream.str()); } if (!Options.SourcePath.empty()) - Mod->setSourceFileName(Options.SourcePath); + Mod->setSourceFileName(moduleSourceName(Options.SourcePath)); Mod->setTargetTriple(Triple(Options.Target)); Mod->setDataLayout(Machine.createDataLayout()); if (std::optional Failure = verify(*Mod, "verification_failed")) @@ -767,7 +894,9 @@ compileOptimizedPartitions(const EmitOptions &Options, SharedFailure Failures; forEachPartition(Count, [&](size_t I) { std::string Key = hashParts({Salt, "summary", Plan.Modules[I]}); - if ((Summaries[I] = Cache.lookup(Key, "bc"))) { + // A reused summary has no block counts to record, so a recording build + // annotates every partition again. + if (!Recorder && (Summaries[I] = Cache.lookup(Key, "bc"))) { if (Annotating) { std::unique_ptr Counts = Cache.lookup(Key, "match"); if (!Counts || !Match.deserialize(Counts->getBuffer())) @@ -809,12 +938,13 @@ compileOptimizedPartitions(const EmitOptions &Options, // link; the post-link optimization reads the annotated weights. Pipeline P(*Machine, Level, profileOptions(Options)); if (Annotating) - P.countAnnotated(PartitionMatch.Annotated); + P.afterProfileAnnotation(PartitionMatch); ModulePassManager MPM = P.PB.buildThinLTOPreLinkDefaultPipeline(Level); MPM.addPass(UnnameStructTypes()); + // The module hash is set at the thin link instead (see Names). MPM.addPass(BitcodeWriterPass(Stream, /*ShouldPreserveUseListOrder=*/false, /*EmitSummaryIndex=*/true, - /*EmitModuleHash=*/true)); + /*EmitModuleHash=*/false)); MPM.run(Mod, P.MAM); } if (Annotating) { @@ -921,6 +1051,24 @@ compileOptimizedPartitions(const EmitOptions &Options, std::vector Names(Count); for (size_t I = 0; I < Count; ++I) Names[I] = "partition-" + std::to_string(I); + // An internal function or variable that another partition imports a + // reference to is renamed .llvm.. + // A hash of the module's contents would change that name in every + // importing partition after any edit of the exporting one, and compile + // them all again. Each module's hash is its partition number instead, + // which keeps the names unique within the program and the same across + // builds. + Config.CombinedIndexHook = [&Names](const ModuleSummaryIndex &Index, + const DenseSet &) { + StringMap &Paths = + const_cast(Index).modulePaths(); + for (size_t I = 0; I < Names.size(); ++I) { + auto It = Paths.find(Names[I]); + if (It != Paths.end()) + It->second = {0, static_cast(I + 1), 0, 0, 0}; + } + return true; + }; limitImports(); if (!Options.ProfileUse.empty()) limitProfileImports(); @@ -997,6 +1145,17 @@ compilePartitions(const EmitOptions &Options, const PartitionPlan &Plan, Profile.getError().message()}; Salt = hashParts({Salt, "profile-use", (*Profile)->getBuffer()}); } + // The block counts change the partitions they annotate. + if (!Options.ProfileBlocks.empty()) { + ErrorOr> Blocks = + MemoryBuffer::getFile(Options.ProfileBlocks, /*IsText=*/false, + /*RequiresNullTerminator=*/false); + if (!Blocks) + return EmitFailure{"profile_blocks_read_failed", + Options.ProfileBlocks + ": " + + Blocks.getError().message()}; + Salt = hashParts({Salt, "profile-blocks", (*Blocks)->getBuffer()}); + } for (const std::string &Path : Options.ImportBitcode) { ErrorOr> Bitcode = MemoryBuffer::getFile(Path, /*IsText=*/false, @@ -1060,8 +1219,22 @@ int emitPartitions(const EmitOptions &Options) { MemoryBuffer::getFile(Options.Input, /*IsText=*/false, /*RequiresNullTerminator=*/false); if (Input) { + bool Optimized = Options.OptLevel != "0"; std::optional Text = partitionProgramText( - (*Input)->getBuffer(), static_cast(Options.Outputs.size())); + (*Input)->getBuffer(), static_cast(Options.Outputs.size()), + /*SkipDominated=*/Optimized); + // An optimized partition is simplified before the thin link and + // optimized again after it, so one that holds more than two thirds of + // the program's function text takes longer than optimizing the module + // whole: such a module is optimized whole, and its code is generated in + // parts. + if (Text && Optimized && dominatedPartitions(Text->FunctionBytes)) { + EmitOptions Whole = Options; + Whole.SplitCodegen = true; + Whole.CacheDirectory.clear(); + Input->reset(); + return emit(Whole); + } if (Text) { Input->reset(); PartitionPlan Plan{std::move(Text->Sources), @@ -1082,6 +1255,8 @@ int emitPartitions(const EmitOptions &Options) { Failure = writePartitionObjects(Options, Objects, Temporaries); for (size_t I = 0; !Failure && I < Temporaries.size(); ++I) Failure = publish(Temporaries[I], Options.Outputs[I]); + if (!Failure) + Failure = writeRecordedBlocks(Options); if (Failure) { for (const SmallString<256> &Temporary : Temporaries) sys::fs::remove(Temporary); @@ -1276,7 +1451,7 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine, { Pipeline P(Machine, Level, profileOptions(Options)); if (Annotating) - P.countAnnotated(Match.Annotated); + P.afterProfileAnnotation(Match); P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM); } if (std::optional Failure = @@ -1338,8 +1513,100 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine, return Failure; } +// Lists the annotated and the mismatched functions of one module in +// `Result`, each with the sum of its counters in the profile. `ProfileNames` +// maps the module's symbol names, which the mismatch warnings use, to +// profile names. +std::optional +weighProfileMatch(const std::string &ProfileUse, + const StringMap &ProfileNames, + const std::vector &AnnotatedNames, + const std::vector &MismatchedNames, + ModuleProfileMatch &Result) { + Expected> Reader = + IndexedInstrProfReader::create(ProfileUse, *vfs::getRealFileSystem()); + if (!Reader) + return ProfileUse + ": " + toString(Reader.takeError()); + StringMap Counts; + for (const NamedInstrProfRecord &Record : **Reader) { + uint64_t Sum = 0; + for (uint64_t Count : Record.Counts) + Sum += Count; + Counts[Record.Name] += Sum; + } + if (Error E = (*Reader)->getError()) + return ProfileUse + ": " + toString(std::move(E)); + auto CountsOf = [&Counts](StringRef Name) -> uint64_t { + auto It = Counts.find(Name); + return It == Counts.end() ? 0 : It->second; + }; + for (const std::string &Name : AnnotatedNames) + Result.AnnotatedCounts.emplace_back(Name, CountsOf(Name)); + for (const std::string &Name : MismatchedNames) { + auto It = ProfileNames.find(Name); + std::string ProfileName = It == ProfileNames.end() ? Name : It->second; + uint64_t Sum = CountsOf(ProfileName); + Result.MismatchedCounts.emplace_back(std::move(ProfileName), Sum); + } + return std::nullopt; +} + } // namespace +std::optional optimizeModule(Module &M, TargetMachine &Machine, + const std::string &OptLevel, + bool Instrument, + const std::string &ProfileGenerate, + const std::string &ProfileUse, + ModuleProfileMatch &Result) { + OptimizationLevel Level = optimizationLevel(OptLevel); + std::optional Profile; + if (Instrument) { + Profile = PGOOptions(ProfileGenerate, "", "", "", PGOOptions::IRInstr); + } else if (!ProfileUse.empty()) { + EmitOptions Inspected; + Inspected.ProfileUse = ProfileUse; + if (std::optional Failure = inspectProfile(Inspected)) + return Failure->Message; + reportMissingProfiles(); + Profile = PGOOptions(ProfileUse, "", "", "", PGOOptions::IRUse); + } + bool Annotating = Profile && Profile->Action == PGOOptions::IRUse; + ProfileMatch Match; + // The annotation reports a mismatched function by its symbol name; the + // profile names local functions after their source file. + StringMap ProfileNames; + std::vector MismatchedNames; + std::vector AnnotatedNames; + if (Annotating) { + for (const Function &F : M) + if (!F.isDeclaration()) + ProfileNames[F.getName()] = getIRPGOFuncName(F); + M.getContext().setDiagnosticHandler( + std::make_unique(Match, &MismatchedNames)); + } + { + Pipeline P(Machine, Level, std::move(Profile)); + if (Annotating) { + P.afterProfileAnnotation(Match); + P.collectAnnotated(AnnotatedNames); + } + P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM); + } + Result.Annotated = Match.Annotated.load(); + Result.Mismatched = Match.Mismatched.load(); + Result.Missing = Match.Missing.load(); + if (Annotating) { + if (std::optional Failure = weighProfileMatch( + ProfileUse, ProfileNames, AnnotatedNames, MismatchedNames, Result)) + return Failure; + } + if (std::optional Failure = + verify(M, "post_optimization_verification_failed")) + return Failure->Message; + return std::nullopt; +} + int emit(const EmitOptions &Options) { if (!supportsTarget(Options.Target)) return reportError("unsupported_target", @@ -1379,6 +1646,12 @@ int emit(const EmitOptions &Options) { return reportError("invalid_import", "runtime bitcode import requires an optimized build", Options.DiagnosticFormat); + if (Options.ProfileUse.empty() && + (!Options.ProfileBlocks.empty() || !Options.RecordProfileBlocks.empty())) + return reportError("invalid_profile_blocks", + "--profile-blocks and --record-profile-blocks require " + "--profile-use", + Options.DiagnosticFormat); if (!Options.ProfileUse.empty()) reportMissingProfiles(); if (std::optional Failure = inspectProfile(Options)) @@ -1405,7 +1678,7 @@ int emit(const EmitOptions &Options) { return reportError("invalid_ir", Stream.str(), Options.DiagnosticFormat); } if (!Options.SourcePath.empty()) - Mod->setSourceFileName(Options.SourcePath); + Mod->setSourceFileName(moduleSourceName(Options.SourcePath)); std::string LookupError; std::unique_ptr Machine = @@ -1451,8 +1724,11 @@ int emit(const EmitOptions &Options) { : CodeGenFileType::AssemblyFile; if (Options.SplitCodegen) { ProfileMatch Match; - if (std::optional Failure = - emitSplitCodegen(Options, *Mod, *Machine, Level, Match)) + std::optional Failure = + emitSplitCodegen(Options, *Mod, *Machine, Level, Match); + if (!Failure) + Failure = writeRecordedBlocks(Options); + if (Failure) return reportError(Failure->Code, Failure->Message, Options.DiagnosticFormat); if (!Options.ProfileUse.empty()) @@ -1466,6 +1742,8 @@ int emit(const EmitOptions &Options) { Options.Outputs[0], Temporaries, Match); if (!Failure) Failure = publish(Temporaries[0], Options.Outputs[0]); + if (!Failure) + Failure = writeRecordedBlocks(Options); if (Failure) { for (const SmallString<256> &Temporary : Temporaries) sys::fs::remove(Temporary); diff --git a/native/llvm-codegen/src/emit.h b/native/llvm-codegen/src/emit.h index 1bdc578df..926f7d6ad 100644 --- a/native/llvm-codegen/src/emit.h +++ b/native/llvm-codegen/src/emit.h @@ -1,9 +1,16 @@ #pragma once +#include #include #include +#include #include +namespace llvm { +class Module; +class TargetMachine; +} // namespace llvm + namespace scriptc { struct EmitOptions { @@ -28,6 +35,13 @@ struct EmitOptions { std::string ProfileGenerate; // Indexed profile (.profdata) that guides optimization. Empty: none. std::string ProfileUse; + // With ProfileUse: block counts (profile-blocks.h) that annotate the + // functions whose control flow changed since the profile was collected. + // Empty: none. + std::string ProfileBlocks; + // With ProfileUse: writes the block counts of the functions the profile + // annotates to this path. Empty: none. + std::string RecordProfileBlocks; // With several outputs: optimize the module whole and divide only its code // generation among the outputs. bool SplitCodegen = false; @@ -36,4 +50,32 @@ struct EmitOptions { std::optional parseEmitOptions(int Argc, char **Argv); int emit(const EmitOptions &Options); +// How a profile matched the functions of one optimized module: functions +// that received its counts, functions whose control flow changed since it +// was collected, and functions it has no record of. The annotated and the +// mismatched functions are also listed by profile name, with the sum of the +// profile's counters for each, so a changed hot function can weigh more than +// a changed cold one. +struct ModuleProfileMatch { + uint64_t Annotated = 0; + uint64_t Mismatched = 0; + uint64_t Missing = 0; + std::vector> AnnotatedCounts; + std::vector> MismatchedCounts; +}; + +// Optimizes `M` with the per-module default pipeline at `OptLevel`, as `emit` +// does for a whole program, with IR instrumentation when `Instrument` is set +// or annotation from the indexed `ProfileUse` profile. Instrumentation writes +// raw profiles to the `ProfileGenerate` pattern; an empty pattern names no +// file, so the program that links the module names it. Returns an error +// message, or nothing on success. +std::optional optimizeModule(llvm::Module &M, + llvm::TargetMachine &Machine, + const std::string &OptLevel, + bool Instrument, + const std::string &ProfileGenerate, + const std::string &ProfileUse, + ModuleProfileMatch &Match); + } // namespace scriptc diff --git a/native/llvm-codegen/src/main.cpp b/native/llvm-codegen/src/main.cpp index 57416106a..d9481395d 100644 --- a/native/llvm-codegen/src/main.cpp +++ b/native/llvm-codegen/src/main.cpp @@ -58,6 +58,9 @@ int version(int Argc, char **Argv) { {"supported_targets", std::move(SupportedTargets)}, {"default_target", scriptc::DefaultTarget}, {"data_layout", Machine->createDataLayout().getStringRepresentation()}, + // Commands and options added within one protocol version, for callers + // that also run against older helpers. + {"features", json::Array{"runtime-unit-profile", "profile-functions"}}, }; outs() << formatv("{0}\n", json::Value(std::move(Response))); return 0; @@ -80,6 +83,8 @@ int main(int Argc, char **Argv) { return scriptc::pruneImport(Argc, Argv); if (Argc >= 2 && StringRef(Argv[1]) == "merge-profiles") return scriptc::mergeProfiles(Argc, Argv); + if (Argc >= 2 && StringRef(Argv[1]) == "profile-functions") + return scriptc::profileFunctions(Argc, Argv); if (Argc >= 2 && StringRef(Argv[1]) == "emit") { std::optional Options = scriptc::parseEmitOptions(Argc, Argv); @@ -95,5 +100,6 @@ int main(int Argc, char **Argv) { } return scriptc::reportError("usage", "expected 'version --format=json', 'emit', 'runtime-unit', " - "'prune-import', or 'merge-profiles'"); + "'prune-import', 'merge-profiles', or " + "'profile-functions'"); } diff --git a/native/llvm-codegen/src/partition.cpp b/native/llvm-codegen/src/partition.cpp index ef5230129..7174e2b24 100644 --- a/native/llvm-codegen/src/partition.cpp +++ b/native/llvm-codegen/src/partition.cpp @@ -36,6 +36,10 @@ struct Symbol { // Local definitions are rewritten with hidden visibility. std::string PromotedHeader; std::vector Refs; + // The functions its body calls directly (sorted). + std::vector Calls; + // A variable defined with "constant", whose loads the optimizer can fold. + bool Constant = false; std::vector Metadata; // The source-module group the emitter wrote before this definition, or // empty. @@ -74,6 +78,8 @@ uint32_t stableHash(StringRef Name) { return Hash; } +constexpr unsigned MinGroupedPartitions = 32; + // Places the functions whose root carries a source-module group by group: a // module's functions call each other most, so keeping them together leaves // fewer calls for the thin link to import across partitions. A group larger @@ -83,11 +89,11 @@ uint32_t stableHash(StringRef Name) { // lighter of two partitions chosen by hashes of its key, after the functions // placed by name. The placement depends only on the module text and the // partition count. Modules with fewer than MinGroupedPartitions partitions -// keep the placement by name: with few partitions per core, the balance of -// the largest partitions decides the build time more than the imports. +// keep the placement by name, which balanceRoots evens out: with few +// partitions per core, the balance of the largest partitions decides the +// build time more than the imports. void placeGroups(std::vector &Symbols, ArrayRef Roots, unsigned Count) { - constexpr unsigned MinGroupedPartitions = 32; if (Count < MinGroupedPartitions) return; auto Bytes = [](const Symbol &S) -> uint64_t { @@ -153,6 +159,67 @@ void placeGroups(std::vector &Symbols, ArrayRef Roots, Symbols[I].Partition = Where[Keys[I]]; } +// In modules of fewer than MinGroupedPartitions partitions, which +// placeGroups leaves placed by name, the functions of each root (a +// function and the chain that follows it) move together to another partition +// when their name's partition would hold more than 1.25 times the mean +// function bytes: each root probes the next partitions in turn, keeping the +// first with room, else its own. An edit then recompiles about a mean +// partition, and the largest partition, which bounds a cold build, stays +// near the mean unless one root is larger. Roots and their bytes, and so +// placement, depend only on the module text. +void balanceRoots(std::vector &Symbols, ArrayRef Roots, + ArrayRef Live, unsigned Count) { + if (Count >= MinGroupedPartitions) + return; + DenseMap RootBytes; + uint64_t Total = 0; + for (unsigned I = 0; I < Symbols.size(); ++I) + if (Symbols[I].Kind == SymbolKind::Function && Live[I]) + RootBytes[Roots[I]] += Symbols[I].Header.size() + Symbols[I].Body.size(); + // Each root counts in whole 4 KiB units, so that an edit that grows a root + // by less than that leaves every comparison, and every other root's + // partition, unchanged. + constexpr uint64_t Unit = 4096; + for (auto &Entry : RootBytes) { + Entry.second = (Entry.second + Unit - 1) / Unit * Unit; + Total += Entry.second; + } + std::vector Order; + for (auto &Entry : RootBytes) + Order.push_back(Entry.first); + llvm::sort(Order, [&](unsigned A, unsigned B) { + return Symbols[A].Name < Symbols[B].Name; + }); + uint64_t Cap = Total * 5 / 4 / Count; + std::vector Load(Count, 0); + DenseMap Placed; + // Roots of more than half the cap are placed first, in name order, so + // that small roots fill around them; the others follow in name order. + auto Place = [&](unsigned Root) { + uint64_t Bytes = RootBytes[Root]; + unsigned Home = stableHash(Symbols[Root].Name) % Count, Chosen = Home; + for (unsigned K = 0; K < Count; ++K) { + unsigned P = (Home + K) % Count; + if (Load[P] + Bytes <= Cap) { + Chosen = P; + break; + } + } + Load[Chosen] += Bytes; + Placed[Root] = Chosen; + }; + for (unsigned Root : Order) + if (RootBytes[Root] * 2 > Cap) + Place(Root); + for (unsigned Root : Order) + if (RootBytes[Root] * 2 <= Cap) + Place(Root); + for (unsigned I = 0; I < Symbols.size(); ++I) + if (Symbols[I].Kind == SymbolKind::Function && Live[I]) + Symbols[I].Partition = Placed[Roots[I]]; +} + bool startsWithLinkage(StringRef Rest) { for (StringRef Linkage : {"linkonce", "weak", "common", "appending", "extern_weak", @@ -278,6 +345,7 @@ bool parseGlobal(StringRef Line, Symbol &S) { } if (Cursor.consume_front("constant ")) { Keyword = "constant"; + S.Constant = true; break; } size_t Space = Cursor.find(' '); @@ -325,10 +393,13 @@ bool parseGlobal(StringRef Line, Symbol &S) { } // Collects the symbols (sorted) and metadata nodes (in order of first -// appearance) that Text references outside quoted strings. +// appearance) that Text references outside quoted strings, and in Calls the +// symbols it names as direct callees, which the emitter writes followed by +// "(". void scanReferences(StringRef Text, const StringMap &Index, std::vector &Refs, - std::vector &Metadata) { + std::vector &Metadata, + std::vector *Calls = nullptr) { DenseSet Seen(Metadata.begin(), Metadata.end()); bool Quoted = false; for (size_t I = 0; I < Text.size(); ++I) { @@ -343,8 +414,12 @@ void scanReferences(StringRef Text, const StringMap &Index, size_t Length = nameLength(Text, I + 1); if (Length != 0) { auto It = Index.find(Text.substr(I + 1, Length)); - if (It != Index.end()) + if (It != Index.end()) { Refs.push_back(It->second); + if (Calls && I + 1 + Length < Text.size() && + Text[I + 1 + Length] == '(') + Calls->push_back(It->second); + } I += Length; } } else if (C == '!' && I + 1 < Text.size() && isDigit(Text[I + 1])) { @@ -359,6 +434,10 @@ void scanReferences(StringRef Text, const StringMap &Index, } llvm::sort(Refs); Refs.erase(std::unique(Refs.begin(), Refs.end()), Refs.end()); + if (Calls) { + llvm::sort(*Calls); + Calls->erase(std::unique(Calls->begin(), Calls->end()), Calls->end()); + } } // Appends Text with every unquoted metadata reference renumbered by Map. @@ -392,8 +471,18 @@ void appendRenumbered(StringRef Text, const DenseMap &Map, } // namespace +bool dominatedPartitions(ArrayRef FunctionBytes) { + uint64_t Total = 0, Largest = 0; + for (uint64_t Bytes : FunctionBytes) { + Total += Bytes; + Largest = std::max(Largest, Bytes); + } + return Largest * 3 > Total * 2; +} + std::optional partitionProgramText(StringRef Text, - unsigned Count) { + unsigned Count, + bool SkipDominated) { if (Count < 2) return std::nullopt; std::vector Header, Types, Attributes, NamedMetadata; @@ -529,7 +618,7 @@ std::optional partitionProgramText(StringRef Text, scanReferences(S.Header, Index, S.Refs, S.Metadata); if (!S.Body.empty()) { std::vector Refs; - scanReferences(S.Body, Index, Refs, S.Metadata); + scanReferences(S.Body, Index, Refs, S.Metadata, &S.Calls); S.Refs.insert(S.Refs.end(), Refs.begin(), Refs.end()); llvm::sort(S.Refs); S.Refs.erase(std::unique(S.Refs.begin(), S.Refs.end()), S.Refs.end()); @@ -584,15 +673,36 @@ std::optional partitionProgramText(StringRef Text, // inlining candidate of it, or data only it uses; keep it with that // definition so its partition can inline it without importing, and so a // helper an edit adds lands in the partition of the code that uses it. A - // chain through a global continues to the function that uses the global. + // function that its referrer only takes the address of is not an inlining + // candidate: a closure body whose address its creator passes to the + // runtime, or a compiled regular expression's matcher that a mutable + // global points to, gives the referrer nothing to inline, and joining it + // would make every edit of the referrer optimize it again. Such a function + // is placed by its own name, unless its body is under AddressOnlyBytes (a + // trampoline or thread entry, which costs little to optimize again). A + // chain through a constant global, whose loads the optimizer can turn into + // direct calls, continues to the function that uses the global. // A function body larger than this is past LLVM's inline cost limit even // for the only call to a local function, so it is placed by its own name. - // A chain stops where it would leave a function's source-module group: - // module initializers, which only the entry point calls, keep their own - // partitions instead of all sharing the entry point's. + // A chain stops where it would leave a function's source-module group for + // a function outside every group. In a module of more than + // GroupedChainBytes of LLVM text, it also stops where it would join a + // function of another group: module initializers, which only the entry + // point calls, keep their own partitions instead of all sharing the entry + // point's. In a smaller module, a function that one function of another + // source file calls stays with it, so it can still be inlined there as in + // the whole module; when that gathers most of the program into one + // partition, the program is optimized whole (see emitPartitions). constexpr size_t MaxInlinedBodyBytes = 256 * 1024; + constexpr size_t GroupedChainBytes = 1024 * 1024; + constexpr size_t AddressOnlyBytes = 512; + bool GroupedChains = Text.size() > GroupedChainBytes; constexpr unsigned Shared = ~1u; std::vector Referrer(Symbols.size(), NoPartition); + // Whether a definition may follow its first referrer: it is not a + // function, or the referrer calls it or is a constant global, or its body + // is under AddressOnlyBytes. + std::vector MayFollow(Symbols.size(), 0); for (unsigned I = 0; I < Symbols.size(); ++I) { const Symbol &S = Symbols[I]; if (S.Kind == SymbolKind::Declaration || !Live[I]) @@ -600,18 +710,29 @@ std::optional partitionProgramText(StringRef Text, for (unsigned Ref : S.Refs) { if (Ref == I) continue; - Referrer[Ref] = Referrer[Ref] == NoPartition ? I : Shared; + if (Referrer[Ref] == NoPartition) { + Referrer[Ref] = I; + MayFollow[Ref] = + Symbols[Ref].Kind != SymbolKind::Function || + Symbols[Ref].Body.size() < AddressOnlyBytes || + (S.Kind == SymbolKind::Function + ? std::binary_search(S.Calls.begin(), S.Calls.end(), Ref) + : S.Constant); + } else { + Referrer[Ref] = Shared; + } } } auto Follows = [&](unsigned Child, unsigned Parent) { const Symbol &S = Symbols[Child]; - if (!S.Local || Parent == NoPartition || Parent == Shared || - Symbols[Parent].Kind == SymbolKind::Declaration) + if (!S.Local || !MayFollow[Child] || Parent == NoPartition || + Parent == Shared || Symbols[Parent].Kind == SymbolKind::Declaration) return false; if (S.Kind == SymbolKind::Function && (S.Body.size() > MaxInlinedBodyBytes || (Symbols[Parent].Kind == SymbolKind::Function && !S.Group.empty() && - S.Group != Symbols[Parent].Group))) + S.Group != Symbols[Parent].Group && + (GroupedChains || Symbols[Parent].Group.empty())))) return false; return true; }; @@ -637,6 +758,7 @@ std::optional partitionProgramText(StringRef Text, Symbols[I].Partition = stableHash(Symbols[Root].Name) % Count; } placeGroups(Symbols, Roots, Count); + balanceRoots(Symbols, Roots, Live, Count); // A global follows the function its chain ends at, then the first // function that references it, then its own name. for (unsigned I = 0; I < Symbols.size(); ++I) @@ -663,10 +785,41 @@ std::optional partitionProgramText(StringRef Text, Result.PublicSymbols.insert(S.Name); Result.Sources.resize(Count); + Result.FunctionBytes.assign(Count, 0); std::vector> Owned(Count); for (unsigned I = 0; I < Symbols.size(); ++I) - if (Symbols[I].Kind != SymbolKind::Declaration && Live[I]) + if (Symbols[I].Kind != SymbolKind::Declaration && Live[I]) { Owned[Symbols[I].Partition].push_back(I); + if (Symbols[I].Kind == SymbolKind::Function) + Result.FunctionBytes[Symbols[I].Partition] += + Symbols[I].Header.size() + Symbols[I].Body.size(); + } + // The caller optimizes such a module whole, so its partitions' text is + // not needed. + if (SkipDominated && dominatedPartitions(Result.FunctionBytes)) { + Result.Sources.clear(); + return Result; + } + + // An internal definition stays internal, so its partition's optimizer can + // inline a function into its last caller and delete it as the whole module + // would. It is promoted to hidden visibility only when a definition of + // another partition or a metadata node names it. + std::vector Exported(Symbols.size(), 0); + for (unsigned I : NamedSymbols) + Exported[I] = 1; + for (auto &Entry : Metadata) + for (unsigned I : Entry.second.Symbols) + Exported[I] = 1; + for (unsigned I = 0; I < Symbols.size(); ++I) { + const Symbol &S = Symbols[I]; + if (S.Kind == SymbolKind::Declaration || !Live[I]) + continue; + for (unsigned Ref : S.Refs) + if (Symbols[Ref].Kind != SymbolKind::Declaration && + Symbols[Ref].Partition != S.Partition) + Exported[Ref] = 1; + } auto Assemble = [&](unsigned P) { std::vector Needed(Symbols.size(), 0); @@ -731,7 +884,7 @@ std::optional partitionProgramText(StringRef Text, const Symbol &S = Symbols[I]; if (S.Kind != Kind) continue; - Line(S.Local ? StringRef(S.PromotedHeader) : S.Header); + Line(S.Local && Exported[I] ? StringRef(S.PromotedHeader) : S.Header); if (!S.Body.empty()) Line(S.Body); } diff --git a/native/llvm-codegen/src/partition.h b/native/llvm-codegen/src/partition.h index dc000d751..0756469e8 100644 --- a/native/llvm-codegen/src/partition.h +++ b/native/llvm-codegen/src/partition.h @@ -1,8 +1,10 @@ #pragma once +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" +#include #include #include #include @@ -14,9 +16,11 @@ struct ProgramPartitions { // One complete LLVM assembly module per partition. std::vector Sources; // Definitions that were not internal in the original module. Every other - // definition was internal and is promoted to hidden visibility only so the - // partitions can reference each other. + // definition was internal, and is promoted to hidden visibility only when + // another partition references it. llvm::StringSet<> PublicSymbols; + // The bytes of the function definitions placed in each partition. + std::vector FunctionBytes; }; // Divides the LLVM assembly scriptc emits into Count modules without parsing @@ -27,13 +31,21 @@ struct ProgramPartitions { // functions are placed by a stable hash of their name. An internal // definition referenced by exactly one other definition stays with it, also // through a global, unless it is a function whose body is too large to -// inline or that would join a function of another source-module group; a -// global otherwise stays with the first function that references it. +// inline, that would leave its source-module group for a function outside +// every group, or, in a module of more than 1 MiB, that would join a function +// of another source-module group; a global otherwise stays with the first +// function that references it. // Internal definitions that no non-internal definition, named metadata or // metadata node reaches are left out of every partition. Returns nullopt // for any construct outside the emitter's narrow top-level grammar; callers -// then split a parsed module. +// then split a parsed module. With SkipDominated, a division that +// dominatedPartitions rejects has no Sources. std::optional partitionProgramText(llvm::StringRef Text, - unsigned Count); + unsigned Count, + bool SkipDominated); + +// Whether the largest partition would hold more than two thirds of the +// program's function bytes. +bool dominatedPartitions(llvm::ArrayRef FunctionBytes); } // namespace scriptc diff --git a/native/llvm-codegen/src/profile-blocks.cpp b/native/llvm-codegen/src/profile-blocks.cpp new file mode 100644 index 000000000..4c94577a3 --- /dev/null +++ b/native/llvm-codegen/src/profile-blocks.cpp @@ -0,0 +1,503 @@ +#include "profile-blocks.h" + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/ProfDataUtils.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" +#include "llvm/Transforms/Utils/SampleProfileInference.h" + +#include + +using namespace llvm; + +namespace scriptc { + +namespace { + +constexpr StringRef Header = "scriptc-profile-blocks 1"; + +// An edge whose branch weight is below this fraction of its branch's total +// counts as unlikely in flow inference. +constexpr uint64_t UnlikelyRatio = 1000; + +// Alignments of more blocks than this product fall back to matching labels +// that occur once on each side. +constexpr uint64_t MaxAlignmentCells = uint64_t(1) << 22; + +void writeEscaped(raw_ostream &OS, StringRef Text) { + if (Text.empty()) { + OS << '\\'; + return; + } + for (unsigned char C : Text) { + if (C <= ' ' || C >= 0x7f || C == '\\') + OS << '\\' << hexdigit(C >> 4, /*LowerCase=*/true) + << hexdigit(C & 15, /*LowerCase=*/true); + else + OS << C; + } +} + +bool readEscaped(StringRef Token, std::string &Out) { + Out.clear(); + if (Token == "\\") + return true; + for (size_t I = 0; I < Token.size(); ++I) { + if (Token[I] != '\\') { + Out.push_back(Token[I]); + continue; + } + if (I + 2 >= Token.size()) + return false; + unsigned High = hexDigitValue(Token[I + 1]); + unsigned Low = hexDigitValue(Token[I + 2]); + if (High > 15 || Low > 15) + return false; + Out.push_back(static_cast(High << 4 | Low)); + I += 2; + } + return true; +} + +// A block label's kind: the label without its digits. The emitter numbers +// each function's blocks in order (`if.t3`, `loop.c4`), so a block added or +// removed earlier in the function renumbers the blocks after it but keeps +// their kinds. +std::string kindOf(StringRef Label) { + std::string Kind; + Kind.reserve(Label.size()); + for (char C : Label) + if (!isDigit(C)) + Kind.push_back(C); + return Kind; +} + +struct Token { + std::string Kind; + StringRef Label; + size_t Successors; +}; + +bool sameKind(const Token &A, const Token &B) { + return A.Successors == B.Successors && A.Kind == B.Kind; +} + +// Pairs every block of Blocks with the recorded block that holds its place, +// or -1: the longest common subsequence of the two block orders by kind and +// successor count, which prefers pairs whose labels are equal. Common +// leading and trailing runs pair up first; a remaining middle too large to +// align pairs the labels that occur once on each side. +std::vector alignBlocks(const FunctionBlocks &Record, + ArrayRef Blocks) { + std::vector Old, New; + Old.reserve(Record.Blocks.size()); + for (const BlockRecord &Block : Record.Blocks) + Old.push_back({kindOf(Block.Label), Block.Label, Block.Successors.size()}); + New.reserve(Blocks.size()); + for (const BasicBlock *Block : Blocks) { + SmallPtrSet Distinct(succ_begin(Block), + succ_end(Block)); + New.push_back( + {kindOf(Block->getName()), Block->getName(), Distinct.size()}); + } + std::vector Match(New.size(), -1); + size_t Begin = 0; + while (Begin < Old.size() && Begin < New.size() && + sameKind(Old[Begin], New[Begin])) { + Match[Begin] = static_cast(Begin); + ++Begin; + } + size_t OldEnd = Old.size(), NewEnd = New.size(); + while (OldEnd > Begin && NewEnd > Begin && + sameKind(Old[OldEnd - 1], New[NewEnd - 1])) { + --OldEnd; + --NewEnd; + Match[NewEnd] = static_cast(OldEnd); + } + size_t Rows = OldEnd - Begin, Columns = NewEnd - Begin; + if (Rows == 0 || Columns == 0) + return Match; + auto Score = [&](size_t I, size_t J) -> uint32_t { + const Token &A = Old[Begin + I], &B = New[Begin + J]; + if (!sameKind(A, B)) + return 0; + return A.Label == B.Label ? 2 : 1; + }; + if (uint64_t(Rows) * Columns <= MaxAlignmentCells) { + // Best[I][J]: the best score of the middle's suffixes from I and J. + size_t Width = Columns + 1; + std::vector Best((Rows + 1) * Width, 0); + for (size_t I = Rows; I-- > 0;) + for (size_t J = Columns; J-- > 0;) { + uint32_t Value = + std::max(Best[(I + 1) * Width + J], Best[I * Width + J + 1]); + if (uint32_t S = Score(I, J)) + Value = std::max(Value, S + Best[(I + 1) * Width + J + 1]); + Best[I * Width + J] = Value; + } + size_t I = 0, J = 0; + while (I < Rows && J < Columns) { + uint32_t S = Score(I, J); + if (S && Best[I * Width + J] == S + Best[(I + 1) * Width + J + 1]) { + Match[Begin + J] = static_cast(Begin + I); + ++I; + ++J; + } else if (Best[I * Width + J] == Best[(I + 1) * Width + J]) { + ++I; + } else { + ++J; + } + } + return Match; + } + StringMap OldLabels, NewLabels; + for (size_t I = Begin; I < OldEnd; ++I) { + auto [It, Inserted] = OldLabels.try_emplace(Old[I].Label, I); + if (!Inserted) + It->second = -1; + } + for (size_t J = Begin; J < NewEnd; ++J) { + auto [It, Inserted] = NewLabels.try_emplace(New[J].Label, J); + if (!Inserted) + It->second = -1; + } + for (const auto &Entry : NewLabels) { + auto It = OldLabels.find(Entry.getKey()); + if (Entry.second < 0 || It == OldLabels.end() || It->second < 0 || + !sameKind(Old[It->second], New[Entry.second])) + continue; + Match[Entry.second] = It->second; + } + return Match; +} + +// LLVM's profile inference costs, with recorded edge counts that cost as much +// to change as recorded block counts. Its default leaves edges free to +// change, and its rebalancing of the flow below a counted block spreads it +// evenly over the block's edges whatever their counts. +ProfiParams inferenceParams() { + ProfiParams Params; + Params.EvenFlowDistribution = true; + Params.RebalanceUnknown = false; + Params.JoinIslands = true; + Params.CostBlockInc = 10; + Params.CostBlockDec = 20; + Params.CostBlockEntryInc = 40; + Params.CostBlockEntryDec = 10; + Params.CostBlockZeroInc = 11; + Params.CostBlockUnknownInc = 0; + Params.CostJumpInc = 10; + Params.CostJumpFTInc = 10; + Params.CostJumpDec = 20; + Params.CostJumpFTDec = 20; + Params.CostJumpUnknownInc = 0; + Params.CostJumpUnknownFTInc = 0; + return Params; +} + +void markFrequency(Function &F, uint64_t EntryCount, uint64_t MaxCount, + const ProfileSummaryInfo &Summary) { + if (Summary.isHotCount(EntryCount)) + F.addFnAttr(Attribute::InlineHint); + else if (Summary.isColdCount(MaxCount) && !F.hasFnAttribute(Attribute::Hot)) + F.addFnAttr(Attribute::Cold); +} + +} // namespace + +std::unique_ptr readProfileBlocks(StringRef Path, + std::string &Error) { + ErrorOr> Buffer = MemoryBuffer::getFile(Path); + if (!Buffer) { + Error = Path.str() + ": " + Buffer.getError().message(); + return nullptr; + } + auto Blocks = std::make_unique(); + StringRef Rest = (*Buffer)->getBuffer(); + size_t Line = 0; + auto Fail = [&](StringRef Why) { + Error = (Path + ":" + Twine(Line) + ": " + Why).str(); + return nullptr; + }; + auto NextLine = [&]() { + auto [Current, Remaining] = Rest.split('\n'); + Rest = Remaining; + ++Line; + return Current; + }; + if (NextLine() != Header) + return Fail("not a scriptc profile blocks file"); + SmallVector Fields; + std::string Name, Label; + while (!Rest.empty()) { + Fields.clear(); + NextLine().split(Fields, ' ', -1, /*KeepEmpty=*/false); + uint64_t Entry = 0, Count = 0; + if (Fields.size() != 4 || Fields[0] != "f" || + !readEscaped(Fields[1], Name) || Fields[2].getAsInteger(10, Entry) || + Fields[3].getAsInteger(10, Count)) + return Fail("expected a function line"); + FunctionBlocks &Function = (*Blocks)[Name]; + Function.EntryCount = Entry; + Function.Blocks.resize(Count); + for (BlockRecord &Block : Function.Blocks) { + Fields.clear(); + if (Rest.empty()) + return Fail("expected a block line"); + NextLine().split(Fields, ' ', -1, /*KeepEmpty=*/false); + if (Fields.size() < 2 || !readEscaped(Fields[0], Label) || + Fields[1].getAsInteger(10, Block.Count)) + return Fail("expected a block line"); + Block.Label = Label; + for (StringRef Field : ArrayRef(Fields).drop_front(2)) { + auto [Position, EdgeCount] = Field.split(':'); + uint32_t Successor = 0; + uint64_t Value = 0; + if (Position.getAsInteger(10, Successor) || + EdgeCount.getAsInteger(10, Value) || Successor >= Count) + return Fail("expected a successor and its edge count"); + Block.Successors.push_back({Successor, Value}); + } + } + } + return Blocks; +} + +void annotateFromBlocks(Function &F, const FunctionBlocks &Record, + const ProfileSummaryInfo &Summary) { + if (Record.EntryCount == 0 || Record.Blocks.empty()) { + F.setEntryCount(Function::ProfileCount(0, Function::PCT_Real)); + if (Summary.hasProfileSummary() && !F.hasFnAttribute(Attribute::Hot)) + F.addFnAttr(Attribute::Cold); + return; + } + std::vector Order; + for (const BasicBlock &Block : F) + Order.push_back(&Block); + std::vector Match = alignBlocks(Record, Order); + std::vector Paired(Record.Blocks.size(), false); + for (int64_t Recorded : Match) + if (Recorded >= 0) + Paired[Recorded] = true; + + // Flow runs through the blocks that the entry reaches and that reach an + // exit, in the function's order, as LLVM's profile inference requires. + df_iterator_default_set Forward, Backward; + for (const BasicBlock *Block : depth_first_ext(&F, Forward)) + (void)Block; + for (const BasicBlock &Block : F) + if (succ_empty(&Block)) + for (const BasicBlock *Reaching : + inverse_depth_first_ext(&Block, Backward)) + (void)Reaching; + DenseMap Position, Node; + FlowFunction Flow; + for (size_t I = 0; I < Order.size(); ++I) { + Position[Order[I]] = I; + if (!Forward.contains(Order[I]) || !Backward.contains(Order[I])) + continue; + Node[Order[I]] = Flow.Blocks.size(); + FlowBlock Block; + Block.Index = Flow.Blocks.size(); + if (Match[I] >= 0) { + Block.Weight = Record.Blocks[Match[I]].Count; + Block.HasUnknownWeight = false; + } else if (I == 0) { + Block.Weight = Record.EntryCount; + Block.HasUnknownWeight = false; + } + Flow.Blocks.push_back(Block); + } + uint64_t EntryCount = Record.EntryCount, MaxCount = Record.EntryCount; + if (Flow.Blocks.size() > 1 && Node.count(&F.getEntryBlock())) { + for (const BasicBlock *Block : Order) { + auto Source = Node.find(Block); + if (Source == Node.end()) + continue; + int64_t From = Match[Position[Block]]; + SmallVector Targets, Successors; + for (const BasicBlock *Successor : successors(Block)) { + Successors.push_back(Successor); + if (!is_contained(Targets, Successor)) + Targets.push_back(Successor); + } + SmallVector Weights; + extractBranchWeights(*Block->getTerminator(), Weights); + const BlockRecord *Recorded = From >= 0 ? &Record.Blocks[From] : nullptr; + for (size_t K = 0; K < Targets.size(); ++K) { + auto Target = Node.find(Targets[K]); + if (Target == Node.end()) + continue; + FlowJump Jump; + Jump.Source = Source->second; + Jump.Target = Target->second; + if (Recorded != nullptr) { + // An edge to the recorded target's pair takes that edge's count. + // An edge to an unpaired target takes the count of the recorded + // edge in its place among the block's targets, which a paired + // block has as many of, when that edge's target is unpaired too: + // a target whose label changed keeps its edge's count. + int64_t To = Match[Position[Targets[K]]]; + auto Edge = llvm::find_if(Recorded->Successors, [&](const auto &E) { + return To >= 0 && E.first == static_cast(To); + }); + if (Edge == Recorded->Successors.end() && To < 0 && + Recorded->Successors.size() == Targets.size() && + !Paired[Recorded->Successors[K].first]) + Edge = Recorded->Successors.begin() + K; + if (Edge != Recorded->Successors.end()) { + Jump.Weight = Edge->second; + Jump.HasUnknownWeight = false; + } + } + // An edge without a recorded count that the function's own branch + // weights (from llvm.expect) make unlikely takes flow only when no + // other path can. + if (Jump.HasUnknownWeight && Weights.size() == Successors.size()) { + uint64_t Total = 0, Weight = 0; + for (size_t I = 0; I < Successors.size(); ++I) { + Total += Weights[I]; + if (Successors[I] == Targets[K]) + Weight += Weights[I]; + } + Jump.IsUnlikely = Weight * UnlikelyRatio < Total; + } + Flow.Jumps.push_back(Jump); + } + } + for (FlowJump &Jump : Flow.Jumps) { + Flow.Blocks[Jump.Source].SuccJumps.push_back(&Jump); + Flow.Blocks[Jump.Target].PredJumps.push_back(&Jump); + } + applyFlowInference(inferenceParams(), Flow); + EntryCount = Flow.Blocks[0].Flow; + MaxCount = 0; + for (const FlowBlock &Block : Flow.Blocks) + MaxCount = std::max(MaxCount, Block.Flow); + if (MaxCount > 0 && EntryCount == 0) + EntryCount = 1; + // Branch weights as the profile annotation sets them: the first + // successor position of each target takes the edge's count. + DenseMap, uint64_t> JumpFlow; + for (const FlowJump &Jump : Flow.Jumps) + JumpFlow[{Jump.Source, Jump.Target}] = Jump.Flow; + for (BasicBlock &Block : F) { + auto Source = Node.find(&Block); + Instruction *Terminator = Block.getTerminator(); + if (Source == Node.end() || Flow.Blocks[Source->second].Flow == 0 || + Terminator->getNumSuccessors() < 2 || + !(isa(Terminator) || isa(Terminator) || + isa(Terminator) || isa(Terminator) || + isa(Terminator))) + continue; + SmallVector Counts(Terminator->getNumSuccessors(), 0); + SmallPtrSet Seen; + uint64_t Max = 0; + for (unsigned I = 0; I < Terminator->getNumSuccessors(); ++I) { + const BasicBlock *Successor = Terminator->getSuccessor(I); + auto Target = Node.find(Successor); + if (Target == Node.end() || !Seen.insert(Successor).second) + continue; + Counts[I] = JumpFlow.lookup({Source->second, Target->second}); + Max = std::max(Max, Counts[I]); + } + if (Max > 0) + setProfMetadata(Terminator, Counts, Max); + } + } + F.setEntryCount(Function::ProfileCount(EntryCount, Function::PCT_Real)); + markFrequency(F, EntryCount, MaxCount, Summary); +} + +void BlockRecorder::add(const Function &F) { + std::optional Entry = F.getEntryCount(); + if (!Entry || Entry->isSynthetic()) + return; + std::string Text; + raw_string_ostream OS(Text); + OS << "f "; + writeEscaped(OS, F.getName()); + if (Entry->getCount() == 0) { + OS << " 0 0\n"; + } else { + OS << ' ' << Entry->getCount() << ' ' << F.size() << '\n'; + Function &Mutable = const_cast(F); + DominatorTree Dominators(Mutable); + LoopInfo Loops(Dominators); + BranchProbabilityInfo Probabilities(F, Loops); + BlockFrequencyInfo Frequencies(F, Probabilities, Loops); + DenseMap Position; + uint32_t Next = 0; + for (const BasicBlock &Block : F) + Position[&Block] = Next++; + for (const BasicBlock &Block : F) { + uint64_t Count = Frequencies.getBlockProfileCount(&Block).value_or(0); + writeEscaped(OS, Block.getName()); + OS << ' ' << Count; + // Edges to one target through several successor positions add up. + SmallVector, 4> Edges; + const Instruction *Terminator = Block.getTerminator(); + for (unsigned I = 0, E = Terminator->getNumSuccessors(); I < E; ++I) { + uint32_t Target = Position.lookup(Terminator->getSuccessor(I)); + uint64_t EdgeCount = + Probabilities.getEdgeProbability(&Block, I).scale(Count); + auto It = llvm::find_if(Edges, [&](const auto &Edge) { + return Edge.first == Target; + }); + if (It == Edges.end()) + Edges.push_back({Target, EdgeCount}); + else + It->second += EdgeCount; + } + for (auto [Target, EdgeCount] : Edges) + OS << ' ' << Target << ':' << EdgeCount; + OS << '\n'; + } + } + std::lock_guard Guard(Lock); + Records[F.getName().str()] = std::move(Text); +} + +std::optional BlockRecorder::write(StringRef Path) { + SmallString<256> Temporary(Path); + Temporary += ".tmp-%%%%%%"; + int Fd = -1; + if (std::error_code EC = sys::fs::createUniqueFile(Temporary, Fd, Temporary)) + return EC.message(); + { + raw_fd_ostream Stream(Fd, /*shouldClose=*/true); + Stream << Header << '\n'; + std::lock_guard Guard(Lock); + for (const auto &[Name, Text] : Records) + Stream << Text; + Stream.close(); + if (Stream.has_error()) { + std::string Message = Stream.error().message(); + Stream.clear_error(); + sys::fs::remove(Temporary); + return Message; + } + } + if (std::error_code EC = sys::fs::rename(Temporary, Path)) { + sys::fs::remove(Temporary); + return EC.message(); + } + return std::nullopt; +} + +} // namespace scriptc diff --git a/native/llvm-codegen/src/profile-blocks.h b/native/llvm-codegen/src/profile-blocks.h new file mode 100644 index 000000000..1db71b53e --- /dev/null +++ b/native/llvm-codegen/src/profile-blocks.h @@ -0,0 +1,83 @@ +#pragma once + +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace llvm { +class Function; +class ProfileSummaryInfo; +} // namespace llvm + +namespace scriptc { + +// A profile's block counts (a .blocks file), which annotate the functions +// whose control flow changed since the profile was collected. The profile +// annotation matches a function only when its control flow hashes as it did +// then; the block counts describe each function by its blocks' labels, which +// an unrelated change to the function leaves mostly as they were. +// +// The file is text. Its first line is `scriptc-profile-blocks 1`; each +// function follows as a line `f ` and then one +// line per block in the function's order: `