diff --git a/.github/workflows/_memwaste.yml b/.github/workflows/_memwaste.yml new file mode 100644 index 000000000..174490db6 --- /dev/null +++ b/.github/workflows/_memwaste.yml @@ -0,0 +1,183 @@ +# Reusable: the waste sanitizer (memory + CPU waste), REPORT-ONLY. +# +# What runs (scripts/memwaste.sh, memwaste-scaling.py, memwaste-ratchet.py): +# mode=pr Linux: the event lane (memory waste + CPU work) indexing this +# repository at the checked-out commit, and the CPU scaling lane +# (src/ replicated 2x against 4x, per-function counter ratios). +# mode=full the same, plus the event lane on macOS and Windows and the access +# lane (read/write verdicts per allocation) on Linux. +# +# The corpus is the repository itself: pinned by the commit, no network, and a +# real mix of C, C++, Python, JavaScript and vendored code. +# +# REPORT-ONLY: every job uploads its report and prints the ratchet comparison +# against scripts/memwaste-baseline.json, but nothing here can turn a PR red -- +# the ratchet runs without --enforce and the PR's ci-ok gate does not depend on +# these jobs. A job fails only when the MEASUREMENT is unsound (a table +# overflowed, frees the layer never saw): that is a defect of the tool, and a +# red job is how it gets noticed. Switching the ratchet to --enforce is a +# separate, deliberate change once the numbers have been stable across reruns. +name: Memwaste + +on: + workflow_call: + inputs: + mode: + description: "pr (Linux event + scaling) or full (all platforms, access lane)" + type: string + default: pr + +permissions: + contents: read + +jobs: + event-linux: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm + - name: Event lane on this repository + run: | + # `cmd | tee` reports TEE's status: without this a failing lane + # (a missing exec bit, a build error) passes the step and only + # surfaces as the ratchet not finding its JSON. + set -o pipefail + mkdir -p "$RUNNER_TEMP/memwaste-event" + scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \ + | tee "$RUNNER_TEMP/memwaste-event/report.txt" + - name: Ratchet (report-only) + run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue ${{ inputs.mode }}-linux + - name: Upload report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: memwaste-event-linux-${{ inputs.mode }} + path: | + ${{ runner.temp }}/memwaste-event/report.txt + ${{ runner.temp }}/memwaste-event/memwaste.json + ${{ runner.temp }}/memwaste-event/index.json + if-no-files-found: warn + + scaling-linux: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm + - name: CPU scaling lane (src/ at 2x against 4x) + # Report-only: the script exits 1 when it flags a super-linear function. + continue-on-error: true + run: | + # `cmd | tee` reports TEE's status: without this a failing lane + # (a missing exec bit, a build error) passes the step and only + # surfaces as the ratchet not finding its JSON. + set -o pipefail + mkdir -p "$RUNNER_TEMP/memwaste-scaling" + python3 scripts/memwaste-scaling.py --corpus src --replicas 2 \ + --out "$RUNNER_TEMP/memwaste-scaling" | tee "$RUNNER_TEMP/memwaste-scaling/report.txt" + - name: Upload report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: memwaste-scaling-linux-${{ inputs.mode }} + path: ${{ runner.temp }}/memwaste-scaling/report.txt + if-no-files-found: warn + + access-linux: + if: ${{ inputs.mode == 'full' }} + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm + - name: Access lane on this repository + run: | + # `cmd | tee` reports TEE's status: without this a failing lane + # (a missing exec bit, a build error) passes the step and only + # surfaces as the ratchet not finding its JSON. + set -o pipefail + mkdir -p "$RUNNER_TEMP/memwaste-access" + scripts/memwaste.sh "$GITHUB_WORKSPACE" --lane access --top 40 --sort untouched \ + --out "$RUNNER_TEMP/memwaste-access" | tee "$RUNNER_TEMP/memwaste-access/report.txt" + - name: Upload report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: memwaste-access-linux-${{ inputs.mode }} + path: | + ${{ runner.temp }}/memwaste-access/report.txt + ${{ runner.temp }}/memwaste-access/memwaste.json + if-no-files-found: warn + + event-macos: + if: ${{ inputs.mode == 'full' }} + runs-on: macos-14 + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Event lane on this repository + run: | + # `cmd | tee` reports TEE's status: without this a failing lane + # (a missing exec bit, a build error) passes the step and only + # surfaces as the ratchet not finding its JSON. + set -o pipefail + mkdir -p "$RUNNER_TEMP/memwaste-event" + scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \ + | tee "$RUNNER_TEMP/memwaste-event/report.txt" + - name: Ratchet (report-only) + run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue full-macos + - name: Upload report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: memwaste-event-macos-${{ inputs.mode }} + path: | + ${{ runner.temp }}/memwaste-event/report.txt + ${{ runner.temp }}/memwaste-event/memwaste.json + if-no-files-found: warn + + event-windows: + if: ${{ inputs.mode == 'full' }} + runs-on: windows-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2 + with: + msystem: CLANG64 + path-type: inherit + install: >- + mingw-w64-clang-x86_64-clang + mingw-w64-clang-x86_64-llvm + mingw-w64-clang-x86_64-zlib + mingw-w64-clang-x86_64-python3 + make + git + coreutils + - name: Event lane on this repository + shell: msys2 {0} + run: | + # `cmd | tee` reports TEE's status: without this a failing lane + # (a missing exec bit, a build error) passes the step and only + # surfaces as the ratchet not finding its JSON. + set -o pipefail + out="$(cygpath -u "$RUNNER_TEMP")/memwaste-event" + mkdir -p "$out" + CC=clang CXX=clang++ scripts/memwaste.sh "$(cygpath -u "$GITHUB_WORKSPACE")" --top 40 --out "$out" \ + | tee "$out/report.txt" + - name: Ratchet (report-only) + shell: msys2 {0} + run: python3 scripts/memwaste-ratchet.py --out "$(cygpath -u "$RUNNER_TEMP")/memwaste-event" --venue full-windows + - name: Upload report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: memwaste-event-windows-${{ inputs.mode }} + path: | + ${{ runner.temp }}/memwaste-event/report.txt + ${{ runner.temp }}/memwaste-event/memwaste.json + if-no-files-found: warn diff --git a/.github/workflows/_security.yml b/.github/workflows/_security.yml index dd6f3400c..ce044eed2 100644 --- a/.github/workflows/_security.yml +++ b/.github/workflows/_security.yml @@ -41,6 +41,14 @@ jobs: codeql-gate: runs-on: ubuntu-latest timeout-minutes: 240 + # Without security-events:read this job inherits the workflow's + # contents:read, the code-scanning alert API answers 403, and the gate + # below reported "0 alerts" on every run it has ever made. actions:read is + # what the CodeQL-run poll above needs for the same reason. + permissions: + contents: read + security-events: read + actions: read steps: - name: Wait for CodeQL on current commit (max 150 min) env: @@ -77,11 +85,43 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + # FAIL CLOSED. The previous form was `gh api ... 2>/dev/null || echo "0"`, + # which looks defensive and is the opposite. gh writes the API error + # BODY to stdout, so a 403 made ALERTS the string + # {"message":"Resource not accessible by integration",...,"status":"403"}0 + # every `[` comparison then failed as non-integer, the `if` took its + # false branch, and the step printed "CodeQL gate passed (0 alerts)" + # and exited 0. That is exactly how a security gate should NEVER fail: + # silently, in the direction of "everything is fine". An unreadable + # alert list is now an error, not a zero. + # NOTE: this runs in a command substitution, so it must `return`, not + # `exit` -- an `exit` here would only end the subshell and let the + # caller continue with an empty count, recreating the very bug this + # replaces. The callers below check the status explicitly. + count_open_alerts() { + local body status + body=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' \ + --jq 'length' 2>/dev/null) + status=$? + if [ $status -ne 0 ]; then + echo "BLOCKED: cannot read code scanning alerts (gh exit $status)." >&2 + echo " Refusing to treat an unreadable alert list as zero." >&2 + echo " Most likely cause: the job lacks security-events:read." >&2 + return 1 + fi + case "$body" in + ''|*[!0-9]*) + echo "BLOCKED: alert count was not a number: ${body:-}" >&2 + return 1 + ;; + esac + printf '%s' "$body" + } echo "Waiting 60s for alert API to settle..." sleep 60 - ALERTS=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0") + ALERTS=$(count_open_alerts) || exit 1 sleep 15 - ALERTS2=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0") + ALERTS2=$(count_open_alerts) || exit 1 [ "$ALERTS" -lt "$ALERTS2" ] && ALERTS=$ALERTS2 if [ "$ALERTS" -gt 0 ]; then echo "BLOCKED: $ALERTS open alert(s)" diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index d915eb243..7ccd56bf5 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -107,6 +107,32 @@ jobs: if: startsWith(matrix.os, 'ubuntu') run: sudo apt-get update && sudo apt-get install -y zlib1g-dev ccache + # Ubuntu 23.10+ ships kernel.apparmor_restrict_unprivileged_userns=1, which + # blocks unshare(CLONE_NEWUSER) for unprivileged users. Without this, the + # #1830 single-uid-userns smoke test SKIPS on every 24.04 leg, and its only + # real-namespace coverage lived on the two ubuntu-22.04 legs -- which are + # broad-matrix only (never PR CI) and which GitHub retires on 2027-04-17. + # A deliberate relaxation of a security check was therefore guarded by a + # test that ran in no PR and was on a clock. Hosted runners have + # passwordless sudo, so one sysctl puts it on the CORE matrix instead. + # Best-effort: a runner that refuses simply returns to skipping. + - name: Permit unprivileged user namespaces (Ubuntu) + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || \ + echo "could not lift the userns restriction; namespace tests will skip" + # Then SAY whether it worked. The runner prints per-shard aggregates + # ("N passed, M skipped") and never individual test names, so a + # namespace test that silently returns to skipping is invisible in a + # green log -- the same false assurance that let #1830 ship with a + # test that never executed. This line is the evidence. Probed exactly + # as the test does it: unprivileged, no sudo. + if unshare -U -r true 2>/dev/null; then + echo "CBM_USERNS_AVAILABLE=yes" + else + echo "CBM_USERNS_AVAILABLE=no (namespace-dependent tests will skip)" + fi + - name: Install ccache (macOS) if: startsWith(matrix.os, 'macos') run: command -v ccache >/dev/null 2>&1 || brew install ccache diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index dbaa52a6c..e83a82d06 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -29,7 +29,7 @@ jobs: run: sudo apt-get update && sudo apt-get install -y zlib1g-dev - name: Initialize CodeQL - uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 with: languages: c-cpp build-mode: manual @@ -38,6 +38,6 @@ jobs: run: scripts/build.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 with: category: "/language:c-cpp" diff --git a/.github/workflows/dry-run.yml b/.github/workflows/dry-run.yml index 43a6b6bd9..600dbad04 100644 --- a/.github/workflows/dry-run.yml +++ b/.github/workflows/dry-run.yml @@ -45,6 +45,10 @@ on: description: 'Skip candidate scan; mark unscanned and default to stripped' type: boolean default: false + skip_memwaste: + description: 'Skip the waste sanitizer (report-only) and the fuzz smoke' + type: boolean + default: false soak_level: description: 'Soak: full (quick+asan), quick (10min), none' type: choice @@ -58,6 +62,13 @@ jobs: # ── Security (independent island — does not block main pipeline) ── security: uses: ./.github/workflows/_security.yml + # The called job requests security-events:read to count code-scanning + # alerts; a reusable workflow cannot request more than its caller grants, + # so withholding this here fails the whole run at startup. + permissions: + contents: read + security-events: read + actions: read secrets: inherit # ── Lint (cppcheck + clang-format) ──────────────────────────── @@ -100,6 +111,38 @@ jobs: with: broad_platforms: true + # ── Waste sanitizer, REPORT-ONLY (independent island) ───────── + # Event lane on Linux, macOS and Windows, access lane and the per-function + # scaling lane on Linux. Uploads reports; fails only when the measurement is + # unsound (a defect of the tool itself). + memwaste: + if: ${{ inputs.skip_memwaste != true }} + uses: ./.github/workflows/_memwaste.yml + with: + mode: full + + # ── Fuzz smoke (independent island) ─────────────────────────── + # A fixed number of executions from a fixed seed per target, so the verdict is + # a function of the commit (O9). A crash here is a real defect in a parser + # that reads untrusted input; its reproducer is uploaded. + fuzz: + if: ${{ inputs.skip_memwaste != true }} + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev clang llvm libclang-rt-dev + - name: Fuzz smoke (extract, cypher, config) + run: scripts/fuzz.sh all --runs 20000 --seed 1 + - name: Upload reproducers + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fuzz-artifacts + path: build/fuzz/artifacts-*/ + if-no-files-found: ignore + # ── Soak tests (optional, parallel with smoke) ──────────────── soak: if: ${{ inputs.soak_level != 'none' && !cancelled() && needs.build.result != 'failure' && needs.build.result != 'skipped' }} diff --git a/.github/workflows/nightly-soak.yml b/.github/workflows/nightly-soak.yml index 4c8abeb28..2bd5bc7e8 100644 --- a/.github/workflows/nightly-soak.yml +++ b/.github/workflows/nightly-soak.yml @@ -35,6 +35,27 @@ permissions: contents: read jobs: + # Fuzz exploration: new inputs are the point here, so it runs for a time + # budget instead of a fixed count. NON-GATING like the soak; a crash uploads + # its reproducer, which becomes a fixed-seed regression input once fixed. + fuzz: + if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev clang llvm libclang-rt-dev + - name: Fuzz exploration (30 min per target) + run: scripts/fuzz.sh all --max-total-time 1800 --seed 0 + - name: Upload reproducers + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fuzz-artifacts-nightly + path: build/fuzz/artifacts-*/ + if-no-files-found: ignore + soak: uses: ./.github/workflows/_soak.yml with: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c18a45d13..8cb464b19 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -22,6 +22,13 @@ concurrency: jobs: security: uses: ./.github/workflows/_security.yml + # The called job requests security-events:read to count code-scanning + # alerts; a reusable workflow cannot request more than its caller grants, + # so withholding this here fails the whole run at startup. + permissions: + contents: read + security-events: read + actions: read secrets: inherit lint: @@ -57,9 +64,29 @@ jobs: run: | # The full .diff endpoint rejects large-but-valid PRs at 20k lines. # The paginated files endpoint remains filename-only for this gate. - FILES=$(gh api --paginate "repos/$REPO/pulls/$PR/files?per_page=100" --jq '.[].filename') + if FILES=$(gh api --paginate "repos/$REPO/pulls/$PR/files?per_page=100" --jq '.[].filename'); then + echo "file list: pulls/files API" + else + # The files endpoint renders the diff server-side too, and answers + # 422 "this diff is taking too long to generate" for a PR that + # regenerates a vendored parser (#2246: a 42 MB sql/parser.c) -- + # so does the compare endpoint. The PR merge ref's first parent is + # the base, so a name-only diff across it is the same file list. + # depth 2 + blob:none fetches commits and trees only (well under + # 1 MB, under a second); nothing from the PR is checked out or run. + echo "::notice::pulls/files API failed; file list taken from refs/pull/$PR/merge" + CHANGES_GIT="$RUNNER_TEMP/changes.git" + git init -q --bare "$CHANGES_GIT" + git --git-dir="$CHANGES_GIT" fetch -q --depth=2 --filter=blob:none \ + "https://github.com/$REPO" "refs/pull/$PR/merge" + FILES=$(git --git-dir="$CHANGES_GIT" diff --name-only --no-renames FETCH_HEAD^1 FETCH_HEAD) + fi printf '%s\n' "$FILES" - if printf '%s\n' "$FILES" | grep -qE '^(src/|internal/|install\.(sh|ps1)|scripts/build\.sh|scripts/smoke-test\.sh|scripts/smoke-local\.sh|scripts/smoke-fixture-server\.py|scripts/gen-third-party-notices\.sh|scripts/env\.sh|test-infrastructure/vm/(vm-smoke\.sh|windows-user-path-guard\.ps1)|Makefile\.cbm)'; then + # A here-string, not a pipe: under this shell's pipefail, grep -q + # exits on its first match, printf dies of SIGPIPE once the list + # outgrows the pipe buffer, and the MATCH is then reported as a + # failure -- product=false, smoke skipped, on the largest PRs. + if grep -qE '^(src/|internal/|install\.(sh|ps1)|scripts/build\.sh|scripts/smoke-test\.sh|scripts/smoke-local\.sh|scripts/smoke-fixture-server\.py|scripts/gen-third-party-notices\.sh|scripts/env\.sh|test-infrastructure/vm/(vm-smoke\.sh|windows-user-path-guard\.ps1)|Makefile\.cbm)' <<<"$FILES"; then echo "product=true" >> "$GITHUB_OUTPUT" else echo "product=false" >> "$GITHUB_OUTPUT" @@ -147,6 +174,17 @@ jobs: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1000M + # ── Waste sanitizer, REPORT-ONLY: memory + CPU waste on this repository and + # the per-function scaling lane (Linux). Deliberately NOT in ci-ok's needs: + # it uploads evidence and prints the ratchet, it never blocks a merge. Only + # product-code PRs pay for it. ── + memwaste: + needs: [changes] + if: ${{ !cancelled() && needs.changes.outputs.product == 'true' }} + uses: ./.github/workflows/_memwaste.yml + with: + mode: pr + ci-ok: # The one required context (besides dco) — fails unless every PR stage # succeeded, so matrix renames can never silently deadlock merges. `skipped` diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 03abb518e..c0e796364 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,13 @@ jobs: # ── Security (starts immediately, blocks only verify) ─────────── security: uses: ./.github/workflows/_security.yml + # The called job requests security-events:read to count code-scanning + # alerts; a reusable workflow cannot request more than its caller grants, + # so withholding this here fails the whole run at startup. + permissions: + contents: read + security-events: read + actions: read secrets: inherit # ── 0. Preflight: refuse malformed dispatch inputs ────────────── @@ -138,6 +145,15 @@ jobs: version: ${{ inputs.version }} use_release_artifacts: true + # ── Waste sanitizer, REPORT-ONLY ──────────────────────────────── + # The same full run as the dry run, on the release commit. Not in + # release-draft's needs: the report is evidence for the release notes, never + # a publication gate. + memwaste: + uses: ./.github/workflows/_memwaste.yml + with: + mode: full + # ── 6. Create DRAFT release ──────────────────────────────────── # Requires smoke to have actually SUCCEEDED, not merely "not failed". The bare # !cancelled() && !failure() form is fail-OPEN: a skipped smoke is neither diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index a208a62ca..f2d8bcb92 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -27,6 +27,6 @@ jobs: publish_results: true - name: Upload SARIF results - uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4 + uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4 with: sarif_file: results.sarif diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index aa91f91c3..e9b4f9357 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -1,7 +1,7 @@ # Smoke invariants across the WIDEST set of GitHub-hosted runners. Builds the # production source path with explicit fault-injection test seams and runs # scripts/smoke-invariants.sh -# (version/help, MCP initialize handshake [#513], all 14 tools invocable, malformed- +# (version/help, MCP initialize handshake [#513], all registered tools invocable, malformed- # input resilience, clean EOF exit, shared-lib resolution, install dry-run). # # Maximizing platforms is the point: ubuntu-22.04 (older glibc → AlmaLinux/#182 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96ce8f861..93596d06f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ src/ foundation/ Arena allocator, hash table, string utils, platform compat store/ SQLite graph storage (WAL mode, FTS5) cypher/ Cypher query → SQL translation - mcp/ MCP server (JSON-RPC 2.0 over stdio, 14 tools) + mcp/ MCP server (JSON-RPC 2.0 over stdio, 17 tools) pipeline/ Multi-pass indexing pipeline pass_*.c Individual pipeline passes (definitions, calls, usages, etc.) httplink.c HTTP route extraction (Go/Express/Laravel/Ktor/Python) @@ -149,6 +149,25 @@ If in doubt, open an issue and ask. - Run `scripts/test.sh` and `scripts/lint.sh` before submitting - Keep PRs focused — avoid unrelated reformatting or refactoring +## AI-assisted and AI-authored contributions + +Contributions written with the help of an AI tool, or written by an AI agent acting for a person, are welcome here on the same terms as any other. What we review is the change, not who or what typed it. Three things make that work: + +- **Say so.** If an agent wrote the PR or is answering in the thread, disclose it — a line in the PR description or at the top of a comment is enough ("written with X", "this account is operated by an AI agent on behalf of Y"). Disclosure is never held against a contribution. Finding out later is. +- **A person is accountable.** The [DCO sign-off](#license-and-sign-off-dco--required-on-every-commit) is a legal certification, and only a person can make it. The human behind the account certifies every commit, answers for it, and is who we talk to if something goes wrong. An agent may carry the conversation; it cannot carry the responsibility. +- **The bar does not move.** Every claim in a PR must be something a reviewer can check: the failing test, the command that was run, the numbers and the machine they came from. "The model says it works" is not evidence, and a confident description does not replace a test that fails without the fix. We read AI-written PRs exactly as carefully as human-written ones — no more suspiciously, and no less. + +Two practical notes: + +- `Co-authored-by:` trailers naming a tool are fine. Links to private chat or agent sessions are not — they are dead to everyone else and do not belong in permanent history; we will ask you to remove them. +- High-volume, low-effort submissions — many near-identical PRs, changes nobody ran, replies that do not engage with the review — will be closed, whoever or whatever produced them. One careful PR is worth more than twenty generated ones. + +### Who speaks for the project + +Only the maintainer account ([@DeusData](https://github.com/DeusData)) speaks for this project. A review, an approval, a "this will be merged" or a "this is a duplicate" from any other account is that person's opinion, however official it sounds — helpful reviews from the community are very welcome, and that is what they are. + +**Impersonating a maintainer is not acceptable**, by a person or by an agent: presenting yourself as part of the maintainer team, posting approvals or change requests styled as maintainer decisions, or telling other contributors what the project has decided. We dismiss such reviews, hide the comments, say publicly in the affected threads that they were not ours, and report and block the account. If you are unsure whether a response you received is from the project, check the account name — and ask. + ## Security We take security seriously. All PRs go through: diff --git a/Makefile.cbm b/Makefile.cbm index 35844a529..7a3cd31e1 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -76,14 +76,15 @@ endif # the prod build uses mimalloc (MI_OVERRIDE=1); the test build is CRT+ASan, where # binding would create an alloc/free mismatch, so the guard is prod-only. CFLAGS_PROD = $(CFLAGS_COMMON) -O2 -DCBM_BIND_TS_ALLOCATOR=1 $(TEST_SEAM_DEFINE) \ - $(GLOBAL_OVERRIDE_DEFINE) $(CFLAGS_EXTRA) -CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 + $(GLOBAL_OVERRIDE_DEFINE) $(MEMWASTE_CFLAGS) $(CFLAGS_EXTRA) +CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 $(MEMWASTE_VENDOR_CFLAGS) # Test flags: debug + sanitizers (override SANITIZE= to disable on Windows) SANITIZE = -fsanitize=address,undefined -fno-omit-frame-pointer EDITOR_TEST_DEFINES = -DCBM_JSON_LIKE_ENABLE_TEST_API=1 \ -DCBM_TOML_EDIT_ENABLE_TEST_API=1 -DCBM_YAML_ENABLE_TEST_API=1 \ -DCBM_TEXT_EDIT_ENABLE_TEST_API=1 -DCBM_CLI_ENABLE_TEST_API=1 \ + -DCBM_CONFIG_EDIT_PATH_ENABLE_TEST_API=1 \ -DCBM_DIAGNOSTICS_ENABLE_TEST_API=1 -DCBM_ENABLE_TEST_SEAMS=1 # The build system is the single source of truth for "is this binary # instrumented": compiler-specific probes (__SANITIZE_ADDRESS__) miss @@ -215,6 +216,75 @@ ELF_HARDENING_FLAGS := -Wl,-z,noexecstack -Wl,-z,separate-code \ endif endif +# MEMWASTE=1: the waste-sanitizer flavour (src/foundation/mem_events.h). Built +# like production -- -O2, mimalloc, the real allocator routing -- plus the event +# layer and its observers, because waste measured under ASan's allocator would +# describe ASan. Frame pointers stay so the macOS observer can walk to the +# caller. On Linux this is also what finally LINKS the --wrap observation shim: +# its flags and source were defined here but never reached a link line, so the +# Linux profiler had been reporting nothing. +# +# Every object is built this way, VENDORED ones included (grammars, the +# tree-sitter runtime, SQLite, tre, lz4, zstd, the C++ preprocessor): waste in +# a vendored parser is waste in this program. mimalloc alone is left as it is -- +# its reads and writes are the allocator's own bookkeeping, not program data. +# +# MEMWASTE_ACCESS=1 adds the access lane (reads and writes of every tracked +# block through the compiler's load/store callbacks; clang only). It builds at +# -O1 without vectorisation: at -O2 the callbacks describe the OPTIMISED program, +# and reads the optimiser removed would be reported as never read. +# MEMWASTE_PROFILE=1 adds clang's execution counters (-fprofile-instr-generate) +# for the CPU scaling lane (scripts/memwaste-scaling.py). +# Each flavour needs its own BUILD_DIR: vendored objects are not rebuilt when +# only the flavour changes. +# +# The libc wrappers (mem_override_libc.c) need --wrap on Linux and Windows; +# macOS defines the symbols instead. STATIC=1 (musl) is not a memwaste target. +MEMWASTE ?= 0 +MEMWASTE_ACCESS ?= 0 +MEMWASTE_PROFILE ?= 0 +ifeq ($(MEMWASTE_ACCESS),1) +override MEMWASTE := 1 +endif +ifeq ($(MEMWASTE_PROFILE),1) +override MEMWASTE := 1 +endif +MEMWASTE_CFLAGS := +MEMWASTE_VENDOR_CFLAGS := +MEMWASTE_LDFLAGS := +MEMWASTE_SRCS := +MEMWASTE_ALLOC_SYMS_LINUX := malloc calloc realloc free strdup strndup posix_memalign \ + aligned_alloc memalign +MEMWASTE_LIBC_SYMS_LINUX := memcpy memmove memset memcmp strlen strcmp strncmp \ + __memcpy_chk __memmove_chk __memset_chk read __read_chk write \ + pread pwrite pread64 pwrite64 open open64 __open_2 __open64_2 \ + stat lstat fopen fopen64 fread __fread_chk fwrite opendir readdir \ + pthread_mutex_lock +MEMWASTE_LIBC_SYMS_WIN := memcpy memmove memset memcmp strlen strcmp strncmp _wfopen fread fwrite +ifeq ($(MEMWASTE),1) +MEMWASTE_CFLAGS := -DCBM_MEMWASTE=1 -fno-omit-frame-pointer +MEMWASTE_VENDOR_CFLAGS := -DCBM_MEMWASTE=1 -fno-omit-frame-pointer +ifeq ($(IS_LINUX),yes) +MEMWASTE_LDFLAGS := $(foreach sym,$(MEMWASTE_ALLOC_SYMS_LINUX) $(MEMWASTE_LIBC_SYMS_LINUX),-Wl,--wrap=$(sym)) +MEMWASTE_SRCS := src/foundation/mem_override_posix.c +endif +ifeq ($(IS_MINGW),yes) +MEMWASTE_LDFLAGS := $(foreach sym,$(MEMWASTE_LIBC_SYMS_WIN),-Wl,--wrap=$(sym)) +endif +endif +ifeq ($(MEMWASTE_ACCESS),1) +MEMWASTE_ACCESS_FLAGS := -DCBM_MEMWASTE_ACCESS=1 -O1 -fno-vectorize -fno-slp-vectorize \ + -fsanitize-coverage=inline-bool-flag,trace-loads,trace-stores +MEMWASTE_CFLAGS += $(MEMWASTE_ACCESS_FLAGS) +MEMWASTE_VENDOR_CFLAGS += $(MEMWASTE_ACCESS_FLAGS) +endif +ifeq ($(MEMWASTE_PROFILE),1) +MEMWASTE_PROFILE_FLAGS := -DCBM_MEMWASTE_PROFILE=1 -fprofile-instr-generate -fprofile-update=atomic +MEMWASTE_CFLAGS += $(MEMWASTE_PROFILE_FLAGS) +MEMWASTE_VENDOR_CFLAGS += $(MEMWASTE_PROFILE_FLAGS) +MEMWASTE_LDFLAGS += $(MEMWASTE_PROFILE_FLAGS) +endif + # The POSIX wrap shim exists only so the profiler can observe allocations. It # must never reach a sanitized build: the Linux/macOS test builds are CRT+ASan, # and redirecting malloc into mimalloc underneath ASan's own interception mixes @@ -224,7 +294,7 @@ endif # instead of the system libstdc++ (scripts/msan.sh overrides both vars). CXX_STDLIB ?= -lstdc++ CXX_STDLIB_FLAGS ?= -LDFLAGS = -lm $(CXX_STDLIB) -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS) +LDFLAGS = -lm $(CXX_STDLIB) -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) $(MIMALLOC_WRAP_FLAGS) $(MEMWASTE_LDFLAGS) $(ELF_HARDENING_FLAGS) LDFLAGS_TEST = -lm $(CXX_STDLIB) -lpthread -lz $(SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS) LDFLAGS_TSAN = -lm $(CXX_STDLIB) -lpthread -lz $(TSAN_SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS) @@ -247,10 +317,15 @@ FOUNDATION_SRCS = \ src/foundation/compat_fs.c \ src/foundation/compat_regex.c \ src/foundation/mem.c \ + src/foundation/mem_core.c \ + src/foundation/mem_events.c \ + src/foundation/mem_override_darwin.c \ + src/foundation/mem_override_libc.c \ src/foundation/diagnostics.c \ src/foundation/profile.c \ src/foundation/dump_verify.c \ src/foundation/limits.c \ + src/foundation/index_policy.c \ src/foundation/subprocess.c \ src/foundation/sha256.c \ src/foundation/secure_random.c \ @@ -274,6 +349,8 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/extract_k8s.c \ $(CBM_DIR)/extract_dbt.c \ $(CBM_DIR)/helpers.c \ + $(CBM_DIR)/result_compact.c \ + $(CBM_DIR)/result_spill.c \ $(CBM_DIR)/lang_specs.c \ $(CBM_DIR)/macro_table.c \ $(CBM_DIR)/iris_export_xml.c \ @@ -372,6 +449,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_gitdiff.c \ src/pipeline/pass_configures.c \ src/pipeline/pass_configlink.c \ + src/pipeline/pass_doclinks.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ src/pipeline/pass_envscan.c \ @@ -409,7 +487,7 @@ GIT_SRCS = src/git/git_context.c CLI_SRCS = src/cli/cli.c src/cli/progress_sink.c src/cli/hook_augment.c src/cli/client_adapter.c \ src/cli/agent_clients.c src/cli/agent_profiles.c \ src/cli/config_json_like.c src/cli/config_toml_edit.c src/cli/config_yaml_edit.c \ - src/cli/config_text_edit.c src/cli/activation_transaction.c + src/cli/config_text_edit.c src/cli/config_edit_path.c src/cli/activation_transaction.c # UI module (graph visualization) UI_SRCS = \ @@ -468,10 +546,19 @@ ifneq ($(filter yes,$(IS_MINGW) $(IS_LINUX)),) MIMALLOC_OVERRIDE_DEFINE := -DMI_MALLOC_OVERRIDE=1 GLOBAL_OVERRIDE_DEFINE := -DCBM_MEM_GLOBAL_OVERRIDE=1 endif +# MI_DEBUG=0: the production allocator is mimalloc's RELEASE build. Its header +# defaults to MI_DEBUG=2 unless NDEBUG is defined, and nothing here defined it, +# so every shipped binary up to v0.11.0 ran mimalloc's debug build: internal +# assertions, fine-grained statistics, padding on every block and a memset of +# every freed block. The waste sanitizer measured it on the Go corpus as 92 GB +# (macOS) and 444 GB (Linux, where mimalloc also serves libc's malloc) of byte +# work inside the allocator (2026-09-17). Committed-bytes accounting, which the +# memory budget reads, is kept in every build (subproc stats, not MI_STAT). +# The TEST build below keeps mimalloc's debug checks. MIMALLOC_CFLAGS = -std=c11 -O2 -w \ -Ivendored/mimalloc/include \ -Ivendored/mimalloc/src \ - -DMI_OVERRIDE=1 \ + -DMI_OVERRIDE=1 -DMI_DEBUG=0 \ $(MIMALLOC_OVERRIDE_DEFINE) MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \ -Ivendored/mimalloc/include \ @@ -492,7 +579,7 @@ MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \ SQLITE3_OMIT_FLAGS = -DSQLITE_OMIT_LOAD_EXTENSION SQLITE3_SRC = vendored/sqlite3/sqlite3.c SQLITE3_CFLAGS = -std=c11 -O2 -w -DSQLITE_DQS=0 -DSQLITE_THREADSAFE=1 -DSQLITE_ENABLE_FTS5 \ - $(SQLITE3_OMIT_FLAGS) + $(SQLITE3_OMIT_FLAGS) $(MEMWASTE_VENDOR_CFLAGS) SQLITE3_CFLAGS_TEST = -std=c11 -g -O1 -w -DSQLITE_DQS=0 -DSQLITE_THREADSAFE=1 -DSQLITE_ENABLE_FTS5 \ $(SQLITE3_OMIT_FLAGS) @@ -504,7 +591,7 @@ TRE_CFLAGS = -std=c11 -g -O1 -w -Ivendored/tre YYJSON_SRC = vendored/yyjson/yyjson.c # All production sources -PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC) +PROD_SRCS = $(MEMWASTE_SRCS) $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC) EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \ $(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) @@ -532,6 +619,7 @@ TEST_FOUNDATION_SRCS = \ tests/test_str_intern.c \ tests/test_log.c \ tests/test_str_util.c \ + tests/test_index_policy.c \ tests/test_workspace.c \ tests/test_platform.c \ tests/test_diagnostics.c \ @@ -588,7 +676,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c @@ -637,7 +725,7 @@ TEST_CLI_SRCS = tests/test_cli.c tests/test_agent_clients.c tests/test_agent_pro tests/test_config_toml_edit.c tests/test_config_yaml_edit.c tests/test_config_text_edit.c \ tests/test_activation_transaction.c -TEST_MEM_SRCS = tests/test_mem.c +TEST_MEM_SRCS = tests/test_mem.c tests/test_mem_events.c TEST_UI_SRCS = tests/test_ui.c TEST_HTTPD_SRCS = tests/test_httpd.c @@ -742,7 +830,8 @@ BUILD_DIR = build/c # ── Object file compilation (grammars need relaxed warnings) ───── # Grammar + tree-sitter runtime: compiled without -Werror (upstream code has warnings) -GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) +GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \ + $(MEMWASTE_VENDOR_CFLAGS) GRAMMAR_CFLAGS_TEST = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \ -DCBM_ENABLE_TEST_SEAMS=1 $(SANITIZED_DEFINE) $(SANITIZE) GRAMMAR_CFLAGS_TSAN = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \ @@ -760,7 +849,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o # ── Targets ────────────────────────────────────────────────────── -.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security +.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz $(BUILD_DIR): mkdir -p $(BUILD_DIR) @@ -775,7 +864,24 @@ test-foundation: $(BUILD_DIR)/test-foundation # ── Grammar/TS/LSP object files (compiled with relaxed warnings) ─ -$(BUILD_DIR)/%.o: $(CBM_DIR)/%.c | $(BUILD_DIR) +# Each grammar_.c is a unity object that #includes its own vendored +# parser.c and, where the grammar has one, scanner.c. Those includes were +# prerequisites of nothing, so editing a vendored scanner left the object +# untouched and the edit silently absent from the binary: on 2026-09-19 a fix +# to properties/scanner.c appeared not to work at all, and `nm` showed the old +# symbol still sitting in prod_grammar_properties.o. MANIFEST.md's "Local +# source patches" table means patching a vendored scanner is expected, so this +# trap is reachable by design. Same reasoning as LSP_UNITY_DEPS and +# TS_RUNTIME_DEPS above; secondary expansion is what lets one pattern rule name +# each grammar's own directory ($$* is the stem, grammar_properties -> +# properties). Non-grammar objects match no directory and gain no prerequisite. +GRAMMAR_VENDOR_DIR = $(CBM_DIR)/vendored/grammars/$$(subst grammar_,,$$*) +GRAMMAR_UNITY_DEPS = $$(wildcard $(GRAMMAR_VENDOR_DIR)/*.c) \ + $$(wildcard $(GRAMMAR_VENDOR_DIR)/*.h) \ + $$(wildcard $(GRAMMAR_VENDOR_DIR)/tree_sitter/*.h) +.SECONDEXPANSION: + +$(BUILD_DIR)/%.o: $(CBM_DIR)/%.c $(GRAMMAR_UNITY_DEPS) | $(BUILD_DIR) $(CC) $(GRAMMAR_CFLAGS_TEST) -c -o $@ $< $(BUILD_DIR)/ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR) @@ -787,7 +893,7 @@ $(BUILD_DIR)/lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR) $(BUILD_DIR)/preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR) $(CXX) $(CXXFLAGS_TEST) -w -I$(CBM_DIR)/vendored -c -o $@ $< -$(BUILD_DIR)/tsan_%.o: $(CBM_DIR)/%.c | $(BUILD_DIR) +$(BUILD_DIR)/tsan_%.o: $(CBM_DIR)/%.c $(GRAMMAR_UNITY_DEPS) | $(BUILD_DIR) $(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $< $(BUILD_DIR)/tsan_ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR) @@ -864,7 +970,7 @@ $(BUILD_DIR)/tsan_tre.o: $(TRE_SRC) | $(BUILD_DIR) $(CC) $(TRE_CFLAGS) $(TSAN_SANITIZE) -c -o $@ $< $(BUILD_DIR)/prod_tre.o: $(TRE_SRC) | $(BUILD_DIR) - $(CC) $(TRE_CFLAGS) -O2 -c -o $@ $< + $(CC) $(TRE_CFLAGS) -O2 $(MEMWASTE_VENDOR_CFLAGS) -c -o $@ $< endif # Vendored LZ4 (test build) @@ -1017,7 +1123,7 @@ TS_RUNTIME_OBJ_PROD = $(BUILD_DIR)/prod_ts_runtime.o LSP_OBJ_PROD = $(BUILD_DIR)/prod_lsp_all.o PP_OBJ_PROD = $(BUILD_DIR)/prod_preprocessor.o -$(BUILD_DIR)/prod_%.o: $(CBM_DIR)/%.c | $(BUILD_DIR) +$(BUILD_DIR)/prod_%.o: $(CBM_DIR)/%.c $(GRAMMAR_UNITY_DEPS) | $(BUILD_DIR) $(CC) $(GRAMMAR_CFLAGS) -c -o $@ $< $(BUILD_DIR)/prod_ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR) @@ -1032,14 +1138,14 @@ $(BUILD_DIR)/prod_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR) # Vendored LZ4 (compiled separately, not unity-built via lz4_store.c) LZ4_OBJ_PROD = $(BUILD_DIR)/prod_lz4.o $(BUILD_DIR)/prod_lz4hc.o $(BUILD_DIR)/prod_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR) - $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -c -o $@ $< + $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR) -c -o $@ $< $(BUILD_DIR)/prod_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR) - $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $< + $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/lz4 -c -o $@ $< # Vendored zstd (compiled separately, not unity-built via zstd_store.c) ZSTD_OBJ_PROD = $(BUILD_DIR)/prod_zstd.o $(BUILD_DIR)/prod_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR) - $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $< + $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/zstd -c -o $@ $< OBJS_VENDORED_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LSP_OBJ_PROD) $(PP_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ) @@ -1061,7 +1167,7 @@ MAIN_SRC = src/main.c # for trap debugging -- silently re-run the previously instrumented binary, # which during this bisect produced a "crashes without sanitizers too" result # that was pure artifact and sent the investigation down the wrong path. -BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE) +BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)|MEMWASTE=$(MEMWASTE)|MEMWASTE_ACCESS=$(MEMWASTE_ACCESS)|MEMWASTE_PROFILE=$(MEMWASTE_PROFILE) .PHONY: build-config-check build-config-check: $(BUILD_DIR)/.build-config: build-config-check | $(BUILD_DIR) @@ -1187,8 +1293,17 @@ lint-no-suppress: lint: lint-tidy lint-cppcheck lint-format lint-no-suppress @echo "=== All linters passed ===" +# Memory-core linter: memory is allocated through src/foundation/mem_core.h, +# not through raw malloc/calloc/realloc/free/strdup. A checked-in baseline +# (scripts/memory-core-baseline.txt) records each file's raw-site count and the +# gate goes red the moment any file grows. Files only ever go down; lower the +# baseline line in the same change that migrates the file. +lint-memory-core: + @echo "=== memory-core linter ===" + @python3 scripts/lint-memory-core.py + # CI linters (no clang-tidy — platform-dependent, enforced locally via pre-commit) -lint-ci: lint-cppcheck lint-format lint-no-suppress +lint-ci: lint-cppcheck lint-format lint-no-suppress lint-memory-core @echo "=== CI linters passed ===" # ── Local memory-diagnostic lanes (not PR-CI gates by decision: the diag @@ -1248,6 +1363,28 @@ test-lsan: cd $(CURDIR) && ASAN_OPTIONS="detect_leaks=1:halt_on_error=1" \ ./build/lsan/test-runner $(LSAN_SUITES) +# ── Coverage-guided fuzzing (libFuzzer) ───────────────────────── +# One binary, three targets (tests/fuzz/cbm_fuzz.c): the per-file extractor, +# the Cypher parser, the TOML/YAML/JSON config editors. ASan+UBSan as in the +# test lane, plus fuzzer-no-link on EVERY object -- the vendored grammars and +# the tree-sitter runtime feed coverage back, which is where the parser state +# space lives. Needs a clang that ships libFuzzer (Homebrew LLVM on macOS; Apple +# clang does not). Run through scripts/fuzz.sh. +FUZZ_CC ?= clang +FUZZ_CXX ?= clang++ +FUZZ_SANITIZE = -fsanitize=address,undefined,fuzzer-no-link -fno-omit-frame-pointer +FUZZ_SRC = tests/fuzz/cbm_fuzz.c +fuzz: + $(MAKE) -f Makefile.cbm build/fuzz/cbm-fuzz BUILD_DIR=build/fuzz \ + CC=$(FUZZ_CC) CXX=$(FUZZ_CXX) SANITIZE="$(FUZZ_SANITIZE)" + +$(BUILD_DIR)/cbm-fuzz: $(FUZZ_SRC) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR) + $(CC) $(CFLAGS_TEST) -fsanitize=fuzzer -o $@ \ + $(FUZZ_SRC) $(PROD_SRCS) \ + $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \ + $(OBJS_VENDORED_TEST) \ + $(LDFLAGS_TEST) + # Path-sensitive memory analysis: leak paths, null derefs, uninitialized reads. # Non-gating: triage findings against the code (this lane's first run produced # 9 real fixes and 6 recorded false positives — see the analyzer batch commit). diff --git a/README.md b/README.md index c6ab67ee2..9c82c18ae 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![GitHub Release](https://img.shields.io/github/v/release/DeusData/codebase-memory-mcp?style=flat&color=blue)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE) [![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/dry-run.yml?label=CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/dry-run.yml) -[![Tests](https://img.shields.io/badge/tests-6768_passing-brightgreen)](https://github.com/DeusData/codebase-memory-mcp) +[![Tests](https://img.shields.io/badge/tests-8050_passing-brightgreen)](https://github.com/DeusData/codebase-memory-mcp) [![Languages](https://img.shields.io/badge/languages-162-orange)](https://github.com/DeusData/codebase-memory-mcp) [![Hybrid LSP](https://img.shields.io/badge/Hybrid_LSP-10_languages-blue)](#hybrid-lsp) [![Agents](https://img.shields.io/badge/agent_surfaces-45-purple)](https://github.com/DeusData/codebase-memory-mcp) @@ -16,7 +16,7 @@ **The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. Ships as a native executable with a small verified runtime-asset set for macOS, Linux, and Windows — download, run `install`, done. -High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 162 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No language runtime, hosted service, or API key. Plug and play across 45 supported automatic/conditional client surfaces. +High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 162 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 17 MCP tools. No language runtime, hosted service, or API key. Plug and play across 45 supported automatic/conditional client surfaces. > **Research** — The design and benchmarks behind this project are described in the preprint [*Codebase-Memory: Tree-Sitter-Based Knowledge Graphs for LLM Code Exploration via MCP*](https://arxiv.org/abs/2603.27277) (arXiv:2603.27277). Evaluated across 31 real-world repositories: 83% answer quality, 10× fewer tokens, 2.1× fewer tool calls vs. file-by-file exploration. @@ -37,7 +37,7 @@ High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-si - **45 supported automatic/conditional client surfaces** — `install` configures detected clients and safely activates conditional clients only when their documented platform, marker, or explicit existing config path is present. See [Multi-Agent Support](#multi-agent-support) for the complete matrix and manual/UI-only boundaries. - **Built-in graph visualization** — 3D interactive UI at `localhost:9749`, served from the binary itself. - **Infrastructure-as-code indexing** — Dockerfiles, Kubernetes manifests, and Kustomize overlays indexed as graph nodes with cross-references. `Resource` nodes for K8s kinds, `Module` nodes for Kustomize overlays with `IMPORTS` edges to referenced resources. -- **15 MCP tools** — search, trace, architecture, impact analysis, targeted index-coverage checks, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. +- **17 MCP tools** — search, trace, architecture, impact analysis, targeted index-coverage checks, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. ## Quick Start @@ -425,7 +425,7 @@ scripts/build.sh # without the UI (development only) Every platform ships **one self-contained executable**: the graph UI and the agent integration templates are linked into the binary, so an extracted archive is immediately complete. -Run the test suite (6,768 tests across 120 suites): +Run the test suite (8,060 tests across 141 suites): ```bash scripts/test.sh # full: clean sanitizer build + all suites + guards @@ -461,7 +461,7 @@ Add to `~/.claude.json` (user scope) or project `.mcp.json`: } ``` -Restart your agent. Verify with `/mcp` — you should see `codebase-memory-mcp` with 15 tools. +Restart your agent. Verify with `/mcp` — you should see `codebase-memory-mcp` with 17 tools. @@ -673,6 +673,7 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool | `list_projects` | List all indexed projects with node/edge counts. | | `delete_project` | Remove a project and all its graph data. | | `index_status` | Check indexing status of a project. | +| `check_index_coverage` | Check whether exact paths or a scope are indexed and fresh. A clean result means no recorded gap, not proof of completeness. | ### Querying @@ -683,7 +684,9 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool | `detect_changes` | Map git diff to affected symbols + blast radius with risk classification. | | `query_graph` | Execute Cypher-like graph queries (read-only). | | `get_graph_schema` | Node/edge counts, relationship patterns, property definitions per label. Run this first. | +| `compare_graphs` | Compare two indexed snapshots: node/edge additions and removals between a base and a target. | | `get_code_snippet` | Read source code for a function by qualified name. | +| `get_file_outline` | Declaration outline of one repository-relative file in source order, with optional label filter and paging. | | `get_architecture` | Codebase overview: languages, packages, routes, hotspots, clusters, ADR. | | `search_code` | Grep-like text search within indexed project files. | | `manage_adr` | CRUD for Architecture Decision Records (`get` reads, `update` replaces the whole document, `set_sections` rewrites only the named sections and leaves every other byte untouched, `sections` lists headings). Query modes do not wait behind a same-project reindex; writes remain serialized. | @@ -701,7 +704,7 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool ### Edge Types -`CONTAINS_PACKAGE`, `CONTAINS_FOLDER`, `CONTAINS_FILE`, `DEFINES`, `DEFINES_METHOD`, `IMPORTS`, `CALLS`, `CALL_REFERENCE`, `HTTP_CALLS`, `ASYNC_CALLS`, `IMPLEMENTS`, `HANDLES`, `USAGE`, `CONFIGURES`, `WRITES`, `MEMBER_OF`, `TESTS`, `USES_TYPE`, `FILE_CHANGES_WITH` +`CONTAINS_PACKAGE`, `CONTAINS_FOLDER`, `CONTAINS_FILE`, `DEFINES`, `DEFINES_METHOD`, `IMPORTS`, `CALLS`, `CALL_REFERENCE`, `HTTP_CALLS`, `ASYNC_CALLS`, `IMPLEMENTS`, `HANDLES`, `USAGE`, `CONFIGURES`, `REFERENCES_FILE`, `WRITES`, `MEMBER_OF`, `TESTS`, `USES_TYPE`, `FILE_CHANGES_WITH` ### Qualified Names @@ -733,9 +736,15 @@ codebase-memory-mcp config set auto_index true # auto-index on session codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true) codebase-memory-mcp config set watcher_enabled false # stop the watcher thread entirely (default: true) +codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit +codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit codebase-memory-mcp config reset auto_index # reset to default ``` +The two `index_max_*` settings default to `off`. Exceeding one fails the complete +index attempt rather than publishing a partial graph; an existing serving index +is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). + ### Environment Variables | Variable | Default | Description | @@ -843,7 +852,7 @@ Also supported (not yet benchmarked): Ada, Agda, Apex, Assembly (NASM), Astro, A src/ main.c Entry point (MCP stdio server + CLI + install/update/config) daemon/ Per-account session coordination, IPC, lifecycle, shared jobs/watchers - mcp/ MCP server (15 tools, JSON-RPC 2.0, session detection, auto-index) + mcp/ MCP server (17 tools, JSON-RPC 2.0, session detection, auto-index) cli/ Install/uninstall/update/config (45 client surfaces, hooks, instructions) store/ SQLite graph storage (nodes, edges, traversal, search, Louvain) pipeline/ Multi-pass indexing (structure → definitions → calls → HTTP links → config → tests) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 862d591c3..038256976 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -89,6 +89,8 @@ Current keys: | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | | `auto_watch` | `true` | Register the session's project with the background git watcher on connect. Set `false` to keep a session from registering its project (the watcher still runs for other projects). | | `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. | +| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | +| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | > **`watcher_enabled` vs `auto_watch`.** `watcher_enabled` controls whether the > watcher *subsystem* starts at all (the background poll thread). `auto_watch` is @@ -116,6 +118,14 @@ Current keys: > `auto_index` still runs, and `index_repository` stays available for manual > reindexing. +The two `index_max_*` settings are independent and disabled by default. They +apply to explicit indexing, automatic indexing, and watcher re-indexing, but not +to `cross-repo-intelligence`, which does not scan repository source files. +Equality is allowed; exceeding either setting fails the complete index request +and preserves any previously serving database. See +[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and +error-response details. + ## 3. UI Settings The optional built-in graph UI stores its settings in: diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md new file mode 100644 index 000000000..a3d57e015 --- /dev/null +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -0,0 +1,68 @@ +# Index resource limits + +Index resource limits are optional operator controls for repositories whose +discovery breadth is not known in advance. They are disabled by default so +existing large-repository workloads retain their current behavior. + +## Discovery settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_max_files` | `off` | `off` or `1..10000000` | Accepted source-file count | +| `index_max_source_mb` | `off` | `off` or `1..1048576` | Accepted source-file bytes | + +Set or reset them with the normal configuration command: + +```bash +codebase-memory-mcp config set index_max_files 250000 +codebase-memory-mcp config set index_max_source_mb 16384 +codebase-memory-mcp config reset index_max_files +``` + +Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero, +negative values, suffixes, trailing characters, and values outside the stated +ranges are rejected without changing the stored value. + +## Counting and failure semantics + +`index_max_files` counts a file only after it passes directory pruning, ignore +rules, filename and suffix filters, language detection, and the existing +per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same +accepted set. + +Equality is allowed. The first file or byte that makes an observed value greater +than its limit stops discovery. CBM discards the partial file list and does not +publish a partial graph as a complete index. + +For an explicit MCP request the error payload contains: + +```json +{ + "status": "error", + "code": "resource_limit_exceeded", + "stage": "discovery", + "resource": "files", + "observed": 250001, + "limit": 250000, + "unit": "files", + "retryable": true, + "serving_index_preserved": true, + "message": "Index discovery exceeded index_max_files" +} +``` + +The previous database remains available because publication occurs only after a +complete discovery and successful staged build. If no previous database exists, +`serving_index_preserved` is false. + +## Trust and compatibility + +Limits are read from the CLI-managed `_config.db`; they are not MCP request +arguments. A supervised parent replaces any caller-supplied internal policy +before spawning its worker, and the worker rejects a missing or incomplete +parent policy. + +These settings do not replace or increase `auto_index_limit`, change the 512 MiB +single-file cap, alter workspace-root authorization, or affect +`cross-repo-intelligence`. With both settings `off`, discovery follows the +existing unbounded path. diff --git a/docs/index.html b/docs/index.html index 56496d5d8..fdfde941e 100644 --- a/docs/index.html +++ b/docs/index.html @@ -56,7 +56,7 @@ "featureList": [ "Indexes 158 programming languages via vendored tree-sitter grammars", "Hybrid LSP semantic type resolution for Python, TypeScript/JavaScript, PHP, C#, Go, C/C++, Java, Kotlin, and Rust", - "15 MCP tools for structural search, call-path tracing, targeted coverage checks, and Cypher graph queries", + "17 MCP tools for structural search, call-path tracing, targeted coverage checks, and Cypher graph queries", "Semantic vector code search via bundled nomic-embed-code embeddings (no API key, fully local)", "Semantic graph edges (SEMANTICALLY_RELATED) and near-clone detection (SIMILAR_TO, MinHash + LSH)", "Cross-service linking for HTTP, gRPC, GraphQL, tRPC, and pub/sub channels with confidence scoring", @@ -739,8 +739,8 @@

The optional UI variant adds one verified, content-addressed frontend pack and serves an interactive 3D graph at localhost:9749 to explore nodes, edges, and clusters visually.

-

15 MCP tools

-

search_graph, trace_path, detect_changes, query_graph (Cypher), get_architecture, get_code_snippet, check_index_coverage, manage_adr, and 7 more.

+

17 MCP tools

+

search_graph, trace_path, detect_changes, query_graph (Cypher), get_architecture, get_code_snippet, check_index_coverage, manage_adr, and 9 more.

Cypher graph queries

diff --git a/flake.nix b/flake.nix index 50f6319ba..5a0afbb98 100644 --- a/flake.nix +++ b/flake.nix @@ -63,7 +63,7 @@ packages = forAllSystems (pkgs: rec { default = pkgs.stdenv.mkDerivation { pname = "codebase-memory-mcp"; - version = "0.10.8"; + version = "0.11.0"; src = ./.; diff --git a/graph-ui/package-lock.json b/graph-ui/package-lock.json index 1065f3783..b5441de62 100644 --- a/graph-ui/package-lock.json +++ b/graph-ui/package-lock.json @@ -33,7 +33,7 @@ "tailwindcss": "^4.1.0", "typescript": "^5.7.0", "vite": "^6.4.3", - "vitest": "^4.1.0" + "vitest": "^4.1.11" } }, "node_modules/@adobe/css-tools": { @@ -3603,16 +3603,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz", - "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -3621,13 +3621,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz", - "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.8", + "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -3648,9 +3648,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz", - "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", "dev": true, "license": "MIT", "dependencies": { @@ -3661,13 +3661,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz", - "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.8", + "@vitest/utils": "4.1.11", "pathe": "^2.0.3" }, "funding": { @@ -3675,14 +3675,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz", - "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -3691,9 +3691,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz", - "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", "dev": true, "license": "MIT", "funding": { @@ -3701,13 +3701,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz", - "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.8", + "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -5831,9 +5831,9 @@ } }, "node_modules/tinyrainbow": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", - "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", "dev": true, "license": "MIT", "engines": { @@ -6141,19 +6141,19 @@ } }, "node_modules/vitest": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz", - "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.8", - "@vitest/mocker": "4.1.8", - "@vitest/pretty-format": "4.1.8", - "@vitest/runner": "4.1.8", - "@vitest/snapshot": "4.1.8", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -6181,12 +6181,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.8", - "@vitest/browser-preview": "4.1.8", - "@vitest/browser-webdriverio": "4.1.8", - "@vitest/coverage-istanbul": "4.1.8", - "@vitest/coverage-v8": "4.1.8", - "@vitest/ui": "4.1.8", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" diff --git a/graph-ui/package.json b/graph-ui/package.json index b1aa3998b..b023560e8 100644 --- a/graph-ui/package.json +++ b/graph-ui/package.json @@ -37,7 +37,7 @@ "tailwindcss": "^4.1.0", "typescript": "^5.7.0", "vite": "^6.4.3", - "vitest": "^4.1.0" + "vitest": "^4.1.11" }, "overrides": { "form-data": ">=4.0.6", diff --git a/install.ps1 b/install.ps1 index d63f1001d..3b087321b 100644 --- a/install.ps1 +++ b/install.ps1 @@ -7,8 +7,22 @@ $ErrorActionPreference = "Stop" -# Enforce TLS 1.2+ (older PowerShell defaults to TLS 1.0 which GitHub rejects) -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -bor [Net.SecurityProtocolType]::Tls13 +# Enforce TLS 1.2+ (older PowerShell defaults to TLS 1.0 which GitHub rejects). +# +# TLS 1.3 is added ONLY where schannel can actually negotiate it: Windows 11 and +# Server 2022 (build 20348+). On Windows 10 the enum value still exists from +# .NET Framework 4.8 onward, so the assignment succeeds and nothing warns -- but +# the handshake then fails outright with "The request was aborted: Could not +# create SSL/TLS secure channel". An unsupported protocol flag in this bitmask +# is a hard failure, not a graceful downgrade, so every Windows 10 user was +# blocked at the first download (#1856). The enum-name probe additionally keeps +# the script parsing on .NET Framework 4.7, where Tls13 is not defined at all. +$CbmProtocols = [Net.SecurityProtocolType]::Tls12 +if ([Environment]::OSVersion.Version.Build -ge 20348 -and + ([enum]::GetNames([Net.SecurityProtocolType]) -contains 'Tls13')) { + $CbmProtocols = $CbmProtocols -bor [Net.SecurityProtocolType]::Tls13 +} +[Net.ServicePointManager]::SecurityProtocol = $CbmProtocols Add-Type -AssemblyName System.Net.Http $Repo = "DeusData/codebase-memory-mcp" diff --git a/internal/cbm/arena.h b/internal/cbm/arena.h index 5c6bef9f0..a451e4cf1 100644 --- a/internal/cbm/arena.h +++ b/internal/cbm/arena.h @@ -1,41 +1,7 @@ -#ifndef CBM_ARENA_H -#define CBM_ARENA_H - -#include - -// CBMArena is a simple bump allocator that allocates from fixed-size blocks. -// All memory is freed at once via cbm_arena_destroy(). Individual frees are not -// supported — this is by design for per-file extraction where all data has the -// same lifetime. -#define CBM_ARENA_MAX_BLOCKS 256 -#define CBM_ARENA_DEFAULT_BLOCK_SIZE (64 * 1024) // 64KB initial - -typedef struct { - char *blocks[CBM_ARENA_MAX_BLOCKS]; - size_t block_sizes[CBM_ARENA_MAX_BLOCKS]; // per-block sizes (for stats) - int nblocks; - size_t block_size; - size_t used; // bytes used in current block - size_t total_alloc; // cumulative bytes allocated (for stats) -} CBMArena; - -// Initialize an arena with the default block size. -void cbm_arena_init(CBMArena *a); - -// Allocate n bytes from the arena. Returns NULL on OOM or block exhaustion. -// All returned pointers are 8-byte aligned. -void *cbm_arena_alloc(CBMArena *a, size_t n); - -// Duplicate a string into arena memory. Returns arena-owned copy. -char *cbm_arena_strdup(CBMArena *a, const char *s); - -// Duplicate a string of known length into arena memory. NUL-terminates. -char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len); - -// sprintf into arena memory. Returns arena-owned string. -char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) __attribute__((format(printf, 2, 3))); - -// Free all blocks. Arena is invalid after this call. -void cbm_arena_destroy(CBMArena *a); - -#endif // CBM_ARENA_H +/* The arena lives in src/foundation/arena.h. This file used to be a stale + * copy with the same include guard and a shorter API; whichever header a + * translation unit included first won, silently. One definition now. */ +#ifndef CBM_INTERNAL_ARENA_SHIM_H +#define CBM_INTERNAL_ARENA_SHIM_H +#include "../../src/foundation/arena.h" /* relative: the lsp_all unit has no -Isrc */ +#endif /* CBM_INTERNAL_ARENA_SHIM_H */ diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 0bd718f82..b54fb54a7 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -1,7 +1,10 @@ /* Full declaration set for the same CBMArena, and it must precede cbm.h: * internal/cbm/arena.h declares a subset and the two share the CBM_ARENA_H * guard, so whichever is included first is the one this file sees. */ -#include "foundation/arena.h" // cbm_arena_init_sized +#include "foundation/arena.h" // cbm_arena_init_sized +#include "foundation/mem_core.h" // class accounting for the bound allocators +#include "foundation/mem_events.h" // waste sanitizer: the bound allocators bypass every observer +#include "foundation/log.h" // cbm_log_warn -- extract.lsp.skipped #include "cbm.h" #include "arena.h" // CBMArena, cbm_arena_init/alloc/strdup/destroy #include "helpers.h" @@ -122,7 +125,79 @@ void cbm_reset_profile(void) { } \ } while (0) +/* A definition name never spans lines. Config extractors hand over a node's + * whole text — a YAML block key, a Makefile recipe (`endif\n\n$(obj)/x.h`), a + * ktest .conf directive — and the line break rode into the graph (2026-09-16 + * probe: 8,301 Field nodes on elastic/elasticsearch, 91 kernel nodes). Cut at + * the first line break and drop trailing blanks; the qualified name carries + * the same text as its last segment and is cut the same way. */ +static const char *cbm_first_line(CBMArena *a, const char *text) { + if (!text) { + return text; + } + const char *nl = strpbrk(text, "\r\n"); + if (!nl) { + return text; + } + size_t n = (size_t)(nl - text); + while (n > 0 && (text[n - 1] == ' ' || text[n - 1] == '\t')) { + n--; + } + char *cut = (char *)cbm_arena_alloc(a, n + 1); + if (!cut) { + return text; + } + memcpy(cut, text, n); + cut[n] = '\0'; + return cut; +} + +/* JS/TS files: a definition named by a literal token is the walker naming a + * function or variable from an object key or a numeric pattern element — + * `{}` (1,534 incoming CALLS on microsoft/TypeScript's test baselines), `1` + * (38 nodes), a quoted string. An identifier starts with a letter, `_`, `$`, + * `#` (private members) or a non-ASCII byte; anything else is not a name. */ +static bool cbm_js_family_path(const char *path) { + if (!path) { + return false; + } + const char *dot = strrchr(path, '.'); + if (!dot) { + return false; + } + static const char *const exts[] = {".js", ".mjs", ".cjs", ".jsx", ".ts", + ".mts", ".cts", ".tsx", ".ets", NULL}; + for (int i = 0; exts[i]; i++) { + if (strcmp(dot, exts[i]) == 0) { + return true; + } + } + return false; +} + +static bool cbm_js_name_is_junk(const char *name) { + if (!name || !name[0]) { + return false; /* empty names are handled by the callers' own rules */ + } + unsigned char c = (unsigned char)name[0]; + bool identifier_start = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || + c == '$' || c == '#' || c >= 0x80; + /* Member names JS spells without an identifier start are still names: + * computed keys (`[Symbol.iterator]`), string-literal keys (`"my-key"`, + * `'x'`) and escaped identifiers (`А`). Measured on the TypeScript + * corpus: 1,531 `[…]` members, 97 quoted members, 154 escaped + * identifiers — all real definitions. What remains ({…} patterns, numeric + * literals, parenthesised types, `...rest`) is a token, not a name. */ + bool member_key_start = c == '[' || c == '"' || c == '\'' || c == '\\'; + return !identifier_start && !member_key_start; +} + void cbm_defs_push(CBMDefArray *arr, CBMArena *a, CBMDefinition def) { + def.name = cbm_first_line(a, def.name); + def.qualified_name = cbm_first_line(a, def.qualified_name); + if (cbm_js_family_path(def.file_path) && cbm_js_name_is_junk(def.name)) { + return; + } GROW_ARRAY(arr, a); arr->items[arr->count++] = def; } @@ -213,9 +288,93 @@ static const char *cbm_string_read(void *payload, uint32_t byte, TSPoint point, // --- Parse timeout callback --- +/* Budget for the tree-sitter progress callback. The PRIMARY gate is per-thread + * CPU time: a worker descheduled under CI contention burns WALL time but not + * CPU, so a starved-but-parseable file must NOT be abandoned merely because the + * 5 s budget elapsed in wall-clock against near-zero CPU. That false "parse + * timeout" silently dropped a file's defs and, with them, every cross-file edge + * those defs anchored (e.g. a Celery.send_task definition backing an ASYNC_CALLS + * edge). A generous WALL ceiling stays as a backstop so a genuinely + * stuck/spinning parse still terminates in bounded time. */ +#define CBM_PARSE_WALL_CEILING_FACTOR 12ULL /* ~60 s ceiling for the 5 s CPU budget */ +/* The unified walk has NO budget: it walks every node of every file. + * + * It used to stop on a thread-CPU deadline, which made the GRAPH a function of + * how fast the machine happened to be running. Two indexes of the same C# + * corpus, same machine, minutes apart, stopped the walk of + * src/tests/JIT/jit64/opt/cse/hugeSimpleExpr1.cs at node 2,660,352 and at node + * 2,574,336 (measured 2026-09-19). That file's tail is all field assignments, + * so the two runs disagreed by 10 WRITES edges: a graph that differed run to + * run for a reason no user could see or reproduce. A budget in NODES would at + * least have been reproducible, but it still answers "what is in this repo?" + * with "depends how much we felt like reading" — so there is no budget at all. + * Everything the walk does is a pure function of the tree, and now so is when + * it stops: at the end. + * + * What the deadline was protecting against is real and is handled where it + * belongs — in the cost per node, not in a clock. Generated blobs (2.7-8.2 M + * node trees) used to cost 18-48 us per node because usage stamping called + * ts_node_parent, which descends from the root; the walk now carries its own + * parent chain, so a deep tree costs what a shallow one does. + * + * CBM_WALK_MAX_NODES reinstates a budget for an operator who needs one + * (0 = the default, no budget). */ +#define CBM_WALK_MAX_NODES_DEFAULT 0u + +/* Read fresh each call, like cbm_max_file_bytes: cheap next to a file walk, and + * no memoized copy to go stale between runs in the same process. */ +static uint32_t cbm_walk_max_nodes(void) { + const char *raw = getenv("CBM_WALK_MAX_NODES"); + if (raw && raw[0]) { + errno = 0; + char *end = NULL; + unsigned long v = strtoul(raw, &end, 10); + if (errno == 0 && end != raw && *end == '\0' && v <= UINT32_MAX) { + return (uint32_t)v; /* 0 is a legitimate choice: no budget */ + } + /* Unparseable / out-of-range → the default, never an accidental 0. */ + } + return CBM_WALK_MAX_NODES_DEFAULT; +} + +typedef struct { + uint64_t cpu_deadline_ns; // trip once this thread's CPU time passes it + uint64_t wall_ceiling_ns; // hard wall backstop for a spinning/stuck parse + /* Set by the callback when IT ended the parse. ts_parser_parse_with_options + * returns NULL for several reasons and the budget is only one of them, but + * the failure used to be reported as "parse timeout" whenever a budget was + * configured at all. A 7-line .properties file was accused of timing out + * (2026-09-19); whatever really happened to it, saying "timeout" sent the + * next reader looking at the clock. */ + bool tripped; +} CBMParseBudget; + +#ifdef CBM_ENABLE_TEST_SEAMS +/* Deterministic RED-repro seam (armed by CBM_TEST_WALL_STALL_ON in + * cbm_extract_file_ex): when set, the timeout callback reads the WALL clock this + * many ns ahead of reality while CPU time is untouched — emulating a worker + * descheduled long enough for the old wall-only budget to elapse against + * near-zero CPU. With the CPU-time budget the parse still completes; a wall-only + * / tight-ceiling budget drops the file. Thread-local so it cannot leak across + * worker threads. Compiled only into seam-enabled test artifacts. */ +static CBM_TLS uint64_t tl_parse_wall_seam_offset_ns = 0; +#endif + +/* tree-sitter's TSProgressCallback mandates a non-const TSParseState*; a const + * parameter here would not match the opts.progress_callback assignment below. */ +// cppcheck-suppress constParameterCallback static bool cbm_timeout_cb(TSParseState *state) { - uint64_t deadline = *(uint64_t *)state->payload; - return now_ns() > deadline; + CBMParseBudget *budget = (CBMParseBudget *)state->payload; + uint64_t wall = now_ns(); +#ifdef CBM_ENABLE_TEST_SEAMS + wall += tl_parse_wall_seam_offset_ns; +#endif + bool over = + cbm_thread_cpu_time_ns() > budget->cpu_deadline_ns || wall > budget->wall_ceiling_ns; + if (over) { + budget->tripped = true; /* so a NULL tree can name its real cause */ + } + return over; } // --- Thread-local parser pool --- @@ -258,6 +417,20 @@ static TSParser *get_thread_parser(const TSLanguage *ts_lang, CBMLanguage lang) * to mimalloc would mismatch ASan/CRT frees — there these binds compile to * no-ops and the build stays unchanged. */ +/* SQLite on a dedicated mimalloc heap per thread: ON only in the index worker, + * whose default heap holds the graph (SQLite churn on that heap paid a page + * walk per allocation: 132 s vs 9.7 s on the kernel's coverage publish). OFF + * everywhere else: the daemon runs a thread per connection, and a heap + * created per such thread pins every SQLite block the shared connection + * keeps (page cache, statement cache) to pages nobody's heap owns any more -- + * the Linux soak grew 180 KB per query, 11 -> 144 MB in ten minutes, where + * the default thread heap had been flat (2026-09-14). */ +static _Atomic int g_sqlite_dedicated_heap; + +void cbm_sqlite_dedicated_heap(bool on) { + atomic_store_explicit(&g_sqlite_dedicated_heap, on ? 1 : 0, memory_order_relaxed); +} + #if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR #include @@ -270,16 +443,76 @@ static TSParser *get_thread_parser(const TSLanguage *ts_lang, CBMLanguage lang) * hook here the biggest per-request allocations in the process — SQLite's page * cache and its query working set — are invisible to the attribution profile * (#581). */ +/* SQLite allocates from a mimalloc heap of its own, one per thread. Its page + * queues then hold SQLite blocks only. Sharing the thread's default heap with + * the graph -- 40M+ blocks, millions of pages once the graph moved onto the + * core -- made every statement-journal chunk of the coverage publish step pay + * a walk over the graph's pages: 132 s on the kernel where v0.10.8, whose + * graph lived outside mimalloc, took 9.7 s (sampled 2026-09-14). mi_free + * works across heaps, so xFree and cross-thread frees are unchanged; a + * thread's heap is released with the thread. */ +static _Thread_local mi_heap_t *tl_sqlite_heap; + +/* NULL = the calling thread's default heap (mi_malloc); see the switch above. */ +static mi_heap_t *sqlite_heap(void) { + if (!atomic_load_explicit(&g_sqlite_dedicated_heap, memory_order_relaxed)) { + return NULL; + } + if (!tl_sqlite_heap) { + tl_sqlite_heap = mi_heap_new(); + } + return tl_sqlite_heap; +} + +#if defined(CBM_MEMWASTE) && CBM_MEMWASTE +#define BOUND_ALLOC(block, n, flags) \ + do { \ + if ((block) && cbm_memev_enabled()) { \ + cbm_memev_alloc_ex((block), (n), mi_usable_size(block), __builtin_return_address(0), \ + (unsigned)(flags)); \ + } \ + } while (0) +#define BOUND_REALLOC(old_block, grown, n) \ + do { \ + if ((grown) && cbm_memev_enabled()) { \ + cbm_memev_realloc((old_block), (grown), (n), mi_usable_size(grown), \ + __builtin_return_address(0)); \ + } \ + } while (0) +#define BOUND_FREE(block) cbm_memev_free(block) +#else +#define BOUND_ALLOC(block, n, flags) ((void)0) +#define BOUND_REALLOC(old_block, grown, n) ((void)0) +#define BOUND_FREE(block) ((void)0) +#endif + static void *cbm_sqlite_malloc(int n) { - void *block = mi_malloc((size_t)n); + mi_heap_t *heap = sqlite_heap(); + void *block = heap ? mi_heap_malloc(heap, (size_t)n) : mi_malloc((size_t)n); + if (block) { + cbm_mem_class_add_external(CBM_MEM_CLASS_STORE, mi_usable_size(block)); + } + BOUND_ALLOC(block, (size_t)n, 0); return block; } static void cbm_sqlite_free(void *p) { + if (p) { + cbm_mem_class_remove_external(CBM_MEM_CLASS_STORE, mi_usable_size(p)); + BOUND_FREE(p); + } mi_free(p); } static void *cbm_sqlite_realloc(void *p, int n) { - if (p) {} - void *grown = mi_realloc(p, (size_t)n); + size_t old_size = p ? mi_usable_size(p) : 0; + mi_heap_t *heap = sqlite_heap(); + CBM_MEMEV_BACKING(1); + void *grown = heap ? mi_heap_realloc(heap, p, (size_t)n) : mi_realloc(p, (size_t)n); + CBM_MEMEV_BACKING(-1); + if (grown) { + cbm_mem_class_remove_external(CBM_MEM_CLASS_STORE, old_size); + cbm_mem_class_add_external(CBM_MEM_CLASS_STORE, mi_usable_size(grown)); + } + BOUND_REALLOC(p, grown, (size_t)n); return grown; } static int cbm_sqlite_size(void *p) { @@ -292,18 +525,37 @@ static int cbm_sqlite_roundup(int n) { * through these, and they too skip the interposer. */ static void *cbm_ts_malloc(size_t n) { void *block = mi_malloc(n); + if (block) { + cbm_mem_class_add_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(block)); + } + BOUND_ALLOC(block, n, 0); return block; } static void *cbm_ts_calloc(size_t count, size_t size) { void *block = mi_calloc(count, size); + if (block) { + cbm_mem_class_add_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(block)); + } + BOUND_ALLOC(block, count * size, CBM_MEMEV_ZEROED); return block; } static void *cbm_ts_realloc(void *p, size_t n) { - if (p) {} + size_t old_size = p ? mi_usable_size(p) : 0; + CBM_MEMEV_BACKING(1); void *grown = mi_realloc(p, n); + CBM_MEMEV_BACKING(-1); + if (grown) { + cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, old_size); + cbm_mem_class_add_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(grown)); + } + BOUND_REALLOC(p, grown, n); return grown; } static void cbm_ts_free(void *p) { + if (p) { + cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(p)); + BOUND_FREE(p); + } mi_free(p); } @@ -375,6 +627,120 @@ void cbm_reset_thread_parser(void) { } } +enum { FIELD_CACHE_SLOTS = 512, FIELD_NAME_CACHED_MAX = 31 }; +typedef struct { + const TSLanguage *lang; + uint32_t len; + TSFieldId id; + char name[FIELD_NAME_CACHED_MAX + 1]; +} field_id_slot_t; +/* On the HEAP behind a thread-local pointer, not a thread-local array: glibc + * takes the static TLS block out of each thread's stack allocation, so a 24 KB + * array here is 24 KB every thread must fit — and the threads that ask for a + * small stack (the 64 KB parent-death watchdog) then fail to start at all, + * with EINVAL surfacing nowhere near the growth that caused it. See + * cbm_thread_create's fallback for the other half of that lesson. */ +static CBM_TLS field_id_slot_t *tl_field_ids; + +static field_id_slot_t *field_cache(void) { + if (!tl_field_ids) { + tl_field_ids = cbm_calloc(CBM_MEM_CLASS_OTHER, FIELD_CACHE_SLOTS * sizeof(field_id_slot_t)); + } + return tl_field_ids; +} + +static void cbm_ts_field_cache_release_thread(void) { + cbm_free(CBM_MEM_CLASS_OTHER, tl_field_ids); + tl_field_ids = NULL; +} + +TSNode cbm_ts_child_by_field_name(TSNode node, const char *name, uint32_t name_length) { + if (!name || name_length == 0 || name_length > FIELD_NAME_CACHED_MAX || ts_node_is_null(node)) { + return (ts_node_child_by_field_name)(node, name, name_length); /* the real one */ + } + field_id_slot_t *cache = field_cache(); + if (!cache) { + return (ts_node_child_by_field_name)(node, name, name_length); /* uncached, still correct */ + } + const TSLanguage *lang = ts_node_language(node); + uint64_t h = 0xcbf29ce484222325ULL ^ (uint64_t)(uintptr_t)lang; + for (uint32_t i = 0; i < name_length; i++) { + h ^= (uint8_t)name[i]; + h *= 0x100000001b3ULL; + } + field_id_slot_t *slot = &cache[h & (FIELD_CACHE_SLOTS - 1)]; + if (slot->lang != lang || slot->len != name_length || + memcmp(slot->name, name, name_length) != 0) { + /* The content is compared, never just the pointer: callers also pass + * names built in reused buffers. */ + slot->id = ts_language_field_id_for_name(lang, name, name_length); + slot->lang = lang; + slot->len = name_length; + memcpy(slot->name, name, name_length); + } + return ts_node_child_by_field_id(node, slot->id); +} + +static CBM_TLS TSTreeCursor tl_cursor; +static CBM_TLS bool tl_cursor_live = false; + +TSTreeCursor *cbm_thread_cursor(TSNode node) { + if (tl_cursor_live) { + ts_tree_cursor_reset(&tl_cursor, node); + } else { + tl_cursor = ts_tree_cursor_new(node); + tl_cursor_live = true; + } + return &tl_cursor; +} + +/* Per-depth cursors for recursive walks (cbm_cursor_acquire). A walk at depth d + * holds slot d while it recurses into depth d + 1; the LSP call walkers used to + * create and delete a cursor at EVERY node -- 49.8 M cursor stacks on the Go + * corpus (waste sanitizer, 2026-09-17). Deeper than the pool: private cursors. */ +enum { CURSOR_POOL_DEPTH = 128 }; +/* Heap-backed for the same reason as the field cache: 4 KB of cursors plus + * their state in static TLS is 4 KB charged to every thread in the image, and + * it is the kind of growth that makes a small-stack thread refuse to start. */ +typedef struct { + TSTreeCursor cursors[CURSOR_POOL_DEPTH]; + uint8_t state[CURSOR_POOL_DEPTH]; /* bit0 live, bit1 busy */ +} cursor_pool_t; +static CBM_TLS cursor_pool_t *tl_cursor_pool; +enum { CURSOR_LIVE = 1, CURSOR_BUSY = 2 }; + +TSTreeCursor *cbm_cursor_acquire(cbm_cursor_lease_t *lease, int depth, TSNode node) { + if (!tl_cursor_pool) { + tl_cursor_pool = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(cursor_pool_t)); + } + cursor_pool_t *pool = tl_cursor_pool; + if (pool && depth >= 0 && depth < CURSOR_POOL_DEPTH && !(pool->state[depth] & CURSOR_BUSY)) { + if (pool->state[depth] & CURSOR_LIVE) { + ts_tree_cursor_reset(&pool->cursors[depth], node); + } else { + pool->cursors[depth] = ts_tree_cursor_new(node); + } + pool->state[depth] = CURSOR_LIVE | CURSOR_BUSY; + lease->slot = depth; + lease->cursor = &pool->cursors[depth]; + return lease->cursor; + } + /* No pool (or the slot is taken): a private cursor is always correct. */ + lease->private_cursor = ts_tree_cursor_new(node); + lease->slot = -1; + lease->cursor = &lease->private_cursor; + return lease->cursor; +} + +void cbm_cursor_release(cbm_cursor_lease_t *lease) { + if (lease->slot < 0) { + ts_tree_cursor_delete(&lease->private_cursor); + } else if (tl_cursor_pool) { + tl_cursor_pool->state[lease->slot] &= (uint8_t)~CURSOR_BUSY; + } + lease->cursor = NULL; +} + void cbm_destroy_thread_parser(void) { // Full cleanup: delete the parser. Call on worker thread exit. if (tl_parser) { @@ -382,6 +748,29 @@ void cbm_destroy_thread_parser(void) { tl_parser = NULL; tl_parser_lang = CBM_LANG_COUNT; } + if (tl_cursor_live) { + ts_tree_cursor_delete(&tl_cursor); + tl_cursor_live = false; + } + if (tl_cursor_pool) { + bool busy = false; + for (int d = 0; d < CURSOR_POOL_DEPTH; d++) { + /* a busy slot belongs to a walk still on this stack: leave it */ + if (tl_cursor_pool->state[d] == CURSOR_LIVE) { + ts_tree_cursor_delete(&tl_cursor_pool->cursors[d]); + tl_cursor_pool->state[d] = 0; + } else if (tl_cursor_pool->state[d] & CURSOR_BUSY) { + busy = true; + } + } + /* Outstanding leases point INTO this block, so it is only released once + * no walk still holds a slot. */ + if (!busy) { + cbm_free(CBM_MEM_CLASS_OTHER, tl_cursor_pool); + tl_cursor_pool = NULL; + } + } + cbm_ts_field_cache_release_thread(); } void cbm_shutdown(void) { @@ -1041,14 +1430,89 @@ static void cbm_mark_pp_error_rows(TSNode n, uint8_t *rows, uint32_t row_count, /* Recovery subtraction (#963): tree-sitter error recovery plus the * ERROR-descending def walker often still extract constructs INSIDE a failed * region (verified: a function in an #ifdef-split ERROR region and even a - * `def broken(:` both came back as defs). A region whose every line is - * covered by definitions that START inside it is definitely recovered — its - * constructs ARE in the graph — so flagging it would be a false miss. - * Container defs (Module/Package) are ignored: a file-spanning Module node is - * not evidence the region's constructs survived. Conservative: partially - * covered regions stay flagged. */ -static bool cbm_region_is_recovered(uint32_t rs, uint32_t re, const CBMDefArray *defs) { - enum { MAX_COVER_DEFS = 256 }; + * `def broken(:` both came back as defs). The lines a definition that STARTS + * inside the region covers are therefore in the graph, and only the lines no + * definition covers can honestly be called missed. Container defs + * (Module/Package) are ignored: a file-spanning Module node is not evidence + * the region's constructs survived. + * + * This used to be all-or-nothing — a region stayed flagged whole unless every + * line was covered. On torvalds/linux (2026-09-16 probe) one ERROR node that + * swallowed the second half of kernel/sched/core.c (lines 5522–11284) survived + * because of the comment and macro lines between its 286 extracted functions, + * and the coverage report told a reader 68 % of the scheduler was unindexed. + * Now the uncovered gaps are reported instead, and a gap holding only blank, + * comment or preprocessor lines is not a miss at all. */ +static uint32_t *cbm_line_offsets(const char *src, int src_len, uint32_t *out_lines) { + uint32_t lines = 1; + for (int i = 0; i < src_len; i++) { + if (src[i] == '\n') { + lines++; + } + } + uint32_t *offs = + (uint32_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(lines + 1) * sizeof(uint32_t)); + if (!offs) { + *out_lines = 0; + return NULL; + } + uint32_t n = 0; + offs[n++] = 0; + for (int i = 0; i < src_len && n < lines; i++) { + if (src[i] == '\n') { + offs[n++] = (uint32_t)i + 1; + } + } + offs[n] = (uint32_t)src_len; + *out_lines = n; + return offs; +} + +/* A 1-based line that holds nothing a reader would call missed code: blank, + * a comment line, or a preprocessor directive (the import pass owns #include; + * #if/#endif carry no construct of their own). */ +static bool cbm_line_is_inert(const char *src, const uint32_t *offs, uint32_t nlines, + uint32_t line) { + if (!src || !offs || line == 0 || line > nlines) { + return true; + } + const char *p = src + offs[line - 1]; + const char *end = src + offs[line]; + while (p < end && (*p == ' ' || *p == '\t' || *p == '\r')) { + p++; + } + if (p >= end || *p == '\n') { + return true; + } + if (p[0] == '/' && p + 1 < end && (p[1] == '/' || p[1] == '*')) { + return true; + } + return p[0] == '*' || p[0] == '#'; +} + +static void cbm_regions_emit_gap(cbm_error_regions_t *out, uint32_t gs, uint32_t ge, + const char *src, const uint32_t *offs, uint32_t nlines) { + bool live = false; + for (uint32_t l = gs; l <= ge && !live; l++) { + live = !cbm_line_is_inert(src, offs, nlines, l); + } + if (!live) { + return; + } + if (out->count < CBM_MAX_ERROR_REGIONS) { + out->starts[out->count] = gs; + out->ends[out->count] = ge; + out->count++; + } else { + out->dropped++; + } +} + +/* Replace [rs, re] by the sub-ranges no in-region definition covers. */ +static void cbm_region_uncovered_gaps(uint32_t rs, uint32_t re, const CBMDefArray *defs, + const char *src, const uint32_t *offs, uint32_t nlines, + cbm_error_regions_t *out) { + enum { MAX_COVER_DEFS = 1024 }; uint32_t starts[MAX_COVER_DEFS]; uint32_t ends[MAX_COVER_DEFS]; int n = 0; @@ -1060,14 +1524,25 @@ static bool cbm_region_is_recovered(uint32_t rs, uint32_t re, const CBMDefArray if (d->start_line < rs || d->start_line > re) { continue; /* recovery evidence must originate inside the region */ } + /* A one-line definition salvaged from inside an error region is the + * walker's guess at broken text (`def x(:` comes back as a def, a + * mis-parsed `static _Thread_local int x` as a Variable named `int`, + * and Python's synthetic builtins all sit on line 1); only a construct + * whose body spans lines proves it was understood. Conservative: real + * one-line variables and macros inside a failed region keep their line + * flagged. */ + if (d->end_line <= d->start_line) { + continue; + } starts[n] = d->start_line; - ends[n] = d->end_line < d->start_line ? d->start_line : d->end_line; + ends[n] = d->end_line; n++; } if (n == 0) { - return false; + cbm_regions_emit_gap(out, rs, re, src, offs, nlines); + return; } - /* Insertion-sort by start, then sweep for gaps in [rs, re]. */ + /* Insertion-sort by start, then sweep and emit the gaps in [rs, re]. */ for (int i = 1; i < n; i++) { uint32_t s = starts[i]; uint32_t e = ends[i]; @@ -1080,16 +1555,18 @@ static bool cbm_region_is_recovered(uint32_t rs, uint32_t re, const CBMDefArray starts[j + 1] = s; ends[j + 1] = e; } - uint32_t covered_to = rs - 1; + uint32_t cursor = rs; for (int i = 0; i < n; i++) { - if (starts[i] > covered_to + 1) { - return false; /* uncovered gap */ + if (starts[i] > cursor) { + cbm_regions_emit_gap(out, cursor, starts[i] - 1, src, offs, nlines); } - if (ends[i] > covered_to) { - covered_to = ends[i]; + if (ends[i] + 1 > cursor) { + cursor = ends[i] + 1; } } - return covered_to >= re; + if (cursor <= re) { + cbm_regions_emit_gap(out, cursor, re, src, offs, nlines); + } } /* #961: true when 1-based `line` of `src` contains `name` (used to verify a @@ -1197,6 +1674,63 @@ static bool cbm_span_contains_callable_def(const char *src, int src_len, uint32_ return false; } +/* #1989: type-def counterpart of cbm_span_contains_callable_def. A rescued + * class/struct/enum definition has its name followed by a base clause (':') or + * a body ('{'), not a parameter list, so the callable validator would reject + * every type def pulled from the expanded tree. */ +static bool cbm_span_contains_type_def(const char *src, int src_len, uint32_t start_line, + uint32_t end_line, const char *name) { + if (!src || src_len <= 0 || !name || !name[0] || start_line == 0 || end_line < start_line) { + return false; + } + int span_start = 0; + uint32_t line = 1; + while (span_start < src_len && line < start_line) { + if (src[span_start++] == '\n') { + line++; + } + } + if (line != start_line) { + return false; + } + int span_end = span_start; + while (span_end < src_len && line <= end_line) { + if (src[span_end++] == '\n') { + line++; + } + } + size_t name_len = strlen(name); + for (int pos = span_start; pos + (int)name_len <= span_end; pos++) { + if (strncmp(src + pos, name, name_len) != 0 || + (pos > 0 && cbm_identifier_char(src[pos - 1])) || + (pos + (int)name_len < src_len && cbm_identifier_char(src[pos + name_len]))) { + continue; + } + int open = pos + (int)name_len; + while (open < span_end && isspace((unsigned char)src[open])) { + open++; + } + // Base clause (": public B"), body ("{"), or template args ("<..."). + if (open < span_end && (src[open] == ':' || src[open] == '{' || src[open] == '<')) { + return true; + } + } + return false; +} + +/* #1989: label buckets for the misparse-correction gates. A raw def extracted + * from a broken `class MOD_API Foo` shape lands under a callable label while + * the expanded tree yields a proper type label for the same name. */ +static bool cbm_def_label_is_type(const char *label) { + return label != NULL && (strcmp(label, "Class") == 0 || strcmp(label, "Enum") == 0 || + strcmp(label, "Interface") == 0 || strcmp(label, "Type") == 0 || + strcmp(label, "Struct") == 0 || strcmp(label, "Union") == 0); +} + +static bool cbm_def_label_is_callable(const char *label) { + return label != NULL && (strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0); +} + /* Remap an expanded-source definition back to the original input file. Every * line in the definition must be attributable to the main file; generated * macro bodies, included headers, and ambiguous spans fail closed. */ @@ -1224,16 +1758,19 @@ static bool cbm_remap_preprocessed_def(CBMDefinition *def, const CBMPreprocessed return true; } -static void cbm_subtract_recovered_regions(cbm_error_regions_t *regs, const CBMDefArray *defs) { - int kept = 0; - for (int i = 0; i < regs->count; i++) { - if (!cbm_region_is_recovered(regs->starts[i], regs->ends[i], defs)) { - regs->starts[kept] = regs->starts[i]; - regs->ends[kept] = regs->ends[i]; - kept++; - } +static void cbm_subtract_recovered_regions(cbm_error_regions_t *regs, const CBMDefArray *defs, + const char *src, int src_len) { + if (regs->count <= 0) { + return; } - regs->count = kept; + uint32_t nlines = 0; + uint32_t *offs = cbm_line_offsets(src, src_len, &nlines); + cbm_error_regions_t out = {{0}, {0}, 0, regs->dropped}; + for (int i = 0; i < regs->count; i++) { + cbm_region_uncovered_gaps(regs->starts[i], regs->ends[i], defs, src, offs, nlines, &out); + } + cbm_free(CBM_MEM_CLASS_EXTRACT, offs); + *regs = out; } /* #1071: a function-like macro invocation whose argument is a type token @@ -1533,6 +2070,7 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage * that bound and so a singleton OS allocation. One file in twelve thousand * pays it, which is why the cost is accepted. */ enum { CBM_EXTRACT_SCRATCH_BLOCK = CBM_SZ_512 * CBM_SZ_1K }; +enum { CBM_EXTRACT_SCRATCH_KEEP_BYTES = 4 * CBM_SZ_1K * CBM_SZ_1K }; static CBMFileResult *extract_file_ex_body(const char *source, int source_len, CBMLanguage language, const char *project, const char *rel_path, @@ -1542,13 +2080,12 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C const CBMReturnTypeTable *return_type_table, CBMArena *scratch) { // Allocate result on heap (arena inside for all string data) - enum { SINGLE = 1 }; - CBMFileResult *result = (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult)); + CBMFileResult *result = cbm_result_alloc(); if (!result) { return NULL; } - cbm_arena_init(&result->arena); + cbm_work_arena_take(&result->arena); CBMArena *a = &result->arena; /* Crash-quarantine hard guard (Stage 3c): a file the supervisor pinned as a @@ -1620,26 +2157,79 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C }; TSParseOptions opts = {0}; - uint64_t deadline_ns = 0; // cppcheck-suppress unreadVariable + CBMParseBudget budget = {0}; // cppcheck-suppress unreadVariable + uint64_t budget_ns = 0; if (timeout_micros > 0) { - deadline_ns = t0 + ((uint64_t)timeout_micros * USEC_TO_NSEC); - opts.payload = &deadline_ns; + budget_ns = (uint64_t)timeout_micros * USEC_TO_NSEC; + // Descheduling burns wall time but not CPU: gate on this thread's CPU + // time so a starved-but-parseable file is not abandoned, with a generous + // wall ceiling as a backstop against a genuinely spinning/stuck parse. + budget.cpu_deadline_ns = cbm_thread_cpu_time_ns() + budget_ns; + budget.wall_ceiling_ns = t0 + budget_ns * CBM_PARSE_WALL_CEILING_FACTOR; + opts.payload = &budget; opts.progress_callback = cbm_timeout_cb; +#ifdef CBM_ENABLE_TEST_SEAMS + tl_parse_wall_seam_offset_ns = 0; + const char *stall_on = getenv("CBM_TEST_WALL_STALL_ON"); + if (stall_on && stall_on[0] && rel_path && strstr(rel_path, stall_on)) { + // Push the wall reading past the 1x budget (the old wall-only budget + // trips) but well under the generous ceiling (the CPU-time budget + // survives): budget + 1 s, deterministic, no real timing involved. + tl_parse_wall_seam_offset_ns = budget_ns + NSEC_PER_SEC; + } +#endif } TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts); uint64_t t1 = now_ns(); +#ifdef CBM_ENABLE_TEST_SEAMS + t1 += tl_parse_wall_seam_offset_ns; /* the stall seam inflates every wall reading */ +#endif if (!tree) { result->has_error = true; - result->error_msg = - cbm_arena_strdup(a, timeout_micros > 0 ? "parse timeout" : "parse failed"); + /* Only the budget's own callback may call this a timeout. Any other + * NULL is a parse failure, and saying so is the difference between a + * reader chasing the clock and a reader chasing the real cause. */ + result->error_msg = cbm_arena_strdup(a, budget.tripped ? "parse timeout" : "parse failed"); cbm_index_mark_done(rel_path); return result; } TSNode root = ts_tree_root_node(tree); + /* No file is disqualified from the LSP walks by how long its parse took. + * + * The rule used to be "a parse that spent more than half its budget means + * this tree is too heavy for the walks" — a clock deciding which files get + * type-aware refinement, so the same repo could come back with different + * graphs. It was added on 2026-09-14 against a 23 MB single-expression C# + * JIT test that cost 354 s in the per-file walk and then crashed the + * cross-file resolve. + * + * Removing it was checked against exactly that file and that path + * (2026-09-19): the whole 12k-file C# corpus twice, cross-file resolve + * included, and hugeexpr1.cs on its own — no crash, same 1,224,981 nodes / + * 5,794,873 edges as with the rule in place, same ~100 s. Synthetic C# with + * valid nesting 10,000 levels deep indexes fine; past roughly that depth + * tree-sitter itself stops producing a usable tree, so a tree deep enough + * to endanger a recursive walk never reaches one. The 354 s is gone for a + * different reason: the walk that spent it was C#'s own, climbing with + * ts_node_parent, and it now uses the cursor (extract_usages.c). + * + * CBM_TEST_LSP_SKIP_ON still names a file for the tests that pin the + * skipped-file behaviour itself. */ +#ifdef CBM_ENABLE_TEST_SEAMS + { + const char *skip_on = getenv("CBM_TEST_LSP_SKIP_ON"); + if (skip_on && skip_on[0] && rel_path && strstr(rel_path, skip_on)) { + result->lsp_skipped = true; /* the test names the file; no timing involved */ + cbm_log_warn("extract.lsp.skipped", "reason", "test_seam", "path", + rel_path ? rel_path : ""); + } + } +#endif + // Compute module QN. Java/Go derive the module from the CONTAINING // DIRECTORY (package semantics) rather than baking the filename stem in, // so def QNs, the LSP caller_qn, and the textual calls-enclosing QN all @@ -1662,6 +2252,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C .root = root, .macro_table = macro_table, .return_type_table = return_type_table, + .walk_budget_nodes = cbm_walk_max_nodes(), }; // Run extractors: defs + imports use separate walks (unique recursion patterns), @@ -1669,6 +2260,14 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C cbm_extract_definitions(&ctx); cbm_extract_imports(&ctx); cbm_extract_unified(&ctx); + result->tree_nodes = ts_node_descendant_count(root); + result->walk_nodes_visited = ctx.walk_nodes_visited; + if (ctx.walk_budget_exhausted) { + result->walk_truncated = true; + result->lsp_skipped = true; + cbm_log_warn("extract.walk.truncated", "reason", "node_budget", "path", + rel_path ? rel_path : ""); + } // Channel detection (Socket.IO / EventEmitter) — JS/TS only. cbm_extract_channels(&ctx); @@ -1688,7 +2287,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // LSP type-aware call/usage resolution (per-file). Runs in every mode; // refines the tree-sitter + textual-resolution graph with type info. uint64_t lsp_start = now_ns(); - { + if (!result->lsp_skipped) { if (language == CBM_LANG_GO) { cbm_run_go_lsp(a, result, source, source_len, root); } @@ -1727,15 +2326,15 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C if (language == CBM_LANG_CSHARP) { cbm_run_cs_lsp(a, result, source, source_len, root); } - } - if (language == CBM_LANG_JAVA) { - cbm_run_java_lsp(a, result, source, source_len, root); - } - if (language == CBM_LANG_KOTLIN) { - cbm_run_kotlin_lsp(a, result, source, source_len, root); - } - if (language == CBM_LANG_RUST) { - cbm_run_rust_lsp(a, result, source, source_len, root); + if (language == CBM_LANG_JAVA) { + cbm_run_java_lsp(a, result, source, source_len, root); + } + if (language == CBM_LANG_KOTLIN) { + cbm_run_kotlin_lsp(a, result, source, source_len, root); + } + if (language == CBM_LANG_RUST) { + cbm_run_rust_lsp(a, result, source, source_len, root); + } } atomic_fetch_add(&total_lsp_ns, now_ns() - lsp_start); @@ -1757,6 +2356,14 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // Defs keep original-source line numbers; only CALLS are extracted from expanded source. if (language == CBM_LANG_C || language == CBM_LANG_CPP || language == CBM_LANG_CUDA) { uint64_t pp_start = now_ns(); + /* #1989: collect build-system export-macro candidates from the raw + * source. They are predefined empty in the expanded buffer (see + * preprocessor.cpp) so `class MOD_API Foo` parses with its real name, + * and the rescue loop below can adopt corrected type defs. Bounded: + * at most CBM_EXPORT_MACRO_MAX names per file. */ + char export_cands[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int export_cand_count = + cbm_export_macro_candidates(source, source_len, export_cands, CBM_EXPORT_MACRO_MAX); CBMPreprocessedSource *preprocessed = cbm_preprocess_with_map( source, source_len, rel_path, extra_defines, include_paths, language != CBM_LANG_C); if (preprocessed && preprocessed->source) { @@ -1837,17 +2444,52 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C * recover defs from it — adopting ONLY those that * intersect a raw ERROR region, whose name is visible on * the raw source line, and whose QN the raw pass did not - * already extract. */ - if (ts_node_has_error(root)) { + * already extract. + * + * #1989: build-system export macros (_API, + * _EXPORT, ...) are empty on the real compile line but + * opaque to tree-sitter, so `class MOD_API Foo` misparses + * with the macro as the type name — often with NO raw ERROR + * region at all (the class_specifier "succeeds" with the + * wrong name; enums and free functions do error out). When + * export-macro candidates were collected for this file, also + * adopt expanded defs that replace a raw def literally named + * as a candidate, or that correct a raw misparse label (the + * raw pass extracted the same name as Function/Method from + * the broken function_definition shape; the expanded tree + * yields a proper type def). Superseded raw defs are + * dropped so the corrected definition is not shadowed by + * the qualified-name dedup. With no candidates the block + * reduces to the original #961 behavior. */ + if (ts_node_has_error(root) || export_cand_count > 0) { cbm_error_regions_t raw_regs = {{0}, {0}, 0, 0}; cbm_collect_error_regions(root, &raw_regs, source, source_len); - if (raw_regs.count > 0) { + if (raw_regs.count > 0 || export_cand_count > 0) { int defs_before = result->defs.count; cbm_extract_definitions(&pp_ctx); int w = defs_before; + char *superseded = NULL; + if (export_cand_count > 0 && defs_before > 0) { + superseded = + (char *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)defs_before); + } + /* #1989: spans of defs already adopted from the + * expanded tree. A def nested inside one of these + * (an inline method of a rescued class) was dropped + * by the raw pass — the misparsed + * `class MOD_API Foo {...}` parsed as a + * function_definition, and nested defs in its body + * are never walked. Bounded: one span per adopted + * def, capped at 64. */ + struct { + uint32_t start; + uint32_t end; + } rescued_spans[64]; + int rescued_count = 0; for (int i = defs_before; i < result->defs.count; i++) { CBMDefinition *d = &result->defs.items[i]; bool adopt = false; + int supersede_j = -1; if (cbm_remap_preprocessed_def(d, preprocessed)) { for (int rj = 0; rj < raw_regs.count && !adopt; rj++) { if (d->start_line <= raw_regs.ends[rj] && @@ -1855,26 +2497,159 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C adopt = true; } } + if (!adopt && export_cand_count > 0 && d->name) { + for (int ci = 0; ci < export_cand_count && !adopt; ci++) { + for (int j = 0; j < defs_before && !adopt; j++) { + CBMDefinition *r = &result->defs.items[j]; + if (!r->name || + strcmp(r->name, export_cands[ci]) != 0) { + continue; + } + if (r->start_line <= d->end_line && + d->start_line <= r->end_line && + cbm_span_contains_type_def( + source, source_len, d->start_line, + d->end_line, d->name)) { + adopt = true; + supersede_j = j; + } + } + } + if (!adopt && d->label && cbm_def_label_is_type(d->label)) { + for (int j = 0; j < defs_before && !adopt; j++) { + CBMDefinition *r = &result->defs.items[j]; + /* Widened past callables (#1989): the + * misparse also mints value-label raw + * defs for types (`class X_API FEmpty {}` + * -> Variable FEmpty). Same name, + * overlapping span, raw label is NOT a + * type => raw is the misparse artifact. */ + if (!r->name || !r->label || !d->name || + strcmp(r->name, d->name) != 0 || + cbm_def_label_is_type(r->label)) { + continue; + } + if (r->start_line <= d->end_line && + d->start_line <= r->end_line) { + adopt = true; + supersede_j = j; + } + } + } + /* Nested rescue (#1989): a def nested inside + * an already-adopted def's span — the inline + * method of a rescued class, dropped by the + * raw pass because the class parsed as a + * function body. The same validation gates + * (name-on-line, shape, QN dedup) still run + * below before it lands. */ + if (!adopt && rescued_count > 0) { + for (int k = 0; k < rescued_count && !adopt; k++) { + if (d->start_line >= rescued_spans[k].start && + d->end_line <= rescued_spans[k].end) { + adopt = true; + } + } + } + } } - if (adopt && (!d->name || - !cbm_line_contains(source, source_len, d->start_line, - d->name) || - !cbm_span_contains_callable_def( - source, source_len, d->start_line, d->end_line, - d->name))) { + if (adopt && + (!d->name || + !cbm_line_contains(source, source_len, d->start_line, + d->name) || + (!cbm_span_contains_callable_def(source, source_len, + d->start_line, d->end_line, + d->name) && + !cbm_span_contains_type_def(source, source_len, d->start_line, + d->end_line, d->name)))) { adopt = false; } for (int j = 0; j < defs_before && adopt; j++) { + if (j == supersede_j || (superseded && superseded[j])) { + continue; // being replaced by this very def + } const char *q = result->defs.items[j].qualified_name; if (q && d->qualified_name && strcmp(q, d->qualified_name) == 0) { + /* #1989: a raw def holding the same QN is + * usually a reject — unless it is the + * misparse artifact this very def came to + * replace (raw "Function FCalc" from the + * broken `class MOD_API FCalc` shape vs the + * rescued "Class FCalc"). Same name, same + * span, raw label is not a type: supersede + * instead of dropping the correction. */ + CBMDefinition *r = &result->defs.items[j]; + if (d->label && cbm_def_label_is_type(d->label) && + r->label && r->name && d->name && + strcmp(r->name, d->name) == 0 && + !cbm_def_label_is_type(r->label) && + r->start_line == d->start_line && + r->end_line == d->end_line && superseded) { + superseded[j] = 1; + continue; + } adopt = false; } } if (adopt) { + if (supersede_j >= 0 && supersede_j < defs_before && + superseded) { + superseded[supersede_j] = 1; + } + /* Phantom-artifact suppression (#1989): on the + * raw tree `class MOD_API Foo : public Base + * {...}` parses as a function_definition whose + * declarator is the BASE-CLASS name, so the raw + * pass mints a bogus Function def named after + * the base spanning the whole class. Fingerprint: + * a raw callable def with an IDENTICAL span to + * the adopted type def whose name is one of the + * adopted def's base classes. A real method is + * a strict subset of the class span and never + * carries a base-class name. */ + if (d->label && cbm_def_label_is_type(d->label) && superseded && + d->base_classes) { + for (int j = 0; j < defs_before; j++) { + CBMDefinition *r = &result->defs.items[j]; + if (!r->label || !r->name || + !cbm_def_label_is_callable(r->label) || + r->start_line != d->start_line || + r->end_line != d->end_line) { + continue; + } + for (const char **b = d->base_classes; *b; b++) { + if (strcmp(*b, r->name) == 0) { + superseded[j] = 1; + break; + } + } + } + } + if (rescued_count < 64) { + rescued_spans[rescued_count].start = d->start_line; + rescued_spans[rescued_count].end = d->end_line; + rescued_count++; + } result->defs.items[w++] = *d; } } + if (superseded) { + int rw = 0; + for (int j = 0; j < defs_before; j++) { + if (!superseded[j]) { + result->defs.items[rw++] = result->defs.items[j]; + } + } + int shift = defs_before - rw; + if (shift > 0 && w > defs_before) { + memmove(&result->defs.items[rw], + &result->defs.items[defs_before], + (size_t)(w - defs_before) * sizeof(CBMDefinition)); + } + w -= shift; + cbm_free(CBM_MEM_CLASS_EXTRACT, superseded); + } result->defs.count = w; } } @@ -2060,7 +2835,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C * splitting a brace inside a recovered function looks unrecovered: * the refinement keeps only the discarded branch, the function starts * above it, and the evidence falls outside the range. */ - cbm_subtract_recovered_regions(®s, &result->defs); + cbm_subtract_recovered_regions(®s, &result->defs, source, source_len); /* Phase 2: cut what is left down to the lines the preprocessed parse * could not explain. */ if (pp_line_map) { @@ -2108,26 +2883,141 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C return result; } +/* ── Per-worker working arena (see cbm.h) ───────────────────────────── */ +static CBM_TLS CBMArena tl_work_arena; +static CBM_TLS bool tl_work_arena_live = false; + +/* The traversal scratch arena is kept per worker the same way. A thread keeps + * one only after it has given a working arena back (so only pipeline workers, + * which call cbm_work_arena_release when they end), and only while it holds no + * more than CBM_EXTRACT_SCRATCH_KEEP_BYTES (an outsized file's arena is not + * worth holding). Before this every file created and destroyed a + * 512 KB block -- 28,145 of them on the Go corpus, 14 GB allocated, 99.9 % + * never written and 7,136 never touched (waste sanitizer, 2026-09-17). */ +/* The parked arena lives on the HEAP, reached through a thread-local pointer, + * not inline in thread-local storage: a CBMArena is ~4 KB (256 block pointers + * and 256 sizes), and static TLS is charged to every thread AND taken out of + * each thread's stack allocation — enough of it and a small-stack thread stops + * being creatable at all (PR #2233). The holder is allocated once per thread + * and reused, so parking still costs no allocation per file. */ +static CBM_TLS CBMArena *tl_scratch_slot; +static CBM_TLS bool tl_scratch_live = false; +static CBM_TLS bool tl_scratch_keep = false; + +void cbm_work_arena_take(CBMArena *into) { + if (tl_work_arena_live) { + *into = tl_work_arena; + tl_work_arena_live = false; + cbm_arena_rewind(into); + return; + } + cbm_arena_init(into); +} + +void cbm_work_arena_release(void) { + if (tl_work_arena_live) { + cbm_arena_destroy(&tl_work_arena); + tl_work_arena_live = false; + } + if (tl_scratch_live && tl_scratch_slot) { + cbm_arena_destroy(tl_scratch_slot); + tl_scratch_live = false; + } + cbm_free(CBM_MEM_CLASS_OTHER, tl_scratch_slot); + tl_scratch_slot = NULL; + cbm_result_compact_release_thread(); + tl_scratch_keep = false; +} + +bool cbm_work_arena_keeping(void) { + return tl_scratch_keep; +} + +void cbm_work_arena_keep_begin(void) { + tl_scratch_keep = true; +} + +void cbm_work_arena_give(CBMArena *from) { + if (!from || from->nblocks == 0) { + return; + } + tl_scratch_keep = true; /* a pipeline worker: its release call will come */ + if (tl_work_arena_live || cbm_arena_capacity(from) > (size_t)CBM_WORK_ARENA_KEEP_BYTES) { + cbm_arena_destroy(from); + return; + } + tl_work_arena = *from; + tl_work_arena_live = true; + memset(from, 0, sizeof(*from)); +} + /* Public entry. Owns the traversal scratch arena for the whole of one file's * extraction: created here, handed to the body as ctx->scratch, destroyed on * the way out. The body has seven early returns, so bracketing it in a wrapper - * is what keeps that to one create and one destroy. If the arena cannot be - * created, the body is handed NULL and the traversal stacks fall back to the - * result arena, which is what shipped before #1997. */ + * is what keeps that to one create and one destroy. The arena is LAZY: it takes + * its first block when a traversal stack first needs one, so a file that builds + * no stack (most non-code files) costs nothing -- opened eagerly, 3.3 GB of + * 512 KB blocks were never read or written on the Go corpus (waste sanitizer, + * 2026-09-17). If the block cannot be allocated, the stack's allocation fails + * and it stops growing, exactly as on any later out-of-memory. */ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLanguage language, const char *project, const char *rel_path, int64_t timeout_micros, const char **extra_defines, const char **include_paths, const CBMMacroTable *macro_table, const CBMReturnTypeTable *return_type_table) { CBMArena scratch; - cbm_arena_init_sized(&scratch, CBM_EXTRACT_SCRATCH_BLOCK); - CBMFileResult *result = extract_file_ex_body( - source, source_len, language, project, rel_path, timeout_micros, extra_defines, - include_paths, macro_table, return_type_table, scratch.nblocks > 0 ? &scratch : NULL); - cbm_arena_destroy(&scratch); + if (tl_scratch_live && tl_scratch_slot) { + scratch = *tl_scratch_slot; + tl_scratch_live = false; + cbm_arena_rewind(&scratch); + } else { + cbm_arena_init_lazy(&scratch, CBM_EXTRACT_SCRATCH_BLOCK); + } + CBMFileResult *result = extract_file_ex_body(source, source_len, language, project, rel_path, + timeout_micros, extra_defines, include_paths, + macro_table, return_type_table, &scratch); + /* !tl_scratch_live: a nested extraction (an embedded language inside this + * file) may already have parked its own; never overwrite it. */ + /* Kept up to CBM_EXTRACT_SCRATCH_KEEP_BYTES, grown blocks included: the + * definitions walk draws its frames from this arena too, so a file with + * many definitions grows it past the first block, and dropping every grown + * arena turned those files into fresh 512 KB blocks for the next file. */ + if (tl_scratch_keep && !tl_scratch_live && scratch.nblocks > 0 && + cbm_arena_capacity(&scratch) <= (size_t)CBM_EXTRACT_SCRATCH_KEEP_BYTES) { + if (!tl_scratch_slot) { + tl_scratch_slot = cbm_alloc(CBM_MEM_CLASS_OTHER, sizeof(CBMArena)); + } + if (tl_scratch_slot) { + *tl_scratch_slot = scratch; + tl_scratch_live = true; + } else { + cbm_arena_destroy(&scratch); /* no holder: keep nothing, stay correct */ + } + } else { + cbm_arena_destroy(&scratch); + } return result; } +CBMFileResult *cbm_result_alloc(void) { + /* The one raw allocation of a result: cbm_free_result releases it with + * the matching free. Extraction and the spill loader both come here. */ + enum { SINGLE = 1 }; + return (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult)); +} + +void cbm_result_release_owned(CBMFileResult *result) { + if (!result) { + return; + } + for (int i = 0; i < result->owned_result_count; i++) { + cbm_free_result(result->owned_results[i]); + } + free(result->owned_results); + result->owned_results = NULL; + result->owned_result_count = 0; +} + void cbm_free_result(CBMFileResult *result) { if (!result) { return; @@ -2136,12 +3026,7 @@ void cbm_free_result(CBMFileResult *result) { ts_tree_delete(result->cached_tree); result->cached_tree = NULL; } - for (int i = 0; i < result->owned_result_count; i++) { - cbm_free_result(result->owned_results[i]); - } - free(result->owned_results); - result->owned_results = NULL; - result->owned_result_count = 0; + cbm_result_release_owned(result); cbm_arena_destroy(&result->arena); free(result); } diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index a5bdba999..a00192a18 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -6,6 +6,17 @@ #include "arena.h" #include "tree_sitter/api.h" +/* Field lookups by NAME resolve the name with a linear strncmp scan over the + * grammar's field table on every call -- 951 M strncmp calls on the Go corpus + * (waste sanitizer, 2026-09-17) across ~1,200 call sites. Tree-sitter's own + * implementation is exactly "field id for name, then child by field id"; this + * caches the first half per thread (language, name) and keeps the second. + * Every extractor includes this header, so every call site gets it. */ +TSNode cbm_ts_child_by_field_name(TSNode node, const char *name, uint32_t name_length); +/* Variadic: call sites spell the name and its length as ONE macro argument + * (TS_FIELD("body") expands to "body", 4), which must expand before the call. */ +#define ts_node_child_by_field_name(...) cbm_ts_child_by_field_name(__VA_ARGS__) + // Language enum mirrors lang.Language in Go. // Order must match lang_specs.c tables. typedef enum { @@ -248,31 +259,41 @@ typedef enum { } CBMSourceOrigin; typedef struct { - const char *callee_name; // raw callee text ("pkg.Func", "foo") - const char *enclosing_func_qn; // QN of enclosing function (or module QN) - const char *first_string_arg; // first string literal argument (URL, topic, key) or NULL - const char *second_arg_name; // second argument identifier (handler ref) or NULL - CBMCallArg args[CBM_MAX_CALL_ARGS]; // first N arguments with expressions - int arg_count; // number of captured arguments - int loop_depth; // enclosing loop nesting at the call site - int branch_depth; // enclosing branch nesting at the call site - int start_line; // 1-based source line of the call (for def range-match) - uint32_t site_start_byte; // exact AST occurrence span; end > start when present - uint32_t site_end_byte; // exclusive byte offset in the source file - CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer - bool is_method; // method/member call with an UNRESOLVED receiver. Perl: - // arrow/method call ($obj->m). TS/JS/TSX: member call - // x.foo() whose receiver is not this/super. Python: - // x.foo() where x is not self/cls/super() and is not - // rooted in an imported name. Read by the weak-member - // guard and by the pxc synthetic-carrier dedup key in - // pass_lsp_cross.c. Default false. - bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit - // C++ operator). Never fall back to textual resolution. - bool callee_is_locally_bound; // bare call foo() whose callee identifier is bound as a - // parameter of an enclosing function, so it cannot be the - // module-level foo. Python only today. Read by the - // weak-local-binding guard. Default false. + const char *callee_name; // raw callee text ("pkg.Func", "foo") + const char *enclosing_func_qn; // QN of enclosing function (or module QN) + const char *first_string_arg; // first string literal argument (URL, topic, key) or NULL + const char *second_arg_name; // second argument identifier (handler ref) or NULL + /* First arg_count captured arguments, arena-allocated on first capture; + * NULL when arg_count == 0. Was an inline args[CBM_MAX_CALL_ARGS] (256 of + * the record's 320 bytes) -- the Go corpus census (2026-09-13) put 825k + * calls at 251 MB with most of that empty slots. Readers index it exactly + * as before; only `sizeof` changed. */ + CBMCallArg *args; + int arg_count; // number of captured arguments (<= CBM_MAX_CALL_ARGS) + int loop_depth; // enclosing loop nesting at the call site + int branch_depth; // enclosing branch nesting at the call site + int start_line; // 1-based source line of the call (for def range-match) + uint32_t site_start_byte; // exact AST occurrence span; end > start when present + uint32_t site_end_byte; // exclusive byte offset in the source file + CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer + bool is_method; // method/member call with an UNRESOLVED receiver. Perl: + // arrow/method call ($obj->m). TS/JS/TSX: member call + // x.foo() whose receiver is not this/super. Python: + // x.foo() where x is not self/cls/super() and is not + // rooted in an imported name. Read by the weak-member + // guard and by the pxc synthetic-carrier dedup key in + // pass_lsp_cross.c. Default false. + bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit + // C++ operator). Never fall back to textual resolution. + bool callee_is_locally_bound; // bare call foo() whose callee identifier is bound as a + // parameter of an enclosing function, so it cannot be the + // module-level foo. Python only today. Read by the + // weak-local-binding guard. Default false. + bool receiver_is_self_attribute; // Python member call whose receiver is an attribute + // chain rooted at self/cls but not self/cls itself + // (self.compiler.apply_converters()). An object the + // class owns, not a parameter: read by the weak-member + // guard's unique-name exemption. Default false. } CBMCall; typedef struct { @@ -286,16 +307,18 @@ typedef enum { } CBMUsageKind; typedef struct { - const char *ref_name; // referenced identifier - const char *enclosing_func_qn; // QN of enclosing function (or module QN) + const char *ref_name; // referenced identifier + const char *enclosing_func_qn; // QN of enclosing function (or module QN) + /* Fixed-width fields grouped so the record packs to 40 bytes (was 48; the + * Go corpus holds 4.68M of these). Field meanings unchanged. */ + uint32_t lexical_scope_id; // extraction-local scope instance; never graph identity + uint32_t site_start_byte; // exact reference-token span; end > start when present + uint32_t site_end_byte; // exclusive byte offset in the source file CBMUsageKind kind; // ordinary USAGE or explicit callable reference + CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer bool may_be_call_reference; // syntactic candidate; exact LSP proof may upgrade its edge bool semantic_reference_blocked; // lexical evidence blocks only unproven textual fallback bool semantic_reference_local_shadow; // blocker belongs to a non-module lexical scope - uint32_t lexical_scope_id; // extraction-local scope instance; never graph identity - uint32_t site_start_byte; // exact reference-token span; end > start when present - uint32_t site_end_byte; // exclusive byte offset in the source file - CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer bool is_member_access; // token is the member half of a selector/attribute // (Go x.f — field_identifier). The extractor strips // the receiver, so this is the only surviving record @@ -541,7 +564,22 @@ typedef struct CBMFileResult { int error_region_count; bool is_test_file; int imports_count; - TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree) + TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree) + /* The parse alone used more than its share of the per-file budget: the + * per-file LSP walk and the cross-file resolve skip this file (its + * unified-extractor defs stay). Set by cbm_extract_file_ex, honoured by + * cbm_pxc_dispatch_file -- one site for every language. */ + bool lsp_skipped; + /* The unified walk stopped at its CPU budget: defs/calls/usages found up + * to that point are kept, the rest of the file is not walked. Implies + * lsp_skipped. */ + bool walk_truncated; + /* Size of this file's parse tree, and how much of it the unified walk got + * through. Reported for a truncated or LSP-skipped file so the coverage + * report says how much of it is missing, instead of leaving the gap + * silent. */ + uint32_t tree_nodes; + uint32_t walk_nodes_visited; CBMLanguage cached_lang; // language of cached tree (for parser selection) // Retained source bytes — copied into `arena` by the parallel @@ -632,6 +670,19 @@ typedef struct { * class-body variable def records which class declares it (parent_class) * without changing its module-level qualified name. NULL elsewhere. */ const char *var_parent_class; + /* Per-file walk budget in VISITED NODES (0 = unbounded). The unified cursor + * walk stops once it is spent, so no single file can hold a worker for + * minutes: a 23 MB single-expression C# test file cost 346 s in usage + * stamping alone (tree-sitter's ts_node_parent descends from the root, + * quadratic on a deep tree; 2026-09-14). What was extracted before the stop + * is kept, and the file is named in the coverage report. Counted in nodes + * rather than CPU time so that the same file always stops at the same node + * — see CBM_WALK_MAX_NODES_DEFAULT for what a clock did here. */ + uint32_t walk_budget_nodes; + bool walk_budget_exhausted; + /* How many nodes the unified walk actually visited (whether or not it ran + * out of budget) — the measurement the budget has to be expressed in. */ + uint32_t walk_nodes_visited; } CBMExtractCtx; // --- Public API --- @@ -645,6 +696,12 @@ typedef struct { // also calls it so non-main entry points (pipeline passes) still get the binds. // In the test build (no CBM_BIND_TS_ALLOCATOR) this is a no-op. void cbm_alloc_init(void); +/* SQLite allocates from a dedicated mimalloc heap per thread while on; the + * index worker turns it on (its default heap holds the graph). Off elsewhere: + * a thread-per-connection daemon would pin connection-lifetime blocks to + * dead threads. The switch exists in every build; it changes nothing where + * the allocator binds are compiled out. */ +void cbm_sqlite_dedicated_heap(bool on); // Initialize the library. Call once at startup. Returns 0 on success. int cbm_init(void); @@ -675,6 +732,41 @@ void cbm_index_mark_done(const char *rel_path); // Extract all data from one file. Caller must call cbm_free_result(). // source must remain valid for the duration of the call. // timeout_micros: per-file parse timeout in microseconds (0 = no timeout). +/* Compact a finished result: copy everything reachable from it -- every + * record array at exact count, every string once (interned by content within + * the file), the retained source -- into one exact-size arena, and destroy the + * working arena the extractors wrote into. Measured on the Go corpus + * (2026-09-13): 14.8 GB written per index, 3.4 GB reachable; the rest was + * node-text copies and abandoned array generations no one could free because + * the result owned the arena. Call once, after the last per-file write and + * before the result is cached for later passes. Later appends into the arena + * still work (growth restarts at the default block). A composite's owned + * per-unit results are released: after the deep copy nothing points at them. + * On allocation failure the result is left exactly as it was. */ +void cbm_result_compact(CBMFileResult *result); + +/* The working arena the extractors write into, per worker thread. Extraction + * takes it (rewound, pages still mapped) instead of allocating a fresh arena + * per file; compaction returns it instead of destroying it. Reusing the same + * addresses directly is what stops the purge/re-commit churn that kept a + * kernel worker at 15 GB resident with 4-5 GB charged. An arena that grew past + * CBM_WORK_ARENA_KEEP_BYTES (one giant file) is destroyed, not kept. */ +enum { CBM_WORK_ARENA_KEEP_BYTES = 16 * 1024 * 1024 }; +void cbm_work_arena_take(CBMArena *into); +void cbm_work_arena_give(CBMArena *from); +/* Drop this thread's kept working arena (end of an extraction pass). */ +void cbm_work_arena_release(void); +/* True on a thread that has given a working arena back, i.e. a pipeline + * worker whose cbm_work_arena_release is guaranteed to run: only such a thread + * may keep per-thread scratch between files. */ +bool cbm_work_arena_keeping(void); +/* Let this thread keep its per-file extraction scratch between the files of a + * sequential loop; the caller must call cbm_work_arena_release on the same + * thread when the loop ends (every return path). */ +void cbm_work_arena_keep_begin(void); +/* Free the compaction scratch this thread kept (cbm_work_arena_release calls it). */ +void cbm_result_compact_release_thread(void); + CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage language, const char *project, const char *rel_path, int64_t timeout_micros, const char **extra_defines, // NULL-terminated, or NULL @@ -696,6 +788,12 @@ CBMFileResult *cbm_extract_file_ex( // Free all memory associated with a result. void cbm_free_result(CBMFileResult *result); +/* Allocate an empty result; cbm_free_result releases it. */ +CBMFileResult *cbm_result_alloc(void); + +/* Release a composite result's per-unit results (the owner of that array). */ +void cbm_result_release_owned(CBMFileResult *result); + // Free only the cached tree from a result (caller retained it for reuse). void cbm_free_tree(CBMFileResult *result); @@ -710,6 +808,25 @@ void cbm_reset_thread_parser(void); // Destroy the thread-local parser. Call on worker thread exit. void cbm_destroy_thread_parser(void); +// This thread's reusable tree cursor, reset to `node`. Child walks run once per +// visited node; a cursor per walk was a malloc + free of its stack each time +// (49.8 M on the Go corpus, waste sanitizer 2026-09-17). Valid until the next +// call on this thread; released with the thread parser. +TSTreeCursor *cbm_thread_cursor(TSNode node); + +// Reusable cursors for RECURSIVE walks: one per recursion depth on this thread, +// so a walk that recurses from inside its child loop never shares a cursor with +// its own callers. A slot already in use (another walker nested on this thread) +// hands out a private cursor instead; release deletes it. Released with the +// thread parser. +typedef struct { + TSTreeCursor *cursor; + TSTreeCursor private_cursor; + int slot; /* -1: private */ +} cbm_cursor_lease_t; +TSTreeCursor *cbm_cursor_acquire(cbm_cursor_lease_t *lease, int depth, TSNode node); +void cbm_cursor_release(cbm_cursor_lease_t *lease); + // Shutdown the library. Call once at exit. void cbm_shutdown(void); diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index c22a11e01..e1de896b8 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -258,9 +258,18 @@ static char *extract_constructor_callee(CBMArena *a, TSNode node, const char *so } // Try common field-based callee resolution (function, name, method fields). +static TSNode unwrap_await_callee(TSNode node) { + if (ts_node_is_null(node) || strcmp(ts_node_type(node), "await_expression") != 0 || + ts_node_named_child_count(node) == 0) { + return node; + } + return ts_node_named_child(node, 0); +} + static char *extract_callee_from_fields(CBMArena *a, TSNode node, const char *source) { // Try "function" field TSNode func_node = ts_node_child_by_field_name(node, TS_FIELD("function")); + func_node = unwrap_await_callee(func_node); if (!ts_node_is_null(func_node)) { const char *fk = ts_node_type(func_node); if (strcmp(fk, "selector_expression") == 0) { @@ -2059,6 +2068,20 @@ static void extract_call_args(CBMExtractCtx *ctx, TSNode args, CBMCall *call) { for (uint32_t ai = 0; ai < argc && call->arg_count < CBM_MAX_CALL_ARGS; ai++) { TSNode arg_node = ts_node_named_child(args, ai); const char *ak = ts_node_type(arg_node); + /* tree-sitter lists a comment inside the argument list as a named + * child (Java block_comment/line_comment, C-family comment). It is + * not an argument: taken as one, a leading block comment became + * args[0] and the Route pass read its text as a URL. */ + if (strcmp(ak, "comment") == 0 || strcmp(ak, "line_comment") == 0 || + strcmp(ak, "block_comment") == 0) { + continue; + } + if (!call->args) { + call->args = cbm_arena_calloc(ctx->arena, CBM_MAX_CALL_ARGS * sizeof(CBMCallArg)); + if (!call->args) { + return; + } + } CBMCallArg *ca = &call->args[call->arg_count]; memset(ca, 0, sizeof(*ca)); @@ -2257,6 +2280,61 @@ static const char *extract_binary_concat_suffix(CBMExtractCtx *ctx, TSNode node) } // Try to extract URL/topic from a positional argument (string or constant). +/* Swift names its call arguments, so each one is a value_argument that may lead + * with a value_argument_label — the `from:` in `data(from: url)`. Return the + * value itself, and leave any other node exactly as it came in. */ +static TSNode swift_argument_value(TSNode arg) { + if (strcmp(ts_node_type(arg), "value_argument") != 0 || ts_node_named_child_count(arg) == 0) { + return arg; + } + TSNode val = ts_node_named_child(arg, 0); + if (strcmp(ts_node_type(val), "value_argument_label") == 0 && + ts_node_named_child_count(arg) > 1) { + val = ts_node_named_child(arg, 1); + } + return val; +} + +/* Swift has no URL literal, so almost no real code passes a bare string to a + * request. It writes `URL(string: "https://…")!` instead, and the literal then + * sits two levels down: past the trailing `!`, which the grammar models as a + * postfix_expression, and inside the constructor's own argument list. + * + * Unwrap both so that literal is as reachable as a bare one. Only the three + * Foundation types that take a URL string are unwrapped — any other call keeps + * its own meaning, and a non-literal argument such as `URL(string: base + path)` + * falls through to the ordinary handling unchanged. */ +static TSNode swift_unwrap_url_constructor(CBMExtractCtx *ctx, TSNode arg) { + /* Step past a trailing "!" or "?". */ + if (strcmp(ts_node_type(arg), "postfix_expression") == 0) { + TSNode target = ts_node_child_by_field_name(arg, TS_FIELD("target")); + if (!ts_node_is_null(target)) { + arg = target; + } + } + if (strcmp(ts_node_type(arg), "call_expression") != 0) { + return arg; + } + TSNode callee = ts_node_named_child(arg, 0); + if (ts_node_is_null(callee) || strcmp(ts_node_type(callee), "simple_identifier") != 0) { + return arg; + } + const char *name = cbm_node_text(ctx->arena, callee, ctx->source); + if (!name || (strcmp(name, "URL") != 0 && strcmp(name, "URLComponents") != 0 && + strcmp(name, "URLRequest") != 0)) { + return arg; + } + TSNode suffix = cbm_find_child_by_kind(arg, "call_suffix"); + if (ts_node_is_null(suffix)) { + return arg; + } + TSNode inner = cbm_find_child_by_kind(suffix, "value_arguments"); + if (ts_node_is_null(inner) || ts_node_named_child_count(inner) == 0) { + return arg; + } + return swift_argument_value(ts_node_named_child(inner, 0)); +} + static const char *extract_positional_url(CBMExtractCtx *ctx, TSNode arg, const char *ak) { /* JS/TS template literals: `/things/${id}` normalizes to "/things/{}" so the * client URL joins the server route's canonical placeholder (issue #1006). */ @@ -2300,15 +2378,12 @@ static const char *extract_url_or_topic_arg(CBMExtractCtx *ctx, TSNode args) { } /* Swift wraps each argument in a value_argument that may lead with its * label, so `data(from: url)` would otherwise yield the label `from` - * rather than the value. Step past a leading value_argument_label. */ - if (strcmp(ts_node_type(arg), "value_argument") == 0 && - ts_node_named_child_count(arg) > 0) { - TSNode val = ts_node_named_child(arg, 0); - if (strcmp(ts_node_type(val), "value_argument_label") == 0 && - ts_node_named_child_count(arg) > 1) { - val = ts_node_named_child(arg, 1); - } - arg = val; + * rather than the value. */ + arg = swift_argument_value(arg); + /* A Swift URL is usually built rather than written bare, and the + * literal then sits inside that constructor. */ + if (ctx->language == CBM_LANG_SWIFT) { + arg = swift_unwrap_url_constructor(ctx, arg); } const char *ak = ts_node_type(arg); @@ -2950,6 +3025,28 @@ static char *resolve_objectscript_instance_call(CBMArena *a, TSNode node, const * statically-known type here, so the call must not bind by short name alone. * Note `self.client.send()` is NOT exempt: the receiver is `self.client`, an * attribute of unknown type, not `self` itself. */ +/* True when a Python attribute-call receiver is an attribute chain ROOTED at + * self/cls but is not self/cls itself: `self.compiler.apply_converters()` has + * receiver `self.compiler`, an object the class owns. The weak-member guard's + * unique-name exemption keys on this shape (see cbm_weak_member_unique_name_ + * exempt): a bare parameter (`accelerator.backward()`) carries no ownership + * evidence and stays suppressed. Direct `self.m()` is already exempt via + * python_receiver_is_exempt and is deliberately NOT flagged here. */ +static bool python_receiver_rooted_at_self(CBMExtractCtx *ctx, TSNode receiver) { + if (ts_node_is_null(receiver) || strcmp(ts_node_type(receiver), "attribute") != 0) { + return false; + } + TSNode root = receiver; + while (!ts_node_is_null(root) && strcmp(ts_node_type(root), "attribute") == 0) { + root = ts_node_child_by_field_name(root, TS_FIELD("object")); + } + if (ts_node_is_null(root) || strcmp(ts_node_type(root), "identifier") != 0) { + return false; + } + char *name = cbm_node_text(ctx->arena, root, ctx->source); + return name && (strcmp(name, "self") == 0 || strcmp(name, "cls") == 0); +} + static bool python_receiver_is_exempt(CBMExtractCtx *ctx, TSNode receiver) { if (ts_node_is_null(receiver)) { return false; @@ -3515,6 +3612,7 @@ static CBMPrimaryCalleeSelection select_primary_callee(CBMExtractCtx *ctx, TSNod } selection.expr = language_specific_callee_expr(ctx->language, node); + selection.expr = unwrap_await_callee(selection.expr); if (is_dynamic_callee_expr(ctx, selection.expr)) { selection.expr = (TSNode){0}; return selection; @@ -3679,6 +3777,7 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "attribute") == 0) { TSNode obj = ts_node_child_by_field_name(fn, TS_FIELD("object")); call.is_method = !python_receiver_is_exempt(ctx, obj); + call.receiver_is_self_attribute = python_receiver_rooted_at_self(ctx, obj); } else if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "identifier") == 0) { call.callee_is_locally_bound = python_callee_is_bound_parameter(ctx, state, fn); } @@ -3768,6 +3867,13 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML if (strcmp(ack, "method_arg") != 0) { continue; } + if (!call.args) { + call.args = cbm_arena_calloc(ctx->arena, + CBM_MAX_CALL_ARGS * sizeof(CBMCallArg)); + if (!call.args) { + break; + } + } CBMCallArg *ca = &call.args[call.arg_count]; memset(ca, 0, sizeof(*ca)); ca->index = call.arg_count; diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index b49147dc6..916f1136f 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -3477,6 +3477,159 @@ static TSNode find_function_params(TSNode func_node, CBMLanguage lang) { return params; } +/* ── C-family declared return type ────────────────────────────────── + * The C-family grammars split a declared return type three ways: the `type` + * field (`char`), sibling type_qualifier nodes (`const`), and the + * pointer/reference declarators wrapping the function declarator (`*`). Taking + * only the `type` field published `const char *get_name(void)` as "char". + * + * Canonical spelling: leading cv-qualifiers in source order, the base type text + * verbatim, then one space and the declarator markers outermost-first with no + * space between them — `const char *`, `char **`, `Text &`, `Text *&`. A + * qualifier on a pointer level follows its `*` and is separated from the next + * marker by a space: `char *const *`. A qualifier written after the base type + * (`char const *`) is normalized to the leading position. */ + +/* Output sink: measures when buf is NULL, writes otherwise. Rendering twice + * sizes the arena allocation exactly without a second copy of the logic. */ +typedef struct { + char *buf; + size_t len; +} c_rt_out_t; + +static void c_rt_put(c_rt_out_t *out, const char *text, size_t n) { + if (out->buf) { + memcpy(out->buf + out->len, text, n); + } + out->len += n; +} + +static void c_rt_put_str(c_rt_out_t *out, const char *text) { + c_rt_put(out, text, strlen(text)); +} + +static void c_rt_put_node(c_rt_out_t *out, TSNode node, const char *source) { + uint32_t start = ts_node_start_byte(node); + uint32_t end = ts_node_end_byte(node); + if (end > start) { + c_rt_put(out, source + start, (size_t)(end - start)); + } +} + +/* type_qualifier also covers keywords that are not part of the type + * (`constexpr`, `_Noreturn`, `mutable`, `__extension__`, …). Keep only the ones + * that are, so `constexpr int f()` still returns "int". */ +static bool is_c_return_cv_qualifier(TSNode node, const char *source) { + static const char *const kept[] = {"const", "volatile", "restrict", "__restrict", + "__restrict__", "_Atomic", NULL}; + if (strcmp(ts_node_type(node), "type_qualifier") != 0) { + return false; + } + uint32_t start = ts_node_start_byte(node); + uint32_t end = ts_node_end_byte(node); + size_t len = end > start ? (size_t)(end - start) : 0; + for (const char *const *k = kept; *k; k++) { + if (strlen(*k) == len && memcmp(source + start, *k, len) == 0) { + return true; + } + } + return false; +} + +static bool is_c_declarator_lang(CBMLanguage lang) { + return lang == CBM_LANG_C || lang == CBM_LANG_CPP || lang == CBM_LANG_CUDA || + lang == CBM_LANG_GLSL || lang == CBM_LANG_HLSL || lang == CBM_LANG_ISPC || + lang == CBM_LANG_SLANG || lang == CBM_LANG_OBJC; +} + +/* Render the canonical return type into `out`; returns how many qualifiers and + * markers were added around the base type (0 = the base text alone is already + * the whole type). The declarator walk is one strict child chain, so it is + * O(depth) with no recursion and needs no depth cap. It stops at the first node + * that is neither a pointer nor a reference declarator: for a function returning + * a function pointer (`int (*f(void))(int)`) that is the outer + * function_declarator, which leaves the base type as it was. */ +static size_t c_rt_render(c_rt_out_t *out, TSNode func_node, TSNode type_node, TSNode declarator, + const char *source) { + size_t added = 0; + uint32_t decl_start = ts_node_start_byte(declarator); + uint32_t nc = ts_node_named_child_count(func_node); + for (uint32_t i = 0; i < nc; i++) { + TSNode ch = ts_node_named_child(func_node, i); + if (ts_node_start_byte(ch) >= decl_start) { + break; + } + if (is_c_return_cv_qualifier(ch, source)) { + c_rt_put_node(out, ch, source); + c_rt_put_str(out, " "); + added++; + } + } + c_rt_put_node(out, type_node, source); + + bool need_space = true; + TSNode decl = declarator; + while (!ts_node_is_null(decl)) { + const char *dk = ts_node_type(decl); + bool is_ref = strcmp(dk, "reference_declarator") == 0; + if (!is_ref && strcmp(dk, "pointer_declarator") != 0) { + break; + } + if (need_space) { + c_rt_put_str(out, " "); + need_space = false; + } + /* A reference_declarator opens with its `&` / `&&` token. */ + TSNode marker = ts_node_child(decl, 0); + if (is_ref && !ts_node_is_null(marker) && !ts_node_is_named(marker)) { + c_rt_put_node(out, marker, source); + } else { + c_rt_put_str(out, is_ref ? "&" : "*"); + } + added++; + uint32_t dn = ts_node_named_child_count(decl); + for (uint32_t i = 0; i < dn; i++) { + TSNode q = ts_node_named_child(decl, i); + if (is_c_return_cv_qualifier(q, source)) { + c_rt_put_node(out, q, source); + need_space = true; + added++; + } + } + /* tree-sitter-cpp/-cuda give a reference_declarator's inner declarator no + * `declarator` field (see find_c_params); it is the one named child. */ + TSNode inner = ts_node_child_by_field_name(decl, TS_FIELD("declarator")); + if (ts_node_is_null(inner) && is_ref && dn > 0) { + inner = ts_node_named_child(decl, 0); + } + decl = inner; + } + return added; +} + +/* Declared return type of a C-family function/method node whose `type` field is + * `type_node`. Any other language, and any type with nothing around its base + * type, gets the base type text exactly as before. */ +static char *c_declared_return_type(CBMExtractCtx *ctx, TSNode func_node, TSNode type_node) { + CBMArena *a = ctx->arena; + TSNode declarator = ts_node_child_by_field_name(func_node, TS_FIELD("declarator")); + if (!is_c_declarator_lang(ctx->language) || ts_node_is_null(declarator)) { + return cbm_node_text(a, type_node, ctx->source); + } + c_rt_out_t out = {NULL, 0}; + if (c_rt_render(&out, func_node, type_node, declarator, ctx->source) == 0) { + return cbm_node_text(a, type_node, ctx->source); + } + out.buf = (char *)cbm_arena_alloc(a, out.len + NULL_TERM); + if (!out.buf) { + return cbm_node_text(a, type_node, ctx->source); + } + out.len = 0; + (void)c_rt_render(&out, func_node, type_node, declarator, ctx->source); + out.buf[out.len] = '\0'; + return out.buf; +} + // C++: resolve trailing return type (auto f() -> Type) on a declarator node. // Updates def->return_type and def->return_types if trailing type found. static void resolve_cpp_trailing_return(CBMArena *a, TSNode func_node, const char *source, @@ -3712,7 +3865,7 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec for (const char **f = rt_fields; *f; f++) { TSNode rt = ts_node_child_by_field_name(func_node, *f, (uint32_t)strlen(*f)); if (!ts_node_is_null(rt)) { - def.return_type = cbm_node_text(a, rt, ctx->source); + def.return_type = c_declared_return_type(ctx, func_node, rt); def.return_types = extract_return_types(a, rt, ctx->source, ctx->language); break; } @@ -4964,7 +5117,7 @@ static void push_method_def(CBMExtractCtx *ctx, TSNode child, TSNode class_node, for (const char **f = rt_fields; *f; f++) { TSNode rt = ts_node_child_by_field_name(child, *f, (uint32_t)strlen(*f)); if (!ts_node_is_null(rt)) { - def.return_type = cbm_node_text(a, rt, ctx->source); + def.return_type = c_declared_return_type(ctx, child, rt); break; } } @@ -7053,6 +7206,13 @@ typedef struct { int cap; const char *path; // for the WARN when the ceiling is hit (may be NULL) bool warned; + /* The per-file traversal scratch (ctx->scratch) when there is one: frames + * then come from memory the thread reuses file after file, where a malloc + * of 256 frames per file was 28 k allocations and 255 MB never written on + * the Go corpus (waste sanitizer, 2026-09-17). Growth copies into a + * doubled buffer and abandons the old one to the arena, like TSNodeStack. + * NULL: the heap, freed by the walk. */ + CBMArena *arena; } wd_stack_t; // Generous safety ceiling (frames), env-overridable via CBM_WALK_DEFS_MAX. @@ -7082,7 +7242,16 @@ static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) { } return; // bounded: stop growing (warned, not silent) } - walk_defs_frame_t *nd = safe_realloc(s->data, (size_t)ncap * sizeof(walk_defs_frame_t)); + walk_defs_frame_t *nd = NULL; + if (s->arena) { + nd = (walk_defs_frame_t *)cbm_arena_alloc(s->arena, + (size_t)ncap * sizeof(walk_defs_frame_t)); + if (nd && s->top > 0) { + memcpy(nd, s->data, (size_t)s->top * sizeof(walk_defs_frame_t)); + } + } else { + nd = safe_realloc(s->data, (size_t)ncap * sizeof(walk_defs_frame_t)); + } if (!nd) { /* OOM — safe_realloc already freed the old buffer. Bail cleanly: drop * pending frames so the walk_defs loop drains and exits without a NULL @@ -7747,6 +7916,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, (void)depth_unused; wd_stack_t s = {0}; s.path = ctx->rel_path; + s.arena = ctx->scratch; wd_push(&s, root, ctx->enclosing_class_qn); while (s.top > 0) { @@ -7899,7 +8069,9 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, * collection is mandatory (see wd_push_children_reverse). */ wd_push_children_reverse(&s, node, frame.enclosing_class_qn); } - free(s.data); + if (!s.arena) { + free(s.data); + } } void cbm_extract_definitions_without_module(CBMExtractCtx *ctx) { diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 544020db3..fcdfc8e09 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -133,6 +133,19 @@ static void parse_go_import_spec(CBMExtractCtx *ctx, TSNode spec) { return; } + /* cgo: `import "C"` names the cgo PSEUDO-package, not a real one — "C" is + * reserved by go/build and no package with that import path can exist. + * Emitting it as an ordinary import makes the import resolver fall through + * to its symbol-name fallback (cbm_pipeline_resolve_import_node Strategy 3, + * pass_pkgmap.c), which matches the literal name "C" against every project + * node called C and picks the lexicographically smallest: on a real Go repo + * all 27 cgo files pointed their IMPORTS edge at the same unrelated `C` + * member of a test helper. Drop it — the C side is not a graph package, and + * the file's remaining imports are unaffected. */ + if (strcmp(path, "C") == 0) { + return; + } + TSNode name_node = ts_node_child_by_field_name(spec, TS_FIELD("name")); const char *local_name = !ts_node_is_null(name_node) ? cbm_node_text(a, name_node, ctx->source) : path_last(a, path); diff --git a/internal/cbm/extract_node_stack.h b/internal/cbm/extract_node_stack.h index 0013d53b1..30a27968b 100644 --- a/internal/cbm/extract_node_stack.h +++ b/internal/cbm/extract_node_stack.h @@ -12,9 +12,9 @@ * * Growth abandons the old buffer in that scratch arena, which is free because * the arena dies with the file. The initial capacities below are unchanged, but - * they are no longer what a small file costs: cbm_extract_file_ex creates the - * scratch arena's first block for every call, including calls that build no - * stack at all, and reclaims it on the way out. + * they are no longer what a small file costs: cbm_extract_file_ex hands every + * call a lazy scratch arena, whose first block is taken by the first stack + * built, so a call that builds no stack allocates nothing. */ #ifndef CBM_EXTRACT_NODE_STACK_H #define CBM_EXTRACT_NODE_STACK_H @@ -84,13 +84,14 @@ static inline TSNode ts_nstack_pop(TSNodeStack *s) { */ static inline void ts_nstack_push_children(TSNodeStack *s, TSNode node) { int base = s->count; - TSTreeCursor cursor = ts_tree_cursor_new(node); - if (ts_tree_cursor_goto_first_child(&cursor)) { + /* The thread's reusable cursor (cbm_thread_cursor): the walk completes + * before anything else on this thread can ask for it. */ + TSTreeCursor *cursor = cbm_thread_cursor(node); + if (ts_tree_cursor_goto_first_child(cursor)) { do { - ts_nstack_push(s, ts_tree_cursor_current_node(&cursor)); - } while (ts_tree_cursor_goto_next_sibling(&cursor)); + ts_nstack_push(s, ts_tree_cursor_current_node(cursor)); + } while (ts_tree_cursor_goto_next_sibling(cursor)); } - ts_tree_cursor_delete(&cursor); /* Reverse [base, count) so the first child pops first (forward order). */ int lo = base, hi = s->count - 1; while (lo < hi) { diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index 5c2905f9c..086f501a0 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -5,6 +5,8 @@ #include "lang_specs.h" // CBMLangSpec, cbm_lang_spec, CBM_LANG_* #include "tree_sitter/api.h" // TSNode, TSTreeCursor, ts_tree_cursor_*, ts_node_* #include "foundation/constants.h" +#include "foundation/compat.h" // cbm_thread_cpu_time_ns +#include enum { MAX_INFRA_BINDINGS = 8 }; @@ -2596,9 +2598,33 @@ void cbm_extract_unified(CBMExtractCtx *ctx) { state.branch_depth = 0; uint32_t depth = 0; + uint32_t visited = 0; +#ifdef CBM_ENABLE_TEST_SEAMS + /* CBM_TEST_WALK_BUDGET_NODES=: spend the budget after n nodes. Production + * now counts nodes too, so this seam only shortens the budget; it no longer + * has to stand in for a mechanism of its own. */ + uint32_t budget_nodes = ctx->walk_budget_nodes; + { + const char *seam = getenv("CBM_TEST_WALK_BUDGET_NODES"); + if (seam && seam[0]) { + budget_nodes = (uint32_t)strtoul(seam, NULL, 10); + } + } +#else + const uint32_t budget_nodes = ctx->walk_budget_nodes; +#endif for (;;) { TSNode node = ts_tree_cursor_current_node(&cursor); + visited++; + /* The budget is spent in nodes, so the walk of a given file always ends + * on the same node — on any machine, under any load. A CPU deadline + * here made the graph differ between two runs on one machine; see + * CBM_WALK_MAX_NODES_DEFAULT (cbm.c). */ + if (budget_nodes != 0 && visited > budget_nodes) { + ctx->walk_budget_exhausted = true; + break; + } bool trivia = is_unified_trivia_node(node); if (!trivia) { /* Trivia consumes no semantic state. Scope expiry may be deferred @@ -2652,6 +2678,7 @@ void cbm_extract_unified(CBMExtractCtx *ctx) { } cbm_finalize_lexical_usages(ctx, &state); + ctx->walk_nodes_visited = visited; ts_tree_cursor_delete(&occurrence_cursor); ts_tree_cursor_delete(&cursor); } diff --git a/internal/cbm/extract_usages.c b/internal/cbm/extract_usages.c index fd83864ac..f6b391315 100644 --- a/internal/cbm/extract_usages.c +++ b/internal/cbm/extract_usages.c @@ -1355,20 +1355,20 @@ static bool is_direct_argument_value(TSNode node) { return !ts_node_is_null(grandparent) && is_argument_container_kind(ts_node_type(grandparent)); } -/* Cursor-backed counterpart for the unified walker. `cursor` must currently - * point at `node`; it is consumed while walking toward the argument owner. */ -static bool is_direct_argument_value_cursor(TSNode node, TSTreeCursor *cursor) { - TSNode current = node; - while (!ts_node_is_null(current)) { - TSNode parent; - const char *field; - if (!occurrence_parent(cursor, current, &parent, &field)) - return false; +/* The body of the walk above, entered one level in: for a caller that has + * ALREADY stepped the cursor onto `parent` and knows which `field` of it the + * node below occupies. A caller that climbed to find its site has that pair in + * hand, and re-deriving it would mean either stepping the cursor twice or + * paying ts_node_parent for what the cursor just told us. */ +static bool is_direct_argument_value_from(TSNode parent, const char *field, TSTreeCursor *cursor) { + for (;;) { const char *parent_kind = ts_node_type(parent); if (is_labeled_argument_kind(parent_kind)) { if (!field || strcmp(field, "value") != 0) return false; - current = parent; + /* The labeled argument now has to be the argument value itself. */ + if (!occurrence_parent(cursor, parent, &parent, &field)) + return false; continue; } if (field && strcmp(field, "arguments") == 0) @@ -1389,7 +1389,16 @@ static bool is_direct_argument_value_cursor(TSNode node, TSTreeCursor *cursor) { return occurrence_parent(cursor, parent, &grandparent, &argument_field) && is_argument_container_kind(ts_node_type(grandparent)); } - return false; +} + +/* Cursor-backed counterpart for the unified walker. `cursor` must currently + * point at `node`; it is consumed while walking toward the argument owner. */ +static bool is_direct_argument_value_cursor(TSNode node, TSTreeCursor *cursor) { + TSNode parent; + const char *field; + if (!occurrence_parent(cursor, node, &parent, &field)) + return false; + return is_direct_argument_value_from(parent, field, cursor); } static bool is_direct_argument_value_walk(TSNode node, WalkState *state) { @@ -1443,15 +1452,36 @@ static bool language_may_stamp_exact_callable_value_candidate(CBMLanguage langua * arguments are narrow syntactic candidates only. A language LSP must still * prove one target at this exact occurrence before the graph upgrades USAGE to * CALL_REFERENCE; unresolved, reassigned, and composite expressions stay USAGE. */ -static TSNode csharp_callable_value_site(TSNode node) { - usage_slow_parent_fallback_test_note(); +/* One step up from `site`, on the walk cursor when the caller has one. Every + * other language branch in call_reference_candidate_site climbs this way; C# + * used ts_node_parent, which tree-sitter answers by descending from the ROOT, + * so each step costs O(depth) with a child scan at every level. On the + * generated .NET JIT tests — single expressions megabytes deep — that was + * 18-48 us per visited node and ~85% of the whole unified walk (sampled + * 2026-09-19: stamp_usage_site -> ts_node_parent -> + * ts_node_child_with_descendant). Those files are also the ones the old CPU + * deadline used to cut off mid-walk, so the slow path and the nondeterminism + * it forced were the same defect seen from two ends. */ +static bool csharp_site_parent(TSTreeCursor *cursor, TSNode site, TSNode *parent, + const char **field) { + if (!cursor) { + usage_slow_parent_fallback_test_note(); + } + return occurrence_parent(cursor, site, parent, field); +} + +static TSNode csharp_callable_value_site(TSNode node, TSNode parent, const char *parent_field, + TSTreeCursor *cursor) { const char *kind = ts_node_type(node); if (strcmp(kind, "identifier") != 0 && strcmp(kind, "simple_identifier") != 0) { return (TSNode){0}; } + /* The caller has already resolved node's parent (and the field it occupies) + * with the cursor, so the climb starts from that pair instead of asking for + * it again. Invariant below: `parent`/`field` always describe `site`. */ TSNode site = node; - TSNode parent = ts_node_parent(site); + const char *field = parent_field; if (!ts_node_is_null(parent) && strcmp(ts_node_type(parent), "generic_name") == 0) { TSNode name = ts_node_child_by_field_name(parent, TS_FIELD("name")); if (ts_node_is_null(name) && ts_node_named_child_count(parent) > 0) { @@ -1461,7 +1491,9 @@ static TSNode csharp_callable_value_site(TSNode node) { return (TSNode){0}; } site = parent; - parent = ts_node_parent(site); + if (!csharp_site_parent(cursor, site, &parent, &field)) { + return (TSNode){0}; + } } if (!ts_node_is_null(parent) && strcmp(ts_node_type(parent), "member_access_expression") == 0) { @@ -1470,7 +1502,9 @@ static TSNode csharp_callable_value_site(TSNode node) { return (TSNode){0}; } site = parent; - parent = ts_node_parent(site); + if (!csharp_site_parent(cursor, site, &parent, &field)) { + return (TSNode){0}; + } } while (!ts_node_is_null(parent) && @@ -1483,12 +1517,17 @@ static TSNode csharp_callable_value_site(TSNode node) { return (TSNode){0}; } site = parent; - parent = ts_node_parent(site); + if (!csharp_site_parent(cursor, site, &parent, &field)) { + return (TSNode){0}; + } } /* The wrapper chain proves admission, but the semantic row is keyed to the * terminal method-name identifier. Keep the raw carrier on that leaf. */ - return is_direct_argument_value(site) ? node : (TSNode){0}; + if (ts_node_is_null(parent)) { + return (TSNode){0}; + } + return is_direct_argument_value_from(parent, field, cursor) ? node : (TSNode){0}; } /* Climb out of the parentheses wrapping a direct argument and return the @@ -1545,10 +1584,12 @@ static TSNode call_reference_candidate_site(CBMExtractCtx *ctx, TSNode node, con const char *parent_field = NULL; if (!cursor) { usage_slow_parent_fallback_test_note(); - parent = ts_node_parent(node); - } else { - (void)occurrence_parent(cursor, node, &parent, &parent_field); } + /* Resolve the parent AND the field node occupies in it, on both paths. The + * C# site walk below carries that field into the argument test, and + * ts_node_parent on its own would leave it NULL there — silently losing the + * "this node IS the arguments" case whenever no cursor is available. */ + (void)occurrence_parent(cursor, node, &parent, &parent_field); bool ts_family = ctx->language == CBM_LANG_JAVASCRIPT || ctx->language == CBM_LANG_TYPESCRIPT || ctx->language == CBM_LANG_TSX || ctx->language == CBM_LANG_ARKTS; if (ts_family && strcmp(kind, "property_identifier") == 0 && !ts_node_is_null(parent) && @@ -1588,7 +1629,7 @@ static TSNode call_reference_candidate_site(CBMExtractCtx *ctx, TSNode node, con return is_direct_argument_value_walk(node, state) ? node : (TSNode){0}; } if (ctx->language == CBM_LANG_CSHARP) { - return csharp_callable_value_site(node); + return csharp_callable_value_site(node, parent, parent_field, cursor); } if (strcmp(kind, "identifier") != 0 && strcmp(kind, "simple_identifier") != 0) { return (TSNode){0}; diff --git a/internal/cbm/lsp/c_lsp.c b/internal/cbm/lsp/c_lsp.c index c5991561e..374f6f404 100644 --- a/internal/cbm/lsp/c_lsp.c +++ b/internal/cbm/lsp/c_lsp.c @@ -63,6 +63,7 @@ void c_lsp_init(CLSPContext *ctx, CBMArena *arena, const char *source, int sourc ctx->source = source; ctx->source_len = source_len; ctx->registry = registry; + ctx->registry_head = (CBMTypeRegistry *)registry; ctx->module_qn = module_qn; ctx->module_qn_len = module_qn ? strlen(module_qn) : 0; ctx->enclosing_func_qn = module_qn; @@ -2819,10 +2820,10 @@ static const CBMRegisteredFunc *c_lookup_member_depth(CLSPContext *ctx, const ch size_t slen = strlen(shortn); const char *best_qn = NULL; CBMTypeShortIter it; - cbm_registry_types_by_short_name(ctx->registry, shortn, &it); + cbm_registry_types_by_short_name_chain(ctx->registry, shortn, &it); int i; while ((i = cbm_type_short_iter_next(&it)) >= 0) { - const char *q = ctx->registry->types[i].qualified_name; + const char *q = it.reg->types[i].qualified_name; if (!q) { continue; } @@ -3670,12 +3671,12 @@ static const CBMRegisteredFunc *c_lookup_free_operator(CLSPContext *ctx, const c const char *right_ns = extract_namespace_from_qn(ctx->arena, right_qn); const CBMRegisteredFunc *match = NULL; CBMFreeFuncIter it; - cbm_registry_free_funcs_by_short_name(ctx->registry, operator_name, &it); + cbm_registry_free_funcs_by_short_name_chain(ctx->registry, operator_name, &it); for (int index = cbm_free_func_iter_next(&it); index >= 0; index = cbm_free_func_iter_next(&it)) { - if (index >= ctx->registry->func_count) + if (index >= it.reg->func_count) continue; - const CBMRegisteredFunc *candidate = &ctx->registry->funcs[index]; + const CBMRegisteredFunc *candidate = &it.reg->funcs[index]; if (candidate->receiver_type || !candidate->short_name || strcmp(candidate->short_name, operator_name) != 0 || !candidate->qualified_name) { continue; @@ -3730,13 +3731,13 @@ static const CBMRegisteredFunc *c_lookup_free_unary_operator(CLSPContext *ctx, const CBMRegisteredFunc *match = NULL; const CBMType *actuals[1] = {operand_type}; CBMFreeFuncIter it; - cbm_registry_free_funcs_by_short_name(ctx->registry, operator_name, &it); + cbm_registry_free_funcs_by_short_name_chain(ctx->registry, operator_name, &it); for (int index = cbm_free_func_iter_next(&it); index >= 0; index = cbm_free_func_iter_next(&it)) { - if (index >= ctx->registry->func_count) { + if (index >= it.reg->func_count) { continue; } - const CBMRegisteredFunc *candidate = &ctx->registry->funcs[index]; + const CBMRegisteredFunc *candidate = &it.reg->funcs[index]; if (candidate->receiver_type || !candidate->short_name || strcmp(candidate->short_name, operator_name) != 0 || !candidate->qualified_name) { continue; @@ -4882,18 +4883,16 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) { // indexing bitcoin). Cross-phase template deduction then relies on the // positional fallback, which is graceful degradation. if (ctx->in_template && ctx->template_param_count > 0 && !ctx->registry_shared) { - // Find the registered function and set type_param_names - for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->func_count; ri++) { - CBMRegisteredFunc *rf = &((CBMTypeRegistry *)ctx->registry)->funcs[ri]; - if (strcmp(rf->qualified_name, func_qn) == 0 && !rf->type_param_names) { - const char **tpn = (const char **)cbm_arena_alloc( - ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *)); - for (int ti = 0; ti < ctx->template_param_count; ti++) - tpn[ti] = ctx->template_param_names[ti]; - tpn[ctx->template_param_count] = NULL; - rf->type_param_names = tpn; - break; - } + // Set type_param_names on the registered function -- on the head's + // own copy of it (copy-on-write), never on a shared base entry. + CBMRegisteredFunc *rf = cbm_registry_func_for_update(ctx->registry_head, func_qn); + if (rf && !rf->type_param_names) { + const char **tpn = (const char **)cbm_arena_alloc( + ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *)); + for (int ti = 0; ti < ctx->template_param_count; ti++) + tpn[ti] = ctx->template_param_names[ti]; + tpn[ctx->template_param_count] = NULL; + rf->type_param_names = tpn; } } @@ -4929,12 +4928,9 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) { } // Set min_params on the registered function (for default-arg overload matching) if (total_params > 0 && defaulted_params > 0) { - for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->func_count; ri++) { - CBMRegisteredFunc *rf = &((CBMTypeRegistry *)ctx->registry)->funcs[ri]; - if (strcmp(rf->qualified_name, func_qn) == 0 && rf->min_params < 0) { - rf->min_params = total_params - defaulted_params; - break; - } + CBMRegisteredFunc *rf = cbm_registry_func_for_update(ctx->registry_head, func_qn); + if (rf && rf->min_params < 0) { + rf->min_params = total_params - defaulted_params; } } @@ -5059,7 +5055,7 @@ static void c_process_body_child(CLSPContext *ctx, TSNode child) { tpn[ctx->template_param_count] = NULL; rf.type_param_names = tpn; } - cbm_registry_add_func((CBMTypeRegistry *)ctx->registry, rf); + cbm_registry_add_func(ctx->registry_head, rf); } } } @@ -5161,14 +5157,7 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) { // Store template param names on the registered type (for substitution) if (ctx->in_template && ctx->template_param_names && ctx->template_param_count > 0) { - CBMRegisteredType *rt = NULL; - for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->type_count; ri++) { - if (strcmp(((CBMTypeRegistry *)ctx->registry)->types[ri].qualified_name, - class_qn) == 0) { - rt = &((CBMTypeRegistry *)ctx->registry)->types[ri]; - break; - } - } + CBMRegisteredType *rt = cbm_registry_type_for_update(ctx->registry_head, class_qn); if (rt && !rt->type_param_names) { const char **tpn = (const char **)cbm_arena_alloc( ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *)); @@ -5341,14 +5330,25 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) { should_upgrade = true; } if (should_upgrade) { - // Update existing entry's signature return type - const CBMType **new_rets = (const CBMType **)cbm_arena_alloc( - ctx->arena, 2 * sizeof(const CBMType *)); - new_rets[0] = actual_ret; - new_rets[1] = NULL; - CBMRegisteredFunc *mut = (CBMRegisteredFunc *)existing; - mut->signature = cbm_type_func_replace_returns( - ctx->arena, existing->signature, new_rets); + /* Refine the return type on the HEAD copy only + * (copy-on-write). `existing` may live in the + * sealed shared base; writing a scratch-arena + * signature into it was read by other workers + * after this file's arena died (ASan heap-use- + * after-free in c_adl_resolve on dotnet/runtime, + * 2026-09-14). NULL = the head is sealed too: + * skip the refinement. */ + CBMRegisteredFunc *mut = cbm_registry_func_for_update( + ctx->registry_head, existing->qualified_name); + if (mut) { + const CBMType **new_rets = + (const CBMType **)cbm_arena_alloc( + ctx->arena, 2 * sizeof(const CBMType *)); + new_rets[0] = actual_ret; + new_rets[1] = NULL; + mut->signature = cbm_type_func_replace_returns( + ctx->arena, mut->signature, new_rets); + } } break; } @@ -5364,7 +5364,7 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) { rf.receiver_type = ctx->enclosing_class_qn; rf.signature = cbm_type_func(ctx->arena, NULL, NULL, rets); rf.min_params = -1; - cbm_registry_add_func((CBMTypeRegistry *)ctx->registry, rf); + cbm_registry_add_func(ctx->registry_head, rf); break; } else if (strcmp(dk, "reference_declarator") == 0) { actual_ret = cbm_type_reference(ctx->arena, actual_ret); diff --git a/internal/cbm/lsp/c_lsp.h b/internal/cbm/lsp/c_lsp.h index 9aa936960..af81d5bbd 100644 --- a/internal/cbm/lsp/c_lsp.h +++ b/internal/cbm/lsp/c_lsp.h @@ -13,6 +13,11 @@ typedef struct { const char *source; int source_len; const CBMTypeRegistry *registry; + /* The writable head of the registry chain: the per-file overlay the + * dispatcher hands in, or the per-file registry on the non-cross path. + * Every refinement (lazy add, min_params, template params) goes here + * via cbm_registry_*_for_update; nothing behind ->fallback is written. */ + CBMTypeRegistry *registry_head; CBMScope *current_scope; // Include map: header_path -> namespace QN prefix @@ -94,9 +99,9 @@ typedef struct { // READ-ONLY across resolve workers — never mutate it // (and never store per-worker arena pointers into it) bool debug; - int eval_depth; // recursion depth for c_eval_expr_type (crash guard) - int eval_steps; // total expression eval calls for current file (hang guard) - int walk_depth; // c_resolve_calls_in_node self-recursion (AST nesting) + int eval_depth; // recursion depth for c_eval_expr_type (crash guard) + int eval_steps; // total expression eval calls for current file (hang guard) + int walk_depth; // c_resolve_calls_in_node self-recursion (AST nesting) int control_flow_depth; // if/loop/switch/catch nesting; assignments merge fail-closed } CLSPContext; diff --git a/internal/cbm/lsp/cs_lsp.c b/internal/cbm/lsp/cs_lsp.c index 0beed0a56..fc3b548ef 100644 --- a/internal/cbm/lsp/cs_lsp.c +++ b/internal/cbm/lsp/cs_lsp.c @@ -492,10 +492,10 @@ const char *cs_resolve_type_name(CSLSPContext *ctx, const char *raw) { CBMTypeShortIter ts_it; int i; if (ctx->registry) { - cbm_registry_types_by_short_name(ctx->registry, the_short, &ts_it); + cbm_registry_types_by_short_name_chain(ctx->registry, the_short, &ts_it); } while (ctx->registry && (i = cbm_type_short_iter_next(&ts_it)) >= 0) { - const CBMRegisteredType *cand = &ctx->registry->types[i]; + const CBMRegisteredType *cand = &ts_it.reg->types[i]; if (!cand->short_name || strcmp(cand->short_name, the_short) != 0) continue; int score = 0; @@ -643,12 +643,12 @@ static const CBMRegisteredFunc *cs_lookup_extension(CSLSPContext *ctx, const cha * registration-order tie-break exactly (bucket chains are not in funcs[] * order). */ CBMFreeFuncIter ext_it; - cbm_registry_free_funcs_by_short_name(ctx->registry, method_name, &ext_it); + cbm_registry_free_funcs_by_short_name_chain(ctx->registry, method_name, &ext_it); const CBMRegisteredFunc *best = NULL; int best_idx = -1; int i; while ((i = cbm_free_func_iter_next(&ext_it)) >= 0) { - const CBMRegisteredFunc *cand = &ctx->registry->funcs[i]; + const CBMRegisteredFunc *cand = &ext_it.reg->funcs[i]; if (!cand->short_name || strcmp(cand->short_name, method_name) != 0) continue; if (cand->receiver_type) @@ -1522,8 +1522,12 @@ static const char *cs_resolve_callable_name(CSLSPContext *ctx, const char *name) * generated fixtures can register receiverless callables. Only accept a * unique short-name match; ambiguity is intentionally not guessed. */ const CBMRegisteredFunc *only = NULL; - for (int i = 0; ctx->registry && i < ctx->registry->func_count; i++) { - const CBMRegisteredFunc *candidate = &ctx->registry->funcs[i]; + CBMFreeFuncIter all_funcs; + if (ctx->registry) { + cbm_registry_all_funcs_chain(ctx->registry, &all_funcs); + } + for (int i = -1; ctx->registry && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) { + const CBMRegisteredFunc *candidate = &all_funcs.reg->funcs[i]; if (candidate->receiver_type || !candidate->short_name || strcmp(candidate->short_name, name) != 0) { continue; @@ -2142,8 +2146,12 @@ static void cs_resolve_invocation(CSLSPContext *ctx, TSNode call) { /* Last resort: any free function with this short name in registry. */ const CBMRegisteredFunc *best = NULL; int best_score = -1; - for (int i = 0; ctx->registry && i < ctx->registry->func_count; i++) { - const CBMRegisteredFunc *cand = &ctx->registry->funcs[i]; + CBMFreeFuncIter all_funcs; + if (ctx->registry) { + cbm_registry_all_funcs_chain(ctx->registry, &all_funcs); + } + for (int i = -1; ctx->registry && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) { + const CBMRegisteredFunc *cand = &all_funcs.reg->funcs[i]; if (cand->receiver_type) continue; if (!cand->short_name || strcmp(cand->short_name, bare) != 0) diff --git a/internal/cbm/lsp/go_lsp.c b/internal/cbm/lsp/go_lsp.c index 7efa273e1..ba7df163b 100644 --- a/internal/cbm/lsp/go_lsp.c +++ b/internal/cbm/lsp/go_lsp.c @@ -1,12 +1,14 @@ #include "go_lsp.h" #include "lsp_node_iter.h" +#include "../../../src/foundation/hash_table.h" #include "../helpers.h" #include #include #include +#include // Forward declarations -static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node); +static void resolve_calls_in_node_inner(GoLSPContext *ctx, TSNode node); /* Depth-guarded entry for the AST call-resolution walk. The walk recurses once * per nesting level; a deeply-nested or cyclic file can overflow the native @@ -15,7 +17,7 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node); * crash. The cap is CBM_LSP_MAX_WALK_DEPTH, env-overridable via the same name. * The walk_depth-- runs after the inner returns, so early returns in the body * never leak the counter. */ -static void resolve_calls_in_node(GoLSPContext* ctx, TSNode node) { +static void resolve_calls_in_node(GoLSPContext *ctx, TSNode node) { if (ctx->walk_depth >= cbm_lsp_max_walk_depth()) return; ctx->walk_depth++; @@ -25,14 +27,16 @@ static void resolve_calls_in_node(GoLSPContext* ctx, TSNode node) { static void emit_resolved_call(GoLSPContext *ctx, const char *callee_qn, const char *strategy, float confidence, TSNode site); static const char *go_exact_callable_target(GoLSPContext *ctx, TSNode node); -static const CBMType* go_lookup_field(GoLSPContext* ctx, const char* type_qn, const char* field_name, int depth); -static void extract_type_params_from_ast(CBMArena* arena, CBMTypeRegistry* reg, - TSNode root, const char* source, const char* module_qn); +static const CBMType *go_lookup_field(GoLSPContext *ctx, const char *type_qn, + const char *field_name, int depth); +static void extract_type_params_from_ast(CBMArena *arena, CBMTypeRegistry *reg, TSNode root, + const char *source, const char *module_qn); // --- Initialization --- -void go_lsp_init(GoLSPContext* ctx, CBMArena* arena, const char* source, int source_len, - const CBMTypeRegistry* registry, const char* package_qn, CBMResolvedCallArray* out) { +void go_lsp_init(GoLSPContext *ctx, CBMArena *arena, const char *source, int source_len, + const CBMTypeRegistry *registry, const char *package_qn, + CBMResolvedCallArray *out) { memset(ctx, 0, sizeof(GoLSPContext)); ctx->arena = arena; ctx->source = source; @@ -43,21 +47,24 @@ void go_lsp_init(GoLSPContext* ctx, CBMArena* arena, const char* source, int sou ctx->current_scope = cbm_scope_push(arena, NULL); // root scope { - const char* debug_env = getenv("CBM_LSP_DEBUG"); + const char *debug_env = getenv("CBM_LSP_DEBUG"); ctx->debug = (debug_env && debug_env[0]); } } -void go_lsp_add_import(GoLSPContext* ctx, const char* local_name, const char* pkg_qn) { +void go_lsp_add_import(GoLSPContext *ctx, const char *local_name, const char *pkg_qn) { // Store in parallel arrays (arena-allocated, grow by doubling) if (ctx->import_count % 32 == 0) { int new_cap = ctx->import_count + 32; - const char** new_names = (const char**)cbm_arena_alloc(ctx->arena, (new_cap + 1) * sizeof(const char*)); - const char** new_qns = (const char**)cbm_arena_alloc(ctx->arena, (new_cap + 1) * sizeof(const char*)); - if (!new_names || !new_qns) return; + const char **new_names = + (const char **)cbm_arena_alloc(ctx->arena, (new_cap + 1) * sizeof(const char *)); + const char **new_qns = + (const char **)cbm_arena_alloc(ctx->arena, (new_cap + 1) * sizeof(const char *)); + if (!new_names || !new_qns) + return; if (ctx->import_local_names && ctx->import_count > 0) { - memcpy(new_names, ctx->import_local_names, ctx->import_count * sizeof(const char*)); - memcpy(new_qns, ctx->import_package_qns, ctx->import_count * sizeof(const char*)); + memcpy(new_names, ctx->import_local_names, ctx->import_count * sizeof(const char *)); + memcpy(new_qns, ctx->import_package_qns, ctx->import_count * sizeof(const char *)); } ctx->import_local_names = new_names; ctx->import_package_qns = new_qns; @@ -69,13 +76,13 @@ void go_lsp_add_import(GoLSPContext* ctx, const char* local_name, const char* pk // --- Helper: get node text --- -static char* lsp_node_text(GoLSPContext* ctx, TSNode node) { +static char *lsp_node_text(GoLSPContext *ctx, TSNode node) { return cbm_node_text(ctx->arena, node, ctx->source); } // --- Helper: resolve import alias to package QN --- -static const char* resolve_import(GoLSPContext* ctx, const char* local_name) { +static const char *resolve_import(GoLSPContext *ctx, const char *local_name) { for (int i = 0; i < ctx->import_count; i++) { if (strcmp(ctx->import_local_names[i], local_name) == 0) { return ctx->import_package_qns[i]; @@ -86,31 +93,26 @@ static const char* resolve_import(GoLSPContext* ctx, const char* local_name) { // --- Helper: check if name is a Go builtin --- -static bool is_go_builtin_func(const char* name) { - static const char* builtins[] = { - "make", "new", "append", "len", "cap", "delete", - "close", "copy", "panic", "recover", "print", "println", - "complex", "real", "imag", "min", "max", "clear", - NULL - }; - for (const char** b = builtins; *b; b++) { - if (strcmp(name, *b) == 0) return true; +static bool is_go_builtin_func(const char *name) { + static const char *builtins[] = {"make", "new", "append", "len", "cap", + "delete", "close", "copy", "panic", "recover", + "print", "println", "complex", "real", "imag", + "min", "max", "clear", NULL}; + for (const char **b = builtins; *b; b++) { + if (strcmp(name, *b) == 0) + return true; } return false; } // --- Helper: check if name is a Go builtin type --- -static const CBMType* resolve_builtin_type(GoLSPContext* ctx, const char* name) { - static const char* builtin_types[] = { - "int", "int8", "int16", "int32", "int64", - "uint", "uint8", "uint16", "uint32", "uint64", - "float32", "float64", "complex64", "complex128", - "string", "bool", "byte", "rune", "error", - "uintptr", "any", - NULL - }; - for (const char** b = builtin_types; *b; b++) { +static const CBMType *resolve_builtin_type(GoLSPContext *ctx, const char *name) { + static const char *builtin_types[] = { + "int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", + "uint32", "uint64", "float32", "float64", "complex64", "complex128", "string", "bool", + "byte", "rune", "error", "uintptr", "any", NULL}; + for (const char **b = builtin_types; *b; b++) { if (strcmp(name, *b) == 0) { return cbm_type_builtin(ctx->arena, name); } @@ -120,20 +122,23 @@ static const CBMType* resolve_builtin_type(GoLSPContext* ctx, const char* name) // --- go_parse_type_node: AST type node -> CBMType --- -const CBMType* go_parse_type_node(GoLSPContext* ctx, TSNode node) { - if (ts_node_is_null(node)) return cbm_type_unknown(); +const CBMType *go_parse_type_node(GoLSPContext *ctx, TSNode node) { + if (ts_node_is_null(node)) + return cbm_type_unknown(); - const char* kind = ts_node_type(node); + const char *kind = ts_node_type(node); // type_identifier: simple named type if (strcmp(kind, "type_identifier") == 0) { - char* name = lsp_node_text(ctx, node); - if (!name) return cbm_type_unknown(); - const CBMType* builtin = resolve_builtin_type(ctx, name); - if (builtin) return builtin; + char *name = lsp_node_text(ctx, node); + if (!name) + return cbm_type_unknown(); + const CBMType *builtin = resolve_builtin_type(ctx, name); + if (builtin) + return builtin; // Resolve as local type: package_qn.TypeName return cbm_type_named(ctx->arena, - cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, name)); + cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, name)); } // qualified_type: pkg.Type @@ -141,12 +146,12 @@ const CBMType* go_parse_type_node(GoLSPContext* ctx, TSNode node) { TSNode pkg_node = ts_node_child_by_field_name(node, "package", 7); TSNode name_node = ts_node_child_by_field_name(node, "name", 4); if (!ts_node_is_null(pkg_node) && !ts_node_is_null(name_node)) { - char* pkg = lsp_node_text(ctx, pkg_node); - char* name = lsp_node_text(ctx, name_node); - const char* pkg_qn = resolve_import(ctx, pkg); + char *pkg = lsp_node_text(ctx, pkg_node); + char *name = lsp_node_text(ctx, name_node); + const char *pkg_qn = resolve_import(ctx, pkg); if (pkg_qn) { return cbm_type_named(ctx->arena, - cbm_arena_sprintf(ctx->arena, "%s.%s", pkg_qn, name)); + cbm_arena_sprintf(ctx->arena, "%s.%s", pkg_qn, name)); } } return cbm_type_unknown(); @@ -157,7 +162,7 @@ const CBMType* go_parse_type_node(GoLSPContext* ctx, TSNode node) { uint32_t nc = ts_node_named_child_count(node); if (nc > 0) { return cbm_type_pointer(ctx->arena, - go_parse_type_node(ctx, ts_node_named_child(node, nc - 1))); + go_parse_type_node(ctx, ts_node_named_child(node, nc - 1))); } return cbm_type_unknown(); } @@ -184,9 +189,8 @@ const CBMType* go_parse_type_node(GoLSPContext* ctx, TSNode node) { if (strcmp(kind, "map_type") == 0) { TSNode key = ts_node_child_by_field_name(node, "key", 3); TSNode value = ts_node_child_by_field_name(node, "value", 5); - return cbm_type_map(ctx->arena, - go_parse_type_node(ctx, key), - go_parse_type_node(ctx, value)); + return cbm_type_map(ctx->arena, go_parse_type_node(ctx, key), + go_parse_type_node(ctx, value)); } // channel_type: chan T @@ -196,11 +200,13 @@ const CBMType* go_parse_type_node(GoLSPContext* ctx, TSNode node) { value = ts_node_named_child(node, ts_node_named_child_count(node) - 1); } // Determine direction from text - char* text = lsp_node_text(ctx, node); + char *text = lsp_node_text(ctx, node); int dir = 0; if (text) { - if (strncmp(text, "chan<-", 6) == 0 || strncmp(text, "chan <-", 7) == 0) dir = 1; - else if (strncmp(text, "<-chan", 6) == 0 || strncmp(text, "<- chan", 7) == 0) dir = 2; + if (strncmp(text, "chan<-", 6) == 0 || strncmp(text, "chan <-", 7) == 0) + dir = 1; + else if (strncmp(text, "<-chan", 6) == 0 || strncmp(text, "<- chan", 7) == 0) + dir = 2; } return cbm_type_channel(ctx->arena, go_parse_type_node(ctx, value), dir); } @@ -212,7 +218,7 @@ const CBMType* go_parse_type_node(GoLSPContext* ctx, TSNode node) { // interface_type if (strcmp(kind, "interface_type") == 0) { - CBMType* t = (CBMType*)cbm_arena_alloc(ctx->arena, sizeof(CBMType)); + CBMType *t = (CBMType *)cbm_arena_alloc(ctx->arena, sizeof(CBMType)); memset(t, 0, sizeof(CBMType)); t->kind = CBM_TYPE_INTERFACE; return t; @@ -220,7 +226,7 @@ const CBMType* go_parse_type_node(GoLSPContext* ctx, TSNode node) { // struct_type if (strcmp(kind, "struct_type") == 0) { - CBMType* t = (CBMType*)cbm_arena_alloc(ctx->arena, sizeof(CBMType)); + CBMType *t = (CBMType *)cbm_arena_alloc(ctx->arena, sizeof(CBMType)); memset(t, 0, sizeof(CBMType)); t->kind = CBM_TYPE_STRUCT; return t; @@ -247,12 +253,13 @@ const CBMType* go_parse_type_node(GoLSPContext* ctx, TSNode node) { // parameter_list used as result type (multi-return) if (strcmp(kind, "parameter_list") == 0) { int count = 0; - const CBMType* elems[16]; + const CBMType *elems[16]; uint32_t nc = ts_node_child_count(node); for (uint32_t i = 0; i < nc && count < 16; i++) { TSNode child = ts_node_child(node, i); - if (ts_node_is_null(child) || !ts_node_is_named(child)) continue; - const char* ck = ts_node_type(child); + if (ts_node_is_null(child) || !ts_node_is_named(child)) + continue; + const char *ck = ts_node_type(child); if (strcmp(ck, "parameter_declaration") == 0) { TSNode tn = ts_node_child_by_field_name(child, "type", 4); if (!ts_node_is_null(tn)) { @@ -262,8 +269,10 @@ const CBMType* go_parse_type_node(GoLSPContext* ctx, TSNode node) { elems[count++] = go_parse_type_node(ctx, child); } } - if (count == 1) return elems[0]; - if (count > 1) return cbm_type_tuple(ctx->arena, elems, count); + if (count == 1) + return elems[0]; + if (count > 1) + return cbm_type_tuple(ctx->arena, elems, count); } return cbm_type_unknown(); @@ -273,14 +282,17 @@ const CBMType* go_parse_type_node(GoLSPContext* ctx, TSNode node) { // Unify a parameter type pattern (containing TYPE_PARAM) against a concrete argument type. // Fills inferred[i] with the concrete type bound to type_param_names[i]. -static void go_unify_type(const CBMType* param_type, const CBMType* arg_type, - const char** type_param_names, const CBMType** inferred, int param_count) { - if (!param_type || !arg_type || cbm_type_is_unknown(arg_type)) return; +static void go_unify_type(const CBMType *param_type, const CBMType *arg_type, + const char **type_param_names, const CBMType **inferred, + int param_count) { + if (!param_type || !arg_type || cbm_type_is_unknown(arg_type)) + return; if (param_type->kind == CBM_TYPE_TYPE_PARAM) { for (int i = 0; i < param_count; i++) { if (strcmp(param_type->data.type_param.name, type_param_names[i]) == 0) { - if (!inferred[i]) inferred[i] = arg_type; + if (!inferred[i]) + inferred[i] = arg_type; break; } } @@ -289,36 +301,41 @@ static void go_unify_type(const CBMType* param_type, const CBMType* arg_type, // Structural matching — recurse into composite types if (param_type->kind == CBM_TYPE_SLICE && arg_type->kind == CBM_TYPE_SLICE) { - go_unify_type(param_type->data.slice.elem, arg_type->data.slice.elem, - type_param_names, inferred, param_count); + go_unify_type(param_type->data.slice.elem, arg_type->data.slice.elem, type_param_names, + inferred, param_count); } if (param_type->kind == CBM_TYPE_POINTER && arg_type->kind == CBM_TYPE_POINTER) { - go_unify_type(param_type->data.pointer.elem, arg_type->data.pointer.elem, - type_param_names, inferred, param_count); + go_unify_type(param_type->data.pointer.elem, arg_type->data.pointer.elem, type_param_names, + inferred, param_count); } if (param_type->kind == CBM_TYPE_MAP && arg_type->kind == CBM_TYPE_MAP) { - go_unify_type(param_type->data.map.key, arg_type->data.map.key, - type_param_names, inferred, param_count); - go_unify_type(param_type->data.map.value, arg_type->data.map.value, - type_param_names, inferred, param_count); + go_unify_type(param_type->data.map.key, arg_type->data.map.key, type_param_names, inferred, + param_count); + go_unify_type(param_type->data.map.value, arg_type->data.map.value, type_param_names, + inferred, param_count); } if (param_type->kind == CBM_TYPE_CHANNEL && arg_type->kind == CBM_TYPE_CHANNEL) { - go_unify_type(param_type->data.channel.elem, arg_type->data.channel.elem, - type_param_names, inferred, param_count); + go_unify_type(param_type->data.channel.elem, arg_type->data.channel.elem, type_param_names, + inferred, param_count); } if (param_type->kind == CBM_TYPE_FUNC && arg_type->kind == CBM_TYPE_FUNC) { // Match param types if (param_type->data.func.param_types && arg_type->data.func.param_types) { - for (int i = 0; param_type->data.func.param_types[i] && arg_type->data.func.param_types[i]; i++) { - go_unify_type(param_type->data.func.param_types[i], arg_type->data.func.param_types[i], - type_param_names, inferred, param_count); + for (int i = 0; + param_type->data.func.param_types[i] && arg_type->data.func.param_types[i]; i++) { + go_unify_type(param_type->data.func.param_types[i], + arg_type->data.func.param_types[i], type_param_names, inferred, + param_count); } } // Match return types if (param_type->data.func.return_types && arg_type->data.func.return_types) { - for (int i = 0; param_type->data.func.return_types[i] && arg_type->data.func.return_types[i]; i++) { - go_unify_type(param_type->data.func.return_types[i], arg_type->data.func.return_types[i], - type_param_names, inferred, param_count); + for (int i = 0; + param_type->data.func.return_types[i] && arg_type->data.func.return_types[i]; + i++) { + go_unify_type(param_type->data.func.return_types[i], + arg_type->data.func.return_types[i], type_param_names, inferred, + param_count); } } } @@ -326,33 +343,39 @@ static void go_unify_type(const CBMType* param_type, const CBMType* arg_type, // --- go_eval_expr_type: recursive expression type evaluator --- -const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { - if (ts_node_is_null(node)) return cbm_type_unknown(); +const CBMType *go_eval_expr_type(GoLSPContext *ctx, TSNode node) { + if (ts_node_is_null(node)) + return cbm_type_unknown(); - const char* kind = ts_node_type(node); + const char *kind = ts_node_type(node); // --- Identifier: scope lookup --- if (strcmp(kind, "identifier") == 0) { - char* name = lsp_node_text(ctx, node); - if (!name) return cbm_type_unknown(); + char *name = lsp_node_text(ctx, node); + if (!name) + return cbm_type_unknown(); // Check scope first - const CBMType* t = cbm_scope_lookup(ctx->current_scope, name); + const CBMType *t = cbm_scope_lookup(ctx->current_scope, name); if (cbm_scope_contains(ctx->current_scope, name)) return t ? t : cbm_type_unknown(); // Check if it's a package-level function - const CBMRegisteredFunc* f = cbm_registry_lookup_symbol(ctx->registry, ctx->package_qn, name); - if (f && f->signature) return f->signature; + const CBMRegisteredFunc *f = + cbm_registry_lookup_symbol(ctx->registry, ctx->package_qn, name); + if (f && f->signature) + return f->signature; // Check if it's a builtin type (for type conversions like string(x), int(x)) - const CBMType* bt = resolve_builtin_type(ctx, name); - if (bt) return cbm_type_named(ctx->arena, name); + const CBMType *bt = resolve_builtin_type(ctx, name); + if (bt) + return cbm_type_named(ctx->arena, name); // Check if it's a registered type (for type conversions like MyType(x)) - const char* type_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, name); - const CBMRegisteredType* rt = cbm_registry_lookup_type(ctx->registry, type_qn); - if (rt) return cbm_type_named(ctx->arena, type_qn); + const char *type_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, name); + const CBMRegisteredType *rt = cbm_registry_lookup_type(ctx->registry, type_qn); + if (rt) + return cbm_type_named(ctx->arena, type_qn); return cbm_type_unknown(); } @@ -361,49 +384,57 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { if (strcmp(kind, "selector_expression") == 0) { TSNode operand = ts_node_child_by_field_name(node, "operand", 7); TSNode field = ts_node_child_by_field_name(node, "field", 5); - if (ts_node_is_null(operand) || ts_node_is_null(field)) return cbm_type_unknown(); + if (ts_node_is_null(operand) || ts_node_is_null(field)) + return cbm_type_unknown(); - char* field_name = lsp_node_text(ctx, field); - if (!field_name) return cbm_type_unknown(); + char *field_name = lsp_node_text(ctx, field); + if (!field_name) + return cbm_type_unknown(); // Check if operand is an import alias (pkg.Symbol) if (strcmp(ts_node_type(operand), "identifier") == 0) { - char* pkg_name = lsp_node_text(ctx, operand); + char *pkg_name = lsp_node_text(ctx, operand); if (pkg_name) { - const char* pkg_qn = resolve_import(ctx, pkg_name); + const char *pkg_qn = resolve_import(ctx, pkg_name); if (pkg_qn) { // Look up pkg.Symbol as a function or type - const CBMRegisteredFunc* f = cbm_registry_lookup_symbol(ctx->registry, pkg_qn, field_name); - if (f && f->signature) return f->signature; + const CBMRegisteredFunc *f = + cbm_registry_lookup_symbol(ctx->registry, pkg_qn, field_name); + if (f && f->signature) + return f->signature; // Check if it's a type - char* type_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", pkg_qn, field_name); - const CBMRegisteredType* rt = cbm_registry_lookup_type(ctx->registry, type_qn); - if (rt) return cbm_type_named(ctx->arena, type_qn); + char *type_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", pkg_qn, field_name); + const CBMRegisteredType *rt = cbm_registry_lookup_type(ctx->registry, type_qn); + if (rt) + return cbm_type_named(ctx->arena, type_qn); return cbm_type_unknown(); } } } // Evaluate operand type - const CBMType* recv_type = go_eval_expr_type(ctx, operand); - if (cbm_type_is_unknown(recv_type)) return cbm_type_unknown(); + const CBMType *recv_type = go_eval_expr_type(ctx, operand); + if (cbm_type_is_unknown(recv_type)) + return cbm_type_unknown(); // Auto-deref pointers for method calls - const CBMType* base_type = recv_type; + const CBMType *base_type = recv_type; if (base_type->kind == CBM_TYPE_POINTER) { base_type = cbm_type_deref(base_type); } if (base_type->kind == CBM_TYPE_NAMED) { - const char* type_qn = base_type->data.named.qualified_name; + const char *type_qn = base_type->data.named.qualified_name; // Look up method/field (methods recurse through embeddings) - const CBMRegisteredFunc* method = go_lookup_field_or_method(ctx, type_qn, field_name); - if (method && method->signature) return method->signature; + const CBMRegisteredFunc *method = go_lookup_field_or_method(ctx, type_qn, field_name); + if (method && method->signature) + return method->signature; // Check struct fields (with embedded field promotion) - const CBMType* field_type = go_lookup_field(ctx, type_qn, field_name, 0); - if (field_type && !cbm_type_is_unknown(field_type)) return field_type; + const CBMType *field_type = go_lookup_field(ctx, type_qn, field_name, 0); + if (field_type && !cbm_type_is_unknown(field_type)) + return field_type; } return cbm_type_unknown(); @@ -413,43 +444,45 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { if (strcmp(kind, "call_expression") == 0) { TSNode func_node = ts_node_child_by_field_name(node, "function", 8); TSNode args_node = ts_node_child_by_field_name(node, "arguments", 9); - if (ts_node_is_null(func_node)) return cbm_type_unknown(); + if (ts_node_is_null(func_node)) + return cbm_type_unknown(); // Check for builtin calls if (strcmp(ts_node_type(func_node), "identifier") == 0) { - char* name = lsp_node_text(ctx, func_node); + char *name = lsp_node_text(ctx, func_node); if (name && is_go_builtin_func(name)) { return go_eval_builtin_call(ctx, name, args_node); } } // Evaluate function type - const CBMType* func_type = go_eval_expr_type(ctx, func_node); + const CBMType *func_type = go_eval_expr_type(ctx, func_node); // If it's a FUNC type, return its return type - if (func_type && func_type->kind == CBM_TYPE_FUNC && - func_type->data.func.return_types && func_type->data.func.return_types[0]) { + if (func_type && func_type->kind == CBM_TYPE_FUNC && func_type->data.func.return_types && + func_type->data.func.return_types[0]) { // Check for explicit type arguments: call_expression has type_arguments field - // Go tree-sitter: Func[T1, T2](args) → call_expression { function, type_arguments, arguments } + // Go tree-sitter: Func[T1, T2](args) → call_expression { function, type_arguments, + // arguments } TSNode targs_node = ts_node_child_by_field_name(node, "type_arguments", 14); if (!ts_node_is_null(targs_node)) { // Look up the registered function to get type_param_names - const CBMRegisteredFunc* rfunc = NULL; - char* func_name = lsp_node_text(ctx, func_node); + const CBMRegisteredFunc *rfunc = NULL; + char *func_name = lsp_node_text(ctx, func_node); if (func_name) { - const char* func_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", - ctx->package_qn, func_name); + const char *func_qn = + cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, func_name); rfunc = cbm_registry_lookup_func(ctx->registry, func_qn); // If not found as local, try via import (selector_expression: pkg.Func) if (!rfunc && strcmp(ts_node_type(func_node), "selector_expression") == 0) { TSNode operand = ts_node_child_by_field_name(func_node, "operand", 7); TSNode field = ts_node_child_by_field_name(func_node, "field", 5); if (!ts_node_is_null(operand) && !ts_node_is_null(field)) { - char* pkg = lsp_node_text(ctx, operand); - char* fn = lsp_node_text(ctx, field); + char *pkg = lsp_node_text(ctx, operand); + char *fn = lsp_node_text(ctx, field); if (pkg && fn) { - const char* pkg_qn = resolve_import(ctx, pkg); + const char *pkg_qn = resolve_import(ctx, pkg); if (pkg_qn) { rfunc = cbm_registry_lookup_symbol(ctx->registry, pkg_qn, fn); } @@ -460,40 +493,44 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { if (rfunc && rfunc->type_param_names) { // Parse type arguments from AST - const CBMType* type_args[16]; + const CBMType *type_args[16]; int targ_count = 0; uint32_t ta_nc = ts_node_child_count(targs_node); for (uint32_t ti = 0; ti < ta_nc && targ_count < 15; ti++) { TSNode targ = ts_node_child(targs_node, ti); - if (ts_node_is_null(targ) || !ts_node_is_named(targ)) continue; + if (ts_node_is_null(targ) || !ts_node_is_named(targ)) + continue; type_args[targ_count++] = go_parse_type_node(ctx, targ); } // Count type params int param_count = 0; - while (rfunc->type_param_names[param_count]) param_count++; + while (rfunc->type_param_names[param_count]) + param_count++; if (targ_count > 0 && targ_count == param_count) { - const CBMType** targ_arr = (const CBMType**)cbm_arena_alloc( - ctx->arena, (targ_count + 1) * sizeof(const CBMType*)); - for (int ti = 0; ti < targ_count; ti++) targ_arr[ti] = type_args[ti]; + const CBMType **targ_arr = (const CBMType **)cbm_arena_alloc( + ctx->arena, (targ_count + 1) * sizeof(const CBMType *)); + for (int ti = 0; ti < targ_count; ti++) + targ_arr[ti] = type_args[ti]; targ_arr[targ_count] = NULL; // Substitute type params in return type(s) int ret_count = 0; - while (func_type->data.func.return_types[ret_count]) ret_count++; + while (func_type->data.func.return_types[ret_count]) + ret_count++; if (ret_count == 1) { return cbm_type_substitute(ctx->arena, - func_type->data.func.return_types[0], - rfunc->type_param_names, targ_arr); + func_type->data.func.return_types[0], + rfunc->type_param_names, targ_arr); } // Multi-return: substitute all - const CBMType** new_rets = (const CBMType**)cbm_arena_alloc( - ctx->arena, (ret_count + 1) * sizeof(const CBMType*)); + const CBMType **new_rets = (const CBMType **)cbm_arena_alloc( + ctx->arena, (ret_count + 1) * sizeof(const CBMType *)); for (int ri = 0; ri < ret_count; ri++) { - new_rets[ri] = cbm_type_substitute(ctx->arena, - func_type->data.func.return_types[ri], + new_rets[ri] = cbm_type_substitute( + ctx->arena, func_type->data.func.return_types[ri], rfunc->type_param_names, targ_arr); } new_rets[ret_count] = NULL; @@ -505,20 +542,20 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { // Implicit generics: infer type args from argument types if (ts_node_is_null(targs_node)) { // Look up registered function to check for type_param_names - const CBMRegisteredFunc* rfunc = NULL; - char* func_name = lsp_node_text(ctx, func_node); + const CBMRegisteredFunc *rfunc = NULL; + char *func_name = lsp_node_text(ctx, func_node); if (func_name) { - const char* func_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", - ctx->package_qn, func_name); + const char *func_qn = + cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, func_name); rfunc = cbm_registry_lookup_func(ctx->registry, func_qn); if (!rfunc && strcmp(ts_node_type(func_node), "selector_expression") == 0) { TSNode operand = ts_node_child_by_field_name(func_node, "operand", 7); TSNode field = ts_node_child_by_field_name(func_node, "field", 5); if (!ts_node_is_null(operand) && !ts_node_is_null(field)) { - char* pkg = lsp_node_text(ctx, operand); - char* fn = lsp_node_text(ctx, field); + char *pkg = lsp_node_text(ctx, operand); + char *fn = lsp_node_text(ctx, field); if (pkg && fn) { - const char* pkg_qn = resolve_import(ctx, pkg); + const char *pkg_qn = resolve_import(ctx, pkg); if (pkg_qn) rfunc = cbm_registry_lookup_symbol(ctx->registry, pkg_qn, fn); } @@ -528,39 +565,49 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { if (rfunc && rfunc->type_param_names && func_type->data.func.param_types) { int tpc = 0; - while (rfunc->type_param_names[tpc]) tpc++; + while (rfunc->type_param_names[tpc]) + tpc++; // Check if any return type uses a type param bool has_type_param = false; int ret_count = 0; - while (func_type->data.func.return_types[ret_count]) ret_count++; + while (func_type->data.func.return_types[ret_count]) + ret_count++; for (int ri = 0; ri < ret_count && !has_type_param; ri++) { // Quick check: walk return type tree for TYPE_PARAM nodes - const CBMType* rt = func_type->data.func.return_types[ri]; - if (rt->kind == CBM_TYPE_TYPE_PARAM) has_type_param = true; + const CBMType *rt = func_type->data.func.return_types[ri]; + if (rt->kind == CBM_TYPE_TYPE_PARAM) + has_type_param = true; else if (rt->kind == CBM_TYPE_SLICE && rt->data.slice.elem && - rt->data.slice.elem->kind == CBM_TYPE_TYPE_PARAM) has_type_param = true; + rt->data.slice.elem->kind == CBM_TYPE_TYPE_PARAM) + has_type_param = true; else if (rt->kind == CBM_TYPE_POINTER && rt->data.pointer.elem && - rt->data.pointer.elem->kind == CBM_TYPE_TYPE_PARAM) has_type_param = true; + rt->data.pointer.elem->kind == CBM_TYPE_TYPE_PARAM) + has_type_param = true; else if (rt->kind == CBM_TYPE_MAP) { - if (rt->data.map.key && rt->data.map.key->kind == CBM_TYPE_TYPE_PARAM) has_type_param = true; - if (rt->data.map.value && rt->data.map.value->kind == CBM_TYPE_TYPE_PARAM) has_type_param = true; + if (rt->data.map.key && rt->data.map.key->kind == CBM_TYPE_TYPE_PARAM) + has_type_param = true; + if (rt->data.map.value && + rt->data.map.value->kind == CBM_TYPE_TYPE_PARAM) + has_type_param = true; } } if (has_type_param && tpc > 0 && tpc <= 16) { // Evaluate argument types and unify - const CBMType* inferred[16] = {0}; + const CBMType *inferred[16] = {0}; if (!ts_node_is_null(args_node)) { uint32_t argc = ts_node_named_child_count(args_node); int pi = 0; - for (uint32_t ai = 0; ai < argc && func_type->data.func.param_types[pi]; ai++) { + for (uint32_t ai = 0; ai < argc && func_type->data.func.param_types[pi]; + ai++) { TSNode arg = ts_node_named_child(args_node, ai); - if (ts_node_is_null(arg)) continue; - const CBMType* arg_type = go_eval_expr_type(ctx, arg); - go_unify_type(func_type->data.func.param_types[pi], - arg_type, rfunc->type_param_names, inferred, tpc); + if (ts_node_is_null(arg)) + continue; + const CBMType *arg_type = go_eval_expr_type(ctx, arg); + go_unify_type(func_type->data.func.param_types[pi], arg_type, + rfunc->type_param_names, inferred, tpc); pi++; } } @@ -568,25 +615,29 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { // Check if all type params were inferred bool all_inferred = true; for (int i = 0; i < tpc; i++) { - if (!inferred[i]) { all_inferred = false; break; } + if (!inferred[i]) { + all_inferred = false; + break; + } } if (all_inferred) { - const CBMType** targ_arr = (const CBMType**)cbm_arena_alloc( - ctx->arena, (tpc + 1) * sizeof(const CBMType*)); - for (int i = 0; i < tpc; i++) targ_arr[i] = inferred[i]; + const CBMType **targ_arr = (const CBMType **)cbm_arena_alloc( + ctx->arena, (tpc + 1) * sizeof(const CBMType *)); + for (int i = 0; i < tpc; i++) + targ_arr[i] = inferred[i]; targ_arr[tpc] = NULL; if (ret_count == 1) { return cbm_type_substitute(ctx->arena, - func_type->data.func.return_types[0], - rfunc->type_param_names, targ_arr); + func_type->data.func.return_types[0], + rfunc->type_param_names, targ_arr); } - const CBMType** new_rets = (const CBMType**)cbm_arena_alloc( - ctx->arena, (ret_count + 1) * sizeof(const CBMType*)); + const CBMType **new_rets = (const CBMType **)cbm_arena_alloc( + ctx->arena, (ret_count + 1) * sizeof(const CBMType *)); for (int ri = 0; ri < ret_count; ri++) { - new_rets[ri] = cbm_type_substitute(ctx->arena, - func_type->data.func.return_types[ri], + new_rets[ri] = cbm_type_substitute( + ctx->arena, func_type->data.func.return_types[ri], rfunc->type_param_names, targ_arr); } new_rets[ret_count] = NULL; @@ -601,7 +652,8 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { return func_type->data.func.return_types[0]; } int count = 0; - while (func_type->data.func.return_types[count]) count++; + while (func_type->data.func.return_types[count]) + count++; if (count > 1) { return cbm_type_tuple(ctx->arena, func_type->data.func.return_types, count); } @@ -615,7 +667,7 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { // Type conversion with composite type syntax: []byte(s), map[K]V(x), etc. // The function node is a type node (slice_type, map_type, etc.) { - const char* fk = ts_node_type(func_node); + const char *fk = ts_node_type(func_node); if (strcmp(fk, "slice_type") == 0 || strcmp(fk, "array_type") == 0 || strcmp(fk, "map_type") == 0 || strcmp(fk, "pointer_type") == 0 || strcmp(fk, "channel_type") == 0) { @@ -638,14 +690,16 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { // --- Unary expression: &x, *x, <-ch, !x --- if (strcmp(kind, "unary_expression") == 0) { TSNode operand = ts_node_child_by_field_name(node, "operand", 7); - if (ts_node_is_null(operand)) return cbm_type_unknown(); + if (ts_node_is_null(operand)) + return cbm_type_unknown(); // Get operator for (uint32_t i = 0; i < ts_node_child_count(node); i++) { TSNode child = ts_node_child(node, i); if (!ts_node_is_named(child)) { - char* op = lsp_node_text(ctx, child); - if (!op) continue; + char *op = lsp_node_text(ctx, child); + if (!op) + continue; if (strcmp(op, "&") == 0) { return cbm_type_pointer(ctx->arena, go_eval_expr_type(ctx, operand)); } @@ -653,7 +707,7 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { return cbm_type_deref(go_eval_expr_type(ctx, operand)); } if (strcmp(op, "<-") == 0) { - const CBMType* ch_type = go_eval_expr_type(ctx, operand); + const CBMType *ch_type = go_eval_expr_type(ctx, operand); if (ch_type && ch_type->kind == CBM_TYPE_CHANNEL) { return ch_type->data.channel.elem; } @@ -671,11 +725,15 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { // --- Index expression: a[i] --- if (strcmp(kind, "index_expression") == 0) { TSNode operand = ts_node_child_by_field_name(node, "operand", 7); - if (ts_node_is_null(operand)) return cbm_type_unknown(); - const CBMType* op_type = go_eval_expr_type(ctx, operand); - if (!op_type) return cbm_type_unknown(); - if (op_type->kind == CBM_TYPE_MAP) return op_type->data.map.value; - if (op_type->kind == CBM_TYPE_SLICE) return op_type->data.slice.elem; + if (ts_node_is_null(operand)) + return cbm_type_unknown(); + const CBMType *op_type = go_eval_expr_type(ctx, operand); + if (!op_type) + return cbm_type_unknown(); + if (op_type->kind == CBM_TYPE_MAP) + return op_type->data.map.value; + if (op_type->kind == CBM_TYPE_SLICE) + return op_type->data.slice.elem; return cbm_type_unknown(); } @@ -701,24 +759,25 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { for (uint32_t i = 0; i < ts_node_child_count(node); i++) { TSNode child = ts_node_child(node, i); if (!ts_node_is_named(child)) { - char* op = lsp_node_text(ctx, child); - if (op && (strcmp(op, "==") == 0 || strcmp(op, "!=") == 0 || - strcmp(op, "<") == 0 || strcmp(op, ">") == 0 || - strcmp(op, "<=") == 0 || strcmp(op, ">=") == 0 || + char *op = lsp_node_text(ctx, child); + if (op && (strcmp(op, "==") == 0 || strcmp(op, "!=") == 0 || strcmp(op, "<") == 0 || + strcmp(op, ">") == 0 || strcmp(op, "<=") == 0 || strcmp(op, ">=") == 0 || strcmp(op, "&&") == 0 || strcmp(op, "||") == 0)) { return cbm_type_builtin(ctx->arena, "bool"); } break; } } - if (!ts_node_is_null(left)) return go_eval_expr_type(ctx, left); + if (!ts_node_is_null(left)) + return go_eval_expr_type(ctx, left); return cbm_type_unknown(); } // --- Slice expression: a[low:high] --- if (strcmp(kind, "slice_expression") == 0) { TSNode operand = ts_node_child_by_field_name(node, "operand", 7); - if (!ts_node_is_null(operand)) return go_eval_expr_type(ctx, operand); + if (!ts_node_is_null(operand)) + return go_eval_expr_type(ctx, operand); return cbm_type_unknown(); } @@ -746,7 +805,7 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { TSNode body = ts_node_child_by_field_name(node, "body", 4); if (!ts_node_is_null(body)) { // Push child scope (inherits all outer bindings via parent chain) - CBMScope* saved = ctx->current_scope; + CBMScope *saved = ctx->current_scope; ctx->current_scope = cbm_scope_push(ctx->arena, ctx->current_scope); // Bind closure parameters @@ -755,16 +814,18 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { uint32_t nc = ts_node_child_count(params); for (uint32_t i = 0; i < nc; i++) { TSNode param = ts_node_child(params, i); - if (ts_node_is_null(param) || !ts_node_is_named(param)) continue; - if (strcmp(ts_node_type(param), "parameter_declaration") != 0) continue; + if (ts_node_is_null(param) || !ts_node_is_named(param)) + continue; + if (strcmp(ts_node_type(param), "parameter_declaration") != 0) + continue; TSNode type_node = ts_node_child_by_field_name(param, "type", 4); - const CBMType* pt = go_parse_type_node(ctx, type_node); + const CBMType *pt = go_parse_type_node(ctx, type_node); uint32_t pnc = ts_node_child_count(param); for (uint32_t j = 0; j < pnc; j++) { TSNode ch = ts_node_child(param, j); if (!ts_node_is_null(ch) && ts_node_is_named(ch) && strcmp(ts_node_type(ch), "identifier") == 0) { - char* pname = lsp_node_text(ctx, ch); + char *pname = lsp_node_text(ctx, ch); if (pname && strcmp(pname, "_") != 0) cbm_scope_bind(ctx->current_scope, pname, pt); } @@ -779,15 +840,17 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { } // Build full FUNC type with param/return types from AST - const CBMType* pt_arr[16]; + const CBMType *pt_arr[16]; int pt_count = 0; TSNode params2 = ts_node_child_by_field_name(node, "parameters", 10); if (!ts_node_is_null(params2)) { uint32_t nc2 = ts_node_child_count(params2); for (uint32_t i = 0; i < nc2 && pt_count < 15; i++) { TSNode p = ts_node_child(params2, i); - if (ts_node_is_null(p) || !ts_node_is_named(p)) continue; - if (strcmp(ts_node_type(p), "parameter_declaration") != 0) continue; + if (ts_node_is_null(p) || !ts_node_is_named(p)) + continue; + if (strcmp(ts_node_type(p), "parameter_declaration") != 0) + continue; TSNode pt = ts_node_child_by_field_name(p, "type", 4); if (!ts_node_is_null(pt)) pt_arr[pt_count++] = go_parse_type_node(ctx, pt); @@ -795,7 +858,7 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { } pt_arr[pt_count] = NULL; - const CBMType* rt_arr[16]; + const CBMType *rt_arr[16]; int rt_count = 0; TSNode result = ts_node_child_by_field_name(node, "result", 6); if (!ts_node_is_null(result)) { @@ -803,9 +866,11 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { uint32_t rnc = ts_node_child_count(result); for (uint32_t i = 0; i < rnc && rt_count < 15; i++) { TSNode rc = ts_node_child(result, i); - if (ts_node_is_null(rc) || !ts_node_is_named(rc)) continue; + if (ts_node_is_null(rc) || !ts_node_is_named(rc)) + continue; TSNode rt = ts_node_child_by_field_name(rc, "type", 4); - if (ts_node_is_null(rt)) rt = rc; + if (ts_node_is_null(rt)) + rt = rc; rt_arr[rt_count++] = go_parse_type_node(ctx, rt); } } else { @@ -814,9 +879,8 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { } rt_arr[rt_count] = NULL; - return cbm_type_func(ctx->arena, NULL, - pt_count > 0 ? (const CBMType**)pt_arr : NULL, - rt_count > 0 ? (const CBMType**)rt_arr : NULL); + return cbm_type_func(ctx->arena, NULL, pt_count > 0 ? (const CBMType **)pt_arr : NULL, + rt_count > 0 ? (const CBMType **)rt_arr : NULL); } return cbm_type_unknown(); @@ -824,7 +888,7 @@ const CBMType* go_eval_expr_type(GoLSPContext* ctx, TSNode node) { // --- go_eval_builtin_call --- -const CBMType* go_eval_builtin_call(GoLSPContext* ctx, const char* name, TSNode args) { +const CBMType *go_eval_builtin_call(GoLSPContext *ctx, const char *name, TSNode args) { // make(Type, ...) -> Type if (strcmp(name, "make") == 0 && !ts_node_is_null(args)) { uint32_t nc = ts_node_named_child_count(args); @@ -877,17 +941,20 @@ const CBMType* go_eval_builtin_call(GoLSPContext* ctx, const char* name, TSNode // the file's imports; returns NULL when no alias segment is involved. static const char *go_requalify_via_imports(GoLSPContext *ctx, const char *type_qn) { - if (!ctx || !type_qn || !type_qn[0] || ctx->import_count <= 0) return NULL; + if (!ctx || !type_qn || !type_qn[0] || ctx->import_count <= 0) + return NULL; for (int j = 0; j < ctx->import_count; j++) { const char *alias = ctx->import_local_names[j]; const char *alias_qn = ctx->import_package_qns[j]; - if (!alias || !alias[0] || !alias_qn || strchr(alias, '.')) continue; + if (!alias || !alias[0] || !alias_qn || strchr(alias, '.')) + continue; size_t alias_len = strlen(alias); /* Last occurrence of a "alias" segment in type_qn. */ const char *hit = NULL; for (const char *p = type_qn;;) { p = strstr(p, "."); - if (!p) break; + if (!p) + break; p++; if (strncmp(p, alias, alias_len) == 0 && p[alias_len] == '.') { hit = p; @@ -902,26 +969,31 @@ static const char *go_requalify_via_imports(GoLSPContext *ctx, const char *type_ return NULL; } -static const CBMType* go_lookup_field(GoLSPContext* ctx, - const char* type_qn, const char* field_name, int depth) { - if (!type_qn || !field_name || depth > 5) return NULL; +static const CBMType *go_lookup_field(GoLSPContext *ctx, const char *type_qn, + const char *field_name, int depth) { + if (!type_qn || !field_name || depth > 5) + return NULL; - const CBMRegisteredType* rt = cbm_registry_lookup_type(ctx->registry, type_qn); + const CBMRegisteredType *rt = cbm_registry_lookup_type(ctx->registry, type_qn); if (!rt && depth == 0) { /* Import-alias re-qualification: field texts from cross-package defs * may embed an alias segment only this file's import map resolves. */ - const char* alt_qn = go_requalify_via_imports(ctx, type_qn); - if (alt_qn) rt = cbm_registry_lookup_type(ctx->registry, alt_qn); + const char *alt_qn = go_requalify_via_imports(ctx, type_qn); + if (alt_qn) + rt = cbm_registry_lookup_type(ctx->registry, alt_qn); } - if (!rt) return NULL; + if (!rt) + return NULL; // Follow alias chain - if (rt->alias_of) return go_lookup_field(ctx, rt->alias_of, field_name, depth + 1); + if (rt->alias_of) + return go_lookup_field(ctx, rt->alias_of, field_name, depth + 1); // Direct field lookup if (rt->field_names) { for (int i = 0; rt->field_names[i]; i++) { - if (strcmp(rt->field_names[i], field_name) == 0 && rt->field_types && rt->field_types[i]) { + if (strcmp(rt->field_names[i], field_name) == 0 && rt->field_types && + rt->field_types[i]) { return rt->field_types[i]; } } @@ -930,8 +1002,9 @@ static const CBMType* go_lookup_field(GoLSPContext* ctx, // Promoted fields from embedded types if (rt->embedded_types) { for (int i = 0; rt->embedded_types[i]; i++) { - const CBMType* f = go_lookup_field(ctx, rt->embedded_types[i], field_name, depth + 1); - if (f) return f; + const CBMType *f = go_lookup_field(ctx, rt->embedded_types[i], field_name, depth + 1); + if (f) + return f; } } @@ -940,40 +1013,49 @@ static const CBMType* go_lookup_field(GoLSPContext* ctx, // --- go_lookup_field_or_method: method sets + embedding --- -static const CBMRegisteredFunc* go_lookup_field_or_method_depth(GoLSPContext* ctx, - const char* type_qn, const char* member_name, int depth) { - if (!type_qn || !member_name) return NULL; - if (depth > CBM_LSP_MAX_LOOKUP_DEPTH) return NULL; +static const CBMRegisteredFunc *go_lookup_field_or_method_depth(GoLSPContext *ctx, + const char *type_qn, + const char *member_name, + int depth) { + if (!type_qn || !member_name) + return NULL; + if (depth > CBM_LSP_MAX_LOOKUP_DEPTH) + return NULL; // Direct method lookup - const CBMRegisteredFunc* f = cbm_registry_lookup_method(ctx->registry, type_qn, member_name); - if (f) return f; + const CBMRegisteredFunc *f = cbm_registry_lookup_method(ctx->registry, type_qn, member_name); + if (f) + return f; /* Import-alias re-qualification fallback: NAMED receivers built from * cross-package field type texts can carry a ".svc.Svc" QN; * retry the method set on the import-resolved QN (see * go_requalify_via_imports). */ if (depth == 0) { - const char* alt_qn = go_requalify_via_imports(ctx, type_qn); + const char *alt_qn = go_requalify_via_imports(ctx, type_qn); if (alt_qn) { f = go_lookup_field_or_method_depth(ctx, alt_qn, member_name, depth + 1); - if (f) return f; + if (f) + return f; } } - const CBMRegisteredType* rt = cbm_registry_lookup_type(ctx->registry, type_qn); + const CBMRegisteredType *rt = cbm_registry_lookup_type(ctx->registry, type_qn); if (rt) { // Follow type alias chain if (rt->alias_of) { f = go_lookup_field_or_method_depth(ctx, rt->alias_of, member_name, depth + 1); - if (f) return f; + if (f) + return f; } // Check embedded types (promoted methods) if (rt->embedded_types) { for (int i = 0; rt->embedded_types[i]; i++) { - f = go_lookup_field_or_method_depth(ctx, rt->embedded_types[i], member_name, depth + 1); - if (f) return f; + f = go_lookup_field_or_method_depth(ctx, rt->embedded_types[i], member_name, + depth + 1); + if (f) + return f; } } } @@ -981,8 +1063,8 @@ static const CBMRegisteredFunc* go_lookup_field_or_method_depth(GoLSPContext* ct return NULL; } -const CBMRegisteredFunc* go_lookup_field_or_method(GoLSPContext* ctx, - const char* type_qn, const char* member_name) { +const CBMRegisteredFunc *go_lookup_field_or_method(GoLSPContext *ctx, const char *type_qn, + const char *member_name) { return go_lookup_field_or_method_depth(ctx, type_qn, member_name, 0); } @@ -1058,17 +1140,19 @@ static void go_scope_bind_value(GoLSPContext *ctx, const char *name, const CBMTy // --- go_process_statement: bind variables from statements --- -void go_process_statement(GoLSPContext* ctx, TSNode node) { - if (ts_node_is_null(node)) return; - const char* kind = ts_node_type(node); +void go_process_statement(GoLSPContext *ctx, TSNode node) { + if (ts_node_is_null(node)) + return; + const char *kind = ts_node_type(node); // short_var_declaration: a, b := expr if (strcmp(kind, "short_var_declaration") == 0) { TSNode left = ts_node_child_by_field_name(node, "left", 4); TSNode right = ts_node_child_by_field_name(node, "right", 5); - if (ts_node_is_null(left) || ts_node_is_null(right)) return; + if (ts_node_is_null(left) || ts_node_is_null(right)) + return; - const CBMType* rhs_type = NULL; + const CBMType *rhs_type = NULL; TSNode single_rhs = {0}; // Check if RHS is an expression_list (multiple values) @@ -1091,11 +1175,13 @@ void go_process_statement(GoLSPContext* ctx, TSNode node) { uint32_t lhs_count = ts_node_named_child_count(left); for (uint32_t i = 0; i < lhs_count; i++) { TSNode lhs_var = ts_node_named_child(left, i); - if (strcmp(ts_node_type(lhs_var), "identifier") != 0) continue; - char* var_name = lsp_node_text(ctx, lhs_var); - if (!var_name || strcmp(var_name, "_") == 0) continue; + if (strcmp(ts_node_type(lhs_var), "identifier") != 0) + continue; + char *var_name = lsp_node_text(ctx, lhs_var); + if (!var_name || strcmp(var_name, "_") == 0) + continue; - const CBMType* var_type = cbm_type_unknown(); + const CBMType *var_type = cbm_type_unknown(); if (rhs_type) { if (rhs_type->kind == CBM_TYPE_TUPLE && (int)i < rhs_type->data.tuple.count) { var_type = rhs_type->data.tuple.elems[i]; @@ -1106,7 +1192,7 @@ void go_process_statement(GoLSPContext* ctx, TSNode node) { go_scope_bind_value(ctx, var_name, var_type, i == 0 ? rhs_callable : NULL); } } else if (strcmp(ts_node_type(left), "identifier") == 0) { - char* var_name = lsp_node_text(ctx, left); + char *var_name = lsp_node_text(ctx, left); if (var_name && strcmp(var_name, "_") != 0 && rhs_type) { go_scope_bind_value(ctx, var_name, rhs_type, rhs_callable); } @@ -1156,7 +1242,7 @@ void go_process_statement(GoLSPContext* ctx, TSNode node) { TSNode type_node = ts_node_child_by_field_name(node, "type", 4); TSNode value_node = ts_node_child_by_field_name(node, "value", 5); - const CBMType* var_type = cbm_type_unknown(); + const CBMType *var_type = cbm_type_unknown(); TSNode callable_value = value_node; if (!ts_node_is_null(value_node) && strcmp(ts_node_type(value_node), "expression_list") == 0) { @@ -1181,9 +1267,10 @@ void go_process_statement(GoLSPContext* ctx, TSNode node) { uint32_t vnc = ts_node_child_count(node); for (uint32_t vi = 0; vi < vnc; vi++) { TSNode ch = ts_node_child(node, vi); - if (ts_node_is_null(ch) || !ts_node_is_named(ch)) continue; + if (ts_node_is_null(ch) || !ts_node_is_named(ch)) + continue; if (strcmp(ts_node_type(ch), "identifier") == 0) { - char* var_name = lsp_node_text(ctx, ch); + char *var_name = lsp_node_text(ctx, ch); if (var_name && strcmp(var_name, "_") != 0) { go_scope_bind_value(ctx, var_name, var_type, callable_qn); } @@ -1197,9 +1284,10 @@ void go_process_statement(GoLSPContext* ctx, TSNode node) { TSNode name_node = ts_node_child_by_field_name(node, "name", 4); TSNode type_node = ts_node_child_by_field_name(node, "type", 4); TSNode value_node = ts_node_child_by_field_name(node, "value", 5); - if (ts_node_is_null(name_node)) return; + if (ts_node_is_null(name_node)) + return; - const CBMType* const_type = cbm_type_unknown(); + const CBMType *const_type = cbm_type_unknown(); TSNode callable_value = value_node; if (!ts_node_is_null(value_node) && strcmp(ts_node_type(value_node), "expression_list") == 0) { @@ -1222,7 +1310,7 @@ void go_process_statement(GoLSPContext* ctx, TSNode node) { ts_node_is_null(callable_value) ? NULL : go_exact_callable_target(ctx, callable_value); if (strcmp(ts_node_type(name_node), "identifier") == 0) { - char* name = lsp_node_text(ctx, name_node); + char *name = lsp_node_text(ctx, name_node); if (name && strcmp(name, "_") != 0) go_scope_bind_value(ctx, name, const_type, callable_qn); } @@ -1233,13 +1321,14 @@ void go_process_statement(GoLSPContext* ctx, TSNode node) { if (strcmp(kind, "range_clause") == 0) { TSNode left = ts_node_child_by_field_name(node, "left", 4); TSNode right = ts_node_child_by_field_name(node, "right", 5); - if (ts_node_is_null(right)) return; + if (ts_node_is_null(right)) + return; - const CBMType* container_type = go_eval_expr_type(ctx, right); + const CBMType *container_type = go_eval_expr_type(ctx, right); // Determine key and value types based on container type - const CBMType* key_type = cbm_type_unknown(); - const CBMType* val_type = cbm_type_unknown(); + const CBMType *key_type = cbm_type_unknown(); + const CBMType *val_type = cbm_type_unknown(); if (container_type) { switch (container_type->kind) { @@ -1268,9 +1357,11 @@ void go_process_statement(GoLSPContext* ctx, TSNode node) { uint32_t lhs_count = ts_node_named_child_count(left); for (uint32_t i = 0; i < lhs_count; i++) { TSNode var_node = ts_node_named_child(left, i); - if (strcmp(ts_node_type(var_node), "identifier") != 0) continue; - char* var_name = lsp_node_text(ctx, var_node); - if (!var_name || strcmp(var_name, "_") == 0) continue; + if (strcmp(ts_node_type(var_node), "identifier") != 0) + continue; + char *var_name = lsp_node_text(ctx, var_node); + if (!var_name || strcmp(var_name, "_") == 0) + continue; cbm_scope_bind(ctx->current_scope, var_name, i == 0 ? key_type : val_type); } } @@ -1286,7 +1377,8 @@ void go_process_statement(GoLSPContext* ctx, TSNode node) { static void go_emit_resolved_kind(GoLSPContext *ctx, const char *callee_qn, const char *strategy, float confidence, const char *source_name, CBMResolvedKind kind, TSNode site) { - if (!ctx->resolved_calls || !callee_qn || !ctx->enclosing_func_qn) return; + if (!ctx->resolved_calls || !callee_qn || !ctx->enclosing_func_qn) + return; CBMResolvedCall rc = {0}; rc.caller_qn = ctx->enclosing_func_qn; @@ -1345,7 +1437,8 @@ static void emit_unresolved_reference(GoLSPContext *ctx, const char *source_name // Emit a diagnostic for an unresolved call (confidence 0.0). static void emit_unresolved_call(GoLSPContext *ctx, const char *expr_text, const char *reason, TSNode site) { - if (!ctx->resolved_calls || !ctx->enclosing_func_qn) return; + if (!ctx->resolved_calls || !ctx->enclosing_func_qn) + return; CBMResolvedCall rc = {0}; rc.caller_qn = ctx->enclosing_func_qn; @@ -1428,9 +1521,10 @@ static void go_invalidate_control_flow_aliases(GoLSPContext *ctx, TSNode node, i // --- Walk call expressions and resolve them --- -static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { - if (ts_node_is_null(node)) return; - const char* kind = ts_node_type(node); +static void resolve_calls_in_node_inner(GoLSPContext *ctx, TSNode node) { + if (ts_node_is_null(node)) + return; + const char *kind = ts_node_type(node); // Process statements to build scope go_process_statement(ctx, node); @@ -1446,22 +1540,23 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { go_resolve_value_references_at(ctx, node); TSNode func_node = ts_node_child_by_field_name(node, "function", 8); if (!ts_node_is_null(func_node)) { - const char* fk = ts_node_type(func_node); + const char *fk = ts_node_type(func_node); // selector_expression: obj.Method() or pkg.Func() if (strcmp(fk, "selector_expression") == 0) { TSNode operand = ts_node_child_by_field_name(func_node, "operand", 7); TSNode field = ts_node_child_by_field_name(func_node, "field", 5); if (!ts_node_is_null(operand) && !ts_node_is_null(field)) { - char* field_name = lsp_node_text(ctx, field); + char *field_name = lsp_node_text(ctx, field); // Check if operand is a package import if (strcmp(ts_node_type(operand), "identifier") == 0) { - char* pkg_name = lsp_node_text(ctx, operand); + char *pkg_name = lsp_node_text(ctx, operand); if (pkg_name) { - const char* pkg_qn = resolve_import(ctx, pkg_name); + const char *pkg_qn = resolve_import(ctx, pkg_name); if (pkg_qn && field_name) { - const CBMRegisteredFunc* f = cbm_registry_lookup_symbol(ctx->registry, pkg_qn, field_name); + const CBMRegisteredFunc *f = + cbm_registry_lookup_symbol(ctx->registry, pkg_qn, field_name); if (f) { emit_resolved_call(ctx, f->qualified_name, "lsp_direct", 0.95f, node); @@ -1479,14 +1574,15 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { // Type-based method dispatch if (field_name) { - const CBMType* recv_type = go_eval_expr_type(ctx, operand); - const CBMType* base = recv_type; - if (base && base->kind == CBM_TYPE_POINTER) base = cbm_type_deref(base); + const CBMType *recv_type = go_eval_expr_type(ctx, operand); + const CBMType *base = recv_type; + if (base && base->kind == CBM_TYPE_POINTER) + base = cbm_type_deref(base); if (base && base->kind == CBM_TYPE_NAMED) { const char *recv_qn = base->data.named.qualified_name; - const CBMRegisteredType *receiver_type = cbm_registry_lookup_type( - ctx->registry, recv_qn); + const CBMRegisteredType *receiver_type = + cbm_registry_lookup_type(ctx->registry, recv_qn); const char *alt_qn = NULL; /* Re-qualify NAMED receivers that embed an import * alias segment (cross-package field type texts) — @@ -1495,7 +1591,8 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { alt_qn = go_requalify_via_imports(ctx, recv_qn); if (alt_qn) { receiver_type = cbm_registry_lookup_type(ctx->registry, alt_qn); - if (receiver_type) recv_qn = alt_qn; + if (receiver_type) + recv_qn = alt_qn; } } /* Registered interface receivers must reach the @@ -1503,13 +1600,12 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { * method registrations are signatures, not concrete * dispatch targets. */ if (!receiver_type || !receiver_type->is_interface) { - const CBMRegisteredFunc *method = go_lookup_field_or_method( - ctx, recv_qn, field_name); + const CBMRegisteredFunc *method = + go_lookup_field_or_method(ctx, recv_qn, field_name); if (method) { const char *strategy = "lsp_type_dispatch"; if (method->receiver_type && - strcmp(method->receiver_type, - recv_qn) != 0) { + strcmp(method->receiver_type, recv_qn) != 0) { strategy = "lsp_embed_dispatch"; } emit_resolved_call(ctx, method->qualified_name, strategy, 0.95f, @@ -1519,15 +1615,16 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { } } - // Interface dispatch: NAMED type that is an interface, or bare INTERFACE type + // Interface dispatch: NAMED type that is an interface, or bare INTERFACE + // type if (base && field_name) { bool is_iface = (base->kind == CBM_TYPE_INTERFACE); - const char* iface_qn = NULL; + const char *iface_qn = NULL; if (!is_iface && base->kind == CBM_TYPE_NAMED) { - const CBMRegisteredType* rt = cbm_registry_lookup_type(ctx->registry, - base->data.named.qualified_name); + const CBMRegisteredType *rt = cbm_registry_lookup_type( + ctx->registry, base->data.named.qualified_name); if (!rt) { - const char* alt_qn = go_requalify_via_imports( + const char *alt_qn = go_requalify_via_imports( ctx, base->data.named.qualified_name); if (alt_qn) rt = cbm_registry_lookup_type(ctx->registry, alt_qn); @@ -1538,32 +1635,47 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { } } if (is_iface) { - // Try interface satisfaction: find concrete types implementing this interface - const CBMRegisteredType* iface_rt = iface_qn ? - cbm_registry_lookup_type(ctx->registry, iface_qn) : NULL; - if (iface_rt && iface_rt->method_names && iface_rt->method_names[0]) { + // Try interface satisfaction: find concrete types implementing this + // interface + const CBMRegisteredType *iface_rt = + iface_qn ? cbm_registry_lookup_type(ctx->registry, iface_qn) + : NULL; + if (iface_rt && iface_rt->method_names && + iface_rt->method_names[0]) { // Count interface methods int iface_mcount = 0; - while (iface_rt->method_names[iface_mcount]) iface_mcount++; + while (iface_rt->method_names[iface_mcount]) + iface_mcount++; // Scan all registered types for satisfaction - const char* sole_impl_qn = NULL; + const char *sole_impl_qn = NULL; int impl_count = 0; // Skip stdlib types when interface is from a project package - bool iface_is_project = iface_qn && strchr(iface_qn, '/') != NULL; - for (int ti = 0; ti < ctx->registry->type_count && impl_count < 2; ti++) { - const CBMRegisteredType* cand = &ctx->registry->types[ti]; - if (cand->is_interface) continue; - if (!cand->qualified_name) continue; - if (cand->alias_of) continue; - // For project interfaces, skip stdlib candidates (no '/' in QN) - if (iface_is_project && !strchr(cand->qualified_name, '/')) continue; + bool iface_is_project = + iface_qn && strchr(iface_qn, '/') != NULL; + CBMTypeShortIter all_types; + cbm_registry_all_types_chain(ctx->registry, &all_types); + for (int ti = -1; + impl_count < 2 && + (ti = cbm_type_short_iter_next(&all_types)) >= 0;) { + const CBMRegisteredType *cand = &all_types.reg->types[ti]; + if (cand->is_interface) + continue; + if (!cand->qualified_name) + continue; + if (cand->alias_of) + continue; + // For project interfaces, skip stdlib candidates (no '/' in + // QN) + if (iface_is_project && !strchr(cand->qualified_name, '/')) + continue; // Check if candidate has all interface methods bool satisfies = true; for (int mi = 0; mi < iface_mcount; mi++) { - if (!cbm_registry_lookup_method(ctx->registry, - cand->qualified_name, iface_rt->method_names[mi])) { + if (!cbm_registry_lookup_method( + ctx->registry, cand->qualified_name, + iface_rt->method_names[mi])) { satisfies = false; break; } @@ -1576,14 +1688,15 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { if (impl_count == 1 && sole_impl_qn) { // Single implementer: resolve to concrete method - const CBMRegisteredFunc* concrete_method = - cbm_registry_lookup_method(ctx->registry, sole_impl_qn, field_name); + const CBMRegisteredFunc *concrete_method = + cbm_registry_lookup_method(ctx->registry, sole_impl_qn, + field_name); if (concrete_method) { // Sole-implementer interface dispatch is an unambiguous - // resolution (exactly one concrete method); rank it at least - // as high as a direct type dispatch (0.95) so the concrete - // `Type.method` wins over the interface-method type_dispatch - // for the same call site. + // resolution (exactly one concrete method); rank it at + // least as high as a direct type dispatch (0.95) so the + // concrete `Type.method` wins over the interface-method + // type_dispatch for the same call site. emit_resolved_call(ctx, concrete_method->qualified_name, "lsp_interface_resolve", 0.95f, node); @@ -1611,7 +1724,7 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { field_name), "method_not_found", node); } else if (cbm_type_is_unknown(recv_type)) { - char* operand_text = lsp_node_text(ctx, operand); + char *operand_text = lsp_node_text(ctx, operand); emit_unresolved_call( ctx, cbm_arena_sprintf(ctx->arena, "%s.%s", @@ -1624,7 +1737,7 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { // Direct function call: FuncName() if (strcmp(fk, "identifier") == 0) { - char* name = lsp_node_text(ctx, func_node); + char *name = lsp_node_text(ctx, func_node); if (name && !is_go_builtin_func(name)) { if (cbm_scope_contains(ctx->current_scope, name)) { const char *alias_target = @@ -1638,7 +1751,8 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { goto recurse; } // Package-local function - const CBMRegisteredFunc* f = cbm_registry_lookup_symbol(ctx->registry, ctx->package_qn, name); + const CBMRegisteredFunc *f = + cbm_registry_lookup_symbol(ctx->registry, ctx->package_qn, name); if (f) { emit_resolved_call(ctx, f->qualified_name, "lsp_direct", 0.95f, node); } else { @@ -1651,18 +1765,16 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { recurse:; // Push scope for blocks and statements that introduce variables - bool push_scope = (strcmp(kind, "block") == 0 || - strcmp(kind, "if_statement") == 0 || - strcmp(kind, "for_statement") == 0 || - strcmp(kind, "expression_switch_statement") == 0); + bool push_scope = + (strcmp(kind, "block") == 0 || strcmp(kind, "if_statement") == 0 || + strcmp(kind, "for_statement") == 0 || strcmp(kind, "expression_switch_statement") == 0); if (push_scope) { ctx->current_scope = cbm_scope_push(ctx->arena, ctx->current_scope); } // Process initializer field for if/for/switch statements. // Go patterns: if err := f(); ..., for i := 0; ..., switch v := x; v { ... } - if (strcmp(kind, "if_statement") == 0 || - strcmp(kind, "for_statement") == 0 || + if (strcmp(kind, "if_statement") == 0 || strcmp(kind, "for_statement") == 0 || strcmp(kind, "expression_switch_statement") == 0) { TSNode init = ts_node_child_by_field_name(node, "initializer", 11); if (!ts_node_is_null(init)) { @@ -1688,14 +1800,15 @@ recurse:; // identifier (operand "animal") // type_case (case clause, may repeat) if (strcmp(kind, "type_switch_statement") == 0) { - const char* switch_var = NULL; + const char *switch_var = NULL; // Pass 1: find switch variable and operand type uint32_t nc2 = ts_node_child_count(node); bool found_assign = false; for (uint32_t i = 0; i < nc2; i++) { TSNode child = ts_node_child(node, i); - if (ts_node_is_null(child)) continue; - const char* ck = ts_node_type(child); + if (ts_node_is_null(child)) + continue; + const char *ck = ts_node_type(child); // expression_list before := is the LHS (variable name) if (!found_assign && strcmp(ck, "expression_list") == 0) { @@ -1705,8 +1818,9 @@ recurse:; } // := operator marks the assignment if (!ts_node_is_named(child)) { - char* tok = lsp_node_text(ctx, child); - if (tok && strcmp(tok, ":=") == 0) found_assign = true; + char *tok = lsp_node_text(ctx, child); + if (tok && strcmp(tok, ":=") == 0) + found_assign = true; } // identifier after := is the operand being type-switched if (found_assign && strcmp(ck, "identifier") == 0) { @@ -1717,25 +1831,27 @@ recurse:; // Pass 2: process each type_case with narrowed scope for (uint32_t i = 0; i < nc2; i++) { TSNode child = ts_node_child(node, i); - if (ts_node_is_null(child) || !ts_node_is_named(child)) continue; - if (strcmp(ts_node_type(child), "type_case") != 0) continue; + if (ts_node_is_null(child) || !ts_node_is_named(child)) + continue; + if (strcmp(ts_node_type(child), "type_case") != 0) + continue; - CBMScope* saved = ctx->current_scope; + CBMScope *saved = ctx->current_scope; ctx->current_scope = cbm_scope_push(ctx->arena, ctx->current_scope); // Find case type and bind switch variable uint32_t cc_count = ts_node_child_count(child); for (uint32_t j = 0; j < cc_count; j++) { TSNode cc_child = ts_node_child(child, j); - if (ts_node_is_null(cc_child) || !ts_node_is_named(cc_child)) continue; - const char* cc_kind = ts_node_type(cc_child); + if (ts_node_is_null(cc_child) || !ts_node_is_named(cc_child)) + continue; + const char *cc_kind = ts_node_type(cc_child); if (strcmp(cc_kind, "type_identifier") == 0 || strcmp(cc_kind, "qualified_type") == 0 || - strcmp(cc_kind, "pointer_type") == 0 || - strcmp(cc_kind, "slice_type") == 0) { + strcmp(cc_kind, "pointer_type") == 0 || strcmp(cc_kind, "slice_type") == 0) { if (switch_var) { cbm_scope_bind(ctx->current_scope, switch_var, - go_parse_type_node(ctx, cc_child)); + go_parse_type_node(ctx, cc_child)); } break; } @@ -1744,20 +1860,22 @@ recurse:; // Recurse into case body statements for (uint32_t j = 0; j < cc_count; j++) { TSNode cc_child = ts_node_child(child, j); - if (ts_node_is_null(cc_child) || !ts_node_is_named(cc_child)) continue; - const char* cc_kind = ts_node_type(cc_child); + if (ts_node_is_null(cc_child) || !ts_node_is_named(cc_child)) + continue; + const char *cc_kind = ts_node_type(cc_child); // Skip type nodes, process everything else (expression_statement, etc.) if (strcmp(cc_kind, "type_identifier") == 0 || strcmp(cc_kind, "qualified_type") == 0 || - strcmp(cc_kind, "pointer_type") == 0 || - strcmp(cc_kind, "slice_type") == 0) continue; + strcmp(cc_kind, "pointer_type") == 0 || strcmp(cc_kind, "slice_type") == 0) + continue; resolve_calls_in_node(ctx, cc_child); } ctx->current_scope = saved; } - if (push_scope) ctx->current_scope = cbm_scope_pop(ctx->current_scope); + if (push_scope) + ctx->current_scope = cbm_scope_pop(ctx->current_scope); return; } @@ -1767,19 +1885,21 @@ recurse:; uint32_t nc3 = ts_node_child_count(node); for (uint32_t i = 0; i < nc3; i++) { TSNode child = ts_node_child(node, i); - if (ts_node_is_null(child) || !ts_node_is_named(child)) continue; - const char* ck3 = ts_node_type(child); + if (ts_node_is_null(child) || !ts_node_is_named(child)) + continue; + const char *ck3 = ts_node_type(child); if (strcmp(ck3, "communication_case") == 0 || strcmp(ck3, "default_case") == 0) { - CBMScope* saved = ctx->current_scope; + CBMScope *saved = ctx->current_scope; ctx->current_scope = cbm_scope_push(ctx->arena, ctx->current_scope); // Process children: receive_statement binds vars, then recurse body uint32_t cc_count = ts_node_child_count(child); for (uint32_t j = 0; j < cc_count; j++) { TSNode cc_child = ts_node_child(child, j); - if (ts_node_is_null(cc_child) || !ts_node_is_named(cc_child)) continue; - const char* cc_kind = ts_node_type(cc_child); + if (ts_node_is_null(cc_child) || !ts_node_is_named(cc_child)) + continue; + const char *cc_kind = ts_node_type(cc_child); if (strcmp(cc_kind, "receive_statement") == 0) { // receive_statement: left := <-right @@ -1788,10 +1908,10 @@ recurse:; TSNode left = ts_node_child_by_field_name(cc_child, "left", 4); TSNode right = ts_node_child_by_field_name(cc_child, "right", 5); if (!ts_node_is_null(right)) { - const CBMType* right_type = go_eval_expr_type(ctx, right); + const CBMType *right_type = go_eval_expr_type(ctx, right); // right_type might be a channel (if right is the channel) // or already the elem type (if right is a <-ch unary expr) - const CBMType* recv_type = right_type; + const CBMType *recv_type = right_type; if (right_type && right_type->kind == CBM_TYPE_CHANNEL) { recv_type = right_type->data.channel.elem; } @@ -1800,19 +1920,21 @@ recurse:; uint32_t lhs_count = ts_node_named_child_count(left); for (uint32_t k = 0; k < lhs_count; k++) { TSNode var_node = ts_node_named_child(left, k); - if (strcmp(ts_node_type(var_node), "identifier") != 0) continue; - char* var_name = lsp_node_text(ctx, var_node); - if (!var_name || strcmp(var_name, "_") == 0) continue; + if (strcmp(ts_node_type(var_node), "identifier") != 0) + continue; + char *var_name = lsp_node_text(ctx, var_node); + if (!var_name || strcmp(var_name, "_") == 0) + continue; if (k == 0) { cbm_scope_bind(ctx->current_scope, var_name, recv_type); } else { // second var is the ok bool cbm_scope_bind(ctx->current_scope, var_name, - cbm_type_builtin(ctx->arena, "bool")); + cbm_type_builtin(ctx->arena, "bool")); } } } else if (strcmp(ts_node_type(left), "identifier") == 0) { - char* var_name = lsp_node_text(ctx, left); + char *var_name = lsp_node_text(ctx, left); if (var_name && strcmp(var_name, "_") != 0) { cbm_scope_bind(ctx->current_scope, var_name, recv_type); } @@ -1828,20 +1950,24 @@ recurse:; ctx->current_scope = saved; } } - if (push_scope) ctx->current_scope = cbm_scope_pop(ctx->current_scope); + if (push_scope) + ctx->current_scope = cbm_scope_pop(ctx->current_scope); return; } // Recurse into children via a cursor (O(n)); ts_node_child(node,i) is O(i) // → O(n²) on a wide node. + // The cursor is this depth's pooled one (cbm_cursor_acquire): the recursion + // below runs one level deeper and never touches it. { - TSTreeCursor cursor = ts_tree_cursor_new(node); - if (ts_tree_cursor_goto_first_child(&cursor)) { + cbm_cursor_lease_t lease; + TSTreeCursor *cursor = cbm_cursor_acquire(&lease, ctx->walk_depth, node); + if (ts_tree_cursor_goto_first_child(cursor)) { do { - resolve_calls_in_node(ctx, ts_tree_cursor_current_node(&cursor)); - } while (ts_tree_cursor_goto_next_sibling(&cursor)); + resolve_calls_in_node(ctx, ts_tree_cursor_current_node(cursor)); + } while (ts_tree_cursor_goto_next_sibling(cursor)); } - ts_tree_cursor_delete(&cursor); + cbm_cursor_release(&lease); } if (push_scope) { @@ -1851,13 +1977,15 @@ recurse:; // --- Process a single function body --- -static void process_function(GoLSPContext* ctx, TSNode func_node) { +static void process_function(GoLSPContext *ctx, TSNode func_node) { // Set enclosing function QN TSNode name_node = ts_node_child_by_field_name(func_node, "name", 4); - if (ts_node_is_null(name_node)) return; + if (ts_node_is_null(name_node)) + return; - char* func_name = lsp_node_text(ctx, name_node); - if (!func_name || !func_name[0]) return; + char *func_name = lsp_node_text(ctx, name_node); + if (!func_name || !func_name[0]) + return; // Enclosing-function QN must be the BARE package.Func form (no receiver // type segment). The textual call events (extract_unified.c) source calls @@ -1870,7 +1998,7 @@ static void process_function(GoLSPContext* ctx, TSNode func_node) { ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, func_name); // Push function scope - CBMScope* saved_scope = ctx->current_scope; + CBMScope *saved_scope = ctx->current_scope; ctx->current_scope = cbm_scope_push(ctx->arena, ctx->current_scope); // Bind parameters into scope (including variadic) @@ -1879,15 +2007,17 @@ static void process_function(GoLSPContext* ctx, TSNode func_node) { uint32_t nc = ts_node_child_count(params); for (uint32_t i = 0; i < nc; i++) { TSNode param = ts_node_child(params, i); - if (ts_node_is_null(param) || !ts_node_is_named(param)) continue; - const char* pk = ts_node_type(param); + if (ts_node_is_null(param) || !ts_node_is_named(param)) + continue; + const char *pk = ts_node_type(param); bool is_variadic = (strcmp(pk, "variadic_parameter_declaration") == 0); - if (!is_variadic && strcmp(pk, "parameter_declaration") != 0) continue; + if (!is_variadic && strcmp(pk, "parameter_declaration") != 0) + continue; // Get type TSNode type_node = ts_node_child_by_field_name(param, "type", 4); - const CBMType* param_type = go_parse_type_node(ctx, type_node); + const CBMType *param_type = go_parse_type_node(ctx, type_node); // Variadic: ...T is []T in the function body if (is_variadic && param_type) { @@ -1898,9 +2028,10 @@ static void process_function(GoLSPContext* ctx, TSNode func_node) { uint32_t pnc = ts_node_child_count(param); for (uint32_t j = 0; j < pnc; j++) { TSNode child = ts_node_child(param, j); - if (ts_node_is_null(child) || !ts_node_is_named(child)) continue; + if (ts_node_is_null(child) || !ts_node_is_named(child)) + continue; if (strcmp(ts_node_type(child), "identifier") == 0) { - char* pname = lsp_node_text(ctx, child); + char *pname = lsp_node_text(ctx, child); if (pname && strcmp(pname, "_") != 0) { cbm_scope_bind(ctx->current_scope, pname, param_type); } @@ -1913,23 +2044,26 @@ static void process_function(GoLSPContext* ctx, TSNode func_node) { TSNode result_node = ts_node_child_by_field_name(func_node, "result", 6); if (!ts_node_is_null(result_node)) { // result can be a parameter_list (named returns) or a simple type - const char* rk = ts_node_type(result_node); + const char *rk = ts_node_type(result_node); if (strcmp(rk, "parameter_list") == 0) { uint32_t rnc = ts_node_child_count(result_node); for (uint32_t i = 0; i < rnc; i++) { TSNode rparam = ts_node_child(result_node, i); - if (ts_node_is_null(rparam) || !ts_node_is_named(rparam)) continue; - if (strcmp(ts_node_type(rparam), "parameter_declaration") != 0) continue; + if (ts_node_is_null(rparam) || !ts_node_is_named(rparam)) + continue; + if (strcmp(ts_node_type(rparam), "parameter_declaration") != 0) + continue; TSNode rtype = ts_node_child_by_field_name(rparam, "type", 4); - const CBMType* ret_type = go_parse_type_node(ctx, rtype); + const CBMType *ret_type = go_parse_type_node(ctx, rtype); uint32_t rpnc = ts_node_child_count(rparam); for (uint32_t j = 0; j < rpnc; j++) { TSNode child = ts_node_child(rparam, j); - if (ts_node_is_null(child) || !ts_node_is_named(child)) continue; + if (ts_node_is_null(child) || !ts_node_is_named(child)) + continue; if (strcmp(ts_node_type(child), "identifier") == 0) { - char* rname = lsp_node_text(ctx, child); + char *rname = lsp_node_text(ctx, child); if (rname && strcmp(rname, "_") != 0) { cbm_scope_bind(ctx->current_scope, rname, ret_type); } @@ -1946,11 +2080,13 @@ static void process_function(GoLSPContext* ctx, TSNode func_node) { uint32_t rnc = ts_node_child_count(recv); for (uint32_t i = 0; i < rnc; i++) { TSNode rp = ts_node_child(recv, i); - if (ts_node_is_null(rp) || !ts_node_is_named(rp)) continue; - if (strcmp(ts_node_type(rp), "parameter_declaration") != 0) continue; + if (ts_node_is_null(rp) || !ts_node_is_named(rp)) + continue; + if (strcmp(ts_node_type(rp), "parameter_declaration") != 0) + continue; TSNode rtype = ts_node_child_by_field_name(rp, "type", 4); - const CBMType* recv_type = go_parse_type_node(ctx, rtype); + const CBMType *recv_type = go_parse_type_node(ctx, rtype); // Find receiver name uint32_t rpnc = ts_node_child_count(rp); @@ -1958,7 +2094,7 @@ static void process_function(GoLSPContext* ctx, TSNode func_node) { TSNode rc = ts_node_child(rp, j); if (!ts_node_is_null(rc) && ts_node_is_named(rc) && strcmp(ts_node_type(rc), "identifier") == 0) { - char* rname = lsp_node_text(ctx, rc); + char *rname = lsp_node_text(ctx, rc); if (rname && strcmp(rname, "_") != 0) { cbm_scope_bind(ctx->current_scope, rname, recv_type); } @@ -1980,24 +2116,26 @@ static void process_function(GoLSPContext* ctx, TSNode func_node) { // --- Process entire file --- -void go_lsp_process_file(GoLSPContext* ctx, TSNode root) { - if (ts_node_is_null(root)) return; +void go_lsp_process_file(GoLSPContext *ctx, TSNode root) { + if (ts_node_is_null(root)) + return; // Collect top-level children once (O(n)); ts_node_child(root,i) is O(i) → O(n²). uint32_t kn = 0; - TSNode* kids = cbm_lsp_collect_children(ctx->arena, root, &kn); + TSNode *kids = cbm_lsp_collect_children(ctx->arena, root, &kn); // Pass 1: Bind package-level var/const declarations into root scope. // Must happen before processing functions so all globals are visible. for (uint32_t i = 0; i < kn; i++) { TSNode child = kids[i]; - const char* kind = ts_node_type(child); + const char *kind = ts_node_type(child); if (strcmp(kind, "var_declaration") == 0 || strcmp(kind, "const_declaration") == 0) { uint32_t vnc = ts_node_child_count(child); for (uint32_t j = 0; j < vnc; j++) { TSNode spec = ts_node_child(child, j); - if (ts_node_is_null(spec) || !ts_node_is_named(spec)) continue; + if (ts_node_is_null(spec) || !ts_node_is_named(spec)) + continue; go_process_statement(ctx, spec); } } @@ -2006,10 +2144,9 @@ void go_lsp_process_file(GoLSPContext* ctx, TSNode root) { // Pass 2: Process function/method bodies for (uint32_t i = 0; i < kn; i++) { TSNode child = kids[i]; - const char* kind = ts_node_type(child); + const char *kind = ts_node_type(child); - if (strcmp(kind, "function_declaration") == 0 || - strcmp(kind, "method_declaration") == 0) { + if (strcmp(kind, "function_declaration") == 0 || strcmp(kind, "method_declaration") == 0) { process_function(ctx, child); } } @@ -2017,8 +2154,9 @@ void go_lsp_process_file(GoLSPContext* ctx, TSNode root) { // --- Helper: parse Go return type text into CBMType --- -const CBMType* cbm_parse_return_type_text(CBMArena* a, const char* text, const char* module_qn) { - if (!text || !text[0]) return cbm_type_unknown(); +const CBMType *cbm_parse_return_type_text(CBMArena *a, const char *text, const char *module_qn) { + if (!text || !text[0]) + return cbm_type_unknown(); // Pointer: *Foo if (text[0] == '*') { @@ -2031,15 +2169,13 @@ const CBMType* cbm_parse_return_type_text(CBMArena* a, const char* text, const c } // Builtin types - static const char* builtins[] = { - "int","int8","int16","int32","int64", - "uint","uint8","uint16","uint32","uint64", - "float32","float64","string","bool","byte","rune","error", - "any","uintptr", - NULL - }; - for (const char** b = builtins; *b; b++) { - if (strcmp(text, *b) == 0) return cbm_type_builtin(a, text); + static const char *builtins[] = {"int", "int8", "int16", "int32", "int64", + "uint", "uint8", "uint16", "uint32", "uint64", + "float32", "float64", "string", "bool", "byte", + "rune", "error", "any", "uintptr", NULL}; + for (const char **b = builtins; *b; b++) { + if (strcmp(text, *b) == 0) + return cbm_type_builtin(a, text); } // Named type: assume local to module @@ -2057,23 +2193,80 @@ static const CBMType *cbm_go_lsp_parse_signature_param_text(CBMArena *arena, con return cbm_parse_return_type_text(arena, text, module_qn); } +// --- Shared Go stdlib registry --- + +/* The Go stdlib (2,328 functions, 321 types) registered ONCE per process into + * a finalized, sealed registry that every per-file registry chains to as its + * fallback -- the overlay contract of type_registry.h: lookups and the chain + * iterators reach it, refinements copy into the head, nothing writes it. + * Registered into each file's registry instead, it was 13.4 k re-registrations + * on the Go corpus, 2.5 GB of copies, and every lookup of the unfinalized + * per-file registry scanned the stdlib linearly (waste sanitizer, 2026-09-17). + * Threads racing the first build each build a copy and the first to publish + * wins; the others free theirs -- no thread waits. NULL on allocation + * failure: the caller registers the stdlib into its own registry as before. */ +typedef struct { + CBMArena arena; + CBMTypeRegistry reg; +} go_stdlib_holder_t; + +static _Atomic(go_stdlib_holder_t *) g_go_stdlib_shared; + +static const CBMTypeRegistry *go_stdlib_shared(void) { + go_stdlib_holder_t *ready = atomic_load_explicit(&g_go_stdlib_shared, memory_order_acquire); + if (ready) + return &ready->reg; + go_stdlib_holder_t *built = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*built)); + if (!built) + return NULL; + cbm_arena_init(&built->arena); + if (built->arena.nblocks == 0) { + cbm_free(CBM_MEM_CLASS_OTHER, built); + return NULL; + } + cbm_registry_init(&built->reg, &built->arena); + cbm_go_stdlib_register(&built->reg, &built->arena); + cbm_registry_finalize(&built->reg); + built->reg.read_only = true; + go_stdlib_holder_t *expected = NULL; + if (atomic_compare_exchange_strong_explicit(&g_go_stdlib_shared, &expected, built, + memory_order_acq_rel, memory_order_acquire)) { + return &built->reg; + } + cbm_arena_destroy(&built->arena); + cbm_free(CBM_MEM_CLASS_OTHER, built); + return &expected->reg; +} + +/* A per-file registry sees the stdlib: chained to the shared copy, or + * registered into it when the shared copy cannot be built. */ +static void go_registry_with_stdlib(CBMTypeRegistry *reg, CBMArena *arena) { + const CBMTypeRegistry *shared = go_stdlib_shared(); + if (shared) { + reg->fallback = shared; + } else { + cbm_go_stdlib_register(reg, arena); + } +} + // --- Entry point: build registry from file defs + run LSP --- -void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, - const char* source, int source_len, TSNode root) { +void cbm_run_go_lsp(CBMArena *arena, CBMFileResult *result, const char *source, int source_len, + TSNode root) { CBMTypeRegistry reg; cbm_registry_init(®, arena); - // Register Go stdlib types/functions - cbm_go_stdlib_register(®, arena); + // Go stdlib types/functions: the shared sealed registry, chained + go_registry_with_stdlib(®, arena); - const char* module_qn = result->module_qn; + const char *module_qn = result->module_qn; // Phase 1: Register all types and functions from the file's own definitions for (int i = 0; i < result->defs.count; i++) { - CBMDefinition* d = &result->defs.items[i]; - if (!d->qualified_name || !d->name) continue; + CBMDefinition *d = &result->defs.items[i]; + if (!d->qualified_name || !d->name) + continue; // Register every type-like container (Class/Struct/Type/Interface/Enum/ // Trait). Struct included so a Go `type T struct {...}` (now labelled @@ -2088,13 +2281,16 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, // Populate embedded_types from base_classes (Go struct embedding) if (d->base_classes) { int bc_count = 0; - while (d->base_classes[bc_count]) bc_count++; + while (d->base_classes[bc_count]) + bc_count++; if (bc_count > 0) { - const char** embedded = (const char**)cbm_arena_alloc(arena, (bc_count + 1) * sizeof(const char*)); + const char **embedded = (const char **)cbm_arena_alloc( + arena, (bc_count + 1) * sizeof(const char *)); for (int j = 0; j < bc_count; j++) { - const char* bc = d->base_classes[j]; + const char *bc = d->base_classes[j]; // Strip pointer prefix for embedded *Type - while (bc[0] == '*') bc++; + while (bc[0] == '*') + bc++; // Qualify the embedded type name embedded[j] = cbm_arena_sprintf(arena, "%s.%s", module_qn, bc); } @@ -2114,20 +2310,23 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, rf.short_name = d->name; // Build FUNC type from return_types - const CBMType** ret_types = NULL; + const CBMType **ret_types = NULL; if (d->return_types) { int count = 0; - while (d->return_types[count]) count++; + while (d->return_types[count]) + count++; if (count > 0) { - ret_types = (const CBMType**)cbm_arena_alloc(arena, (count + 1) * sizeof(const CBMType*)); + ret_types = (const CBMType **)cbm_arena_alloc( + arena, (count + 1) * sizeof(const CBMType *)); for (int j = 0; j < count; j++) { - ret_types[j] = cbm_parse_return_type_text(arena, d->return_types[j], module_qn); + ret_types[j] = + cbm_parse_return_type_text(arena, d->return_types[j], module_qn); } ret_types[count] = NULL; } } else if (d->return_type && d->return_type[0]) { // Fallback: single return_type string - ret_types = (const CBMType**)cbm_arena_alloc(arena, 2 * sizeof(const CBMType*)); + ret_types = (const CBMType **)cbm_arena_alloc(arena, 2 * sizeof(const CBMType *)); ret_types[0] = cbm_parse_return_type_text(arena, d->return_type, module_qn); ret_types[1] = NULL; } @@ -2136,7 +2335,7 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, * is present, including a non-NULL carrier with an explicit zero * count. Its slots deliberately have no names because legacy name * extraction is not guaranteed to have the same multiplicity. */ - const CBMType** param_types_arr = NULL; + const CBMType **param_types_arr = NULL; const char **param_names_arr = NULL; bool has_ordered_params = d->signature_param_types != NULL || d->signature_param_count > 0; @@ -2148,11 +2347,14 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, } else if (d->param_types) { param_names_arr = d->param_names; int count = 0; - while (d->param_types[count]) count++; + while (d->param_types[count]) + count++; if (count > 0) { - param_types_arr = (const CBMType**)cbm_arena_alloc(arena, (count + 1) * sizeof(const CBMType*)); + param_types_arr = (const CBMType **)cbm_arena_alloc( + arena, (count + 1) * sizeof(const CBMType *)); for (int j = 0; j < count; j++) { - param_types_arr[j] = cbm_parse_return_type_text(arena, d->param_types[j], module_qn); + param_types_arr[j] = + cbm_parse_return_type_text(arena, d->param_types[j], module_qn); } param_types_arr[count] = NULL; } @@ -2204,28 +2406,32 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, { // Collect top-level children once (O(n)); ts_node_child(root,i) is O(i) → O(n²). uint32_t rkn = 0; - TSNode* rkids = cbm_lsp_collect_children(arena, root, &rkn); + TSNode *rkids = cbm_lsp_collect_children(arena, root, &rkn); for (uint32_t i = 0; i < rkn; i++) { TSNode top = rkids[i]; - const char* tk = ts_node_type(top); + const char *tk = ts_node_type(top); // type_declaration contains type_spec children - if (strcmp(tk, "type_declaration") != 0) continue; + if (strcmp(tk, "type_declaration") != 0) + continue; uint32_t td_nc = ts_node_child_count(top); for (uint32_t j = 0; j < td_nc; j++) { TSNode spec = ts_node_child(top, j); - if (ts_node_is_null(spec) || !ts_node_is_named(spec)) continue; - const char* spec_kind = ts_node_type(spec); + if (ts_node_is_null(spec) || !ts_node_is_named(spec)) + continue; + const char *spec_kind = ts_node_type(spec); // type_alias: type Foo = Bar if (strcmp(spec_kind, "type_alias") == 0) { TSNode alias_name = ts_node_child_by_field_name(spec, "name", 4); TSNode alias_type = ts_node_child_by_field_name(spec, "type", 4); if (!ts_node_is_null(alias_name) && !ts_node_is_null(alias_type)) { - char* aname = cbm_node_text(arena, alias_name, source); - char* atarget = cbm_node_text(arena, alias_type, source); + char *aname = cbm_node_text(arena, alias_name, source); + char *atarget = cbm_node_text(arena, alias_type, source); if (aname && aname[0] && atarget && atarget[0]) { - const char* alias_type_qn = cbm_arena_sprintf(arena, "%s.%s", module_qn, aname); - const char* alias_target_qn = cbm_arena_sprintf(arena, "%s.%s", module_qn, atarget); + const char *alias_type_qn = + cbm_arena_sprintf(arena, "%s.%s", module_qn, aname); + const char *alias_target_qn = + cbm_arena_sprintf(arena, "%s.%s", module_qn, atarget); bool found_a = false; for (int ti = 0; ti < reg.type_count; ti++) { if (reg.types[ti].qualified_name && @@ -2248,30 +2454,34 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, continue; } - if (strcmp(spec_kind, "type_spec") != 0) continue; + if (strcmp(spec_kind, "type_spec") != 0) + continue; TSNode name_node = ts_node_child_by_field_name(spec, "name", 4); TSNode type_node = ts_node_child_by_field_name(spec, "type", 4); - if (ts_node_is_null(name_node) || ts_node_is_null(type_node)) continue; + if (ts_node_is_null(name_node) || ts_node_is_null(type_node)) + continue; - char* type_name = cbm_node_text(arena,name_node, source); - if (!type_name || !type_name[0]) continue; - const char* type_qn = cbm_arena_sprintf(arena, "%s.%s", module_qn, type_name); + char *type_name = cbm_node_text(arena, name_node, source); + if (!type_name || !type_name[0]) + continue; + const char *type_qn = cbm_arena_sprintf(arena, "%s.%s", module_qn, type_name); // Interface type: extract method names for satisfaction checking if (strcmp(ts_node_type(type_node), "interface_type") == 0) { - const char* iface_methods[64]; + const char *iface_methods[64]; int iface_method_count = 0; uint32_t inl_nc = ts_node_named_child_count(type_node); for (uint32_t k = 0; k < inl_nc && iface_method_count < 63; k++) { TSNode child = ts_node_named_child(type_node, k); - if (ts_node_is_null(child)) continue; - const char* ck = ts_node_type(child); + if (ts_node_is_null(child)) + continue; + const char *ck = ts_node_type(child); // method_spec: MethodName(params) returns if (strcmp(ck, "method_spec") == 0 || strcmp(ck, "method_elem") == 0) { TSNode mname = ts_node_child_by_field_name(child, "name", 4); if (!ts_node_is_null(mname)) { - char* mn = cbm_node_text(arena, mname, source); + char *mn = cbm_node_text(arena, mname, source); if (mn && mn[0]) { iface_methods[iface_method_count++] = mn; } @@ -2281,9 +2491,10 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, if (iface_method_count > 0) { for (int ti = 0; ti < reg.type_count; ti++) { if (!reg.types[ti].qualified_name || - strcmp(reg.types[ti].qualified_name, type_qn) != 0) continue; - const char** names = (const char**)cbm_arena_alloc(arena, - (iface_method_count + 1) * sizeof(const char*)); + strcmp(reg.types[ti].qualified_name, type_qn) != 0) + continue; + const char **names = (const char **)cbm_arena_alloc( + arena, (iface_method_count + 1) * sizeof(const char *)); for (int mi = 0; mi < iface_method_count; mi++) { names[mi] = iface_methods[mi]; } @@ -2295,7 +2506,8 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, continue; } - if (strcmp(ts_node_type(type_node), "struct_type") != 0) continue; + if (strcmp(ts_node_type(type_node), "struct_type") != 0) + continue; // Find field_declaration_list inside struct_type TSNode field_list = ts_node_child_by_field_name(type_node, "body", 4); @@ -2304,13 +2516,14 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, if (ts_node_named_child_count(type_node) > 0) field_list = ts_node_named_child(type_node, 0); } - if (ts_node_is_null(field_list)) continue; + if (ts_node_is_null(field_list)) + continue; // Scan field_declarations for embeds and named fields - const char* embeds[16]; + const char *embeds[16]; int embed_count = 0; - const char* fld_names[64]; - const CBMType* fld_types[64]; + const char *fld_names[64]; + const CBMType *fld_types[64]; int fld_count = 0; // Create a temporary LSP context for parsing field types @@ -2325,8 +2538,10 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, uint32_t fl_nc = ts_node_child_count(field_list); for (uint32_t k = 0; k < fl_nc; k++) { TSNode field = ts_node_child(field_list, k); - if (ts_node_is_null(field) || !ts_node_is_named(field)) continue; - if (strcmp(ts_node_type(field), "field_declaration") != 0) continue; + if (ts_node_is_null(field) || !ts_node_is_named(field)) + continue; + if (strcmp(ts_node_type(field), "field_declaration") != 0) + continue; TSNode fname = ts_node_child_by_field_name(field, "name", 4); TSNode ftype = ts_node_child_by_field_name(field, "type", 4); @@ -2334,16 +2549,19 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, if (ts_node_is_null(fname) && !ts_node_is_null(ftype)) { // Embedded field: has type but no name if (embed_count < 15) { - char* embed_text = cbm_node_text(arena, ftype, source); + char *embed_text = cbm_node_text(arena, ftype, source); if (embed_text && embed_text[0]) { - const char* et = embed_text; - while (*et == '*') et++; - embeds[embed_count++] = cbm_arena_sprintf(arena, "%s.%s", module_qn, et); + const char *et = embed_text; + while (*et == '*') + et++; + embeds[embed_count++] = + cbm_arena_sprintf(arena, "%s.%s", module_qn, et); } } - } else if (!ts_node_is_null(fname) && !ts_node_is_null(ftype) && fld_count < 63) { + } else if (!ts_node_is_null(fname) && !ts_node_is_null(ftype) && + fld_count < 63) { // Named field: name + type - char* fn = cbm_node_text(arena, fname, source); + char *fn = cbm_node_text(arena, fname, source); if (fn && fn[0]) { fld_names[fld_count] = fn; fld_types[fld_count] = go_parse_type_node(&tmp_ctx, ftype); @@ -2355,20 +2573,22 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, // Find the registered type and update it for (int ti = 0; ti < reg.type_count; ti++) { if (!reg.types[ti].qualified_name || - strcmp(reg.types[ti].qualified_name, type_qn) != 0) continue; + strcmp(reg.types[ti].qualified_name, type_qn) != 0) + continue; if (embed_count > 0) { - const char** arr = (const char**)cbm_arena_alloc(arena, - (embed_count + 1) * sizeof(const char*)); - for (int ei = 0; ei < embed_count; ei++) arr[ei] = embeds[ei]; + const char **arr = (const char **)cbm_arena_alloc( + arena, (embed_count + 1) * sizeof(const char *)); + for (int ei = 0; ei < embed_count; ei++) + arr[ei] = embeds[ei]; arr[embed_count] = NULL; reg.types[ti].embedded_types = arr; } if (fld_count > 0) { - const char** names = (const char**)cbm_arena_alloc(arena, - (fld_count + 1) * sizeof(const char*)); - const CBMType** types = (const CBMType**)cbm_arena_alloc(arena, - (fld_count + 1) * sizeof(const CBMType*)); + const char **names = (const char **)cbm_arena_alloc( + arena, (fld_count + 1) * sizeof(const char *)); + const CBMType **types = (const CBMType **)cbm_arena_alloc( + arena, (fld_count + 1) * sizeof(const CBMType *)); for (int fi = 0; fi < fld_count; fi++) { names[fi] = fld_names[fi]; types[fi] = fld_types[fi]; @@ -2393,7 +2613,7 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, // Add imports for (int i = 0; i < result->imports.count; i++) { - CBMImport* imp = &result->imports.items[i]; + CBMImport *imp = &result->imports.items[i]; if (imp->local_name && imp->module_path) { go_lsp_add_import(&lsp_ctx, imp->local_name, imp->module_path); } @@ -2406,30 +2626,35 @@ void cbm_run_go_lsp(CBMArena* arena, CBMFileResult* result, // --- Cross-file LSP: parse source, build registry from defs, run LSP --- // Helper: split "|"-separated string into array of CBMType*. -static const CBMType** split_pipe_types(CBMArena* a, const char* text, const char* module_qn) { - if (!text || !text[0]) return NULL; +static const CBMType **split_pipe_types(CBMArena *a, const char *text, const char *module_qn) { + if (!text || !text[0]) + return NULL; // Count separators int count = 1; - for (const char* p = text; *p; p++) { - if (*p == '|') count++; + for (const char *p = text; *p; p++) { + if (*p == '|') + count++; } - const CBMType** arr = (const CBMType**)cbm_arena_alloc(a, (count + 1) * sizeof(const CBMType*)); - if (!arr) return NULL; + const CBMType **arr = + (const CBMType **)cbm_arena_alloc(a, (count + 1) * sizeof(const CBMType *)); + if (!arr) + return NULL; // Split and parse each type - char* buf = cbm_arena_strdup(a, text); + char *buf = cbm_arena_strdup(a, text); int idx = 0; - char* start = buf; - for (char* p = buf; ; p++) { + char *start = buf; + for (char *p = buf;; p++) { if (*p == '|' || *p == '\0') { char save = *p; *p = '\0'; if (start[0]) { arr[idx++] = cbm_parse_return_type_text(a, start, module_qn); } - if (save == '\0') break; + if (save == '\0') + break; start = p + 1; } } @@ -2438,28 +2663,32 @@ static const CBMType** split_pipe_types(CBMArena* a, const char* text, const cha } // Helper: split "|"-separated string into array of const char*. -static const char** split_pipe_strings(CBMArena* a, const char* text) { - if (!text || !text[0]) return NULL; +static const char **split_pipe_strings(CBMArena *a, const char *text) { + if (!text || !text[0]) + return NULL; int count = 1; - for (const char* p = text; *p; p++) { - if (*p == '|') count++; + for (const char *p = text; *p; p++) { + if (*p == '|') + count++; } - const char** arr = (const char**)cbm_arena_alloc(a, (count + 1) * sizeof(const char*)); - if (!arr) return NULL; + const char **arr = (const char **)cbm_arena_alloc(a, (count + 1) * sizeof(const char *)); + if (!arr) + return NULL; - char* buf = cbm_arena_strdup(a, text); + char *buf = cbm_arena_strdup(a, text); int idx = 0; - char* start = buf; - for (char* p = buf; ; p++) { + char *start = buf; + for (char *p = buf;; p++) { if (*p == '|' || *p == '\0') { char save = *p; *p = '\0'; if (start[0]) { arr[idx++] = cbm_arena_strdup(a, start); } - if (save == '\0') break; + if (save == '\0') + break; start = p + 1; } } @@ -2470,32 +2699,40 @@ static const char** split_pipe_strings(CBMArena* a, const char* text) { // Helper: parse "|"-separated "name:type" field definitions and populate a registered type. // Format: "Binder:Binder|Name:string|Count:int" // type_text is resolved relative to def_module_qn. -static void parse_field_defs_into_type(CBMArena* arena, CBMTypeRegistry* reg, - const char* type_qn, const char* field_defs, const char* def_module_qn) { - if (!field_defs || !field_defs[0] || !type_qn) return; +/* Parse "name:type|name:type" into NULL-terminated name/type arrays. False when + * nothing parsed (or on allocation failure): the type keeps no field info. */ +static bool parse_field_defs(CBMArena *arena, const char *field_defs, const char *def_module_qn, + const char ***names_out, const CBMType ***types_out) { // Count fields int count = 1; - for (const char* p = field_defs; *p; p++) { - if (*p == '|') count++; + for (const char *p = field_defs; *p; p++) { + if (*p == '|') + count++; } - if (count > 63) count = 63; + if (count > 63) + count = 63; - const char** names = (const char**)cbm_arena_alloc(arena, (count + 1) * sizeof(const char*)); - const CBMType** types = (const CBMType**)cbm_arena_alloc(arena, (count + 1) * sizeof(const CBMType*)); - if (!names || !types) return; + const char **names = (const char **)cbm_arena_alloc(arena, (count + 1) * sizeof(const char *)); + const CBMType **types = + (const CBMType **)cbm_arena_alloc(arena, (count + 1) * sizeof(const CBMType *)); + if (!names || !types) + return false; - char* buf = cbm_arena_strdup(arena, field_defs); + char *buf = cbm_arena_strdup(arena, field_defs); int idx = 0; - char* start = buf; - for (char* p = buf; ; p++) { + char *start = buf; + for (char *p = buf;; p++) { if (*p == '|' || *p == '\0') { char save = *p; *p = '\0'; // Parse "name:type" - char* colon = NULL; - for (char* q = start; *q; q++) { - if (*q == ':') { colon = q; break; } + char *colon = NULL; + for (char *q = start; *q; q++) { + if (*q == ':') { + colon = q; + break; + } } if (colon && idx < count) { *colon = '\0'; @@ -2503,14 +2740,25 @@ static void parse_field_defs_into_type(CBMArena* arena, CBMTypeRegistry* reg, types[idx] = cbm_parse_return_type_text(arena, colon + 1, def_module_qn); idx++; } - if (save == '\0') break; + if (save == '\0') + break; start = p + 1; } } names[idx] = NULL; types[idx] = NULL; + *names_out = names; + *types_out = types; + return idx > 0; +} - if (idx > 0) { +static void parse_field_defs_into_type(CBMArena *arena, CBMTypeRegistry *reg, const char *type_qn, + const char *field_defs, const char *def_module_qn) { + if (!field_defs || !field_defs[0] || !type_qn) + return; + const char **names = NULL; + const CBMType **types = NULL; + if (parse_field_defs(arena, field_defs, def_module_qn, &names, &types)) { // Find the registered type and update field info for (int ti = 0; ti < reg->type_count; ti++) { if (reg->types[ti].qualified_name && @@ -2528,16 +2776,19 @@ static void parse_field_defs_into_type(CBMArena* arena, CBMTypeRegistry* reg, // Helper: parse a tree-sitter type AST node with type param awareness. // Like go_parse_type_node but checks type_identifier against type_param_names first, // and works without a full GoLSPContext (uses raw arena/source/module_qn). -static const CBMType* parse_type_node_with_params(CBMArena* arena, TSNode node, - const char* source, const char* module_qn, const char** type_param_names) { - if (ts_node_is_null(node)) return cbm_type_unknown(); +static const CBMType *parse_type_node_with_params(CBMArena *arena, TSNode node, const char *source, + const char *module_qn, + const char **type_param_names) { + if (ts_node_is_null(node)) + return cbm_type_unknown(); - const char* kind = ts_node_type(node); + const char *kind = ts_node_type(node); // type_identifier: check type params first, then resolve as named type if (strcmp(kind, "type_identifier") == 0) { - char* name = cbm_node_text(arena, node, source); - if (!name) return cbm_type_unknown(); + char *name = cbm_node_text(arena, node, source); + if (!name) + return cbm_type_unknown(); // Check type param names if (type_param_names) { for (int i = 0; type_param_names[i]; i++) { @@ -2547,13 +2798,11 @@ static const CBMType* parse_type_node_with_params(CBMArena* arena, TSNode node, } } // Builtin types - if (strcmp(name, "int") == 0 || strcmp(name, "string") == 0 || - strcmp(name, "bool") == 0 || strcmp(name, "float64") == 0 || - strcmp(name, "float32") == 0 || strcmp(name, "byte") == 0 || - strcmp(name, "rune") == 0 || strcmp(name, "error") == 0 || - strcmp(name, "any") == 0 || strcmp(name, "int8") == 0 || - strcmp(name, "int16") == 0 || strcmp(name, "int32") == 0 || - strcmp(name, "int64") == 0 || strcmp(name, "uint") == 0 || + if (strcmp(name, "int") == 0 || strcmp(name, "string") == 0 || strcmp(name, "bool") == 0 || + strcmp(name, "float64") == 0 || strcmp(name, "float32") == 0 || + strcmp(name, "byte") == 0 || strcmp(name, "rune") == 0 || strcmp(name, "error") == 0 || + strcmp(name, "any") == 0 || strcmp(name, "int8") == 0 || strcmp(name, "int16") == 0 || + strcmp(name, "int32") == 0 || strcmp(name, "int64") == 0 || strcmp(name, "uint") == 0 || strcmp(name, "uint8") == 0 || strcmp(name, "uint16") == 0 || strcmp(name, "uint32") == 0 || strcmp(name, "uint64") == 0 || strcmp(name, "uintptr") == 0) { @@ -2564,8 +2813,8 @@ static const CBMType* parse_type_node_with_params(CBMArena* arena, TSNode node, // type_elem: wrapper in type_arguments if (strcmp(kind, "type_elem") == 0 && ts_node_named_child_count(node) > 0) { - return parse_type_node_with_params(arena, ts_node_named_child(node, 0), - source, module_qn, type_param_names); + return parse_type_node_with_params(arena, ts_node_named_child(node, 0), source, module_qn, + type_param_names); } // qualified_type: pkg.Type @@ -2573,8 +2822,8 @@ static const CBMType* parse_type_node_with_params(CBMArena* arena, TSNode node, TSNode pkg_node = ts_node_child_by_field_name(node, "package", 7); TSNode name_node = ts_node_child_by_field_name(node, "name", 4); if (!ts_node_is_null(pkg_node) && !ts_node_is_null(name_node)) { - char* pkg = cbm_node_text(arena, pkg_node, source); - char* name = cbm_node_text(arena, name_node, source); + char *pkg = cbm_node_text(arena, pkg_node, source); + char *name = cbm_node_text(arena, name_node, source); if (pkg && name) { return cbm_type_named(arena, cbm_arena_sprintf(arena, "%s.%s", pkg, name)); } @@ -2586,8 +2835,9 @@ static const CBMType* parse_type_node_with_params(CBMArena* arena, TSNode node, if (strcmp(kind, "pointer_type") == 0) { uint32_t nc = ts_node_named_child_count(node); if (nc > 0) - return cbm_type_pointer(arena, parse_type_node_with_params(arena, - ts_node_named_child(node, nc - 1), source, module_qn, type_param_names)); + return cbm_type_pointer( + arena, parse_type_node_with_params(arena, ts_node_named_child(node, nc - 1), source, + module_qn, type_param_names)); return cbm_type_unknown(); } @@ -2596,8 +2846,8 @@ static const CBMType* parse_type_node_with_params(CBMArena* arena, TSNode node, TSNode elem = ts_node_child_by_field_name(node, "element", 7); if (ts_node_is_null(elem) && ts_node_named_child_count(node) > 0) elem = ts_node_named_child(node, ts_node_named_child_count(node) - 1); - return cbm_type_slice(arena, parse_type_node_with_params(arena, - elem, source, module_qn, type_param_names)); + return cbm_type_slice( + arena, parse_type_node_with_params(arena, elem, source, module_qn, type_param_names)); } // array_type: [N]T @@ -2605,16 +2855,16 @@ static const CBMType* parse_type_node_with_params(CBMArena* arena, TSNode node, TSNode elem = ts_node_child_by_field_name(node, "element", 7); if (ts_node_is_null(elem) && ts_node_named_child_count(node) > 0) elem = ts_node_named_child(node, ts_node_named_child_count(node) - 1); - return cbm_type_slice(arena, parse_type_node_with_params(arena, - elem, source, module_qn, type_param_names)); + return cbm_type_slice( + arena, parse_type_node_with_params(arena, elem, source, module_qn, type_param_names)); } // map_type: map[K]V if (strcmp(kind, "map_type") == 0) { TSNode key = ts_node_child_by_field_name(node, "key", 3); TSNode value = ts_node_child_by_field_name(node, "value", 5); - return cbm_type_map(arena, - parse_type_node_with_params(arena, key, source, module_qn, type_param_names), + return cbm_type_map( + arena, parse_type_node_with_params(arena, key, source, module_qn, type_param_names), parse_type_node_with_params(arena, value, source, module_qn, type_param_names)); } @@ -2623,14 +2873,17 @@ static const CBMType* parse_type_node_with_params(CBMArena* arena, TSNode node, TSNode value = ts_node_child_by_field_name(node, "value", 5); if (ts_node_is_null(value) && ts_node_named_child_count(node) > 0) value = ts_node_named_child(node, ts_node_named_child_count(node) - 1); - char* text = cbm_node_text(arena, node, source); + char *text = cbm_node_text(arena, node, source); int dir = 0; if (text) { - if (strncmp(text, "chan<-", 6) == 0 || strncmp(text, "chan <-", 7) == 0) dir = 1; - else if (strncmp(text, "<-chan", 6) == 0 || strncmp(text, "<- chan", 7) == 0) dir = 2; + if (strncmp(text, "chan<-", 6) == 0 || strncmp(text, "chan <-", 7) == 0) + dir = 1; + else if (strncmp(text, "<-chan", 6) == 0 || strncmp(text, "<- chan", 7) == 0) + dir = 2; } - return cbm_type_channel(arena, parse_type_node_with_params(arena, - value, source, module_qn, type_param_names), dir); + return cbm_type_channel( + arena, parse_type_node_with_params(arena, value, source, module_qn, type_param_names), + dir); } // function_type: func(T) R — full parsing with type param awareness @@ -2638,51 +2891,54 @@ static const CBMType* parse_type_node_with_params(CBMArena* arena, TSNode node, TSNode params_node = ts_node_child_by_field_name(node, "parameters", 10); TSNode result_node = ts_node_child_by_field_name(node, "result", 6); - const CBMType* param_types_arr[16]; + const CBMType *param_types_arr[16]; int pc = 0; if (!ts_node_is_null(params_node)) { uint32_t pnc = ts_node_child_count(params_node); for (uint32_t i = 0; i < pnc && pc < 15; i++) { TSNode child = ts_node_child(params_node, i); - if (ts_node_is_null(child) || !ts_node_is_named(child)) continue; + if (ts_node_is_null(child) || !ts_node_is_named(child)) + continue; if (strcmp(ts_node_type(child), "parameter_declaration") == 0) { TSNode pt = ts_node_child_by_field_name(child, "type", 4); if (!ts_node_is_null(pt)) - param_types_arr[pc++] = parse_type_node_with_params(arena, - pt, source, module_qn, type_param_names); + param_types_arr[pc++] = parse_type_node_with_params( + arena, pt, source, module_qn, type_param_names); } } } param_types_arr[pc] = NULL; - const CBMType* ret_types_arr[16]; + const CBMType *ret_types_arr[16]; int rc = 0; if (!ts_node_is_null(result_node)) { if (strcmp(ts_node_type(result_node), "parameter_list") == 0) { uint32_t rnc = ts_node_child_count(result_node); for (uint32_t i = 0; i < rnc && rc < 15; i++) { TSNode child = ts_node_child(result_node, i); - if (ts_node_is_null(child) || !ts_node_is_named(child)) continue; + if (ts_node_is_null(child) || !ts_node_is_named(child)) + continue; TSNode rt = ts_node_child_by_field_name(child, "type", 4); - if (ts_node_is_null(rt)) rt = child; - ret_types_arr[rc++] = parse_type_node_with_params(arena, - rt, source, module_qn, type_param_names); + if (ts_node_is_null(rt)) + rt = child; + ret_types_arr[rc++] = + parse_type_node_with_params(arena, rt, source, module_qn, type_param_names); } } else { - ret_types_arr[rc++] = parse_type_node_with_params(arena, - result_node, source, module_qn, type_param_names); + ret_types_arr[rc++] = parse_type_node_with_params(arena, result_node, source, + module_qn, type_param_names); } } ret_types_arr[rc] = NULL; - const CBMType** pt = pc > 0 ? (const CBMType**)param_types_arr : NULL; - const CBMType** rt = rc > 0 ? (const CBMType**)ret_types_arr : NULL; + const CBMType **pt = pc > 0 ? (const CBMType **)param_types_arr : NULL; + const CBMType **rt = rc > 0 ? (const CBMType **)ret_types_arr : NULL; return cbm_type_func(arena, NULL, pt, rt); } // interface_type if (strcmp(kind, "interface_type") == 0) { - CBMType* t = (CBMType*)cbm_arena_alloc(arena, sizeof(CBMType)); + CBMType *t = (CBMType *)cbm_arena_alloc(arena, sizeof(CBMType)); memset(t, 0, sizeof(CBMType)); t->kind = CBM_TYPE_INTERFACE; return t; @@ -2690,15 +2946,16 @@ static const CBMType* parse_type_node_with_params(CBMArena* arena, TSNode node, // parenthesized_type: (T) if (strcmp(kind, "parenthesized_type") == 0 && ts_node_named_child_count(node) > 0) { - return parse_type_node_with_params(arena, ts_node_named_child(node, 0), - source, module_qn, type_param_names); + return parse_type_node_with_params(arena, ts_node_named_child(node, 0), source, module_qn, + type_param_names); } // generic_type: Type[T1, T2] if (strcmp(kind, "generic_type") == 0) { TSNode type_node = ts_node_child_by_field_name(node, "type", 4); if (!ts_node_is_null(type_node)) - return parse_type_node_with_params(arena, type_node, source, module_qn, type_param_names); + return parse_type_node_with_params(arena, type_node, source, module_qn, + type_param_names); } return cbm_type_unknown(); @@ -2742,34 +2999,40 @@ static const CBMType *cbm_go_lsp_keep_known_refinement(const CBMType *candidate, // Scan AST for generic function declarations and set type_param_names + re-parse // return types on matching registered functions. -static void extract_type_params_from_ast(CBMArena* arena, CBMTypeRegistry* reg, - TSNode root, const char* source, const char* module_qn) { +static void extract_type_params_from_ast(CBMArena *arena, CBMTypeRegistry *reg, TSNode root, + const char *source, const char *module_qn) { // Collect top-level children once (O(n)); ts_node_child(root,i) is O(i) → O(n²). uint32_t rkn = 0; - TSNode* rkids = cbm_lsp_collect_children(arena, root, &rkn); + TSNode *rkids = cbm_lsp_collect_children(arena, root, &rkn); for (uint32_t i = 0; i < rkn; i++) { TSNode top = rkids[i]; - if (strcmp(ts_node_type(top), "function_declaration") != 0) continue; + if (strcmp(ts_node_type(top), "function_declaration") != 0) + continue; // Check for type_parameter_list (field name: "type_parameters", 15 chars) TSNode tp_list = ts_node_child_by_field_name(top, "type_parameters", 15); - if (ts_node_is_null(tp_list)) continue; + if (ts_node_is_null(tp_list)) + continue; // Get function name TSNode name_node = ts_node_child_by_field_name(top, "name", 4); - if (ts_node_is_null(name_node)) continue; - char* func_name = cbm_node_text(arena, name_node, source); - if (!func_name || !func_name[0]) continue; + if (ts_node_is_null(name_node)) + continue; + char *func_name = cbm_node_text(arena, name_node, source); + if (!func_name || !func_name[0]) + continue; // Extract type param names from type_parameter_declaration children - const char* params[16]; + const char *params[16]; int param_count = 0; uint32_t tp_nc = ts_node_child_count(tp_list); for (uint32_t j = 0; j < tp_nc && param_count < 15; j++) { TSNode child = ts_node_child(tp_list, j); - if (ts_node_is_null(child) || !ts_node_is_named(child)) continue; - if (strcmp(ts_node_type(child), "type_parameter_declaration") != 0) continue; + if (ts_node_is_null(child) || !ts_node_is_named(child)) + continue; + if (strcmp(ts_node_type(child), "type_parameter_declaration") != 0) + continue; TSNode pname = ts_node_child_by_field_name(child, "name", 4); if (ts_node_is_null(pname)) { // Fallback: first type_identifier child @@ -2784,21 +3047,24 @@ static void extract_type_params_from_ast(CBMArena* arena, CBMTypeRegistry* reg, } } if (!ts_node_is_null(pname)) { - char* pn = cbm_node_text(arena, pname, source); + char *pn = cbm_node_text(arena, pname, source); if (pn && pn[0]) { params[param_count++] = cbm_arena_strdup(arena, pn); } } } - if (param_count == 0) continue; + if (param_count == 0) + continue; params[param_count] = NULL; // Build arena-allocated type_param_names array - const char** tp_names = (const char**)cbm_arena_alloc(arena, (param_count + 1) * sizeof(const char*)); - for (int j = 0; j <= param_count; j++) tp_names[j] = params[j]; + const char **tp_names = + (const char **)cbm_arena_alloc(arena, (param_count + 1) * sizeof(const char *)); + for (int j = 0; j <= param_count; j++) + tp_names[j] = params[j]; // Find the matching registered function and set type_param_names - const char* func_qn = cbm_arena_sprintf(arena, "%s.%s", module_qn, func_name); + const char *func_qn = cbm_arena_sprintf(arena, "%s.%s", module_qn, func_name); for (int fi = 0; fi < reg->func_count; fi++) { if (reg->funcs[fi].qualified_name && strcmp(reg->funcs[fi].qualified_name, func_qn) == 0) { @@ -2807,23 +3073,24 @@ static void extract_type_params_from_ast(CBMArena* arena, CBMTypeRegistry* reg, // Re-parse return types with type param awareness if (reg->funcs[fi].signature && reg->funcs[fi].signature->kind == CBM_TYPE_FUNC && reg->funcs[fi].signature->data.func.return_types) { - const CBMType** old_rets = reg->funcs[fi].signature->data.func.return_types; + const CBMType **old_rets = reg->funcs[fi].signature->data.func.return_types; int ret_count = 0; - while (old_rets[ret_count]) ret_count++; + while (old_rets[ret_count]) + ret_count++; // Check if any return type is a NAMED that matches a type param bool needs_reparse = false; for (int ri = 0; ri < ret_count && !needs_reparse; ri++) { - const CBMType* check = old_rets[ri]; + const CBMType *check = old_rets[ri]; // Unwrap slice/pointer to get inner named type if (check->kind == CBM_TYPE_SLICE && check->data.slice.elem) check = check->data.slice.elem; else if (check->kind == CBM_TYPE_POINTER && check->data.pointer.elem) check = check->data.pointer.elem; if (check->kind == CBM_TYPE_NAMED) { - const char* qn = check->data.named.qualified_name; - const char* dot = strrchr(qn, '.'); - const char* short_name = dot ? dot + 1 : qn; + const char *qn = check->data.named.qualified_name; + const char *dot = strrchr(qn, '.'); + const char *short_name = dot ? dot + 1 : qn; for (int pi = 0; pi < param_count; pi++) { if (strcmp(short_name, tp_names[pi]) == 0) { needs_reparse = true; @@ -2836,17 +3103,19 @@ static void extract_type_params_from_ast(CBMArena* arena, CBMTypeRegistry* reg, if (needs_reparse) { TSNode result_node = ts_node_child_by_field_name(top, "result", 6); if (!ts_node_is_null(result_node)) { - const CBMType** new_rets = (const CBMType**)cbm_arena_alloc(arena, - (ret_count + 1) * sizeof(const CBMType*)); + const CBMType **new_rets = (const CBMType **)cbm_arena_alloc( + arena, (ret_count + 1) * sizeof(const CBMType *)); if (strcmp(ts_node_type(result_node), "parameter_list") == 0) { int idx = 0; uint32_t rnc = ts_node_child_count(result_node); for (uint32_t ri = 0; ri < rnc && idx < ret_count; ri++) { TSNode rchild = ts_node_child(result_node, ri); - if (ts_node_is_null(rchild) || !ts_node_is_named(rchild)) continue; + if (ts_node_is_null(rchild) || !ts_node_is_named(rchild)) + continue; TSNode rtype = ts_node_child_by_field_name(rchild, "type", 4); - if (ts_node_is_null(rtype)) rtype = rchild; + if (ts_node_is_null(rtype)) + rtype = rchild; const CBMType *parsed = parse_type_node_with_params( arena, rtype, source, module_qn, tp_names); new_rets[idx] = @@ -2880,20 +3149,22 @@ static void extract_type_params_from_ast(CBMArena* arena, CBMTypeRegistry* reg, // Also re-parse param types with TYPE_PARAM awareness (for implicit generics) if (reg->funcs[fi].signature && reg->funcs[fi].signature->kind == CBM_TYPE_FUNC && reg->funcs[fi].signature->data.func.param_types) { - const CBMType** old_params = reg->funcs[fi].signature->data.func.param_types; + const CBMType **old_params = reg->funcs[fi].signature->data.func.param_types; int pc = 0; - while (old_params[pc]) pc++; + while (old_params[pc]) + pc++; // Re-parse from AST parameter list TSNode params_node = ts_node_child_by_field_name(top, "parameters", 10); if (!ts_node_is_null(params_node)) { - const CBMType** new_params = (const CBMType**)cbm_arena_alloc(arena, - (pc + 1) * sizeof(const CBMType*)); + const CBMType **new_params = (const CBMType **)cbm_arena_alloc( + arena, (pc + 1) * sizeof(const CBMType *)); int idx = 0; bool overflow = false; uint32_t pnc = ts_node_child_count(params_node); for (uint32_t pi = 0; pi < pnc && !overflow; pi++) { TSNode pchild = ts_node_child(params_node, pi); - if (ts_node_is_null(pchild) || !ts_node_is_named(pchild)) continue; + if (ts_node_is_null(pchild) || !ts_node_is_named(pchild)) + continue; const char *param_kind = ts_node_type(pchild); if (strcmp(param_kind, "parameter_declaration") != 0 && strcmp(param_kind, "variadic_parameter_declaration") != 0) { @@ -2935,37 +3206,37 @@ static void extract_type_params_from_ast(CBMArena* arena, CBMTypeRegistry* reg, } // Forward: tree-sitter Go language (from lang_specs.c) -extern const TSLanguage* tree_sitter_go(void); +extern const TSLanguage *tree_sitter_go(void); -void cbm_run_go_lsp_cross( - CBMArena* arena, - const char* source, int source_len, - const char* module_qn, - CBMLSPDef* defs, int def_count, - const char** import_names, const char** import_qns, int import_count, - TSTree* cached_tree, - CBMResolvedCallArray* out) -{ - if (!source || source_len <= 0) return; +void cbm_run_go_lsp_cross(CBMArena *arena, const char *source, int source_len, + const char *module_qn, CBMLSPDef *defs, int def_count, + const char **import_names, const char **import_qns, int import_count, + TSTree *cached_tree, CBMResolvedCallArray *out) { + if (!source || source_len <= 0) + return; // 1. Use cached tree if available, otherwise parse fresh - TSParser* parser = NULL; - TSTree* tree = cached_tree; + TSParser *parser = NULL; + TSTree *tree = cached_tree; bool owns_tree = false; if (!tree) { parser = ts_parser_new(); - if (!parser) return; + if (!parser) + return; ts_parser_set_language(parser, tree_sitter_go()); tree = ts_parser_parse_string(parser, NULL, source, source_len); owns_tree = true; - if (!tree) { ts_parser_delete(parser); return; } + if (!tree) { + ts_parser_delete(parser); + return; + } } TSNode root = ts_tree_root_node(tree); - // 2. Build registry + // 2. Build registry (the stdlib is the shared sealed registry, chained) CBMTypeRegistry reg; cbm_registry_init(®, arena); - cbm_go_stdlib_register(®, arena); + go_registry_with_stdlib(®, arena); // Register all defs (file-local + cross-file). // Perf: borrow strings from defs[] directly — they live in the @@ -2973,18 +3244,19 @@ void cbm_run_go_lsp_cross( // wasted work. On kubernetes (~110k defs × 11k files × 5 strdups // ~= 6B mallocs) this alone saves ~90s of resolve wall time. for (int i = 0; i < def_count; i++) { - CBMLSPDef* d = &defs[i]; - if (!d->qualified_name || !d->short_name || !d->label) continue; + CBMLSPDef *d = &defs[i]; + if (!d->qualified_name || !d->short_name || !d->label) + continue; - const char* def_mod = d->def_module_qn ? d->def_module_qn : module_qn; + const char *def_mod = d->def_module_qn ? d->def_module_qn : module_qn; // Every type-like container (Type/Class/Struct/Interface/Enum/Trait). // Struct included so Go structs (now labelled "Struct") register as types. if (cbm_label_is_type_like(d->label)) { CBMRegisteredType rt; memset(&rt, 0, sizeof(rt)); - rt.qualified_name = d->qualified_name; // borrowed - rt.short_name = d->short_name; // borrowed + rt.qualified_name = d->qualified_name; // borrowed + rt.short_name = d->short_name; // borrowed rt.is_interface = d->is_interface || strcmp(d->label, "Interface") == 0; rt.embedded_types = split_pipe_strings(arena, d->embedded_types); @@ -3005,11 +3277,11 @@ void cbm_run_go_lsp_cross( if (strcmp(d->label, "Function") == 0 || strcmp(d->label, "Method") == 0) { CBMRegisteredFunc rf; memset(&rf, 0, sizeof(rf)); - rf.qualified_name = d->qualified_name; // borrowed - rf.short_name = d->short_name; // borrowed + rf.qualified_name = d->qualified_name; // borrowed + rf.short_name = d->short_name; // borrowed // Build FUNC type from return_types text - const CBMType** ret_types = split_pipe_types(arena, d->return_types, def_mod); + const CBMType **ret_types = split_pipe_types(arena, d->return_types, def_mod); CBMGoLSPSignatureParserContext param_parser_ctx = {def_mod}; const CBMType **param_types = cbm_type_materialize_signature_params( arena, d->signature_param_types, d->signature_param_count, @@ -3018,14 +3290,14 @@ void cbm_run_go_lsp_cross( // Method receiver if (strcmp(d->label, "Method") == 0 && d->receiver_type && d->receiver_type[0]) { - rf.receiver_type = d->receiver_type; // borrowed + rf.receiver_type = d->receiver_type; // borrowed // Auto-create type entry if not exists if (!cbm_registry_lookup_type(®, rf.receiver_type)) { CBMRegisteredType auto_type; memset(&auto_type, 0, sizeof(auto_type)); auto_type.qualified_name = rf.receiver_type; - const char* dot = strrchr(d->receiver_type, '.'); - auto_type.short_name = dot ? dot + 1 : rf.receiver_type; // borrowed substring + const char *dot = strrchr(d->receiver_type, '.'); + auto_type.short_name = dot ? dot + 1 : rf.receiver_type; // borrowed substring cbm_registry_add_type(®, auto_type); } } @@ -3038,26 +3310,30 @@ void cbm_run_go_lsp_cross( { // Collect top-level children once (O(n)); ts_node_child(root,i) is O(i) → O(n²). uint32_t rkn = 0; - TSNode* rkids = cbm_lsp_collect_children(arena, root, &rkn); + TSNode *rkids = cbm_lsp_collect_children(arena, root, &rkn); for (uint32_t i = 0; i < rkn; i++) { TSNode top = rkids[i]; - if (strcmp(ts_node_type(top), "type_declaration") != 0) continue; + if (strcmp(ts_node_type(top), "type_declaration") != 0) + continue; uint32_t td_nc = ts_node_child_count(top); for (uint32_t j = 0; j < td_nc; j++) { TSNode spec = ts_node_child(top, j); - if (ts_node_is_null(spec) || !ts_node_is_named(spec)) continue; - const char* spec_kind2 = ts_node_type(spec); + if (ts_node_is_null(spec) || !ts_node_is_named(spec)) + continue; + const char *spec_kind2 = ts_node_type(spec); // type_alias: type Foo = Bar if (strcmp(spec_kind2, "type_alias") == 0) { TSNode alias_name = ts_node_child_by_field_name(spec, "name", 4); TSNode alias_type = ts_node_child_by_field_name(spec, "type", 4); if (!ts_node_is_null(alias_name) && !ts_node_is_null(alias_type)) { - char* aname = cbm_node_text(arena, alias_name, source); - char* atarget = cbm_node_text(arena, alias_type, source); + char *aname = cbm_node_text(arena, alias_name, source); + char *atarget = cbm_node_text(arena, alias_type, source); if (aname && aname[0] && atarget && atarget[0]) { - const char* alias_type_qn = cbm_arena_sprintf(arena, "%s.%s", module_qn, aname); - const char* alias_target_qn = cbm_arena_sprintf(arena, "%s.%s", module_qn, atarget); + const char *alias_type_qn = + cbm_arena_sprintf(arena, "%s.%s", module_qn, aname); + const char *alias_target_qn = + cbm_arena_sprintf(arena, "%s.%s", module_qn, atarget); bool found_a = false; for (int ti = 0; ti < reg.type_count; ti++) { if (reg.types[ti].qualified_name && @@ -3080,28 +3356,33 @@ void cbm_run_go_lsp_cross( continue; } - if (strcmp(spec_kind2, "type_spec") != 0) continue; + if (strcmp(spec_kind2, "type_spec") != 0) + continue; TSNode name_node = ts_node_child_by_field_name(spec, "name", 4); TSNode type_node = ts_node_child_by_field_name(spec, "type", 4); - if (ts_node_is_null(name_node) || ts_node_is_null(type_node)) continue; - if (strcmp(ts_node_type(type_node), "struct_type") != 0) continue; + if (ts_node_is_null(name_node) || ts_node_is_null(type_node)) + continue; + if (strcmp(ts_node_type(type_node), "struct_type") != 0) + continue; - char* type_name = cbm_node_text(arena, name_node, source); - if (!type_name || !type_name[0]) continue; - const char* type_qn = cbm_arena_sprintf(arena, "%s.%s", module_qn, type_name); + char *type_name = cbm_node_text(arena, name_node, source); + if (!type_name || !type_name[0]) + continue; + const char *type_qn = cbm_arena_sprintf(arena, "%s.%s", module_qn, type_name); TSNode field_list = ts_node_child_by_field_name(type_node, "body", 4); if (ts_node_is_null(field_list)) { if (ts_node_named_child_count(type_node) > 0) field_list = ts_node_named_child(type_node, 0); } - if (ts_node_is_null(field_list)) continue; + if (ts_node_is_null(field_list)) + continue; - const char* embeds[16]; + const char *embeds[16]; int embed_count = 0; - const char* fld_names[64]; - const CBMType* fld_types[64]; + const char *fld_names[64]; + const CBMType *fld_types[64]; int fld_count = 0; GoLSPContext tmp_ctx; @@ -3118,21 +3399,26 @@ void cbm_run_go_lsp_cross( uint32_t fl_nc = ts_node_child_count(field_list); for (uint32_t k = 0; k < fl_nc; k++) { TSNode field = ts_node_child(field_list, k); - if (ts_node_is_null(field) || !ts_node_is_named(field)) continue; - if (strcmp(ts_node_type(field), "field_declaration") != 0) continue; + if (ts_node_is_null(field) || !ts_node_is_named(field)) + continue; + if (strcmp(ts_node_type(field), "field_declaration") != 0) + continue; TSNode fname = ts_node_child_by_field_name(field, "name", 4); TSNode ftype = ts_node_child_by_field_name(field, "type", 4); if (ts_node_is_null(fname) && !ts_node_is_null(ftype)) { if (embed_count < 15) { - char* embed_text = cbm_node_text(arena, ftype, source); + char *embed_text = cbm_node_text(arena, ftype, source); if (embed_text && embed_text[0]) { - const char* et = embed_text; - while (*et == '*') et++; - embeds[embed_count++] = cbm_arena_sprintf(arena, "%s.%s", module_qn, et); + const char *et = embed_text; + while (*et == '*') + et++; + embeds[embed_count++] = + cbm_arena_sprintf(arena, "%s.%s", module_qn, et); } } - } else if (!ts_node_is_null(fname) && !ts_node_is_null(ftype) && fld_count < 63) { - char* fn = cbm_node_text(arena, fname, source); + } else if (!ts_node_is_null(fname) && !ts_node_is_null(ftype) && + fld_count < 63) { + char *fn = cbm_node_text(arena, fname, source); if (fn && fn[0]) { fld_names[fld_count] = fn; fld_types[fld_count] = go_parse_type_node(&tmp_ctx, ftype); @@ -3143,20 +3429,22 @@ void cbm_run_go_lsp_cross( for (int ti = 0; ti < reg.type_count; ti++) { if (!reg.types[ti].qualified_name || - strcmp(reg.types[ti].qualified_name, type_qn) != 0) continue; + strcmp(reg.types[ti].qualified_name, type_qn) != 0) + continue; if (embed_count > 0) { - const char** arr = (const char**)cbm_arena_alloc(arena, - (embed_count + 1) * sizeof(const char*)); - for (int ei = 0; ei < embed_count; ei++) arr[ei] = embeds[ei]; + const char **arr = (const char **)cbm_arena_alloc( + arena, (embed_count + 1) * sizeof(const char *)); + for (int ei = 0; ei < embed_count; ei++) + arr[ei] = embeds[ei]; arr[embed_count] = NULL; reg.types[ti].embedded_types = arr; } if (fld_count > 0) { - const char** names = (const char**)cbm_arena_alloc(arena, - (fld_count + 1) * sizeof(const char*)); - const CBMType** types = (const CBMType**)cbm_arena_alloc(arena, - (fld_count + 1) * sizeof(const CBMType*)); + const char **names = (const char **)cbm_arena_alloc( + arena, (fld_count + 1) * sizeof(const char *)); + const CBMType **types = (const CBMType **)cbm_arena_alloc( + arena, (fld_count + 1) * sizeof(const CBMType *)); for (int fi = 0; fi < fld_count; fi++) { names[fi] = fld_names[fi]; types[fi] = fld_types[fi]; @@ -3198,7 +3486,8 @@ void cbm_run_go_lsp_cross( // 5. Cleanup — only free if we allocated if (owns_tree) { ts_tree_delete(tree); - if (parser) ts_parser_delete(parser); + if (parser) + ts_parser_delete(parser); } } @@ -3217,23 +3506,47 @@ void cbm_run_go_lsp_cross( * 1b (and not already in defs[] embedded_types/field_defs strings) are * missed. In practice the per-file LSP during extract already captures * those via the def strings. */ -CBMTypeRegistry* cbm_go_build_cross_registry( - CBMArena* arena, CBMLSPDef* defs, int def_count) { - if (!arena) return NULL; - CBMTypeRegistry* reg = (CBMTypeRegistry*)cbm_arena_alloc(arena, sizeof(*reg)); - if (!reg) return NULL; +CBMTypeRegistry *cbm_go_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, int def_count) { + if (!arena) + return NULL; + CBMTypeRegistry *reg = (CBMTypeRegistry *)cbm_arena_alloc(arena, sizeof(*reg)); + if (!reg) + return NULL; cbm_registry_init(reg, arena); cbm_go_stdlib_register(reg, arena); + /* Qualified name -> FIRST type index while the registry is built: it is + * not finalized until the end, so its own lookup scans every type, and the + * receiver check below plus the field-definition lookup made the build + * O(types x (methods + structs)) -- 4.4 s single-threaded on the Go corpus, + * the scaling lane's parse_field_defs_into_type x2.9 (waste sanitizer, + * 2026-09-17). First-match semantics are kept exactly: a receiver's + * placeholder type added before its struct still receives the fields. + * NULL on allocation failure: the linear paths below run as before. */ + CBMHashTable *first_type = cbm_ht_create((uint32_t)(def_count / 4 + 1024)); + int noted_types = 0; +#define GO_NOTE_NEW_TYPES() \ + do { \ + for (; first_type && noted_types < reg->type_count; noted_types++) { \ + const char *tqn = reg->types[noted_types].qualified_name; \ + if (tqn && !cbm_ht_has(first_type, tqn)) { \ + cbm_ht_set(first_type, tqn, (void *)(uintptr_t)(noted_types + 1)); \ + } \ + } \ + } while (0) + GO_NOTE_NEW_TYPES(); + for (int i = 0; i < def_count; i++) { - CBMLSPDef* d = &defs[i]; - if (!d->qualified_name || !d->short_name || !d->label) continue; + CBMLSPDef *d = &defs[i]; + if (!d->qualified_name || !d->short_name || !d->label) + continue; /* Filter to Go defs only — the all_defs[] array is mixed-language. */ - if (d->lang != CBM_LANG_GO) continue; + if (d->lang != CBM_LANG_GO) + continue; /* In the pre-built path every def carries its own def_module_qn * (set by cbm_pxc_collect_all_defs). There is no caller module to * fall back to — this registry is project-wide, not per-file. */ - const char* def_mod = d->def_module_qn ? d->def_module_qn : ""; + const char *def_mod = d->def_module_qn ? d->def_module_qn : ""; // Every type-like container (Type/Class/Struct/Interface/Enum/Trait). // Struct included so Go structs (now labelled "Struct") register as types. @@ -3248,9 +3561,20 @@ CBMTypeRegistry* cbm_go_build_cross_registry( rt.method_names = split_pipe_strings(arena, d->method_names_str); } cbm_registry_add_type(reg, rt); + GO_NOTE_NEW_TYPES(); if (d->field_defs && d->field_defs[0]) { - parse_field_defs_into_type(arena, reg, rt.qualified_name, - d->field_defs, def_mod); + if (first_type) { + const char **names = NULL; + const CBMType **types = NULL; + uintptr_t at = (uintptr_t)cbm_ht_get(first_type, rt.qualified_name); + if (parse_field_defs(arena, d->field_defs, def_mod, &names, &types) && at) { + reg->types[at - 1].field_names = names; + reg->types[at - 1].field_types = types; + } + } else { + parse_field_defs_into_type(arena, reg, rt.qualified_name, d->field_defs, + def_mod); + } } } @@ -3259,7 +3583,7 @@ CBMTypeRegistry* cbm_go_build_cross_registry( memset(&rf, 0, sizeof(rf)); rf.qualified_name = d->qualified_name; /* borrowed */ rf.short_name = d->short_name; - const CBMType** ret_types = split_pipe_types(arena, d->return_types, def_mod); + const CBMType **ret_types = split_pipe_types(arena, d->return_types, def_mod); CBMGoLSPSignatureParserContext param_parser_ctx = {def_mod}; const CBMType **param_types = cbm_type_materialize_signature_params( arena, d->signature_param_types, d->signature_param_count, @@ -3267,44 +3591,51 @@ CBMTypeRegistry* cbm_go_build_cross_registry( rf.signature = cbm_type_func(arena, NULL, param_types, ret_types); if (strcmp(d->label, "Method") == 0 && d->receiver_type && d->receiver_type[0]) { rf.receiver_type = d->receiver_type; - if (!cbm_registry_lookup_type(reg, rf.receiver_type)) { + bool known = first_type ? cbm_ht_has(first_type, rf.receiver_type) + : cbm_registry_lookup_type(reg, rf.receiver_type) != NULL; + if (!known) { CBMRegisteredType auto_type; memset(&auto_type, 0, sizeof(auto_type)); auto_type.qualified_name = rf.receiver_type; - const char* dot = strrchr(d->receiver_type, '.'); + const char *dot = strrchr(d->receiver_type, '.'); auto_type.short_name = dot ? dot + 1 : rf.receiver_type; cbm_registry_add_type(reg, auto_type); + GO_NOTE_NEW_TYPES(); } } cbm_registry_add_func(reg, rf); } } +#undef GO_NOTE_NEW_TYPES + cbm_ht_free(first_type); cbm_registry_finalize(reg); reg->read_only = true; /* seal: shared Tier-2 registry is read-only during resolve */ return reg; } -void cbm_run_go_lsp_cross_with_registry( - CBMArena* arena, - const char* source, int source_len, - const char* module_qn, - CBMTypeRegistry* reg, - const char** import_names, const char** import_qns, int import_count, - TSTree* cached_tree, - CBMResolvedCallArray* out) { - if (!source || source_len <= 0 || !reg) return; +void cbm_run_go_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, + const char *module_qn, CBMTypeRegistry *reg, + const char **import_names, const char **import_qns, + int import_count, TSTree *cached_tree, + CBMResolvedCallArray *out) { + if (!source || source_len <= 0 || !reg) + return; - TSParser* parser = NULL; - TSTree* tree = cached_tree; + TSParser *parser = NULL; + TSTree *tree = cached_tree; bool owns_tree = false; if (!tree) { parser = ts_parser_new(); - if (!parser) return; + if (!parser) + return; ts_parser_set_language(parser, tree_sitter_go()); tree = ts_parser_parse_string(parser, NULL, source, source_len); owns_tree = true; - if (!tree) { ts_parser_delete(parser); return; } + if (!tree) { + ts_parser_delete(parser); + return; + } } TSNode root = ts_tree_root_node(tree); @@ -3322,7 +3653,8 @@ void cbm_run_go_lsp_cross_with_registry( if (owns_tree) { ts_tree_delete(tree); - if (parser) ts_parser_delete(parser); + if (parser) + ts_parser_delete(parser); } } @@ -3353,46 +3685,47 @@ void cbm_run_go_lsp_cross_with_registry( * This is what the user meant: walk the AST ONCE (during extract), * capture everything cross-LSP needs as metadata, and let cross-LSP * be a pure lookup pass. */ -int cbm_go_fast_resolve_qualified_calls( - CBMFileResult* result, - CBMTypeRegistry* reg, - const char** import_names, const char** import_qns, int import_count) { - if (!result || !reg) return 0; +int cbm_go_fast_resolve_qualified_calls(CBMFileResult *result, CBMTypeRegistry *reg, + const char **import_names, const char **import_qns, + int import_count) { + if (!result || !reg) + return 0; /* Snapshot the count: we append to resolved_calls inside the loop, * but only want to scan the original unresolved entries. */ int initial_count = result->resolved_calls.count; int newly_resolved = 0; for (int i = 0; i < initial_count; i++) { - const CBMResolvedCall* uc = &result->resolved_calls.items[i]; - if (!uc->strategy || !uc->callee_qn || !uc->caller_qn) continue; - if (strcmp(uc->strategy, "lsp_unresolved") != 0) continue; + const CBMResolvedCall *uc = &result->resolved_calls.items[i]; + if (!uc->strategy || !uc->callee_qn || !uc->caller_qn) + continue; + if (strcmp(uc->strategy, "lsp_unresolved") != 0) + continue; /* Case 1: callee_qn is already a fully-qualified type/pkg * symbol that per-file LSP couldn't find in its local registry * but the global registry has. (method_not_found with NAMED * receiver, or any other case where callee_qn looks like a * full QN). Direct lookup. */ - const CBMRegisteredFunc* f = cbm_registry_lookup_func(reg, uc->callee_qn); + const CBMRegisteredFunc *f = cbm_registry_lookup_func(reg, uc->callee_qn); /* Case 2: callee_qn is "local_pkg_alias.suffix" — the alias * needs translating through the file's import map to get the * real module QN. */ if (!f) { - const char* dot = strchr(uc->callee_qn, '.'); + const char *dot = strchr(uc->callee_qn, '.'); if (dot) { size_t prefix_len = (size_t)(dot - uc->callee_qn); if (prefix_len > 0 && prefix_len < 256) { char prefix[256]; memcpy(prefix, uc->callee_qn, prefix_len); prefix[prefix_len] = '\0'; - const char* suffix = dot + 1; + const char *suffix = dot + 1; for (int j = 0; j < import_count; j++) { if (import_names[j] && import_qns[j] && strcmp(prefix, import_names[j]) == 0) { char fq[1024]; - int n = snprintf(fq, sizeof(fq), "%s.%s", - import_qns[j], suffix); + int n = snprintf(fq, sizeof(fq), "%s.%s", import_qns[j], suffix); if (n > 0 && n < (int)sizeof(fq)) { f = cbm_registry_lookup_func(reg, fq); } @@ -3411,7 +3744,8 @@ int cbm_go_fast_resolve_qualified_calls( f = cbm_registry_lookup_symbol(reg, result->module_qn, uc->callee_qn); } - if (!f) continue; + if (!f) + continue; /* Emit a resolved entry. cbm_pipeline_find_lsp_resolution * picks the highest-confidence match, so the unresolved entry @@ -3438,18 +3772,17 @@ int cbm_go_fast_resolve_qualified_calls( // --- Batch cross-file LSP --- -void cbm_batch_go_lsp_cross( - CBMArena* arena, - CBMBatchGoLSPFile* files, int file_count, - CBMResolvedCallArray* out) -{ - if (!files || file_count <= 0 || !out) return; +void cbm_batch_go_lsp_cross(CBMArena *arena, CBMBatchGoLSPFile *files, int file_count, + CBMResolvedCallArray *out) { + if (!files || file_count <= 0 || !out) + return; for (int f = 0; f < file_count; f++) { - CBMBatchGoLSPFile* file = &files[f]; + CBMBatchGoLSPFile *file = &files[f]; memset(&out[f], 0, sizeof(CBMResolvedCallArray)); - if (!file->source || file->source_len <= 0 || file->def_count <= 0) continue; + if (!file->source || file->source_len <= 0 || file->def_count <= 0) + continue; // Per-file arena: registry + temp data freed after each file CBMArena file_arena; @@ -3459,29 +3792,24 @@ void cbm_batch_go_lsp_cross( memset(&file_out, 0, sizeof(file_out)); // Delegate to existing per-file function - cbm_run_go_lsp_cross( - &file_arena, - file->source, file->source_len, - file->module_qn, - file->defs, file->def_count, - file->import_names, file->import_qns, file->import_count, - file->cached_tree, - &file_out); + cbm_run_go_lsp_cross(&file_arena, file->source, file->source_len, file->module_qn, + file->defs, file->def_count, file->import_names, file->import_qns, + file->import_count, file->cached_tree, &file_out); // Copy results to output arena (must outlive per-file arena) if (file_out.count > 0) { out[f].count = file_out.count; - out[f].items = (CBMResolvedCall*)cbm_arena_alloc(arena, - file_out.count * sizeof(CBMResolvedCall)); + out[f].items = + (CBMResolvedCall *)cbm_arena_alloc(arena, file_out.count * sizeof(CBMResolvedCall)); for (int j = 0; j < file_out.count; j++) { - CBMResolvedCall* src = &file_out.items[j]; - CBMResolvedCall* dst = &out[f].items[j]; + CBMResolvedCall *src = &file_out.items[j]; + CBMResolvedCall *dst = &out[f].items[j]; memset(dst, 0, sizeof(*dst)); dst->caller_qn = src->caller_qn ? cbm_arena_strdup(arena, src->caller_qn) : NULL; dst->callee_qn = src->callee_qn ? cbm_arena_strdup(arena, src->callee_qn) : NULL; - dst->strategy = src->strategy ? cbm_arena_strdup(arena, src->strategy) : NULL; + dst->strategy = src->strategy ? cbm_arena_strdup(arena, src->strategy) : NULL; dst->confidence = src->confidence; - dst->reason = src->reason ? cbm_arena_strdup(arena, src->reason) : NULL; + dst->reason = src->reason ? cbm_arena_strdup(arena, src->reason) : NULL; dst->kind = src->kind; dst->site_start_byte = src->site_start_byte; dst->site_end_byte = src->site_end_byte; diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 6dbc126e1..8a0396115 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -238,6 +238,7 @@ void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int sou ctx->source = source; ctx->source_len = source_len; ctx->registry = registry; + ctx->registry_head = (CBMTypeRegistry *)registry; ctx->module_qn = module_qn; ctx->resolved_calls = out; ctx->current_scope = py_scope_push_checked(ctx); @@ -394,7 +395,7 @@ static void py_import_syntax_match_add(PyImportSyntaxMatch *match, PyDirectImpor * registry then retains responsibility for proving that the exact target is * an undecorated, unambiguous callable. */ static const char *py_canonical_from_import_qn(PyLSPContext *ctx, TSNode statement, - TSNode imported_name, const char *incoming_qn) { + TSNode imported_name, const char *incoming_qn) { if (!ctx || !incoming_qn || ts_node_is_null(statement) || ts_node_is_null(imported_name)) return NULL; TSNode module = py_from_import_module_node(statement); @@ -407,10 +408,9 @@ static const char *py_canonical_from_import_qn(PyLSPContext *ctx, TSNode stateme while (module_suffix && *module_suffix == '.') module_suffix++; - const char *full_suffix = - module_suffix && module_suffix[0] - ? cbm_arena_sprintf(ctx->arena, "%s.%s", module_suffix, member) - : member; + const char *full_suffix = module_suffix && module_suffix[0] + ? cbm_arena_sprintf(ctx->arena, "%s.%s", module_suffix, member) + : member; if (!full_suffix) return NULL; if (py_qn_has_boundary_suffix(incoming_qn, full_suffix)) @@ -445,17 +445,16 @@ static void py_import_match_statement(PyLSPContext *ctx, TSNode stmt, const char TSNode alias = ts_node_child_by_field_name(item, "alias", 5); if (py_import_node_text_equals(ctx, alias, local)) { const char *canonical = py_canonical_from_import_qn(ctx, stmt, name, qn); - py_import_syntax_match_add( - match, canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED, - canonical ? canonical : qn); + py_import_syntax_match_add(match, + canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED, + canonical ? canonical : qn); } - } else if ((strcmp(kind, "identifier") == 0 || - strcmp(kind, "dotted_name") == 0) && + } else if ((strcmp(kind, "identifier") == 0 || strcmp(kind, "dotted_name") == 0) && py_import_node_text_equals(ctx, item, local)) { const char *canonical = py_canonical_from_import_qn(ctx, stmt, item, qn); - py_import_syntax_match_add( - match, canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED, - canonical ? canonical : qn); + py_import_syntax_match_add(match, + canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED, + canonical ? canonical : qn); } } return; @@ -489,8 +488,7 @@ static void py_import_match_statement(PyLSPContext *ctx, TSNode stmt, const char } } -static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match, - const char **qn_io) { +static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match, const char **qn_io) { if (!match || !qn_io) return PY_DIRECT_IMPORT_UNKNOWN; if (match->count == 0) @@ -503,8 +501,7 @@ static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match, } static PyDirectImportKind py_import_kind_from_statement(PyLSPContext *ctx, TSNode stmt, - const char *local, - const char **qn_io) { + const char *local, const char **qn_io) { const char *qn = qn_io ? *qn_io : NULL; if (!ctx || !local || !qn || !qn_io || ts_node_is_null(stmt)) return PY_DIRECT_IMPORT_UNKNOWN; @@ -513,8 +510,8 @@ static PyDirectImportKind py_import_kind_from_statement(PyLSPContext *ctx, TSNod return py_import_match_result(&match, qn_io); } -static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, TSNode root, - const char *local, const char **qn_io) { +static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, TSNode root, const char *local, + const char **qn_io) { const char *qn = qn_io ? *qn_io : NULL; if (!ctx || !local || !qn || !qn_io || ts_node_is_null(root)) return PY_DIRECT_IMPORT_UNKNOWN; @@ -589,36 +586,34 @@ static void py_bind_import_index(PyLSPContext *ctx, int index, bool synthetic_fa if (!local || !qn || strcmp(local, "*") == 0) return; - PyDirectImportKind direct_kind = ctx->import_kinds - ? (PyDirectImportKind)ctx->import_kinds[index] - : PY_DIRECT_IMPORT_UNKNOWN; + PyDirectImportKind direct_kind = + ctx->import_kinds ? (PyDirectImportKind)ctx->import_kinds[index] : PY_DIRECT_IMPORT_UNKNOWN; bool from_style = direct_kind == PY_FROM_IMPORT || - (direct_kind == PY_DIRECT_IMPORT_UNKNOWN && - import_is_from_style(local, qn)); + (direct_kind == PY_DIRECT_IMPORT_UNKNOWN && import_is_from_style(local, qn)); const CBMType *t; if (direct_kind == PY_DIRECT_IMPORT_UNALIASED) { - // Python binds only the root of an unaliased dotted import. + // Python binds only the root of an unaliased dotted import. t = cbm_type_module(ctx->arena, local); } else if (direct_kind == PY_DIRECT_IMPORT_ALIASED) { t = cbm_type_module(ctx->arena, qn); } else if (from_style) { - // `from X import Y` — bind Y to NAMED(X.Y). Phase 6 attribute - // resolution checks the registry to upgrade to MODULE / class - // / function as appropriate. + // `from X import Y` — bind Y to NAMED(X.Y). Phase 6 attribute + // resolution checks the registry to upgrade to MODULE / class + // / function as appropriate. t = cbm_type_named(ctx->arena, qn); } else if (strchr(qn, '.') != NULL) { - // Dotted path whose tail does NOT match the local name: an - // ALIASED binding — `from X import Y as Z` (Z names function/ - // class X.Y) or `import a.b as z` (z names module a.b). The - // CBMImport shape cannot distinguish the two, but NAMED(qn) - // covers both: phase 6 upgrades it to the registered function/ - // class for the from-import and to MODULE for the module alias. - // Binding MODULE here made `g()` calls on `from m import f as g` - // resolve as calls on a module — lsp=MISS, and the whole CALLS - // edge was lost (#988). + // Dotted path whose tail does NOT match the local name: an + // ALIASED binding — `from X import Y as Z` (Z names function/ + // class X.Y) or `import a.b as z` (z names module a.b). The + // CBMImport shape cannot distinguish the two, but NAMED(qn) + // covers both: phase 6 upgrades it to the registered function/ + // class for the from-import and to MODULE for the module alias. + // Binding MODULE here made `g()` calls on `from m import f as g` + // resolve as calls on a module — lsp=MISS, and the whole CALLS + // edge was lost (#988). t = cbm_type_named(ctx->arena, qn); } else { - // `import X` / `import X as Y` (single segment) — MODULE(X). + // `import X` / `import X as Y` (single segment) — MODULE(X). t = cbm_type_module(ctx->arena, qn); } const CBMRegisteredFunc *imported = @@ -841,8 +836,7 @@ static void py_emit_resolved_call(PyLSPContext *ctx, const char *callee_qn, cons } static void py_emit_resolved_reference(PyLSPContext *ctx, const char *callee_qn, - const char *source_name, TSNode site, - const char *strategy) { + const char *source_name, TSNode site, const char *strategy) { if (!ctx || !ctx->resolved_calls || !callee_qn || !ctx->enclosing_func_qn || ts_node_is_null(site)) { return; @@ -1027,8 +1021,7 @@ static void py_resolve_value_references_at(PyLSPContext *ctx, TSNode call) { if (candidate && cbm_scope_contains(ctx->current_scope, source_name)) { const CBMType *binding = cbm_type_resolve_alias(cbm_scope_lookup(ctx->current_scope, source_name)); - if (binding && binding->kind == CBM_TYPE_NAMED && - binding->data.named.qualified_name && + if (binding && binding->kind == CBM_TYPE_NAMED && binding->data.named.qualified_name && strcmp(binding->data.named.qualified_name, candidate) == 0) { py_emit_unresolved_reference(ctx, candidate, arg); } @@ -1195,16 +1188,10 @@ static void py_register_instance_field(PyLSPContext *ctx, const char *class_qn, return; } - // Find the type entry. cbm_registry_lookup_type returns a const pointer; - // we need a mutable pointer into the registry's array. - CBMRegisteredType *rt = NULL; - for (int i = 0; i < ctx->registry->type_count; i++) { - const char *qn = ctx->registry->types[i].qualified_name; - if (qn && strcmp(qn, class_qn) == 0) { - rt = &ctx->registry->types[i]; - break; - } - } + // The writable entry for the class: the head's own, or a copy of the + // base entry made in the head (copy-on-write). The shared base is never + // written -- it used to be, from every parallel resolve worker. + CBMRegisteredType *rt = cbm_registry_type_for_update(ctx->registry_head, class_qn); if (!rt) return; @@ -1570,11 +1557,12 @@ static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node) const char *prefix = cbm_arena_sprintf(ctx->arena, "%s.", qn); size_t prefix_len = strlen(prefix); bool is_submodule = false; - for (int i = 0; i < ctx->registry->func_count; i++) { - const char *fqn = ctx->registry->funcs[i].qualified_name; + CBMFreeFuncIter all_funcs; + cbm_registry_all_funcs_chain(ctx->registry, &all_funcs); + for (int i = -1; !is_submodule && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) { + const char *fqn = all_funcs.reg->funcs[i].qualified_name; if (fqn && strncmp(fqn, prefix, prefix_len) == 0) { is_submodule = true; - break; } } if (is_submodule) @@ -2600,8 +2588,7 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { const char *local = ctx->import_local_names[i]; const char *import_qn = ctx->import_module_qns[i]; if (local && import_qn && strcmp(local, fname) == 0 && - strcmp(import_qn, qn) == 0 && - py_import_index_is_from_binding(ctx, i)) { + strcmp(import_qn, qn) == 0 && py_import_index_is_from_binding(ctx, i)) { imported_alias = true; break; } @@ -2758,8 +2745,8 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { // Skip if mod is already rooted under the project to avoid // "..mod". if (!(strncmp(mod, ctx->module_qn, root_len) == 0 && mod[root_len] == '.')) { - char *qual_mod = (char *)cbm_arena_alloc(ctx->arena, root_len + 1 + - strlen(mod) + 1); + char *qual_mod = + (char *)cbm_arena_alloc(ctx->arena, root_len + 1 + strlen(mod) + 1); if (qual_mod) { memcpy(qual_mod, ctx->module_qn, root_len); qual_mod[root_len] = '.'; @@ -4047,10 +4034,8 @@ static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char // For methods, bind `self`/`cls` AFTER param walk so the receiver type // wins over the unannotated `self` / `cls` parameter declaration. if (ctx->enclosing_class_qn) { - py_scope_bind(ctx, "self", - cbm_type_named(ctx->arena, ctx->enclosing_class_qn)); - py_scope_bind(ctx, "cls", - cbm_type_named(ctx->arena, ctx->enclosing_class_qn)); + py_scope_bind(ctx, "self", cbm_type_named(ctx->arena, ctx->enclosing_class_qn)); + py_scope_bind(ctx, "cls", cbm_type_named(ctx->arena, ctx->enclosing_class_qn)); } TSNode body = ts_node_child_by_field_name(func_node, "body", 4); @@ -4197,12 +4182,14 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, TSNode root) { if (!ctx || !ctx->registry || !ctx->module_qn) return; size_t prefix_len = strlen(ctx->module_qn); - for (int i = 0; i < ctx->registry->type_count; i++) { - const CBMRegisteredType *type = &ctx->registry->types[i]; + CBMTypeShortIter all_types; + cbm_registry_all_types_chain(ctx->registry, &all_types); + for (int i = -1; (i = cbm_type_short_iter_next(&all_types)) >= 0;) { + const CBMRegisteredType *type = &all_types.reg->types[i]; const char *qn = type->qualified_name; const char *name = type->short_name; - if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || - qn[prefix_len] != '.' || py_root_defines_class_named(ctx, root, name)) { + if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || qn[prefix_len] != '.' || + py_root_defines_class_named(ctx, root, name)) { continue; } py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn)); @@ -4502,8 +4489,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int py_invalidate_possible_bindings(ctx, definition, depth + 1); return; } - if (strcmp(kind, "import_statement") == 0 || - strcmp(kind, "import_from_statement") == 0) { + if (strcmp(kind, "import_statement") == 0 || strcmp(kind, "import_from_statement") == 0) { py_invalidate_import_bindings(ctx, node); return; } @@ -4511,8 +4497,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int TSNode right = ts_node_child_by_field_name(node, "right", 5); if (ts_node_is_null(right)) return; - py_invalidate_binding_target(ctx, ts_node_child_by_field_name(node, "left", 4), - depth + 1); + py_invalidate_binding_target(ctx, ts_node_child_by_field_name(node, "left", 4), depth + 1); py_invalidate_possible_bindings(ctx, right, depth + 1); return; } @@ -4523,8 +4508,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int py_invalidate_possible_bindings(ctx, right, depth + 1); return; } - if (strcmp(kind, "named_expression") == 0 || - strcmp(kind, "assignment_expression") == 0) { + if (strcmp(kind, "named_expression") == 0 || strcmp(kind, "assignment_expression") == 0) { TSNode name = ts_node_child_by_field_name(node, "name", 4); if (ts_node_is_null(name)) name = ts_node_child_by_field_name(node, "left", 4); @@ -4619,8 +4603,7 @@ static void py_replay_import_local(PyLSPContext *ctx, TSNode stmt, const char *l continue; } const char *candidate_qn = ctx->import_module_qns[i]; - PyDirectImportKind kind = - py_import_kind_from_statement(ctx, stmt, local, &candidate_qn); + PyDirectImportKind kind = py_import_kind_from_statement(ctx, stmt, local, &candidate_qn); if (!py_replayable_import_kind(kind)) continue; if (!chosen_qn) { @@ -4642,8 +4625,7 @@ static void py_replay_import_local(PyLSPContext *ctx, TSNode stmt, const char *l py_bind_import_index(ctx, chosen, false); } -static void py_replay_import_statement(PyLSPContext *ctx, TSNode stmt, - unsigned char *consumed) { +static void py_replay_import_statement(PyLSPContext *ctx, TSNode stmt, unsigned char *consumed) { if (py_import_statement_is_wildcard(stmt)) { py_invalidate_module_bindings_for_wildcard(ctx); return; @@ -4686,8 +4668,7 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { py_bind_external_module_classes(ctx, root); unsigned char *consumed_imports = NULL; if (ctx->import_count > 0) { - consumed_imports = - (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count); + consumed_imports = (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count); if (consumed_imports) memset(consumed_imports, 0, (size_t)ctx->import_count); } @@ -4700,8 +4681,7 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { for (uint32_t i = 0; i < nc; i++) { TSNode c = ts_node_named_child(root, i); const char *ck = ts_node_type(c); - if (strcmp(ck, "import_statement") == 0 || - strcmp(ck, "import_from_statement") == 0) { + if (strcmp(ck, "import_statement") == 0 || strcmp(ck, "import_from_statement") == 0) { py_replay_import_statement(ctx, c, consumed_imports); } else if (strcmp(ck, "function_definition") == 0) { py_bind_module_function(ctx, c); diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index 96a85dc5d..939025100 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -44,6 +44,10 @@ typedef struct { const char *source; int source_len; const CBMTypeRegistry *registry; + /* Writable head of the registry chain (the per-file overlay, or the + * per-file registry on the non-cross path); refinements go here via + * cbm_registry_*_for_update, never into anything behind ->fallback. */ + CBMTypeRegistry *registry_head; CBMScope *current_scope; // Import map: local_name -> module_qn (arena-allocated, NULL-terminated). diff --git a/internal/cbm/lsp/ts_lsp.c b/internal/cbm/lsp/ts_lsp.c index fc8b8911f..0072aa999 100644 --- a/internal/cbm/lsp/ts_lsp.c +++ b/internal/cbm/lsp/ts_lsp.c @@ -2800,8 +2800,17 @@ static void resolve_value_references_at(TSLSPContext *ctx, TSNode call_node) { } } +static TSNode unwrap_await_callee(TSNode node) { + if (ts_node_is_null(node) || strcmp(ts_node_type(node), "await_expression") != 0 || + ts_node_named_child_count(node) == 0) { + return node; + } + return ts_node_named_child(node, 0); +} + static void resolve_call_at(TSLSPContext *ctx, TSNode call_node) { TSNode fn = ts_node_child_by_field_name(call_node, "function", TS_LSP_FIELD_LEN("function")); + fn = unwrap_await_callee(fn); if (ts_node_is_null(fn)) return; diff --git a/internal/cbm/lsp/type_registry.c b/internal/cbm/lsp/type_registry.c index 30c551899..4e2f02012 100644 --- a/internal/cbm/lsp/type_registry.c +++ b/internal/cbm/lsp/type_registry.c @@ -301,6 +301,9 @@ static void build_ffunc_short_index(CBMTypeRegistry *reg, CBMArena *idx_arena) { void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name, CBMTypeShortIter *out) { out->reg = reg; + out->chain = false; + out->key = short_name; + out->shadow = NULL; out->hash = fnv1a(short_name); if (reg->type_qn_buckets && reg->type_qn_bucket_count > 0) { if (reg->type_short_buckets && reg->type_short_bucket_count > 0) { @@ -319,7 +322,7 @@ void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *sh } } -int cbm_type_short_iter_next(CBMTypeShortIter *it) { +static int type_short_next_local(CBMTypeShortIter *it) { const CBMTypeRegistry *reg = it->reg; while (it->chain_idx >= 0) { const CBMRegistryHashEntry *e = ®->type_short_entries[it->chain_idx]; @@ -335,9 +338,79 @@ int cbm_type_short_iter_next(CBMTypeShortIter *it) { return -1; } +static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg, + const char *qualified_name); +static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg, + const char *qualified_name); + +/* A base entry is shadowed when the head holds an entry with the same QN. */ +static bool type_shadowed(const CBMTypeRegistry *shadow, const CBMTypeRegistry *reg, int i) { + if (!shadow || shadow == reg || !reg->types[i].qualified_name) { + return false; + } + return lookup_type_self(shadow, reg->types[i].qualified_name) != NULL; +} +static bool func_shadowed(const CBMTypeRegistry *shadow, const CBMTypeRegistry *reg, int i) { + if (!shadow || shadow == reg || !reg->funcs[i].qualified_name) { + return false; + } + return lookup_func_self(shadow, reg->funcs[i].qualified_name) != NULL; +} + +static void type_short_open_linear(const CBMTypeRegistry *reg, CBMTypeShortIter *out) { + out->reg = reg; + out->hash = 0; + out->chain_idx = -1; + out->tail_i = 0; + out->tail_end = reg->type_count; +} + +int cbm_type_short_iter_next(CBMTypeShortIter *it) { + for (;;) { + int p = type_short_next_local(it); + if (p >= 0) { + if (it->chain && type_shadowed(it->shadow, it->reg, p)) { + continue; + } + return p; + } + if (!it->chain || !it->reg->fallback) { + return -1; + } + const CBMTypeRegistry *next = it->reg->fallback; + const char *key = it->key; + const CBMTypeRegistry *shadow = it->shadow; + if (key) { + cbm_registry_types_by_short_name(next, key, it); + } else { + type_short_open_linear(next, it); + } + it->chain = true; + it->key = key; + it->shadow = shadow; + } +} + +void cbm_registry_types_by_short_name_chain(const CBMTypeRegistry *head, const char *short_name, + CBMTypeShortIter *out) { + cbm_registry_types_by_short_name(head, short_name, out); + out->chain = true; + out->shadow = head; +} + +void cbm_registry_all_types_chain(const CBMTypeRegistry *head, CBMTypeShortIter *out) { + type_short_open_linear(head, out); + out->chain = true; + out->key = NULL; + out->shadow = head; +} + void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char *bare, CBMTypeEmbedIter *out) { out->reg = reg; + out->chain = false; + out->key = bare; + out->shadow = NULL; out->hash = fnv1a(bare); out->prev_type = -1; if (reg->type_qn_buckets && reg->type_qn_bucket_count > 0) { @@ -358,7 +431,7 @@ void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char } } -int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) { +static int type_embed_next_local(CBMTypeEmbedIter *it) { const CBMTypeRegistry *reg = it->reg; while (it->chain_idx >= 0) { const CBMRegistryHashEntry *e = ®->type_embed_entries[it->chain_idx]; @@ -377,9 +450,40 @@ int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) { return -1; } +int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) { + for (;;) { + int p = type_embed_next_local(it); + if (p >= 0) { + if (it->chain && type_shadowed(it->shadow, it->reg, p)) { + continue; + } + return p; + } + if (!it->chain || !it->reg->fallback) { + return -1; + } + const CBMTypeRegistry *next = it->reg->fallback; + const char *key = it->key; + const CBMTypeRegistry *shadow = it->shadow; + cbm_registry_types_by_embedded_bare(next, key, it); + it->chain = true; + it->shadow = shadow; + } +} + +void cbm_registry_types_by_embedded_bare_chain(const CBMTypeRegistry *head, const char *bare, + CBMTypeEmbedIter *out) { + cbm_registry_types_by_embedded_bare(head, bare, out); + out->chain = true; + out->shadow = head; +} + void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const char *short_name, CBMFreeFuncIter *out) { out->reg = reg; + out->chain = false; + out->key = short_name; + out->shadow = NULL; out->hash = fnv1a(short_name); if (reg->func_qn_buckets && reg->func_qn_bucket_count > 0) { if (reg->ffunc_short_buckets && reg->ffunc_short_bucket_count > 0) { @@ -397,7 +501,7 @@ void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const cha } } -int cbm_free_func_iter_next(CBMFreeFuncIter *it) { +static int free_func_next_local(CBMFreeFuncIter *it) { const CBMTypeRegistry *reg = it->reg; while (it->chain_idx >= 0) { const CBMRegistryHashEntry *e = ®->ffunc_short_entries[it->chain_idx]; @@ -413,9 +517,57 @@ int cbm_free_func_iter_next(CBMFreeFuncIter *it) { return -1; } +static void free_func_open_linear(const CBMTypeRegistry *reg, CBMFreeFuncIter *out) { + out->reg = reg; + out->hash = 0; + out->chain_idx = -1; + out->tail_i = 0; + out->tail_end = reg->func_count; +} + +int cbm_free_func_iter_next(CBMFreeFuncIter *it) { + for (;;) { + int p = free_func_next_local(it); + if (p >= 0) { + if (it->chain && func_shadowed(it->shadow, it->reg, p)) { + continue; + } + return p; + } + if (!it->chain || !it->reg->fallback) { + return -1; + } + const CBMTypeRegistry *next = it->reg->fallback; + const char *key = it->key; + const CBMTypeRegistry *shadow = it->shadow; + if (key) { + cbm_registry_free_funcs_by_short_name(next, key, it); + } else { + free_func_open_linear(next, it); + } + it->chain = true; + it->key = key; + it->shadow = shadow; + } +} + +void cbm_registry_free_funcs_by_short_name_chain(const CBMTypeRegistry *head, + const char *short_name, CBMFreeFuncIter *out) { + cbm_registry_free_funcs_by_short_name(head, short_name, out); + out->chain = true; + out->shadow = head; +} + +void cbm_registry_all_funcs_chain(const CBMTypeRegistry *head, CBMFreeFuncIter *out) { + free_func_open_linear(head, out); + out->chain = true; + out->key = NULL; + out->shadow = head; +} + void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn, const char *method_name, CBMMethodIter *out) { - memset(out, 0, sizeof(*out)); + memset(out, 0, sizeof(*out)); /* chain=false, shadow=NULL */ out->reg = reg; out->receiver_qn = receiver_qn; out->method_name = method_name; @@ -435,10 +587,7 @@ void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn, } } -int cbm_method_iter_next(CBMMethodIter *it) { - if (!it || !it->reg || !it->receiver_qn || !it->method_name) { - return -1; - } +static int method_next_local(CBMMethodIter *it) { const CBMTypeRegistry *reg = it->reg; while (it->chain_idx >= 0) { const CBMRegistryHashEntry *e = ®->method_entries[it->chain_idx]; @@ -465,6 +614,80 @@ int cbm_method_iter_next(CBMMethodIter *it) { return -1; } +int cbm_method_iter_next(CBMMethodIter *it) { + if (!it || !it->reg || !it->receiver_qn || !it->method_name) { + return -1; + } + for (;;) { + int p = method_next_local(it); + if (p >= 0) { + if (it->chain && func_shadowed(it->shadow, it->reg, p)) { + continue; + } + return p; + } + if (!it->chain || !it->reg->fallback) { + return -1; + } + const CBMTypeRegistry *next = it->reg->fallback; + const CBMTypeRegistry *shadow = it->shadow; + cbm_registry_methods(next, it->receiver_qn, it->method_name, it); + it->chain = true; + it->shadow = shadow; + } +} + +void cbm_registry_methods_chain(const CBMTypeRegistry *head, const char *receiver_qn, + const char *method_name, CBMMethodIter *out) { + cbm_registry_methods(head, receiver_qn, method_name, out); + out->chain = true; + out->shadow = head; +} + +CBMRegisteredFunc *cbm_registry_func_for_update(CBMTypeRegistry *head, const char *qualified_name) { + if (!head || !qualified_name || head->read_only) { + return NULL; + } + const CBMRegisteredFunc *own = lookup_func_self(head, qualified_name); + if (own) { + return (CBMRegisteredFunc *)own; + } + for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) { + const CBMRegisteredFunc *base = lookup_func_self(r, qualified_name); + if (base) { + int before = head->func_count; + cbm_registry_add_func(head, *base); + if (head->func_count == before + 1) { + return &head->funcs[before]; + } + return NULL; + } + } + return NULL; +} + +CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const char *qualified_name) { + if (!head || !qualified_name || head->read_only) { + return NULL; + } + const CBMRegisteredType *own = lookup_type_self(head, qualified_name); + if (own) { + return (CBMRegisteredType *)own; + } + for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) { + const CBMRegisteredType *base = lookup_type_self(r, qualified_name); + if (base) { + int before = head->type_count; + cbm_registry_add_type(head, *base); + if (head->type_count == before + 1) { + return &head->types[before]; + } + return NULL; + } + } + return NULL; +} + void cbm_registry_finalize_into(CBMTypeRegistry *reg, CBMArena *idx_arena) { if (!reg || !idx_arena) return; diff --git a/internal/cbm/lsp/type_registry.h b/internal/cbm/lsp/type_registry.h index 7f7737448..c752350b1 100644 --- a/internal/cbm/lsp/type_registry.h +++ b/internal/cbm/lsp/type_registry.h @@ -242,6 +242,16 @@ typedef struct { int chain_idx; int tail_i; int tail_end; + /* Chain mode (see cbm_registry_*_chain): when this registry is exhausted + * the iterator re-opens on reg->fallback; `reg` then names the registry + * the yielded index belongs to, so callers deref it->reg->types[i], never + * the registry they started from. `key` re-opens the same query on the + * next link; NULL = linear scan. `shadow` is the head: fallback entries + * whose qualified_name the head also holds are skipped (an overlay copy + * hides its base original). */ + bool chain; + const char *key; + const CBMTypeRegistry *shadow; } CBMTypeShortIter; void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name, CBMTypeShortIter *out); @@ -263,6 +273,9 @@ typedef struct { int tail_i; // next tail/linear type index int tail_end; // reg->type_count snapshot int prev_type; // last yielded type index (adjacent-dedup); -1 = none + bool chain; + const char *key; + const CBMTypeRegistry *shadow; } CBMTypeEmbedIter; void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char *bare, CBMTypeEmbedIter *out); @@ -277,6 +290,9 @@ typedef struct { int chain_idx; int tail_i; int tail_end; + bool chain; + const char *key; + const CBMTypeRegistry *shadow; } CBMFreeFuncIter; void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const char *short_name, @@ -296,11 +312,41 @@ typedef struct { int chain_idx; int tail_i; int tail_end; + bool chain; + const CBMTypeRegistry *shadow; } CBMMethodIter; void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn, const char *method_name, CBMMethodIter *out); int cbm_method_iter_next(CBMMethodIter *it); +/* ── Chain-aware iteration and copy-on-write (per-file overlay contract) ── + * + * A resolve walk is handed a per-file OVERLAY registry chained (`fallback`) to + * the immutable shared base. Lookups already chain; these make the index + * iterators and the whole-registry scans chain too, and give a walk one way + * to refine an entry: copy it into the head first. Nothing behind `fallback` + * is ever written. Every yielded index belongs to it->reg at that moment. */ +void cbm_registry_types_by_short_name_chain(const CBMTypeRegistry *head, const char *short_name, + CBMTypeShortIter *out); +void cbm_registry_types_by_embedded_bare_chain(const CBMTypeRegistry *head, const char *bare, + CBMTypeEmbedIter *out); +void cbm_registry_free_funcs_by_short_name_chain(const CBMTypeRegistry *head, + const char *short_name, CBMFreeFuncIter *out); +void cbm_registry_methods_chain(const CBMTypeRegistry *head, const char *receiver_qn, + const char *method_name, CBMMethodIter *out); +/* Linear scans over every func / type in the chain (head first, shadowed + * base entries skipped). Same iterator types, same it->reg contract. */ +void cbm_registry_all_funcs_chain(const CBMTypeRegistry *head, CBMFreeFuncIter *out); +void cbm_registry_all_types_chain(const CBMTypeRegistry *head, CBMTypeShortIter *out); + +/* The writable entry for qualified_name: the head's own entry if it has one, + * else a copy of the first fallback entry added to the head (copy-on-write), + * else NULL. NULL also when the head is sealed (read_only) -- a walk must never + * mutate a shared registry, so the caller skips the refinement, exactly as the + * sealed-registry no-op in cbm_registry_add_func does today. */ +CBMRegisteredFunc *cbm_registry_func_for_update(CBMTypeRegistry *head, const char *qualified_name); +CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const char *qualified_name); + // --- TS-specific helpers (return NULL for types without these signatures) --- // If the type has a call signature (e.g., `interface F { (x:number): string }`), return diff --git a/internal/cbm/preprocessor.cpp b/internal/cbm/preprocessor.cpp index 54cb073fe..9be6298f3 100644 --- a/internal/cbm/preprocessor.cpp +++ b/internal/cbm/preprocessor.cpp @@ -42,6 +42,283 @@ static bool has_preprocessor_work(const char *source, int source_len) { return false; } +// ── Export-macro candidates (#1989) ───────────────────────────────────────── +// Build systems define symbol-export macros empty on the compiler command line +// (UE's UBT: `/D "MODULE_API="`; CMake generate_export_header: `_EXPORT`), +// so they never appear as #define lines in the source. tree-sitter has no +// preprocessor state either, and `class MODULE_API Foo` misparses with the +// macro token as the type name (worse: enums and free functions are lost to +// ERROR regions entirely). To mirror the real compile line, the preprocessed +// second pass predefines the conventional export-macro-shaped identifiers found +// in the file as empty. The shape is deliberately narrow — ALL_CAPS identifier +// ending in a known export suffix, with a non-trivial prefix — and the list is +// capped, so ordinary all-caps identifiers cannot be swept in wholesale. +static const char *kExportMacroSuffixes[] = {"_API", "_EXPORT", "_IMPORT", + "_DLLEXPORT", "_DEPRECATED", NULL}; + +static bool is_export_macro_shape(const char *id, size_t len) { + if (id[0] < 'A' || id[0] > 'Z') { + return false; + } + for (size_t i = 0; i < len; i++) { + char c = id[i]; + bool upper = c >= 'A' && c <= 'Z'; + bool digit = c >= '0' && c <= '9'; + if (!upper && !digit && c != '_') { + return false; + } + } + for (int s = 0; kExportMacroSuffixes[s]; s++) { + size_t slen = strlen(kExportMacroSuffixes[s]); + // Require at least two prefix characters before the suffix so a bare + // "X_API"-style token (single-letter, easily a genuine symbol) stays out. + if (len >= slen + 2 && strncmp(id + len - slen, kExportMacroSuffixes[s], slen) == 0) { + return true; + } + } + return false; +} + +static bool is_identifier_start(char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; +} + +static bool is_identifier_char(char c) { + return is_identifier_start(c) || (c >= '0' && c <= '9'); +} + +// C++ raw string literals (R"delim(...)delim"). Returns the index one past +// the closing quote, or -1 when the form cannot be confidently recognized +// (unterminated, malformed delimiter) — the caller then STOPS collecting so +// the mis-modeled literal cannot poison the scan state of the rest of the +// file (#1989 review). +static int skip_raw_string(const char *source, int source_len, int quote) { + int j = quote + 1; + int delim_start = j; + while (j < source_len) { + char c = source[j]; + if (c == '(') { + break; + } + // Delimiter chars: the standard d-char set excludes space, parens, + // backslash, and the line breaks — a double quote IS a legal d-char + // (R"""(...)""" compiles), so it must not end the delimiter (#1989 + // review round 3). + if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == ')' || c == '\\' || + j - delim_start >= 16) { + return -1; + } + j++; + } + if (j >= source_len || source[j] != '(') { + return -1; + } + int delim_len = j - delim_start; + j++; // past the opening "(" + while (j < source_len) { + if (source[j] == ')') { + int k = j + 1; + int m = 0; + while (m < delim_len && k + m < source_len && + source[k + m] == source[delim_start + m]) { + m++; + } + if (m == delim_len && k + m < source_len && source[k + m] == '"') { + return k + m + 1; + } + } + j++; + } + return -1; // unterminated raw string: uncertain +} + +// If the identifier just scanned is a raw-string prefix (R, LR, uR, UR, u8R) +// immediately followed by a quote, return skip_raw_string(...). Returns 0 for +// an ordinary identifier (including an identifier merely ENDING in R — only a +// standalone prefix token starts a raw string) and -1 on an unrecognizable +// raw form. +static int raw_string_after_prefix(const char *source, int source_len, int start, int end) { + static const char *prefixes[] = {"R", "LR", "uR", "UR", "u8R", NULL}; + size_t idlen = (size_t)(end - start); + bool is_prefix = false; + for (int p = 0; prefixes[p]; p++) { + if (idlen == strlen(prefixes[p]) && strncmp(source + start, prefixes[p], idlen) == 0) { + is_prefix = true; + break; + } + } + if (!is_prefix || end >= source_len || source[end] != '"') { + return 0; + } + return skip_raw_string(source, source_len, end); +} + +// Advance past comments and string/char literals so only real code tokens +// consume the candidate budget (#1989 review). Handles line-spliced // +// comments (a backslash before the newline continues the comment — line +// splicing happens before comment recognition) and, via the identifier-scan +// hook in the callers, C++ raw string literals. A digit separator (1'000'000) +// can still over-skip — that fails safe, toward NOT collecting a candidate. +static int skip_non_code(const char *source, int source_len, int i) { + while (i < source_len) { + char c = source[i]; + if (c == '/' && i + 1 < source_len && source[i + 1] == '/') { + i += 2; + for (;;) { + while (i < source_len && source[i] != '\n') { + i++; + } + if (i >= source_len) { + break; + } + // Line splice: a backslash right before the newline (allowing + // \r\n) means the comment continues on the next line. + int back = i - 1; + if (back >= 0 && source[back] == '\r') { + back--; + } + if (back >= 0 && source[back] == '\\') { + i++; // spliced: keep consuming inside the comment + continue; + } + i++; // real end-of-comment newline + break; + } + } else if (c == '/' && i + 1 < source_len && source[i + 1] == '*') { + i += 2; + while (i + 1 < source_len && !(source[i] == '*' && source[i + 1] == '/')) { + i++; + } + if (i + 1 < source_len) { + i += 2; // past the closing "*/" + } else { + i = source_len; // unterminated comment runs to EOF + } + } else if (c == '"' || c == '\'') { + char quote = c; + i++; + while (i < source_len) { + if (source[i] == '\\') { + i += 2; + continue; + } + if (source[i] == quote) { + i++; + break; + } + i++; + } + } else { + return i; + } + } + return i; +} + +// Count-only scan used by the early gate (no storage). +static bool has_export_macro_candidates(const char *source, int source_len) { + if (!source || source_len <= 0) { + return false; + } + int i = 0; + while (i < source_len) { + i = skip_non_code(source, source_len, i); + if (i >= source_len) { + break; + } + if (!is_identifier_start(source[i])) { + i++; + continue; + } + int start = i; + while (i < source_len && is_identifier_char(source[i])) { + i++; + } + // A raw-string prefix consumes everything up to its closing quote; + // an unrecognizable raw form poisons the scan state, so fail toward + // NOT running the second pass (raw behavior preserved). + int raw = raw_string_after_prefix(source, source_len, start, i); + if (raw < 0) { + return false; + } + if (raw > 0) { + i = raw; + continue; + } + if (is_export_macro_shape(source + start, (size_t)(i - start))) { + return true; + } + } + return false; +} + +static int collect_export_macro_candidates(const char *source, int source_len, + char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out) { + if (!source || source_len <= 0 || !out || max_out <= 0) { + return 0; + } + int stored = 0; + int i = 0; + while (i < source_len) { + i = skip_non_code(source, source_len, i); + if (i >= source_len) { + break; + } + if (!is_identifier_start(source[i])) { + i++; + continue; + } + int start = i; + while (i < source_len && is_identifier_char(source[i])) { + i++; + } + // Raw string (see has_export_macro_candidates): skip its body, or stop + // collecting entirely when the form cannot be confidently recognized — + // continuing would mis-tokenize the rest of the file and could both + // mint phantom candidates and mask real ones (#1989 review). + int raw = raw_string_after_prefix(source, source_len, start, i); + if (raw < 0) { + break; + } + if (raw > 0) { + i = raw; + continue; + } + size_t len = (size_t)(i - start); + /* Length gate FIRST: the dedup below reads out[k][0..len] against rows + * of CBM_EXPORT_MACRO_NAME_MAX bytes — an over-long candidate must be + * rejected before any compare touches them. */ + if (len >= CBM_EXPORT_MACRO_NAME_MAX) { + continue; + } + if (!is_export_macro_shape(source + start, len)) { + continue; + } + bool dup = false; + for (int k = 0; k < stored; k++) { + if (strncmp(out[k], source + start, len) == 0 && out[k][len] == '\0') { + dup = true; + break; + } + } + if (dup) { + continue; + } + if (stored >= max_out) { + break; // bounded: stop collecting once the cap is reached + } + memcpy(out[stored], source + start, len); + out[stored][len] = '\0'; + stored++; + } + return stored; +} + +int cbm_export_macro_candidates(const char *source, int source_len, + char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out) { + return collect_export_macro_candidates(source, source_len, out, max_out); +} + static int count_expanded_lines(const std::string &text) { int count = 1; for (char c : text) { @@ -146,7 +423,11 @@ static bool build_line_map(const std::string &expanded, const std::string &main_ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_len, const char *filename, const char **extra_defines, const char **include_paths, int cpp_mode) { - if (!has_preprocessor_work(source, source_len)) { + // Run the second pass when there are directives to evaluate OR when the file + // carries export-macro-shaped identifiers to predefine empty (#1989) — a UE + // plugin header with only `#pragma once` + `#include` lines still needs it. + if (!has_preprocessor_work(source, source_len) && + !has_export_macro_candidates(source, source_len)) { return NULL; // NULL = no expansion needed, use original } @@ -156,6 +437,28 @@ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_le for (int i = 0; extra_defines[i]; i++) dui.defines.push_back(extra_defines[i]); } + // Predefine collected export-macro candidates as empty, mirroring the + // real compile command line (`/D "MODULE_API="`). Names already provided + // by the caller win — never override an explicit define. + char export_cands[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int export_cand_count = + collect_export_macro_candidates(source, source_len, export_cands, CBM_EXPORT_MACRO_MAX); + for (int i = 0; i < export_cand_count; i++) { + bool provided = false; + for (std::list::const_iterator it = dui.defines.begin(); + it != dui.defines.end(); ++it) { + const std::string &def = *it; + size_t eq = def.find('='); + std::string name = (eq == std::string::npos) ? def : def.substr(0, eq); + if (name == export_cands[i]) { + provided = true; + break; + } + } + if (!provided) { + dui.defines.push_back(std::string(export_cands[i]) + "="); + } + } if (include_paths) { for (int i = 0; include_paths[i]; i++) dui.includePaths.push_back(include_paths[i]); diff --git a/internal/cbm/preprocessor.h b/internal/cbm/preprocessor.h index 72828c012..1b108b335 100644 --- a/internal/cbm/preprocessor.h +++ b/internal/cbm/preprocessor.h @@ -4,6 +4,15 @@ #include #include +/* #1989: build-system export macros (UE "_API", CMake generate_export_header + * "_EXPORT", ...) are defined empty on the real compile command line but are + * invisible to tree-sitter, so `class MOD_API Foo` misparses with the macro token + * as the type name. The preprocessed second pass predefines the conventional + * export-macro-shaped identifiers found in a source file as empty, mirroring the + * real build line. The candidate list is bounded. */ +#define CBM_EXPORT_MACRO_MAX 32 +#define CBM_EXPORT_MACRO_NAME_MAX 96 + #ifdef __cplusplus extern "C" { #endif @@ -15,6 +24,12 @@ typedef struct { int expanded_line_count; } CBMPreprocessedSource; +// Collect conventional export-macro-shaped identifiers (ALL_CAPS ending in +// _API/_EXPORT/_IMPORT/_DLLEXPORT/_DEPRECATED) from the source, up to max_out +// unique names. Returns the number of names stored in out. +int cbm_export_macro_candidates(const char *source, int source_len, + char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out); + // Preprocess C/C++ source: expand macros, evaluate #ifdef, resolve #include. // Returns malloc-allocated expanded source, or NULL if no expansion needed/on failure. // extra_defines: NULL-terminated array of "NAME=VALUE" strings (can be NULL). diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c new file mode 100644 index 000000000..d237b2a17 --- /dev/null +++ b/internal/cbm/result_compact.c @@ -0,0 +1,534 @@ +/* + * result_compact.c — copy a finished CBMFileResult into one exact-size arena. + * + * Why this exists (Go corpus census, 2026-09-13): extraction writes every + * temporary into the result arena — cbm_node_text copies at 496 call sites, + * per-node QN sprintf, enclosing-QN strings — and GROW_ARRAY leaves each + * previous generation of every record array dead behind it. 14.8 GB written, + * 3.4 GB reachable, and all of it retained until after resolve because the + * result owned the arena. Compaction walks what is reachable, measures it, + * copies it into a single block of exactly that size (strings interned by + * content within the file), and destroys the working arena. + * + * Three passes over one traversal: + * COUNT — number of string references, to size the intern table + * MEASURE — bytes every allocation will take (aligned like the arena does) + * COPY — the same allocations, for real, into the fresh arena + * MEASURE and COPY issue identical allocation sequences, so the block fits + * exactly; any failure leaves the result untouched. + */ + +#include "cbm.h" +#include "foundation/arena.h" +#include "foundation/compat.h" /* CBM_TLS */ +#include "foundation/constants.h" +#include "foundation/mem_core.h" +#include "result_spill.h" /* cbm_result_relocate */ + +#include +#include +#include + +enum { CR_ALIGN = 7, CR_MIN_TABLE = 64, CR_TABLE_LOAD = 2 }; + +typedef enum { CR_COUNT = 0, CR_MEASURE, CR_COPY, CR_RELOCATE } cr_phase_t; + +typedef struct { + const char *src; /* NULL = empty slot */ + char *dst; /* copy in the new arena; NULL until COPY reaches it */ + uint64_t hash; + size_t len; /* strlen(src) */ +} cr_slot_t; + +typedef struct { + cr_phase_t phase; + size_t refs; /* COUNT: string references seen */ + size_t bytes; /* MEASURE: total arena bytes */ + CBMArena *dst; /* COPY: the fresh arena */ + bool failed; + cr_slot_t *slots; + size_t cap; /* power of two */ + /* MEASURE records which slot each string reference landed in; COPY, which + * presents the very same references in the same order, replays them + * instead of measuring, hashing and comparing every string a second time + * (20 M repeated strlen + 31 M memcmp on the Go corpus, waste sanitizer + * 2026-09-17). */ + uint32_t *seq; + size_t seq_len; + size_t seq_pos; + /* RELOCATE: pointers in [old_base, old_base + old_len) move by delta. */ + const char *old_base; + size_t old_len; + ptrdiff_t delta; +} cr_ctx_t; + +static bool cr_in_old_block(const cr_ctx_t *c, const void *p) { + const char *cp = (const char *)p; + return cp >= c->old_base && cp < c->old_base + c->old_len; +} + +static size_t cr_aligned(size_t n) { + return (n + CR_ALIGN) & ~(size_t)CR_ALIGN; +} + +static uint64_t cr_hash(const char *s, size_t len) { + uint64_t h = 1469598103934665603ULL; + for (size_t i = 0; i < len; i++) { + h ^= (unsigned char)s[i]; + h *= 1099511628211ULL; + } + return h; +} + +/* Find or insert the slot for s. Never fails: the table is sized from COUNT + * at half load, and the same references are presented again in COPY. */ +static cr_slot_t *cr_slot(cr_ctx_t *c, const char *s) { + size_t len = strlen(s); + uint64_t h = cr_hash(s, len); + size_t mask = c->cap - SKIP_ONE; + size_t i = (size_t)h & mask; + for (;;) { + cr_slot_t *slot = &c->slots[i]; + if (!slot->src) { + slot->src = s; + slot->hash = h; + slot->len = len; + slot->dst = NULL; + return slot; + } + if (slot->hash == h && slot->len == len && + (slot->src == s || memcmp(slot->src, s, len) == 0)) { + return slot; + } + i = (i + SKIP_ONE) & mask; + } +} + +/* A raw allocation of `bytes` in the new arena: MEASURE books it, COPY makes + * it. Zero bytes is no allocation (the arena returns NULL for it too). */ +static void *cr_alloc(cr_ctx_t *c, size_t bytes) { + if (bytes == 0) { + return NULL; + } + if (c->phase == CR_MEASURE) { + c->bytes += cr_aligned(bytes); + return NULL; + } + if (c->phase == CR_COPY) { + void *p = cbm_arena_alloc(c->dst, bytes); + if (!p) { + c->failed = true; + } + return p; + } + return NULL; +} + +/* A string field: interned by content. COPY rewrites the field. */ +static void cr_str(cr_ctx_t *c, const char **field) { + const char *s = *field; + if (!s) { + return; + } + if (c->phase == CR_RELOCATE) { + if (cr_in_old_block(c, s)) { + *field = s + c->delta; + } + return; + } + if (c->phase == CR_COUNT) { + c->refs++; + return; + } + cr_slot_t *slot; + if (c->phase == CR_COPY && c->seq && c->seq_pos < c->seq_len) { + slot = &c->slots[c->seq[c->seq_pos++]]; + } else { + slot = cr_slot(c, s); + } + if (c->phase == CR_MEASURE) { + if (c->seq && c->seq_len < c->refs) { + c->seq[c->seq_len++] = (uint32_t)(slot - c->slots); + } + if (!slot->dst) { + slot->dst = (char *)s; /* mark as booked; reset before COPY */ + c->bytes += cr_aligned(slot->len + SKIP_ONE); + } + return; + } + if (!slot->dst) { + char *copy = (char *)cr_alloc(c, slot->len + SKIP_ONE); + if (!copy) { + return; + } + memcpy(copy, slot->src, slot->len + SKIP_ONE); + slot->dst = copy; + } + *field = slot->dst; +} + +/* A blob (fingerprint, retained source): copied verbatim. */ +static void cr_blob(cr_ctx_t *c, const void **field, size_t bytes) { + if (!*field || bytes == 0) { + return; + } + if (c->phase == CR_RELOCATE) { + if (cr_in_old_block(c, *field)) { + *field = (const char *)*field + c->delta; + } + return; + } + if (c->phase == CR_COPY) { + void *copy = cr_alloc(c, bytes); + if (!copy) { + return; + } + memcpy(copy, *field, bytes); + *field = copy; + } else { + (void)cr_alloc(c, bytes); + } +} + +/* A record array: copied at exact count, then the caller walks its fields. */ +static void cr_array(cr_ctx_t *c, void **items, int count, size_t elem) { + if (!*items || count <= 0) { + if (c->phase == CR_COPY) { + *items = NULL; + } + return; + } + if (c->phase == CR_RELOCATE) { + if (cr_in_old_block(c, *items)) { + *items = (char *)*items + c->delta; + } + return; + } + cr_blob(c, (const void **)items, (size_t)count * elem); +} + +/* A NULL-terminated list of strings: the pointer array plus each string. */ +static void cr_list(cr_ctx_t *c, const char ***field) { + const char **list = *field; + if (!list) { + return; + } + int n = 0; + if (c->phase == CR_RELOCATE) { + /* The array may live in a block we can no longer read at its old + * address: relocate the pointer first, then count and walk it. */ + cr_blob(c, (const void **)field, sizeof(char *)); + list = *field; + while (list[n]) { + n++; + } + for (int i = 0; i < n; i++) { + cr_str(c, &list[i]); + } + return; + } + while (list[n]) { + n++; + } + cr_blob(c, (const void **)field, (size_t)(n + SKIP_ONE) * sizeof(char *)); + const char **walk = *field; /* the copy in COPY, the original otherwise */ + for (int i = 0; i < n && walk; i++) { + cr_str(c, &walk[i]); + } +} + +/* A counted list of strings (signature_param_types). */ +static void cr_counted_list(cr_ctx_t *c, const char ***field, int count) { + if (!*field || count <= 0) { + return; + } + cr_blob(c, (const void **)field, (size_t)count * sizeof(char *)); + /* (RELOCATE: the pointer moved; the walk below now reads the new array.) */ + const char **walk = *field; + for (int i = 0; i < count && walk; i++) { + cr_str(c, &walk[i]); + } +} + +static void cr_walk_def(cr_ctx_t *c, CBMDefinition *d) { + cr_str(c, &d->name); + cr_str(c, &d->qualified_name); + cr_str(c, &d->label); + cr_str(c, &d->file_path); + cr_str(c, &d->signature); + cr_str(c, &d->return_type); + cr_str(c, &d->receiver); + cr_str(c, &d->docstring); + cr_str(c, &d->parent_class); + cr_list(c, &d->decorators); + cr_list(c, &d->base_classes); + cr_list(c, &d->param_names); + cr_list(c, &d->param_types); + cr_counted_list(c, &d->signature_param_types, d->signature_param_count); + cr_list(c, &d->return_types); + cr_str(c, &d->route_path); + cr_str(c, &d->route_method); + cr_blob(c, (const void **)&d->fingerprint, + d->fingerprint_k > 0 ? (size_t)d->fingerprint_k * sizeof(uint32_t) : 0); + cr_str(c, &d->structural_profile); + cr_str(c, &d->body_tokens); + cr_str(c, &d->impl_trait); +} + +static void cr_walk_call(cr_ctx_t *c, CBMCall *call) { + cr_str(c, &call->callee_name); + cr_str(c, &call->enclosing_func_qn); + cr_str(c, &call->first_string_arg); + cr_str(c, &call->second_arg_name); + int argc = call->arg_count; + if (argc > CBM_MAX_CALL_ARGS) { + argc = CBM_MAX_CALL_ARGS; + } + if (call->args && argc > 0) { + cr_array(c, (void **)&call->args, argc, sizeof(CBMCallArg)); + for (int i = 0; i < argc && call->args; i++) { + cr_str(c, &call->args[i].expr); + cr_str(c, &call->args[i].value); + cr_str(c, &call->args[i].keyword); + } + } else if (c->phase == CR_COPY) { + call->args = NULL; + call->arg_count = 0; + } +} + +static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { + cr_array(c, (void **)&r->defs.items, r->defs.count, sizeof(CBMDefinition)); + for (int i = 0; i < r->defs.count && r->defs.items; i++) { + cr_walk_def(c, &r->defs.items[i]); + } + cr_array(c, (void **)&r->calls.items, r->calls.count, sizeof(CBMCall)); + for (int i = 0; i < r->calls.count && r->calls.items; i++) { + cr_walk_call(c, &r->calls.items[i]); + } + cr_array(c, (void **)&r->imports.items, r->imports.count, sizeof(CBMImport)); + for (int i = 0; i < r->imports.count && r->imports.items; i++) { + cr_str(c, &r->imports.items[i].local_name); + cr_str(c, &r->imports.items[i].module_path); + } + cr_array(c, (void **)&r->usages.items, r->usages.count, sizeof(CBMUsage)); + for (int i = 0; i < r->usages.count && r->usages.items; i++) { + cr_str(c, &r->usages.items[i].ref_name); + cr_str(c, &r->usages.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->throws.items, r->throws.count, sizeof(CBMThrow)); + for (int i = 0; i < r->throws.count && r->throws.items; i++) { + cr_str(c, &r->throws.items[i].exception_name); + cr_str(c, &r->throws.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->rw.items, r->rw.count, sizeof(CBMReadWrite)); + for (int i = 0; i < r->rw.count && r->rw.items; i++) { + cr_str(c, &r->rw.items[i].var_name); + cr_str(c, &r->rw.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->type_refs.items, r->type_refs.count, sizeof(CBMTypeRef)); + for (int i = 0; i < r->type_refs.count && r->type_refs.items; i++) { + cr_str(c, &r->type_refs.items[i].type_name); + cr_str(c, &r->type_refs.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->env_accesses.items, r->env_accesses.count, sizeof(CBMEnvAccess)); + for (int i = 0; i < r->env_accesses.count && r->env_accesses.items; i++) { + cr_str(c, &r->env_accesses.items[i].env_key); + cr_str(c, &r->env_accesses.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->type_assigns.items, r->type_assigns.count, sizeof(CBMTypeAssign)); + for (int i = 0; i < r->type_assigns.count && r->type_assigns.items; i++) { + cr_str(c, &r->type_assigns.items[i].var_name); + cr_str(c, &r->type_assigns.items[i].type_name); + cr_str(c, &r->type_assigns.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->impl_traits.items, r->impl_traits.count, sizeof(CBMImplTrait)); + for (int i = 0; i < r->impl_traits.count && r->impl_traits.items; i++) { + cr_str(c, &r->impl_traits.items[i].trait_name); + cr_str(c, &r->impl_traits.items[i].struct_name); + cr_str(c, &r->impl_traits.items[i].struct_qn); + } + cr_array(c, (void **)&r->resolved_calls.items, r->resolved_calls.count, + sizeof(CBMResolvedCall)); + for (int i = 0; i < r->resolved_calls.count && r->resolved_calls.items; i++) { + cr_str(c, &r->resolved_calls.items[i].caller_qn); + cr_str(c, &r->resolved_calls.items[i].callee_qn); + cr_str(c, &r->resolved_calls.items[i].strategy); + cr_str(c, &r->resolved_calls.items[i].reason); + } + cr_array(c, (void **)&r->string_refs.items, r->string_refs.count, sizeof(CBMStringRef)); + for (int i = 0; i < r->string_refs.count && r->string_refs.items; i++) { + cr_str(c, &r->string_refs.items[i].value); + cr_str(c, &r->string_refs.items[i].enclosing_func_qn); + cr_str(c, &r->string_refs.items[i].key_path); + } + cr_array(c, (void **)&r->infra_bindings.items, r->infra_bindings.count, + sizeof(CBMInfraBinding)); + for (int i = 0; i < r->infra_bindings.count && r->infra_bindings.items; i++) { + cr_str(c, &r->infra_bindings.items[i].source_name); + cr_str(c, &r->infra_bindings.items[i].target_url); + cr_str(c, &r->infra_bindings.items[i].broker); + } + cr_array(c, (void **)&r->channels.items, r->channels.count, sizeof(CBMChannel)); + for (int i = 0; i < r->channels.count && r->channels.items; i++) { + cr_str(c, &r->channels.items[i].channel_name); + cr_str(c, &r->channels.items[i].transport); + cr_str(c, &r->channels.items[i].enclosing_func_qn); + } + cr_str(c, &r->module_qn); + cr_str(c, &r->namespace_name); + cr_list(c, &r->exports); + cr_list(c, &r->constants); + cr_list(c, &r->global_vars); + cr_list(c, &r->macros); + cr_str(c, &r->error_msg); + cr_str(c, &r->error_ranges); + cr_blob(c, (const void **)&r->source, r->source ? (size_t)r->source_len + SKIP_ONE : 0); +} + +void cbm_result_relocate(CBMFileResult *result, const char *old_base, size_t len, char *new_base) { + if (!result || !old_base || !new_base || len == 0 || old_base == new_base) { + return; + } + cr_ctx_t c; + memset(&c, 0, sizeof(c)); + c.phase = CR_RELOCATE; + c.old_base = old_base; + c.old_len = len; + c.delta = new_base - old_base; + cr_walk(&c, result); +} + +static size_t cr_pow2_at_least(size_t n) { + size_t cap = CR_MIN_TABLE; + while (cap < n) { + cap *= PAIR_LEN; + } + return cap; +} + +/* The intern table and the replay sequence, kept per pipeline worker: a fresh + * zeroed table per file was 21,875 allocations and 2.0 GB of pure churn on the + * Go corpus (waste sanitizer, 2026-09-17). Kept only on threads whose + * cbm_work_arena_release is guaranteed to run (cbm_work_arena_keeping), and + * only up to CR_KEEP_BYTES; anything else allocates per call as before. */ +enum { CR_KEEP_BYTES = 8 * 1024 * 1024 }; +static CBM_TLS void *tl_cr_buf; +static CBM_TLS size_t tl_cr_bytes; + +void cbm_result_compact_release_thread(void) { + cbm_free(CBM_MEM_CLASS_EXTRACT, tl_cr_buf); + tl_cr_buf = NULL; + tl_cr_bytes = 0; +} + +/* One zeroed block holding the slot table followed by the sequence. Returns + * whether the block is the kept one (then the caller must not free it). */ +static bool cr_scratch_get(cr_ctx_t *c) { + size_t slot_bytes = c->cap * sizeof(cr_slot_t); + size_t bytes = slot_bytes + (c->refs * sizeof(uint32_t)); + bool keep = bytes <= (size_t)CR_KEEP_BYTES && cbm_work_arena_keeping(); + void *buf = NULL; + if (keep && tl_cr_buf && tl_cr_bytes >= bytes) { + buf = tl_cr_buf; + } else if (keep) { + size_t grown = CR_MIN_TABLE; /* powers of two: a growing file mix reallocates rarely */ + while (grown < bytes) { + grown *= PAIR_LEN; + } + if (grown > (size_t)CR_KEEP_BYTES) { + grown = bytes; + } + cbm_result_compact_release_thread(); + tl_cr_buf = cbm_alloc(CBM_MEM_CLASS_EXTRACT, grown); + tl_cr_bytes = tl_cr_buf ? grown : 0; + buf = tl_cr_buf; + } else { + buf = cbm_alloc(CBM_MEM_CLASS_EXTRACT, bytes); + } + if (!buf) { + return false; + } + memset( + buf, 0, + slot_bytes); /* MEASURE needs an empty table; the sequence is written before it is read */ + c->slots = (cr_slot_t *)buf; + c->seq = c->refs ? (uint32_t *)((char *)buf + slot_bytes) : NULL; + return true; +} + +static void cr_scratch_put(cr_ctx_t *c) { + if ((void *)c->slots != tl_cr_buf) { + cbm_free(CBM_MEM_CLASS_EXTRACT, c->slots); + } + c->slots = NULL; + c->seq = NULL; +} + +void cbm_result_compact(CBMFileResult *result) { + if (!result || result->arena.nblocks == 0) { + return; + } + cr_ctx_t c; + memset(&c, 0, sizeof(c)); + + /* Work on a copy of the header: every pointer rewrite lands here and the + * caller's result is replaced only once everything succeeded. */ + CBMFileResult tmp = *result; + + c.phase = CR_COUNT; + cr_walk(&c, &tmp); + + c.cap = cr_pow2_at_least(c.refs * CR_TABLE_LOAD + CR_MIN_TABLE); + if (!cr_scratch_get(&c)) { + return; + } + + c.phase = CR_MEASURE; + cr_walk(&c, &tmp); + for (size_t i = 0; i < c.cap; i++) { + c.slots[i].dst = NULL; /* MEASURE used dst as a booked marker */ + } + + CBMArena fresh; + cbm_arena_init_exact(&fresh, c.bytes); + if (fresh.nblocks == 0) { + cr_scratch_put(&c); + return; + } + + c.phase = CR_COPY; + c.dst = &fresh; + cr_walk(&c, &tmp); + cr_scratch_put(&c); + if (c.failed) { + cbm_arena_destroy(&fresh); + return; + } + + /* Exact-count arrays: nothing may append into the dead headroom. */ + tmp.defs.cap = tmp.defs.count; + tmp.calls.cap = tmp.calls.count; + tmp.imports.cap = tmp.imports.count; + tmp.usages.cap = tmp.usages.count; + tmp.throws.cap = tmp.throws.count; + tmp.rw.cap = tmp.rw.count; + tmp.type_refs.cap = tmp.type_refs.count; + tmp.env_accesses.cap = tmp.env_accesses.count; + tmp.type_assigns.cap = tmp.type_assigns.count; + tmp.impl_traits.cap = tmp.impl_traits.count; + tmp.resolved_calls.cap = tmp.resolved_calls.count; + tmp.string_refs.cap = tmp.string_refs.count; + tmp.infra_bindings.cap = tmp.infra_bindings.count; + tmp.channels.cap = tmp.channels.count; + + /* A composite kept its per-unit results only so shallow-copied strings + * stayed valid; every string is now a copy of its own. */ + cbm_result_release_owned(result); + tmp.owned_results = NULL; + tmp.owned_result_count = 0; + + cbm_work_arena_give(&result->arena); /* kept for this thread's next file */ + tmp.arena = fresh; + *result = tmp; +} diff --git a/internal/cbm/result_spill.c b/internal/cbm/result_spill.c new file mode 100644 index 000000000..852421632 --- /dev/null +++ b/internal/cbm/result_spill.c @@ -0,0 +1,410 @@ +/* + * result_spill.c — see result_spill.h. + * + * Layout on disk, per parked result: + * spill_rec_hdr_t (magic, block length, block base address at park time, + * the CBMFileResult header as it was in memory) + * block bytes (the single compacted arena block) + * + * The header's pointers are meaningless on disk; the loader rebuilds the + * arena at a new address and shifts every pointer by the delta + * (cbm_result_relocate, implemented on the compaction traversal so it sees + * exactly the fields compaction copied). + */ + +#include "result_spill.h" + +#include "foundation/arena.h" +#include "foundation/compat_fs.h" +#include "foundation/compat_thread.h" +#include "foundation/constants.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "foundation/platform.h" /* cbm_fs_free_bytes: the disk ceiling below */ + +/* The park writes a result as an opaque image: the record header (a struct + * copy) and the compacted arena block. Both carry padding bytes no code ever + * wrote -- inside structs, between objects -- and MemorySanitizer tracks + * that mark through the compaction's memcpy, so it refuses the fwrite of an + * image that is read back whole and never interpreted byte by byte (CI MSan + * lane on #2202: offset 4087, then 6714, of a 6,952-byte block). Under MSan + * the image is declared defined right before the write; every other build + * compiles this to nothing. */ +#include "foundation/sanitized.h" /* __has_feature exists everywhere, cppcheck included */ +#if __has_feature(memory_sanitizer) +#include +#define SPILL_IMAGE_DEFINED(p, n) __msan_unpoison((p), (n)) +#else +#define SPILL_IMAGE_DEFINED(p, n) ((void)0) +#endif + +#include +#include +#include +#include +#ifdef _WIN32 +#include +#define SPILL_PID() ((long)_getpid()) +#define SPILL_SEEK(fp, off) _fseeki64((fp), (long long)(off), SEEK_SET) +#else +#include +#define SPILL_PID() ((long)getpid()) +#define SPILL_SEEK(fp, off) fseeko((fp), (off_t)(off), SEEK_SET) +#endif + +static const uint64_t SPILL_MAGIC = 0x5350494C4C524553ULL; /* "SPILLRES" */ + +typedef struct { + uint64_t magic; + uint64_t block_len; + uint64_t old_base; /* (uintptr_t) of the block when parked */ + CBMFileResult header; +} spill_rec_hdr_t; + +typedef struct { + FILE *fp; + cbm_mutex_t mu; /* reads seek; writes append -- one lock per file */ + char path[CBM_SZ_1K]; + uint64_t end; /* bytes written so far (append offset) */ + bool unlinked; /* already removed while open (POSIX); close must not retry */ +} spill_file_t; + +typedef struct { + int writer; /* -1 = empty */ + uint64_t offset; /* record start in that writer's file */ + uint64_t rec_len; /* header + block */ + /* The file's namespace/package, kept in MEMORY while the result itself is + * on disk. The namespace map that import resolution is built on is + * assembled from every file at once, before any of them is reloaded: a + * parked file used to contribute nothing to it, so spilling silently + * changed how imports resolved (see cbm_result_spill_namespace). One short + * string per parked file is a rounding error against the result it + * replaces. */ + char *namespace_name; +} spill_slot_t; + +struct cbm_result_spill { + spill_file_t *files; + int writers; + spill_slot_t *slots; + int slot_count; + _Atomic int64_t parked; + _Atomic int64_t bytes; + _Atomic int64_t loads; + int64_t byte_cap; /* 0 = uncapped (the platform could not say) */ + _Atomic int cap_reported; /* reaching the cap is logged once, not per park */ +}; + +/* Spilling trades a memory problem for a disk problem, so the disk gets a say. + * + * FLOOR: below this much free space, do not start spilling at all — filling the + * user's disk is a worse failure than the memory pressure being relieved, since + * it breaks everything else on the machine and not just this index. + * SHARE: of what IS free, never take more than this fraction, so a runaway + * index cannot consume the rest either. + * Free space is read once, at open: it is this run's budget, not a moving + * target to race against, and a failed write stays handled underneath. */ +enum { SPILL_FREE_FLOOR_GB = 10, SPILL_FREE_SHARE_DIV = 2 }; + +/* Remove spill files left by runs that are no longer here. Only files this + * module names (spill--.bin) are touched, and on Windows an open + * file refuses deletion, which is exactly the liveness check we want: a running + * index keeps its own spill. POSIX runs unlink at open, so this only ever finds + * files from before that behaviour, or from another platform's cache. */ +static void spill_sweep_orphans(const char *spill_dir) { + cbm_dir_t *dir = cbm_opendir(spill_dir); + if (!dir) { + return; + } + int removed = 0; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + if (strncmp(entry->name, "spill-", 6) != 0) { + continue; + } + size_t len = strlen(entry->name); + if (len < 5 || strcmp(entry->name + len - 4, ".bin") != 0) { + continue; + } + char path[CBM_SZ_1K]; + if (snprintf(path, sizeof(path), "%s/%s", spill_dir, entry->name) >= (int)sizeof(path)) { + continue; + } + if (cbm_unlink(path) == 0) { + removed++; + } + } + cbm_closedir(dir); + if (removed > 0) { + char removed_text[CBM_SZ_16]; + snprintf(removed_text, sizeof(removed_text), "%d", removed); + cbm_log_info("mem.spill.orphans_removed", "count", removed_text, "dir", spill_dir); + } +} + +cbm_result_spill_t *cbm_result_spill_open(const char *dir, int writers, int slots) { + if (!dir || !dir[0] || writers <= 0 || slots <= 0) { + return NULL; + } + char spill_dir[CBM_SZ_1K]; + if (snprintf(spill_dir, sizeof(spill_dir), "%s/spill", dir) >= (int)sizeof(spill_dir)) { + return NULL; + } + if (!cbm_mkdir_p(spill_dir, 0700)) { + cbm_log_warn("mem.spill.open_failed", "dir", spill_dir, "reason", "mkdir"); + return NULL; + } + /* Reclaim what earlier runs could not. A run killed outright (OOM killer, + * SIGKILL, power loss) runs no cleanup, and on Windows its spill files + * survive; POSIX runs unlink-at-open below and leaves nothing. Deleting is + * the liveness test: an open file cannot be removed on Windows, so a + * concurrent run's spill refuses and is left alone. */ + spill_sweep_orphans(spill_dir); + /* Ask the disk BEFORE trading memory for it. Refusing here leaves the + * caller in the memory path it was already in (back-pressure, then its own + * budget decision) — the same outcome as a disk that fills mid-run, minus + * the filled disk. */ + size_t free_bytes = cbm_fs_free_bytes(spill_dir); + int64_t byte_cap = 0; + if (free_bytes > 0) { + size_t floor_bytes = (size_t)SPILL_FREE_FLOOR_GB * 1024 * 1024 * 1024; + if (free_bytes < floor_bytes) { + char free_mb[CBM_SZ_32]; + snprintf(free_mb, sizeof(free_mb), "%zu", free_bytes / (1024 * 1024)); + cbm_log_warn("mem.spill.refused", "reason", "low_disk", "free_mb", free_mb, "detail", + "not spilling onto a nearly full disk; memory relief continues through " + "back-pressure instead"); + return NULL; + } + byte_cap = (int64_t)(free_bytes / SPILL_FREE_SHARE_DIV); + } + cbm_result_spill_t *sp = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*sp)); + if (!sp) { + return NULL; + } + sp->files = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)writers * sizeof(spill_file_t)); + sp->slots = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)slots * sizeof(spill_slot_t)); + if (!sp->files || !sp->slots) { + cbm_result_spill_close(sp); + return NULL; + } + sp->writers = writers; + sp->slot_count = slots; + sp->byte_cap = byte_cap; + for (int i = 0; i < slots; i++) { + sp->slots[i].writer = -1; + } + for (int w = 0; w < writers; w++) { + spill_file_t *f = &sp->files[w]; + snprintf(f->path, sizeof(f->path), "%s/spill-%ld-%d.bin", spill_dir, SPILL_PID(), w); + f->fp = cbm_fopen(f->path, "w+b"); + if (!f->fp) { + cbm_log_warn("mem.spill.open_failed", "path", f->path, "reason", "fopen"); + cbm_result_spill_close(sp); + return NULL; + } + cbm_mutex_init(&f->mu); +#ifndef _WIN32 + /* Unlink while we hold it open: the data stays reachable through this + * handle, and the space comes back the moment the process ends — a + * clean exit, an abort, or a SIGKILL from the OOM killer alike. A + * killed run cannot leave gigabytes of spill behind because there is no + * name left to leave it under. Windows refuses to unlink an open file, + * so it keeps the close-path unlink and the sweep below. */ + if (cbm_unlink(f->path) == 0) { + f->unlinked = true; + } +#endif + } + cbm_log_info("mem.spill.open", "dir", spill_dir, "writers", writers > 0 ? "yes" : "no"); + return sp; +} + +bool cbm_result_spill_has(const cbm_result_spill_t *sp, int slot) { + return sp && slot >= 0 && slot < sp->slot_count && sp->slots[slot].writer >= 0; +} + +bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFileResult *result) { + if (!sp || !result || writer < 0 || writer >= sp->writers || slot < 0 || + slot >= sp->slot_count || sp->slots[slot].writer >= 0) { + return false; + } + /* Only a compacted result is one block with every pointer inside it; a + * result that owns sub-results (embedded languages) has more arenas than + * that and stays in memory. A retained parse tree is a re-parse cache, + * not data: it is dropped with the in-memory result below. */ + if (result->arena.nblocks != 1 || result->owned_result_count != 0) { + return false; + } + /* The disk ceiling. Refusing a park leaves the result in memory, which is + * exactly where it would be without spill at all — the caller's + * back-pressure and budget keep deciding from there. What this prevents is + * an index quietly consuming the rest of the user's free space. */ + if (sp->byte_cap > 0) { + int64_t want = (int64_t)(sizeof(spill_rec_hdr_t) + result->arena.used); + if (atomic_load_explicit(&sp->bytes, memory_order_relaxed) + want > sp->byte_cap) { + if (atomic_exchange_explicit(&sp->cap_reported, 1, memory_order_relaxed) == 0) { + char cap_mb[CBM_SZ_32]; + snprintf(cap_mb, sizeof(cap_mb), "%lld", (long long)(sp->byte_cap / (1024 * 1024))); + cbm_log_warn("mem.spill.cap_reached", "cap_mb", cap_mb, "detail", + "spill is at its share of free disk; results stay in memory and " + "back-pressure takes over"); + } + return false; + } + } + spill_file_t *f = &sp->files[writer]; + spill_rec_hdr_t hdr; + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = SPILL_MAGIC; + hdr.block_len = result->arena.used; /* bytes actually written into the block */ + hdr.old_base = (uint64_t)(uintptr_t)result->arena.blocks[0]; + hdr.header = *result; + hdr.header.cached_tree = NULL; /* never on disk: the loader gets no tree */ + SPILL_IMAGE_DEFINED(&hdr, sizeof(hdr)); + SPILL_IMAGE_DEFINED(result->arena.blocks[0], hdr.block_len); + cbm_mutex_lock(&f->mu); + uint64_t offset = f->end; + bool ok = SPILL_SEEK(f->fp, offset) == 0 && fwrite(&hdr, sizeof(hdr), 1, f->fp) == 1 && + (hdr.block_len == 0 || fwrite(result->arena.blocks[0], hdr.block_len, 1, f->fp) == 1); + if (ok) { + f->end = offset + sizeof(hdr) + hdr.block_len; + } + cbm_mutex_unlock(&f->mu); + if (!ok) { + cbm_log_warn("mem.spill.write_failed", "path", f->path); + return false; + } + sp->slots[slot].writer = writer; + sp->slots[slot].offset = offset; + sp->slots[slot].rec_len = sizeof(hdr) + hdr.block_len; + /* Copied BEFORE the result is freed below: the namespace map is built from + * all files at once, long before this slot is loaded back. */ + if (result->namespace_name && result->namespace_name[0]) { + sp->slots[slot].namespace_name = + cbm_mem_strdup(CBM_MEM_CLASS_OTHER, result->namespace_name); + } + atomic_fetch_add_explicit(&sp->parked, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&sp->bytes, (int64_t)(sizeof(hdr) + hdr.block_len), + memory_order_relaxed); + cbm_free_result(result); + return true; +} + +CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot) { + if (!cbm_result_spill_has(sp, slot)) { + return NULL; + } + const spill_slot_t *s = &sp->slots[slot]; + spill_file_t *f = &sp->files[s->writer]; + spill_rec_hdr_t hdr; + cbm_mutex_lock(&f->mu); + bool ok = SPILL_SEEK(f->fp, s->offset) == 0 && fread(&hdr, sizeof(hdr), 1, f->fp) == 1 && + hdr.magic == SPILL_MAGIC; + CBMFileResult *r = NULL; + if (ok) { + r = cbm_result_alloc(); + if (r) { + *r = hdr.header; + r->cached_tree = NULL; + memset(&r->arena, 0, sizeof(r->arena)); + cbm_arena_init_exact(&r->arena, hdr.block_len ? (size_t)hdr.block_len : 1); + if (r->arena.nblocks == 1 && + (hdr.block_len == 0 || + fread(r->arena.blocks[0], (size_t)hdr.block_len, 1, f->fp) == 1)) { + r->arena.used = (size_t)hdr.block_len; + r->arena.total_alloc = (size_t)hdr.block_len; + } else { + ok = false; + } + } else { + ok = false; + } + } + cbm_mutex_unlock(&f->mu); + if (!ok) { + cbm_log_warn("mem.spill.read_failed", "path", f->path); + if (r) { + cbm_free_result(r); + } + return NULL; + } + cbm_result_relocate(r, (const char *)(uintptr_t)hdr.old_base, (size_t)hdr.block_len, + r->arena.blocks[0]); + atomic_fetch_add_explicit((_Atomic int64_t *)&sp->loads, 1, memory_order_relaxed); + return r; +} + +bool cbm_result_spill_peek_header(const cbm_result_spill_t *sp, int slot, CBMFileResult *out) { + if (!out || !cbm_result_spill_has(sp, slot)) { + return false; + } + const spill_slot_t *s = &sp->slots[slot]; + spill_file_t *f = &sp->files[s->writer]; + spill_rec_hdr_t hdr; + cbm_mutex_lock(&f->mu); + bool ok = SPILL_SEEK(f->fp, s->offset) == 0 && fread(&hdr, sizeof(hdr), 1, f->fp) == 1 && + hdr.magic == SPILL_MAGIC; + cbm_mutex_unlock(&f->mu); + if (ok) { + *out = hdr.header; + } + return ok; +} + +void cbm_result_spill_peek_counts(const cbm_result_spill_t *sp, int slot, int *defs, int *impls) { + CBMFileResult hdr; + bool ok = cbm_result_spill_peek_header(sp, slot, &hdr); + if (defs) { + *defs = ok ? hdr.defs.count : 0; + } + if (impls) { + *impls = ok ? hdr.impl_traits.count : 0; + } +} + +void cbm_result_spill_stats(const cbm_result_spill_t *sp, int64_t *parked, int64_t *bytes, + int64_t *loads) { + if (parked) { + *parked = sp ? atomic_load_explicit(&sp->parked, memory_order_relaxed) : 0; + } + if (bytes) { + *bytes = sp ? atomic_load_explicit(&sp->bytes, memory_order_relaxed) : 0; + } + if (loads) { + *loads = sp ? atomic_load_explicit(&sp->loads, memory_order_relaxed) : 0; + } +} + +const char *cbm_result_spill_namespace(const cbm_result_spill_t *sp, int slot) { + if (!sp || !sp->slots || slot < 0 || slot >= sp->slot_count) { + return NULL; + } + return sp->slots[slot].namespace_name; +} + +void cbm_result_spill_close(cbm_result_spill_t *sp) { + if (!sp) { + return; + } + if (sp->files) { + for (int w = 0; w < sp->writers; w++) { + spill_file_t *f = &sp->files[w]; + if (f->fp) { + (void)fclose(f->fp); + if (!f->unlinked) { /* POSIX already removed it at open */ + (void)cbm_unlink(f->path); + } + cbm_mutex_destroy(&f->mu); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, sp->files); + } + if (sp->slots) { + for (int i = 0; i < sp->slot_count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, sp->slots[i].namespace_name); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, sp->slots); + cbm_free(CBM_MEM_CLASS_OTHER, sp); +} diff --git a/internal/cbm/result_spill.h b/internal/cbm/result_spill.h new file mode 100644 index 000000000..75b7a3231 --- /dev/null +++ b/internal/cbm/result_spill.h @@ -0,0 +1,73 @@ +/* + * result_spill.h — per-file extraction results parked on disk when the + * memory budget is hit. + * + * A compacted CBMFileResult is one exact-size arena block plus a header whose + * pointers all point into that block (cbm_result_compact guarantees it). That + * makes a result a relocatable blob: written as header + block, read back at + * any address by shifting every pointer by the base delta. The store is one + * append-only file per writer thread under the cache directory; loads use + * pread and are safe from any thread. Nothing here decides WHEN to spill -- + * the pipeline does, on the budget signal (see cbm_pipeline_ctx_t.spill). + */ +#ifndef CBM_RESULT_SPILL_H +#define CBM_RESULT_SPILL_H + +#include "cbm.h" + +#include +#include +#include + +typedef struct cbm_result_spill cbm_result_spill_t; + +/* One store for one pipeline run: `writers` append-only files under + * /spill--.bin, `slots` result slots (one per input file). + * NULL on failure (no directory, no space): the caller keeps results in + * memory as before. */ +cbm_result_spill_t *cbm_result_spill_open(const char *dir, int writers, int slots); + +/* Park a compacted result in slot `slot` via writer `writer`, then free the + * in-memory result (a retained parse tree goes with it; a loaded result has + * none). Returns false (and leaves the result untouched) when the result is + * not a single-block compaction, owns sub-results, or the write fails. */ +bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFileResult *result); + +/* True when slot `slot` holds a parked result. */ +bool cbm_result_spill_has(const cbm_result_spill_t *sp, int slot); + +/* Load slot `slot` back into memory: a fresh CBMFileResult the caller owns + * (cbm_free_result). NULL when the slot is empty or the read fails. Safe from + * several threads for different slots. */ +CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot); + +/* Read only the parked header of slot `slot` into *out: every count in it + * is valid, every pointer meaningless. False when the slot is empty or the + * read fails. Lets a consumer skip the load when there is nothing to read. */ +bool cbm_result_spill_peek_header(const cbm_result_spill_t *sp, int slot, CBMFileResult *out); + +/* Read only the parked header of slot `slot`: how many defs and impl + * relations it holds (the collector sizes its array before loading). */ +void cbm_result_spill_peek_counts(const cbm_result_spill_t *sp, int slot, int *defs, int *impls); + +/* The namespace/package a parked file declares, or NULL. Kept in MEMORY while + * the result itself is on disk, because import resolution builds its namespace + * map from EVERY file before any parked one is read back. Without it a spilled + * file contributed nothing to that map, so imports resolved differently + * depending on memory pressure -- php measured 57,182 edges in memory against + * 59,379 while spilling, the same binary, 2026-09-18. */ +const char *cbm_result_spill_namespace(const cbm_result_spill_t *sp, int slot); + +/* Counters for the log: results parked, bytes on disk, loads served. */ +void cbm_result_spill_stats(const cbm_result_spill_t *sp, int64_t *parked, int64_t *bytes, + int64_t *loads); + +/* Close and delete the files. */ +void cbm_result_spill_close(cbm_result_spill_t *sp); + +/* Relocation primitive used by the loader: every pointer inside `result` + * that points into [old_base, old_base + len) is shifted to the block now at + * `new_base`. Exposed for tests. */ +void cbm_result_relocate(CBMFileResult *result, const char *old_base, size_t len, char *new_base); + +#endif /* CBM_RESULT_SPILL_H */ diff --git a/internal/cbm/service_patterns.c b/internal/cbm/service_patterns.c index e3740f5aa..c2425e09e 100644 --- a/internal/cbm/service_patterns.c +++ b/internal/cbm/service_patterns.c @@ -140,6 +140,20 @@ static const lib_pattern_t http_libraries[] = { {"socket.http", CBM_SVC_HTTP, NULL}, {"resty.http", CBM_SVC_HTTP, NULL}, + /* Glued-name libraries. match_qn needs an identifier boundary around an + * id, so a library whose own name glues a lowercase prefix/suffix onto + * another id ("grequests", "curlpp") is listed explicitly: no boundary + * rule can tell "grequests" from "myrequests". */ + {"grequests", CBM_SVC_HTTP, NULL}, /* Python: gevent + requests */ + {"txrequests", CBM_SVC_HTTP, NULL}, /* Python: Twisted + requests */ + {"redaxios", CBM_SVC_HTTP, NULL}, /* JS: axios-compatible fetch client */ + {"gaxios", CBM_SVC_HTTP, NULL}, /* JS: Google's axios-style client */ + {"libcurl", CBM_SVC_HTTP, NULL}, /* C, node-libcurl */ + {"curlpp", CBM_SVC_HTTP, NULL}, /* C++ libcurl wrapper */ + {"curlcpp", CBM_SVC_HTTP, NULL}, /* C++ libcurl wrapper */ + {"hyperlocal", CBM_SVC_HTTP, NULL}, /* Rust: hyper over unix sockets */ + {"guzzlehttp", CBM_SVC_HTTP, NULL}, /* PHP: Guzzle's composer vendor */ + {NULL, CBM_SVC_NONE, NULL}, }; @@ -263,6 +277,33 @@ static const lib_pattern_t async_libraries[] = { {"dapr.clients.grpc", CBM_SVC_ASYNC, "dapr"}, {"DaprClient", CBM_SVC_ASYNC, "dapr"}, + /* Glued-name libraries (see the note in http_libraries). */ + {"aiokafka", CBM_SVC_ASYNC, "kafka"}, + {"pykafka", CBM_SVC_ASYNC, "kafka"}, + {"librdkafka", CBM_SVC_ASYNC, "kafka"}, + {"rdkafkacpp", CBM_SVC_ASYNC, "kafka"}, + {"rskafka", CBM_SVC_ASYNC, "kafka"}, + {"aioamqp", CBM_SVC_ASYNC, "rabbitmq"}, + {"amqpstorm", CBM_SVC_ASYNC, "rabbitmq"}, + {"pamqp", CBM_SVC_ASYNC, "rabbitmq"}, + {"pyamqp", CBM_SVC_ASYNC, "rabbitmq"}, + {"amqprs", CBM_SVC_ASYNC, "rabbitmq"}, + {"amqpcpp", CBM_SVC_ASYNC, "rabbitmq"}, + {"pynats", CBM_SVC_ASYNC, "nats"}, + {"jnats", CBM_SVC_ASYNC, "nats"}, + {"aiomqtt", CBM_SVC_ASYNC, "mqtt"}, + {"amqtt", CBM_SVC_ASYNC, "mqtt"}, + {"hbmqtt", CBM_SVC_ASYNC, "mqtt"}, + {"umqtt", CBM_SVC_ASYNC, "mqtt"}, + {"mqttools", CBM_SVC_ASYNC, "mqtt"}, + {"emqtt", CBM_SVC_ASYNC, "mqtt"}, + {"rumqtt", CBM_SVC_ASYNC, "mqtt"}, + {"gomqtt", CBM_SVC_ASYNC, "mqtt"}, + {"libmosquitto", CBM_SVC_ASYNC, "mqtt"}, + {"mosquittopp", CBM_SVC_ASYNC, "mqtt"}, + {"pubsublite", CBM_SVC_ASYNC, "pubsub"}, + {"gocelery", CBM_SVC_ASYNC, "celery"}, + {NULL, CBM_SVC_NONE, NULL}, }; @@ -308,6 +349,13 @@ static const lib_pattern_t config_libraries[] = { {"Application.get_env", CBM_SVC_CONFIG, NULL}, {"Application.fetch_env", CBM_SVC_CONFIG, NULL}, + /* Glued-name accessors (see the note in http_libraries). */ + {"wgetenv", CBM_SVC_CONFIG, NULL}, /* Windows CRT: _wgetenv, _wgetenv_s */ + {"getenvb", CBM_SVC_CONFIG, NULL}, /* Python: os.getenvb */ + {"qgetenv", CBM_SVC_CONFIG, NULL}, /* Qt */ + {"dotenvx", CBM_SVC_CONFIG, NULL}, /* JS: @dotenvx/dotenvx */ + {"phpdotenv", CBM_SVC_CONFIG, NULL}, /* PHP: vlucas/phpdotenv */ + {NULL, CBM_SVC_NONE, NULL}, }; @@ -377,6 +425,11 @@ static const lib_pattern_t route_reg_libraries[] = { {"akka.http.scaladsl.server", CBM_SVC_ROUTE_REG, NULL}, {"play.api.routing", CBM_SVC_ROUTE_REG, NULL}, + /* Glued-name frameworks (see the note in http_libraries). */ + {"apiflask", CBM_SVC_ROUTE_REG, NULL}, /* Python: Flask-based */ + {"honox", CBM_SVC_ROUTE_REG, NULL}, /* JS: Hono meta-framework */ + {"fasthttprouter", CBM_SVC_ROUTE_REG, NULL}, /* Go: httprouter for fasthttp */ + {NULL, CBM_SVC_NONE, NULL}, }; @@ -448,6 +501,11 @@ static const lib_pattern_t graphql_libraries[] = { {"async-graphql", CBM_SVC_GRAPHQL, NULL}, {"juniper", CBM_SVC_GRAPHQL, NULL}, + /* Glued-name libraries (see the note in http_libraries). */ + {"gqlparser", CBM_SVC_GRAPHQL, NULL}, /* Go: vektah/gqlparser */ + {"gqlgenc", CBM_SVC_GRAPHQL, NULL}, /* Go: gqlgen client generator */ + {"aiogqlc", CBM_SVC_GRAPHQL, NULL}, /* Python: asyncio GraphQL client */ + {NULL, CBM_SVC_NONE, NULL}, }; @@ -534,17 +592,67 @@ static const method_suffix_t method_suffixes[] = { /* ── Matching implementation ───────────────────────────────────── */ -/* Check if any library identifier appears as a substring in the QN. - * Case-sensitive: "requests" matches "project.venv.requests.api.get" - * but not "Requests". Library names are specific enough to avoid - * false positives even with substring matching. */ +static bool qn_is_lower(char ch) { + return ch >= 'a' && ch <= 'z'; +} + +static bool is_digit_char(char ch) { + return ch >= '0' && ch <= '9'; +} + +static bool qn_is_alnum(char ch) { + return qn_is_lower(ch) || is_digit_char(ch) || (ch >= 'A' && ch <= 'Z'); +} + +/* True when the occurrence of `id` at `hit` sits on identifier boundaries. + * + * A raw substring match let short ids fire inside unrelated words: "gin." + * in "plugin.", "dio" in "studio", "surf" in "surface", "express" in + * "expression". Every character that is not an ASCII letter or digit is a + * separator ('.', '/', '\\', ':', '_', '-', '$', '@', ...). + * + * BEFORE: start of string, a separator, an id that itself starts with a + * separator ("@trpc/server"), or an id that starts with an + * uppercase letter — a capital opens a new CamelCase word whatever + * precedes it ("AsyncHttpClient", "IHttpClientFactory", + * "NSURLSession"). Rejected: a lowercase/digit-initial id glued to + * a letter or digit ("plugin." / "studio" / "myrequests"). + * AFTER: end of string, a separator, an id that itself ends in a + * separator ("gin."), an uppercase letter (CamelCase glue: + * "GuzzleHttp", "FeignClient", "KafkaProducer", "kafkaProducer") + * or a digit (version suffix: "urllib2", "amqp091-go"). + * Rejected: a lowercase letter continuing the word ("surface", + * "curly", "expression"). */ +static bool qn_hit_on_boundary(const char *qn, const char *hit, const char *id, size_t id_len) { + char first = id[0]; + char last = id[id_len - 1]; + if (hit > qn && qn_is_alnum(hit[-1]) && (qn_is_lower(first) || is_digit_char(first))) { + return false; + } + if (qn_is_alnum(last) && qn_is_lower(hit[id_len])) { + return false; + } + return true; +} + +/* Check if any library identifier appears in the QN on identifier + * boundaries (see qn_hit_on_boundary). Case-sensitive: "requests" matches + * "project.venv.requests.api.get" and "svc.requests_get" but neither + * "Requests" nor "myrequests". */ static const lib_pattern_t *match_qn(const char *qn, const lib_pattern_t *patterns) { if (!qn || !qn[0]) { return NULL; } for (int i = 0; patterns[i].library_id != NULL; i++) { - if (strstr(qn, patterns[i].library_id) != NULL) { - return &patterns[i]; + const char *id = patterns[i].library_id; + size_t id_len = strlen(id); + if (id_len == 0) { + continue; + } + for (const char *hit = strstr(qn, id); hit != NULL; hit = strstr(hit + 1, id)) { + if (qn_hit_on_boundary(qn, hit, id, id_len)) { + return &patterns[i]; + } } } return NULL; @@ -574,10 +682,6 @@ static bool contains_segment(const char *path, const char *segment) { return false; } -static bool is_digit_char(char ch) { - return ch >= '0' && ch <= '9'; -} - static bool has_http_route_marker(const char *path) { if (starts_with_segment(path, "api") || starts_with_segment(path, "apis") || starts_with_segment(path, "graphql") || starts_with_segment(path, "health") || @@ -714,6 +818,25 @@ static const char *strip_string_delimiters(const char *literal, char *buf, size_ return buf; } +/* A comment opens with the slash a route opens with, and an argument list + * that starts with one handed three Java block comments to the Route pass as + * URLs (elasticsearch, 2026-09-16). The wildcard route (slash-star alone) is a + * path, so the block-comment shape needs its closing star-slash; a route + * literal never holds a line break. */ +bool cbm_service_pattern_is_comment_text(const char *text) { + if (!text || text[0] != '/') { + return false; + } + if (strchr(text, '\n') != NULL || strchr(text, '\r') != NULL) { + return true; + } + size_t n = strlen(text); + if (text[1] == '*' && n >= 4 && text[n - 2] == '*' && text[n - 1] == '/') { + return true; + } + return text[1] == '/' && (text[2] == ' ' || text[2] == '\t'); +} + bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name) { char path_buf[1024]; const char *path = strip_string_delimiters(literal, path_buf, sizeof(path_buf)); @@ -729,6 +852,9 @@ bool cbm_service_pattern_is_http_route_literal(const char *literal, const char * if (path[0] != '/') { return false; } + if (cbm_service_pattern_is_comment_text(path)) { + return false; + } if (callee_is_delimiter_or_filesystem_builder(callee_name)) { return false; } diff --git a/internal/cbm/service_patterns.h b/internal/cbm/service_patterns.h index 4642a08de..d9a93c251 100644 --- a/internal/cbm/service_patterns.h +++ b/internal/cbm/service_patterns.h @@ -48,6 +48,12 @@ bool cbm_service_pattern_is_global_fetch(const char *callee_name); * Rejects filesystem paths and non-HTTP string consumers. */ bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name); +/* True when text that reached a path slot is a source comment rather than a + * path: a block comment (slash-star … star-slash), a line comment (slash-slash + * followed by a space), or any text with a line break. The wildcard route + * (slash-star alone) is a path. */ +bool cbm_service_pattern_is_comment_text(const char *text); + /* Per-worker TLS cache for cbm_service_pattern_match results. The * pattern matcher runs once per resolved CALL edge in emit_service_ * edge — that's 6 pattern lists × ~30 patterns × strstr per call ≈ diff --git a/internal/cbm/sqlite_writer.c b/internal/cbm/sqlite_writer.c index 7391803f4..da8aba06c 100644 --- a/internal/cbm/sqlite_writer.c +++ b/internal/cbm/sqlite_writer.c @@ -15,6 +15,8 @@ // - Varints: 1-9 bytes, big-endian, MSB continuation #include "sqlite_writer.h" +#include "foundation/arena.h" /* index-cell arena */ +#include "foundation/compat.h" /* CBM_TLS */ #include "foundation/constants.h" #include "foundation/compat_fs.h" #include "foundation/compat_thread.h" @@ -26,6 +28,7 @@ #include #include #include +#include #ifdef _WIN32 #include @@ -301,29 +304,37 @@ typedef struct { uint8_t *data; int len; int cap; + bool heap; /* data is malloc-owned; otherwise it is the caller's inline space */ } DynBuf; -static void dynbuf_init(DynBuf *b) { - b->data = NULL; +/* Start on caller-provided space (a record builder's stack array). Growth past + * it moves to the heap once; a buffer that never outgrows it never allocates. */ +static void dynbuf_init_inline(DynBuf *b, uint8_t *space, int cap) { + b->data = space; b->len = 0; - b->cap = 0; + b->cap = cap; + b->heap = false; } static bool dynbuf_ensure(DynBuf *b, int needed) { if (b->len + needed <= b->cap) { return true; } - int newcap = b->cap == 0 ? INITIAL_PAGE_CAP : b->cap; + int newcap = b->cap < INITIAL_PAGE_CAP ? INITIAL_PAGE_CAP : b->cap; while (newcap < b->len + needed) { newcap *= GROWTH_FACTOR; } - uint8_t *p = (uint8_t *)realloc(b->data, newcap); + uint8_t *p = b->heap ? (uint8_t *)realloc(b->data, newcap) : (uint8_t *)malloc(newcap); if (!p) { (void)fprintf(stderr, "cbm_write_db: dynbuf realloc failed size=%d\n", newcap); return false; } + if (!b->heap && b->len > 0) { + memcpy(p, b->data, (size_t)b->len); + } b->data = p; b->cap = newcap; + b->heap = true; return true; } @@ -343,22 +354,35 @@ static bool dynbuf_append(DynBuf *b, const void *data, int len) { } static void dynbuf_free(DynBuf *b) { - free(b->data); + if (b->heap) { + free(b->data); + } b->data = NULL; b->len = b->cap = 0; + b->heap = false; } // --- Record builder --- // Builds a SQLite record: header (header_len varint + serial types) + body (values) +/* Inline space for one record. Every node, edge and index row used to allocate + * a 4 KB header buffer and a 4 KB body buffer and free both after copying a few + * dozen bytes out: the waste sanitizer measured 32 M allocations and ~126 GB of + * churn on the Go corpus, ~99 % of it never written (2026-09-17). A header is + * one serial type per column; a body is the row's text. Rows that do not fit + * (long property JSON) still grow onto the heap. */ +enum { REC_HEADER_INLINE = 64, REC_BODY_INLINE = 1536 }; + typedef struct { DynBuf header; // serial type varints DynBuf body; // column values + uint8_t header_space[REC_HEADER_INLINE]; + uint8_t body_space[REC_BODY_INLINE]; } RecordBuilder; static void rec_init(RecordBuilder *r) { - dynbuf_init(&r->header); - dynbuf_init(&r->body); + dynbuf_init_inline(&r->header, r->header_space, REC_HEADER_INLINE); + dynbuf_init_inline(&r->body, r->body_space, REC_BODY_INLINE); } static void rec_free(RecordBuilder *r) { @@ -408,8 +432,7 @@ static void rec_add_blob(RecordBuilder *r, const uint8_t *data, int len) { // Finalize: returns the complete record bytes (header_len + header + body). // Caller must free the returned buffer. -static uint8_t *rec_finalize(RecordBuilder *r, int *out_len) { - *out_len = 0; +static int rec_total_header(const RecordBuilder *r) { int header_content_len = r->header.len; int header_len_varint_len = varint_len(header_content_len + varint_len(header_content_len)); // The header size varint includes itself, so we may need to iterate @@ -420,20 +443,73 @@ static uint8_t *rec_finalize(RecordBuilder *r, int *out_len) { header_len_varint_len = recalc; total_header = header_len_varint_len + header_content_len; } + return total_header; +} - int total = total_header + r->body.len; +// Bytes rec_write produces: the record exactly as rec_finalize returns it. +static int rec_payload_len(const RecordBuilder *r) { + return rec_total_header(r) + r->body.len; +} + +// Write the record (header_len + header + body) at dst; returns its length. +static int rec_write(const RecordBuilder *r, uint8_t *dst) { + int pos = put_varint(dst, rec_total_header(r)); + memcpy(dst + pos, r->header.data, r->header.len); + pos += r->header.len; + memcpy(dst + pos, r->body.data, r->body.len); + return pos + r->body.len; +} + +static uint8_t *rec_finalize(RecordBuilder *r, int *out_len) { + *out_len = 0; + int total = rec_payload_len(r); uint8_t *buf = (uint8_t *)malloc(total); if (!buf) { return NULL; } - int pos = put_varint(buf, total_header); - memcpy(buf + pos, r->header.data, header_content_len); - pos += header_content_len; - memcpy(buf + pos, r->body.data, r->body.len); - *out_len = total; + *out_len = rec_write(r, buf); return buf; } +/* One index build's cells live in an arena: they are all released together + * once the B-tree is written, where a malloc + free per row was 12.1 M + * allocations on the Go corpus (waste sanitizer, 2026-09-17). The writer + * builds one index at a time on one thread; NULL = the malloc path, which the + * unit tests and any other caller keep. */ +enum { CELLS_ARENA_BLOCK = 1024 * 1024 }; +static CBM_TLS CBMArena *tl_cells_arena; + +static uint8_t *cells_alloc(size_t bytes) { + if (tl_cells_arena) { + return (uint8_t *)cbm_arena_alloc(tl_cells_arena, bytes); + } + return (uint8_t *)malloc(bytes); +} + +static void cells_free(uint8_t *cell) { + if (!tl_cells_arena) { + free(cell); + } +} + +// An index leaf cell, varint(payload_len) + record, written in place: building +// the record first and copying it into the cell was a second allocation and a +// copy for every index entry (16 M record buffers of pure churn on the Go +// corpus, waste sanitizer 2026-09-17). Same bytes. +static uint8_t *rec_finalize_index_cell(const RecordBuilder *r, int *out_len) { + *out_len = 0; + int payload_len = rec_payload_len(r); + int total = varint_len(payload_len) + payload_len; + uint8_t *cell = cells_alloc((size_t)total); + if (!cell) { + return NULL; + } + int pos = put_varint(cell, payload_len); + pos += rec_write(r, cell + pos); + *out_len = pos; + return cell; +} + // --- Page builder --- // Accumulates cells (records) into B-tree leaf pages. @@ -460,6 +536,11 @@ typedef struct { PageRef *leaves; int leaf_count; int leaf_cap; + + // One reusable buffer every table cell is assembled in before it is copied + // into the page (grows to the largest cell, freed by pb_free). + uint8_t *cell_buf; + int cell_cap; } PageBuilder; static void pb_init(PageBuilder *pb, FILE *fp, uint32_t start_page, bool is_index) { @@ -475,6 +556,8 @@ static void pb_init(PageBuilder *pb, FILE *fp, uint32_t start_page, bool is_inde pb->leaves = NULL; pb->leaf_count = 0; pb->leaf_cap = 0; + pb->cell_buf = NULL; + pb->cell_cap = 0; } static void pb_free(PageBuilder *pb) { @@ -484,6 +567,21 @@ static void pb_free(PageBuilder *pb) { } free(pb->leaves); } + free(pb->cell_buf); + pb->cell_buf = NULL; + pb->cell_cap = 0; +} + +static uint8_t *pb_cell_scratch(PageBuilder *pb, int len) { + if (pb->cell_cap < len) { + uint8_t *grown = (uint8_t *)realloc(pb->cell_buf, (size_t)len); + if (!grown) { + return NULL; + } + pb->cell_buf = grown; + pb->cell_cap = len; + } + return pb->cell_buf; } // Flush current leaf page to file @@ -736,74 +834,46 @@ static uint32_t pb_build_interior(PageBuilder *pb, bool is_index) { // Build a nodes table record: (id, project, label, name, qualified_name, file_path, start_line, // end_line, properties) -static uint8_t *build_node_record(const CBMDumpNode *n, int *out_len) { - RecordBuilder r; - rec_init(&r); - - rec_add_int(&r, n->id); - rec_add_text(&r, n->project); - rec_add_text(&r, n->label); - rec_add_text(&r, n->name); - rec_add_text(&r, n->qualified_name); - rec_add_text(&r, n->file_path ? n->file_path : ""); - rec_add_int(&r, n->start_line); - rec_add_int(&r, n->end_line); - rec_add_text(&r, n->properties ? n->properties : "{}"); - - uint8_t *data = rec_finalize(&r, out_len); - rec_free(&r); - return data; +static void fill_node_record(RecordBuilder *r, const CBMDumpNode *n) { + rec_add_int(r, n->id); + rec_add_text(r, n->project); + rec_add_text(r, n->label); + rec_add_text(r, n->name); + rec_add_text(r, n->qualified_name); + rec_add_text(r, n->file_path ? n->file_path : ""); + rec_add_int(r, n->start_line); + rec_add_int(r, n->end_line); + rec_add_text(r, n->properties ? n->properties : "{}"); } // Build an edges table record: (id, project, source_id, target_id, type, properties) // url_path_gen and local_name_gen are VIRTUAL generated columns — NOT stored in the record. -static uint8_t *build_edge_record(const CBMDumpEdge *e, int *out_len) { - RecordBuilder r; - rec_init(&r); - - rec_add_int(&r, e->id); - rec_add_text(&r, e->project); - rec_add_int(&r, e->source_id); - rec_add_int(&r, e->target_id); - rec_add_text(&r, e->type); - rec_add_text(&r, e->properties ? e->properties : "{}"); - - uint8_t *data = rec_finalize(&r, out_len); - rec_free(&r); - return data; +static void fill_edge_record(RecordBuilder *r, const CBMDumpEdge *e) { + rec_add_int(r, e->id); + rec_add_text(r, e->project); + rec_add_int(r, e->source_id); + rec_add_int(r, e->target_id); + rec_add_text(r, e->type); + rec_add_text(r, e->properties ? e->properties : "{}"); } // Build a node_vectors table record: (node_id, project, vector) -// Includes node_id in the record body (same pattern as build_node_record). -static uint8_t *build_vector_record(const CBMDumpVector *v, int *out_len) { - RecordBuilder r; - rec_init(&r); - - rec_add_int(&r, v->node_id); - rec_add_text(&r, v->project); - rec_add_blob(&r, v->vector, v->vector_len); - - uint8_t *data = rec_finalize(&r, out_len); - rec_free(&r); - return data; +// Includes node_id in the record body (same pattern as the node record). +static void fill_vector_record(RecordBuilder *r, const CBMDumpVector *v) { + rec_add_int(r, v->node_id); + rec_add_text(r, v->project); + rec_add_blob(r, v->vector, v->vector_len); } // Build a token_vectors table record: (id, project, token, vector, idf) -static uint8_t *build_token_vec_record(const CBMDumpTokenVec *tv, int *out_len) { - RecordBuilder r; - rec_init(&r); - - rec_add_int(&r, tv->id); - rec_add_text(&r, tv->project); - rec_add_text(&r, tv->token); - rec_add_blob(&r, tv->vector, tv->vector_len); +static void fill_token_vec_record(RecordBuilder *r, const CBMDumpTokenVec *tv) { + rec_add_int(r, tv->id); + rec_add_text(r, tv->project); + rec_add_text(r, tv->token); + rec_add_blob(r, tv->vector, tv->vector_len); /* Store IDF as integer × 1000 for fixed-point (avoid float in record) */ enum { IDF_FIXED_POINT_SCALE = 1000 }; - rec_add_int(&r, (int64_t)(tv->idf * IDF_FIXED_POINT_SCALE)); - - uint8_t *data = rec_finalize(&r, out_len); - rec_free(&r); - return data; + rec_add_int(r, (int64_t)(tv->idf * IDF_FIXED_POINT_SCALE)); } // Build a projects table record: (name, indexed_at, root_path) @@ -924,27 +994,8 @@ static uint8_t *build_index_entry_2text_rowid(const char *col1, const char *col2 rec_add_text(&r, col1); rec_add_text(&r, col2); rec_add_int(&r, rowid); - int payload_len = 0; - uint8_t *payload = rec_finalize(&r, &payload_len); + uint8_t *cell = rec_finalize_index_cell(&r, out_len); rec_free(&r); - if (!payload) { - *out_len = 0; - return NULL; - } - - // Index cell: varint(payload_len) + payload - int vl = varint_len(payload_len); - int total = vl + payload_len; - uint8_t *cell = (uint8_t *)malloc(total); - if (!cell) { - free(payload); - *out_len = 0; - return NULL; - } - int pos = put_varint(cell, payload_len); - memcpy(cell + pos, payload, payload_len); - free(payload); - *out_len = total; return cell; } @@ -956,26 +1007,8 @@ static uint8_t *build_index_entry_int_text_rowid(int64_t val, const char *text, rec_add_int(&r, val); rec_add_text(&r, text); rec_add_int(&r, rowid); - int payload_len = 0; - uint8_t *payload = rec_finalize(&r, &payload_len); + uint8_t *cell = rec_finalize_index_cell(&r, out_len); rec_free(&r); - if (!payload) { - *out_len = 0; - return NULL; - } - - int vl = varint_len(payload_len); - int total = vl + payload_len; - uint8_t *cell = (uint8_t *)malloc(total); - if (!cell) { - free(payload); - *out_len = 0; - return NULL; - } - int pos = put_varint(cell, payload_len); - memcpy(cell + pos, payload, payload_len); - free(payload); - *out_len = total; return cell; } @@ -988,26 +1021,8 @@ static uint8_t *build_index_entry_text_int_text_rowid(const char *t1, int64_t va rec_add_int(&r, val); rec_add_text(&r, t2); rec_add_int(&r, rowid); - int payload_len = 0; - uint8_t *payload = rec_finalize(&r, &payload_len); + uint8_t *cell = rec_finalize_index_cell(&r, out_len); rec_free(&r); - if (!payload) { - *out_len = 0; - return NULL; - } - - int vl = varint_len(payload_len); - int total = vl + payload_len; - uint8_t *cell = (uint8_t *)malloc(total); - if (!cell) { - free(payload); - *out_len = 0; - return NULL; - } - int pos = put_varint(cell, payload_len); - memcpy(cell + pos, payload, payload_len); - free(payload); - *out_len = total; return cell; } @@ -1023,26 +1038,8 @@ static uint8_t *build_index_entry_unique_2int_2text_rowid(int64_t v1, int64_t v2 rec_add_text(&r, text); rec_add_text(&r, text2); rec_add_int(&r, rowid); - int payload_len = 0; - uint8_t *payload = rec_finalize(&r, &payload_len); + uint8_t *cell = rec_finalize_index_cell(&r, out_len); rec_free(&r); - if (!payload) { - *out_len = 0; - return NULL; - } - - int vlen = varint_len(payload_len); - int total = vlen + payload_len; - uint8_t *cell = (uint8_t *)malloc(total); - if (!cell) { - free(payload); - *out_len = 0; - return NULL; - } - int pos = put_varint(cell, payload_len); - memcpy(cell + pos, payload, payload_len); - free(payload); - *out_len = total; return cell; } @@ -1115,18 +1112,34 @@ static uint8_t *overflowize_index_cell(FILE *fp, uint32_t *next_page, uint8_t *c uint32_t first_ovfl = write_overflow_pages(fp, next_page, cell + vlen + local, (int)plen - local); int nlen = vlen + local + BTREE_PTR_SIZE; - uint8_t *data = (uint8_t *)malloc((size_t)nlen); + uint8_t *data = cells_alloc((size_t)nlen); if (!data) { return cell; /* fall back to the (broken) inline form on OOM */ } memcpy(data, cell, (size_t)(vlen + local)); put_u32(data + vlen + local, first_ovfl); - free(cell); + cells_free(cell); *cell_len = nlen; return data; } #define TABLE_OVERFLOW_MIN_LOCAL 8199 +// Put a finished table cell on the current leaf, flushing a full leaf first. +static bool pb_place_table_cell(PageBuilder *pb, const uint8_t *cell, int cell_len, + int64_t prev_rowid) { + if (!pb_cell_fits(pb, cell_len) && pb->cell_count > 0) { + if (!pb_ensure_leaf_cap(pb)) { + return false; + } + pb->leaves[pb->leaf_count].max_key = prev_rowid; + pb->leaves[pb->leaf_count].sep_cell = NULL; + pb->leaves[pb->leaf_count].sep_cell_len = 0; + pb_flush_leaf(pb); + } + pb_add_cell(pb, cell, cell_len); + return true; +} + // Add a table cell to the PageBuilder, flushing leaf pages as needed. // If the payload exceeds max_local, overflow pages are written and only the // local portion plus a 4-byte overflow page pointer is stored in the leaf cell. @@ -1160,22 +1173,39 @@ static void pb_add_table_cell_with_flush(PageBuilder *pb, int64_t rowid, const u if (!cell) { return; } - - if (!pb_cell_fits(pb, cell_len) && pb->cell_count > 0) { - if (!pb_ensure_leaf_cap(pb)) { - free(cell); - return; - } - pb->leaves[pb->leaf_count].max_key = prev_rowid; - pb->leaves[pb->leaf_count].sep_cell = NULL; - pb->leaves[pb->leaf_count].sep_cell_len = 0; - pb_flush_leaf(pb); - } - - pb_add_cell(pb, cell, cell_len); + (void)pb_place_table_cell(pb, cell, cell_len, prev_rowid); free(cell); } +// A table record written straight into the page builder's cell buffer: the +// record builder's bytes go to the cell once, where rec_finalize + +// build_table_cell allocated the record, allocated the cell, copied, and freed +// both for every row. Same bytes; a record too large for the page takes the +// overflow path unchanged. False only on allocation failure. +static bool pb_add_table_record_with_flush(PageBuilder *pb, int64_t rowid, const RecordBuilder *r, + int64_t prev_rowid) { + int payload_len = rec_payload_len(r); + if (payload_len > TABLE_OVERFLOW_MAX_LOCAL) { + int rec_len = 0; + uint8_t *rec = rec_finalize((RecordBuilder *)r, &rec_len); + if (!rec) { + return false; + } + pb_add_table_cell_with_flush(pb, rowid, rec, rec_len, prev_rowid); + free(rec); + return true; + } + int total = varint_len(payload_len) + varint_len(rowid) + payload_len; + uint8_t *cell = pb_cell_scratch(pb, total); + if (!cell) { + return false; + } + int pos = put_varint(cell, payload_len); + pos += put_varint(cell + pos, rowid); + pos += rec_write(r, cell + pos); + return pb_place_table_cell(pb, cell, pos, prev_rowid); +} + // Finalize a table PageBuilder: flush last leaf and build interior pages. static uint32_t pb_finalize_table(PageBuilder *pb, uint32_t *next_page, int64_t last_rowid) { if (pb->cell_count > 0) { @@ -1607,21 +1637,8 @@ static uint8_t *ecell_url_path(const CBMDumpEdge *e, int *out_len) { rec_add_null(&r); } rec_add_int(&r, e->id); - int payload_len = 0; - uint8_t *payload = rec_finalize(&r, &payload_len); + uint8_t *cell = rec_finalize_index_cell(&r, out_len); rec_free(&r); - int vlen = varint_len(payload_len); - int total = vlen + payload_len; - uint8_t *cell = (uint8_t *)malloc(total); - if (!cell) { - free(payload); - *out_len = 0; - return NULL; - } - int pos = put_varint(cell, payload_len); - memcpy(cell + pos, payload, payload_len); - free(payload); - *out_len = total; return cell; } @@ -1636,19 +1653,21 @@ static uint32_t build_edge_index_sorted(FILE *fp, uint32_t *next_page, CBMDumpEd } uint8_t **idx_cells = (uint8_t **)malloc(edge_count * sizeof(uint8_t *)); int *idx_lens = (int *)malloc(edge_count * sizeof(int)); + CBMArena cells; + cbm_arena_init_lazy(&cells, CELLS_ARENA_BLOCK); if (!idx_cells || !idx_lens) { free(perm); free(idx_cells); free(idx_lens); return 0; } + tl_cells_arena = &cells; for (int i = 0; i < edge_count; i++) { int si = perm[i]; idx_cells[i] = cell_fn(&edges[si], &idx_lens[i]); if (!idx_cells[i]) { - for (int j = 0; j < i; j++) { - free(idx_cells[j]); - } + tl_cells_arena = NULL; + cbm_arena_destroy(&cells); free(idx_cells); free(idx_lens); free(perm); @@ -1657,9 +1676,8 @@ static uint32_t build_edge_index_sorted(FILE *fp, uint32_t *next_page, CBMDumpEd } free(perm); uint32_t root = write_index_btree(fp, next_page, idx_cells, idx_lens, edge_count); - for (int i = 0; i < edge_count; i++) { - free(idx_cells[i]); - } + tl_cells_arena = NULL; + cbm_arena_destroy(&cells); free(idx_cells); free(idx_lens); return root; @@ -1691,20 +1709,22 @@ static uint32_t build_node_index_sorted(FILE *fp, uint32_t *next_page, CBMDumpNo } uint8_t **idx_cells = (uint8_t **)malloc(node_count * sizeof(uint8_t *)); int *idx_lens = (int *)malloc(node_count * sizeof(int)); + CBMArena cells; + cbm_arena_init_lazy(&cells, CELLS_ARENA_BLOCK); if (!idx_cells || !idx_lens) { free(perm); free(idx_cells); free(idx_lens); return 0; } + tl_cells_arena = &cells; for (int i = 0; i < node_count; i++) { int si = perm[i]; idx_cells[i] = build_index_entry_2text_rowid(nodes[si].project, col_fn(&nodes[si]), nodes[si].id, &idx_lens[i]); if (!idx_cells[i]) { - for (int j = 0; j < i; j++) { - free(idx_cells[j]); - } + tl_cells_arena = NULL; + cbm_arena_destroy(&cells); free(idx_cells); free(idx_lens); free(perm); @@ -1713,9 +1733,8 @@ static uint32_t build_node_index_sorted(FILE *fp, uint32_t *next_page, CBMDumpNo } free(perm); uint32_t root = write_index_btree(fp, next_page, idx_cells, idx_lens, node_count); - for (int i = 0; i < node_count; i++) { - free(idx_cells[i]); - } + tl_cells_arena = NULL; + cbm_arena_destroy(&cells); free(idx_cells); free(idx_lens); return root; @@ -1759,6 +1778,10 @@ static int sync_writer_output(FILE *fp) { } static int discard_writer_output(write_db_ctx_t *w, int rc) { + /* Cleanup runs after the failure that brought us here, and a library + * call may set errno even when it succeeds. Carry the reason across it + * so the caller can report WHY the publish failed. */ + int failure_errno = errno; if (w->fp) { (void)fclose(w->fp); w->fp = NULL; @@ -1766,6 +1789,7 @@ static int discard_writer_output(write_db_ctx_t *w, int rc) { if (w->temp_path[0]) { (void)cbm_unlink(w->temp_path); } + errno = failure_errno; return rc; } @@ -1774,10 +1798,12 @@ static int publish_writer_output(write_db_ctx_t *w) { return discard_writer_output(w, ERR_WRITE_FAILED); } if (fclose(w->fp) != 0) { + int failure_errno = errno; w->fp = NULL; if (w->temp_path[0]) { (void)cbm_unlink(w->temp_path); } + errno = failure_errno; return ERR_WRITE_FAILED; } w->fp = NULL; @@ -1785,7 +1811,12 @@ static int publish_writer_output(write_db_ctx_t *w) { return 0; } if (cbm_rename_replace(w->temp_path, w->final_path) != 0) { + /* cbm_rename_replace translated the platform error into errno so the + * caller can say what denied the publish (#1620). Preserve it across + * the cleanup unlink. */ + int rename_errno = errno; (void)cbm_unlink(w->temp_path); + errno = rename_errno; return ERR_WRITE_FAILED; } /* Sidecars are removed only after the replacement succeeds. On POSIX, @@ -1796,13 +1827,13 @@ static int publish_writer_output(write_db_ctx_t *w) { return 0; } -/* Callback type for building a record from an item at index i. */ -typedef uint8_t *(*build_record_fn)(const void *items, int i, int *out_len); +/* Callback type for filling a record from an item at index i. */ +typedef void (*fill_record_fn)(RecordBuilder *r, const void *items, int i); typedef int64_t (*get_rowid_fn)(const void *items, int i); /* Write a streaming B-tree table from count items, or an empty table if count == 0. */ static int write_one_table(write_db_ctx_t *w, uint32_t *root, const void *items, int count, - build_record_fn build_rec, get_rowid_fn get_id) { + fill_record_fn fill_rec, get_rowid_fn get_id) { if (count <= 0 || !items) { *root = write_table_btree(w->fp, &w->next_page, NULL, NULL, NULL, 0, false); return 0; @@ -1810,15 +1841,17 @@ static int write_one_table(write_db_ctx_t *w, uint32_t *root, const void *items, PageBuilder pb; pb_init(&pb, w->fp, w->next_page, false); for (int i = 0; i < count; i++) { - int rec_len; - uint8_t *rec = build_rec(items, i, &rec_len); - if (!rec) { - return ERR_WRITE_FAILED; - } + RecordBuilder r; + rec_init(&r); + fill_rec(&r, items, i); int64_t rowid = get_id(items, i); int64_t prev_id = i > 0 ? get_id(items, i - SKIP_ONE) : 0; - pb_add_table_cell_with_flush(&pb, rowid, rec, rec_len, prev_id); - free(rec); + bool placed = pb_add_table_record_with_flush(&pb, rowid, &r, prev_id); + rec_free(&r); + if (!placed) { + pb_free(&pb); + return ERR_WRITE_FAILED; + } } *root = pb_finalize_table(&pb, &w->next_page, get_id(items, count - SKIP_ONE)); return 0; @@ -1826,20 +1859,20 @@ static int write_one_table(write_db_ctx_t *w, uint32_t *root, const void *items, /* Adapter functions for write_one_table (nodes are written via the streaming * PageBuilder in cbm_writer_append_nodes, so no node adapter is needed here). */ -static uint8_t *adapt_build_edge(const void *items, int i, int *out_len) { - return build_edge_record(&((const CBMDumpEdge *)items)[i], out_len); +static void adapt_fill_edge(RecordBuilder *r, const void *items, int i) { + fill_edge_record(r, &((const CBMDumpEdge *)items)[i]); } static int64_t adapt_edge_id(const void *items, int i) { return ((const CBMDumpEdge *)items)[i].id; } -static uint8_t *adapt_build_vector(const void *items, int i, int *out_len) { - return build_vector_record(&((const CBMDumpVector *)items)[i], out_len); +static void adapt_fill_vector(RecordBuilder *r, const void *items, int i) { + fill_vector_record(r, &((const CBMDumpVector *)items)[i]); } static int64_t adapt_vector_id(const void *items, int i) { return ((const CBMDumpVector *)items)[i].node_id; } -static uint8_t *adapt_build_token_vec(const void *items, int i, int *out_len) { - return build_token_vec_record(&((const CBMDumpTokenVec *)items)[i], out_len); +static void adapt_fill_token_vec(RecordBuilder *r, const void *items, int i) { + fill_token_vec_record(r, &((const CBMDumpTokenVec *)items)[i]); } static int64_t adapt_token_vec_id(const void *items, int i) { return ((const CBMDumpTokenVec *)items)[i].id; @@ -2066,17 +2099,17 @@ static int write_db_after_nodes(write_db_ctx_t *w, uint32_t nodes_root) { uint32_t vectors_root; uint32_t token_vecs_root; int rc = - write_one_table(w, &edges_root, w->edges, w->edge_count, adapt_build_edge, adapt_edge_id); + write_one_table(w, &edges_root, w->edges, w->edge_count, adapt_fill_edge, adapt_edge_id); if (rc != 0) { return discard_writer_output(w, rc); } - rc = write_one_table(w, &vectors_root, w->vectors, w->vector_count, adapt_build_vector, + rc = write_one_table(w, &vectors_root, w->vectors, w->vector_count, adapt_fill_vector, adapt_vector_id); if (rc != 0) { return discard_writer_output(w, rc); } rc = write_one_table(w, &token_vecs_root, w->token_vecs, w->token_vec_count, - adapt_build_token_vec, adapt_token_vec_id); + adapt_fill_token_vec, adapt_token_vec_id); if (rc != 0) { return discard_writer_output(w, rc); } @@ -2283,13 +2316,22 @@ cbm_db_writer_t *cbm_writer_open(const char *path) { int n = snprintf(w->wc.final_path, sizeof(w->wc.final_path), "%s", path); if (n < 0 || (size_t)n >= sizeof(w->wc.final_path) || make_writer_temp_path(path, w, w->wc.temp_path, sizeof(w->wc.temp_path)) != 0) { + /* Both conditions are truncation and neither sets errno; name the + * reason rather than let the caller report a stale one. */ free(w); + errno = ENAMETOOLONG; return NULL; } FILE *fp = cbm_fopen(w->wc.temp_path, "wb"); if (!fp) { + /* The cleanup below unlinks a file that was never created, so it + * fails and leaves ENOENT behind — which reads as a missing path + * when the real answer is that the directory refused the create. + * That is the #1620 case, so carry the open's reason across it. */ + int open_errno = errno; (void)cbm_unlink(w->wc.temp_path); free(w); + errno = open_errno; return NULL; } w->wc.fp = fp; @@ -2307,16 +2349,18 @@ int cbm_writer_append_nodes(cbm_db_writer_t *w, const CBMDumpNode *nodes, int co return w->err; } for (int i = 0; i < count; i++) { - int rec_len; - uint8_t *rec = build_node_record(&nodes[i], &rec_len); - if (!rec) { + RecordBuilder r; + rec_init(&r); + fill_node_record(&r, &nodes[i]); + /* prev_rowid is the previous node's id (0 for the very first), matching + * the one-shot write_one_table loop — so output is byte-identical. */ + bool placed = + pb_add_table_record_with_flush(&w->nodes_pb, nodes[i].id, &r, w->last_node_rowid); + rec_free(&r); + if (!placed) { w->err = ERR_WRITE_FAILED; return w->err; } - /* prev_rowid is the previous node's id (0 for the very first), matching - * the one-shot write_one_table loop — so output is byte-identical. */ - pb_add_table_cell_with_flush(&w->nodes_pb, nodes[i].id, rec, rec_len, w->last_node_rowid); - free(rec); w->last_node_rowid = nodes[i].id; w->node_rows_written++; } @@ -2355,6 +2399,10 @@ int cbm_writer_finalize(cbm_db_writer_t *w, const char *project, const char *roo write_db_ctx_t wc = w->wc; /* value copy survives free(w) */ free(w); if (err != 0) { + /* A sticky append failure: errno belongs to whatever call failed + * many calls ago, not to the publish. Clear it so the caller does + * not attach a reason this path does not have. */ + errno = 0; return discard_writer_output(&wc, err); } return write_db_after_nodes(&wc, nodes_root); diff --git a/internal/cbm/vendored/grammars/MANIFEST.md b/internal/cbm/vendored/grammars/MANIFEST.md index 4025b5f50..74322763e 100644 --- a/internal/cbm/vendored/grammars/MANIFEST.md +++ b/internal/cbm/vendored/grammars/MANIFEST.md @@ -87,6 +87,7 @@ row instead. | rescript | `rescript/scanner.c`, deserialize | guard `memcpy(state, buffer, n_bytes)` with `if (n_bytes > 0)` | UBSan: zero-length `memcpy` with a NULL `buffer` / `n_bytes == 0` on empty-state deserialize (formal UB, harmless). The sibling serialize copies a fixed `sizeof(ScannerState)` (always > 0, non-NULL src) and needs no guard. | | purescript | `purescript/scanner.c`, serialize | guard `memcpy(buffer, indents->data, to_copy)` with `if (to_copy > 0)` | UBSan: zero-length `memcpy` with a NULL/0-size source when the indent vector is empty (formal UB, harmless) | | plsql | `plsql/parser.c`, include | `#include ` → `#include "tree_sitter/parser.h"` | The older ABI-14 generator emits angle brackets; every other vendored grammar uses the quoted form, which resolves the per-grammar `tree_sitter/` header from the including file's directory | +| properties | `properties/scanner.c`, whole file | move the `reached_eof` flag out of a file-scope `static bool` and into a per-parser payload allocated by `..._external_scanner_create()` | **Data race, and the only one of its kind in 103 vendored scanners.** That flag is how the scanner refuses to emit a second end-of-input `FAKE_EOL`, and upstream keeps it in ONE process-wide object. With several worker threads indexing `.properties` files at once, whichever reached EOF first set the flag, so another thread's `!reached_eof` was false and its parse never received the `FAKE_EOL` it needed to finish — it sat at end-of-input asking for a token that would never arrive. Measured 2026-09-19: ~106 M parse operations on a 253-byte file before the per-file budget stopped it, after which the file was dropped from the graph and java/kotlin indexes differed run to run. Interleaved A/B under identical load, 40 runs each: unpatched 6 stalls, patched 0. The serialize/deserialize wire format is unchanged (the state is still the returned length). Upstream `6310671b24d4` still carries the static, so a re-vendor must re-apply this | | swift | `swift/scanner.c`, `OP_SYMBOL_SUPPRESSOR` + `eat_operators` | `1UL <<` / `1 <<` → `1ULL <<` | UBSan: `1 << suppressor` shifts an `int` by up to `TOKEN_COUNT` bits, undefined once the index reaches 31, while the mask it feeds is `uint64_t`. `1UL << FAKE_TRY_BANG` is the same defect on Windows, where `unsigned long` is 32 bits and `FAKE_TRY_BANG` is 32; the CLANGARM64 leg runs UBSan in trap mode, so there it is an illegal instruction rather than a log line. Upstream already carries both changes: `fb63a7004f07` (2026-04-06, upstream #558) for `eat_operators`, `6ab8d1d74ebd` (2026-08-10) for the `OP_SYMBOL_SUPPRESSOR` entry. Our pin `8abb3e8b3325` (2026-03-20) predates both, so this is a backport rather than a local invention — a re-vendor past 2026-08-10 should delete this row, not re-apply it | ## Vendored from verified upstream diff --git a/internal/cbm/vendored/grammars/properties/scanner.c b/internal/cbm/vendored/grammars/properties/scanner.c index 583456268..5ef7b71b0 100644 --- a/internal/cbm/vendored/grammars/properties/scanner.c +++ b/internal/cbm/vendored/grammars/properties/scanner.c @@ -1,5 +1,7 @@ #include "tree_sitter/parser.h" +#include + #ifdef _MSC_VER #pragma warning(disable : 4100) #elif defined(__GNUC__) || defined(__clang__) @@ -8,21 +10,43 @@ enum TokenType { FAKE_EOL }; -static bool reached_eof = false; +/* The scanner's one bit of state -- whether THIS parse has already emitted the + * end-of-input FAKE_EOL -- lives in the payload, one per parser, which is what + * tree-sitter's create/destroy contract is for. + * + * It used to be a file-scope `static bool`: one object for the entire process, + * shared by every thread. The grammar needs exactly one FAKE_EOL to close a + * file, and `!reached_eof` is how the scanner refuses to emit a second one. So + * when two threads indexed .properties files at the same time, whichever + * reached EOF first set the flag, and the other thread's `!reached_eof` was + * false -- it never received the FAKE_EOL its parse was waiting for and sat at + * end-of-input asking for a token that would never arrive. Measured on + * 2026-09-19: ~106 million parse operations on a 253-byte file before the + * per-file budget stopped it, after which the file was dropped from the graph + * and the index differed from run to run. */ +typedef struct { + bool reached_eof; +} Scanner; -bool tree_sitter_properties_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) { +bool tree_sitter_properties_external_scanner_scan(void *payload, TSLexer *lexer, + const bool *valid_symbols) { + Scanner *scanner = (Scanner *)payload; lexer->result_symbol = FAKE_EOL; - return reached_eof = !reached_eof && valid_symbols[FAKE_EOL] && lexer->eof(lexer); + return scanner->reached_eof = + !scanner->reached_eof && valid_symbols[FAKE_EOL] && lexer->eof(lexer); } +/* Wire format unchanged: the state IS the returned length (1 = already emitted, + * 0 = not), and no bytes are written to the buffer. */ unsigned tree_sitter_properties_external_scanner_serialize(void *payload, char *buffer) { - return reached_eof; + return ((Scanner *)payload)->reached_eof; } -void tree_sitter_properties_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) { - reached_eof = length; +void tree_sitter_properties_external_scanner_deserialize(void *payload, const char *buffer, + unsigned length) { + ((Scanner *)payload)->reached_eof = length; } -void *tree_sitter_properties_external_scanner_create() { return NULL; } +void *tree_sitter_properties_external_scanner_create() { return calloc(1, sizeof(Scanner)); } -void tree_sitter_properties_external_scanner_destroy(void *payload) { } \ No newline at end of file +void tree_sitter_properties_external_scanner_destroy(void *payload) { free(payload); } diff --git a/pkg/chocolatey/codebase-memory-mcp.nuspec b/pkg/chocolatey/codebase-memory-mcp.nuspec index 5ce1deacc..7b6f6cabe 100644 --- a/pkg/chocolatey/codebase-memory-mcp.nuspec +++ b/pkg/chocolatey/codebase-memory-mcp.nuspec @@ -2,7 +2,7 @@ codebase-memory-mcp - 0.10.8 + 0.11.0 codebase-memory-mcp DeusData https://github.com/DeusData/codebase-memory-mcp @@ -28,7 +28,7 @@ After installation, run: to configure your coding agents automatically. - https://github.com/DeusData/codebase-memory-mcp/releases/tag/v0.10.8 + https://github.com/DeusData/codebase-memory-mcp/releases/tag/v0.11.0 diff --git a/pkg/chocolatey/tools/chocolateyInstall.ps1 b/pkg/chocolatey/tools/chocolateyInstall.ps1 index 132b86d90..031ac9e86 100644 --- a/pkg/chocolatey/tools/chocolateyInstall.ps1 +++ b/pkg/chocolatey/tools/chocolateyInstall.ps1 @@ -1,9 +1,9 @@ $ErrorActionPreference = 'Stop' $packageName = 'codebase-memory-mcp' -$version = '0.10.8' +$version = '0.11.0' $url64 = "https://github.com/DeusData/codebase-memory-mcp/releases/download/v${version}/codebase-memory-mcp-windows-amd64.zip" -$checksum64 = 'b43ad982994c4d829670749e08d3b622a74bb20041fc0a7d02bef6113f81c34d' +$checksum64 = '6eb6beaf261b19e419766e78baf93cbc3cf1c6338cff8fb7c0234859f96d1685' $installDir = Join-Path $env:ChocolateyBinRoot $packageName Install-ChocolateyZipPackage ` diff --git a/pkg/go/cmd/codebase-memory-mcp/main.go b/pkg/go/cmd/codebase-memory-mcp/main.go index 91f1adc33..65f6e131a 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main.go +++ b/pkg/go/cmd/codebase-memory-mcp/main.go @@ -36,7 +36,7 @@ import ( const ( repo = "DeusData/codebase-memory-mcp" - version = "0.10.8" + version = "0.11.0" windowsBinaryName = "codebase-memory-mcp.exe" maxRedirects = 5 diff --git a/pkg/npm/README.md b/pkg/npm/README.md index 614b81b1c..e1e5abe3e 100644 --- a/pkg/npm/README.md +++ b/pkg/npm/README.md @@ -7,7 +7,7 @@ **The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. This npm wrapper downloads, verifies, and caches the selected native runtime set: the executable, its authenticated integration asset, and—when requested—the content-addressed UI pack. -High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 162 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 45 automatic/conditional client surfaces. +High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 162 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 17 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 45 automatic/conditional client surfaces. ## Installation @@ -38,7 +38,7 @@ Restart your agent. Say **"Index this project"** — done. - **Lifecycle hooks stay conservative** — Kimi uses `UserPromptSubmit`; on macOS/Linux, GitLab Duo gets a fail-open user `SessionStart`, while Devin gets `UserPromptSubmit`, `PostCompaction`, and a deduplicated `SessionStart` when Claude does not already provide it. Qoder, GitLab Duo, Devin, and Factory hooks are withheld on Windows without a documented shell/executor contract. Cline's auto-activating file hooks are withheld because their context output is not reliably consumed, CodeBuddy beta hooks are not auto-installed, and Cursor context hooks remain withheld. - **Subagent access is explicit** — Claude, Gemini, Kiro, Qwen, CodeBuddy, KiloCode, Mistral Vibe, Grok Build, Qoder, Junie, and Factory get documented graph profiles with the narrowest tool/server filters their schemas support. KiloCode and Vibe enumerate read-only query tools rather than using server wildcards. Cursor, Rovo, Pochi, and Cline use explicit parent handoff where child MCP is unavailable or unsafe; IBM Bob receives no invented hook or agent. - **Manual, UI, cloud, or repository-managed (not counted)** — Qodo, Warp MCP, JetBrains AI/ACP, GitHub Copilot coding agent, Jules, CodeRabbit, Replit, BLACKBOX AI, Plandex, and SWE-agent. Warp is counted above for its detected skill installation; its MCP connection remains manual. -- **15 MCP tools** — search, trace, architecture, impact analysis, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. +- **17 MCP tools** — search, trace, architecture, impact analysis, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. ## Supported Platforms diff --git a/pkg/npm/package.json b/pkg/npm/package.json index f2b122d84..d19488f7f 100644 --- a/pkg/npm/package.json +++ b/pkg/npm/package.json @@ -1,6 +1,6 @@ { "name": "codebase-memory-mcp", - "version": "0.10.8", + "version": "0.11.0", "description": "Fast code intelligence for AI coding agents — installs a verified native runtime set", "mcpName": "io.github.DeusData/codebase-memory-mcp", "license": "MIT", diff --git a/pkg/pypi/pyproject.toml b/pkg/pypi/pyproject.toml index 5821d2a4b..f848e7cd8 100644 --- a/pkg/pypi/pyproject.toml +++ b/pkg/pypi/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "hatchling.build" [project] name = "codebase-memory-mcp" -version = "0.10.8" +version = "0.11.0" description = "Fast code intelligence for AI coding agents — installs a verified native runtime set" readme = "README.md" license = { text = "MIT" } diff --git a/pkg/pypi/src/codebase_memory_mcp/_cli.py b/pkg/pypi/src/codebase_memory_mcp/_cli.py index 8a4aa112c..458b8c350 100644 --- a/pkg/pypi/src/codebase_memory_mcp/_cli.py +++ b/pkg/pypi/src/codebase_memory_mcp/_cli.py @@ -338,7 +338,7 @@ def _version() -> str: from importlib.metadata import version return version("codebase-memory-mcp") except Exception: - return "0.10.8" + return "0.11.0" def _os_name() -> str: diff --git a/scripts/fuzz.sh b/scripts/fuzz.sh new file mode 100755 index 000000000..43e44d8f9 --- /dev/null +++ b/scripts/fuzz.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# fuzz.sh — run the libFuzzer targets of build/fuzz/cbm-fuzz. +# +# Usage: +# scripts/fuzz.sh [--runs N] [--max-total-time S] +# [--jobs N] [--seed N] [--no-build] +# +# Two shapes, one driver: +# --runs N --seed S a fixed number of executions from a fixed seed: the same +# inputs every time, so a smoke in CI is a function of the +# code (O9). This is the default (--runs 20000 --seed 1). +# --max-total-time S exploration for dry-run and nightly: new inputs are the +# point, and a crash is saved with its reproducer. +# +# The checked-in seeds (tests/fuzz/corpus//) are copied into a working +# corpus under build/fuzz/, so a run never rewrites the repository. A crash, +# leak or timeout leaves its input in build/fuzz/artifacts-/ and the +# script exits non-zero; reproduce with +# CBM_FUZZ_TARGET= build/fuzz/cbm-fuzz +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# --help before the positional target: the venue-parity interface probe calls +# every canonical entry as ` --help` with no other argument. +case "${1:-}" in +-h | --help) sed -n '2,21p' "$0"; exit 0 ;; +esac +TARGET="${1:-}" +[ $# -gt 0 ] && shift +RUNS=20000 +SEED=1 +MAX_TIME="" +JOBS=1 +BUILD=1 +while [ $# -gt 0 ]; do + case "$1" in + --runs) RUNS="$2"; shift 2 ;; + --seed) SEED="$2"; shift 2 ;; + --max-total-time) MAX_TIME="$2"; RUNS=""; shift 2 ;; + --jobs) JOBS="$2"; shift 2 ;; + --no-build) BUILD=0; shift ;; + -h | --help) sed -n '2,21p' "$0"; exit 0 ;; + *) echo "fuzz.sh: unknown argument $1. Please consult --help." >&2; exit 2 ;; + esac +done + +case "$TARGET" in +extract | cypher | config) TARGETS=("$TARGET") ;; +all) TARGETS=(extract cypher config) ;; +*) echo "fuzz.sh: target must be extract, cypher, config or all. Please consult --help." >&2 + exit 2 ;; +esac + +BIN="$ROOT/build/fuzz/cbm-fuzz" +if [ "$BUILD" -eq 1 ]; then + make -C "$ROOT" -f Makefile.cbm -j"$(getconf _NPROCESSORS_ONLN)" fuzz >/dev/null +fi +if [ ! -x "$BIN" ]; then + echo "fuzz.sh: $BIN is missing (build failed, or --no-build without a build)" >&2 + exit 2 +fi + +status=0 +for t in "${TARGETS[@]}"; do + corpus="$ROOT/build/fuzz/corpus-$t" + artifacts="$ROOT/build/fuzz/artifacts-$t/" + mkdir -p "$corpus" "$artifacts" + cp -R "$ROOT/tests/fuzz/corpus/$t/." "$corpus/" + args=(-artifact_prefix="$artifacts" -seed="$SEED" -rss_limit_mb=4096 -timeout=30 -print_final_stats=1) + if [ -n "$RUNS" ]; then + args+=(-runs="$RUNS") + else + args+=(-max_total_time="$MAX_TIME") + fi + if [ "$JOBS" -gt 1 ]; then + args+=(-jobs="$JOBS" -workers="$JOBS") + fi + echo "=== fuzz $t: ${RUNS:+$RUNS runs}${MAX_TIME:+$MAX_TIME s} (seed $SEED) ===" + if ! CBM_FUZZ_TARGET="$t" "$BIN" "${args[@]}" "$corpus"; then + echo "fuzz.sh: target $t FAILED -- reproducer in $artifacts" >&2 + status=1 + fi +done +exit "$status" diff --git a/scripts/hooks/pre-commit b/scripts/hooks/pre-commit index fa42d92ae..7b5e1a4ff 100755 --- a/scripts/hooks/pre-commit +++ b/scripts/hooks/pre-commit @@ -17,4 +17,13 @@ echo "pre-commit: security audit (source-level)..." scripts/security-audit.sh echo "pre-commit: building and running tests..." -make -j$(sysctl -n hw.ncpu 2>/dev/null || nproc) -f Makefile.cbm test +( + # Git exports repository-local variables to hooks. The test suite creates + # independent repositories and linked worktrees, so keep the commit's + # index and object paths out of those fixtures. + while IFS= read -r variable; do + unset "$variable" + done < <(git rev-parse --local-env-vars) + + make -j"$(sysctl -n hw.ncpu 2>/dev/null || nproc)" -f Makefile.cbm test +) diff --git a/scripts/lint-memory-core.py b/scripts/lint-memory-core.py new file mode 100644 index 000000000..18c329383 --- /dev/null +++ b/scripts/lint-memory-core.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +""" +lint-memory-core.py — the memory-core linter. + +Memory in this project is allocated through ONE core (src/foundation/mem_core.h), +not through scattered raw malloc/calloc/realloc/free/strdup. This linter is the +gate that keeps that true. + +WHY A RATCHET, NOT A BAN. An audit on 2026-09-13 found ~800 raw allocation +sites in src/ and none of them can be migrated in one change. So the gate works +like every other honest debt gate: a checked-in baseline records how many raw +sites each file has TODAY, and the build goes red the moment any file has MORE +than its baseline, or a file not in the baseline grows one. Files can only ever +go down. When a file is migrated, its baseline line is lowered (or removed) in +the same change -- and --strict turns "below baseline" into a failure too, so +the baseline cannot silently rot behind the code. + +WHAT COUNTS AS RAW. A call to malloc/calloc/realloc/free/strdup/strndup that is +not part of a longer identifier. cbm_alloc, cbm_free, cbm_calloc, mi_malloc, +cbm_arena_alloc and heap_strdup are all NOT matches: the character before the +name is an identifier character. Comments and string literals are stripped +first, so prose that mentions malloc( does not trip the gate -- the security +audit already bit us once on exactly that with fork(. + +EXEMPT. The core itself and the allocator plumbing it sits on: + src/foundation/mem_core.c the route + (arena.c allocates its blocks THROUGH the core -- class arena -- and is scanned) + src/foundation/slab_alloc.c same + src/foundation/mem.c policy/measurement, probes with malloc + src/foundation/mem_override_*.c the --wrap / override shims + src/foundation/compat*.c libc replacement surface + internal/**/vendored/** not ours + vendored/** not ours + +SCOPE. Everything under src/ and internal/ -- cli, mcp, daemon, store, +pipeline, the extraction engine -- so the count is for the whole project, not +one subsystem, and a leak shows up wherever it is. + +Usage: + lint-memory-core.py check against the baseline (CI) + lint-memory-core.py --strict also fail when a file is BELOW baseline + lint-memory-core.py --write-baseline regenerate scripts/memory-core-baseline.txt + lint-memory-core.py --list print every raw site (file:line: call) + +Exit 0 = clean. Exit 1 = a file grew. Exit 2 = usage/IO error. +""" +from __future__ import annotations + +import argparse +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parent.parent +BASELINE = ROOT / "scripts" / "memory-core-baseline.txt" +SCAN_ROOTS = ("src", "internal") +RAW = re.compile(r"(? bool: + """Any vendored/ segment anywhere under internal/ is not ours.""" + return "/vendored/" in rel or rel.startswith("vendored/") + + +def is_exempt(rel: str) -> bool: + return rel in EXEMPT_EXACT or rel.startswith(EXEMPT_PREFIX) or is_vendored(rel) + + +def strip_comments_and_strings(text: str) -> str: + """Blank out comments and string literals, preserving line structure so + reported line numbers stay right. Character-class aware enough for C: + handles escapes inside strings and does not treat // inside a string as a + comment.""" + out = [] + i, n = 0, len(text) + while i < n: + c = text[i] + nxt = text[i + 1] if i + 1 < n else "" + if c == "/" and nxt == "*": + j = text.find("*/", i + 2) + j = n if j < 0 else j + 2 + out.append("".join(ch if ch == "\n" else " " for ch in text[i:j])) + i = j + elif c == "/" and nxt == "/": + j = text.find("\n", i) + j = n if j < 0 else j + out.append(" " * (j - i)) + i = j + elif c == '"' or c == "'": + q = c + j = i + 1 + while j < n and text[j] != q: + if text[j] == "\\": + j += 1 + if j < n and text[j] == "\n": + break + j += 1 + j = min(j + 1, n) + out.append(q + " " * max(0, j - i - 2) + (q if j - i >= 2 else "")) + i = j + else: + out.append(c) + i += 1 + return "".join(out) + + +def scan() -> dict[str, list[tuple[int, str]]]: + hits: dict[str, list[tuple[int, str]]] = {} + for root in SCAN_ROOTS: + base = ROOT / root + if not base.exists(): + continue + for path in sorted(base.rglob("*")): + if path.suffix not in (".c", ".h") or not path.is_file(): + continue + rel = path.relative_to(ROOT).as_posix() + if is_exempt(rel): + continue + text = strip_comments_and_strings(path.read_text(encoding="utf-8", errors="replace")) + for lineno, line in enumerate(text.splitlines(), 1): + for m in RAW.finditer(line): + hits.setdefault(rel, []).append((lineno, m.group(1))) + return hits + + +def read_baseline() -> dict[str, int]: + if not BASELINE.exists(): + return {} + out: dict[str, int] = {} + for raw in BASELINE.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + rel, _, count = line.rpartition("\t") + if not rel: + continue + try: + out[rel] = int(count) + except ValueError: + continue + return out + + +def write_baseline(hits: dict[str, list]) -> None: + lines = [ + "# memory-core-baseline.txt -- raw allocator call sites per file.", + "# Generated by scripts/lint-memory-core.py --write-baseline.", + "# A file may only ever go DOWN. Lower a line in the same change that", + "# migrates the file to src/foundation/mem_core.h; never raise one.", + "", + ] + for rel in sorted(hits): + lines.append(f"{rel}\t{len(hits[rel])}") + BASELINE.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--write-baseline", action="store_true") + ap.add_argument("--strict", action="store_true", help="fail when a file is BELOW its baseline") + ap.add_argument("--list", action="store_true", help="print every raw site") + args = ap.parse_args() + + hits = scan() + total = sum(len(v) for v in hits.values()) + + if args.write_baseline: + write_baseline(hits) + print(f"memory-core baseline written: {len(hits)} files, {total} raw sites") + return 0 + + if args.list: + for rel in sorted(hits): + for lineno, call in hits[rel]: + print(f"{rel}:{lineno}: {call}(") + print(f"-- {total} raw sites in {len(hits)} files") + return 0 + + baseline = read_baseline() + if not baseline: + print(f"ERROR: no baseline at {BASELINE.relative_to(ROOT)}; run --write-baseline", file=sys.stderr) + return 2 + + grew: list[str] = [] + shrank: list[str] = [] + for rel, sites in sorted(hits.items()): + now = len(sites) + allowed = baseline.get(rel, 0) + if now > allowed: + # Show the LAST sites: new code is usually appended, so these are the + # likeliest culprits. The linter cannot know which sites are new + # without a diff; --list prints them all. + delta = now - allowed + tail = ", ".join(f"{ln}:{c}" for ln, c in sites[-max(delta, 1):][-6:]) + grew.append(f" {rel}: grew by {delta} ({allowed} -> {now}); latest sites: {tail}" + f" [run --list for all]") + elif now < allowed: + shrank.append(f" {rel}: {now} (baseline {allowed}) -- lower the baseline") + for rel, allowed in sorted(baseline.items()): + if rel not in hits and allowed > 0: + shrank.append(f" {rel}: 0 (baseline {allowed}) -- remove the line") + + if grew: + print("memory-core linter FAILED: raw allocator use grew. Allocate through") + print("src/foundation/mem_core.h (cbm_alloc/cbm_calloc/cbm_realloc/cbm_free/") + print("cbm_mem_strdup) instead of malloc/calloc/realloc/free/strdup.") + print("\n".join(grew)) + if shrank: + print("memory-core ratchet: these files improved; tighten scripts/memory-core-baseline.txt:") + print("\n".join(shrank)) + if grew or (args.strict and shrank): + return 1 + print(f"memory-core linter passed: {total} raw sites across {len(hits)} files, none grew") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/memlab-drive.py b/scripts/memlab-drive.py index cef508810..bac474126 100644 --- a/scripts/memlab-drive.py +++ b/scripts/memlab-drive.py @@ -32,6 +32,16 @@ def rpc(proc, payload, timeout_note): return message +def tool_failed(reply): + """A JSON-RPC error, or a tool result the server marked isError: MCP reports + a failed tool call as a normal result, so checking only "error" counts every + failure as served (a lab measuring nothing looked healthy).""" + if "error" in reply: + return True + result = reply.get("result") + return isinstance(result, dict) and bool(result.get("isError")) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("binary") @@ -63,14 +73,16 @@ def main(): "params": {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "memlab", "version": "1"}}}, "initialize") + project = None if not args.skip_index: indexed = rpc(proc, {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "index_repository", - "arguments": {"path": args.corpus}}}, + "arguments": {"repo_path": args.corpus}}}, "index_repository") - if "error" in indexed: - print(f"index failed: {indexed['error']}", file=sys.stderr) + if tool_failed(indexed): + print(f"index failed: {indexed.get('error') or indexed.get('result')}", file=sys.stderr) return 3 + project = (indexed.get("result", {}).get("structuredContent") or {}).get("project") halfway = 2 + args.requests // 2 for i in range(2, args.requests + 2): @@ -84,11 +96,15 @@ def main(): "list_projects": {}, "get_graph_schema": {}, "search_code": {"pattern": "Widget"}}.get(args.tool, {}) + if project and args.tool != "list_projects": + arguments = dict(arguments, project=project) reply = rpc(proc, {"jsonrpc": "2.0", "id": i, "method": "tools/call", "params": {"name": args.tool, "arguments": arguments}}, f"request {i}") served += 1 - if "error" in reply: + if tool_failed(reply): + if failures == 0: + print(f"first failure (request {i}): {json.dumps(reply)[:600]}", file=sys.stderr, flush=True) failures += 1 finally: try: diff --git a/scripts/memlab-report.py b/scripts/memlab-report.py index cf538fa7d..eae4bccf8 100644 --- a/scripts/memlab-report.py +++ b/scripts/memlab-report.py @@ -1,19 +1,20 @@ #!/usr/bin/env python3 """Analyse #581 memory-lab output: attribute growth to allocation sites. -Reads the JSONL the profiler appends (CBM_MEM_PROFILE_OUT) and the mem.census -lines from a daemon log, and prints: +Reads the waste layer's dumps (CBM_MEMWASTE_OUT, one at every request-stage phase +mark; see scripts/memlab.sh) and the mem.census lines from a daemon log, and +prints: * retention per request, as a linear fit rather than first-vs-last, so noise does not masquerade as a trend * the call sites whose live bytes grow across the run, ranked by growth - * an explicit unattributed remainder: process growth the profiler could not - account for. That number is the honesty check — a report that attributed - everything while tracking only >=32 KiB allocations would be lying. + * an explicit unattributed remainder: process growth the layer could not + account for (allocator metadata, fragmentation, memory outside malloc). + That number is the honesty check. Usage: - memlab-report.py [--census ] [--top N] - memlab-report.py --diff # platform comparison + memlab-report.py [--census ] [--binary BIN] [--top N] + memlab-report.py --diff [--binary BIN] # platform comparison """ import argparse import json @@ -23,9 +24,11 @@ from collections import defaultdict def load_profile(path): - """Return (summaries, site_series). Records are appended over time, so the - Nth occurrence of a site is its Nth sample.""" - summaries, series = [], defaultdict(list) + """Return (summaries, site_series) for the process with the most dumps. + Every dump lists every site, so the Nth occurrence of a site is its Nth + sample; a site first seen late is padded with zeros so series align.""" + procs = defaultdict(lambda: {"summaries": [], "series": defaultdict(list)}) + cur = None with open(path, "r", errors="replace") as handle: for line in handle: line = line.strip() @@ -35,11 +38,31 @@ def load_profile(path): record = json.loads(line) except json.JSONDecodeError: continue - if "site" in record: - series[record["site"]].append(record) - else: - summaries.append(record) - return summaries, series + if "memwaste" in record: + cur = procs[record.get("pid", 0)] + cur["summaries"].append(record) + elif cur is not None and "site" in record and "work" not in record: + samples = cur["series"][record["site"]] + while len(samples) < len(cur["summaries"]) - 1: + samples.append({"live_bytes": 0}) + samples.append(record) + if not procs: + return [], {} + best = max(procs.values(), key=lambda p: len(p["summaries"])) + return best["summaries"], best["series"] + + +def load_symbols(binary, summaries, sites): + """Symbolise site addresses with the waste report's own symboliser.""" + if not binary or not summaries: + return {} + import importlib.util + import os + spec = importlib.util.spec_from_file_location( + "memwaste_report", os.path.join(os.path.dirname(os.path.abspath(__file__)), "memwaste-report.py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.symbolise(binary, summaries[-1].get("image_base", "0x0"), list(sites)) def linear_slope(values): @@ -57,7 +80,7 @@ def linear_slope(values): def parse_census(path): """Pull the pool totals out of mem.census log lines.""" - fields = ("priv_kb", "rss_kb", "crt_kb", "biggest_kb", "prof_live_kb") + fields = ("priv_kb", "rss_kb", "crt_kb", "biggest_kb") rows = [] pattern = re.compile(r"(\w+)=(\S+)") with open(path, "r", errors="replace") as handle: @@ -71,16 +94,16 @@ def parse_census(path): return rows -def report(path, census_path, top): +def report(path, census_path, top, binary=None): summaries, series = load_profile(path) if not summaries and not series: - print(f"{path}: no profiler records — was CBM_MEM_PROFILE=1 set?") + print(f"{path}: no waste-layer records -- was the memwaste flavour run with CBM_MEMWASTE=1?") return + names = load_symbols(binary, summaries, series.keys()) last = summaries[-1] if summaries else {} print(f"=== {path} ===") if last: - print(f"threshold : {last.get('threshold', 0)} bytes " - f"(allocations below this are invisible by design)") + print(f"samples : {len(summaries)} dumps (every allocation size is tracked)") print(f"sites seen : {last.get('sites', 0)}") print(f"attributed live: {last.get('live_bytes', 0)/1048576:.1f} MB " f"in {last.get('live_blocks', 0)} blocks") @@ -94,23 +117,23 @@ def report(path, census_path, top): slope = linear_slope(live) if slope <= 0 and live[-1] <= live[0]: continue - growth.append((slope, live[-1] - live[0], live[-1], samples[-1])) + growth.append((slope, live[-1] - live[0], live[-1], names.get(site, site))) growth.sort(key=lambda row: row[0], reverse=True) print(f"\n--- sites whose retained bytes GROW (top {top}) ---") if not growth: print("none — no tracked site retains more over time.") - for slope, delta, live, sample in growth[:top]: - frames = " <- ".join(sample.get("frames", [])[:4]) + for slope, delta, live, name in growth[:top]: print(f" +{slope/1024:8.1f} KB/sample total +{delta/1048576:6.2f} MB " f"live {live/1048576:6.2f} MB") - print(f" {frames}") + print(f" {name}") if census_path: rows = parse_census(census_path) if rows: priv = [r["priv_kb"] for r in rows if "priv_kb" in r] - prof = [r["prof_live_kb"] for r in rows if "prof_live_kb" in r] + # attributed = what the waste layer holds live, sampled at its own dumps + prof = [int(h.get("live_bytes", 0)) // 1024 for h in summaries] print(f"\n--- pools vs attribution ({len(rows)} samples) ---") if priv: print(f" private committed : {priv[0]/1024:.1f} -> {priv[-1]/1024:.1f} MB " @@ -119,24 +142,27 @@ def report(path, census_path, top): unattributed = (priv[-1] - priv[0]) - (prof[-1] - prof[0]) print(f" attributed growth : {(prof[-1]-prof[0])/1024:+.1f} MB") print(f" UNATTRIBUTED : {unattributed/1024:+.1f} MB " - "<- below-threshold allocations, allocator metadata, or fragmentation") + "<- allocator metadata, fragmentation, or memory outside malloc (mmap, stacks)") -def diff(path_a, path_b, top): - """Same sites, two platforms: what does one retain that the other does not?""" - _, series_a = load_profile(path_a) +def diff(path_a, path_b, top, binary=None): + """Same sites, two platforms: what does one retain that the other does not? + Raw addresses differ between binaries, so sites are matched by symbol name + when --binary is given (the same binary for both runs) and by address otherwise.""" + summaries_a, series_a = load_profile(path_a) _, series_b = load_profile(path_b) + names = load_symbols(binary, summaries_a, set(series_a) | set(series_b)) rows = [] for site, samples in series_a.items(): a_live = samples[-1]["live_bytes"] b_live = series_b[site][-1]["live_bytes"] if site in series_b else 0 if a_live - b_live > 0: - rows.append((a_live - b_live, a_live, b_live, samples[-1])) + rows.append((a_live - b_live, a_live, b_live, site)) rows.sort(key=lambda row: row[0], reverse=True) print(f"=== {path_a} minus {path_b} (top {top}) ===") - for delta, a_live, b_live, sample in rows[:top]: + for delta, a_live, b_live, site in rows[:top]: print(f" +{delta/1048576:7.2f} MB A={a_live/1048576:.2f} MB B={b_live/1048576:.2f} MB") - print(f" {' <- '.join(sample.get('frames', [])[:4])}") + print(f" {names.get(site, site)}") def main(): @@ -145,11 +171,12 @@ def main(): parser.add_argument("--census") parser.add_argument("--diff") parser.add_argument("--top", type=int, default=12) + parser.add_argument("--binary", help="the memwaste binary, for symbol names") args = parser.parse_args() if args.diff: - diff(args.profile, args.diff, args.top) + diff(args.profile, args.diff, args.top, args.binary) else: - report(args.profile, args.census, args.top) + report(args.profile, args.census, args.top, args.binary) return 0 diff --git a/scripts/memlab.sh b/scripts/memlab.sh index 373742556..d3136b43c 100644 --- a/scripts/memlab.sh +++ b/scripts/memlab.sh @@ -10,9 +10,14 @@ # # Usage: scripts/memlab.sh [requests] [label] # -# Output: memlab-