trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-11-06 08:35:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fix/510-enclosing-gitignore-not-inherited

Browse Source
This commit is contained in:
Martin Vogel
2026-09-25 19:07:17 +02:00
committed by GitHub
parent 282fbfcdab da3ea36339
commit 65fc0bb4f7
252 changed files with 32822 additions and 3408 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/_memwaste.yml
+183
View File
@@ -0,0 +1,183 @@
# Reusable: the waste sanitizer (memory + CPU waste), REPORT-ONLY.
#
# What runs (scripts/memwaste.sh, memwaste-scaling.py, memwaste-ratchet.py):
# mode=pr Linux: the event lane (memory waste + CPU work) indexing this
# repository at the checked-out commit, and the CPU scaling lane
# (src/ replicated 2x against 4x, per-function counter ratios).
# mode=full the same, plus the event lane on macOS and Windows and the access
# lane (read/write verdicts per allocation) on Linux.
#
# The corpus is the repository itself: pinned by the commit, no network, and a
# real mix of C, C++, Python, JavaScript and vendored code.
#
# REPORT-ONLY: every job uploads its report and prints the ratchet comparison
# against scripts/memwaste-baseline.json, but nothing here can turn a PR red --
# the ratchet runs without --enforce and the PR's ci-ok gate does not depend on
# these jobs. A job fails only when the MEASUREMENT is unsound (a table
# overflowed, frees the layer never saw): that is a defect of the tool, and a
# red job is how it gets noticed. Switching the ratchet to --enforce is a
# separate, deliberate change once the numbers have been stable across reruns.
name: Memwaste
on:
workflow_call:
inputs:
mode:
description: "pr (Linux event + scaling) or full (all platforms, access lane)"
type: string
default: pr
permissions:
contents: read
jobs:
event-linux:
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install deps
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
- name: Event lane on this repository
run: |
# `cmd | tee` reports TEE's status: without this a failing lane
# (a missing exec bit, a build error) passes the step and only
# surfaces as the ratchet not finding its JSON.
set -o pipefail
mkdir -p "$RUNNER_TEMP/memwaste-event"
scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \
| tee "$RUNNER_TEMP/memwaste-event/report.txt"
- name: Ratchet (report-only)
run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue ${{ inputs.mode }}-linux
- name: Upload report
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: memwaste-event-linux-${{ inputs.mode }}
path: |
${{ runner.temp }}/memwaste-event/report.txt
${{ runner.temp }}/memwaste-event/memwaste.json
${{ runner.temp }}/memwaste-event/index.json
if-no-files-found: warn
scaling-linux:
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install deps
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
- name: CPU scaling lane (src/ at 2x against 4x)
# Report-only: the script exits 1 when it flags a super-linear function.
continue-on-error: true
run: |
# `cmd | tee` reports TEE's status: without this a failing lane
# (a missing exec bit, a build error) passes the step and only
# surfaces as the ratchet not finding its JSON.
set -o pipefail
mkdir -p "$RUNNER_TEMP/memwaste-scaling"
python3 scripts/memwaste-scaling.py --corpus src --replicas 2 \
--out "$RUNNER_TEMP/memwaste-scaling" | tee "$RUNNER_TEMP/memwaste-scaling/report.txt"
- name: Upload report
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: memwaste-scaling-linux-${{ inputs.mode }}
path: ${{ runner.temp }}/memwaste-scaling/report.txt
if-no-files-found: warn
access-linux:
if: ${{ inputs.mode == 'full' }}
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install deps
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
- name: Access lane on this repository
run: |
# `cmd | tee` reports TEE's status: without this a failing lane
# (a missing exec bit, a build error) passes the step and only
# surfaces as the ratchet not finding its JSON.
set -o pipefail
mkdir -p "$RUNNER_TEMP/memwaste-access"
scripts/memwaste.sh "$GITHUB_WORKSPACE" --lane access --top 40 --sort untouched \
--out "$RUNNER_TEMP/memwaste-access" | tee "$RUNNER_TEMP/memwaste-access/report.txt"
- name: Upload report
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: memwaste-access-linux-${{ inputs.mode }}
path: |
${{ runner.temp }}/memwaste-access/report.txt
${{ runner.temp }}/memwaste-access/memwaste.json
if-no-files-found: warn
event-macos:
if: ${{ inputs.mode == 'full' }}
runs-on: macos-14
timeout-minutes: 60
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Event lane on this repository
run: |
# `cmd | tee` reports TEE's status: without this a failing lane
# (a missing exec bit, a build error) passes the step and only
# surfaces as the ratchet not finding its JSON.
set -o pipefail
mkdir -p "$RUNNER_TEMP/memwaste-event"
scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \
| tee "$RUNNER_TEMP/memwaste-event/report.txt"
- name: Ratchet (report-only)
run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue full-macos
- name: Upload report
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: memwaste-event-macos-${{ inputs.mode }}
path: |
${{ runner.temp }}/memwaste-event/report.txt
${{ runner.temp }}/memwaste-event/memwaste.json
if-no-files-found: warn
event-windows:
if: ${{ inputs.mode == 'full' }}
runs-on: windows-latest
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2
with:
msystem: CLANG64
path-type: inherit
install: >-
mingw-w64-clang-x86_64-clang
mingw-w64-clang-x86_64-llvm
mingw-w64-clang-x86_64-zlib
mingw-w64-clang-x86_64-python3
make
git
coreutils
- name: Event lane on this repository
shell: msys2 {0}
run: |
# `cmd | tee` reports TEE's status: without this a failing lane
# (a missing exec bit, a build error) passes the step and only
# surfaces as the ratchet not finding its JSON.
set -o pipefail
out="$(cygpath -u "$RUNNER_TEMP")/memwaste-event"
mkdir -p "$out"
CC=clang CXX=clang++ scripts/memwaste.sh "$(cygpath -u "$GITHUB_WORKSPACE")" --top 40 --out "$out" \
| tee "$out/report.txt"
- name: Ratchet (report-only)
shell: msys2 {0}
run: python3 scripts/memwaste-ratchet.py --out "$(cygpath -u "$RUNNER_TEMP")/memwaste-event" --venue full-windows
- name: Upload report
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: memwaste-event-windows-${{ inputs.mode }}
path: |
${{ runner.temp }}/memwaste-event/report.txt
${{ runner.temp }}/memwaste-event/memwaste.json
if-no-files-found: warn
.github/workflows/_security.yml
+42 -2
View File
@@ -41,6 +41,14 @@ jobs:
codeql-gate:
runs-on: ubuntu-latest
timeout-minutes: 240
# Without security-events:read this job inherits the workflow's
# contents:read, the code-scanning alert API answers 403, and the gate
# below reported "0 alerts" on every run it has ever made. actions:read is
# what the CodeQL-run poll above needs for the same reason.
permissions:
contents: read
security-events: read
actions: read
steps:
- name: Wait for CodeQL on current commit (max 150 min)
env:
@@ -77,11 +85,43 @@ jobs:
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# FAIL CLOSED. The previous form was `gh api ... 2>/dev/null || echo "0"`,
# which looks defensive and is the opposite. gh writes the API error
# BODY to stdout, so a 403 made ALERTS the string
# {"message":"Resource not accessible by integration",...,"status":"403"}0
# every `[` comparison then failed as non-integer, the `if` took its
# false branch, and the step printed "CodeQL gate passed (0 alerts)"
# and exited 0. That is exactly how a security gate should NEVER fail:
# silently, in the direction of "everything is fine". An unreadable
# alert list is now an error, not a zero.
# NOTE: this runs in a command substitution, so it must `return`, not
# `exit` -- an `exit` here would only end the subshell and let the
# caller continue with an empty count, recreating the very bug this
# replaces. The callers below check the status explicitly.
count_open_alerts() {
local body status
body=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' \
--jq 'length' 2>/dev/null)
status=$?
if [ $status -ne 0 ]; then
echo "BLOCKED: cannot read code scanning alerts (gh exit $status)." >&2
echo " Refusing to treat an unreadable alert list as zero." >&2
echo " Most likely cause: the job lacks security-events:read." >&2
return 1
fi
case "$body" in
''|*[!0-9]*)
echo "BLOCKED: alert count was not a number: ${body:-<empty>}" >&2
return 1
;;
esac
printf '%s' "$body"
}
echo "Waiting 60s for alert API to settle..."
sleep 60
ALERTS=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0")
ALERTS=$(count_open_alerts) || exit 1
sleep 15
ALERTS2=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0")
ALERTS2=$(count_open_alerts) || exit 1
[ "$ALERTS" -lt "$ALERTS2" ] && ALERTS=$ALERTS2
if [ "$ALERTS" -gt 0 ]; then
echo "BLOCKED: $ALERTS open alert(s)"
.github/workflows/_test.yml
+26
View File
@@ -107,6 +107,32 @@ jobs:
if: startsWith(matrix.os, 'ubuntu')
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev ccache
# Ubuntu 23.10+ ships kernel.apparmor_restrict_unprivileged_userns=1, which
# blocks unshare(CLONE_NEWUSER) for unprivileged users. Without this, the
# #1830 single-uid-userns smoke test SKIPS on every 24.04 leg, and its only
# real-namespace coverage lived on the two ubuntu-22.04 legs -- which are
# broad-matrix only (never PR CI) and which GitHub retires on 2027-04-17.
# A deliberate relaxation of a security check was therefore guarded by a
# test that ran in no PR and was on a clock. Hosted runners have
# passwordless sudo, so one sysctl puts it on the CORE matrix instead.
# Best-effort: a runner that refuses simply returns to skipping.
- name: Permit unprivileged user namespaces (Ubuntu)
if: startsWith(matrix.os, 'ubuntu')
run: |
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || \
echo "could not lift the userns restriction; namespace tests will skip"
# Then SAY whether it worked. The runner prints per-shard aggregates
# ("N passed, M skipped") and never individual test names, so a
# namespace test that silently returns to skipping is invisible in a
# green log -- the same false assurance that let #1830 ship with a
# test that never executed. This line is the evidence. Probed exactly
# as the test does it: unprivileged, no sudo.
if unshare -U -r true 2>/dev/null; then
echo "CBM_USERNS_AVAILABLE=yes"
else
echo "CBM_USERNS_AVAILABLE=no (namespace-dependent tests will skip)"
fi
- name: Install ccache (macOS)
if: startsWith(matrix.os, 'macos')
run: command -v ccache >/dev/null 2>&1 || brew install ccache
.github/workflows/codeql.yml
+2 -2
View File
@@ -29,7 +29,7 @@ jobs:
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev
- name: Initialize CodeQL
uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
with:
languages: c-cpp
build-mode: manual
@@ -38,6 +38,6 @@ jobs:
run: scripts/build.sh
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
with:
category: "/language:c-cpp"
.github/workflows/dry-run.yml
+43
View File
@@ -45,6 +45,10 @@ on:
description: 'Skip candidate scan; mark unscanned and default to stripped'
type: boolean
default: false
skip_memwaste:
description: 'Skip the waste sanitizer (report-only) and the fuzz smoke'
type: boolean
default: false
soak_level:
description: 'Soak: full (quick+asan), quick (10min), none'
type: choice
@@ -58,6 +62,13 @@ jobs:
# ── Security (independent island — does not block main pipeline) ──
security:
uses: ./.github/workflows/_security.yml
# The called job requests security-events:read to count code-scanning
# alerts; a reusable workflow cannot request more than its caller grants,
# so withholding this here fails the whole run at startup.
permissions:
contents: read
security-events: read
actions: read
secrets: inherit
# ── Lint (cppcheck + clang-format) ────────────────────────────
@@ -100,6 +111,38 @@ jobs:
with:
broad_platforms: true
# ── Waste sanitizer, REPORT-ONLY (independent island) ─────────
# Event lane on Linux, macOS and Windows, access lane and the per-function
# scaling lane on Linux. Uploads reports; fails only when the measurement is
# unsound (a defect of the tool itself).
memwaste:
if: ${{ inputs.skip_memwaste != true }}
uses: ./.github/workflows/_memwaste.yml
with:
mode: full
# ── Fuzz smoke (independent island) ───────────────────────────
# A fixed number of executions from a fixed seed per target, so the verdict is
# a function of the commit (O9). A crash here is a real defect in a parser
# that reads untrusted input; its reproducer is uploaded.
fuzz:
if: ${{ inputs.skip_memwaste != true }}
runs-on: ubuntu-latest
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install deps
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev clang llvm libclang-rt-dev
- name: Fuzz smoke (extract, cypher, config)
run: scripts/fuzz.sh all --runs 20000 --seed 1
- name: Upload reproducers
if: ${{ failure() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: fuzz-artifacts
path: build/fuzz/artifacts-*/
if-no-files-found: ignore
# ── Soak tests (optional, parallel with smoke) ────────────────
soak:
if: ${{ inputs.soak_level != 'none' && !cancelled() && needs.build.result != 'failure' && needs.build.result != 'skipped' }}
.github/workflows/nightly-soak.yml
+21
View File
@@ -35,6 +35,27 @@ permissions:
contents: read
jobs:
# Fuzz exploration: new inputs are the point here, so it runs for a time
# budget instead of a fixed count. NON-GATING like the soak; a crash uploads
# its reproducer, which becomes a fixed-seed regression input once fixed.
fuzz:
if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install deps
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev clang llvm libclang-rt-dev
- name: Fuzz exploration (30 min per target)
run: scripts/fuzz.sh all --max-total-time 1800 --seed 0
- name: Upload reproducers
if: ${{ failure() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: fuzz-artifacts-nightly
path: build/fuzz/artifacts-*/
if-no-files-found: ignore
soak:
uses: ./.github/workflows/_soak.yml
with:
.github/workflows/pr.yml
+40 -2
View File
@@ -22,6 +22,13 @@ concurrency:
jobs:
security:
uses: ./.github/workflows/_security.yml
# The called job requests security-events:read to count code-scanning
# alerts; a reusable workflow cannot request more than its caller grants,
# so withholding this here fails the whole run at startup.
permissions:
contents: read
security-events: read
actions: read
secrets: inherit
lint:
@@ -57,9 +64,29 @@ jobs:
run: |
# The full .diff endpoint rejects large-but-valid PRs at 20k lines.
# The paginated files endpoint remains filename-only for this gate.
FILES=$(gh api --paginate "repos/$REPO/pulls/$PR/files?per_page=100" --jq '.[].filename')
if FILES=$(gh api --paginate "repos/$REPO/pulls/$PR/files?per_page=100" --jq '.[].filename'); then
echo "file list: pulls/files API"
else
# The files endpoint renders the diff server-side too, and answers
# 422 "this diff is taking too long to generate" for a PR that
# regenerates a vendored parser (#2246: a 42 MB sql/parser.c) --
# so does the compare endpoint. The PR merge ref's first parent is
# the base, so a name-only diff across it is the same file list.
# depth 2 + blob:none fetches commits and trees only (well under
# 1 MB, under a second); nothing from the PR is checked out or run.
echo "::notice::pulls/files API failed; file list taken from refs/pull/$PR/merge"
CHANGES_GIT="$RUNNER_TEMP/changes.git"
git init -q --bare "$CHANGES_GIT"
git --git-dir="$CHANGES_GIT" fetch -q --depth=2 --filter=blob:none \
"https://github.com/$REPO" "refs/pull/$PR/merge"
FILES=$(git --git-dir="$CHANGES_GIT" diff --name-only --no-renames FETCH_HEAD^1 FETCH_HEAD)
fi
printf '%s\n' "$FILES"
if printf '%s\n' "$FILES" | grep -qE '^(src/|internal/|install\.(sh|ps1)|scripts/build\.sh|scripts/smoke-test\.sh|scripts/smoke-local\.sh|scripts/smoke-fixture-server\.py|scripts/gen-third-party-notices\.sh|scripts/env\.sh|test-infrastructure/vm/(vm-smoke\.sh|windows-user-path-guard\.ps1)|Makefile\.cbm)'; then
# A here-string, not a pipe: under this shell's pipefail, grep -q
# exits on its first match, printf dies of SIGPIPE once the list
# outgrows the pipe buffer, and the MATCH is then reported as a
# failure -- product=false, smoke skipped, on the largest PRs.
if grep -qE '^(src/|internal/|install\.(sh|ps1)|scripts/build\.sh|scripts/smoke-test\.sh|scripts/smoke-local\.sh|scripts/smoke-fixture-server\.py|scripts/gen-third-party-notices\.sh|scripts/env\.sh|test-infrastructure/vm/(vm-smoke\.sh|windows-user-path-guard\.ps1)|Makefile\.cbm)' <<<"$FILES"; then
echo "product=true" >> "$GITHUB_OUTPUT"
else
echo "product=false" >> "$GITHUB_OUTPUT"
@@ -147,6 +174,17 @@ jobs:
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 1000M
# ── Waste sanitizer, REPORT-ONLY: memory + CPU waste on this repository and
# the per-function scaling lane (Linux). Deliberately NOT in ci-ok's needs:
# it uploads evidence and prints the ratchet, it never blocks a merge. Only
# product-code PRs pay for it. ──
memwaste:
needs: [changes]
if: ${{ !cancelled() && needs.changes.outputs.product == 'true' }}
uses: ./.github/workflows/_memwaste.yml
with:
mode: pr
ci-ok:
# The one required context (besides dco) — fails unless every PR stage
# succeeded, so matrix renames can never silently deadlock merges. `skipped`
.github/workflows/release.yml
+16
View File
@@ -44,6 +44,13 @@ jobs:
# ── Security (starts immediately, blocks only verify) ───────────
security:
uses: ./.github/workflows/_security.yml
# The called job requests security-events:read to count code-scanning
# alerts; a reusable workflow cannot request more than its caller grants,
# so withholding this here fails the whole run at startup.
permissions:
contents: read
security-events: read
actions: read
secrets: inherit
# ── 0. Preflight: refuse malformed dispatch inputs ──────────────
@@ -138,6 +145,15 @@ jobs:
version: ${{ inputs.version }}
use_release_artifacts: true
# ── Waste sanitizer, REPORT-ONLY ────────────────────────────────
# The same full run as the dry run, on the release commit. Not in
# release-draft's needs: the report is evidence for the release notes, never
# a publication gate.
memwaste:
uses: ./.github/workflows/_memwaste.yml
with:
mode: full
# ── 6. Create DRAFT release ────────────────────────────────────
# Requires smoke to have actually SUCCEEDED, not merely "not failed". The bare
# !cancelled() && !failure() form is fail-OPEN: a skipped smoke is neither
.github/workflows/scorecard.yml
+1 -1
View File
@@ -27,6 +27,6 @@ jobs:
publish_results: true
- name: Upload SARIF results
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4
with:
sarif_file: results.sarif
.github/workflows/smoke.yml
+1 -1
View File
@@ -1,7 +1,7 @@
# Smoke invariants across the WIDEST set of GitHub-hosted runners. Builds the
# production source path with explicit fault-injection test seams and runs
# scripts/smoke-invariants.sh
# (version/help, MCP initialize handshake [#513], all 14 tools invocable, malformed-
# (version/help, MCP initialize handshake [#513], all registered tools invocable, malformed-
# input resilience, clean EOF exit, shared-lib resolution, install dry-run).
#
# Maximizing platforms is the point: ubuntu-22.04 (older glibc → AlmaLinux/#182
CONTRIBUTING.md
+20 -1
View File
@@ -55,7 +55,7 @@ src/
foundation/ Arena allocator, hash table, string utils, platform compat
store/ SQLite graph storage (WAL mode, FTS5)
cypher/ Cypher query → SQL translation
mcp/ MCP server (JSON-RPC 2.0 over stdio, 14 tools)
mcp/ MCP server (JSON-RPC 2.0 over stdio, 17 tools)
pipeline/ Multi-pass indexing pipeline
pass_*.c Individual pipeline passes (definitions, calls, usages, etc.)
httplink.c HTTP route extraction (Go/Express/Laravel/Ktor/Python)
@@ -149,6 +149,25 @@ If in doubt, open an issue and ask.
- Run `scripts/test.sh` and `scripts/lint.sh` before submitting
- Keep PRs focused — avoid unrelated reformatting or refactoring
## AI-assisted and AI-authored contributions
Contributions written with the help of an AI tool, or written by an AI agent acting for a person, are welcome here on the same terms as any other. What we review is the change, not who or what typed it. Three things make that work:
- **Say so.** If an agent wrote the PR or is answering in the thread, disclose it — a line in the PR description or at the top of a comment is enough ("written with X", "this account is operated by an AI agent on behalf of Y"). Disclosure is never held against a contribution. Finding out later is.
- **A person is accountable.** The [DCO sign-off](#license-and-sign-off-dco--required-on-every-commit) is a legal certification, and only a person can make it. The human behind the account certifies every commit, answers for it, and is who we talk to if something goes wrong. An agent may carry the conversation; it cannot carry the responsibility.
- **The bar does not move.** Every claim in a PR must be something a reviewer can check: the failing test, the command that was run, the numbers and the machine they came from. "The model says it works" is not evidence, and a confident description does not replace a test that fails without the fix. We read AI-written PRs exactly as carefully as human-written ones — no more suspiciously, and no less.
Two practical notes:
- `Co-authored-by:` trailers naming a tool are fine. Links to private chat or agent sessions are not — they are dead to everyone else and do not belong in permanent history; we will ask you to remove them.
- High-volume, low-effort submissions — many near-identical PRs, changes nobody ran, replies that do not engage with the review — will be closed, whoever or whatever produced them. One careful PR is worth more than twenty generated ones.
### Who speaks for the project
Only the maintainer account ([@DeusData](https://github.com/DeusData)) speaks for this project. A review, an approval, a "this will be merged" or a "this is a duplicate" from any other account is that person's opinion, however official it sounds — helpful reviews from the community are very welcome, and that is what they are.
**Impersonating a maintainer is not acceptable**, by a person or by an agent: presenting yourself as part of the maintainer team, posting approvals or change requests styled as maintainer decisions, or telling other contributors what the project has decided. We dismiss such reviews, hide the comments, say publicly in the affected threads that they were not ours, and report and block the account. If you are unsure whether a response you received is from the project, check the account name — and ask.
## Security
We take security seriously. All PRs go through:
Makefile.cbm
+157 -20
View File
@@ -76,14 +76,15 @@ endif
# the prod build uses mimalloc (MI_OVERRIDE=1); the test build is CRT+ASan, where
# binding would create an alloc/free mismatch, so the guard is prod-only.
CFLAGS_PROD = $(CFLAGS_COMMON) -O2 -DCBM_BIND_TS_ALLOCATOR=1 $(TEST_SEAM_DEFINE) \
$(GLOBAL_OVERRIDE_DEFINE) $(CFLAGS_EXTRA)
CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2
$(GLOBAL_OVERRIDE_DEFINE) $(MEMWASTE_CFLAGS) $(CFLAGS_EXTRA)
CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 $(MEMWASTE_VENDOR_CFLAGS)
# Test flags: debug + sanitizers (override SANITIZE= to disable on Windows)
SANITIZE = -fsanitize=address,undefined -fno-omit-frame-pointer
EDITOR_TEST_DEFINES = -DCBM_JSON_LIKE_ENABLE_TEST_API=1 \
-DCBM_TOML_EDIT_ENABLE_TEST_API=1 -DCBM_YAML_ENABLE_TEST_API=1 \
-DCBM_TEXT_EDIT_ENABLE_TEST_API=1 -DCBM_CLI_ENABLE_TEST_API=1 \
-DCBM_CONFIG_EDIT_PATH_ENABLE_TEST_API=1 \
-DCBM_DIAGNOSTICS_ENABLE_TEST_API=1 -DCBM_ENABLE_TEST_SEAMS=1
# The build system is the single source of truth for "is this binary
# instrumented": compiler-specific probes (__SANITIZE_ADDRESS__) miss
@@ -215,6 +216,75 @@ ELF_HARDENING_FLAGS := -Wl,-z,noexecstack -Wl,-z,separate-code \
endif
endif
# MEMWASTE=1: the waste-sanitizer flavour (src/foundation/mem_events.h). Built
# like production -- -O2, mimalloc, the real allocator routing -- plus the event
# layer and its observers, because waste measured under ASan's allocator would
# describe ASan. Frame pointers stay so the macOS observer can walk to the
# caller. On Linux this is also what finally LINKS the --wrap observation shim:
# its flags and source were defined here but never reached a link line, so the
# Linux profiler had been reporting nothing.
#
# Every object is built this way, VENDORED ones included (grammars, the
# tree-sitter runtime, SQLite, tre, lz4, zstd, the C++ preprocessor): waste in
# a vendored parser is waste in this program. mimalloc alone is left as it is --
# its reads and writes are the allocator's own bookkeeping, not program data.
#
# MEMWASTE_ACCESS=1 adds the access lane (reads and writes of every tracked
# block through the compiler's load/store callbacks; clang only). It builds at
# -O1 without vectorisation: at -O2 the callbacks describe the OPTIMISED program,
# and reads the optimiser removed would be reported as never read.
# MEMWASTE_PROFILE=1 adds clang's execution counters (-fprofile-instr-generate)
# for the CPU scaling lane (scripts/memwaste-scaling.py).
# Each flavour needs its own BUILD_DIR: vendored objects are not rebuilt when
# only the flavour changes.
#
# The libc wrappers (mem_override_libc.c) need --wrap on Linux and Windows;
# macOS defines the symbols instead. STATIC=1 (musl) is not a memwaste target.
MEMWASTE ?= 0
MEMWASTE_ACCESS ?= 0
MEMWASTE_PROFILE ?= 0
ifeq ($(MEMWASTE_ACCESS),1)
override MEMWASTE := 1
endif
ifeq ($(MEMWASTE_PROFILE),1)
override MEMWASTE := 1
endif
MEMWASTE_CFLAGS :=
MEMWASTE_VENDOR_CFLAGS :=
MEMWASTE_LDFLAGS :=
MEMWASTE_SRCS :=
MEMWASTE_ALLOC_SYMS_LINUX := malloc calloc realloc free strdup strndup posix_memalign \
aligned_alloc memalign
MEMWASTE_LIBC_SYMS_LINUX := memcpy memmove memset memcmp strlen strcmp strncmp \
__memcpy_chk __memmove_chk __memset_chk read __read_chk write \
pread pwrite pread64 pwrite64 open open64 __open_2 __open64_2 \
stat lstat fopen fopen64 fread __fread_chk fwrite opendir readdir \
pthread_mutex_lock
MEMWASTE_LIBC_SYMS_WIN := memcpy memmove memset memcmp strlen strcmp strncmp _wfopen fread fwrite
ifeq ($(MEMWASTE),1)
MEMWASTE_CFLAGS := -DCBM_MEMWASTE=1 -fno-omit-frame-pointer
MEMWASTE_VENDOR_CFLAGS := -DCBM_MEMWASTE=1 -fno-omit-frame-pointer
ifeq ($(IS_LINUX),yes)
MEMWASTE_LDFLAGS := $(foreach sym,$(MEMWASTE_ALLOC_SYMS_LINUX) $(MEMWASTE_LIBC_SYMS_LINUX),-Wl,--wrap=$(sym))
MEMWASTE_SRCS := src/foundation/mem_override_posix.c
endif
ifeq ($(IS_MINGW),yes)
MEMWASTE_LDFLAGS := $(foreach sym,$(MEMWASTE_LIBC_SYMS_WIN),-Wl,--wrap=$(sym))
endif
endif
ifeq ($(MEMWASTE_ACCESS),1)
MEMWASTE_ACCESS_FLAGS := -DCBM_MEMWASTE_ACCESS=1 -O1 -fno-vectorize -fno-slp-vectorize \
-fsanitize-coverage=inline-bool-flag,trace-loads,trace-stores
MEMWASTE_CFLAGS += $(MEMWASTE_ACCESS_FLAGS)
MEMWASTE_VENDOR_CFLAGS += $(MEMWASTE_ACCESS_FLAGS)
endif
ifeq ($(MEMWASTE_PROFILE),1)
MEMWASTE_PROFILE_FLAGS := -DCBM_MEMWASTE_PROFILE=1 -fprofile-instr-generate -fprofile-update=atomic
MEMWASTE_CFLAGS += $(MEMWASTE_PROFILE_FLAGS)
MEMWASTE_VENDOR_CFLAGS += $(MEMWASTE_PROFILE_FLAGS)
MEMWASTE_LDFLAGS += $(MEMWASTE_PROFILE_FLAGS)
endif
# The POSIX wrap shim exists only so the profiler can observe allocations. It
# must never reach a sanitized build: the Linux/macOS test builds are CRT+ASan,
# and redirecting malloc into mimalloc underneath ASan's own interception mixes
@@ -224,7 +294,7 @@ endif
# instead of the system libstdc++ (scripts/msan.sh overrides both vars).
CXX_STDLIB ?= -lstdc++
CXX_STDLIB_FLAGS ?=
LDFLAGS = -lm $(CXX_STDLIB) -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS)
LDFLAGS = -lm $(CXX_STDLIB) -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) $(MIMALLOC_WRAP_FLAGS) $(MEMWASTE_LDFLAGS) $(ELF_HARDENING_FLAGS)
LDFLAGS_TEST = -lm $(CXX_STDLIB) -lpthread -lz $(SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS)
LDFLAGS_TSAN = -lm $(CXX_STDLIB) -lpthread -lz $(TSAN_SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS)
@@ -247,10 +317,15 @@ FOUNDATION_SRCS = \
src/foundation/compat_fs.c \
src/foundation/compat_regex.c \
src/foundation/mem.c \
src/foundation/mem_core.c \
src/foundation/mem_events.c \
src/foundation/mem_override_darwin.c \
src/foundation/mem_override_libc.c \
src/foundation/diagnostics.c \
src/foundation/profile.c \
src/foundation/dump_verify.c \
src/foundation/limits.c \
src/foundation/index_policy.c \
src/foundation/subprocess.c \
src/foundation/sha256.c \
src/foundation/secure_random.c \
@@ -274,6 +349,8 @@ EXTRACTION_SRCS = \
$(CBM_DIR)/extract_k8s.c \
$(CBM_DIR)/extract_dbt.c \
$(CBM_DIR)/helpers.c \
$(CBM_DIR)/result_compact.c \
$(CBM_DIR)/result_spill.c \
$(CBM_DIR)/lang_specs.c \
$(CBM_DIR)/macro_table.c \
$(CBM_DIR)/iris_export_xml.c \
@@ -372,6 +449,7 @@ PIPELINE_SRCS = \
src/pipeline/pass_gitdiff.c \
src/pipeline/pass_configures.c \
src/pipeline/pass_configlink.c \
src/pipeline/pass_doclinks.c \
src/pipeline/pass_route_nodes.c \
src/pipeline/pass_enrichment.c \
src/pipeline/pass_envscan.c \
@@ -409,7 +487,7 @@ GIT_SRCS = src/git/git_context.c
CLI_SRCS = src/cli/cli.c src/cli/progress_sink.c src/cli/hook_augment.c src/cli/client_adapter.c \
src/cli/agent_clients.c src/cli/agent_profiles.c \
src/cli/config_json_like.c src/cli/config_toml_edit.c src/cli/config_yaml_edit.c \
src/cli/config_text_edit.c src/cli/activation_transaction.c
src/cli/config_text_edit.c src/cli/config_edit_path.c src/cli/activation_transaction.c
# UI module (graph visualization)
UI_SRCS = \
@@ -468,10 +546,19 @@ ifneq ($(filter yes,$(IS_MINGW) $(IS_LINUX)),)
MIMALLOC_OVERRIDE_DEFINE := -DMI_MALLOC_OVERRIDE=1
GLOBAL_OVERRIDE_DEFINE := -DCBM_MEM_GLOBAL_OVERRIDE=1
endif
# MI_DEBUG=0: the production allocator is mimalloc's RELEASE build. Its header
# defaults to MI_DEBUG=2 unless NDEBUG is defined, and nothing here defined it,
# so every shipped binary up to v0.11.0 ran mimalloc's debug build: internal
# assertions, fine-grained statistics, padding on every block and a memset of
# every freed block. The waste sanitizer measured it on the Go corpus as 92 GB
# (macOS) and 444 GB (Linux, where mimalloc also serves libc's malloc) of byte
# work inside the allocator (2026-09-17). Committed-bytes accounting, which the
# memory budget reads, is kept in every build (subproc stats, not MI_STAT).
# The TEST build below keeps mimalloc's debug checks.
MIMALLOC_CFLAGS = -std=c11 -O2 -w \
-Ivendored/mimalloc/include \
-Ivendored/mimalloc/src \
-DMI_OVERRIDE=1 \
-DMI_OVERRIDE=1 -DMI_DEBUG=0 \
$(MIMALLOC_OVERRIDE_DEFINE)
MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \
-Ivendored/mimalloc/include \
@@ -492,7 +579,7 @@ MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \
SQLITE3_OMIT_FLAGS = -DSQLITE_OMIT_LOAD_EXTENSION
SQLITE3_SRC = vendored/sqlite3/sqlite3.c
SQLITE3_CFLAGS = -std=c11 -O2 -w -DSQLITE_DQS=0 -DSQLITE_THREADSAFE=1 -DSQLITE_ENABLE_FTS5 \
$(SQLITE3_OMIT_FLAGS)
$(SQLITE3_OMIT_FLAGS) $(MEMWASTE_VENDOR_CFLAGS)
SQLITE3_CFLAGS_TEST = -std=c11 -g -O1 -w -DSQLITE_DQS=0 -DSQLITE_THREADSAFE=1 -DSQLITE_ENABLE_FTS5 \
$(SQLITE3_OMIT_FLAGS)
@@ -504,7 +591,7 @@ TRE_CFLAGS = -std=c11 -g -O1 -w -Ivendored/tre
YYJSON_SRC = vendored/yyjson/yyjson.c
# All production sources
PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)
PROD_SRCS = $(MEMWASTE_SRCS) $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)
EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \
$(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC)
@@ -532,6 +619,7 @@ TEST_FOUNDATION_SRCS = \
tests/test_str_intern.c \
tests/test_log.c \
tests/test_str_util.c \
tests/test_index_policy.c \
tests/test_workspace.c \
tests/test_platform.c \
tests/test_diagnostics.c \
@@ -588,7 +676,7 @@ TEST_DISCOVER_SRCS = \
TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c
TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c
TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c
TEST_WATCHER_SRCS = tests/test_watcher.c
@@ -637,7 +725,7 @@ TEST_CLI_SRCS = tests/test_cli.c tests/test_agent_clients.c tests/test_agent_pro
tests/test_config_toml_edit.c tests/test_config_yaml_edit.c tests/test_config_text_edit.c \
tests/test_activation_transaction.c
TEST_MEM_SRCS = tests/test_mem.c
TEST_MEM_SRCS = tests/test_mem.c tests/test_mem_events.c
TEST_UI_SRCS = tests/test_ui.c
TEST_HTTPD_SRCS = tests/test_httpd.c
@@ -742,7 +830,8 @@ BUILD_DIR = build/c
# ── Object file compilation (grammars need relaxed warnings) ─────
# Grammar + tree-sitter runtime: compiled without -Werror (upstream code has warnings)
GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC)
GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
$(MEMWASTE_VENDOR_CFLAGS)
GRAMMAR_CFLAGS_TEST = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
-DCBM_ENABLE_TEST_SEAMS=1 $(SANITIZED_DEFINE) $(SANITIZE)
GRAMMAR_CFLAGS_TSAN = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
@@ -760,7 +849,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o
# ── Targets ──────────────────────────────────────────────────────
.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security
.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz
$(BUILD_DIR):
mkdir -p $(BUILD_DIR)
@@ -775,7 +864,24 @@ test-foundation: $(BUILD_DIR)/test-foundation
# ── Grammar/TS/LSP object files (compiled with relaxed warnings) ─
$(BUILD_DIR)/%.o: $(CBM_DIR)/%.c | $(BUILD_DIR)
# Each grammar_<lang>.c is a unity object that #includes its own vendored
# parser.c and, where the grammar has one, scanner.c. Those includes were
# prerequisites of nothing, so editing a vendored scanner left the object
# untouched and the edit silently absent from the binary: on 2026-09-19 a fix
# to properties/scanner.c appeared not to work at all, and `nm` showed the old
# symbol still sitting in prod_grammar_properties.o. MANIFEST.md's "Local
# source patches" table means patching a vendored scanner is expected, so this
# trap is reachable by design. Same reasoning as LSP_UNITY_DEPS and
# TS_RUNTIME_DEPS above; secondary expansion is what lets one pattern rule name
# each grammar's own directory ($$* is the stem, grammar_properties ->
# properties). Non-grammar objects match no directory and gain no prerequisite.
GRAMMAR_VENDOR_DIR = $(CBM_DIR)/vendored/grammars/$$(subst grammar_,,$$*)
GRAMMAR_UNITY_DEPS = $$(wildcard $(GRAMMAR_VENDOR_DIR)/*.c) \
$$(wildcard $(GRAMMAR_VENDOR_DIR)/*.h) \
$$(wildcard $(GRAMMAR_VENDOR_DIR)/tree_sitter/*.h)
.SECONDEXPANSION:
$(BUILD_DIR)/%.o: $(CBM_DIR)/%.c $(GRAMMAR_UNITY_DEPS) | $(BUILD_DIR)
$(CC) $(GRAMMAR_CFLAGS_TEST) -c -o $@ $<
$(BUILD_DIR)/ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR)
@@ -787,7 +893,7 @@ $(BUILD_DIR)/lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)
$(BUILD_DIR)/preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
$(CXX) $(CXXFLAGS_TEST) -w -I$(CBM_DIR)/vendored -c -o $@ $<
$(BUILD_DIR)/tsan_%.o: $(CBM_DIR)/%.c | $(BUILD_DIR)
$(BUILD_DIR)/tsan_%.o: $(CBM_DIR)/%.c $(GRAMMAR_UNITY_DEPS) | $(BUILD_DIR)
$(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $<
$(BUILD_DIR)/tsan_ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR)
@@ -864,7 +970,7 @@ $(BUILD_DIR)/tsan_tre.o: $(TRE_SRC) | $(BUILD_DIR)
$(CC) $(TRE_CFLAGS) $(TSAN_SANITIZE) -c -o $@ $<
$(BUILD_DIR)/prod_tre.o: $(TRE_SRC) | $(BUILD_DIR)
$(CC) $(TRE_CFLAGS) -O2 -c -o $@ $<
$(CC) $(TRE_CFLAGS) -O2 $(MEMWASTE_VENDOR_CFLAGS) -c -o $@ $<
endif
# Vendored LZ4 (test build)
@@ -1017,7 +1123,7 @@ TS_RUNTIME_OBJ_PROD = $(BUILD_DIR)/prod_ts_runtime.o
LSP_OBJ_PROD = $(BUILD_DIR)/prod_lsp_all.o
PP_OBJ_PROD = $(BUILD_DIR)/prod_preprocessor.o
$(BUILD_DIR)/prod_%.o: $(CBM_DIR)/%.c | $(BUILD_DIR)
$(BUILD_DIR)/prod_%.o: $(CBM_DIR)/%.c $(GRAMMAR_UNITY_DEPS) | $(BUILD_DIR)
$(CC) $(GRAMMAR_CFLAGS) -c -o $@ $<
$(BUILD_DIR)/prod_ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR)
@@ -1032,14 +1138,14 @@ $(BUILD_DIR)/prod_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
# Vendored LZ4 (compiled separately, not unity-built via lz4_store.c)
LZ4_OBJ_PROD = $(BUILD_DIR)/prod_lz4.o $(BUILD_DIR)/prod_lz4hc.o
$(BUILD_DIR)/prod_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)
$(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -c -o $@ $<
$(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR) -c -o $@ $<
$(BUILD_DIR)/prod_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)
$(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
$(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
# Vendored zstd (compiled separately, not unity-built via zstd_store.c)
ZSTD_OBJ_PROD = $(BUILD_DIR)/prod_zstd.o
$(BUILD_DIR)/prod_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)
$(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
$(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
OBJS_VENDORED_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LSP_OBJ_PROD) $(PP_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ)
@@ -1061,7 +1167,7 @@ MAIN_SRC = src/main.c
# for trap debugging -- silently re-run the previously instrumented binary,
# which during this bisect produced a "crashes without sanitizers too" result
# that was pure artifact and sent the investigation down the wrong path.
BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)
BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)|MEMWASTE=$(MEMWASTE)|MEMWASTE_ACCESS=$(MEMWASTE_ACCESS)|MEMWASTE_PROFILE=$(MEMWASTE_PROFILE)
.PHONY: build-config-check
build-config-check:
$(BUILD_DIR)/.build-config: build-config-check | $(BUILD_DIR)
@@ -1187,8 +1293,17 @@ lint-no-suppress:
lint: lint-tidy lint-cppcheck lint-format lint-no-suppress
@echo "=== All linters passed ==="
# Memory-core linter: memory is allocated through src/foundation/mem_core.h,
# not through raw malloc/calloc/realloc/free/strdup. A checked-in baseline
# (scripts/memory-core-baseline.txt) records each file's raw-site count and the
# gate goes red the moment any file grows. Files only ever go down; lower the
# baseline line in the same change that migrates the file.
lint-memory-core:
@echo "=== memory-core linter ==="
@python3 scripts/lint-memory-core.py
# CI linters (no clang-tidy — platform-dependent, enforced locally via pre-commit)
lint-ci: lint-cppcheck lint-format lint-no-suppress
lint-ci: lint-cppcheck lint-format lint-no-suppress lint-memory-core
@echo "=== CI linters passed ==="
# ── Local memory-diagnostic lanes (not PR-CI gates by decision: the diag
@@ -1248,6 +1363,28 @@ test-lsan:
cd $(CURDIR) && ASAN_OPTIONS="detect_leaks=1:halt_on_error=1" \
./build/lsan/test-runner $(LSAN_SUITES)
# ── Coverage-guided fuzzing (libFuzzer) ─────────────────────────
# One binary, three targets (tests/fuzz/cbm_fuzz.c): the per-file extractor,
# the Cypher parser, the TOML/YAML/JSON config editors. ASan+UBSan as in the
# test lane, plus fuzzer-no-link on EVERY object -- the vendored grammars and
# the tree-sitter runtime feed coverage back, which is where the parser state
# space lives. Needs a clang that ships libFuzzer (Homebrew LLVM on macOS; Apple
# clang does not). Run through scripts/fuzz.sh.
FUZZ_CC ?= clang
FUZZ_CXX ?= clang++
FUZZ_SANITIZE = -fsanitize=address,undefined,fuzzer-no-link -fno-omit-frame-pointer
FUZZ_SRC = tests/fuzz/cbm_fuzz.c
fuzz:
$(MAKE) -f Makefile.cbm build/fuzz/cbm-fuzz BUILD_DIR=build/fuzz \
CC=$(FUZZ_CC) CXX=$(FUZZ_CXX) SANITIZE="$(FUZZ_SANITIZE)"
$(BUILD_DIR)/cbm-fuzz: $(FUZZ_SRC) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR)
$(CC) $(CFLAGS_TEST) -fsanitize=fuzzer -o $@ \
$(FUZZ_SRC) $(PROD_SRCS) \
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
$(OBJS_VENDORED_TEST) \
$(LDFLAGS_TEST)
# Path-sensitive memory analysis: leak paths, null derefs, uninitialized reads.
# Non-gating: triage findings against the code (this lane's first run produced
# 9 real fixes and 6 recorded false positives — see the analyzer batch commit).
README.md
+16 -7
View File
@@ -3,7 +3,7 @@
[![GitHub Release](https://img.shields.io/github/v/release/DeusData/codebase-memory-mcp?style=flat&color=blue)](https://github.com/DeusData/codebase-memory-mcp/releases/latest)
[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
[![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/dry-run.yml?label=CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/dry-run.yml)
[![Tests](https://img.shields.io/badge/tests-6768_passing-brightgreen)](https://github.com/DeusData/codebase-memory-mcp)
[![Tests](https://img.shields.io/badge/tests-8050_passing-brightgreen)](https://github.com/DeusData/codebase-memory-mcp)
[![Languages](https://img.shields.io/badge/languages-162-orange)](https://github.com/DeusData/codebase-memory-mcp)
[![Hybrid LSP](https://img.shields.io/badge/Hybrid_LSP-10_languages-blue)](#hybrid-lsp)
[![Agents](https://img.shields.io/badge/agent_surfaces-45-purple)](https://github.com/DeusData/codebase-memory-mcp)
@@ -16,7 +16,7 @@
**The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. Ships as a native executable with a small verified runtime-asset set for macOS, Linux, and Windows — download, run `install`, done.
High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 162 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No language runtime, hosted service, or API key. Plug and play across 45 supported automatic/conditional client surfaces.
High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 162 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 17 MCP tools. No language runtime, hosted service, or API key. Plug and play across 45 supported automatic/conditional client surfaces.
> **Research** — The design and benchmarks behind this project are described in the preprint [*Codebase-Memory: Tree-Sitter-Based Knowledge Graphs for LLM Code Exploration via MCP*](https://arxiv.org/abs/2603.27277) (arXiv:2603.27277). Evaluated across 31 real-world repositories: 83% answer quality, 10× fewer tokens, 2.1× fewer tool calls vs. file-by-file exploration.
@@ -37,7 +37,7 @@ High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-si
- **45 supported automatic/conditional client surfaces** — `install` configures detected clients and safely activates conditional clients only when their documented platform, marker, or explicit existing config path is present. See [Multi-Agent Support](#multi-agent-support) for the complete matrix and manual/UI-only boundaries.
- **Built-in graph visualization** — 3D interactive UI at `localhost:9749`, served from the binary itself.
- **Infrastructure-as-code indexing** — Dockerfiles, Kubernetes manifests, and Kustomize overlays indexed as graph nodes with cross-references. `Resource` nodes for K8s kinds, `Module` nodes for Kustomize overlays with `IMPORTS` edges to referenced resources.
- **15 MCP tools** — search, trace, architecture, impact analysis, targeted index-coverage checks, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more.
- **17 MCP tools** — search, trace, architecture, impact analysis, targeted index-coverage checks, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more.
## Quick Start
@@ -425,7 +425,7 @@ scripts/build.sh # without the UI (development only)
Every platform ships **one self-contained executable**: the graph UI and the agent integration templates are linked into the binary, so an extracted archive is immediately complete.
Run the test suite (6,768 tests across 120 suites):
Run the test suite (8,060 tests across 141 suites):
```bash
scripts/test.sh # full: clean sanitizer build + all suites + guards
@@ -461,7 +461,7 @@ Add to `~/.claude.json` (user scope) or project `.mcp.json`:
}
```
Restart your agent. Verify with `/mcp` — you should see `codebase-memory-mcp` with 15 tools.
Restart your agent. Verify with `/mcp` — you should see `codebase-memory-mcp` with 17 tools.
</details>
@@ -673,6 +673,7 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool
| `list_projects` | List all indexed projects with node/edge counts. |
| `delete_project` | Remove a project and all its graph data. |
| `index_status` | Check indexing status of a project. |
| `check_index_coverage` | Check whether exact paths or a scope are indexed and fresh. A clean result means no recorded gap, not proof of completeness. |
### Querying
@@ -683,7 +684,9 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool
| `detect_changes` | Map git diff to affected symbols + blast radius with risk classification. |
| `query_graph` | Execute Cypher-like graph queries (read-only). |
| `get_graph_schema` | Node/edge counts, relationship patterns, property definitions per label. Run this first. |
| `compare_graphs` | Compare two indexed snapshots: node/edge additions and removals between a base and a target. |
| `get_code_snippet` | Read source code for a function by qualified name. |
| `get_file_outline` | Declaration outline of one repository-relative file in source order, with optional label filter and paging. |
| `get_architecture` | Codebase overview: languages, packages, routes, hotspots, clusters, ADR. |
| `search_code` | Grep-like text search within indexed project files. |
| `manage_adr` | CRUD for Architecture Decision Records (`get` reads, `update` replaces the whole document, `set_sections` rewrites only the named sections and leaves every other byte untouched, `sections` lists headings). Query modes do not wait behind a same-project reindex; writes remain serialized. |
@@ -701,7 +704,7 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool
### Edge Types
`CONTAINS_PACKAGE`, `CONTAINS_FOLDER`, `CONTAINS_FILE`, `DEFINES`, `DEFINES_METHOD`, `IMPORTS`, `CALLS`, `CALL_REFERENCE`, `HTTP_CALLS`, `ASYNC_CALLS`, `IMPLEMENTS`, `HANDLES`, `USAGE`, `CONFIGURES`, `WRITES`, `MEMBER_OF`, `TESTS`, `USES_TYPE`, `FILE_CHANGES_WITH`
`CONTAINS_PACKAGE`, `CONTAINS_FOLDER`, `CONTAINS_FILE`, `DEFINES`, `DEFINES_METHOD`, `IMPORTS`, `CALLS`, `CALL_REFERENCE`, `HTTP_CALLS`, `ASYNC_CALLS`, `IMPLEMENTS`, `HANDLES`, `USAGE`, `CONFIGURES`, `REFERENCES_FILE`, `WRITES`, `MEMBER_OF`, `TESTS`, `USES_TYPE`, `FILE_CHANGES_WITH`
### Qualified Names
@@ -733,9 +736,15 @@ codebase-memory-mcp config set auto_index true # auto-index on session
codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index
codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true)
codebase-memory-mcp config set watcher_enabled false # stop the watcher thread entirely (default: true)
codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit
codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit
codebase-memory-mcp config reset auto_index # reset to default
```
The two `index_max_*` settings default to `off`. Exceeding one fails the complete
index attempt rather than publishing a partial graph; an existing serving index
is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md).
### Environment Variables
| Variable | Default | Description |
@@ -843,7 +852,7 @@ Also supported (not yet benchmarked): Ada, Agda, Apex, Assembly (NASM), Astro, A
src/
main.c Entry point (MCP stdio server + CLI + install/update/config)
daemon/ Per-account session coordination, IPC, lifecycle, shared jobs/watchers
mcp/ MCP server (15 tools, JSON-RPC 2.0, session detection, auto-index)
mcp/ MCP server (17 tools, JSON-RPC 2.0, session detection, auto-index)
cli/ Install/uninstall/update/config (45 client surfaces, hooks, instructions)
store/ SQLite graph storage (nodes, edges, traversal, search, Louvain)
pipeline/ Multi-pass indexing (structure → definitions → calls → HTTP links → config → tests)
docs/CONFIGURATION.md
+10
View File
@@ -89,6 +89,8 @@ Current keys:
| `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. |
| `auto_watch` | `true` | Register the session's project with the background git watcher on connect. Set `false` to keep a session from registering its project (the watcher still runs for other projects). |
| `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. |
| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. |
| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. |
> **`watcher_enabled` vs `auto_watch`.** `watcher_enabled` controls whether the
> watcher *subsystem* starts at all (the background poll thread). `auto_watch` is
@@ -116,6 +118,14 @@ Current keys:
> `auto_index` still runs, and `index_repository` stays available for manual
> reindexing.
The two `index_max_*` settings are independent and disabled by default. They
apply to explicit indexing, automatic indexing, and watcher re-indexing, but not
to `cross-repo-intelligence`, which does not scan repository source files.
Equality is allowed; exceeding either setting fails the complete index request
and preserves any previously serving database. See
[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and
error-response details.
## 3. UI Settings
The optional built-in graph UI stores its settings in:
docs/INDEX_RESOURCE_LIMITS.md
+68
View File
@@ -0,0 +1,68 @@
# Index resource limits
Index resource limits are optional operator controls for repositories whose
discovery breadth is not known in advance. They are disabled by default so
existing large-repository workloads retain their current behavior.
## Discovery settings
| Key | Default | Accepted value | Protects |
|---|---:|---:|---|
| `index_max_files` | `off` | `off` or `1..10000000` | Accepted source-file count |
| `index_max_source_mb` | `off` | `off` or `1..1048576` | Accepted source-file bytes |
Set or reset them with the normal configuration command:
```bash
codebase-memory-mcp config set index_max_files 250000
codebase-memory-mcp config set index_max_source_mb 16384
codebase-memory-mcp config reset index_max_files
```
Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero,
negative values, suffixes, trailing characters, and values outside the stated
ranges are rejected without changing the stored value.
## Counting and failure semantics
`index_max_files` counts a file only after it passes directory pruning, ignore
rules, filename and suffix filters, language detection, and the existing
per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same
accepted set.
Equality is allowed. The first file or byte that makes an observed value greater
than its limit stops discovery. CBM discards the partial file list and does not
publish a partial graph as a complete index.
For an explicit MCP request the error payload contains:
```json
{
"status": "error",
"code": "resource_limit_exceeded",
"stage": "discovery",
"resource": "files",
"observed": 250001,
"limit": 250000,
"unit": "files",
"retryable": true,
"serving_index_preserved": true,
"message": "Index discovery exceeded index_max_files"
}
```
The previous database remains available because publication occurs only after a
complete discovery and successful staged build. If no previous database exists,
`serving_index_preserved` is false.
## Trust and compatibility
Limits are read from the CLI-managed `_config.db`; they are not MCP request
arguments. A supervised parent replaces any caller-supplied internal policy
before spawning its worker, and the worker rejects a missing or incomplete
parent policy.
These settings do not replace or increase `auto_index_limit`, change the 512 MiB
single-file cap, alter workspace-root authorization, or affect
`cross-repo-intelligence`. With both settings `off`, discovery follows the
existing unbounded path.
docs/index.html
+3 -3
View File
@@ -56,7 +56,7 @@
"featureList": [
"Indexes 158 programming languages via vendored tree-sitter grammars",
"Hybrid LSP semantic type resolution for Python, TypeScript/JavaScript, PHP, C#, Go, C/C++, Java, Kotlin, and Rust",
"15 MCP tools for structural search, call-path tracing, targeted coverage checks, and Cypher graph queries",
"17 MCP tools for structural search, call-path tracing, targeted coverage checks, and Cypher graph queries",
"Semantic vector code search via bundled nomic-embed-code embeddings (no API key, fully local)",
"Semantic graph edges (SEMANTICALLY_RELATED) and near-clone detection (SIMILAR_TO, MinHash + LSH)",
"Cross-service linking for HTTP, gRPC, GraphQL, tRPC, and pub/sub channels with confidence scoring",
@@ -739,8 +739,8 @@
<p>The optional UI variant adds one verified, content-addressed frontend pack and serves an interactive 3D graph at <code>localhost:9749</code> to explore nodes, edges, and clusters visually.</p>
</div>
<div class="feature">
<h3>15 MCP tools</h3>
<p><code>search_graph</code>, <code>trace_path</code>, <code>detect_changes</code>, <code>query_graph</code> (Cypher), <code>get_architecture</code>, <code>get_code_snippet</code>, <code>check_index_coverage</code>, <code>manage_adr</code>, and 7 more.</p>
<h3>17 MCP tools</h3>
<p><code>search_graph</code>, <code>trace_path</code>, <code>detect_changes</code>, <code>query_graph</code> (Cypher), <code>get_architecture</code>, <code>get_code_snippet</code>, <code>check_index_coverage</code>, <code>manage_adr</code>, and 9 more.</p>
</div>
<div class="feature">
<h3>Cypher graph queries</h3>
flake.nix
+1 -1
View File
@@ -63,7 +63,7 @@
packages = forAllSystems (pkgs: rec {
default = pkgs.stdenv.mkDerivation {
pname = "codebase-memory-mcp";
version = "0.10.8";
version = "0.11.0";
src = ./.;
graph-ui/package-lock.json
Generated
+48 -48
View File
@@ -33,7 +33,7 @@
"tailwindcss": "^4.1.0",
"typescript": "^5.7.0",
"vite": "^6.4.3",
"vitest": "^4.1.0"
"vitest": "^4.1.11"
}
},
"node_modules/@adobe/css-tools": {
@@ -3603,16 +3603,16 @@
}
},
"node_modules/@vitest/expect": {
"version": "4.1.8",
"resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz",
"integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==",
"version": "4.1.11",
"resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz",
"integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@standard-schema/spec": "^1.1.0",
"@types/chai": "^5.2.2",
"@vitest/spy": "4.1.8",
"@vitest/utils": "4.1.8",
"@vitest/spy": "4.1.11",
"@vitest/utils": "4.1.11",
"chai": "^6.2.2",
"tinyrainbow": "^3.1.0"
},
@@ -3621,13 +3621,13 @@
}
},
"node_modules/@vitest/mocker": {
"version": "4.1.8",
"resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz",
"integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==",
"version": "4.1.11",
"resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz",
"integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@vitest/spy": "4.1.8",
"@vitest/spy": "4.1.11",
"estree-walker": "^3.0.3",
"magic-string": "^0.30.21"
},
@@ -3648,9 +3648,9 @@
}
},
"node_modules/@vitest/pretty-format": {
"version": "4.1.8",
"resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz",
"integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==",
"version": "4.1.11",
"resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz",
"integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -3661,13 +3661,13 @@
}
},
"node_modules/@vitest/runner": {
"version": "4.1.8",
"resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz",
"integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==",
"version": "4.1.11",
"resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz",
"integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@vitest/utils": "4.1.8",
"@vitest/utils": "4.1.11",
"pathe": "^2.0.3"
},
"funding": {
@@ -3675,14 +3675,14 @@
}
},
"node_modules/@vitest/snapshot": {
"version": "4.1.8",
"resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz",
"integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==",
"version": "4.1.11",
"resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz",
"integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==",
"dev": true,
"license": "MIT",
"dependencies": {
"@vitest/pretty-format": "4.1.8",
"@vitest/utils": "4.1.8",
"@vitest/pretty-format": "4.1.11",
"@vitest/utils": "4.1.11",
"magic-string": "^0.30.21",
"pathe": "^2.0.3"
},
@@ -3691,9 +3691,9 @@
}
},
"node_modules/@vitest/spy": {
"version": "4.1.8",
"resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz",
"integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==",
"version": "4.1.11",
"resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz",
"integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==",
"dev": true,
"license": "MIT",
"funding": {
@@ -3701,13 +3701,13 @@
}
},
"node_modules/@vitest/utils": {
"version": "4.1.8",
"resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz",
"integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==",
"version": "4.1.11",
"resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz",
"integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@vitest/pretty-format": "4.1.8",
"@vitest/pretty-format": "4.1.11",
"convert-source-map": "^2.0.0",
"tinyrainbow": "^3.1.0"
},
@@ -5831,9 +5831,9 @@
}
},
"node_modules/tinyrainbow": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz",
"integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==",
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz",
"integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==",
"dev": true,
"license": "MIT",
"engines": {
@@ -6141,19 +6141,19 @@
}
},
"node_modules/vitest": {
"version": "4.1.8",
"resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz",
"integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==",
"version": "4.1.11",
"resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz",
"integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@vitest/expect": "4.1.8",
"@vitest/mocker": "4.1.8",
"@vitest/pretty-format": "4.1.8",
"@vitest/runner": "4.1.8",
"@vitest/snapshot": "4.1.8",
"@vitest/spy": "4.1.8",
"@vitest/utils": "4.1.8",
"@vitest/expect": "4.1.11",
"@vitest/mocker": "4.1.11",
"@vitest/pretty-format": "4.1.11",
"@vitest/runner": "4.1.11",
"@vitest/snapshot": "4.1.11",
"@vitest/spy": "4.1.11",
"@vitest/utils": "4.1.11",
"es-module-lexer": "^2.0.0",
"expect-type": "^1.3.0",
"magic-string": "^0.30.21",
@@ -6181,12 +6181,12 @@
"@edge-runtime/vm": "*",
"@opentelemetry/api": "^1.9.0",
"@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0",
"@vitest/browser-playwright": "4.1.8",
"@vitest/browser-preview": "4.1.8",
"@vitest/browser-webdriverio": "4.1.8",
"@vitest/coverage-istanbul": "4.1.8",
"@vitest/coverage-v8": "4.1.8",
"@vitest/ui": "4.1.8",
"@vitest/browser-playwright": "4.1.11",
"@vitest/browser-preview": "4.1.11",
"@vitest/browser-webdriverio": "4.1.11",
"@vitest/coverage-istanbul": "4.1.11",
"@vitest/coverage-v8": "4.1.11",
"@vitest/ui": "4.1.11",
"happy-dom": "*",
"jsdom": "*",
"vite": "^6.0.0 || ^7.0.0 || ^8.0.0"
graph-ui/package.json
+1 -1
View File
@@ -37,7 +37,7 @@
"tailwindcss": "^4.1.0",
"typescript": "^5.7.0",
"vite": "^6.4.3",
"vitest": "^4.1.0"
"vitest": "^4.1.11"
},
"overrides": {
"form-data": ">=4.0.6",
install.ps1
+16 -2
View File
@@ -7,8 +7,22 @@
$ErrorActionPreference = "Stop"
# Enforce TLS 1.2+ (older PowerShell defaults to TLS 1.0 which GitHub rejects)
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -bor [Net.SecurityProtocolType]::Tls13
# Enforce TLS 1.2+ (older PowerShell defaults to TLS 1.0 which GitHub rejects).
#
# TLS 1.3 is added ONLY where schannel can actually negotiate it: Windows 11 and
# Server 2022 (build 20348+). On Windows 10 the enum value still exists from
# .NET Framework 4.8 onward, so the assignment succeeds and nothing warns -- but
# the handshake then fails outright with "The request was aborted: Could not
# create SSL/TLS secure channel". An unsupported protocol flag in this bitmask
# is a hard failure, not a graceful downgrade, so every Windows 10 user was
# blocked at the first download (#1856). The enum-name probe additionally keeps
# the script parsing on .NET Framework 4.7, where Tls13 is not defined at all.
$CbmProtocols = [Net.SecurityProtocolType]::Tls12
if ([Environment]::OSVersion.Version.Build -ge 20348 -and
([enum]::GetNames([Net.SecurityProtocolType]) -contains 'Tls13')) {
$CbmProtocols = $CbmProtocols -bor [Net.SecurityProtocolType]::Tls13
}
[Net.ServicePointManager]::SecurityProtocol = $CbmProtocols
Add-Type -AssemblyName System.Net.Http
$Repo = "DeusData/codebase-memory-mcp"
internal/cbm/arena.h
+7 -41
View File
@@ -1,41 +1,7 @@
#ifndef CBM_ARENA_H
#define CBM_ARENA_H
#include <stddef.h>
// CBMArena is a simple bump allocator that allocates from fixed-size blocks.
// All memory is freed at once via cbm_arena_destroy(). Individual frees are not
// supported — this is by design for per-file extraction where all data has the
// same lifetime.
#define CBM_ARENA_MAX_BLOCKS 256
#define CBM_ARENA_DEFAULT_BLOCK_SIZE (64 * 1024) // 64KB initial
typedef struct {
char *blocks[CBM_ARENA_MAX_BLOCKS];
size_t block_sizes[CBM_ARENA_MAX_BLOCKS]; // per-block sizes (for stats)
int nblocks;
size_t block_size;
size_t used; // bytes used in current block
size_t total_alloc; // cumulative bytes allocated (for stats)
} CBMArena;
// Initialize an arena with the default block size.
void cbm_arena_init(CBMArena *a);
// Allocate n bytes from the arena. Returns NULL on OOM or block exhaustion.
// All returned pointers are 8-byte aligned.
void *cbm_arena_alloc(CBMArena *a, size_t n);
// Duplicate a string into arena memory. Returns arena-owned copy.
char *cbm_arena_strdup(CBMArena *a, const char *s);
// Duplicate a string of known length into arena memory. NUL-terminates.
char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len);
// sprintf into arena memory. Returns arena-owned string.
char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) __attribute__((format(printf, 2, 3)));
// Free all blocks. Arena is invalid after this call.
void cbm_arena_destroy(CBMArena *a);
#endif // CBM_ARENA_H
/* The arena lives in src/foundation/arena.h. This file used to be a stale
* copy with the same include guard and a shorter API; whichever header a
* translation unit included first won, silently. One definition now. */
#ifndef CBM_INTERNAL_ARENA_SHIM_H
#define CBM_INTERNAL_ARENA_SHIM_H
#include "../../src/foundation/arena.h" /* relative: the lsp_all unit has no -Isrc */
#endif /* CBM_INTERNAL_ARENA_SHIM_H */
internal/cbm/cbm.c
+960 -75
View File
File diff suppressed because it is too large Load Diff
internal/cbm/cbm.h
+149 -32
View File
@@ -6,6 +6,17 @@
#include "arena.h"
#include "tree_sitter/api.h"
/* Field lookups by NAME resolve the name with a linear strncmp scan over the
* grammar's field table on every call -- 951 M strncmp calls on the Go corpus
* (waste sanitizer, 2026-09-17) across ~1,200 call sites. Tree-sitter's own
* implementation is exactly "field id for name, then child by field id"; this
* caches the first half per thread (language, name) and keeps the second.
* Every extractor includes this header, so every call site gets it. */
TSNode cbm_ts_child_by_field_name(TSNode node, const char *name, uint32_t name_length);
/* Variadic: call sites spell the name and its length as ONE macro argument
* (TS_FIELD("body") expands to "body", 4), which must expand before the call. */
#define ts_node_child_by_field_name(...) cbm_ts_child_by_field_name(__VA_ARGS__)
// Language enum mirrors lang.Language in Go.
// Order must match lang_specs.c tables.
typedef enum {
@@ -248,31 +259,41 @@ typedef enum {
} CBMSourceOrigin;
typedef struct {
const char *callee_name; // raw callee text ("pkg.Func", "foo")
const char *enclosing_func_qn; // QN of enclosing function (or module QN)
const char *first_string_arg; // first string literal argument (URL, topic, key) or NULL
const char *second_arg_name; // second argument identifier (handler ref) or NULL
CBMCallArg args[CBM_MAX_CALL_ARGS]; // first N arguments with expressions
int arg_count; // number of captured arguments
int loop_depth; // enclosing loop nesting at the call site
int branch_depth; // enclosing branch nesting at the call site
int start_line; // 1-based source line of the call (for def range-match)
uint32_t site_start_byte; // exact AST occurrence span; end > start when present
uint32_t site_end_byte; // exclusive byte offset in the source file
CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer
bool is_method; // method/member call with an UNRESOLVED receiver. Perl:
// arrow/method call ($obj->m). TS/JS/TSX: member call
// x.foo() whose receiver is not this/super. Python:
// x.foo() where x is not self/cls/super() and is not
// rooted in an imported name. Read by the weak-member
// guard and by the pxc synthetic-carrier dedup key in
// pass_lsp_cross.c. Default false.
bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit
// C++ operator). Never fall back to textual resolution.
bool callee_is_locally_bound; // bare call foo() whose callee identifier is bound as a
// parameter of an enclosing function, so it cannot be the
// module-level foo. Python only today. Read by the
// weak-local-binding guard. Default false.
const char *callee_name; // raw callee text ("pkg.Func", "foo")
const char *enclosing_func_qn; // QN of enclosing function (or module QN)
const char *first_string_arg; // first string literal argument (URL, topic, key) or NULL
const char *second_arg_name; // second argument identifier (handler ref) or NULL
/* First arg_count captured arguments, arena-allocated on first capture;
* NULL when arg_count == 0. Was an inline args[CBM_MAX_CALL_ARGS] (256 of
* the record's 320 bytes) -- the Go corpus census (2026-09-13) put 825k
* calls at 251 MB with most of that empty slots. Readers index it exactly
* as before; only `sizeof` changed. */
CBMCallArg *args;
int arg_count; // number of captured arguments (<= CBM_MAX_CALL_ARGS)
int loop_depth; // enclosing loop nesting at the call site
int branch_depth; // enclosing branch nesting at the call site
int start_line; // 1-based source line of the call (for def range-match)
uint32_t site_start_byte; // exact AST occurrence span; end > start when present
uint32_t site_end_byte; // exclusive byte offset in the source file
CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer
bool is_method; // method/member call with an UNRESOLVED receiver. Perl:
// arrow/method call ($obj->m). TS/JS/TSX: member call
// x.foo() whose receiver is not this/super. Python:
// x.foo() where x is not self/cls/super() and is not
// rooted in an imported name. Read by the weak-member
// guard and by the pxc synthetic-carrier dedup key in
// pass_lsp_cross.c. Default false.
bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit
// C++ operator). Never fall back to textual resolution.
bool callee_is_locally_bound; // bare call foo() whose callee identifier is bound as a
// parameter of an enclosing function, so it cannot be the
// module-level foo. Python only today. Read by the
// weak-local-binding guard. Default false.
bool receiver_is_self_attribute; // Python member call whose receiver is an attribute
// chain rooted at self/cls but not self/cls itself
// (self.compiler.apply_converters()). An object the
// class owns, not a parameter: read by the weak-member
// guard's unique-name exemption. Default false.
} CBMCall;
typedef struct {
@@ -286,16 +307,18 @@ typedef enum {
} CBMUsageKind;
typedef struct {
const char *ref_name; // referenced identifier
const char *enclosing_func_qn; // QN of enclosing function (or module QN)
const char *ref_name; // referenced identifier
const char *enclosing_func_qn; // QN of enclosing function (or module QN)
/* Fixed-width fields grouped so the record packs to 40 bytes (was 48; the
* Go corpus holds 4.68M of these). Field meanings unchanged. */
uint32_t lexical_scope_id; // extraction-local scope instance; never graph identity
uint32_t site_start_byte; // exact reference-token span; end > start when present
uint32_t site_end_byte; // exclusive byte offset in the source file
CBMUsageKind kind; // ordinary USAGE or explicit callable reference
CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer
bool may_be_call_reference; // syntactic candidate; exact LSP proof may upgrade its edge
bool semantic_reference_blocked; // lexical evidence blocks only unproven textual fallback
bool semantic_reference_local_shadow; // blocker belongs to a non-module lexical scope
uint32_t lexical_scope_id; // extraction-local scope instance; never graph identity
uint32_t site_start_byte; // exact reference-token span; end > start when present
uint32_t site_end_byte; // exclusive byte offset in the source file
CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer
bool is_member_access; // token is the member half of a selector/attribute
// (Go x.f — field_identifier). The extractor strips
// the receiver, so this is the only surviving record
@@ -541,7 +564,22 @@ typedef struct CBMFileResult {
int error_region_count;
bool is_test_file;
int imports_count;
TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree)
TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree)
/* The parse alone used more than its share of the per-file budget: the
* per-file LSP walk and the cross-file resolve skip this file (its
* unified-extractor defs stay). Set by cbm_extract_file_ex, honoured by
* cbm_pxc_dispatch_file -- one site for every language. */
bool lsp_skipped;
/* The unified walk stopped at its CPU budget: defs/calls/usages found up
* to that point are kept, the rest of the file is not walked. Implies
* lsp_skipped. */
bool walk_truncated;
/* Size of this file's parse tree, and how much of it the unified walk got
* through. Reported for a truncated or LSP-skipped file so the coverage
* report says how much of it is missing, instead of leaving the gap
* silent. */
uint32_t tree_nodes;
uint32_t walk_nodes_visited;
CBMLanguage cached_lang; // language of cached tree (for parser selection)
// Retained source bytes — copied into `arena` by the parallel
@@ -632,6 +670,19 @@ typedef struct {
* class-body variable def records which class declares it (parent_class)
* without changing its module-level qualified name. NULL elsewhere. */
const char *var_parent_class;
/* Per-file walk budget in VISITED NODES (0 = unbounded). The unified cursor
* walk stops once it is spent, so no single file can hold a worker for
* minutes: a 23 MB single-expression C# test file cost 346 s in usage
* stamping alone (tree-sitter's ts_node_parent descends from the root,
* quadratic on a deep tree; 2026-09-14). What was extracted before the stop
* is kept, and the file is named in the coverage report. Counted in nodes
* rather than CPU time so that the same file always stops at the same node
* — see CBM_WALK_MAX_NODES_DEFAULT for what a clock did here. */
uint32_t walk_budget_nodes;
bool walk_budget_exhausted;
/* How many nodes the unified walk actually visited (whether or not it ran
* out of budget) — the measurement the budget has to be expressed in. */
uint32_t walk_nodes_visited;
} CBMExtractCtx;
// --- Public API ---
@@ -645,6 +696,12 @@ typedef struct {
// also calls it so non-main entry points (pipeline passes) still get the binds.
// In the test build (no CBM_BIND_TS_ALLOCATOR) this is a no-op.
void cbm_alloc_init(void);
/* SQLite allocates from a dedicated mimalloc heap per thread while on; the
* index worker turns it on (its default heap holds the graph). Off elsewhere:
* a thread-per-connection daemon would pin connection-lifetime blocks to
* dead threads. The switch exists in every build; it changes nothing where
* the allocator binds are compiled out. */
void cbm_sqlite_dedicated_heap(bool on);
// Initialize the library. Call once at startup. Returns 0 on success.
int cbm_init(void);
@@ -675,6 +732,41 @@ void cbm_index_mark_done(const char *rel_path);
// Extract all data from one file. Caller must call cbm_free_result().
// source must remain valid for the duration of the call.
// timeout_micros: per-file parse timeout in microseconds (0 = no timeout).
/* Compact a finished result: copy everything reachable from it -- every
* record array at exact count, every string once (interned by content within
* the file), the retained source -- into one exact-size arena, and destroy the
* working arena the extractors wrote into. Measured on the Go corpus
* (2026-09-13): 14.8 GB written per index, 3.4 GB reachable; the rest was
* node-text copies and abandoned array generations no one could free because
* the result owned the arena. Call once, after the last per-file write and
* before the result is cached for later passes. Later appends into the arena
* still work (growth restarts at the default block). A composite's owned
* per-unit results are released: after the deep copy nothing points at them.
* On allocation failure the result is left exactly as it was. */
void cbm_result_compact(CBMFileResult *result);
/* The working arena the extractors write into, per worker thread. Extraction
* takes it (rewound, pages still mapped) instead of allocating a fresh arena
* per file; compaction returns it instead of destroying it. Reusing the same
* addresses directly is what stops the purge/re-commit churn that kept a
* kernel worker at 15 GB resident with 4-5 GB charged. An arena that grew past
* CBM_WORK_ARENA_KEEP_BYTES (one giant file) is destroyed, not kept. */
enum { CBM_WORK_ARENA_KEEP_BYTES = 16 * 1024 * 1024 };
void cbm_work_arena_take(CBMArena *into);
void cbm_work_arena_give(CBMArena *from);
/* Drop this thread's kept working arena (end of an extraction pass). */
void cbm_work_arena_release(void);
/* True on a thread that has given a working arena back, i.e. a pipeline
* worker whose cbm_work_arena_release is guaranteed to run: only such a thread
* may keep per-thread scratch between files. */
bool cbm_work_arena_keeping(void);
/* Let this thread keep its per-file extraction scratch between the files of a
* sequential loop; the caller must call cbm_work_arena_release on the same
* thread when the loop ends (every return path). */
void cbm_work_arena_keep_begin(void);
/* Free the compaction scratch this thread kept (cbm_work_arena_release calls it). */
void cbm_result_compact_release_thread(void);
CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage language,
const char *project, const char *rel_path, int64_t timeout_micros,
const char **extra_defines, // NULL-terminated, or NULL
@@ -696,6 +788,12 @@ CBMFileResult *cbm_extract_file_ex(
// Free all memory associated with a result.
void cbm_free_result(CBMFileResult *result);
/* Allocate an empty result; cbm_free_result releases it. */
CBMFileResult *cbm_result_alloc(void);
/* Release a composite result's per-unit results (the owner of that array). */
void cbm_result_release_owned(CBMFileResult *result);
// Free only the cached tree from a result (caller retained it for reuse).
void cbm_free_tree(CBMFileResult *result);
@@ -710,6 +808,25 @@ void cbm_reset_thread_parser(void);
// Destroy the thread-local parser. Call on worker thread exit.
void cbm_destroy_thread_parser(void);
// This thread's reusable tree cursor, reset to `node`. Child walks run once per
// visited node; a cursor per walk was a malloc + free of its stack each time
// (49.8 M on the Go corpus, waste sanitizer 2026-09-17). Valid until the next
// call on this thread; released with the thread parser.
TSTreeCursor *cbm_thread_cursor(TSNode node);
// Reusable cursors for RECURSIVE walks: one per recursion depth on this thread,
// so a walk that recurses from inside its child loop never shares a cursor with
// its own callers. A slot already in use (another walker nested on this thread)
// hands out a private cursor instead; release deletes it. Released with the
// thread parser.
typedef struct {
TSTreeCursor *cursor;
TSTreeCursor private_cursor;
int slot; /* -1: private */
} cbm_cursor_lease_t;
TSTreeCursor *cbm_cursor_acquire(cbm_cursor_lease_t *lease, int depth, TSNode node);
void cbm_cursor_release(cbm_cursor_lease_t *lease);
// Shutdown the library. Call once at exit.
void cbm_shutdown(void);
internal/cbm/extract_calls.c
+115 -9
View File
@@ -258,9 +258,18 @@ static char *extract_constructor_callee(CBMArena *a, TSNode node, const char *so
}
// Try common field-based callee resolution (function, name, method fields).
static TSNode unwrap_await_callee(TSNode node) {
if (ts_node_is_null(node) || strcmp(ts_node_type(node), "await_expression") != 0 ||
ts_node_named_child_count(node) == 0) {
return node;
}
return ts_node_named_child(node, 0);
}
static char *extract_callee_from_fields(CBMArena *a, TSNode node, const char *source) {
// Try "function" field
TSNode func_node = ts_node_child_by_field_name(node, TS_FIELD("function"));
func_node = unwrap_await_callee(func_node);
if (!ts_node_is_null(func_node)) {
const char *fk = ts_node_type(func_node);
if (strcmp(fk, "selector_expression") == 0) {
@@ -2059,6 +2068,20 @@ static void extract_call_args(CBMExtractCtx *ctx, TSNode args, CBMCall *call) {
for (uint32_t ai = 0; ai < argc && call->arg_count < CBM_MAX_CALL_ARGS; ai++) {
TSNode arg_node = ts_node_named_child(args, ai);
const char *ak = ts_node_type(arg_node);
/* tree-sitter lists a comment inside the argument list as a named
* child (Java block_comment/line_comment, C-family comment). It is
* not an argument: taken as one, a leading block comment became
* args[0] and the Route pass read its text as a URL. */
if (strcmp(ak, "comment") == 0 || strcmp(ak, "line_comment") == 0 ||
strcmp(ak, "block_comment") == 0) {
continue;
}
if (!call->args) {
call->args = cbm_arena_calloc(ctx->arena, CBM_MAX_CALL_ARGS * sizeof(CBMCallArg));
if (!call->args) {
return;
}
}
CBMCallArg *ca = &call->args[call->arg_count];
memset(ca, 0, sizeof(*ca));
@@ -2257,6 +2280,61 @@ static const char *extract_binary_concat_suffix(CBMExtractCtx *ctx, TSNode node)
}
// Try to extract URL/topic from a positional argument (string or constant).
/* Swift names its call arguments, so each one is a value_argument that may lead
* with a value_argument_label — the `from:` in `data(from: url)`. Return the
* value itself, and leave any other node exactly as it came in. */
static TSNode swift_argument_value(TSNode arg) {
if (strcmp(ts_node_type(arg), "value_argument") != 0 || ts_node_named_child_count(arg) == 0) {
return arg;
}
TSNode val = ts_node_named_child(arg, 0);
if (strcmp(ts_node_type(val), "value_argument_label") == 0 &&
ts_node_named_child_count(arg) > 1) {
val = ts_node_named_child(arg, 1);
}
return val;
}
/* Swift has no URL literal, so almost no real code passes a bare string to a
* request. It writes `URL(string: "https://…")!` instead, and the literal then
* sits two levels down: past the trailing `!`, which the grammar models as a
* postfix_expression, and inside the constructor's own argument list.
*
* Unwrap both so that literal is as reachable as a bare one. Only the three
* Foundation types that take a URL string are unwrapped — any other call keeps
* its own meaning, and a non-literal argument such as `URL(string: base + path)`
* falls through to the ordinary handling unchanged. */
static TSNode swift_unwrap_url_constructor(CBMExtractCtx *ctx, TSNode arg) {
/* Step past a trailing "!" or "?". */
if (strcmp(ts_node_type(arg), "postfix_expression") == 0) {
TSNode target = ts_node_child_by_field_name(arg, TS_FIELD("target"));
if (!ts_node_is_null(target)) {
arg = target;
}
}
if (strcmp(ts_node_type(arg), "call_expression") != 0) {
return arg;
}
TSNode callee = ts_node_named_child(arg, 0);
if (ts_node_is_null(callee) || strcmp(ts_node_type(callee), "simple_identifier") != 0) {
return arg;
}
const char *name = cbm_node_text(ctx->arena, callee, ctx->source);
if (!name || (strcmp(name, "URL") != 0 && strcmp(name, "URLComponents") != 0 &&
strcmp(name, "URLRequest") != 0)) {
return arg;
}
TSNode suffix = cbm_find_child_by_kind(arg, "call_suffix");
if (ts_node_is_null(suffix)) {
return arg;
}
TSNode inner = cbm_find_child_by_kind(suffix, "value_arguments");
if (ts_node_is_null(inner) || ts_node_named_child_count(inner) == 0) {
return arg;
}
return swift_argument_value(ts_node_named_child(inner, 0));
}
static const char *extract_positional_url(CBMExtractCtx *ctx, TSNode arg, const char *ak) {
/* JS/TS template literals: `/things/${id}` normalizes to "/things/{}" so the
* client URL joins the server route's canonical placeholder (issue #1006). */
@@ -2300,15 +2378,12 @@ static const char *extract_url_or_topic_arg(CBMExtractCtx *ctx, TSNode args) {
}
/* Swift wraps each argument in a value_argument that may lead with its
* label, so `data(from: url)` would otherwise yield the label `from`
* rather than the value. Step past a leading value_argument_label. */
if (strcmp(ts_node_type(arg), "value_argument") == 0 &&
ts_node_named_child_count(arg) > 0) {
TSNode val = ts_node_named_child(arg, 0);
if (strcmp(ts_node_type(val), "value_argument_label") == 0 &&
ts_node_named_child_count(arg) > 1) {
val = ts_node_named_child(arg, 1);
}
arg = val;
* rather than the value. */
arg = swift_argument_value(arg);
/* A Swift URL is usually built rather than written bare, and the
* literal then sits inside that constructor. */
if (ctx->language == CBM_LANG_SWIFT) {
arg = swift_unwrap_url_constructor(ctx, arg);
}
const char *ak = ts_node_type(arg);
@@ -2950,6 +3025,28 @@ static char *resolve_objectscript_instance_call(CBMArena *a, TSNode node, const
* statically-known type here, so the call must not bind by short name alone.
* Note `self.client.send()` is NOT exempt: the receiver is `self.client`, an
* attribute of unknown type, not `self` itself. */
/* True when a Python attribute-call receiver is an attribute chain ROOTED at
* self/cls but is not self/cls itself: `self.compiler.apply_converters()` has
* receiver `self.compiler`, an object the class owns. The weak-member guard's
* unique-name exemption keys on this shape (see cbm_weak_member_unique_name_
* exempt): a bare parameter (`accelerator.backward()`) carries no ownership
* evidence and stays suppressed. Direct `self.m()` is already exempt via
* python_receiver_is_exempt and is deliberately NOT flagged here. */
static bool python_receiver_rooted_at_self(CBMExtractCtx *ctx, TSNode receiver) {
if (ts_node_is_null(receiver) || strcmp(ts_node_type(receiver), "attribute") != 0) {
return false;
}
TSNode root = receiver;
while (!ts_node_is_null(root) && strcmp(ts_node_type(root), "attribute") == 0) {
root = ts_node_child_by_field_name(root, TS_FIELD("object"));
}
if (ts_node_is_null(root) || strcmp(ts_node_type(root), "identifier") != 0) {
return false;
}
char *name = cbm_node_text(ctx->arena, root, ctx->source);
return name && (strcmp(name, "self") == 0 || strcmp(name, "cls") == 0);
}
static bool python_receiver_is_exempt(CBMExtractCtx *ctx, TSNode receiver) {
if (ts_node_is_null(receiver)) {
return false;
@@ -3515,6 +3612,7 @@ static CBMPrimaryCalleeSelection select_primary_callee(CBMExtractCtx *ctx, TSNod
}
selection.expr = language_specific_callee_expr(ctx->language, node);
selection.expr = unwrap_await_callee(selection.expr);
if (is_dynamic_callee_expr(ctx, selection.expr)) {
selection.expr = (TSNode){0};
return selection;
@@ -3679,6 +3777,7 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML
if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "attribute") == 0) {
TSNode obj = ts_node_child_by_field_name(fn, TS_FIELD("object"));
call.is_method = !python_receiver_is_exempt(ctx, obj);
call.receiver_is_self_attribute = python_receiver_rooted_at_self(ctx, obj);
} else if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "identifier") == 0) {
call.callee_is_locally_bound = python_callee_is_bound_parameter(ctx, state, fn);
}
@@ -3768,6 +3867,13 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML
if (strcmp(ack, "method_arg") != 0) {
continue;
}
if (!call.args) {
call.args = cbm_arena_calloc(ctx->arena,
CBM_MAX_CALL_ARGS * sizeof(CBMCallArg));
if (!call.args) {
break;
}
}
CBMCallArg *ca = &call.args[call.arg_count];
memset(ca, 0, sizeof(*ca));
ca->index = call.arg_count;
internal/cbm/extract_defs.c
+176 -4
View File
@@ -3477,6 +3477,159 @@ static TSNode find_function_params(TSNode func_node, CBMLanguage lang) {
return params;
}
/* ── C-family declared return type ──────────────────────────────────
* The C-family grammars split a declared return type three ways: the `type`
* field (`char`), sibling type_qualifier nodes (`const`), and the
* pointer/reference declarators wrapping the function declarator (`*`). Taking
* only the `type` field published `const char *get_name(void)` as "char".
*
* Canonical spelling: leading cv-qualifiers in source order, the base type text
* verbatim, then one space and the declarator markers outermost-first with no
* space between them — `const char *`, `char **`, `Text &`, `Text *&`. A
* qualifier on a pointer level follows its `*` and is separated from the next
* marker by a space: `char *const *`. A qualifier written after the base type
* (`char const *`) is normalized to the leading position. */
/* Output sink: measures when buf is NULL, writes otherwise. Rendering twice
* sizes the arena allocation exactly without a second copy of the logic. */
typedef struct {
char *buf;
size_t len;
} c_rt_out_t;
static void c_rt_put(c_rt_out_t *out, const char *text, size_t n) {
if (out->buf) {
memcpy(out->buf + out->len, text, n);
}
out->len += n;
}
static void c_rt_put_str(c_rt_out_t *out, const char *text) {
c_rt_put(out, text, strlen(text));
}
static void c_rt_put_node(c_rt_out_t *out, TSNode node, const char *source) {
uint32_t start = ts_node_start_byte(node);
uint32_t end = ts_node_end_byte(node);
if (end > start) {
c_rt_put(out, source + start, (size_t)(end - start));
}
}
/* type_qualifier also covers keywords that are not part of the type
* (`constexpr`, `_Noreturn`, `mutable`, `__extension__`, …). Keep only the ones
* that are, so `constexpr int f()` still returns "int". */
static bool is_c_return_cv_qualifier(TSNode node, const char *source) {
static const char *const kept[] = {"const", "volatile", "restrict", "__restrict",
"__restrict__", "_Atomic", NULL};
if (strcmp(ts_node_type(node), "type_qualifier") != 0) {
return false;
}
uint32_t start = ts_node_start_byte(node);
uint32_t end = ts_node_end_byte(node);
size_t len = end > start ? (size_t)(end - start) : 0;
for (const char *const *k = kept; *k; k++) {
if (strlen(*k) == len && memcmp(source + start, *k, len) == 0) {
return true;
}
}
return false;
}
static bool is_c_declarator_lang(CBMLanguage lang) {
return lang == CBM_LANG_C || lang == CBM_LANG_CPP || lang == CBM_LANG_CUDA ||
lang == CBM_LANG_GLSL || lang == CBM_LANG_HLSL || lang == CBM_LANG_ISPC ||
lang == CBM_LANG_SLANG || lang == CBM_LANG_OBJC;
}
/* Render the canonical return type into `out`; returns how many qualifiers and
* markers were added around the base type (0 = the base text alone is already
* the whole type). The declarator walk is one strict child chain, so it is
* O(depth) with no recursion and needs no depth cap. It stops at the first node
* that is neither a pointer nor a reference declarator: for a function returning
* a function pointer (`int (*f(void))(int)`) that is the outer
* function_declarator, which leaves the base type as it was. */
static size_t c_rt_render(c_rt_out_t *out, TSNode func_node, TSNode type_node, TSNode declarator,
const char *source) {
size_t added = 0;
uint32_t decl_start = ts_node_start_byte(declarator);
uint32_t nc = ts_node_named_child_count(func_node);
for (uint32_t i = 0; i < nc; i++) {
TSNode ch = ts_node_named_child(func_node, i);
if (ts_node_start_byte(ch) >= decl_start) {
break;
}
if (is_c_return_cv_qualifier(ch, source)) {
c_rt_put_node(out, ch, source);
c_rt_put_str(out, " ");
added++;
}
}
c_rt_put_node(out, type_node, source);
bool need_space = true;
TSNode decl = declarator;
while (!ts_node_is_null(decl)) {
const char *dk = ts_node_type(decl);
bool is_ref = strcmp(dk, "reference_declarator") == 0;
if (!is_ref && strcmp(dk, "pointer_declarator") != 0) {
break;
}
if (need_space) {
c_rt_put_str(out, " ");
need_space = false;
}
/* A reference_declarator opens with its `&` / `&&` token. */
TSNode marker = ts_node_child(decl, 0);
if (is_ref && !ts_node_is_null(marker) && !ts_node_is_named(marker)) {
c_rt_put_node(out, marker, source);
} else {
c_rt_put_str(out, is_ref ? "&" : "*");
}
added++;
uint32_t dn = ts_node_named_child_count(decl);
for (uint32_t i = 0; i < dn; i++) {
TSNode q = ts_node_named_child(decl, i);
if (is_c_return_cv_qualifier(q, source)) {
c_rt_put_node(out, q, source);
need_space = true;
added++;
}
}
/* tree-sitter-cpp/-cuda give a reference_declarator's inner declarator no
* `declarator` field (see find_c_params); it is the one named child. */
TSNode inner = ts_node_child_by_field_name(decl, TS_FIELD("declarator"));
if (ts_node_is_null(inner) && is_ref && dn > 0) {
inner = ts_node_named_child(decl, 0);
}
decl = inner;
}
return added;
}
/* Declared return type of a C-family function/method node whose `type` field is
* `type_node`. Any other language, and any type with nothing around its base
* type, gets the base type text exactly as before. */
static char *c_declared_return_type(CBMExtractCtx *ctx, TSNode func_node, TSNode type_node) {
CBMArena *a = ctx->arena;
TSNode declarator = ts_node_child_by_field_name(func_node, TS_FIELD("declarator"));
if (!is_c_declarator_lang(ctx->language) || ts_node_is_null(declarator)) {
return cbm_node_text(a, type_node, ctx->source);
}
c_rt_out_t out = {NULL, 0};
if (c_rt_render(&out, func_node, type_node, declarator, ctx->source) == 0) {
return cbm_node_text(a, type_node, ctx->source);
}
out.buf = (char *)cbm_arena_alloc(a, out.len + NULL_TERM);
if (!out.buf) {
return cbm_node_text(a, type_node, ctx->source);
}
out.len = 0;
(void)c_rt_render(&out, func_node, type_node, declarator, ctx->source);
out.buf[out.len] = '\0';
return out.buf;
}
// C++: resolve trailing return type (auto f() -> Type) on a declarator node.
// Updates def->return_type and def->return_types if trailing type found.
static void resolve_cpp_trailing_return(CBMArena *a, TSNode func_node, const char *source,
@@ -3712,7 +3865,7 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
for (const char **f = rt_fields; *f; f++) {
TSNode rt = ts_node_child_by_field_name(func_node, *f, (uint32_t)strlen(*f));
if (!ts_node_is_null(rt)) {
def.return_type = cbm_node_text(a, rt, ctx->source);
def.return_type = c_declared_return_type(ctx, func_node, rt);
def.return_types = extract_return_types(a, rt, ctx->source, ctx->language);
break;
}
@@ -4964,7 +5117,7 @@ static void push_method_def(CBMExtractCtx *ctx, TSNode child, TSNode class_node,
for (const char **f = rt_fields; *f; f++) {
TSNode rt = ts_node_child_by_field_name(child, *f, (uint32_t)strlen(*f));
if (!ts_node_is_null(rt)) {
def.return_type = cbm_node_text(a, rt, ctx->source);
def.return_type = c_declared_return_type(ctx, child, rt);
break;
}
}
@@ -7053,6 +7206,13 @@ typedef struct {
int cap;
const char *path; // for the WARN when the ceiling is hit (may be NULL)
bool warned;
/* The per-file traversal scratch (ctx->scratch) when there is one: frames
* then come from memory the thread reuses file after file, where a malloc
* of 256 frames per file was 28 k allocations and 255 MB never written on
* the Go corpus (waste sanitizer, 2026-09-17). Growth copies into a
* doubled buffer and abandons the old one to the arena, like TSNodeStack.
* NULL: the heap, freed by the walk. */
CBMArena *arena;
} wd_stack_t;
// Generous safety ceiling (frames), env-overridable via CBM_WALK_DEFS_MAX.
@@ -7082,7 +7242,16 @@ static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) {
}
return; // bounded: stop growing (warned, not silent)
}
walk_defs_frame_t *nd = safe_realloc(s->data, (size_t)ncap * sizeof(walk_defs_frame_t));
walk_defs_frame_t *nd = NULL;
if (s->arena) {
nd = (walk_defs_frame_t *)cbm_arena_alloc(s->arena,
(size_t)ncap * sizeof(walk_defs_frame_t));
if (nd && s->top > 0) {
memcpy(nd, s->data, (size_t)s->top * sizeof(walk_defs_frame_t));
}
} else {
nd = safe_realloc(s->data, (size_t)ncap * sizeof(walk_defs_frame_t));
}
if (!nd) {
/* OOM — safe_realloc already freed the old buffer. Bail cleanly: drop
* pending frames so the walk_defs loop drains and exits without a NULL
@@ -7747,6 +7916,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
(void)depth_unused;
wd_stack_t s = {0};
s.path = ctx->rel_path;
s.arena = ctx->scratch;
wd_push(&s, root, ctx->enclosing_class_qn);
while (s.top > 0) {
@@ -7899,7 +8069,9 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
* collection is mandatory (see wd_push_children_reverse). */
wd_push_children_reverse(&s, node, frame.enclosing_class_qn);
}
free(s.data);
if (!s.arena) {
free(s.data);
}
}
void cbm_extract_definitions_without_module(CBMExtractCtx *ctx) {
internal/cbm/extract_imports.c
+13
View File
@@ -133,6 +133,19 @@ static void parse_go_import_spec(CBMExtractCtx *ctx, TSNode spec) {
return;
}
/* cgo: `import "C"` names the cgo PSEUDO-package, not a real one — "C" is
* reserved by go/build and no package with that import path can exist.
* Emitting it as an ordinary import makes the import resolver fall through
* to its symbol-name fallback (cbm_pipeline_resolve_import_node Strategy 3,
* pass_pkgmap.c), which matches the literal name "C" against every project
* node called C and picks the lexicographically smallest: on a real Go repo
* all 27 cgo files pointed their IMPORTS edge at the same unrelated `C`
* member of a test helper. Drop it — the C side is not a graph package, and
* the file's remaining imports are unaffected. */
if (strcmp(path, "C") == 0) {
return;
}
TSNode name_node = ts_node_child_by_field_name(spec, TS_FIELD("name"));
const char *local_name =
!ts_node_is_null(name_node) ? cbm_node_text(a, name_node, ctx->source) : path_last(a, path);
internal/cbm/extract_node_stack.h
+9 -8
View File
@@ -12,9 +12,9 @@
*
* Growth abandons the old buffer in that scratch arena, which is free because
* the arena dies with the file. The initial capacities below are unchanged, but
* they are no longer what a small file costs: cbm_extract_file_ex creates the
* scratch arena's first block for every call, including calls that build no
* stack at all, and reclaims it on the way out.
* they are no longer what a small file costs: cbm_extract_file_ex hands every
* call a lazy scratch arena, whose first block is taken by the first stack
* built, so a call that builds no stack allocates nothing.
*/
#ifndef CBM_EXTRACT_NODE_STACK_H
#define CBM_EXTRACT_NODE_STACK_H
@@ -84,13 +84,14 @@ static inline TSNode ts_nstack_pop(TSNodeStack *s) {
*/
static inline void ts_nstack_push_children(TSNodeStack *s, TSNode node) {
int base = s->count;
TSTreeCursor cursor = ts_tree_cursor_new(node);
if (ts_tree_cursor_goto_first_child(&cursor)) {
/* The thread's reusable cursor (cbm_thread_cursor): the walk completes
* before anything else on this thread can ask for it. */
TSTreeCursor *cursor = cbm_thread_cursor(node);
if (ts_tree_cursor_goto_first_child(cursor)) {
do {
ts_nstack_push(s, ts_tree_cursor_current_node(&cursor));
} while (ts_tree_cursor_goto_next_sibling(&cursor));
ts_nstack_push(s, ts_tree_cursor_current_node(cursor));
} while (ts_tree_cursor_goto_next_sibling(cursor));
}
ts_tree_cursor_delete(&cursor);
/* Reverse [base, count) so the first child pops first (forward order). */
int lo = base, hi = s->count - 1;
while (lo < hi) {
internal/cbm/extract_unified.c
+27
View File
@@ -5,6 +5,8 @@
#include "lang_specs.h" // CBMLangSpec, cbm_lang_spec, CBM_LANG_*
#include "tree_sitter/api.h" // TSNode, TSTreeCursor, ts_tree_cursor_*, ts_node_*
#include "foundation/constants.h"
#include "foundation/compat.h" // cbm_thread_cpu_time_ns
#include <stdlib.h>
enum { MAX_INFRA_BINDINGS = 8 };
@@ -2596,9 +2598,33 @@ void cbm_extract_unified(CBMExtractCtx *ctx) {
state.branch_depth = 0;
uint32_t depth = 0;
uint32_t visited = 0;
#ifdef CBM_ENABLE_TEST_SEAMS
/* CBM_TEST_WALK_BUDGET_NODES=<n>: spend the budget after n nodes. Production
* now counts nodes too, so this seam only shortens the budget; it no longer
* has to stand in for a mechanism of its own. */
uint32_t budget_nodes = ctx->walk_budget_nodes;
{
const char *seam = getenv("CBM_TEST_WALK_BUDGET_NODES");
if (seam && seam[0]) {
budget_nodes = (uint32_t)strtoul(seam, NULL, 10);
}
}
#else
const uint32_t budget_nodes = ctx->walk_budget_nodes;
#endif
for (;;) {
TSNode node = ts_tree_cursor_current_node(&cursor);
visited++;
/* The budget is spent in nodes, so the walk of a given file always ends
* on the same node — on any machine, under any load. A CPU deadline
* here made the graph differ between two runs on one machine; see
* CBM_WALK_MAX_NODES_DEFAULT (cbm.c). */
if (budget_nodes != 0 && visited > budget_nodes) {
ctx->walk_budget_exhausted = true;
break;
}
bool trivia = is_unified_trivia_node(node);
if (!trivia) {
/* Trivia consumes no semantic state. Scope expiry may be deferred
@@ -2652,6 +2678,7 @@ void cbm_extract_unified(CBMExtractCtx *ctx) {
}
cbm_finalize_lexical_usages(ctx, &state);
ctx->walk_nodes_visited = visited;
ts_tree_cursor_delete(&occurrence_cursor);
ts_tree_cursor_delete(&cursor);
}
internal/cbm/extract_usages.c
+63 -22
View File
@@ -1355,20 +1355,20 @@ static bool is_direct_argument_value(TSNode node) {
return !ts_node_is_null(grandparent) && is_argument_container_kind(ts_node_type(grandparent));
}
/* Cursor-backed counterpart for the unified walker. `cursor` must currently
* point at `node`; it is consumed while walking toward the argument owner. */
static bool is_direct_argument_value_cursor(TSNode node, TSTreeCursor *cursor) {
TSNode current = node;
while (!ts_node_is_null(current)) {
TSNode parent;
const char *field;
if (!occurrence_parent(cursor, current, &parent, &field))
return false;
/* The body of the walk above, entered one level in: for a caller that has
* ALREADY stepped the cursor onto `parent` and knows which `field` of it the
* node below occupies. A caller that climbed to find its site has that pair in
* hand, and re-deriving it would mean either stepping the cursor twice or
* paying ts_node_parent for what the cursor just told us. */
static bool is_direct_argument_value_from(TSNode parent, const char *field, TSTreeCursor *cursor) {
for (;;) {
const char *parent_kind = ts_node_type(parent);
if (is_labeled_argument_kind(parent_kind)) {
if (!field || strcmp(field, "value") != 0)
return false;
current = parent;
/* The labeled argument now has to be the argument value itself. */
if (!occurrence_parent(cursor, parent, &parent, &field))
return false;
continue;
}
if (field && strcmp(field, "arguments") == 0)
@@ -1389,7 +1389,16 @@ static bool is_direct_argument_value_cursor(TSNode node, TSTreeCursor *cursor) {
return occurrence_parent(cursor, parent, &grandparent, &argument_field) &&
is_argument_container_kind(ts_node_type(grandparent));
}
return false;
}
/* Cursor-backed counterpart for the unified walker. `cursor` must currently
* point at `node`; it is consumed while walking toward the argument owner. */
static bool is_direct_argument_value_cursor(TSNode node, TSTreeCursor *cursor) {
TSNode parent;
const char *field;
if (!occurrence_parent(cursor, node, &parent, &field))
return false;
return is_direct_argument_value_from(parent, field, cursor);
}
static bool is_direct_argument_value_walk(TSNode node, WalkState *state) {
@@ -1443,15 +1452,36 @@ static bool language_may_stamp_exact_callable_value_candidate(CBMLanguage langua
* arguments are narrow syntactic candidates only. A language LSP must still
* prove one target at this exact occurrence before the graph upgrades USAGE to
* CALL_REFERENCE; unresolved, reassigned, and composite expressions stay USAGE. */
static TSNode csharp_callable_value_site(TSNode node) {
usage_slow_parent_fallback_test_note();
/* One step up from `site`, on the walk cursor when the caller has one. Every
* other language branch in call_reference_candidate_site climbs this way; C#
* used ts_node_parent, which tree-sitter answers by descending from the ROOT,
* so each step costs O(depth) with a child scan at every level. On the
* generated .NET JIT tests — single expressions megabytes deep — that was
* 18-48 us per visited node and ~85% of the whole unified walk (sampled
* 2026-09-19: stamp_usage_site -> ts_node_parent ->
* ts_node_child_with_descendant). Those files are also the ones the old CPU
* deadline used to cut off mid-walk, so the slow path and the nondeterminism
* it forced were the same defect seen from two ends. */
static bool csharp_site_parent(TSTreeCursor *cursor, TSNode site, TSNode *parent,
const char **field) {
if (!cursor) {
usage_slow_parent_fallback_test_note();
}
return occurrence_parent(cursor, site, parent, field);
}
static TSNode csharp_callable_value_site(TSNode node, TSNode parent, const char *parent_field,
TSTreeCursor *cursor) {
const char *kind = ts_node_type(node);
if (strcmp(kind, "identifier") != 0 && strcmp(kind, "simple_identifier") != 0) {
return (TSNode){0};
}
/* The caller has already resolved node's parent (and the field it occupies)
* with the cursor, so the climb starts from that pair instead of asking for
* it again. Invariant below: `parent`/`field` always describe `site`. */
TSNode site = node;
TSNode parent = ts_node_parent(site);
const char *field = parent_field;
if (!ts_node_is_null(parent) && strcmp(ts_node_type(parent), "generic_name") == 0) {
TSNode name = ts_node_child_by_field_name(parent, TS_FIELD("name"));
if (ts_node_is_null(name) && ts_node_named_child_count(parent) > 0) {
@@ -1461,7 +1491,9 @@ static TSNode csharp_callable_value_site(TSNode node) {
return (TSNode){0};
}
site = parent;
parent = ts_node_parent(site);
if (!csharp_site_parent(cursor, site, &parent, &field)) {
return (TSNode){0};
}
}
if (!ts_node_is_null(parent) && strcmp(ts_node_type(parent), "member_access_expression") == 0) {
@@ -1470,7 +1502,9 @@ static TSNode csharp_callable_value_site(TSNode node) {
return (TSNode){0};
}
site = parent;
parent = ts_node_parent(site);
if (!csharp_site_parent(cursor, site, &parent, &field)) {
return (TSNode){0};
}
}
while (!ts_node_is_null(parent) &&
@@ -1483,12 +1517,17 @@ static TSNode csharp_callable_value_site(TSNode node) {
return (TSNode){0};
}
site = parent;
parent = ts_node_parent(site);
if (!csharp_site_parent(cursor, site, &parent, &field)) {
return (TSNode){0};
}
}
/* The wrapper chain proves admission, but the semantic row is keyed to the
* terminal method-name identifier. Keep the raw carrier on that leaf. */
return is_direct_argument_value(site) ? node : (TSNode){0};
if (ts_node_is_null(parent)) {
return (TSNode){0};
}
return is_direct_argument_value_from(parent, field, cursor) ? node : (TSNode){0};
}
/* Climb out of the parentheses wrapping a direct argument and return the
@@ -1545,10 +1584,12 @@ static TSNode call_reference_candidate_site(CBMExtractCtx *ctx, TSNode node, con
const char *parent_field = NULL;
if (!cursor) {
usage_slow_parent_fallback_test_note();
parent = ts_node_parent(node);
} else {
(void)occurrence_parent(cursor, node, &parent, &parent_field);
}
/* Resolve the parent AND the field node occupies in it, on both paths. The
* C# site walk below carries that field into the argument test, and
* ts_node_parent on its own would leave it NULL there — silently losing the
* "this node IS the arguments" case whenever no cursor is available. */
(void)occurrence_parent(cursor, node, &parent, &parent_field);
bool ts_family = ctx->language == CBM_LANG_JAVASCRIPT || ctx->language == CBM_LANG_TYPESCRIPT ||
ctx->language == CBM_LANG_TSX || ctx->language == CBM_LANG_ARKTS;
if (ts_family && strcmp(kind, "property_identifier") == 0 && !ts_node_is_null(parent) &&
@@ -1588,7 +1629,7 @@ static TSNode call_reference_candidate_site(CBMExtractCtx *ctx, TSNode node, con
return is_direct_argument_value_walk(node, state) ? node : (TSNode){0};
}
if (ctx->language == CBM_LANG_CSHARP) {
return csharp_callable_value_site(node);
return csharp_callable_value_site(node, parent, parent_field, cursor);
}
if (strcmp(kind, "identifier") != 0 && strcmp(kind, "simple_identifier") != 0) {
return (TSNode){0};
internal/cbm/lsp/c_lsp.c
+44 -44
View File
@@ -63,6 +63,7 @@ void c_lsp_init(CLSPContext *ctx, CBMArena *arena, const char *source, int sourc
ctx->source = source;
ctx->source_len = source_len;
ctx->registry = registry;
ctx->registry_head = (CBMTypeRegistry *)registry;
ctx->module_qn = module_qn;
ctx->module_qn_len = module_qn ? strlen(module_qn) : 0;
ctx->enclosing_func_qn = module_qn;
@@ -2819,10 +2820,10 @@ static const CBMRegisteredFunc *c_lookup_member_depth(CLSPContext *ctx, const ch
size_t slen = strlen(shortn);
const char *best_qn = NULL;
CBMTypeShortIter it;
cbm_registry_types_by_short_name(ctx->registry, shortn, &it);
cbm_registry_types_by_short_name_chain(ctx->registry, shortn, &it);
int i;
while ((i = cbm_type_short_iter_next(&it)) >= 0) {
const char *q = ctx->registry->types[i].qualified_name;
const char *q = it.reg->types[i].qualified_name;
if (!q) {
continue;
}
@@ -3670,12 +3671,12 @@ static const CBMRegisteredFunc *c_lookup_free_operator(CLSPContext *ctx, const c
const char *right_ns = extract_namespace_from_qn(ctx->arena, right_qn);
const CBMRegisteredFunc *match = NULL;
CBMFreeFuncIter it;
cbm_registry_free_funcs_by_short_name(ctx->registry, operator_name, &it);
cbm_registry_free_funcs_by_short_name_chain(ctx->registry, operator_name, &it);
for (int index = cbm_free_func_iter_next(&it); index >= 0;
index = cbm_free_func_iter_next(&it)) {
if (index >= ctx->registry->func_count)
if (index >= it.reg->func_count)
continue;
const CBMRegisteredFunc *candidate = &ctx->registry->funcs[index];
const CBMRegisteredFunc *candidate = &it.reg->funcs[index];
if (candidate->receiver_type || !candidate->short_name ||
strcmp(candidate->short_name, operator_name) != 0 || !candidate->qualified_name) {
continue;
@@ -3730,13 +3731,13 @@ static const CBMRegisteredFunc *c_lookup_free_unary_operator(CLSPContext *ctx,
const CBMRegisteredFunc *match = NULL;
const CBMType *actuals[1] = {operand_type};
CBMFreeFuncIter it;
cbm_registry_free_funcs_by_short_name(ctx->registry, operator_name, &it);
cbm_registry_free_funcs_by_short_name_chain(ctx->registry, operator_name, &it);
for (int index = cbm_free_func_iter_next(&it); index >= 0;
index = cbm_free_func_iter_next(&it)) {
if (index >= ctx->registry->func_count) {
if (index >= it.reg->func_count) {
continue;
}
const CBMRegisteredFunc *candidate = &ctx->registry->funcs[index];
const CBMRegisteredFunc *candidate = &it.reg->funcs[index];
if (candidate->receiver_type || !candidate->short_name ||
strcmp(candidate->short_name, operator_name) != 0 || !candidate->qualified_name) {
continue;
@@ -4882,18 +4883,16 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) {
// indexing bitcoin). Cross-phase template deduction then relies on the
// positional fallback, which is graceful degradation.
if (ctx->in_template && ctx->template_param_count > 0 && !ctx->registry_shared) {
// Find the registered function and set type_param_names
for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->func_count; ri++) {
CBMRegisteredFunc *rf = &((CBMTypeRegistry *)ctx->registry)->funcs[ri];
if (strcmp(rf->qualified_name, func_qn) == 0 && !rf->type_param_names) {
const char **tpn = (const char **)cbm_arena_alloc(
ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *));
for (int ti = 0; ti < ctx->template_param_count; ti++)
tpn[ti] = ctx->template_param_names[ti];
tpn[ctx->template_param_count] = NULL;
rf->type_param_names = tpn;
break;
}
// Set type_param_names on the registered function -- on the head's
// own copy of it (copy-on-write), never on a shared base entry.
CBMRegisteredFunc *rf = cbm_registry_func_for_update(ctx->registry_head, func_qn);
if (rf && !rf->type_param_names) {
const char **tpn = (const char **)cbm_arena_alloc(
ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *));
for (int ti = 0; ti < ctx->template_param_count; ti++)
tpn[ti] = ctx->template_param_names[ti];
tpn[ctx->template_param_count] = NULL;
rf->type_param_names = tpn;
}
}
@@ -4929,12 +4928,9 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) {
}
// Set min_params on the registered function (for default-arg overload matching)
if (total_params > 0 && defaulted_params > 0) {
for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->func_count; ri++) {
CBMRegisteredFunc *rf = &((CBMTypeRegistry *)ctx->registry)->funcs[ri];
if (strcmp(rf->qualified_name, func_qn) == 0 && rf->min_params < 0) {
rf->min_params = total_params - defaulted_params;
break;
}
CBMRegisteredFunc *rf = cbm_registry_func_for_update(ctx->registry_head, func_qn);
if (rf && rf->min_params < 0) {
rf->min_params = total_params - defaulted_params;
}
}
@@ -5059,7 +5055,7 @@ static void c_process_body_child(CLSPContext *ctx, TSNode child) {
tpn[ctx->template_param_count] = NULL;
rf.type_param_names = tpn;
}
cbm_registry_add_func((CBMTypeRegistry *)ctx->registry, rf);
cbm_registry_add_func(ctx->registry_head, rf);
}
}
}
@@ -5161,14 +5157,7 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) {
// Store template param names on the registered type (for substitution)
if (ctx->in_template && ctx->template_param_names && ctx->template_param_count > 0) {
CBMRegisteredType *rt = NULL;
for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->type_count; ri++) {
if (strcmp(((CBMTypeRegistry *)ctx->registry)->types[ri].qualified_name,
class_qn) == 0) {
rt = &((CBMTypeRegistry *)ctx->registry)->types[ri];
break;
}
}
CBMRegisteredType *rt = cbm_registry_type_for_update(ctx->registry_head, class_qn);
if (rt && !rt->type_param_names) {
const char **tpn = (const char **)cbm_arena_alloc(
ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *));
@@ -5341,14 +5330,25 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) {
should_upgrade = true;
}
if (should_upgrade) {
// Update existing entry's signature return type
const CBMType **new_rets = (const CBMType **)cbm_arena_alloc(
ctx->arena, 2 * sizeof(const CBMType *));
new_rets[0] = actual_ret;
new_rets[1] = NULL;
CBMRegisteredFunc *mut = (CBMRegisteredFunc *)existing;
mut->signature = cbm_type_func_replace_returns(
ctx->arena, existing->signature, new_rets);
/* Refine the return type on the HEAD copy only
* (copy-on-write). `existing` may live in the
* sealed shared base; writing a scratch-arena
* signature into it was read by other workers
* after this file's arena died (ASan heap-use-
* after-free in c_adl_resolve on dotnet/runtime,
* 2026-09-14). NULL = the head is sealed too:
* skip the refinement. */
CBMRegisteredFunc *mut = cbm_registry_func_for_update(
ctx->registry_head, existing->qualified_name);
if (mut) {
const CBMType **new_rets =
(const CBMType **)cbm_arena_alloc(
ctx->arena, 2 * sizeof(const CBMType *));
new_rets[0] = actual_ret;
new_rets[1] = NULL;
mut->signature = cbm_type_func_replace_returns(
ctx->arena, mut->signature, new_rets);
}
}
break;
}
@@ -5364,7 +5364,7 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) {
rf.receiver_type = ctx->enclosing_class_qn;
rf.signature = cbm_type_func(ctx->arena, NULL, NULL, rets);
rf.min_params = -1;
cbm_registry_add_func((CBMTypeRegistry *)ctx->registry, rf);
cbm_registry_add_func(ctx->registry_head, rf);
break;
} else if (strcmp(dk, "reference_declarator") == 0) {
actual_ret = cbm_type_reference(ctx->arena, actual_ret);
internal/cbm/lsp/c_lsp.h
+8 -3
View File
@@ -13,6 +13,11 @@ typedef struct {
const char *source;
int source_len;
const CBMTypeRegistry *registry;
/* The writable head of the registry chain: the per-file overlay the
* dispatcher hands in, or the per-file registry on the non-cross path.
* Every refinement (lazy add, min_params, template params) goes here
* via cbm_registry_*_for_update; nothing behind ->fallback is written. */
CBMTypeRegistry *registry_head;
CBMScope *current_scope;
// Include map: header_path -> namespace QN prefix
@@ -94,9 +99,9 @@ typedef struct {
// READ-ONLY across resolve workers — never mutate it
// (and never store per-worker arena pointers into it)
bool debug;
int eval_depth; // recursion depth for c_eval_expr_type (crash guard)
int eval_steps; // total expression eval calls for current file (hang guard)
int walk_depth; // c_resolve_calls_in_node self-recursion (AST nesting)
int eval_depth; // recursion depth for c_eval_expr_type (crash guard)
int eval_steps; // total expression eval calls for current file (hang guard)
int walk_depth; // c_resolve_calls_in_node self-recursion (AST nesting)
int control_flow_depth; // if/loop/switch/catch nesting; assignments merge fail-closed
} CLSPContext;
internal/cbm/lsp/cs_lsp.c
+16 -8
View File
@@ -492,10 +492,10 @@ const char *cs_resolve_type_name(CSLSPContext *ctx, const char *raw) {
CBMTypeShortIter ts_it;
int i;
if (ctx->registry) {
cbm_registry_types_by_short_name(ctx->registry, the_short, &ts_it);
cbm_registry_types_by_short_name_chain(ctx->registry, the_short, &ts_it);
}
while (ctx->registry && (i = cbm_type_short_iter_next(&ts_it)) >= 0) {
const CBMRegisteredType *cand = &ctx->registry->types[i];
const CBMRegisteredType *cand = &ts_it.reg->types[i];
if (!cand->short_name || strcmp(cand->short_name, the_short) != 0)
continue;
int score = 0;
@@ -643,12 +643,12 @@ static const CBMRegisteredFunc *cs_lookup_extension(CSLSPContext *ctx, const cha
* registration-order tie-break exactly (bucket chains are not in funcs[]
* order). */
CBMFreeFuncIter ext_it;
cbm_registry_free_funcs_by_short_name(ctx->registry, method_name, &ext_it);
cbm_registry_free_funcs_by_short_name_chain(ctx->registry, method_name, &ext_it);
const CBMRegisteredFunc *best = NULL;
int best_idx = -1;
int i;
while ((i = cbm_free_func_iter_next(&ext_it)) >= 0) {
const CBMRegisteredFunc *cand = &ctx->registry->funcs[i];
const CBMRegisteredFunc *cand = &ext_it.reg->funcs[i];
if (!cand->short_name || strcmp(cand->short_name, method_name) != 0)
continue;
if (cand->receiver_type)
@@ -1522,8 +1522,12 @@ static const char *cs_resolve_callable_name(CSLSPContext *ctx, const char *name)
* generated fixtures can register receiverless callables. Only accept a
* unique short-name match; ambiguity is intentionally not guessed. */
const CBMRegisteredFunc *only = NULL;
for (int i = 0; ctx->registry && i < ctx->registry->func_count; i++) {
const CBMRegisteredFunc *candidate = &ctx->registry->funcs[i];
CBMFreeFuncIter all_funcs;
if (ctx->registry) {
cbm_registry_all_funcs_chain(ctx->registry, &all_funcs);
}
for (int i = -1; ctx->registry && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) {
const CBMRegisteredFunc *candidate = &all_funcs.reg->funcs[i];
if (candidate->receiver_type || !candidate->short_name ||
strcmp(candidate->short_name, name) != 0) {
continue;
@@ -2142,8 +2146,12 @@ static void cs_resolve_invocation(CSLSPContext *ctx, TSNode call) {
/* Last resort: any free function with this short name in registry. */
const CBMRegisteredFunc *best = NULL;
int best_score = -1;
for (int i = 0; ctx->registry && i < ctx->registry->func_count; i++) {
const CBMRegisteredFunc *cand = &ctx->registry->funcs[i];
CBMFreeFuncIter all_funcs;
if (ctx->registry) {
cbm_registry_all_funcs_chain(ctx->registry, &all_funcs);
}
for (int i = -1; ctx->registry && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) {
const CBMRegisteredFunc *cand = &all_funcs.reg->funcs[i];
if (cand->receiver_type)
continue;
if (!cand->short_name || strcmp(cand->short_name, bare) != 0)
internal/cbm/lsp/go_lsp.c
+1017 -689
View File
File diff suppressed because it is too large Load Diff
internal/cbm/lsp/py_lsp.c
+61 -81
View File
@@ -238,6 +238,7 @@ void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int sou
ctx->source = source;
ctx->source_len = source_len;
ctx->registry = registry;
ctx->registry_head = (CBMTypeRegistry *)registry;
ctx->module_qn = module_qn;
ctx->resolved_calls = out;
ctx->current_scope = py_scope_push_checked(ctx);
@@ -394,7 +395,7 @@ static void py_import_syntax_match_add(PyImportSyntaxMatch *match, PyDirectImpor
* registry then retains responsibility for proving that the exact target is
* an undecorated, unambiguous callable. */
static const char *py_canonical_from_import_qn(PyLSPContext *ctx, TSNode statement,
TSNode imported_name, const char *incoming_qn) {
TSNode imported_name, const char *incoming_qn) {
if (!ctx || !incoming_qn || ts_node_is_null(statement) || ts_node_is_null(imported_name))
return NULL;
TSNode module = py_from_import_module_node(statement);
@@ -407,10 +408,9 @@ static const char *py_canonical_from_import_qn(PyLSPContext *ctx, TSNode stateme
while (module_suffix && *module_suffix == '.')
module_suffix++;
const char *full_suffix =
module_suffix && module_suffix[0]
? cbm_arena_sprintf(ctx->arena, "%s.%s", module_suffix, member)
: member;
const char *full_suffix = module_suffix && module_suffix[0]
? cbm_arena_sprintf(ctx->arena, "%s.%s", module_suffix, member)
: member;
if (!full_suffix)
return NULL;
if (py_qn_has_boundary_suffix(incoming_qn, full_suffix))
@@ -445,17 +445,16 @@ static void py_import_match_statement(PyLSPContext *ctx, TSNode stmt, const char
TSNode alias = ts_node_child_by_field_name(item, "alias", 5);
if (py_import_node_text_equals(ctx, alias, local)) {
const char *canonical = py_canonical_from_import_qn(ctx, stmt, name, qn);
py_import_syntax_match_add(
match, canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED,
canonical ? canonical : qn);
py_import_syntax_match_add(match,
canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED,
canonical ? canonical : qn);
}
} else if ((strcmp(kind, "identifier") == 0 ||
strcmp(kind, "dotted_name") == 0) &&
} else if ((strcmp(kind, "identifier") == 0 || strcmp(kind, "dotted_name") == 0) &&
py_import_node_text_equals(ctx, item, local)) {
const char *canonical = py_canonical_from_import_qn(ctx, stmt, item, qn);
py_import_syntax_match_add(
match, canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED,
canonical ? canonical : qn);
py_import_syntax_match_add(match,
canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED,
canonical ? canonical : qn);
}
}
return;
@@ -489,8 +488,7 @@ static void py_import_match_statement(PyLSPContext *ctx, TSNode stmt, const char
}
}
static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match,
const char **qn_io) {
static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match, const char **qn_io) {
if (!match || !qn_io)
return PY_DIRECT_IMPORT_UNKNOWN;
if (match->count == 0)
@@ -503,8 +501,7 @@ static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match,
}
static PyDirectImportKind py_import_kind_from_statement(PyLSPContext *ctx, TSNode stmt,
const char *local,
const char **qn_io) {
const char *local, const char **qn_io) {
const char *qn = qn_io ? *qn_io : NULL;
if (!ctx || !local || !qn || !qn_io || ts_node_is_null(stmt))
return PY_DIRECT_IMPORT_UNKNOWN;
@@ -513,8 +510,8 @@ static PyDirectImportKind py_import_kind_from_statement(PyLSPContext *ctx, TSNod
return py_import_match_result(&match, qn_io);
}
static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, TSNode root,
const char *local, const char **qn_io) {
static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, TSNode root, const char *local,
const char **qn_io) {
const char *qn = qn_io ? *qn_io : NULL;
if (!ctx || !local || !qn || !qn_io || ts_node_is_null(root))
return PY_DIRECT_IMPORT_UNKNOWN;
@@ -589,36 +586,34 @@ static void py_bind_import_index(PyLSPContext *ctx, int index, bool synthetic_fa
if (!local || !qn || strcmp(local, "*") == 0)
return;
PyDirectImportKind direct_kind = ctx->import_kinds
? (PyDirectImportKind)ctx->import_kinds[index]
: PY_DIRECT_IMPORT_UNKNOWN;
PyDirectImportKind direct_kind =
ctx->import_kinds ? (PyDirectImportKind)ctx->import_kinds[index] : PY_DIRECT_IMPORT_UNKNOWN;
bool from_style = direct_kind == PY_FROM_IMPORT ||
(direct_kind == PY_DIRECT_IMPORT_UNKNOWN &&
import_is_from_style(local, qn));
(direct_kind == PY_DIRECT_IMPORT_UNKNOWN && import_is_from_style(local, qn));
const CBMType *t;
if (direct_kind == PY_DIRECT_IMPORT_UNALIASED) {
// Python binds only the root of an unaliased dotted import.
// Python binds only the root of an unaliased dotted import.
t = cbm_type_module(ctx->arena, local);
} else if (direct_kind == PY_DIRECT_IMPORT_ALIASED) {
t = cbm_type_module(ctx->arena, qn);
} else if (from_style) {
// `from X import Y` — bind Y to NAMED(X.Y). Phase 6 attribute
// resolution checks the registry to upgrade to MODULE / class
// / function as appropriate.
// `from X import Y` — bind Y to NAMED(X.Y). Phase 6 attribute
// resolution checks the registry to upgrade to MODULE / class
// / function as appropriate.
t = cbm_type_named(ctx->arena, qn);
} else if (strchr(qn, '.') != NULL) {
// Dotted path whose tail does NOT match the local name: an
// ALIASED binding — `from X import Y as Z` (Z names function/
// class X.Y) or `import a.b as z` (z names module a.b). The
// CBMImport shape cannot distinguish the two, but NAMED(qn)
// covers both: phase 6 upgrades it to the registered function/
// class for the from-import and to MODULE for the module alias.
// Binding MODULE here made `g()` calls on `from m import f as g`
// resolve as calls on a module — lsp=MISS, and the whole CALLS
// edge was lost (#988).
// Dotted path whose tail does NOT match the local name: an
// ALIASED binding — `from X import Y as Z` (Z names function/
// class X.Y) or `import a.b as z` (z names module a.b). The
// CBMImport shape cannot distinguish the two, but NAMED(qn)
// covers both: phase 6 upgrades it to the registered function/
// class for the from-import and to MODULE for the module alias.
// Binding MODULE here made `g()` calls on `from m import f as g`
// resolve as calls on a module — lsp=MISS, and the whole CALLS
// edge was lost (#988).
t = cbm_type_named(ctx->arena, qn);
} else {
// `import X` / `import X as Y` (single segment) — MODULE(X).
// `import X` / `import X as Y` (single segment) — MODULE(X).
t = cbm_type_module(ctx->arena, qn);
}
const CBMRegisteredFunc *imported =
@@ -841,8 +836,7 @@ static void py_emit_resolved_call(PyLSPContext *ctx, const char *callee_qn, cons
}
static void py_emit_resolved_reference(PyLSPContext *ctx, const char *callee_qn,
const char *source_name, TSNode site,
const char *strategy) {
const char *source_name, TSNode site, const char *strategy) {
if (!ctx || !ctx->resolved_calls || !callee_qn || !ctx->enclosing_func_qn ||
ts_node_is_null(site)) {
return;
@@ -1027,8 +1021,7 @@ static void py_resolve_value_references_at(PyLSPContext *ctx, TSNode call) {
if (candidate && cbm_scope_contains(ctx->current_scope, source_name)) {
const CBMType *binding =
cbm_type_resolve_alias(cbm_scope_lookup(ctx->current_scope, source_name));
if (binding && binding->kind == CBM_TYPE_NAMED &&
binding->data.named.qualified_name &&
if (binding && binding->kind == CBM_TYPE_NAMED && binding->data.named.qualified_name &&
strcmp(binding->data.named.qualified_name, candidate) == 0) {
py_emit_unresolved_reference(ctx, candidate, arg);
}
@@ -1195,16 +1188,10 @@ static void py_register_instance_field(PyLSPContext *ctx, const char *class_qn,
return;
}
// Find the type entry. cbm_registry_lookup_type returns a const pointer;
// we need a mutable pointer into the registry's array.
CBMRegisteredType *rt = NULL;
for (int i = 0; i < ctx->registry->type_count; i++) {
const char *qn = ctx->registry->types[i].qualified_name;
if (qn && strcmp(qn, class_qn) == 0) {
rt = &ctx->registry->types[i];
break;
}
}
// The writable entry for the class: the head's own, or a copy of the
// base entry made in the head (copy-on-write). The shared base is never
// written -- it used to be, from every parallel resolve worker.
CBMRegisteredType *rt = cbm_registry_type_for_update(ctx->registry_head, class_qn);
if (!rt)
return;
@@ -1570,11 +1557,12 @@ static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node)
const char *prefix = cbm_arena_sprintf(ctx->arena, "%s.", qn);
size_t prefix_len = strlen(prefix);
bool is_submodule = false;
for (int i = 0; i < ctx->registry->func_count; i++) {
const char *fqn = ctx->registry->funcs[i].qualified_name;
CBMFreeFuncIter all_funcs;
cbm_registry_all_funcs_chain(ctx->registry, &all_funcs);
for (int i = -1; !is_submodule && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) {
const char *fqn = all_funcs.reg->funcs[i].qualified_name;
if (fqn && strncmp(fqn, prefix, prefix_len) == 0) {
is_submodule = true;
break;
}
}
if (is_submodule)
@@ -2600,8 +2588,7 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) {
const char *local = ctx->import_local_names[i];
const char *import_qn = ctx->import_module_qns[i];
if (local && import_qn && strcmp(local, fname) == 0 &&
strcmp(import_qn, qn) == 0 &&
py_import_index_is_from_binding(ctx, i)) {
strcmp(import_qn, qn) == 0 && py_import_index_is_from_binding(ctx, i)) {
imported_alias = true;
break;
}
@@ -2758,8 +2745,8 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) {
// Skip if mod is already rooted under the project to avoid
// "<root>.<root>.mod".
if (!(strncmp(mod, ctx->module_qn, root_len) == 0 && mod[root_len] == '.')) {
char *qual_mod = (char *)cbm_arena_alloc(ctx->arena, root_len + 1 +
strlen(mod) + 1);
char *qual_mod =
(char *)cbm_arena_alloc(ctx->arena, root_len + 1 + strlen(mod) + 1);
if (qual_mod) {
memcpy(qual_mod, ctx->module_qn, root_len);
qual_mod[root_len] = '.';
@@ -4047,10 +4034,8 @@ static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char
// For methods, bind `self`/`cls` AFTER param walk so the receiver type
// wins over the unannotated `self` / `cls` parameter declaration.
if (ctx->enclosing_class_qn) {
py_scope_bind(ctx, "self",
cbm_type_named(ctx->arena, ctx->enclosing_class_qn));
py_scope_bind(ctx, "cls",
cbm_type_named(ctx->arena, ctx->enclosing_class_qn));
py_scope_bind(ctx, "self", cbm_type_named(ctx->arena, ctx->enclosing_class_qn));
py_scope_bind(ctx, "cls", cbm_type_named(ctx->arena, ctx->enclosing_class_qn));
}
TSNode body = ts_node_child_by_field_name(func_node, "body", 4);
@@ -4197,12 +4182,14 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, TSNode root) {
if (!ctx || !ctx->registry || !ctx->module_qn)
return;
size_t prefix_len = strlen(ctx->module_qn);
for (int i = 0; i < ctx->registry->type_count; i++) {
const CBMRegisteredType *type = &ctx->registry->types[i];
CBMTypeShortIter all_types;
cbm_registry_all_types_chain(ctx->registry, &all_types);
for (int i = -1; (i = cbm_type_short_iter_next(&all_types)) >= 0;) {
const CBMRegisteredType *type = &all_types.reg->types[i];
const char *qn = type->qualified_name;
const char *name = type->short_name;
if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 ||
qn[prefix_len] != '.' || py_root_defines_class_named(ctx, root, name)) {
if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || qn[prefix_len] != '.' ||
py_root_defines_class_named(ctx, root, name)) {
continue;
}
py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn));
@@ -4502,8 +4489,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int
py_invalidate_possible_bindings(ctx, definition, depth + 1);
return;
}
if (strcmp(kind, "import_statement") == 0 ||
strcmp(kind, "import_from_statement") == 0) {
if (strcmp(kind, "import_statement") == 0 || strcmp(kind, "import_from_statement") == 0) {
py_invalidate_import_bindings(ctx, node);
return;
}
@@ -4511,8 +4497,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int
TSNode right = ts_node_child_by_field_name(node, "right", 5);
if (ts_node_is_null(right))
return;
py_invalidate_binding_target(ctx, ts_node_child_by_field_name(node, "left", 4),
depth + 1);
py_invalidate_binding_target(ctx, ts_node_child_by_field_name(node, "left", 4), depth + 1);
py_invalidate_possible_bindings(ctx, right, depth + 1);
return;
}
@@ -4523,8 +4508,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int
py_invalidate_possible_bindings(ctx, right, depth + 1);
return;
}
if (strcmp(kind, "named_expression") == 0 ||
strcmp(kind, "assignment_expression") == 0) {
if (strcmp(kind, "named_expression") == 0 || strcmp(kind, "assignment_expression") == 0) {
TSNode name = ts_node_child_by_field_name(node, "name", 4);
if (ts_node_is_null(name))
name = ts_node_child_by_field_name(node, "left", 4);
@@ -4619,8 +4603,7 @@ static void py_replay_import_local(PyLSPContext *ctx, TSNode stmt, const char *l
continue;
}
const char *candidate_qn = ctx->import_module_qns[i];
PyDirectImportKind kind =
py_import_kind_from_statement(ctx, stmt, local, &candidate_qn);
PyDirectImportKind kind = py_import_kind_from_statement(ctx, stmt, local, &candidate_qn);
if (!py_replayable_import_kind(kind))
continue;
if (!chosen_qn) {
@@ -4642,8 +4625,7 @@ static void py_replay_import_local(PyLSPContext *ctx, TSNode stmt, const char *l
py_bind_import_index(ctx, chosen, false);
}
static void py_replay_import_statement(PyLSPContext *ctx, TSNode stmt,
unsigned char *consumed) {
static void py_replay_import_statement(PyLSPContext *ctx, TSNode stmt, unsigned char *consumed) {
if (py_import_statement_is_wildcard(stmt)) {
py_invalidate_module_bindings_for_wildcard(ctx);
return;
@@ -4686,8 +4668,7 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) {
py_bind_external_module_classes(ctx, root);
unsigned char *consumed_imports = NULL;
if (ctx->import_count > 0) {
consumed_imports =
(unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count);
consumed_imports = (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count);
if (consumed_imports)
memset(consumed_imports, 0, (size_t)ctx->import_count);
}
@@ -4700,8 +4681,7 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) {
for (uint32_t i = 0; i < nc; i++) {
TSNode c = ts_node_named_child(root, i);
const char *ck = ts_node_type(c);
if (strcmp(ck, "import_statement") == 0 ||
strcmp(ck, "import_from_statement") == 0) {
if (strcmp(ck, "import_statement") == 0 || strcmp(ck, "import_from_statement") == 0) {
py_replay_import_statement(ctx, c, consumed_imports);
} else if (strcmp(ck, "function_definition") == 0) {
py_bind_module_function(ctx, c);
internal/cbm/lsp/py_lsp.h
+4
View File
@@ -44,6 +44,10 @@ typedef struct {
const char *source;
int source_len;
const CBMTypeRegistry *registry;
/* Writable head of the registry chain (the per-file overlay, or the
* per-file registry on the non-cross path); refinements go here via
* cbm_registry_*_for_update, never into anything behind ->fallback. */
CBMTypeRegistry *registry_head;
CBMScope *current_scope;
// Import map: local_name -> module_qn (arena-allocated, NULL-terminated).
internal/cbm/lsp/ts_lsp.c
+9
View File
@@ -2800,8 +2800,17 @@ static void resolve_value_references_at(TSLSPContext *ctx, TSNode call_node) {
}
}
static TSNode unwrap_await_callee(TSNode node) {
if (ts_node_is_null(node) || strcmp(ts_node_type(node), "await_expression") != 0 ||
ts_node_named_child_count(node) == 0) {
return node;
}
return ts_node_named_child(node, 0);
}
static void resolve_call_at(TSLSPContext *ctx, TSNode call_node) {
TSNode fn = ts_node_child_by_field_name(call_node, "function", TS_LSP_FIELD_LEN("function"));
fn = unwrap_await_callee(fn);
if (ts_node_is_null(fn))
return;
internal/cbm/lsp/type_registry.c
+231 -8
View File
@@ -301,6 +301,9 @@ static void build_ffunc_short_index(CBMTypeRegistry *reg, CBMArena *idx_arena) {
void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
CBMTypeShortIter *out) {
out->reg = reg;
out->chain = false;
out->key = short_name;
out->shadow = NULL;
out->hash = fnv1a(short_name);
if (reg->type_qn_buckets && reg->type_qn_bucket_count > 0) {
if (reg->type_short_buckets && reg->type_short_bucket_count > 0) {
@@ -319,7 +322,7 @@ void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *sh
}
}
int cbm_type_short_iter_next(CBMTypeShortIter *it) {
static int type_short_next_local(CBMTypeShortIter *it) {
const CBMTypeRegistry *reg = it->reg;
while (it->chain_idx >= 0) {
const CBMRegistryHashEntry *e = &reg->type_short_entries[it->chain_idx];
@@ -335,9 +338,79 @@ int cbm_type_short_iter_next(CBMTypeShortIter *it) {
return -1;
}
static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg,
const char *qualified_name);
static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg,
const char *qualified_name);
/* A base entry is shadowed when the head holds an entry with the same QN. */
static bool type_shadowed(const CBMTypeRegistry *shadow, const CBMTypeRegistry *reg, int i) {
if (!shadow || shadow == reg || !reg->types[i].qualified_name) {
return false;
}
return lookup_type_self(shadow, reg->types[i].qualified_name) != NULL;
}
static bool func_shadowed(const CBMTypeRegistry *shadow, const CBMTypeRegistry *reg, int i) {
if (!shadow || shadow == reg || !reg->funcs[i].qualified_name) {
return false;
}
return lookup_func_self(shadow, reg->funcs[i].qualified_name) != NULL;
}
static void type_short_open_linear(const CBMTypeRegistry *reg, CBMTypeShortIter *out) {
out->reg = reg;
out->hash = 0;
out->chain_idx = -1;
out->tail_i = 0;
out->tail_end = reg->type_count;
}
int cbm_type_short_iter_next(CBMTypeShortIter *it) {
for (;;) {
int p = type_short_next_local(it);
if (p >= 0) {
if (it->chain && type_shadowed(it->shadow, it->reg, p)) {
continue;
}
return p;
}
if (!it->chain || !it->reg->fallback) {
return -1;
}
const CBMTypeRegistry *next = it->reg->fallback;
const char *key = it->key;
const CBMTypeRegistry *shadow = it->shadow;
if (key) {
cbm_registry_types_by_short_name(next, key, it);
} else {
type_short_open_linear(next, it);
}
it->chain = true;
it->key = key;
it->shadow = shadow;
}
}
void cbm_registry_types_by_short_name_chain(const CBMTypeRegistry *head, const char *short_name,
CBMTypeShortIter *out) {
cbm_registry_types_by_short_name(head, short_name, out);
out->chain = true;
out->shadow = head;
}
void cbm_registry_all_types_chain(const CBMTypeRegistry *head, CBMTypeShortIter *out) {
type_short_open_linear(head, out);
out->chain = true;
out->key = NULL;
out->shadow = head;
}
void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char *bare,
CBMTypeEmbedIter *out) {
out->reg = reg;
out->chain = false;
out->key = bare;
out->shadow = NULL;
out->hash = fnv1a(bare);
out->prev_type = -1;
if (reg->type_qn_buckets && reg->type_qn_bucket_count > 0) {
@@ -358,7 +431,7 @@ void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char
}
}
int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) {
static int type_embed_next_local(CBMTypeEmbedIter *it) {
const CBMTypeRegistry *reg = it->reg;
while (it->chain_idx >= 0) {
const CBMRegistryHashEntry *e = &reg->type_embed_entries[it->chain_idx];
@@ -377,9 +450,40 @@ int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) {
return -1;
}
int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) {
for (;;) {
int p = type_embed_next_local(it);
if (p >= 0) {
if (it->chain && type_shadowed(it->shadow, it->reg, p)) {
continue;
}
return p;
}
if (!it->chain || !it->reg->fallback) {
return -1;
}
const CBMTypeRegistry *next = it->reg->fallback;
const char *key = it->key;
const CBMTypeRegistry *shadow = it->shadow;
cbm_registry_types_by_embedded_bare(next, key, it);
it->chain = true;
it->shadow = shadow;
}
}
void cbm_registry_types_by_embedded_bare_chain(const CBMTypeRegistry *head, const char *bare,
CBMTypeEmbedIter *out) {
cbm_registry_types_by_embedded_bare(head, bare, out);
out->chain = true;
out->shadow = head;
}
void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
CBMFreeFuncIter *out) {
out->reg = reg;
out->chain = false;
out->key = short_name;
out->shadow = NULL;
out->hash = fnv1a(short_name);
if (reg->func_qn_buckets && reg->func_qn_bucket_count > 0) {
if (reg->ffunc_short_buckets && reg->ffunc_short_bucket_count > 0) {
@@ -397,7 +501,7 @@ void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const cha
}
}
int cbm_free_func_iter_next(CBMFreeFuncIter *it) {
static int free_func_next_local(CBMFreeFuncIter *it) {
const CBMTypeRegistry *reg = it->reg;
while (it->chain_idx >= 0) {
const CBMRegistryHashEntry *e = &reg->ffunc_short_entries[it->chain_idx];
@@ -413,9 +517,57 @@ int cbm_free_func_iter_next(CBMFreeFuncIter *it) {
return -1;
}
static void free_func_open_linear(const CBMTypeRegistry *reg, CBMFreeFuncIter *out) {
out->reg = reg;
out->hash = 0;
out->chain_idx = -1;
out->tail_i = 0;
out->tail_end = reg->func_count;
}
int cbm_free_func_iter_next(CBMFreeFuncIter *it) {
for (;;) {
int p = free_func_next_local(it);
if (p >= 0) {
if (it->chain && func_shadowed(it->shadow, it->reg, p)) {
continue;
}
return p;
}
if (!it->chain || !it->reg->fallback) {
return -1;
}
const CBMTypeRegistry *next = it->reg->fallback;
const char *key = it->key;
const CBMTypeRegistry *shadow = it->shadow;
if (key) {
cbm_registry_free_funcs_by_short_name(next, key, it);
} else {
free_func_open_linear(next, it);
}
it->chain = true;
it->key = key;
it->shadow = shadow;
}
}
void cbm_registry_free_funcs_by_short_name_chain(const CBMTypeRegistry *head,
const char *short_name, CBMFreeFuncIter *out) {
cbm_registry_free_funcs_by_short_name(head, short_name, out);
out->chain = true;
out->shadow = head;
}
void cbm_registry_all_funcs_chain(const CBMTypeRegistry *head, CBMFreeFuncIter *out) {
free_func_open_linear(head, out);
out->chain = true;
out->key = NULL;
out->shadow = head;
}
void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn,
const char *method_name, CBMMethodIter *out) {
memset(out, 0, sizeof(*out));
memset(out, 0, sizeof(*out)); /* chain=false, shadow=NULL */
out->reg = reg;
out->receiver_qn = receiver_qn;
out->method_name = method_name;
@@ -435,10 +587,7 @@ void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn,
}
}
int cbm_method_iter_next(CBMMethodIter *it) {
if (!it || !it->reg || !it->receiver_qn || !it->method_name) {
return -1;
}
static int method_next_local(CBMMethodIter *it) {
const CBMTypeRegistry *reg = it->reg;
while (it->chain_idx >= 0) {
const CBMRegistryHashEntry *e = &reg->method_entries[it->chain_idx];
@@ -465,6 +614,80 @@ int cbm_method_iter_next(CBMMethodIter *it) {
return -1;
}
int cbm_method_iter_next(CBMMethodIter *it) {
if (!it || !it->reg || !it->receiver_qn || !it->method_name) {
return -1;
}
for (;;) {
int p = method_next_local(it);
if (p >= 0) {
if (it->chain && func_shadowed(it->shadow, it->reg, p)) {
continue;
}
return p;
}
if (!it->chain || !it->reg->fallback) {
return -1;
}
const CBMTypeRegistry *next = it->reg->fallback;
const CBMTypeRegistry *shadow = it->shadow;
cbm_registry_methods(next, it->receiver_qn, it->method_name, it);
it->chain = true;
it->shadow = shadow;
}
}
void cbm_registry_methods_chain(const CBMTypeRegistry *head, const char *receiver_qn,
const char *method_name, CBMMethodIter *out) {
cbm_registry_methods(head, receiver_qn, method_name, out);
out->chain = true;
out->shadow = head;
}
CBMRegisteredFunc *cbm_registry_func_for_update(CBMTypeRegistry *head, const char *qualified_name) {
if (!head || !qualified_name || head->read_only) {
return NULL;
}
const CBMRegisteredFunc *own = lookup_func_self(head, qualified_name);
if (own) {
return (CBMRegisteredFunc *)own;
}
for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) {
const CBMRegisteredFunc *base = lookup_func_self(r, qualified_name);
if (base) {
int before = head->func_count;
cbm_registry_add_func(head, *base);
if (head->func_count == before + 1) {
return &head->funcs[before];
}
return NULL;
}
}
return NULL;
}
CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const char *qualified_name) {
if (!head || !qualified_name || head->read_only) {
return NULL;
}
const CBMRegisteredType *own = lookup_type_self(head, qualified_name);
if (own) {
return (CBMRegisteredType *)own;
}
for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) {
const CBMRegisteredType *base = lookup_type_self(r, qualified_name);
if (base) {
int before = head->type_count;
cbm_registry_add_type(head, *base);
if (head->type_count == before + 1) {
return &head->types[before];
}
return NULL;
}
}
return NULL;
}
void cbm_registry_finalize_into(CBMTypeRegistry *reg, CBMArena *idx_arena) {
if (!reg || !idx_arena)
return;
internal/cbm/lsp/type_registry.h
+46
View File
@@ -242,6 +242,16 @@ typedef struct {
int chain_idx;
int tail_i;
int tail_end;
/* Chain mode (see cbm_registry_*_chain): when this registry is exhausted
* the iterator re-opens on reg->fallback; `reg` then names the registry
* the yielded index belongs to, so callers deref it->reg->types[i], never
* the registry they started from. `key` re-opens the same query on the
* next link; NULL = linear scan. `shadow` is the head: fallback entries
* whose qualified_name the head also holds are skipped (an overlay copy
* hides its base original). */
bool chain;
const char *key;
const CBMTypeRegistry *shadow;
} CBMTypeShortIter;
void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
CBMTypeShortIter *out);
@@ -263,6 +273,9 @@ typedef struct {
int tail_i; // next tail/linear type index
int tail_end; // reg->type_count snapshot
int prev_type; // last yielded type index (adjacent-dedup); -1 = none
bool chain;
const char *key;
const CBMTypeRegistry *shadow;
} CBMTypeEmbedIter;
void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char *bare,
CBMTypeEmbedIter *out);
@@ -277,6 +290,9 @@ typedef struct {
int chain_idx;
int tail_i;
int tail_end;
bool chain;
const char *key;
const CBMTypeRegistry *shadow;
} CBMFreeFuncIter;
void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
@@ -296,11 +312,41 @@ typedef struct {
int chain_idx;
int tail_i;
int tail_end;
bool chain;
const CBMTypeRegistry *shadow;
} CBMMethodIter;
void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn,
const char *method_name, CBMMethodIter *out);
int cbm_method_iter_next(CBMMethodIter *it);
/* ── Chain-aware iteration and copy-on-write (per-file overlay contract) ──
*
* A resolve walk is handed a per-file OVERLAY registry chained (`fallback`) to
* the immutable shared base. Lookups already chain; these make the index
* iterators and the whole-registry scans chain too, and give a walk one way
* to refine an entry: copy it into the head first. Nothing behind `fallback`
* is ever written. Every yielded index belongs to it->reg at that moment. */
void cbm_registry_types_by_short_name_chain(const CBMTypeRegistry *head, const char *short_name,
CBMTypeShortIter *out);
void cbm_registry_types_by_embedded_bare_chain(const CBMTypeRegistry *head, const char *bare,
CBMTypeEmbedIter *out);
void cbm_registry_free_funcs_by_short_name_chain(const CBMTypeRegistry *head,
const char *short_name, CBMFreeFuncIter *out);
void cbm_registry_methods_chain(const CBMTypeRegistry *head, const char *receiver_qn,
const char *method_name, CBMMethodIter *out);
/* Linear scans over every func / type in the chain (head first, shadowed
* base entries skipped). Same iterator types, same it->reg contract. */
void cbm_registry_all_funcs_chain(const CBMTypeRegistry *head, CBMFreeFuncIter *out);
void cbm_registry_all_types_chain(const CBMTypeRegistry *head, CBMTypeShortIter *out);
/* The writable entry for qualified_name: the head's own entry if it has one,
* else a copy of the first fallback entry added to the head (copy-on-write),
* else NULL. NULL also when the head is sealed (read_only) -- a walk must never
* mutate a shared registry, so the caller skips the refinement, exactly as the
* sealed-registry no-op in cbm_registry_add_func does today. */
CBMRegisteredFunc *cbm_registry_func_for_update(CBMTypeRegistry *head, const char *qualified_name);
CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const char *qualified_name);
// --- TS-specific helpers (return NULL for types without these signatures) ---
// If the type has a call signature (e.g., `interface F { (x:number): string }`), return
internal/cbm/preprocessor.cpp
+304 -1
View File
@@ -42,6 +42,283 @@ static bool has_preprocessor_work(const char *source, int source_len) {
return false;
}
// ── Export-macro candidates (#1989) ─────────────────────────────────────────
// Build systems define symbol-export macros empty on the compiler command line
// (UE's UBT: `/D "MODULE_API="`; CMake generate_export_header: `<lib>_EXPORT`),
// so they never appear as #define lines in the source. tree-sitter has no
// preprocessor state either, and `class MODULE_API Foo` misparses with the
// macro token as the type name (worse: enums and free functions are lost to
// ERROR regions entirely). To mirror the real compile line, the preprocessed
// second pass predefines the conventional export-macro-shaped identifiers found
// in the file as empty. The shape is deliberately narrow — ALL_CAPS identifier
// ending in a known export suffix, with a non-trivial prefix — and the list is
// capped, so ordinary all-caps identifiers cannot be swept in wholesale.
static const char *kExportMacroSuffixes[] = {"_API", "_EXPORT", "_IMPORT",
"_DLLEXPORT", "_DEPRECATED", NULL};
static bool is_export_macro_shape(const char *id, size_t len) {
if (id[0] < 'A' || id[0] > 'Z') {
return false;
}
for (size_t i = 0; i < len; i++) {
char c = id[i];
bool upper = c >= 'A' && c <= 'Z';
bool digit = c >= '0' && c <= '9';
if (!upper && !digit && c != '_') {
return false;
}
}
for (int s = 0; kExportMacroSuffixes[s]; s++) {
size_t slen = strlen(kExportMacroSuffixes[s]);
// Require at least two prefix characters before the suffix so a bare
// "X_API"-style token (single-letter, easily a genuine symbol) stays out.
if (len >= slen + 2 && strncmp(id + len - slen, kExportMacroSuffixes[s], slen) == 0) {
return true;
}
}
return false;
}
static bool is_identifier_start(char c) {
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_';
}
static bool is_identifier_char(char c) {
return is_identifier_start(c) || (c >= '0' && c <= '9');
}
// C++ raw string literals (R"delim(...)delim"). Returns the index one past
// the closing quote, or -1 when the form cannot be confidently recognized
// (unterminated, malformed delimiter) — the caller then STOPS collecting so
// the mis-modeled literal cannot poison the scan state of the rest of the
// file (#1989 review).
static int skip_raw_string(const char *source, int source_len, int quote) {
int j = quote + 1;
int delim_start = j;
while (j < source_len) {
char c = source[j];
if (c == '(') {
break;
}
// Delimiter chars: the standard d-char set excludes space, parens,
// backslash, and the line breaks — a double quote IS a legal d-char
// (R"""(...)""" compiles), so it must not end the delimiter (#1989
// review round 3).
if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == ')' || c == '\\' ||
j - delim_start >= 16) {
return -1;
}
j++;
}
if (j >= source_len || source[j] != '(') {
return -1;
}
int delim_len = j - delim_start;
j++; // past the opening "("
while (j < source_len) {
if (source[j] == ')') {
int k = j + 1;
int m = 0;
while (m < delim_len && k + m < source_len &&
source[k + m] == source[delim_start + m]) {
m++;
}
if (m == delim_len && k + m < source_len && source[k + m] == '"') {
return k + m + 1;
}
}
j++;
}
return -1; // unterminated raw string: uncertain
}
// If the identifier just scanned is a raw-string prefix (R, LR, uR, UR, u8R)
// immediately followed by a quote, return skip_raw_string(...). Returns 0 for
// an ordinary identifier (including an identifier merely ENDING in R — only a
// standalone prefix token starts a raw string) and -1 on an unrecognizable
// raw form.
static int raw_string_after_prefix(const char *source, int source_len, int start, int end) {
static const char *prefixes[] = {"R", "LR", "uR", "UR", "u8R", NULL};
size_t idlen = (size_t)(end - start);
bool is_prefix = false;
for (int p = 0; prefixes[p]; p++) {
if (idlen == strlen(prefixes[p]) && strncmp(source + start, prefixes[p], idlen) == 0) {
is_prefix = true;
break;
}
}
if (!is_prefix || end >= source_len || source[end] != '"') {
return 0;
}
return skip_raw_string(source, source_len, end);
}
// Advance past comments and string/char literals so only real code tokens
// consume the candidate budget (#1989 review). Handles line-spliced //
// comments (a backslash before the newline continues the comment — line
// splicing happens before comment recognition) and, via the identifier-scan
// hook in the callers, C++ raw string literals. A digit separator (1'000'000)
// can still over-skip — that fails safe, toward NOT collecting a candidate.
static int skip_non_code(const char *source, int source_len, int i) {
while (i < source_len) {
char c = source[i];
if (c == '/' && i + 1 < source_len && source[i + 1] == '/') {
i += 2;
for (;;) {
while (i < source_len && source[i] != '\n') {
i++;
}
if (i >= source_len) {
break;
}
// Line splice: a backslash right before the newline (allowing
// \r\n) means the comment continues on the next line.
int back = i - 1;
if (back >= 0 && source[back] == '\r') {
back--;
}
if (back >= 0 && source[back] == '\\') {
i++; // spliced: keep consuming inside the comment
continue;
}
i++; // real end-of-comment newline
break;
}
} else if (c == '/' && i + 1 < source_len && source[i + 1] == '*') {
i += 2;
while (i + 1 < source_len && !(source[i] == '*' && source[i + 1] == '/')) {
i++;
}
if (i + 1 < source_len) {
i += 2; // past the closing "*/"
} else {
i = source_len; // unterminated comment runs to EOF
}
} else if (c == '"' || c == '\'') {
char quote = c;
i++;
while (i < source_len) {
if (source[i] == '\\') {
i += 2;
continue;
}
if (source[i] == quote) {
i++;
break;
}
i++;
}
} else {
return i;
}
}
return i;
}
// Count-only scan used by the early gate (no storage).
static bool has_export_macro_candidates(const char *source, int source_len) {
if (!source || source_len <= 0) {
return false;
}
int i = 0;
while (i < source_len) {
i = skip_non_code(source, source_len, i);
if (i >= source_len) {
break;
}
if (!is_identifier_start(source[i])) {
i++;
continue;
}
int start = i;
while (i < source_len && is_identifier_char(source[i])) {
i++;
}
// A raw-string prefix consumes everything up to its closing quote;
// an unrecognizable raw form poisons the scan state, so fail toward
// NOT running the second pass (raw behavior preserved).
int raw = raw_string_after_prefix(source, source_len, start, i);
if (raw < 0) {
return false;
}
if (raw > 0) {
i = raw;
continue;
}
if (is_export_macro_shape(source + start, (size_t)(i - start))) {
return true;
}
}
return false;
}
static int collect_export_macro_candidates(const char *source, int source_len,
char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out) {
if (!source || source_len <= 0 || !out || max_out <= 0) {
return 0;
}
int stored = 0;
int i = 0;
while (i < source_len) {
i = skip_non_code(source, source_len, i);
if (i >= source_len) {
break;
}
if (!is_identifier_start(source[i])) {
i++;
continue;
}
int start = i;
while (i < source_len && is_identifier_char(source[i])) {
i++;
}
// Raw string (see has_export_macro_candidates): skip its body, or stop
// collecting entirely when the form cannot be confidently recognized —
// continuing would mis-tokenize the rest of the file and could both
// mint phantom candidates and mask real ones (#1989 review).
int raw = raw_string_after_prefix(source, source_len, start, i);
if (raw < 0) {
break;
}
if (raw > 0) {
i = raw;
continue;
}
size_t len = (size_t)(i - start);
/* Length gate FIRST: the dedup below reads out[k][0..len] against rows
* of CBM_EXPORT_MACRO_NAME_MAX bytes — an over-long candidate must be
* rejected before any compare touches them. */
if (len >= CBM_EXPORT_MACRO_NAME_MAX) {
continue;
}
if (!is_export_macro_shape(source + start, len)) {
continue;
}
bool dup = false;
for (int k = 0; k < stored; k++) {
if (strncmp(out[k], source + start, len) == 0 && out[k][len] == '\0') {
dup = true;
break;
}
}
if (dup) {
continue;
}
if (stored >= max_out) {
break; // bounded: stop collecting once the cap is reached
}
memcpy(out[stored], source + start, len);
out[stored][len] = '\0';
stored++;
}
return stored;
}
int cbm_export_macro_candidates(const char *source, int source_len,
char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out) {
return collect_export_macro_candidates(source, source_len, out, max_out);
}
static int count_expanded_lines(const std::string &text) {
int count = 1;
for (char c : text) {
@@ -146,7 +423,11 @@ static bool build_line_map(const std::string &expanded, const std::string &main_
CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_len,
const char *filename, const char **extra_defines,
const char **include_paths, int cpp_mode) {
if (!has_preprocessor_work(source, source_len)) {
// Run the second pass when there are directives to evaluate OR when the file
// carries export-macro-shaped identifiers to predefine empty (#1989) — a UE
// plugin header with only `#pragma once` + `#include` lines still needs it.
if (!has_preprocessor_work(source, source_len) &&
!has_export_macro_candidates(source, source_len)) {
return NULL; // NULL = no expansion needed, use original
}
@@ -156,6 +437,28 @@ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_le
for (int i = 0; extra_defines[i]; i++)
dui.defines.push_back(extra_defines[i]);
}
// Predefine collected export-macro candidates as empty, mirroring the
// real compile command line (`/D "MODULE_API="`). Names already provided
// by the caller win — never override an explicit define.
char export_cands[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX];
int export_cand_count =
collect_export_macro_candidates(source, source_len, export_cands, CBM_EXPORT_MACRO_MAX);
for (int i = 0; i < export_cand_count; i++) {
bool provided = false;
for (std::list<std::string>::const_iterator it = dui.defines.begin();
it != dui.defines.end(); ++it) {
const std::string &def = *it;
size_t eq = def.find('=');
std::string name = (eq == std::string::npos) ? def : def.substr(0, eq);
if (name == export_cands[i]) {
provided = true;
break;
}
}
if (!provided) {
dui.defines.push_back(std::string(export_cands[i]) + "=");
}
}
if (include_paths) {
for (int i = 0; include_paths[i]; i++)
dui.includePaths.push_back(include_paths[i]);
internal/cbm/preprocessor.h
+15
View File
@@ -4,6 +4,15 @@
#include <stddef.h>
#include <stdint.h>
/* #1989: build-system export macros (UE "<MOD>_API", CMake generate_export_header
* "<lib>_EXPORT", ...) are defined empty on the real compile command line but are
* invisible to tree-sitter, so `class MOD_API Foo` misparses with the macro token
* as the type name. The preprocessed second pass predefines the conventional
* export-macro-shaped identifiers found in a source file as empty, mirroring the
* real build line. The candidate list is bounded. */
#define CBM_EXPORT_MACRO_MAX 32
#define CBM_EXPORT_MACRO_NAME_MAX 96
#ifdef __cplusplus
extern "C" {
#endif
@@ -15,6 +24,12 @@ typedef struct {
int expanded_line_count;
} CBMPreprocessedSource;
// Collect conventional export-macro-shaped identifiers (ALL_CAPS ending in
// _API/_EXPORT/_IMPORT/_DLLEXPORT/_DEPRECATED) from the source, up to max_out
// unique names. Returns the number of names stored in out.
int cbm_export_macro_candidates(const char *source, int source_len,
char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out);
// Preprocess C/C++ source: expand macros, evaluate #ifdef, resolve #include.
// Returns malloc-allocated expanded source, or NULL if no expansion needed/on failure.
// extra_defines: NULL-terminated array of "NAME=VALUE" strings (can be NULL).
internal/cbm/result_compact.c
+534
View File
@@ -0,0 +1,534 @@
/*
* result_compact.c — copy a finished CBMFileResult into one exact-size arena.
*
* Why this exists (Go corpus census, 2026-09-13): extraction writes every
* temporary into the result arena — cbm_node_text copies at 496 call sites,
* per-node QN sprintf, enclosing-QN strings — and GROW_ARRAY leaves each
* previous generation of every record array dead behind it. 14.8 GB written,
* 3.4 GB reachable, and all of it retained until after resolve because the
* result owned the arena. Compaction walks what is reachable, measures it,
* copies it into a single block of exactly that size (strings interned by
* content within the file), and destroys the working arena.
*
* Three passes over one traversal:
* COUNT — number of string references, to size the intern table
* MEASURE — bytes every allocation will take (aligned like the arena does)
* COPY — the same allocations, for real, into the fresh arena
* MEASURE and COPY issue identical allocation sequences, so the block fits
* exactly; any failure leaves the result untouched.
*/
#include "cbm.h"
#include "foundation/arena.h"
#include "foundation/compat.h" /* CBM_TLS */
#include "foundation/constants.h"
#include "foundation/mem_core.h"
#include "result_spill.h" /* cbm_result_relocate */
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
enum { CR_ALIGN = 7, CR_MIN_TABLE = 64, CR_TABLE_LOAD = 2 };
typedef enum { CR_COUNT = 0, CR_MEASURE, CR_COPY, CR_RELOCATE } cr_phase_t;
typedef struct {
const char *src; /* NULL = empty slot */
char *dst; /* copy in the new arena; NULL until COPY reaches it */
uint64_t hash;
size_t len; /* strlen(src) */
} cr_slot_t;
typedef struct {
cr_phase_t phase;
size_t refs; /* COUNT: string references seen */
size_t bytes; /* MEASURE: total arena bytes */
CBMArena *dst; /* COPY: the fresh arena */
bool failed;
cr_slot_t *slots;
size_t cap; /* power of two */
/* MEASURE records which slot each string reference landed in; COPY, which
* presents the very same references in the same order, replays them
* instead of measuring, hashing and comparing every string a second time
* (20 M repeated strlen + 31 M memcmp on the Go corpus, waste sanitizer
* 2026-09-17). */
uint32_t *seq;
size_t seq_len;
size_t seq_pos;
/* RELOCATE: pointers in [old_base, old_base + old_len) move by delta. */
const char *old_base;
size_t old_len;
ptrdiff_t delta;
} cr_ctx_t;
static bool cr_in_old_block(const cr_ctx_t *c, const void *p) {
const char *cp = (const char *)p;
return cp >= c->old_base && cp < c->old_base + c->old_len;
}
static size_t cr_aligned(size_t n) {
return (n + CR_ALIGN) & ~(size_t)CR_ALIGN;
}
static uint64_t cr_hash(const char *s, size_t len) {
uint64_t h = 1469598103934665603ULL;
for (size_t i = 0; i < len; i++) {
h ^= (unsigned char)s[i];
h *= 1099511628211ULL;
}
return h;
}
/* Find or insert the slot for s. Never fails: the table is sized from COUNT
* at half load, and the same references are presented again in COPY. */
static cr_slot_t *cr_slot(cr_ctx_t *c, const char *s) {
size_t len = strlen(s);
uint64_t h = cr_hash(s, len);
size_t mask = c->cap - SKIP_ONE;
size_t i = (size_t)h & mask;
for (;;) {
cr_slot_t *slot = &c->slots[i];
if (!slot->src) {
slot->src = s;
slot->hash = h;
slot->len = len;
slot->dst = NULL;
return slot;
}
if (slot->hash == h && slot->len == len &&
(slot->src == s || memcmp(slot->src, s, len) == 0)) {
return slot;
}
i = (i + SKIP_ONE) & mask;
}
}
/* A raw allocation of `bytes` in the new arena: MEASURE books it, COPY makes
* it. Zero bytes is no allocation (the arena returns NULL for it too). */
static void *cr_alloc(cr_ctx_t *c, size_t bytes) {
if (bytes == 0) {
return NULL;
}
if (c->phase == CR_MEASURE) {
c->bytes += cr_aligned(bytes);
return NULL;
}
if (c->phase == CR_COPY) {
void *p = cbm_arena_alloc(c->dst, bytes);
if (!p) {
c->failed = true;
}
return p;
}
return NULL;
}
/* A string field: interned by content. COPY rewrites the field. */
static void cr_str(cr_ctx_t *c, const char **field) {
const char *s = *field;
if (!s) {
return;
}
if (c->phase == CR_RELOCATE) {
if (cr_in_old_block(c, s)) {
*field = s + c->delta;
}
return;
}
if (c->phase == CR_COUNT) {
c->refs++;
return;
}
cr_slot_t *slot;
if (c->phase == CR_COPY && c->seq && c->seq_pos < c->seq_len) {
slot = &c->slots[c->seq[c->seq_pos++]];
} else {
slot = cr_slot(c, s);
}
if (c->phase == CR_MEASURE) {
if (c->seq && c->seq_len < c->refs) {
c->seq[c->seq_len++] = (uint32_t)(slot - c->slots);
}
if (!slot->dst) {
slot->dst = (char *)s; /* mark as booked; reset before COPY */
c->bytes += cr_aligned(slot->len + SKIP_ONE);
}
return;
}
if (!slot->dst) {
char *copy = (char *)cr_alloc(c, slot->len + SKIP_ONE);
if (!copy) {
return;
}
memcpy(copy, slot->src, slot->len + SKIP_ONE);
slot->dst = copy;
}
*field = slot->dst;
}
/* A blob (fingerprint, retained source): copied verbatim. */
static void cr_blob(cr_ctx_t *c, const void **field, size_t bytes) {
if (!*field || bytes == 0) {
return;
}
if (c->phase == CR_RELOCATE) {
if (cr_in_old_block(c, *field)) {
*field = (const char *)*field + c->delta;
}
return;
}
if (c->phase == CR_COPY) {
void *copy = cr_alloc(c, bytes);
if (!copy) {
return;
}
memcpy(copy, *field, bytes);
*field = copy;
} else {
(void)cr_alloc(c, bytes);
}
}
/* A record array: copied at exact count, then the caller walks its fields. */
static void cr_array(cr_ctx_t *c, void **items, int count, size_t elem) {
if (!*items || count <= 0) {
if (c->phase == CR_COPY) {
*items = NULL;
}
return;
}
if (c->phase == CR_RELOCATE) {
if (cr_in_old_block(c, *items)) {
*items = (char *)*items + c->delta;
}
return;
}
cr_blob(c, (const void **)items, (size_t)count * elem);
}
/* A NULL-terminated list of strings: the pointer array plus each string. */
static void cr_list(cr_ctx_t *c, const char ***field) {
const char **list = *field;
if (!list) {
return;
}
int n = 0;
if (c->phase == CR_RELOCATE) {
/* The array may live in a block we can no longer read at its old
* address: relocate the pointer first, then count and walk it. */
cr_blob(c, (const void **)field, sizeof(char *));
list = *field;
while (list[n]) {
n++;
}
for (int i = 0; i < n; i++) {
cr_str(c, &list[i]);
}
return;
}
while (list[n]) {
n++;
}
cr_blob(c, (const void **)field, (size_t)(n + SKIP_ONE) * sizeof(char *));
const char **walk = *field; /* the copy in COPY, the original otherwise */
for (int i = 0; i < n && walk; i++) {
cr_str(c, &walk[i]);
}
}
/* A counted list of strings (signature_param_types). */
static void cr_counted_list(cr_ctx_t *c, const char ***field, int count) {
if (!*field || count <= 0) {
return;
}
cr_blob(c, (const void **)field, (size_t)count * sizeof(char *));
/* (RELOCATE: the pointer moved; the walk below now reads the new array.) */
const char **walk = *field;
for (int i = 0; i < count && walk; i++) {
cr_str(c, &walk[i]);
}
}
static void cr_walk_def(cr_ctx_t *c, CBMDefinition *d) {
cr_str(c, &d->name);
cr_str(c, &d->qualified_name);
cr_str(c, &d->label);
cr_str(c, &d->file_path);
cr_str(c, &d->signature);
cr_str(c, &d->return_type);
cr_str(c, &d->receiver);
cr_str(c, &d->docstring);
cr_str(c, &d->parent_class);
cr_list(c, &d->decorators);
cr_list(c, &d->base_classes);
cr_list(c, &d->param_names);
cr_list(c, &d->param_types);
cr_counted_list(c, &d->signature_param_types, d->signature_param_count);
cr_list(c, &d->return_types);
cr_str(c, &d->route_path);
cr_str(c, &d->route_method);
cr_blob(c, (const void **)&d->fingerprint,
d->fingerprint_k > 0 ? (size_t)d->fingerprint_k * sizeof(uint32_t) : 0);
cr_str(c, &d->structural_profile);
cr_str(c, &d->body_tokens);
cr_str(c, &d->impl_trait);
}
static void cr_walk_call(cr_ctx_t *c, CBMCall *call) {
cr_str(c, &call->callee_name);
cr_str(c, &call->enclosing_func_qn);
cr_str(c, &call->first_string_arg);
cr_str(c, &call->second_arg_name);
int argc = call->arg_count;
if (argc > CBM_MAX_CALL_ARGS) {
argc = CBM_MAX_CALL_ARGS;
}
if (call->args && argc > 0) {
cr_array(c, (void **)&call->args, argc, sizeof(CBMCallArg));
for (int i = 0; i < argc && call->args; i++) {
cr_str(c, &call->args[i].expr);
cr_str(c, &call->args[i].value);
cr_str(c, &call->args[i].keyword);
}
} else if (c->phase == CR_COPY) {
call->args = NULL;
call->arg_count = 0;
}
}
static void cr_walk(cr_ctx_t *c, CBMFileResult *r) {
cr_array(c, (void **)&r->defs.items, r->defs.count, sizeof(CBMDefinition));
for (int i = 0; i < r->defs.count && r->defs.items; i++) {
cr_walk_def(c, &r->defs.items[i]);
}
cr_array(c, (void **)&r->calls.items, r->calls.count, sizeof(CBMCall));
for (int i = 0; i < r->calls.count && r->calls.items; i++) {
cr_walk_call(c, &r->calls.items[i]);
}
cr_array(c, (void **)&r->imports.items, r->imports.count, sizeof(CBMImport));
for (int i = 0; i < r->imports.count && r->imports.items; i++) {
cr_str(c, &r->imports.items[i].local_name);
cr_str(c, &r->imports.items[i].module_path);
}
cr_array(c, (void **)&r->usages.items, r->usages.count, sizeof(CBMUsage));
for (int i = 0; i < r->usages.count && r->usages.items; i++) {
cr_str(c, &r->usages.items[i].ref_name);
cr_str(c, &r->usages.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->throws.items, r->throws.count, sizeof(CBMThrow));
for (int i = 0; i < r->throws.count && r->throws.items; i++) {
cr_str(c, &r->throws.items[i].exception_name);
cr_str(c, &r->throws.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->rw.items, r->rw.count, sizeof(CBMReadWrite));
for (int i = 0; i < r->rw.count && r->rw.items; i++) {
cr_str(c, &r->rw.items[i].var_name);
cr_str(c, &r->rw.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->type_refs.items, r->type_refs.count, sizeof(CBMTypeRef));
for (int i = 0; i < r->type_refs.count && r->type_refs.items; i++) {
cr_str(c, &r->type_refs.items[i].type_name);
cr_str(c, &r->type_refs.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->env_accesses.items, r->env_accesses.count, sizeof(CBMEnvAccess));
for (int i = 0; i < r->env_accesses.count && r->env_accesses.items; i++) {
cr_str(c, &r->env_accesses.items[i].env_key);
cr_str(c, &r->env_accesses.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->type_assigns.items, r->type_assigns.count, sizeof(CBMTypeAssign));
for (int i = 0; i < r->type_assigns.count && r->type_assigns.items; i++) {
cr_str(c, &r->type_assigns.items[i].var_name);
cr_str(c, &r->type_assigns.items[i].type_name);
cr_str(c, &r->type_assigns.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->impl_traits.items, r->impl_traits.count, sizeof(CBMImplTrait));
for (int i = 0; i < r->impl_traits.count && r->impl_traits.items; i++) {
cr_str(c, &r->impl_traits.items[i].trait_name);
cr_str(c, &r->impl_traits.items[i].struct_name);
cr_str(c, &r->impl_traits.items[i].struct_qn);
}
cr_array(c, (void **)&r->resolved_calls.items, r->resolved_calls.count,
sizeof(CBMResolvedCall));
for (int i = 0; i < r->resolved_calls.count && r->resolved_calls.items; i++) {
cr_str(c, &r->resolved_calls.items[i].caller_qn);
cr_str(c, &r->resolved_calls.items[i].callee_qn);
cr_str(c, &r->resolved_calls.items[i].strategy);
cr_str(c, &r->resolved_calls.items[i].reason);
}
cr_array(c, (void **)&r->string_refs.items, r->string_refs.count, sizeof(CBMStringRef));
for (int i = 0; i < r->string_refs.count && r->string_refs.items; i++) {
cr_str(c, &r->string_refs.items[i].value);
cr_str(c, &r->string_refs.items[i].enclosing_func_qn);
cr_str(c, &r->string_refs.items[i].key_path);
}
cr_array(c, (void **)&r->infra_bindings.items, r->infra_bindings.count,
sizeof(CBMInfraBinding));
for (int i = 0; i < r->infra_bindings.count && r->infra_bindings.items; i++) {
cr_str(c, &r->infra_bindings.items[i].source_name);
cr_str(c, &r->infra_bindings.items[i].target_url);
cr_str(c, &r->infra_bindings.items[i].broker);
}
cr_array(c, (void **)&r->channels.items, r->channels.count, sizeof(CBMChannel));
for (int i = 0; i < r->channels.count && r->channels.items; i++) {
cr_str(c, &r->channels.items[i].channel_name);
cr_str(c, &r->channels.items[i].transport);
cr_str(c, &r->channels.items[i].enclosing_func_qn);
}
cr_str(c, &r->module_qn);
cr_str(c, &r->namespace_name);
cr_list(c, &r->exports);
cr_list(c, &r->constants);
cr_list(c, &r->global_vars);
cr_list(c, &r->macros);
cr_str(c, &r->error_msg);
cr_str(c, &r->error_ranges);
cr_blob(c, (const void **)&r->source, r->source ? (size_t)r->source_len + SKIP_ONE : 0);
}
void cbm_result_relocate(CBMFileResult *result, const char *old_base, size_t len, char *new_base) {
if (!result || !old_base || !new_base || len == 0 || old_base == new_base) {
return;
}
cr_ctx_t c;
memset(&c, 0, sizeof(c));
c.phase = CR_RELOCATE;
c.old_base = old_base;
c.old_len = len;
c.delta = new_base - old_base;
cr_walk(&c, result);
}
static size_t cr_pow2_at_least(size_t n) {
size_t cap = CR_MIN_TABLE;
while (cap < n) {
cap *= PAIR_LEN;
}
return cap;
}
/* The intern table and the replay sequence, kept per pipeline worker: a fresh
* zeroed table per file was 21,875 allocations and 2.0 GB of pure churn on the
* Go corpus (waste sanitizer, 2026-09-17). Kept only on threads whose
* cbm_work_arena_release is guaranteed to run (cbm_work_arena_keeping), and
* only up to CR_KEEP_BYTES; anything else allocates per call as before. */
enum { CR_KEEP_BYTES = 8 * 1024 * 1024 };
static CBM_TLS void *tl_cr_buf;
static CBM_TLS size_t tl_cr_bytes;
void cbm_result_compact_release_thread(void) {
cbm_free(CBM_MEM_CLASS_EXTRACT, tl_cr_buf);
tl_cr_buf = NULL;
tl_cr_bytes = 0;
}
/* One zeroed block holding the slot table followed by the sequence. Returns
* whether the block is the kept one (then the caller must not free it). */
static bool cr_scratch_get(cr_ctx_t *c) {
size_t slot_bytes = c->cap * sizeof(cr_slot_t);
size_t bytes = slot_bytes + (c->refs * sizeof(uint32_t));
bool keep = bytes <= (size_t)CR_KEEP_BYTES && cbm_work_arena_keeping();
void *buf = NULL;
if (keep && tl_cr_buf && tl_cr_bytes >= bytes) {
buf = tl_cr_buf;
} else if (keep) {
size_t grown = CR_MIN_TABLE; /* powers of two: a growing file mix reallocates rarely */
while (grown < bytes) {
grown *= PAIR_LEN;
}
if (grown > (size_t)CR_KEEP_BYTES) {
grown = bytes;
}
cbm_result_compact_release_thread();
tl_cr_buf = cbm_alloc(CBM_MEM_CLASS_EXTRACT, grown);
tl_cr_bytes = tl_cr_buf ? grown : 0;
buf = tl_cr_buf;
} else {
buf = cbm_alloc(CBM_MEM_CLASS_EXTRACT, bytes);
}
if (!buf) {
return false;
}
memset(
buf, 0,
slot_bytes); /* MEASURE needs an empty table; the sequence is written before it is read */
c->slots = (cr_slot_t *)buf;
c->seq = c->refs ? (uint32_t *)((char *)buf + slot_bytes) : NULL;
return true;
}
static void cr_scratch_put(cr_ctx_t *c) {
if ((void *)c->slots != tl_cr_buf) {
cbm_free(CBM_MEM_CLASS_EXTRACT, c->slots);
}
c->slots = NULL;
c->seq = NULL;
}
void cbm_result_compact(CBMFileResult *result) {
if (!result || result->arena.nblocks == 0) {
return;
}
cr_ctx_t c;
memset(&c, 0, sizeof(c));
/* Work on a copy of the header: every pointer rewrite lands here and the
* caller's result is replaced only once everything succeeded. */
CBMFileResult tmp = *result;
c.phase = CR_COUNT;
cr_walk(&c, &tmp);
c.cap = cr_pow2_at_least(c.refs * CR_TABLE_LOAD + CR_MIN_TABLE);
if (!cr_scratch_get(&c)) {
return;
}
c.phase = CR_MEASURE;
cr_walk(&c, &tmp);
for (size_t i = 0; i < c.cap; i++) {
c.slots[i].dst = NULL; /* MEASURE used dst as a booked marker */
}
CBMArena fresh;
cbm_arena_init_exact(&fresh, c.bytes);
if (fresh.nblocks == 0) {
cr_scratch_put(&c);
return;
}
c.phase = CR_COPY;
c.dst = &fresh;
cr_walk(&c, &tmp);
cr_scratch_put(&c);
if (c.failed) {
cbm_arena_destroy(&fresh);
return;
}
/* Exact-count arrays: nothing may append into the dead headroom. */
tmp.defs.cap = tmp.defs.count;
tmp.calls.cap = tmp.calls.count;
tmp.imports.cap = tmp.imports.count;
tmp.usages.cap = tmp.usages.count;
tmp.throws.cap = tmp.throws.count;
tmp.rw.cap = tmp.rw.count;
tmp.type_refs.cap = tmp.type_refs.count;
tmp.env_accesses.cap = tmp.env_accesses.count;
tmp.type_assigns.cap = tmp.type_assigns.count;
tmp.impl_traits.cap = tmp.impl_traits.count;
tmp.resolved_calls.cap = tmp.resolved_calls.count;
tmp.string_refs.cap = tmp.string_refs.count;
tmp.infra_bindings.cap = tmp.infra_bindings.count;
tmp.channels.cap = tmp.channels.count;
/* A composite kept its per-unit results only so shallow-copied strings
* stayed valid; every string is now a copy of its own. */
cbm_result_release_owned(result);
tmp.owned_results = NULL;
tmp.owned_result_count = 0;
cbm_work_arena_give(&result->arena); /* kept for this thread's next file */
tmp.arena = fresh;
*result = tmp;
}
internal/cbm/result_spill.c
+410
View File
@@ -0,0 +1,410 @@
/*
* result_spill.c — see result_spill.h.
*
* Layout on disk, per parked result:
* spill_rec_hdr_t (magic, block length, block base address at park time,
* the CBMFileResult header as it was in memory)
* block bytes (the single compacted arena block)
*
* The header's pointers are meaningless on disk; the loader rebuilds the
* arena at a new address and shifts every pointer by the delta
* (cbm_result_relocate, implemented on the compaction traversal so it sees
* exactly the fields compaction copied).
*/
#include "result_spill.h"
#include "foundation/arena.h"
#include "foundation/compat_fs.h"
#include "foundation/compat_thread.h"
#include "foundation/constants.h"
#include "foundation/log.h"
#include "foundation/mem_core.h"
#include "foundation/platform.h" /* cbm_fs_free_bytes: the disk ceiling below */
/* The park writes a result as an opaque image: the record header (a struct
* copy) and the compacted arena block. Both carry padding bytes no code ever
* wrote -- inside structs, between objects -- and MemorySanitizer tracks
* that mark through the compaction's memcpy, so it refuses the fwrite of an
* image that is read back whole and never interpreted byte by byte (CI MSan
* lane on #2202: offset 4087, then 6714, of a 6,952-byte block). Under MSan
* the image is declared defined right before the write; every other build
* compiles this to nothing. */
#include "foundation/sanitized.h" /* __has_feature exists everywhere, cppcheck included */
#if __has_feature(memory_sanitizer)
#include <sanitizer/msan_interface.h>
#define SPILL_IMAGE_DEFINED(p, n) __msan_unpoison((p), (n))
#else
#define SPILL_IMAGE_DEFINED(p, n) ((void)0)
#endif
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef _WIN32
#include <process.h>
#define SPILL_PID() ((long)_getpid())
#define SPILL_SEEK(fp, off) _fseeki64((fp), (long long)(off), SEEK_SET)
#else
#include <unistd.h>
#define SPILL_PID() ((long)getpid())
#define SPILL_SEEK(fp, off) fseeko((fp), (off_t)(off), SEEK_SET)
#endif
static const uint64_t SPILL_MAGIC = 0x5350494C4C524553ULL; /* "SPILLRES" */
typedef struct {
uint64_t magic;
uint64_t block_len;
uint64_t old_base; /* (uintptr_t) of the block when parked */
CBMFileResult header;
} spill_rec_hdr_t;
typedef struct {
FILE *fp;
cbm_mutex_t mu; /* reads seek; writes append -- one lock per file */
char path[CBM_SZ_1K];
uint64_t end; /* bytes written so far (append offset) */
bool unlinked; /* already removed while open (POSIX); close must not retry */
} spill_file_t;
typedef struct {
int writer; /* -1 = empty */
uint64_t offset; /* record start in that writer's file */
uint64_t rec_len; /* header + block */
/* The file's namespace/package, kept in MEMORY while the result itself is
* on disk. The namespace map that import resolution is built on is
* assembled from every file at once, before any of them is reloaded: a
* parked file used to contribute nothing to it, so spilling silently
* changed how imports resolved (see cbm_result_spill_namespace). One short
* string per parked file is a rounding error against the result it
* replaces. */
char *namespace_name;
} spill_slot_t;
struct cbm_result_spill {
spill_file_t *files;
int writers;
spill_slot_t *slots;
int slot_count;
_Atomic int64_t parked;
_Atomic int64_t bytes;
_Atomic int64_t loads;
int64_t byte_cap; /* 0 = uncapped (the platform could not say) */
_Atomic int cap_reported; /* reaching the cap is logged once, not per park */
};
/* Spilling trades a memory problem for a disk problem, so the disk gets a say.
*
* FLOOR: below this much free space, do not start spilling at all — filling the
* user's disk is a worse failure than the memory pressure being relieved, since
* it breaks everything else on the machine and not just this index.
* SHARE: of what IS free, never take more than this fraction, so a runaway
* index cannot consume the rest either.
* Free space is read once, at open: it is this run's budget, not a moving
* target to race against, and a failed write stays handled underneath. */
enum { SPILL_FREE_FLOOR_GB = 10, SPILL_FREE_SHARE_DIV = 2 };
/* Remove spill files left by runs that are no longer here. Only files this
* module names (spill-<pid>-<writer>.bin) are touched, and on Windows an open
* file refuses deletion, which is exactly the liveness check we want: a running
* index keeps its own spill. POSIX runs unlink at open, so this only ever finds
* files from before that behaviour, or from another platform's cache. */
static void spill_sweep_orphans(const char *spill_dir) {
cbm_dir_t *dir = cbm_opendir(spill_dir);
if (!dir) {
return;
}
int removed = 0;
cbm_dirent_t *entry;
while ((entry = cbm_readdir(dir)) != NULL) {
if (strncmp(entry->name, "spill-", 6) != 0) {
continue;
}
size_t len = strlen(entry->name);
if (len < 5 || strcmp(entry->name + len - 4, ".bin") != 0) {
continue;
}
char path[CBM_SZ_1K];
if (snprintf(path, sizeof(path), "%s/%s", spill_dir, entry->name) >= (int)sizeof(path)) {
continue;
}
if (cbm_unlink(path) == 0) {
removed++;
}
}
cbm_closedir(dir);
if (removed > 0) {
char removed_text[CBM_SZ_16];
snprintf(removed_text, sizeof(removed_text), "%d", removed);
cbm_log_info("mem.spill.orphans_removed", "count", removed_text, "dir", spill_dir);
}
}
cbm_result_spill_t *cbm_result_spill_open(const char *dir, int writers, int slots) {
if (!dir || !dir[0] || writers <= 0 || slots <= 0) {
return NULL;
}
char spill_dir[CBM_SZ_1K];
if (snprintf(spill_dir, sizeof(spill_dir), "%s/spill", dir) >= (int)sizeof(spill_dir)) {
return NULL;
}
if (!cbm_mkdir_p(spill_dir, 0700)) {
cbm_log_warn("mem.spill.open_failed", "dir", spill_dir, "reason", "mkdir");
return NULL;
}
/* Reclaim what earlier runs could not. A run killed outright (OOM killer,
* SIGKILL, power loss) runs no cleanup, and on Windows its spill files
* survive; POSIX runs unlink-at-open below and leaves nothing. Deleting is
* the liveness test: an open file cannot be removed on Windows, so a
* concurrent run's spill refuses and is left alone. */
spill_sweep_orphans(spill_dir);
/* Ask the disk BEFORE trading memory for it. Refusing here leaves the
* caller in the memory path it was already in (back-pressure, then its own
* budget decision) — the same outcome as a disk that fills mid-run, minus
* the filled disk. */
size_t free_bytes = cbm_fs_free_bytes(spill_dir);
int64_t byte_cap = 0;
if (free_bytes > 0) {
size_t floor_bytes = (size_t)SPILL_FREE_FLOOR_GB * 1024 * 1024 * 1024;
if (free_bytes < floor_bytes) {
char free_mb[CBM_SZ_32];
snprintf(free_mb, sizeof(free_mb), "%zu", free_bytes / (1024 * 1024));
cbm_log_warn("mem.spill.refused", "reason", "low_disk", "free_mb", free_mb, "detail",
"not spilling onto a nearly full disk; memory relief continues through "
"back-pressure instead");
return NULL;
}
byte_cap = (int64_t)(free_bytes / SPILL_FREE_SHARE_DIV);
}
cbm_result_spill_t *sp = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*sp));
if (!sp) {
return NULL;
}
sp->files = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)writers * sizeof(spill_file_t));
sp->slots = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)slots * sizeof(spill_slot_t));
if (!sp->files || !sp->slots) {
cbm_result_spill_close(sp);
return NULL;
}
sp->writers = writers;
sp->slot_count = slots;
sp->byte_cap = byte_cap;
for (int i = 0; i < slots; i++) {
sp->slots[i].writer = -1;
}
for (int w = 0; w < writers; w++) {
spill_file_t *f = &sp->files[w];
snprintf(f->path, sizeof(f->path), "%s/spill-%ld-%d.bin", spill_dir, SPILL_PID(), w);
f->fp = cbm_fopen(f->path, "w+b");
if (!f->fp) {
cbm_log_warn("mem.spill.open_failed", "path", f->path, "reason", "fopen");
cbm_result_spill_close(sp);
return NULL;
}
cbm_mutex_init(&f->mu);
#ifndef _WIN32
/* Unlink while we hold it open: the data stays reachable through this
* handle, and the space comes back the moment the process ends — a
* clean exit, an abort, or a SIGKILL from the OOM killer alike. A
* killed run cannot leave gigabytes of spill behind because there is no
* name left to leave it under. Windows refuses to unlink an open file,
* so it keeps the close-path unlink and the sweep below. */
if (cbm_unlink(f->path) == 0) {
f->unlinked = true;
}
#endif
}
cbm_log_info("mem.spill.open", "dir", spill_dir, "writers", writers > 0 ? "yes" : "no");
return sp;
}
bool cbm_result_spill_has(const cbm_result_spill_t *sp, int slot) {
return sp && slot >= 0 && slot < sp->slot_count && sp->slots[slot].writer >= 0;
}
bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFileResult *result) {
if (!sp || !result || writer < 0 || writer >= sp->writers || slot < 0 ||
slot >= sp->slot_count || sp->slots[slot].writer >= 0) {
return false;
}
/* Only a compacted result is one block with every pointer inside it; a
* result that owns sub-results (embedded languages) has more arenas than
* that and stays in memory. A retained parse tree is a re-parse cache,
* not data: it is dropped with the in-memory result below. */
if (result->arena.nblocks != 1 || result->owned_result_count != 0) {
return false;
}
/* The disk ceiling. Refusing a park leaves the result in memory, which is
* exactly where it would be without spill at all — the caller's
* back-pressure and budget keep deciding from there. What this prevents is
* an index quietly consuming the rest of the user's free space. */
if (sp->byte_cap > 0) {
int64_t want = (int64_t)(sizeof(spill_rec_hdr_t) + result->arena.used);
if (atomic_load_explicit(&sp->bytes, memory_order_relaxed) + want > sp->byte_cap) {
if (atomic_exchange_explicit(&sp->cap_reported, 1, memory_order_relaxed) == 0) {
char cap_mb[CBM_SZ_32];
snprintf(cap_mb, sizeof(cap_mb), "%lld", (long long)(sp->byte_cap / (1024 * 1024)));
cbm_log_warn("mem.spill.cap_reached", "cap_mb", cap_mb, "detail",
"spill is at its share of free disk; results stay in memory and "
"back-pressure takes over");
}
return false;
}
}
spill_file_t *f = &sp->files[writer];
spill_rec_hdr_t hdr;
memset(&hdr, 0, sizeof(hdr));
hdr.magic = SPILL_MAGIC;
hdr.block_len = result->arena.used; /* bytes actually written into the block */
hdr.old_base = (uint64_t)(uintptr_t)result->arena.blocks[0];
hdr.header = *result;
hdr.header.cached_tree = NULL; /* never on disk: the loader gets no tree */
SPILL_IMAGE_DEFINED(&hdr, sizeof(hdr));
SPILL_IMAGE_DEFINED(result->arena.blocks[0], hdr.block_len);
cbm_mutex_lock(&f->mu);
uint64_t offset = f->end;
bool ok = SPILL_SEEK(f->fp, offset) == 0 && fwrite(&hdr, sizeof(hdr), 1, f->fp) == 1 &&
(hdr.block_len == 0 || fwrite(result->arena.blocks[0], hdr.block_len, 1, f->fp) == 1);
if (ok) {
f->end = offset + sizeof(hdr) + hdr.block_len;
}
cbm_mutex_unlock(&f->mu);
if (!ok) {
cbm_log_warn("mem.spill.write_failed", "path", f->path);
return false;
}
sp->slots[slot].writer = writer;
sp->slots[slot].offset = offset;
sp->slots[slot].rec_len = sizeof(hdr) + hdr.block_len;
/* Copied BEFORE the result is freed below: the namespace map is built from
* all files at once, long before this slot is loaded back. */
if (result->namespace_name && result->namespace_name[0]) {
sp->slots[slot].namespace_name =
cbm_mem_strdup(CBM_MEM_CLASS_OTHER, result->namespace_name);
}
atomic_fetch_add_explicit(&sp->parked, 1, memory_order_relaxed);
atomic_fetch_add_explicit(&sp->bytes, (int64_t)(sizeof(hdr) + hdr.block_len),
memory_order_relaxed);
cbm_free_result(result);
return true;
}
CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot) {
if (!cbm_result_spill_has(sp, slot)) {
return NULL;
}
const spill_slot_t *s = &sp->slots[slot];
spill_file_t *f = &sp->files[s->writer];
spill_rec_hdr_t hdr;
cbm_mutex_lock(&f->mu);
bool ok = SPILL_SEEK(f->fp, s->offset) == 0 && fread(&hdr, sizeof(hdr), 1, f->fp) == 1 &&
hdr.magic == SPILL_MAGIC;
CBMFileResult *r = NULL;
if (ok) {
r = cbm_result_alloc();
if (r) {
*r = hdr.header;
r->cached_tree = NULL;
memset(&r->arena, 0, sizeof(r->arena));
cbm_arena_init_exact(&r->arena, hdr.block_len ? (size_t)hdr.block_len : 1);
if (r->arena.nblocks == 1 &&
(hdr.block_len == 0 ||
fread(r->arena.blocks[0], (size_t)hdr.block_len, 1, f->fp) == 1)) {
r->arena.used = (size_t)hdr.block_len;
r->arena.total_alloc = (size_t)hdr.block_len;
} else {
ok = false;
}
} else {
ok = false;
}
}
cbm_mutex_unlock(&f->mu);
if (!ok) {
cbm_log_warn("mem.spill.read_failed", "path", f->path);
if (r) {
cbm_free_result(r);
}
return NULL;
}
cbm_result_relocate(r, (const char *)(uintptr_t)hdr.old_base, (size_t)hdr.block_len,
r->arena.blocks[0]);
atomic_fetch_add_explicit((_Atomic int64_t *)&sp->loads, 1, memory_order_relaxed);
return r;
}
bool cbm_result_spill_peek_header(const cbm_result_spill_t *sp, int slot, CBMFileResult *out) {
if (!out || !cbm_result_spill_has(sp, slot)) {
return false;
}
const spill_slot_t *s = &sp->slots[slot];
spill_file_t *f = &sp->files[s->writer];
spill_rec_hdr_t hdr;
cbm_mutex_lock(&f->mu);
bool ok = SPILL_SEEK(f->fp, s->offset) == 0 && fread(&hdr, sizeof(hdr), 1, f->fp) == 1 &&
hdr.magic == SPILL_MAGIC;
cbm_mutex_unlock(&f->mu);
if (ok) {
*out = hdr.header;
}
return ok;
}
void cbm_result_spill_peek_counts(const cbm_result_spill_t *sp, int slot, int *defs, int *impls) {
CBMFileResult hdr;
bool ok = cbm_result_spill_peek_header(sp, slot, &hdr);
if (defs) {
*defs = ok ? hdr.defs.count : 0;
}
if (impls) {
*impls = ok ? hdr.impl_traits.count : 0;
}
}
void cbm_result_spill_stats(const cbm_result_spill_t *sp, int64_t *parked, int64_t *bytes,
int64_t *loads) {
if (parked) {
*parked = sp ? atomic_load_explicit(&sp->parked, memory_order_relaxed) : 0;
}
if (bytes) {
*bytes = sp ? atomic_load_explicit(&sp->bytes, memory_order_relaxed) : 0;
}
if (loads) {
*loads = sp ? atomic_load_explicit(&sp->loads, memory_order_relaxed) : 0;
}
}
const char *cbm_result_spill_namespace(const cbm_result_spill_t *sp, int slot) {
if (!sp || !sp->slots || slot < 0 || slot >= sp->slot_count) {
return NULL;
}
return sp->slots[slot].namespace_name;
}
void cbm_result_spill_close(cbm_result_spill_t *sp) {
if (!sp) {
return;
}
if (sp->files) {
for (int w = 0; w < sp->writers; w++) {
spill_file_t *f = &sp->files[w];
if (f->fp) {
(void)fclose(f->fp);
if (!f->unlinked) { /* POSIX already removed it at open */
(void)cbm_unlink(f->path);
}
cbm_mutex_destroy(&f->mu);
}
}
cbm_free(CBM_MEM_CLASS_OTHER, sp->files);
}
if (sp->slots) {
for (int i = 0; i < sp->slot_count; i++) {
cbm_free(CBM_MEM_CLASS_OTHER, sp->slots[i].namespace_name);
}
}
cbm_free(CBM_MEM_CLASS_OTHER, sp->slots);
cbm_free(CBM_MEM_CLASS_OTHER, sp);
}
internal/cbm/result_spill.h
+73
View File
@@ -0,0 +1,73 @@
/*
* result_spill.h — per-file extraction results parked on disk when the
* memory budget is hit.
*
* A compacted CBMFileResult is one exact-size arena block plus a header whose
* pointers all point into that block (cbm_result_compact guarantees it). That
* makes a result a relocatable blob: written as header + block, read back at
* any address by shifting every pointer by the base delta. The store is one
* append-only file per writer thread under the cache directory; loads use
* pread and are safe from any thread. Nothing here decides WHEN to spill --
* the pipeline does, on the budget signal (see cbm_pipeline_ctx_t.spill).
*/
#ifndef CBM_RESULT_SPILL_H
#define CBM_RESULT_SPILL_H
#include "cbm.h"
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
typedef struct cbm_result_spill cbm_result_spill_t;
/* One store for one pipeline run: `writers` append-only files under
* <dir>/spill-<pid>-<w>.bin, `slots` result slots (one per input file).
* NULL on failure (no directory, no space): the caller keeps results in
* memory as before. */
cbm_result_spill_t *cbm_result_spill_open(const char *dir, int writers, int slots);
/* Park a compacted result in slot `slot` via writer `writer`, then free the
* in-memory result (a retained parse tree goes with it; a loaded result has
* none). Returns false (and leaves the result untouched) when the result is
* not a single-block compaction, owns sub-results, or the write fails. */
bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFileResult *result);
/* True when slot `slot` holds a parked result. */
bool cbm_result_spill_has(const cbm_result_spill_t *sp, int slot);
/* Load slot `slot` back into memory: a fresh CBMFileResult the caller owns
* (cbm_free_result). NULL when the slot is empty or the read fails. Safe from
* several threads for different slots. */
CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot);
/* Read only the parked header of slot `slot` into *out: every count in it
* is valid, every pointer meaningless. False when the slot is empty or the
* read fails. Lets a consumer skip the load when there is nothing to read. */
bool cbm_result_spill_peek_header(const cbm_result_spill_t *sp, int slot, CBMFileResult *out);
/* Read only the parked header of slot `slot`: how many defs and impl
* relations it holds (the collector sizes its array before loading). */
void cbm_result_spill_peek_counts(const cbm_result_spill_t *sp, int slot, int *defs, int *impls);
/* The namespace/package a parked file declares, or NULL. Kept in MEMORY while
* the result itself is on disk, because import resolution builds its namespace
* map from EVERY file before any parked one is read back. Without it a spilled
* file contributed nothing to that map, so imports resolved differently
* depending on memory pressure -- php measured 57,182 edges in memory against
* 59,379 while spilling, the same binary, 2026-09-18. */
const char *cbm_result_spill_namespace(const cbm_result_spill_t *sp, int slot);
/* Counters for the log: results parked, bytes on disk, loads served. */
void cbm_result_spill_stats(const cbm_result_spill_t *sp, int64_t *parked, int64_t *bytes,
int64_t *loads);
/* Close and delete the files. */
void cbm_result_spill_close(cbm_result_spill_t *sp);
/* Relocation primitive used by the loader: every pointer inside `result`
* that points into [old_base, old_base + len) is shifted to the block now at
* `new_base`. Exposed for tests. */
void cbm_result_relocate(CBMFileResult *result, const char *old_base, size_t len, char *new_base);
#endif /* CBM_RESULT_SPILL_H */
internal/cbm/service_patterns.c
+136 -10
View File
@@ -140,6 +140,20 @@ static const lib_pattern_t http_libraries[] = {
{"socket.http", CBM_SVC_HTTP, NULL},
{"resty.http", CBM_SVC_HTTP, NULL},
/* Glued-name libraries. match_qn needs an identifier boundary around an
* id, so a library whose own name glues a lowercase prefix/suffix onto
* another id ("grequests", "curlpp") is listed explicitly: no boundary
* rule can tell "grequests" from "myrequests". */
{"grequests", CBM_SVC_HTTP, NULL}, /* Python: gevent + requests */
{"txrequests", CBM_SVC_HTTP, NULL}, /* Python: Twisted + requests */
{"redaxios", CBM_SVC_HTTP, NULL}, /* JS: axios-compatible fetch client */
{"gaxios", CBM_SVC_HTTP, NULL}, /* JS: Google's axios-style client */
{"libcurl", CBM_SVC_HTTP, NULL}, /* C, node-libcurl */
{"curlpp", CBM_SVC_HTTP, NULL}, /* C++ libcurl wrapper */
{"curlcpp", CBM_SVC_HTTP, NULL}, /* C++ libcurl wrapper */
{"hyperlocal", CBM_SVC_HTTP, NULL}, /* Rust: hyper over unix sockets */
{"guzzlehttp", CBM_SVC_HTTP, NULL}, /* PHP: Guzzle's composer vendor */
{NULL, CBM_SVC_NONE, NULL},
};
@@ -263,6 +277,33 @@ static const lib_pattern_t async_libraries[] = {
{"dapr.clients.grpc", CBM_SVC_ASYNC, "dapr"},
{"DaprClient", CBM_SVC_ASYNC, "dapr"},
/* Glued-name libraries (see the note in http_libraries). */
{"aiokafka", CBM_SVC_ASYNC, "kafka"},
{"pykafka", CBM_SVC_ASYNC, "kafka"},
{"librdkafka", CBM_SVC_ASYNC, "kafka"},
{"rdkafkacpp", CBM_SVC_ASYNC, "kafka"},
{"rskafka", CBM_SVC_ASYNC, "kafka"},
{"aioamqp", CBM_SVC_ASYNC, "rabbitmq"},
{"amqpstorm", CBM_SVC_ASYNC, "rabbitmq"},
{"pamqp", CBM_SVC_ASYNC, "rabbitmq"},
{"pyamqp", CBM_SVC_ASYNC, "rabbitmq"},
{"amqprs", CBM_SVC_ASYNC, "rabbitmq"},
{"amqpcpp", CBM_SVC_ASYNC, "rabbitmq"},
{"pynats", CBM_SVC_ASYNC, "nats"},
{"jnats", CBM_SVC_ASYNC, "nats"},
{"aiomqtt", CBM_SVC_ASYNC, "mqtt"},
{"amqtt", CBM_SVC_ASYNC, "mqtt"},
{"hbmqtt", CBM_SVC_ASYNC, "mqtt"},
{"umqtt", CBM_SVC_ASYNC, "mqtt"},
{"mqttools", CBM_SVC_ASYNC, "mqtt"},
{"emqtt", CBM_SVC_ASYNC, "mqtt"},
{"rumqtt", CBM_SVC_ASYNC, "mqtt"},
{"gomqtt", CBM_SVC_ASYNC, "mqtt"},
{"libmosquitto", CBM_SVC_ASYNC, "mqtt"},
{"mosquittopp", CBM_SVC_ASYNC, "mqtt"},
{"pubsublite", CBM_SVC_ASYNC, "pubsub"},
{"gocelery", CBM_SVC_ASYNC, "celery"},
{NULL, CBM_SVC_NONE, NULL},
};
@@ -308,6 +349,13 @@ static const lib_pattern_t config_libraries[] = {
{"Application.get_env", CBM_SVC_CONFIG, NULL},
{"Application.fetch_env", CBM_SVC_CONFIG, NULL},
/* Glued-name accessors (see the note in http_libraries). */
{"wgetenv", CBM_SVC_CONFIG, NULL}, /* Windows CRT: _wgetenv, _wgetenv_s */
{"getenvb", CBM_SVC_CONFIG, NULL}, /* Python: os.getenvb */
{"qgetenv", CBM_SVC_CONFIG, NULL}, /* Qt */
{"dotenvx", CBM_SVC_CONFIG, NULL}, /* JS: @dotenvx/dotenvx */
{"phpdotenv", CBM_SVC_CONFIG, NULL}, /* PHP: vlucas/phpdotenv */
{NULL, CBM_SVC_NONE, NULL},
};
@@ -377,6 +425,11 @@ static const lib_pattern_t route_reg_libraries[] = {
{"akka.http.scaladsl.server", CBM_SVC_ROUTE_REG, NULL},
{"play.api.routing", CBM_SVC_ROUTE_REG, NULL},
/* Glued-name frameworks (see the note in http_libraries). */
{"apiflask", CBM_SVC_ROUTE_REG, NULL}, /* Python: Flask-based */
{"honox", CBM_SVC_ROUTE_REG, NULL}, /* JS: Hono meta-framework */
{"fasthttprouter", CBM_SVC_ROUTE_REG, NULL}, /* Go: httprouter for fasthttp */
{NULL, CBM_SVC_NONE, NULL},
};
@@ -448,6 +501,11 @@ static const lib_pattern_t graphql_libraries[] = {
{"async-graphql", CBM_SVC_GRAPHQL, NULL},
{"juniper", CBM_SVC_GRAPHQL, NULL},
/* Glued-name libraries (see the note in http_libraries). */
{"gqlparser", CBM_SVC_GRAPHQL, NULL}, /* Go: vektah/gqlparser */
{"gqlgenc", CBM_SVC_GRAPHQL, NULL}, /* Go: gqlgen client generator */
{"aiogqlc", CBM_SVC_GRAPHQL, NULL}, /* Python: asyncio GraphQL client */
{NULL, CBM_SVC_NONE, NULL},
};
@@ -534,17 +592,67 @@ static const method_suffix_t method_suffixes[] = {
/* ── Matching implementation ───────────────────────────────────── */
/* Check if any library identifier appears as a substring in the QN.
* Case-sensitive: "requests" matches "project.venv.requests.api.get"
* but not "Requests". Library names are specific enough to avoid
* false positives even with substring matching. */
static bool qn_is_lower(char ch) {
return ch >= 'a' && ch <= 'z';
}
static bool is_digit_char(char ch) {
return ch >= '0' && ch <= '9';
}
static bool qn_is_alnum(char ch) {
return qn_is_lower(ch) || is_digit_char(ch) || (ch >= 'A' && ch <= 'Z');
}
/* True when the occurrence of `id` at `hit` sits on identifier boundaries.
*
* A raw substring match let short ids fire inside unrelated words: "gin."
* in "plugin.", "dio" in "studio", "surf" in "surface", "express" in
* "expression". Every character that is not an ASCII letter or digit is a
* separator ('.', '/', '\\', ':', '_', '-', '$', '@', ...).
*
* BEFORE: start of string, a separator, an id that itself starts with a
* separator ("@trpc/server"), or an id that starts with an
* uppercase letter — a capital opens a new CamelCase word whatever
* precedes it ("AsyncHttpClient", "IHttpClientFactory",
* "NSURLSession"). Rejected: a lowercase/digit-initial id glued to
* a letter or digit ("plugin." / "studio" / "myrequests").
* AFTER: end of string, a separator, an id that itself ends in a
* separator ("gin."), an uppercase letter (CamelCase glue:
* "GuzzleHttp", "FeignClient", "KafkaProducer", "kafkaProducer")
* or a digit (version suffix: "urllib2", "amqp091-go").
* Rejected: a lowercase letter continuing the word ("surface",
* "curly", "expression"). */
static bool qn_hit_on_boundary(const char *qn, const char *hit, const char *id, size_t id_len) {
char first = id[0];
char last = id[id_len - 1];
if (hit > qn && qn_is_alnum(hit[-1]) && (qn_is_lower(first) || is_digit_char(first))) {
return false;
}
if (qn_is_alnum(last) && qn_is_lower(hit[id_len])) {
return false;
}
return true;
}
/* Check if any library identifier appears in the QN on identifier
* boundaries (see qn_hit_on_boundary). Case-sensitive: "requests" matches
* "project.venv.requests.api.get" and "svc.requests_get" but neither
* "Requests" nor "myrequests". */
static const lib_pattern_t *match_qn(const char *qn, const lib_pattern_t *patterns) {
if (!qn || !qn[0]) {
return NULL;
}
for (int i = 0; patterns[i].library_id != NULL; i++) {
if (strstr(qn, patterns[i].library_id) != NULL) {
return &patterns[i];
const char *id = patterns[i].library_id;
size_t id_len = strlen(id);
if (id_len == 0) {
continue;
}
for (const char *hit = strstr(qn, id); hit != NULL; hit = strstr(hit + 1, id)) {
if (qn_hit_on_boundary(qn, hit, id, id_len)) {
return &patterns[i];
}
}
}
return NULL;
@@ -574,10 +682,6 @@ static bool contains_segment(const char *path, const char *segment) {
return false;
}
static bool is_digit_char(char ch) {
return ch >= '0' && ch <= '9';
}
static bool has_http_route_marker(const char *path) {
if (starts_with_segment(path, "api") || starts_with_segment(path, "apis") ||
starts_with_segment(path, "graphql") || starts_with_segment(path, "health") ||
@@ -714,6 +818,25 @@ static const char *strip_string_delimiters(const char *literal, char *buf, size_
return buf;
}
/* A comment opens with the slash a route opens with, and an argument list
* that starts with one handed three Java block comments to the Route pass as
* URLs (elasticsearch, 2026-09-16). The wildcard route (slash-star alone) is a
* path, so the block-comment shape needs its closing star-slash; a route
* literal never holds a line break. */
bool cbm_service_pattern_is_comment_text(const char *text) {
if (!text || text[0] != '/') {
return false;
}
if (strchr(text, '\n') != NULL || strchr(text, '\r') != NULL) {
return true;
}
size_t n = strlen(text);
if (text[1] == '*' && n >= 4 && text[n - 2] == '*' && text[n - 1] == '/') {
return true;
}
return text[1] == '/' && (text[2] == ' ' || text[2] == '\t');
}
bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name) {
char path_buf[1024];
const char *path = strip_string_delimiters(literal, path_buf, sizeof(path_buf));
@@ -729,6 +852,9 @@ bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *
if (path[0] != '/') {
return false;
}
if (cbm_service_pattern_is_comment_text(path)) {
return false;
}
if (callee_is_delimiter_or_filesystem_builder(callee_name)) {
return false;
}
internal/cbm/service_patterns.h
+6
View File
@@ -48,6 +48,12 @@ bool cbm_service_pattern_is_global_fetch(const char *callee_name);
* Rejects filesystem paths and non-HTTP string consumers. */
bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name);
/* True when text that reached a path slot is a source comment rather than a
* path: a block comment (slash-star … star-slash), a line comment (slash-slash
* followed by a space), or any text with a line break. The wildcard route
* (slash-star alone) is a path. */
bool cbm_service_pattern_is_comment_text(const char *text);
/* Per-worker TLS cache for cbm_service_pattern_match results. The
* pattern matcher runs once per resolved CALL edge in emit_service_
* edge — that's 6 pattern lists × ~30 patterns × strstr per call ≈
internal/cbm/sqlite_writer.c
+264 -216
View File
@@ -15,6 +15,8 @@
// - Varints: 1-9 bytes, big-endian, MSB continuation
#include "sqlite_writer.h"
#include "foundation/arena.h" /* index-cell arena */
#include "foundation/compat.h" /* CBM_TLS */
#include "foundation/constants.h"
#include "foundation/compat_fs.h"
#include "foundation/compat_thread.h"
@@ -26,6 +28,7 @@
#include <string.h>
#include <stdint.h>
#include <stdbool.h>
#include <errno.h>
#ifdef _WIN32
#include <io.h>
@@ -301,29 +304,37 @@ typedef struct {
uint8_t *data;
int len;
int cap;
bool heap; /* data is malloc-owned; otherwise it is the caller's inline space */
} DynBuf;
static void dynbuf_init(DynBuf *b) {
b->data = NULL;
/* Start on caller-provided space (a record builder's stack array). Growth past
* it moves to the heap once; a buffer that never outgrows it never allocates. */
static void dynbuf_init_inline(DynBuf *b, uint8_t *space, int cap) {
b->data = space;
b->len = 0;
b->cap = 0;
b->cap = cap;
b->heap = false;
}
static bool dynbuf_ensure(DynBuf *b, int needed) {
if (b->len + needed <= b->cap) {
return true;
}
int newcap = b->cap == 0 ? INITIAL_PAGE_CAP : b->cap;
int newcap = b->cap < INITIAL_PAGE_CAP ? INITIAL_PAGE_CAP : b->cap;
while (newcap < b->len + needed) {
newcap *= GROWTH_FACTOR;
}
uint8_t *p = (uint8_t *)realloc(b->data, newcap);
uint8_t *p = b->heap ? (uint8_t *)realloc(b->data, newcap) : (uint8_t *)malloc(newcap);
if (!p) {
(void)fprintf(stderr, "cbm_write_db: dynbuf realloc failed size=%d\n", newcap);
return false;
}
if (!b->heap && b->len > 0) {
memcpy(p, b->data, (size_t)b->len);
}
b->data = p;
b->cap = newcap;
b->heap = true;
return true;
}
@@ -343,22 +354,35 @@ static bool dynbuf_append(DynBuf *b, const void *data, int len) {
}
static void dynbuf_free(DynBuf *b) {
free(b->data);
if (b->heap) {
free(b->data);
}
b->data = NULL;
b->len = b->cap = 0;
b->heap = false;
}
// --- Record builder ---
// Builds a SQLite record: header (header_len varint + serial types) + body (values)
/* Inline space for one record. Every node, edge and index row used to allocate
* a 4 KB header buffer and a 4 KB body buffer and free both after copying a few
* dozen bytes out: the waste sanitizer measured 32 M allocations and ~126 GB of
* churn on the Go corpus, ~99 % of it never written (2026-09-17). A header is
* one serial type per column; a body is the row's text. Rows that do not fit
* (long property JSON) still grow onto the heap. */
enum { REC_HEADER_INLINE = 64, REC_BODY_INLINE = 1536 };
typedef struct {
DynBuf header; // serial type varints
DynBuf body; // column values
uint8_t header_space[REC_HEADER_INLINE];
uint8_t body_space[REC_BODY_INLINE];
} RecordBuilder;
static void rec_init(RecordBuilder *r) {
dynbuf_init(&r->header);
dynbuf_init(&r->body);
dynbuf_init_inline(&r->header, r->header_space, REC_HEADER_INLINE);
dynbuf_init_inline(&r->body, r->body_space, REC_BODY_INLINE);
}
static void rec_free(RecordBuilder *r) {
@@ -408,8 +432,7 @@ static void rec_add_blob(RecordBuilder *r, const uint8_t *data, int len) {
// Finalize: returns the complete record bytes (header_len + header + body).
// Caller must free the returned buffer.
static uint8_t *rec_finalize(RecordBuilder *r, int *out_len) {
*out_len = 0;
static int rec_total_header(const RecordBuilder *r) {
int header_content_len = r->header.len;
int header_len_varint_len = varint_len(header_content_len + varint_len(header_content_len));
// The header size varint includes itself, so we may need to iterate
@@ -420,20 +443,73 @@ static uint8_t *rec_finalize(RecordBuilder *r, int *out_len) {
header_len_varint_len = recalc;
total_header = header_len_varint_len + header_content_len;
}
return total_header;
}
int total = total_header + r->body.len;
// Bytes rec_write produces: the record exactly as rec_finalize returns it.
static int rec_payload_len(const RecordBuilder *r) {
return rec_total_header(r) + r->body.len;
}
// Write the record (header_len + header + body) at dst; returns its length.
static int rec_write(const RecordBuilder *r, uint8_t *dst) {
int pos = put_varint(dst, rec_total_header(r));
memcpy(dst + pos, r->header.data, r->header.len);
pos += r->header.len;
memcpy(dst + pos, r->body.data, r->body.len);
return pos + r->body.len;
}
static uint8_t *rec_finalize(RecordBuilder *r, int *out_len) {
*out_len = 0;
int total = rec_payload_len(r);
uint8_t *buf = (uint8_t *)malloc(total);
if (!buf) {
return NULL;
}
int pos = put_varint(buf, total_header);
memcpy(buf + pos, r->header.data, header_content_len);
pos += header_content_len;
memcpy(buf + pos, r->body.data, r->body.len);
*out_len = total;
*out_len = rec_write(r, buf);
return buf;
}
/* One index build's cells live in an arena: they are all released together
* once the B-tree is written, where a malloc + free per row was 12.1 M
* allocations on the Go corpus (waste sanitizer, 2026-09-17). The writer
* builds one index at a time on one thread; NULL = the malloc path, which the
* unit tests and any other caller keep. */
enum { CELLS_ARENA_BLOCK = 1024 * 1024 };
static CBM_TLS CBMArena *tl_cells_arena;
static uint8_t *cells_alloc(size_t bytes) {
if (tl_cells_arena) {
return (uint8_t *)cbm_arena_alloc(tl_cells_arena, bytes);
}
return (uint8_t *)malloc(bytes);
}
static void cells_free(uint8_t *cell) {
if (!tl_cells_arena) {
free(cell);
}
}
// An index leaf cell, varint(payload_len) + record, written in place: building
// the record first and copying it into the cell was a second allocation and a
// copy for every index entry (16 M record buffers of pure churn on the Go
// corpus, waste sanitizer 2026-09-17). Same bytes.
static uint8_t *rec_finalize_index_cell(const RecordBuilder *r, int *out_len) {
*out_len = 0;
int payload_len = rec_payload_len(r);
int total = varint_len(payload_len) + payload_len;
uint8_t *cell = cells_alloc((size_t)total);
if (!cell) {
return NULL;
}
int pos = put_varint(cell, payload_len);
pos += rec_write(r, cell + pos);
*out_len = pos;
return cell;
}
// --- Page builder ---
// Accumulates cells (records) into B-tree leaf pages.
@@ -460,6 +536,11 @@ typedef struct {
PageRef *leaves;
int leaf_count;
int leaf_cap;
// One reusable buffer every table cell is assembled in before it is copied
// into the page (grows to the largest cell, freed by pb_free).
uint8_t *cell_buf;
int cell_cap;
} PageBuilder;
static void pb_init(PageBuilder *pb, FILE *fp, uint32_t start_page, bool is_index) {
@@ -475,6 +556,8 @@ static void pb_init(PageBuilder *pb, FILE *fp, uint32_t start_page, bool is_inde
pb->leaves = NULL;
pb->leaf_count = 0;
pb->leaf_cap = 0;
pb->cell_buf = NULL;
pb->cell_cap = 0;
}
static void pb_free(PageBuilder *pb) {
@@ -484,6 +567,21 @@ static void pb_free(PageBuilder *pb) {
}
free(pb->leaves);
}
free(pb->cell_buf);
pb->cell_buf = NULL;
pb->cell_cap = 0;
}
static uint8_t *pb_cell_scratch(PageBuilder *pb, int len) {
if (pb->cell_cap < len) {
uint8_t *grown = (uint8_t *)realloc(pb->cell_buf, (size_t)len);
if (!grown) {
return NULL;
}
pb->cell_buf = grown;
pb->cell_cap = len;
}
return pb->cell_buf;
}
// Flush current leaf page to file
@@ -736,74 +834,46 @@ static uint32_t pb_build_interior(PageBuilder *pb, bool is_index) {
// Build a nodes table record: (id, project, label, name, qualified_name, file_path, start_line,
// end_line, properties)
static uint8_t *build_node_record(const CBMDumpNode *n, int *out_len) {
RecordBuilder r;
rec_init(&r);
rec_add_int(&r, n->id);
rec_add_text(&r, n->project);
rec_add_text(&r, n->label);
rec_add_text(&r, n->name);
rec_add_text(&r, n->qualified_name);
rec_add_text(&r, n->file_path ? n->file_path : "");
rec_add_int(&r, n->start_line);
rec_add_int(&r, n->end_line);
rec_add_text(&r, n->properties ? n->properties : "{}");
uint8_t *data = rec_finalize(&r, out_len);
rec_free(&r);
return data;
static void fill_node_record(RecordBuilder *r, const CBMDumpNode *n) {
rec_add_int(r, n->id);
rec_add_text(r, n->project);
rec_add_text(r, n->label);
rec_add_text(r, n->name);
rec_add_text(r, n->qualified_name);
rec_add_text(r, n->file_path ? n->file_path : "");
rec_add_int(r, n->start_line);
rec_add_int(r, n->end_line);
rec_add_text(r, n->properties ? n->properties : "{}");
}
// Build an edges table record: (id, project, source_id, target_id, type, properties)
// url_path_gen and local_name_gen are VIRTUAL generated columns — NOT stored in the record.
static uint8_t *build_edge_record(const CBMDumpEdge *e, int *out_len) {
RecordBuilder r;
rec_init(&r);
rec_add_int(&r, e->id);
rec_add_text(&r, e->project);
rec_add_int(&r, e->source_id);
rec_add_int(&r, e->target_id);
rec_add_text(&r, e->type);
rec_add_text(&r, e->properties ? e->properties : "{}");
uint8_t *data = rec_finalize(&r, out_len);
rec_free(&r);
return data;
static void fill_edge_record(RecordBuilder *r, const CBMDumpEdge *e) {
rec_add_int(r, e->id);
rec_add_text(r, e->project);
rec_add_int(r, e->source_id);
rec_add_int(r, e->target_id);
rec_add_text(r, e->type);
rec_add_text(r, e->properties ? e->properties : "{}");
}
// Build a node_vectors table record: (node_id, project, vector)
// Includes node_id in the record body (same pattern as build_node_record).
static uint8_t *build_vector_record(const CBMDumpVector *v, int *out_len) {
RecordBuilder r;
rec_init(&r);
rec_add_int(&r, v->node_id);
rec_add_text(&r, v->project);
rec_add_blob(&r, v->vector, v->vector_len);
uint8_t *data = rec_finalize(&r, out_len);
rec_free(&r);
return data;
// Includes node_id in the record body (same pattern as the node record).
static void fill_vector_record(RecordBuilder *r, const CBMDumpVector *v) {
rec_add_int(r, v->node_id);
rec_add_text(r, v->project);
rec_add_blob(r, v->vector, v->vector_len);
}
// Build a token_vectors table record: (id, project, token, vector, idf)
static uint8_t *build_token_vec_record(const CBMDumpTokenVec *tv, int *out_len) {
RecordBuilder r;
rec_init(&r);
rec_add_int(&r, tv->id);
rec_add_text(&r, tv->project);
rec_add_text(&r, tv->token);
rec_add_blob(&r, tv->vector, tv->vector_len);
static void fill_token_vec_record(RecordBuilder *r, const CBMDumpTokenVec *tv) {
rec_add_int(r, tv->id);
rec_add_text(r, tv->project);
rec_add_text(r, tv->token);
rec_add_blob(r, tv->vector, tv->vector_len);
/* Store IDF as integer × 1000 for fixed-point (avoid float in record) */
enum { IDF_FIXED_POINT_SCALE = 1000 };
rec_add_int(&r, (int64_t)(tv->idf * IDF_FIXED_POINT_SCALE));
uint8_t *data = rec_finalize(&r, out_len);
rec_free(&r);
return data;
rec_add_int(r, (int64_t)(tv->idf * IDF_FIXED_POINT_SCALE));
}
// Build a projects table record: (name, indexed_at, root_path)
@@ -924,27 +994,8 @@ static uint8_t *build_index_entry_2text_rowid(const char *col1, const char *col2
rec_add_text(&r, col1);
rec_add_text(&r, col2);
rec_add_int(&r, rowid);
int payload_len = 0;
uint8_t *payload = rec_finalize(&r, &payload_len);
uint8_t *cell = rec_finalize_index_cell(&r, out_len);
rec_free(&r);
if (!payload) {
*out_len = 0;
return NULL;
}
// Index cell: varint(payload_len) + payload
int vl = varint_len(payload_len);
int total = vl + payload_len;
uint8_t *cell = (uint8_t *)malloc(total);
if (!cell) {
free(payload);
*out_len = 0;
return NULL;
}
int pos = put_varint(cell, payload_len);
memcpy(cell + pos, payload, payload_len);
free(payload);
*out_len = total;
return cell;
}
@@ -956,26 +1007,8 @@ static uint8_t *build_index_entry_int_text_rowid(int64_t val, const char *text,
rec_add_int(&r, val);
rec_add_text(&r, text);
rec_add_int(&r, rowid);
int payload_len = 0;
uint8_t *payload = rec_finalize(&r, &payload_len);
uint8_t *cell = rec_finalize_index_cell(&r, out_len);
rec_free(&r);
if (!payload) {
*out_len = 0;
return NULL;
}
int vl = varint_len(payload_len);
int total = vl + payload_len;
uint8_t *cell = (uint8_t *)malloc(total);
if (!cell) {
free(payload);
*out_len = 0;
return NULL;
}
int pos = put_varint(cell, payload_len);
memcpy(cell + pos, payload, payload_len);
free(payload);
*out_len = total;
return cell;
}
@@ -988,26 +1021,8 @@ static uint8_t *build_index_entry_text_int_text_rowid(const char *t1, int64_t va
rec_add_int(&r, val);
rec_add_text(&r, t2);
rec_add_int(&r, rowid);
int payload_len = 0;
uint8_t *payload = rec_finalize(&r, &payload_len);
uint8_t *cell = rec_finalize_index_cell(&r, out_len);
rec_free(&r);
if (!payload) {
*out_len = 0;
return NULL;
}
int vl = varint_len(payload_len);
int total = vl + payload_len;
uint8_t *cell = (uint8_t *)malloc(total);
if (!cell) {
free(payload);
*out_len = 0;
return NULL;
}
int pos = put_varint(cell, payload_len);
memcpy(cell + pos, payload, payload_len);
free(payload);
*out_len = total;
return cell;
}
@@ -1023,26 +1038,8 @@ static uint8_t *build_index_entry_unique_2int_2text_rowid(int64_t v1, int64_t v2
rec_add_text(&r, text);
rec_add_text(&r, text2);
rec_add_int(&r, rowid);
int payload_len = 0;
uint8_t *payload = rec_finalize(&r, &payload_len);
uint8_t *cell = rec_finalize_index_cell(&r, out_len);
rec_free(&r);
if (!payload) {
*out_len = 0;
return NULL;
}
int vlen = varint_len(payload_len);
int total = vlen + payload_len;
uint8_t *cell = (uint8_t *)malloc(total);
if (!cell) {
free(payload);
*out_len = 0;
return NULL;
}
int pos = put_varint(cell, payload_len);
memcpy(cell + pos, payload, payload_len);
free(payload);
*out_len = total;
return cell;
}
@@ -1115,18 +1112,34 @@ static uint8_t *overflowize_index_cell(FILE *fp, uint32_t *next_page, uint8_t *c
uint32_t first_ovfl =
write_overflow_pages(fp, next_page, cell + vlen + local, (int)plen - local);
int nlen = vlen + local + BTREE_PTR_SIZE;
uint8_t *data = (uint8_t *)malloc((size_t)nlen);
uint8_t *data = cells_alloc((size_t)nlen);
if (!data) {
return cell; /* fall back to the (broken) inline form on OOM */
}
memcpy(data, cell, (size_t)(vlen + local));
put_u32(data + vlen + local, first_ovfl);
free(cell);
cells_free(cell);
*cell_len = nlen;
return data;
}
#define TABLE_OVERFLOW_MIN_LOCAL 8199
// Put a finished table cell on the current leaf, flushing a full leaf first.
static bool pb_place_table_cell(PageBuilder *pb, const uint8_t *cell, int cell_len,
int64_t prev_rowid) {
if (!pb_cell_fits(pb, cell_len) && pb->cell_count > 0) {
if (!pb_ensure_leaf_cap(pb)) {
return false;
}
pb->leaves[pb->leaf_count].max_key = prev_rowid;
pb->leaves[pb->leaf_count].sep_cell = NULL;
pb->leaves[pb->leaf_count].sep_cell_len = 0;
pb_flush_leaf(pb);
}
pb_add_cell(pb, cell, cell_len);
return true;
}
// Add a table cell to the PageBuilder, flushing leaf pages as needed.
// If the payload exceeds max_local, overflow pages are written and only the
// local portion plus a 4-byte overflow page pointer is stored in the leaf cell.
@@ -1160,22 +1173,39 @@ static void pb_add_table_cell_with_flush(PageBuilder *pb, int64_t rowid, const u
if (!cell) {
return;
}
if (!pb_cell_fits(pb, cell_len) && pb->cell_count > 0) {
if (!pb_ensure_leaf_cap(pb)) {
free(cell);
return;
}
pb->leaves[pb->leaf_count].max_key = prev_rowid;
pb->leaves[pb->leaf_count].sep_cell = NULL;
pb->leaves[pb->leaf_count].sep_cell_len = 0;
pb_flush_leaf(pb);
}
pb_add_cell(pb, cell, cell_len);
(void)pb_place_table_cell(pb, cell, cell_len, prev_rowid);
free(cell);
}
// A table record written straight into the page builder's cell buffer: the
// record builder's bytes go to the cell once, where rec_finalize +
// build_table_cell allocated the record, allocated the cell, copied, and freed
// both for every row. Same bytes; a record too large for the page takes the
// overflow path unchanged. False only on allocation failure.
static bool pb_add_table_record_with_flush(PageBuilder *pb, int64_t rowid, const RecordBuilder *r,
int64_t prev_rowid) {
int payload_len = rec_payload_len(r);
if (payload_len > TABLE_OVERFLOW_MAX_LOCAL) {
int rec_len = 0;
uint8_t *rec = rec_finalize((RecordBuilder *)r, &rec_len);
if (!rec) {
return false;
}
pb_add_table_cell_with_flush(pb, rowid, rec, rec_len, prev_rowid);
free(rec);
return true;
}
int total = varint_len(payload_len) + varint_len(rowid) + payload_len;
uint8_t *cell = pb_cell_scratch(pb, total);
if (!cell) {
return false;
}
int pos = put_varint(cell, payload_len);
pos += put_varint(cell + pos, rowid);
pos += rec_write(r, cell + pos);
return pb_place_table_cell(pb, cell, pos, prev_rowid);
}
// Finalize a table PageBuilder: flush last leaf and build interior pages.
static uint32_t pb_finalize_table(PageBuilder *pb, uint32_t *next_page, int64_t last_rowid) {
if (pb->cell_count > 0) {
@@ -1607,21 +1637,8 @@ static uint8_t *ecell_url_path(const CBMDumpEdge *e, int *out_len) {
rec_add_null(&r);
}
rec_add_int(&r, e->id);
int payload_len = 0;
uint8_t *payload = rec_finalize(&r, &payload_len);
uint8_t *cell = rec_finalize_index_cell(&r, out_len);
rec_free(&r);
int vlen = varint_len(payload_len);
int total = vlen + payload_len;
uint8_t *cell = (uint8_t *)malloc(total);
if (!cell) {
free(payload);
*out_len = 0;
return NULL;
}
int pos = put_varint(cell, payload_len);
memcpy(cell + pos, payload, payload_len);
free(payload);
*out_len = total;
return cell;
}
@@ -1636,19 +1653,21 @@ static uint32_t build_edge_index_sorted(FILE *fp, uint32_t *next_page, CBMDumpEd
}
uint8_t **idx_cells = (uint8_t **)malloc(edge_count * sizeof(uint8_t *));
int *idx_lens = (int *)malloc(edge_count * sizeof(int));
CBMArena cells;
cbm_arena_init_lazy(&cells, CELLS_ARENA_BLOCK);
if (!idx_cells || !idx_lens) {
free(perm);
free(idx_cells);
free(idx_lens);
return 0;
}
tl_cells_arena = &cells;
for (int i = 0; i < edge_count; i++) {
int si = perm[i];
idx_cells[i] = cell_fn(&edges[si], &idx_lens[i]);
if (!idx_cells[i]) {
for (int j = 0; j < i; j++) {
free(idx_cells[j]);
}
tl_cells_arena = NULL;
cbm_arena_destroy(&cells);
free(idx_cells);
free(idx_lens);
free(perm);
@@ -1657,9 +1676,8 @@ static uint32_t build_edge_index_sorted(FILE *fp, uint32_t *next_page, CBMDumpEd
}
free(perm);
uint32_t root = write_index_btree(fp, next_page, idx_cells, idx_lens, edge_count);
for (int i = 0; i < edge_count; i++) {
free(idx_cells[i]);
}
tl_cells_arena = NULL;
cbm_arena_destroy(&cells);
free(idx_cells);
free(idx_lens);
return root;
@@ -1691,20 +1709,22 @@ static uint32_t build_node_index_sorted(FILE *fp, uint32_t *next_page, CBMDumpNo
}
uint8_t **idx_cells = (uint8_t **)malloc(node_count * sizeof(uint8_t *));
int *idx_lens = (int *)malloc(node_count * sizeof(int));
CBMArena cells;
cbm_arena_init_lazy(&cells, CELLS_ARENA_BLOCK);
if (!idx_cells || !idx_lens) {
free(perm);
free(idx_cells);
free(idx_lens);
return 0;
}
tl_cells_arena = &cells;
for (int i = 0; i < node_count; i++) {
int si = perm[i];
idx_cells[i] = build_index_entry_2text_rowid(nodes[si].project, col_fn(&nodes[si]),
nodes[si].id, &idx_lens[i]);
if (!idx_cells[i]) {
for (int j = 0; j < i; j++) {
free(idx_cells[j]);
}
tl_cells_arena = NULL;
cbm_arena_destroy(&cells);
free(idx_cells);
free(idx_lens);
free(perm);
@@ -1713,9 +1733,8 @@ static uint32_t build_node_index_sorted(FILE *fp, uint32_t *next_page, CBMDumpNo
}
free(perm);
uint32_t root = write_index_btree(fp, next_page, idx_cells, idx_lens, node_count);
for (int i = 0; i < node_count; i++) {
free(idx_cells[i]);
}
tl_cells_arena = NULL;
cbm_arena_destroy(&cells);
free(idx_cells);
free(idx_lens);
return root;
@@ -1759,6 +1778,10 @@ static int sync_writer_output(FILE *fp) {
}
static int discard_writer_output(write_db_ctx_t *w, int rc) {
/* Cleanup runs after the failure that brought us here, and a library
* call may set errno even when it succeeds. Carry the reason across it
* so the caller can report WHY the publish failed. */
int failure_errno = errno;
if (w->fp) {
(void)fclose(w->fp);
w->fp = NULL;
@@ -1766,6 +1789,7 @@ static int discard_writer_output(write_db_ctx_t *w, int rc) {
if (w->temp_path[0]) {
(void)cbm_unlink(w->temp_path);
}
errno = failure_errno;
return rc;
}
@@ -1774,10 +1798,12 @@ static int publish_writer_output(write_db_ctx_t *w) {
return discard_writer_output(w, ERR_WRITE_FAILED);
}
if (fclose(w->fp) != 0) {
int failure_errno = errno;
w->fp = NULL;
if (w->temp_path[0]) {
(void)cbm_unlink(w->temp_path);
}
errno = failure_errno;
return ERR_WRITE_FAILED;
}
w->fp = NULL;
@@ -1785,7 +1811,12 @@ static int publish_writer_output(write_db_ctx_t *w) {
return 0;
}
if (cbm_rename_replace(w->temp_path, w->final_path) != 0) {
/* cbm_rename_replace translated the platform error into errno so the
* caller can say what denied the publish (#1620). Preserve it across
* the cleanup unlink. */
int rename_errno = errno;
(void)cbm_unlink(w->temp_path);
errno = rename_errno;
return ERR_WRITE_FAILED;
}
/* Sidecars are removed only after the replacement succeeds. On POSIX,
@@ -1796,13 +1827,13 @@ static int publish_writer_output(write_db_ctx_t *w) {
return 0;
}
/* Callback type for building a record from an item at index i. */
typedef uint8_t *(*build_record_fn)(const void *items, int i, int *out_len);
/* Callback type for filling a record from an item at index i. */
typedef void (*fill_record_fn)(RecordBuilder *r, const void *items, int i);
typedef int64_t (*get_rowid_fn)(const void *items, int i);
/* Write a streaming B-tree table from count items, or an empty table if count == 0. */
static int write_one_table(write_db_ctx_t *w, uint32_t *root, const void *items, int count,
build_record_fn build_rec, get_rowid_fn get_id) {
fill_record_fn fill_rec, get_rowid_fn get_id) {
if (count <= 0 || !items) {
*root = write_table_btree(w->fp, &w->next_page, NULL, NULL, NULL, 0, false);
return 0;
@@ -1810,15 +1841,17 @@ static int write_one_table(write_db_ctx_t *w, uint32_t *root, const void *items,
PageBuilder pb;
pb_init(&pb, w->fp, w->next_page, false);
for (int i = 0; i < count; i++) {
int rec_len;
uint8_t *rec = build_rec(items, i, &rec_len);
if (!rec) {
return ERR_WRITE_FAILED;
}
RecordBuilder r;
rec_init(&r);
fill_rec(&r, items, i);
int64_t rowid = get_id(items, i);
int64_t prev_id = i > 0 ? get_id(items, i - SKIP_ONE) : 0;
pb_add_table_cell_with_flush(&pb, rowid, rec, rec_len, prev_id);
free(rec);
bool placed = pb_add_table_record_with_flush(&pb, rowid, &r, prev_id);
rec_free(&r);
if (!placed) {
pb_free(&pb);
return ERR_WRITE_FAILED;
}
}
*root = pb_finalize_table(&pb, &w->next_page, get_id(items, count - SKIP_ONE));
return 0;
@@ -1826,20 +1859,20 @@ static int write_one_table(write_db_ctx_t *w, uint32_t *root, const void *items,
/* Adapter functions for write_one_table (nodes are written via the streaming
* PageBuilder in cbm_writer_append_nodes, so no node adapter is needed here). */
static uint8_t *adapt_build_edge(const void *items, int i, int *out_len) {
return build_edge_record(&((const CBMDumpEdge *)items)[i], out_len);
static void adapt_fill_edge(RecordBuilder *r, const void *items, int i) {
fill_edge_record(r, &((const CBMDumpEdge *)items)[i]);
}
static int64_t adapt_edge_id(const void *items, int i) {
return ((const CBMDumpEdge *)items)[i].id;
}
static uint8_t *adapt_build_vector(const void *items, int i, int *out_len) {
return build_vector_record(&((const CBMDumpVector *)items)[i], out_len);
static void adapt_fill_vector(RecordBuilder *r, const void *items, int i) {
fill_vector_record(r, &((const CBMDumpVector *)items)[i]);
}
static int64_t adapt_vector_id(const void *items, int i) {
return ((const CBMDumpVector *)items)[i].node_id;
}
static uint8_t *adapt_build_token_vec(const void *items, int i, int *out_len) {
return build_token_vec_record(&((const CBMDumpTokenVec *)items)[i], out_len);
static void adapt_fill_token_vec(RecordBuilder *r, const void *items, int i) {
fill_token_vec_record(r, &((const CBMDumpTokenVec *)items)[i]);
}
static int64_t adapt_token_vec_id(const void *items, int i) {
return ((const CBMDumpTokenVec *)items)[i].id;
@@ -2066,17 +2099,17 @@ static int write_db_after_nodes(write_db_ctx_t *w, uint32_t nodes_root) {
uint32_t vectors_root;
uint32_t token_vecs_root;
int rc =
write_one_table(w, &edges_root, w->edges, w->edge_count, adapt_build_edge, adapt_edge_id);
write_one_table(w, &edges_root, w->edges, w->edge_count, adapt_fill_edge, adapt_edge_id);
if (rc != 0) {
return discard_writer_output(w, rc);
}
rc = write_one_table(w, &vectors_root, w->vectors, w->vector_count, adapt_build_vector,
rc = write_one_table(w, &vectors_root, w->vectors, w->vector_count, adapt_fill_vector,
adapt_vector_id);
if (rc != 0) {
return discard_writer_output(w, rc);
}
rc = write_one_table(w, &token_vecs_root, w->token_vecs, w->token_vec_count,
adapt_build_token_vec, adapt_token_vec_id);
adapt_fill_token_vec, adapt_token_vec_id);
if (rc != 0) {
return discard_writer_output(w, rc);
}
@@ -2283,13 +2316,22 @@ cbm_db_writer_t *cbm_writer_open(const char *path) {
int n = snprintf(w->wc.final_path, sizeof(w->wc.final_path), "%s", path);
if (n < 0 || (size_t)n >= sizeof(w->wc.final_path) ||
make_writer_temp_path(path, w, w->wc.temp_path, sizeof(w->wc.temp_path)) != 0) {
/* Both conditions are truncation and neither sets errno; name the
* reason rather than let the caller report a stale one. */
free(w);
errno = ENAMETOOLONG;
return NULL;
}
FILE *fp = cbm_fopen(w->wc.temp_path, "wb");
if (!fp) {
/* The cleanup below unlinks a file that was never created, so it
* fails and leaves ENOENT behind — which reads as a missing path
* when the real answer is that the directory refused the create.
* That is the #1620 case, so carry the open's reason across it. */
int open_errno = errno;
(void)cbm_unlink(w->wc.temp_path);
free(w);
errno = open_errno;
return NULL;
}
w->wc.fp = fp;
@@ -2307,16 +2349,18 @@ int cbm_writer_append_nodes(cbm_db_writer_t *w, const CBMDumpNode *nodes, int co
return w->err;
}
for (int i = 0; i < count; i++) {
int rec_len;
uint8_t *rec = build_node_record(&nodes[i], &rec_len);
if (!rec) {
RecordBuilder r;
rec_init(&r);
fill_node_record(&r, &nodes[i]);
/* prev_rowid is the previous node's id (0 for the very first), matching
* the one-shot write_one_table loop — so output is byte-identical. */
bool placed =
pb_add_table_record_with_flush(&w->nodes_pb, nodes[i].id, &r, w->last_node_rowid);
rec_free(&r);
if (!placed) {
w->err = ERR_WRITE_FAILED;
return w->err;
}
/* prev_rowid is the previous node's id (0 for the very first), matching
* the one-shot write_one_table loop — so output is byte-identical. */
pb_add_table_cell_with_flush(&w->nodes_pb, nodes[i].id, rec, rec_len, w->last_node_rowid);
free(rec);
w->last_node_rowid = nodes[i].id;
w->node_rows_written++;
}
@@ -2355,6 +2399,10 @@ int cbm_writer_finalize(cbm_db_writer_t *w, const char *project, const char *roo
write_db_ctx_t wc = w->wc; /* value copy survives free(w) */
free(w);
if (err != 0) {
/* A sticky append failure: errno belongs to whatever call failed
* many calls ago, not to the publish. Clear it so the caller does
* not attach a reason this path does not have. */
errno = 0;
return discard_writer_output(&wc, err);
}
return write_db_after_nodes(&wc, nodes_root);
internal/cbm/vendored/grammars/MANIFEST.md
+1
View File
@@ -87,6 +87,7 @@ row instead.
| rescript | `rescript/scanner.c`, deserialize | guard `memcpy(state, buffer, n_bytes)` with `if (n_bytes > 0)` | UBSan: zero-length `memcpy` with a NULL `buffer` / `n_bytes == 0` on empty-state deserialize (formal UB, harmless). The sibling serialize copies a fixed `sizeof(ScannerState)` (always > 0, non-NULL src) and needs no guard. |
| purescript | `purescript/scanner.c`, serialize | guard `memcpy(buffer, indents->data, to_copy)` with `if (to_copy > 0)` | UBSan: zero-length `memcpy` with a NULL/0-size source when the indent vector is empty (formal UB, harmless) |
| plsql | `plsql/parser.c`, include | `#include <tree_sitter/parser.h>` → `#include "tree_sitter/parser.h"` | The older ABI-14 generator emits angle brackets; every other vendored grammar uses the quoted form, which resolves the per-grammar `tree_sitter/` header from the including file's directory |
| properties | `properties/scanner.c`, whole file | move the `reached_eof` flag out of a file-scope `static bool` and into a per-parser payload allocated by `..._external_scanner_create()` | **Data race, and the only one of its kind in 103 vendored scanners.** That flag is how the scanner refuses to emit a second end-of-input `FAKE_EOL`, and upstream keeps it in ONE process-wide object. With several worker threads indexing `.properties` files at once, whichever reached EOF first set the flag, so another thread's `!reached_eof` was false and its parse never received the `FAKE_EOL` it needed to finish — it sat at end-of-input asking for a token that would never arrive. Measured 2026-09-19: ~106 M parse operations on a 253-byte file before the per-file budget stopped it, after which the file was dropped from the graph and java/kotlin indexes differed run to run. Interleaved A/B under identical load, 40 runs each: unpatched 6 stalls, patched 0. The serialize/deserialize wire format is unchanged (the state is still the returned length). Upstream `6310671b24d4` still carries the static, so a re-vendor must re-apply this |
| swift | `swift/scanner.c`, `OP_SYMBOL_SUPPRESSOR` + `eat_operators` | `1UL <<` / `1 <<` → `1ULL <<` | UBSan: `1 << suppressor` shifts an `int` by up to `TOKEN_COUNT` bits, undefined once the index reaches 31, while the mask it feeds is `uint64_t`. `1UL << FAKE_TRY_BANG` is the same defect on Windows, where `unsigned long` is 32 bits and `FAKE_TRY_BANG` is 32; the CLANGARM64 leg runs UBSan in trap mode, so there it is an illegal instruction rather than a log line. Upstream already carries both changes: `fb63a7004f07` (2026-04-06, upstream #558) for `eat_operators`, `6ab8d1d74ebd` (2026-08-10) for the `OP_SYMBOL_SUPPRESSOR` entry. Our pin `8abb3e8b3325` (2026-03-20) predates both, so this is a backport rather than a local invention — a re-vendor past 2026-08-10 should delete this row, not re-apply it |
## Vendored from verified upstream
internal/cbm/vendored/grammars/properties/scanner.c
Generated
+32 -8
View File
@@ -1,5 +1,7 @@
#include "tree_sitter/parser.h"
#include <stdlib.h>
#ifdef _MSC_VER
#pragma warning(disable : 4100)
#elif defined(__GNUC__) || defined(__clang__)
@@ -8,21 +10,43 @@
enum TokenType { FAKE_EOL };
static bool reached_eof = false;
/* The scanner's one bit of state -- whether THIS parse has already emitted the
* end-of-input FAKE_EOL -- lives in the payload, one per parser, which is what
* tree-sitter's create/destroy contract is for.
*
* It used to be a file-scope `static bool`: one object for the entire process,
* shared by every thread. The grammar needs exactly one FAKE_EOL to close a
* file, and `!reached_eof` is how the scanner refuses to emit a second one. So
* when two threads indexed .properties files at the same time, whichever
* reached EOF first set the flag, and the other thread's `!reached_eof` was
* false -- it never received the FAKE_EOL its parse was waiting for and sat at
* end-of-input asking for a token that would never arrive. Measured on
* 2026-09-19: ~106 million parse operations on a 253-byte file before the
* per-file budget stopped it, after which the file was dropped from the graph
* and the index differed from run to run. */
typedef struct {
bool reached_eof;
} Scanner;
bool tree_sitter_properties_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
bool tree_sitter_properties_external_scanner_scan(void *payload, TSLexer *lexer,
const bool *valid_symbols) {
Scanner *scanner = (Scanner *)payload;
lexer->result_symbol = FAKE_EOL;
return reached_eof = !reached_eof && valid_symbols[FAKE_EOL] && lexer->eof(lexer);
return scanner->reached_eof =
!scanner->reached_eof && valid_symbols[FAKE_EOL] && lexer->eof(lexer);
}
/* Wire format unchanged: the state IS the returned length (1 = already emitted,
* 0 = not), and no bytes are written to the buffer. */
unsigned tree_sitter_properties_external_scanner_serialize(void *payload, char *buffer) {
return reached_eof;
return ((Scanner *)payload)->reached_eof;
}
void tree_sitter_properties_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
reached_eof = length;
void tree_sitter_properties_external_scanner_deserialize(void *payload, const char *buffer,
unsigned length) {
((Scanner *)payload)->reached_eof = length;
}
void *tree_sitter_properties_external_scanner_create() { return NULL; }
void *tree_sitter_properties_external_scanner_create() { return calloc(1, sizeof(Scanner)); }
void tree_sitter_properties_external_scanner_destroy(void *payload) { }
void tree_sitter_properties_external_scanner_destroy(void *payload) { free(payload); }
pkg/chocolatey/codebase-memory-mcp.nuspec
+2 -2
View File
@@ -2,7 +2,7 @@
<package xmlns="http://schemas.microsoft.com/packaging/2015/06/nuspec.xsd">
<metadata>
<id>codebase-memory-mcp</id>
<version>0.10.8</version>
<version>0.11.0</version>
<title>codebase-memory-mcp</title>
<authors>DeusData</authors>
<projectUrl>https://github.com/DeusData/codebase-memory-mcp</projectUrl>
@@ -28,7 +28,7 @@ After installation, run:
to configure your coding agents automatically.
</description>
<releaseNotes>https://github.com/DeusData/codebase-memory-mcp/releases/tag/v0.10.8</releaseNotes>
<releaseNotes>https://github.com/DeusData/codebase-memory-mcp/releases/tag/v0.11.0</releaseNotes>
</metadata>
<files>
<file src="tools\**" target="tools" />
pkg/chocolatey/tools/chocolateyInstall.ps1
+2 -2
View File
@@ -1,9 +1,9 @@
$ErrorActionPreference = 'Stop'
$packageName = 'codebase-memory-mcp'
$version = '0.10.8'
$version = '0.11.0'
$url64 = "https://github.com/DeusData/codebase-memory-mcp/releases/download/v${version}/codebase-memory-mcp-windows-amd64.zip"
$checksum64 = 'b43ad982994c4d829670749e08d3b622a74bb20041fc0a7d02bef6113f81c34d'
$checksum64 = '6eb6beaf261b19e419766e78baf93cbc3cf1c6338cff8fb7c0234859f96d1685'
$installDir = Join-Path $env:ChocolateyBinRoot $packageName
Install-ChocolateyZipPackage `
pkg/go/cmd/codebase-memory-mcp/main.go
+1 -1
View File
@@ -36,7 +36,7 @@ import (
const (
repo = "DeusData/codebase-memory-mcp"
version = "0.10.8"
version = "0.11.0"
windowsBinaryName = "codebase-memory-mcp.exe"
maxRedirects = 5
pkg/npm/README.md
+2 -2
View File
@@ -7,7 +7,7 @@
**The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. This npm wrapper downloads, verifies, and caches the selected native runtime set: the executable, its authenticated integration asset, and—when requested—the content-addressed UI pack.
High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 162 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 45 automatic/conditional client surfaces.
High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 162 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 17 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 45 automatic/conditional client surfaces.
## Installation
@@ -38,7 +38,7 @@ Restart your agent. Say **"Index this project"** — done.
- **Lifecycle hooks stay conservative** — Kimi uses `UserPromptSubmit`; on macOS/Linux, GitLab Duo gets a fail-open user `SessionStart`, while Devin gets `UserPromptSubmit`, `PostCompaction`, and a deduplicated `SessionStart` when Claude does not already provide it. Qoder, GitLab Duo, Devin, and Factory hooks are withheld on Windows without a documented shell/executor contract. Cline's auto-activating file hooks are withheld because their context output is not reliably consumed, CodeBuddy beta hooks are not auto-installed, and Cursor context hooks remain withheld.
- **Subagent access is explicit** — Claude, Gemini, Kiro, Qwen, CodeBuddy, KiloCode, Mistral Vibe, Grok Build, Qoder, Junie, and Factory get documented graph profiles with the narrowest tool/server filters their schemas support. KiloCode and Vibe enumerate read-only query tools rather than using server wildcards. Cursor, Rovo, Pochi, and Cline use explicit parent handoff where child MCP is unavailable or unsafe; IBM Bob receives no invented hook or agent.
- **Manual, UI, cloud, or repository-managed (not counted)** — Qodo, Warp MCP, JetBrains AI/ACP, GitHub Copilot coding agent, Jules, CodeRabbit, Replit, BLACKBOX AI, Plandex, and SWE-agent. Warp is counted above for its detected skill installation; its MCP connection remains manual.
- **15 MCP tools** — search, trace, architecture, impact analysis, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more.
- **17 MCP tools** — search, trace, architecture, impact analysis, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more.
## Supported Platforms
pkg/npm/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "codebase-memory-mcp",
"version": "0.10.8",
"version": "0.11.0",
"description": "Fast code intelligence for AI coding agents — installs a verified native runtime set",
"mcpName": "io.github.DeusData/codebase-memory-mcp",
"license": "MIT",
pkg/pypi/pyproject.toml
+1 -1
View File
@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
[project]
name = "codebase-memory-mcp"
version = "0.10.8"
version = "0.11.0"
description = "Fast code intelligence for AI coding agents — installs a verified native runtime set"
readme = "README.md"
license = { text = "MIT" }
pkg/pypi/src/codebase_memory_mcp/_cli.py
+1 -1
View File
@@ -338,7 +338,7 @@ def _version() -> str:
from importlib.metadata import version
return version("codebase-memory-mcp")
except Exception:
return "0.10.8"
return "0.11.0"
def _os_name() -> str:
scripts/fuzz.sh
Executable
+84
View File
@@ -0,0 +1,84 @@
#!/usr/bin/env bash
# fuzz.sh — run the libFuzzer targets of build/fuzz/cbm-fuzz.
#
# Usage:
# scripts/fuzz.sh <extract|cypher|config|all> [--runs N] [--max-total-time S]
# [--jobs N] [--seed N] [--no-build]
#
# Two shapes, one driver:
# --runs N --seed S a fixed number of executions from a fixed seed: the same
# inputs every time, so a smoke in CI is a function of the
# code (O9). This is the default (--runs 20000 --seed 1).
# --max-total-time S exploration for dry-run and nightly: new inputs are the
# point, and a crash is saved with its reproducer.
#
# The checked-in seeds (tests/fuzz/corpus/<target>/) are copied into a working
# corpus under build/fuzz/, so a run never rewrites the repository. A crash,
# leak or timeout leaves its input in build/fuzz/artifacts-<target>/ and the
# script exits non-zero; reproduce with
# CBM_FUZZ_TARGET=<target> build/fuzz/cbm-fuzz <artifact>
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
# --help before the positional target: the venue-parity interface probe calls
# every canonical entry as `<entry> --help` with no other argument.
case "${1:-}" in
-h | --help) sed -n '2,21p' "$0"; exit 0 ;;
esac
TARGET="${1:-}"
[ $# -gt 0 ] && shift
RUNS=20000
SEED=1
MAX_TIME=""
JOBS=1
BUILD=1
while [ $# -gt 0 ]; do
case "$1" in
--runs) RUNS="$2"; shift 2 ;;
--seed) SEED="$2"; shift 2 ;;
--max-total-time) MAX_TIME="$2"; RUNS=""; shift 2 ;;
--jobs) JOBS="$2"; shift 2 ;;
--no-build) BUILD=0; shift ;;
-h | --help) sed -n '2,21p' "$0"; exit 0 ;;
*) echo "fuzz.sh: unknown argument $1. Please consult --help." >&2; exit 2 ;;
esac
done
case "$TARGET" in
extract | cypher | config) TARGETS=("$TARGET") ;;
all) TARGETS=(extract cypher config) ;;
*) echo "fuzz.sh: target must be extract, cypher, config or all. Please consult --help." >&2
exit 2 ;;
esac
BIN="$ROOT/build/fuzz/cbm-fuzz"
if [ "$BUILD" -eq 1 ]; then
make -C "$ROOT" -f Makefile.cbm -j"$(getconf _NPROCESSORS_ONLN)" fuzz >/dev/null
fi
if [ ! -x "$BIN" ]; then
echo "fuzz.sh: $BIN is missing (build failed, or --no-build without a build)" >&2
exit 2
fi
status=0
for t in "${TARGETS[@]}"; do
corpus="$ROOT/build/fuzz/corpus-$t"
artifacts="$ROOT/build/fuzz/artifacts-$t/"
mkdir -p "$corpus" "$artifacts"
cp -R "$ROOT/tests/fuzz/corpus/$t/." "$corpus/"
args=(-artifact_prefix="$artifacts" -seed="$SEED" -rss_limit_mb=4096 -timeout=30 -print_final_stats=1)
if [ -n "$RUNS" ]; then
args+=(-runs="$RUNS")
else
args+=(-max_total_time="$MAX_TIME")
fi
if [ "$JOBS" -gt 1 ]; then
args+=(-jobs="$JOBS" -workers="$JOBS")
fi
echo "=== fuzz $t: ${RUNS:+$RUNS runs}${MAX_TIME:+$MAX_TIME s} (seed $SEED) ==="
if ! CBM_FUZZ_TARGET="$t" "$BIN" "${args[@]}" "$corpus"; then
echo "fuzz.sh: target $t FAILED -- reproducer in $artifacts" >&2
status=1
fi
done
exit "$status"
scripts/hooks/pre-commit
+10 -1
View File
@@ -17,4 +17,13 @@ echo "pre-commit: security audit (source-level)..."
scripts/security-audit.sh
echo "pre-commit: building and running tests..."
make -j$(sysctl -n hw.ncpu 2>/dev/null || nproc) -f Makefile.cbm test
(
# Git exports repository-local variables to hooks. The test suite creates
# independent repositories and linked worktrees, so keep the commit's
# index and object paths out of those fixtures.
while IFS= read -r variable; do
unset "$variable"
done < <(git rev-parse --local-env-vars)
make -j"$(sysctl -n hw.ncpu 2>/dev/null || nproc)" -f Makefile.cbm test
)
scripts/lint-memory-core.py
+228
View File
@@ -0,0 +1,228 @@
#!/usr/bin/env python3
"""
lint-memory-core.py — the memory-core linter.
Memory in this project is allocated through ONE core (src/foundation/mem_core.h),
not through scattered raw malloc/calloc/realloc/free/strdup. This linter is the
gate that keeps that true.
WHY A RATCHET, NOT A BAN. An audit on 2026-09-13 found ~800 raw allocation
sites in src/ and none of them can be migrated in one change. So the gate works
like every other honest debt gate: a checked-in baseline records how many raw
sites each file has TODAY, and the build goes red the moment any file has MORE
than its baseline, or a file not in the baseline grows one. Files can only ever
go down. When a file is migrated, its baseline line is lowered (or removed) in
the same change -- and --strict turns "below baseline" into a failure too, so
the baseline cannot silently rot behind the code.
WHAT COUNTS AS RAW. A call to malloc/calloc/realloc/free/strdup/strndup that is
not part of a longer identifier. cbm_alloc, cbm_free, cbm_calloc, mi_malloc,
cbm_arena_alloc and heap_strdup are all NOT matches: the character before the
name is an identifier character. Comments and string literals are stripped
first, so prose that mentions malloc( does not trip the gate -- the security
audit already bit us once on exactly that with fork(.
EXEMPT. The core itself and the allocator plumbing it sits on:
src/foundation/mem_core.c the route
(arena.c allocates its blocks THROUGH the core -- class arena -- and is scanned)
src/foundation/slab_alloc.c same
src/foundation/mem.c policy/measurement, probes with malloc
src/foundation/mem_override_*.c the --wrap / override shims
src/foundation/compat*.c libc replacement surface
internal/**/vendored/** not ours
vendored/** not ours
SCOPE. Everything under src/ and internal/ -- cli, mcp, daemon, store,
pipeline, the extraction engine -- so the count is for the whole project, not
one subsystem, and a leak shows up wherever it is.
Usage:
lint-memory-core.py check against the baseline (CI)
lint-memory-core.py --strict also fail when a file is BELOW baseline
lint-memory-core.py --write-baseline regenerate scripts/memory-core-baseline.txt
lint-memory-core.py --list print every raw site (file:line: call)
Exit 0 = clean. Exit 1 = a file grew. Exit 2 = usage/IO error.
"""
from __future__ import annotations
import argparse
import pathlib
import re
import sys
ROOT = pathlib.Path(__file__).resolve().parent.parent
BASELINE = ROOT / "scripts" / "memory-core-baseline.txt"
SCAN_ROOTS = ("src", "internal")
RAW = re.compile(r"(?<![A-Za-z0-9_])(malloc|calloc|realloc|free|strdup|strndup)\s*\(")
EXEMPT_EXACT = {
"src/foundation/mem_core.c",
"src/foundation/mem.c",
}
EXEMPT_PREFIX = (
"src/foundation/mem_override_",
"src/foundation/compat",
"vendored/",
)
def is_vendored(rel: str) -> bool:
"""Any vendored/ segment anywhere under internal/ is not ours."""
return "/vendored/" in rel or rel.startswith("vendored/")
def is_exempt(rel: str) -> bool:
return rel in EXEMPT_EXACT or rel.startswith(EXEMPT_PREFIX) or is_vendored(rel)
def strip_comments_and_strings(text: str) -> str:
"""Blank out comments and string literals, preserving line structure so
reported line numbers stay right. Character-class aware enough for C:
handles escapes inside strings and does not treat // inside a string as a
comment."""
out = []
i, n = 0, len(text)
while i < n:
c = text[i]
nxt = text[i + 1] if i + 1 < n else ""
if c == "/" and nxt == "*":
j = text.find("*/", i + 2)
j = n if j < 0 else j + 2
out.append("".join(ch if ch == "\n" else " " for ch in text[i:j]))
i = j
elif c == "/" and nxt == "/":
j = text.find("\n", i)
j = n if j < 0 else j
out.append(" " * (j - i))
i = j
elif c == '"' or c == "'":
q = c
j = i + 1
while j < n and text[j] != q:
if text[j] == "\\":
j += 1
if j < n and text[j] == "\n":
break
j += 1
j = min(j + 1, n)
out.append(q + " " * max(0, j - i - 2) + (q if j - i >= 2 else ""))
i = j
else:
out.append(c)
i += 1
return "".join(out)
def scan() -> dict[str, list[tuple[int, str]]]:
hits: dict[str, list[tuple[int, str]]] = {}
for root in SCAN_ROOTS:
base = ROOT / root
if not base.exists():
continue
for path in sorted(base.rglob("*")):
if path.suffix not in (".c", ".h") or not path.is_file():
continue
rel = path.relative_to(ROOT).as_posix()
if is_exempt(rel):
continue
text = strip_comments_and_strings(path.read_text(encoding="utf-8", errors="replace"))
for lineno, line in enumerate(text.splitlines(), 1):
for m in RAW.finditer(line):
hits.setdefault(rel, []).append((lineno, m.group(1)))
return hits
def read_baseline() -> dict[str, int]:
if not BASELINE.exists():
return {}
out: dict[str, int] = {}
for raw in BASELINE.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
rel, _, count = line.rpartition("\t")
if not rel:
continue
try:
out[rel] = int(count)
except ValueError:
continue
return out
def write_baseline(hits: dict[str, list]) -> None:
lines = [
"# memory-core-baseline.txt -- raw allocator call sites per file.",
"# Generated by scripts/lint-memory-core.py --write-baseline.",
"# A file may only ever go DOWN. Lower a line in the same change that",
"# migrates the file to src/foundation/mem_core.h; never raise one.",
"",
]
for rel in sorted(hits):
lines.append(f"{rel}\t{len(hits[rel])}")
BASELINE.write_text("\n".join(lines) + "\n", encoding="utf-8")
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--write-baseline", action="store_true")
ap.add_argument("--strict", action="store_true", help="fail when a file is BELOW its baseline")
ap.add_argument("--list", action="store_true", help="print every raw site")
args = ap.parse_args()
hits = scan()
total = sum(len(v) for v in hits.values())
if args.write_baseline:
write_baseline(hits)
print(f"memory-core baseline written: {len(hits)} files, {total} raw sites")
return 0
if args.list:
for rel in sorted(hits):
for lineno, call in hits[rel]:
print(f"{rel}:{lineno}: {call}(")
print(f"-- {total} raw sites in {len(hits)} files")
return 0
baseline = read_baseline()
if not baseline:
print(f"ERROR: no baseline at {BASELINE.relative_to(ROOT)}; run --write-baseline", file=sys.stderr)
return 2
grew: list[str] = []
shrank: list[str] = []
for rel, sites in sorted(hits.items()):
now = len(sites)
allowed = baseline.get(rel, 0)
if now > allowed:
# Show the LAST sites: new code is usually appended, so these are the
# likeliest culprits. The linter cannot know which sites are new
# without a diff; --list prints them all.
delta = now - allowed
tail = ", ".join(f"{ln}:{c}" for ln, c in sites[-max(delta, 1):][-6:])
grew.append(f" {rel}: grew by {delta} ({allowed} -> {now}); latest sites: {tail}"
f" [run --list for all]")
elif now < allowed:
shrank.append(f" {rel}: {now} (baseline {allowed}) -- lower the baseline")
for rel, allowed in sorted(baseline.items()):
if rel not in hits and allowed > 0:
shrank.append(f" {rel}: 0 (baseline {allowed}) -- remove the line")
if grew:
print("memory-core linter FAILED: raw allocator use grew. Allocate through")
print("src/foundation/mem_core.h (cbm_alloc/cbm_calloc/cbm_realloc/cbm_free/")
print("cbm_mem_strdup) instead of malloc/calloc/realloc/free/strdup.")
print("\n".join(grew))
if shrank:
print("memory-core ratchet: these files improved; tighten scripts/memory-core-baseline.txt:")
print("\n".join(shrank))
if grew or (args.strict and shrank):
return 1
print(f"memory-core linter passed: {total} raw sites across {len(hits)} files, none grew")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/memlab-drive.py
+20 -4
View File
@@ -32,6 +32,16 @@ def rpc(proc, payload, timeout_note):
return message
def tool_failed(reply):
"""A JSON-RPC error, or a tool result the server marked isError: MCP reports
a failed tool call as a normal result, so checking only "error" counts every
failure as served (a lab measuring nothing looked healthy)."""
if "error" in reply:
return True
result = reply.get("result")
return isinstance(result, dict) and bool(result.get("isError"))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("binary")
@@ -63,14 +73,16 @@ def main():
"params": {"protocolVersion": "2024-11-05", "capabilities": {},
"clientInfo": {"name": "memlab", "version": "1"}}},
"initialize")
project = None
if not args.skip_index:
indexed = rpc(proc, {"jsonrpc": "2.0", "id": 1, "method": "tools/call",
"params": {"name": "index_repository",
"arguments": {"path": args.corpus}}},
"arguments": {"repo_path": args.corpus}}},
"index_repository")
if "error" in indexed:
print(f"index failed: {indexed['error']}", file=sys.stderr)
if tool_failed(indexed):
print(f"index failed: {indexed.get('error') or indexed.get('result')}", file=sys.stderr)
return 3
project = (indexed.get("result", {}).get("structuredContent") or {}).get("project")
halfway = 2 + args.requests // 2
for i in range(2, args.requests + 2):
@@ -84,11 +96,15 @@ def main():
"list_projects": {},
"get_graph_schema": {},
"search_code": {"pattern": "Widget"}}.get(args.tool, {})
if project and args.tool != "list_projects":
arguments = dict(arguments, project=project)
reply = rpc(proc, {"jsonrpc": "2.0", "id": i, "method": "tools/call",
"params": {"name": args.tool, "arguments": arguments}},
f"request {i}")
served += 1
if "error" in reply:
if tool_failed(reply):
if failures == 0:
print(f"first failure (request {i}): {json.dumps(reply)[:600]}", file=sys.stderr, flush=True)
failures += 1
finally:
try:
scripts/memlab-report.py
+61 -34
View File
@@ -1,19 +1,20 @@
#!/usr/bin/env python3
"""Analyse #581 memory-lab output: attribute growth to allocation sites.
Reads the JSONL the profiler appends (CBM_MEM_PROFILE_OUT) and the mem.census
lines from a daemon log, and prints:
Reads the waste layer's dumps (CBM_MEMWASTE_OUT, one at every request-stage phase
mark; see scripts/memlab.sh) and the mem.census lines from a daemon log, and
prints:
* retention per request, as a linear fit rather than first-vs-last, so noise
does not masquerade as a trend
* the call sites whose live bytes grow across the run, ranked by growth
* an explicit unattributed remainder: process growth the profiler could not
account for. That number is the honesty check — a report that attributed
everything while tracking only >=32 KiB allocations would be lying.
* an explicit unattributed remainder: process growth the layer could not
account for (allocator metadata, fragmentation, memory outside malloc).
That number is the honesty check.
Usage:
memlab-report.py <profile.jsonl> [--census <daemon.log>] [--top N]
memlab-report.py <a.jsonl> --diff <b.jsonl> # platform comparison
memlab-report.py <dump.jsonl> [--census <daemon.log>] [--binary BIN] [--top N]
memlab-report.py <a.jsonl> --diff <b.jsonl> [--binary BIN] # platform comparison
"""
import argparse
import json
@@ -23,9 +24,11 @@ from collections import defaultdict
def load_profile(path):
"""Return (summaries, site_series). Records are appended over time, so the
Nth occurrence of a site is its Nth sample."""
summaries, series = [], defaultdict(list)
"""Return (summaries, site_series) for the process with the most dumps.
Every dump lists every site, so the Nth occurrence of a site is its Nth
sample; a site first seen late is padded with zeros so series align."""
procs = defaultdict(lambda: {"summaries": [], "series": defaultdict(list)})
cur = None
with open(path, "r", errors="replace") as handle:
for line in handle:
line = line.strip()
@@ -35,11 +38,31 @@ def load_profile(path):
record = json.loads(line)
except json.JSONDecodeError:
continue
if "site" in record:
series[record["site"]].append(record)
else:
summaries.append(record)
return summaries, series
if "memwaste" in record:
cur = procs[record.get("pid", 0)]
cur["summaries"].append(record)
elif cur is not None and "site" in record and "work" not in record:
samples = cur["series"][record["site"]]
while len(samples) < len(cur["summaries"]) - 1:
samples.append({"live_bytes": 0})
samples.append(record)
if not procs:
return [], {}
best = max(procs.values(), key=lambda p: len(p["summaries"]))
return best["summaries"], best["series"]
def load_symbols(binary, summaries, sites):
"""Symbolise site addresses with the waste report's own symboliser."""
if not binary or not summaries:
return {}
import importlib.util
import os
spec = importlib.util.spec_from_file_location(
"memwaste_report", os.path.join(os.path.dirname(os.path.abspath(__file__)), "memwaste-report.py"))
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod.symbolise(binary, summaries[-1].get("image_base", "0x0"), list(sites))
def linear_slope(values):
@@ -57,7 +80,7 @@ def linear_slope(values):
def parse_census(path):
"""Pull the pool totals out of mem.census log lines."""
fields = ("priv_kb", "rss_kb", "crt_kb", "biggest_kb", "prof_live_kb")
fields = ("priv_kb", "rss_kb", "crt_kb", "biggest_kb")
rows = []
pattern = re.compile(r"(\w+)=(\S+)")
with open(path, "r", errors="replace") as handle:
@@ -71,16 +94,16 @@ def parse_census(path):
return rows
def report(path, census_path, top):
def report(path, census_path, top, binary=None):
summaries, series = load_profile(path)
if not summaries and not series:
print(f"{path}: no profiler records — was CBM_MEM_PROFILE=1 set?")
print(f"{path}: no waste-layer records -- was the memwaste flavour run with CBM_MEMWASTE=1?")
return
names = load_symbols(binary, summaries, series.keys())
last = summaries[-1] if summaries else {}
print(f"=== {path} ===")
if last:
print(f"threshold : {last.get('threshold', 0)} bytes "
f"(allocations below this are invisible by design)")
print(f"samples : {len(summaries)} dumps (every allocation size is tracked)")
print(f"sites seen : {last.get('sites', 0)}")
print(f"attributed live: {last.get('live_bytes', 0)/1048576:.1f} MB "
f"in {last.get('live_blocks', 0)} blocks")
@@ -94,23 +117,23 @@ def report(path, census_path, top):
slope = linear_slope(live)
if slope <= 0 and live[-1] <= live[0]:
continue
growth.append((slope, live[-1] - live[0], live[-1], samples[-1]))
growth.append((slope, live[-1] - live[0], live[-1], names.get(site, site)))
growth.sort(key=lambda row: row[0], reverse=True)
print(f"\n--- sites whose retained bytes GROW (top {top}) ---")
if not growth:
print("none — no tracked site retains more over time.")
for slope, delta, live, sample in growth[:top]:
frames = " <- ".join(sample.get("frames", [])[:4])
for slope, delta, live, name in growth[:top]:
print(f" +{slope/1024:8.1f} KB/sample total +{delta/1048576:6.2f} MB "
f"live {live/1048576:6.2f} MB")
print(f" {frames}")
print(f" {name}")
if census_path:
rows = parse_census(census_path)
if rows:
priv = [r["priv_kb"] for r in rows if "priv_kb" in r]
prof = [r["prof_live_kb"] for r in rows if "prof_live_kb" in r]
# attributed = what the waste layer holds live, sampled at its own dumps
prof = [int(h.get("live_bytes", 0)) // 1024 for h in summaries]
print(f"\n--- pools vs attribution ({len(rows)} samples) ---")
if priv:
print(f" private committed : {priv[0]/1024:.1f} -> {priv[-1]/1024:.1f} MB "
@@ -119,24 +142,27 @@ def report(path, census_path, top):
unattributed = (priv[-1] - priv[0]) - (prof[-1] - prof[0])
print(f" attributed growth : {(prof[-1]-prof[0])/1024:+.1f} MB")
print(f" UNATTRIBUTED : {unattributed/1024:+.1f} MB "
"<- below-threshold allocations, allocator metadata, or fragmentation")
"<- allocator metadata, fragmentation, or memory outside malloc (mmap, stacks)")
def diff(path_a, path_b, top):
"""Same sites, two platforms: what does one retain that the other does not?"""
_, series_a = load_profile(path_a)
def diff(path_a, path_b, top, binary=None):
"""Same sites, two platforms: what does one retain that the other does not?
Raw addresses differ between binaries, so sites are matched by symbol name
when --binary is given (the same binary for both runs) and by address otherwise."""
summaries_a, series_a = load_profile(path_a)
_, series_b = load_profile(path_b)
names = load_symbols(binary, summaries_a, set(series_a) | set(series_b))
rows = []
for site, samples in series_a.items():
a_live = samples[-1]["live_bytes"]
b_live = series_b[site][-1]["live_bytes"] if site in series_b else 0
if a_live - b_live > 0:
rows.append((a_live - b_live, a_live, b_live, samples[-1]))
rows.append((a_live - b_live, a_live, b_live, site))
rows.sort(key=lambda row: row[0], reverse=True)
print(f"=== {path_a} minus {path_b} (top {top}) ===")
for delta, a_live, b_live, sample in rows[:top]:
for delta, a_live, b_live, site in rows[:top]:
print(f" +{delta/1048576:7.2f} MB A={a_live/1048576:.2f} MB B={b_live/1048576:.2f} MB")
print(f" {' <- '.join(sample.get('frames', [])[:4])}")
print(f" {names.get(site, site)}")
def main():
@@ -145,11 +171,12 @@ def main():
parser.add_argument("--census")
parser.add_argument("--diff")
parser.add_argument("--top", type=int, default=12)
parser.add_argument("--binary", help="the memwaste binary, for symbol names")
args = parser.parse_args()
if args.diff:
diff(args.profile, args.diff, args.top)
diff(args.profile, args.diff, args.top, args.binary)
else:
report(args.profile, args.census, args.top)
report(args.profile, args.census, args.top, args.binary)
return 0
scripts/memlab.sh
+39 -12
View File
@@ -10,9 +10,14 @@
#
# Usage: scripts/memlab.sh <binary> [requests] [label]
#
# Output: memlab-<label>.jsonl (profiler records, one block per sample)
# <binary> must be the waste-sanitizer flavour (make -f Makefile.cbm cbm
# MEMWASTE=1 BUILD_DIR=build/memwaste): the attribution comes from its event
# layer, which observes every allocation on Linux, macOS and Windows alike.
#
# Output: memlab-<label>.jsonl (a waste-layer dump at every request-stage phase mark)
# memlab-<label>.log (daemon log with mem.census lines)
# Analyse with: scripts/memlab-report.py memlab-<label>.jsonl --census memlab-<label>.log
# Analyse with: scripts/memlab-report.py memlab-<label>.jsonl --census memlab-<label>.log \
# --binary <binary>
set -u
BINARY="${1:?usage: memlab.sh <binary> [requests] [label]}"
@@ -24,6 +29,16 @@ if [ ! -x "$BINARY" ]; then
exit 2
fi
# The profiled process must reach a daemon rendezvous and cache this run owns:
# only CBM_RUNTIME_DIR moves the rendezvous, so a private CBM_CACHE_DIR alone
# joined the operator's account daemon (#1691, #1696). The helper also stops
# that daemon before its cache is removed and leaves the root for diagnosis
# when it will not stop.
# shellcheck source=test-runtime.sh
source "$(dirname "$0")/test-runtime.sh"
cbm_test_runtime_init || exit 1
trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
WORK=$(mktemp -d 2>/dev/null || mktemp -d -t memlab)
# Native Windows: the server walks the full ancestor chain of both the binary
@@ -75,7 +90,12 @@ PROFILE_OUT="$PWD/memlab-${LABEL}.jsonl"
RUN_LOG="$PWD/memlab-${LABEL}.log"
rm -f "$PROFILE_OUT" "$RUN_LOG"
cleanup() { rm -rf "$WORK" 2>/dev/null || true; rm -rf "${WIN_ROOT:-}" 2>/dev/null || true; }
cleanup() {
# The helper probes with $BINARY, whose native-Windows copy lives in $WORK.
cbm_test_runtime_cleanup "$BINARY"
rm -rf "$WORK" 2>/dev/null || true
rm -rf "${WIN_ROOT:-}" 2>/dev/null || true
}
trap cleanup EXIT
# A fixed corpus: same file count and content on every platform, so a
@@ -108,15 +128,23 @@ done
echo "=== memlab: binary=$BINARY requests=$REQUESTS label=$LABEL ==="
echo "corpus: $(find "$CORPUS" -name '*.py' | wc -l | tr -d ' ') files"
# The Windows binary needs a native path here; an msys /c/... path is not one.
if command -v cygpath >/dev/null 2>&1 && ! command -v winepath >/dev/null 2>&1; then
# The native Windows binary needs a native path here, under its stamped root;
# an msys /c/... path is not one. Everywhere else the helper's owner-only cache
# (already exported as CBM_CACHE_DIR) is the cache.
if [ -n "${WIN_ROOT:-}" ]; then
mkdir -p "$WORK/cache"
export CBM_CACHE_DIR="$(cygpath -w "$WORK/cache")"
CACHE_HOST="$WORK/cache"
else
export CBM_CACHE_DIR="$WORK/cache"
CACHE_HOST="$CBM_TEST_CACHE_DIR_HOST"
fi
export CBM_MEM_PROFILE=1
export CBM_MEM_PROFILE_OUT="$PROFILE_OUT"
export CBM_MEMWASTE=1
export CBM_MEMWASTE_OUT="$PROFILE_OUT"
# The server marks a memory phase at every request stage, and the layer dumps
# at each mark: live bytes per site become a series over identical requests. The
# series needs neither the fill scan nor the duplicate hash, which only cost time.
export CBM_MEMWASTE_FILL=0
export CBM_MEMWASTE_DUPES=0
export CBM_MEM_CENSUS=1
export CBM_LOG_LEVEL=info
export CBM_LOG_FORMAT=text
@@ -139,7 +167,7 @@ python3 "$(dirname "$0")/memlab-drive.py" "$DRIVE_BINARY" "$DRIVE_CORPUS" "$REQU
RC=$?
# With CBM_CACHE_DIR set the process logs to its own file rather than stderr,
# so fold that in or the census series is invisible.
cat "$WORK"/cache/logs/*.log >> "$RUN_LOG" 2>/dev/null || true
cat "$CACHE_HOST"/logs/*.log >> "$RUN_LOG" 2>/dev/null || true
cat "$WORK/server-stderr.log" >> "$RUN_LOG" 2>/dev/null || true
RESPONSES=$(sed -n "s/.*served=\\([0-9]*\\).*/\\1/p" "$WORK/drive.out" | head -1); RESPONSES=${RESPONSES:-0}
CENSUS=$(grep -c "mem.census" "$RUN_LOG" 2>/dev/null | head -1); CENSUS=${CENSUS:-0}
@@ -155,9 +183,8 @@ if [ "$CENSUS" -eq 0 ]; then
echo "WARN: no census samples — check CBM_MEM_CENSUS wiring" >&2
fi
if [ "$SITES" -eq 0 ]; then
# Not fatal, but never silent: on macOS there is no --wrap, so the
# profiler legitimately has no observation point.
echo "WARN: no profiler records — expected on macOS (no --wrap); a gap anywhere else" >&2
# Never silent: an empty dump means the binary is not the memwaste flavour.
echo "WARN: no waste-layer records -- is $BINARY built with MEMWASTE=1?" >&2
fi
echo "profile: $PROFILE_OUT"
echo "log: $RUN_LOG"
scripts/memory-core-baseline.txt
+90
View File
@@ -0,0 +1,90 @@
# memory-core-baseline.txt -- raw allocator call sites per file.
# Generated by scripts/lint-memory-core.py --write-baseline.
# A file may only ever go DOWN. Lower a line in the same change that
# migrates the file to src/foundation/mem_core.h; never raise one.
internal/cbm/ac.c 20
internal/cbm/arena.c 3
internal/cbm/cbm.c 13
internal/cbm/extract_defs.c 5
internal/cbm/helpers.c 3
internal/cbm/lsp/c_lsp.c 4
internal/cbm/lsp/perl_lsp.c 21
internal/cbm/lsp/py_lsp.c 2
internal/cbm/macro_table.c 1
internal/cbm/service_patterns.c 2
internal/cbm/sqlite_writer.c 74
src/cli/activation_transaction.c 80
src/cli/agent_clients.c 73
src/cli/agent_profiles.c 7
src/cli/cli.c 173
src/cli/client_adapter.c 3
src/cli/config_edit_path.c 2
src/cli/config_json_like.c 142
src/cli/config_text_edit.c 72
src/cli/config_toml_edit.c 85
src/cli/config_yaml_edit.c 117
src/cli/hook_augment.c 31
src/cypher/cypher.c 132
src/daemon/application.c 107
src/daemon/bootstrap.c 13
src/daemon/daemon.c 31
src/daemon/frontend.c 12
src/daemon/ipc.c 146
src/daemon/project_lock.c 6
src/daemon/runtime.c 45
src/daemon/service.c 6
src/daemon/version_cohort.c 9
src/discover/discover.c 31
src/discover/gitignore.c 12
src/discover/userconfig.c 15
src/foundation/diagnostics.c 7
src/foundation/lock_registry.c 18
src/foundation/platform.c 11
src/foundation/platform.h 5
src/foundation/private_file_lock.c 56
src/foundation/str_intern.c 7
src/foundation/subprocess.c 24
src/foundation/win_utf8.h 17
src/foundation/workspace.c 3
src/foundation/yaml.c 10
src/git/git_context.c 18
src/main.c 32
src/mcp/compact_out.c 39
src/mcp/index_supervisor.c 8
src/mcp/mcp.c 784
src/pipeline/artifact.c 38
src/pipeline/fqn.c 23
src/pipeline/lsp_resolve.h 4
src/pipeline/lsp_surface.c 5
src/pipeline/pass_calls.c 23
src/pipeline/pass_compile_commands.c 17
src/pipeline/pass_complexity.c 15
src/pipeline/pass_configlink.c 3
src/pipeline/pass_cross_repo.c 8
src/pipeline/pass_definitions.c 14
src/pipeline/pass_enrichment.c 15
src/pipeline/pass_ensemble_routing.c 10
src/pipeline/pass_envscan.c 2
src/pipeline/pass_githistory.c 27
src/pipeline/pass_importance.c 4
src/pipeline/pass_infrascan.c 2
src/pipeline/pass_k8s.c 6
src/pipeline/pass_lsp_cross.c 50
src/pipeline/pass_parallel.c 31
src/pipeline/pass_pkgmap.c 82
src/pipeline/pass_semantic.c 3
src/pipeline/pass_tests.c 3
src/pipeline/pass_usages.c 4
src/pipeline/path_alias.c 42
src/pipeline/pipeline.c 134
src/pipeline/pipeline_delta.c 19
src/pipeline/pipeline_incremental.c 144
src/pipeline/registry.c 17
src/pipeline/worker_pool.c 2
src/store/store.c 358
src/ui/config.c 12
src/ui/http_server.c 31
src/ui/httpd.c 21
src/ui/layout3d.c 66
src/watcher/watcher.c 33
scripts/memwaste-padding.py
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
"""
memwaste-padding.py — the layout lane of the waste sanitizer: bytes every
instance of a struct carries for alignment alone.
Padding is waste multiplied by the instance count: 4 bytes in a record the graph
buffer holds 15 million of is 57 MB. The compiler knows every hole; this lane
asks it, for EVERY translation unit of the production build -- vendored code
included (grammars, the tree-sitter runtime, SQLite, mimalloc, tre, lz4, zstd,
the C++ preprocessor) -- by replaying the build's own compile commands with
`-fsyntax-only -Wno-everything -Wpadded`.
Per struct it reports the interior holes (removable by ordering fields from the
largest alignment down), the tail padding (removable only when the interior
reordering frees enough bytes), and how many allocation sites name the type in a
sizeof, as a first hint of instance counts. The event lane's per-site requested
sizes are the runtime evidence to pair it with.
Usage:
scripts/memwaste-padding.py # every TU, top 40 by interior padding
scripts/memwaste-padding.py --project-only # skip vendored TUs
scripts/memwaste-padding.py --json
"""
from __future__ import annotations
import argparse
import concurrent.futures as cf
import json
import os
import re
import shlex
import subprocess
import sys
from collections import defaultdict
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SRC_EXT = (".c", ".cc", ".cpp")
KEEP_WITH_ARG = {"-include", "-isystem", "-iquote", "-arch", "-target"}
WARN_RE = re.compile(
r"^(?P<file>[^:]+):(?P<line>\d+):\d+: warning: padding (?:struct|class|size of) "
r"'(?P<type>[^']+)' with (?P<bytes>\d+) bytes? to (?P<what>align '(?P<field>[^']*)'|alignment boundary)")
def build_commands(cc: str, cxx: str) -> list[tuple[str, list[str], str]]:
dry = subprocess.run(["make", "-f", "Makefile.cbm", "-n", "cbm", "BUILD_DIR=build/memwaste-padding-dry",
f"CC={cc}", f"CXX={cxx}", "MEMWASTE=1"],
cwd=ROOT, capture_output=True, text=True, check=False)
text = dry.stdout.replace("\\\n", " ")
jobs = []
seen = set()
for line in text.splitlines():
try:
argv = shlex.split(line)
except ValueError:
continue
if not argv or os.path.basename(argv[0]) not in (cc, cxx, os.path.basename(cc), os.path.basename(cxx)):
continue
flags = []
sources = []
i = 1
while i < len(argv):
a = argv[i]
if a in KEEP_WITH_ARG and i + 1 < len(argv):
flags += [a, argv[i + 1]]
i += 2
continue
if a == "-o":
i += 2
continue
if a.startswith(("-I", "-D", "-U", "-std=")):
flags.append(a)
elif a.endswith(SRC_EXT) and os.path.isfile(os.path.join(ROOT, a)):
sources.append(a)
i += 1
for s in sources:
key = (s, tuple(flags))
if key not in seen:
seen.add(key)
jobs.append((argv[0], flags, s))
return jobs
def run_one(job: tuple[str, list[str], str]) -> tuple[str, str]:
compiler, flags, source = job
res = subprocess.run([compiler, "-fsyntax-only", "-Wno-everything", "-Wpadded"] + flags + [source],
cwd=ROOT, capture_output=True, text=True, check=False)
return source, res.stderr
def is_vendored(path: str) -> bool:
return "/vendored/" in f"/{path}" or path.startswith("vendored/")
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--cc", default="clang")
ap.add_argument("--cxx", default="clang++")
ap.add_argument("--top", type=int, default=40)
ap.add_argument("--project-only", action="store_true")
ap.add_argument("--jobs", type=int, default=os.cpu_count() or 4)
ap.add_argument("--json", action="store_true")
args = ap.parse_args()
jobs = build_commands(args.cc, args.cxx)
if args.project_only:
jobs = [j for j in jobs if not is_vendored(j[2])]
if not jobs:
sys.exit("memwaste-padding: no compile commands found (make -n printed none for the compiler)")
holes: dict[str, dict] = {}
with cf.ThreadPoolExecutor(max_workers=args.jobs) as pool:
for source, stderr in pool.map(run_one, jobs):
for line in stderr.splitlines():
m = WARN_RE.match(line)
if not m:
continue
file = os.path.relpath(os.path.join(ROOT, m.group("file")), ROOT) \
if not os.path.isabs(m.group("file")) else os.path.relpath(m.group("file"), ROOT)
t = holes.setdefault(m.group("type"), {"type": m.group("type"), "interior": {}, "tail": 0,
"where": f"{file}:{m.group('line')}",
"vendored": is_vendored(file)})
n = int(m.group("bytes"))
if m.group("field") is not None:
t["interior"][m.group("field")] = n # a header seen by many TUs reports the same hole
else:
t["tail"] = n
t["where"] = f"{file}:{m.group('line')}"
# Allocation hint: how many allocation expressions size by this type.
sizeof_sites: dict[str, int] = defaultdict(int)
names = {t.split(" ", 1)[-1] for t in holes}
alloc_re = re.compile(r"(?:alloc|calloc|realloc|arena_alloc|CBM_DA_PUSH|cbm_da_push)\w*\s*\([^;]*sizeof\s*\(\s*"
r"(?:struct\s+)?(\w+)\s*\)")
for base in ("src", "internal/cbm"):
for dirpath, _, filenames in os.walk(os.path.join(ROOT, base)):
if "/vendored" in dirpath:
continue
for fn in filenames:
if not fn.endswith((".c", ".h")):
continue
try:
text = open(os.path.join(dirpath, fn), encoding="utf-8", errors="replace").read()
except OSError:
continue
for m in alloc_re.finditer(text):
if m.group(1) in names:
sizeof_sites[m.group(1)] += 1
rows = []
for t in holes.values():
interior = sum(t["interior"].values())
short = t["type"].split(" ", 1)[-1]
rows.append({"type": t["type"], "interior_bytes": interior, "tail_bytes": t["tail"],
"holes": len(t["interior"]), "alloc_sites": sizeof_sites.get(short, 0),
"where": t["where"], "vendored": t["vendored"]})
rows.sort(key=lambda r: (-(r["alloc_sites"] > 0), -r["interior_bytes"], r["type"]))
if args.json:
json.dump({"translation_units": len(jobs), "structs": rows}, sys.stdout, indent=1)
print()
return 0
print(f"translation units checked: {len(jobs)} structs with padding: {len(rows)} "
f"(vendored: {sum(1 for r in rows if r['vendored'])})")
print()
print(f" {'interior B':>10} {'tail B':>6} {'holes':>5} {'alloc sites':>11} struct (declared at)")
for r in rows[: args.top]:
tag = " [vendored]" if r["vendored"] else ""
print(f" {r['interior_bytes']:>10} {r['tail_bytes']:>6} {r['holes']:>5} {r['alloc_sites']:>11} "
f"{r['type']}{tag} ({r['where']})")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/memwaste-ratchet.py
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
"""
memwaste-ratchet.py — hold the waste numbers where they are, lower them as they fall.
Same shape as the memory-core linter's ratchet: a checked-in baseline per venue
(scripts/memwaste-baseline.json), every waste class normalised PER THOUSAND GRAPH
NODES so a corpus that grows does not look like a regression. A class that grows
past its baseline by more than --tolerance fails under --enforce; without it the
comparison is printed and the exit is 0 (report-only rollout). A class that
FALLS is reported so the baseline can be lowered with --record.
The inputs are what scripts/memwaste.sh leaves in its --out directory:
memwaste.json (the report's --json) and index.json (the index result, for the
node count).
Usage:
memwaste-ratchet.py --out DIR --venue pr-linux # compare, report-only
memwaste-ratchet.py --out DIR --venue pr-linux --enforce # compare, gate
memwaste-ratchet.py --out DIR --venue pr-linux --record # write the baseline
"""
from __future__ import annotations
import argparse
import json
import os
import sys
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BASELINE = os.path.join(ROOT, "scripts", "memwaste-baseline.json")
# Classes the ratchet holds. Left out on purpose: allocator-internal byte work
# (it moves with allocator settings), the access-lane classes (one lane only),
# and contended mutex acquisitions and pool imbalance -- those depend on thread
# scheduling, and a gate on them would be a lottery (O9). They stay in the report.
CLASSES = [
"slack_bytes", "never_written_bytes", "over_requested_bytes", "realloc_copy_bytes",
"short_lived", "dup_bytes_max", "retained_bytes_max", "raw_bytes",
"container_idle_peak_bytes", "byte_work_bytes", "byte_work_repeats", "io_tiny_calls",
"path_repeats", "ht_repeat_lookups", "ht_growths",
]
def current(out_dir: str) -> dict:
report = json.load(open(os.path.join(out_dir, "memwaste.json")))
index = json.load(open(os.path.join(out_dir, "index.json")))
nodes = int(index.get("nodes") or 0)
if nodes <= 0:
sys.exit("memwaste-ratchet: index.json has no node count")
# The index worker is the process that allocated the most.
worker = max(report["processes"].values(), key=lambda p: p["header"].get("usable_bytes", 0))
summary = worker["summary"]
return {
"nodes": nodes,
"per_1k_nodes": {c: round(summary.get(c, 0) * 1000.0 / nodes, 3) for c in CLASSES},
}
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--out", required=True, help="memwaste.sh --out directory")
ap.add_argument("--venue", required=True, help="baseline key, e.g. pr-linux, dryrun-macos")
ap.add_argument("--tolerance", type=float, default=0.05, help="allowed growth before a class fails")
ap.add_argument("--enforce", action="store_true")
ap.add_argument("--record", action="store_true")
args = ap.parse_args()
cur = current(args.out)
base_all = json.load(open(BASELINE)) if os.path.exists(BASELINE) else {}
if args.record:
base_all[args.venue] = cur
with open(BASELINE, "w") as fh:
json.dump(base_all, fh, indent=1, sort_keys=True)
fh.write("\n")
print(f"baseline recorded for {args.venue} ({cur['nodes']:,} nodes)")
return 0
base = base_all.get(args.venue)
if not base:
print(f"no baseline for {args.venue}: nothing to hold yet (record one with --record)")
return 0
worse, better = [], []
print(f"{'class':<28} {'baseline/1k':>14} {'now/1k':>14} {'delta':>8}")
for c in CLASSES:
b = base["per_1k_nodes"].get(c, 0.0)
n = cur["per_1k_nodes"].get(c, 0.0)
delta = (n - b) / b if b else (0.0 if n == 0 else float("inf"))
mark = ""
if delta > args.tolerance:
worse.append(c)
mark = " WORSE"
elif delta < -args.tolerance:
better.append(c)
mark = " better"
shown = "new" if delta == float("inf") else f"{delta * 100:+.1f}%"
print(f"{c:<28} {b:>14,.3f} {n:>14,.3f} {shown:>8}{mark}")
if better:
print(f"\n{len(better)} class(es) improved beyond tolerance: lower the baseline with --record")
if worse:
print(f"\n{len(worse)} class(es) grew beyond {args.tolerance * 100:.0f}%: {', '.join(worse)}")
return 1 if args.enforce else 0
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/memwaste-report.py
+392
View File
@@ -0,0 +1,392 @@
#!/usr/bin/env python3
"""
memwaste-report.py — turn waste-sanitizer dumps into ranked, symbolised evidence.
The sanitizer (src/foundation/mem_events.h) appends JSON lines at every phase
mark: a header per dump, then one object per allocation site and one per CPU
work row, all with RAW return addresses. Symbolisation is offline so the hot
path never loads a symbol handler. This tool reads every dump, keeps the LAST
dump of each process for the cumulative counters, the MAXIMUM over its dumps for
the phase snapshots (retained, duplicates), resolves the addresses against the
binary that produced them, and prints:
1. a waste summary per process — one line per waste class, in bytes/calls;
2. the memory table — allocation sites ranked by a chosen waste metric;
3. containers — arenas, dynamic arrays, hash tables: capacity against use;
4. CPU work — byte work, I/O, contention, pool imbalance, repeated work.
Usage:
memwaste-report.py DUMP.jsonl --binary build/memwaste/codebase-memory-mcp
memwaste-report.py DUMP.jsonl --binary ... --sort never_written --top 30
memwaste-report.py DUMP.jsonl --binary ... --json (for the ratchet)
"""
from __future__ import annotations
import argparse
import json
import os
import platform
import re
import shutil
import subprocess
import sys
from collections import defaultdict
MB = 1024.0 * 1024.0
SNAPSHOT_FIELDS = ("retained_bytes", "dup_bytes", "dup_blocks")
WASTE_CLASSES = [
# (key, label, unit)
("slack_bytes", "allocator slack (usable - requested)", "bytes"),
("never_written_bytes", "never-written bytes (fill pattern intact at free)", "bytes"),
("over_requested_bytes", "requested beyond the last written byte", "bytes"),
("zero_untouched_bytes", "calloc tail still zero at free (upper bound)", "bytes"),
("realloc_copy_bytes", "bytes copied by realloc growth", "bytes"),
("short_lived", "short-lived allocations (churn)", "calls"),
("dup_bytes_max", "duplicate small blocks (peak over phases)", "bytes"),
("retained_bytes_max", "live across a phase boundary (peak)", "bytes"),
("raw_bytes", "bytes that never went through the memory core", "bytes"),
("acc_untouched_bytes", "blocks never read or written (access lane)", "bytes"),
("acc_dead_bytes", "blocks written, never read by instrumented code (access lane)", "bytes"),
("acc_opaque_blocks", "blocks used only by code the access lane cannot see (no verdict)", "blocks"),
("acc_idle_bytes", "blocks idle for at least one phase before free (access lane)", "bytes"),
("acc_uninit_reads", "reads before any write, not calloc (access lane)", "blocks"),
("container_idle_peak_bytes", "container capacity left unused (largest instance, per site)", "bytes"),
("byte_work_bytes", "bytes moved/compared by mem*/str* calls in program code", "bytes"),
("allocator_byte_work_bytes", "bytes moved/compared inside the allocator (mimalloc)", "bytes"),
("byte_work_repeats", "strlen of a string just measured", "calls"),
("io_tiny_calls", "I/O calls under 512 bytes", "calls"),
("path_repeats", "open/stat of a path already opened/statted", "calls"),
("mutex_contended", "contended mutex acquisitions", "calls"),
("pool_imbalance_items", "work items an even pool would have spread", "items"),
("pool_ops_imbalance", "worker events an even pool would have spread (work done)", "events"),
("ht_repeat_lookups", "hash lookups of the key object just looked up", "calls"),
("ht_growths", "hash-table growths", "calls"),
]
BYTE_KINDS = {"memcpy", "memmove", "memset", "memcmp", "strlen", "strcmp", "strncmp"}
IO_KINDS = {"read", "write", "pread", "pwrite", "fread", "fwrite"}
PATH_KINDS = {"open", "stat", "fopen", "opendir"}
CONTAINER_KINDS = {"ct_arena", "ct_dyn_array", "ct_hash_table"}
# Symbols of the vendored allocator. Its byte work is real CPU, but the fix is in
# how the allocator is built or configured, not at a program call site.
ALLOCATOR_SYMBOL = re.compile(r"^_?mi_")
def load(path: str):
"""{pid: {"header": last header, "headers": [...], "sites": {addr: row}, "snap_max": {addr: {...}},
"work": {(kind, addr): row}}} — sites/work from the LAST dump of the process."""
procs: dict[int, dict] = {}
cur = None
with open(path, encoding="utf-8", errors="replace") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue # a torn line from a process that died mid-write
if "memwaste" in obj:
pid = int(obj.get("pid", 0))
p = procs.setdefault(pid, {"headers": [], "snap_max": defaultdict(dict)})
p["headers"].append(obj)
p["header"] = obj
p["sites"] = {}
p["work"] = {}
cur = p
elif cur is None:
continue
elif "work" in obj:
cur["work"][(obj["work"], obj["site"])] = obj
elif "site" in obj:
cur["sites"][obj["site"]] = obj
snap = cur["snap_max"][obj["site"]]
for f in SNAPSHOT_FIELDS:
v = obj.get(f, 0)
if v > snap.get(f, 0):
snap[f] = v
snap[f + "_phase"] = cur["header"].get("phase_label", "?")
return procs
def pe_image_base(binary: str) -> int | None:
"""The preferred ImageBase of a PE file, or None for anything else. A PE
symboliser wants VA = ImageBase + RVA, and the dump gives RVA + runtime base."""
try:
with open(binary, "rb") as fh:
head = fh.read(4096)
except OSError:
return None
if head[:2] != b"MZ" or len(head) < 0x40:
return None
pe = int.from_bytes(head[0x3C:0x40], "little")
if head[pe:pe + 4] != b"PE\0\0":
return None
opt = pe + 24
magic = int.from_bytes(head[opt:opt + 2], "little")
if magic == 0x20B: # PE32+
return int.from_bytes(head[opt + 24:opt + 32], "little")
if magic == 0x10B: # PE32
return int.from_bytes(head[opt + 28:opt + 32], "little")
return None
def symbolise(binary: str | None, base: str, addrs: list[str]) -> dict[str, str]:
if not binary or not addrs:
return {}
if not os.path.exists(binary) and os.path.exists(binary + ".exe"):
binary = binary + ".exe"
out: dict[str, str] = {}
uniq = sorted(set(addrs))
if platform.system() == "Darwin" and shutil.which("atos"):
for i in range(0, len(uniq), 2000):
chunk = uniq[i:i + 2000]
res = subprocess.run(["atos", "-o", binary, "-l", base] + chunk,
capture_output=True, text=True, check=False)
for addr, name in zip(chunk, res.stdout.splitlines()):
out[addr] = name.strip()
elif shutil.which("addr2line") or shutil.which("llvm-addr2line"):
tool = shutil.which("llvm-addr2line") or shutil.which("addr2line")
b = int(base, 16)
preferred = pe_image_base(binary) or 0
for i in range(0, len(uniq), 2000):
chunk = uniq[i:i + 2000]
rel = [hex(int(a, 16) - b + preferred) if b else a for a in chunk]
res = subprocess.run([tool, "-f", "-C", "-p", "-e", binary] + rel,
capture_output=True, text=True, check=False)
for addr, name in zip(chunk, res.stdout.splitlines()):
out[addr] = name.strip()
return out
def pct(part: float, whole: float) -> float:
return 100.0 * part / whole if whole else 0.0
def summarise(p: dict, names: dict[str, str]) -> dict:
s = defaultdict(int)
for addr, r in p["sites"].items():
s["slack_bytes"] += max(r["usable_bytes"] - r["requested_bytes"], 0)
for k in ("never_written_bytes", "over_requested_bytes", "zero_untouched_bytes",
"realloc_copy_bytes", "short_lived", "raw_bytes", "acc_untouched_bytes",
"acc_dead_bytes", "acc_idle_bytes", "acc_uninit_reads", "acc_opaque_blocks"):
s[k] += r.get(k, 0)
snap = p["snap_max"].get(addr, {})
s["dup_bytes_max"] += snap.get("dup_bytes", 0)
s["retained_bytes_max"] += snap.get("retained_bytes", 0)
for (kind, addr), w in p["work"].items():
if kind in CONTAINER_KINDS:
s["container_idle_peak_bytes"] += w.get("peak", max(w["bytes"] - w["aux1"], 0))
if kind == "ct_hash_table":
s["ht_growths"] += w["aux2"]
elif kind in BYTE_KINDS:
if ALLOCATOR_SYMBOL.match(names.get(addr, "")):
s["allocator_byte_work_bytes"] += w["bytes"]
else:
s["byte_work_bytes"] += w["bytes"]
s["byte_work_repeats"] += w["repeats"]
elif kind in IO_KINDS:
s["io_tiny_calls"] += w["aux1"]
elif kind in PATH_KINDS:
s["path_repeats"] += w["repeats"]
elif kind == "mutex":
s["mutex_contended"] += w["aux1"]
elif kind == "parallel_for":
s["pool_imbalance_items"] += w["aux1"]
elif kind == "pool_ops":
s["pool_ops_imbalance"] += w["aux1"]
elif kind == "ht_get":
s["ht_repeat_lookups"] += w["repeats"]
return dict(s)
def fmt(v: float, unit: str) -> str:
if unit == "bytes":
return f"{v / MB:,.1f} MB"
return f"{v:,.0f} {unit}"
SITE_SORTS = {
"bytes": lambda r, s: r["usable_bytes"],
"slack": lambda r, s: r["usable_bytes"] - r["requested_bytes"],
"never_written": lambda r, s: r.get("never_written_bytes", 0),
"over_requested": lambda r, s: r.get("over_requested_bytes", 0),
"churn": lambda r, s: r["short_lived"],
"copy": lambda r, s: r["realloc_copy_bytes"],
"live": lambda r, s: r["live_bytes"],
"allocs": lambda r, s: r["allocs"],
"dups": lambda r, s: s.get("dup_bytes", 0),
"retained": lambda r, s: s.get("retained_bytes", 0),
"dead": lambda r, s: r.get("acc_dead_bytes", 0),
"untouched": lambda r, s: r.get("acc_untouched_bytes", 0),
"idle": lambda r, s: r.get("acc_idle_bytes", 0),
"uninit": lambda r, s: r.get("acc_uninit_reads", 0),
}
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("dump")
ap.add_argument("--binary", help="the binary that produced the dump, for symbolisation")
ap.add_argument("--top", type=int, default=20)
ap.add_argument("--sort", choices=sorted(SITE_SORTS), default="bytes")
ap.add_argument("--json", action="store_true", help="emit the merged tables as JSON")
args = ap.parse_args()
procs = load(args.dump)
if not procs:
print("no waste-sanitizer dump found in", args.dump, file=sys.stderr)
return 2
names_by_pid = {}
for pid, p in procs.items():
addrs = list(p["sites"].keys()) + [a for (_, a) in p["work"].keys()]
names_by_pid[pid] = symbolise(args.binary, p["header"].get("image_base", "0x0"), addrs)
if args.json:
out = {"processes": {}}
for pid, p in procs.items():
names = names_by_pid[pid]
out["processes"][str(pid)] = {
"header": p["header"],
"phases": [h.get("phase_label") for h in p["headers"]],
"summary": summarise(p, names),
"sites": [dict(r, name=names.get(a, a),
**{(k if k.endswith("_phase") else k + "_max"): v
for k, v in p["snap_max"].get(a, {}).items()})
for a, r in p["sites"].items()],
"work": [dict(w, name=names.get(a, a)) for (_, a), w in p["work"].items()],
}
json.dump(out, sys.stdout, indent=1)
print()
return 0
# Work-heavy processes first: the index worker is the one that matters.
order = sorted(procs.items(), key=lambda kv: -kv[1]["header"].get("usable_bytes", 0))
for pid, p in order:
h = p["header"]
names = names_by_pid[pid]
s = summarise(p, names)
print("=" * 110)
print(f"process {pid} last dump={h.get('why')} phases={len(p['headers'])} allocs={h['allocs']:,} "
f"cum={h['usable_bytes'] / MB:,.0f} MB live={h['live_bytes'] / MB:,.0f} MB "
f"rss={h.get('rss_bytes', 0) / MB:,.0f} MB sites={h['sites']:,} fill={h.get('fill')} access={h.get('access')} "
f"layer={h.get('layer_vm_bytes', 0) / MB:,.0f} MB/{h.get('layer_threads', 0)} threads")
blind = {k: h.get(k, 0) for k in ("site_table_full", "pointer_table_full", "work_table_full", "untracked_frees")}
if any(blind.values()):
print(" blind spots: " + " ".join(f"{k}={v:,}" for k, v in blind.items()))
if h.get("foreign_frees", 0):
print(f" outside the shim: {h['foreign_frees']:,} frees of blocks another allocator made "
"(C runtime / system DLL; their allocations are invisible to the layer)")
print()
print(" WASTE SUMMARY")
for key, label, unit in WASTE_CLASSES:
v = s.get(key, 0)
if v:
print(f" {fmt(v, unit):>18} {label}")
if len(p["sites"]) == 0 and len(p["work"]) == 0:
continue
print()
print(f" MEMORY — top {args.top} sites by {args.sort}")
print(f" {'cum MB':>9} {'allocs':>12} {'slack%':>6} {'churn%':>6} {'copy MB':>8} {'unwrit MB':>9} "
f"{'overreq MB':>10} {'dup MB':>7} {'retain MB':>9} {'raw%':>5} {'live MB':>8} site")
key = SITE_SORTS[args.sort]
rows = sorted(p["sites"].items(), key=lambda kv: (-key(kv[1], p["snap_max"].get(kv[0], {})), kv[0]))
for addr, r in rows[:args.top]:
snap = p["snap_max"].get(addr, {})
print(f" {r['usable_bytes'] / MB:>9,.1f} {r['allocs']:>12,} "
f"{pct(r['usable_bytes'] - r['requested_bytes'], r['usable_bytes']):>6.1f} "
f"{pct(r['short_lived'], r['allocs']):>6.1f} {r['realloc_copy_bytes'] / MB:>8,.1f} "
f"{r.get('never_written_bytes', 0) / MB:>9,.1f} {r.get('over_requested_bytes', 0) / MB:>10,.1f} "
f"{snap.get('dup_bytes', 0) / MB:>7,.1f} {snap.get('retained_bytes', 0) / MB:>9,.1f} "
f"{pct(r['raw_bytes'], r['usable_bytes']):>5.0f} {r['live_bytes'] / MB:>8,.1f} "
f"{names.get(addr, addr)[:90]}")
if h.get("access"):
# The access lane's verdicts per allocation site. Library-owned blocks (a FILE
# buffer) are not tracked; bytes a library wrote without a store callback are
# recognised through the fill pattern.
acc_cols = [("untouched", "acc_untouched_bytes", "never read or written"),
("dead", "acc_dead_bytes", "written, never read"),
("idle", "acc_idle_bytes", "idle for a phase before free")]
for sort_key, field, label in acc_cols:
rows = [(a, r) for a, r in p["sites"].items() if r.get(field, 0) > 0]
if not rows:
continue
print()
print(f" ACCESS — top {args.top} sites by bytes {label}")
print(f" {'MB':>9} {'blocks':>11} {'allocs':>12} {'uninit rd':>10} site")
blocks_field = field.replace("_bytes", "_blocks")
for a, r in sorted(rows, key=lambda t: (-t[1][field], t[0]))[:args.top]:
print(f" {r[field] / MB:>9,.1f} {r.get(blocks_field, 0):>11,} {r['allocs']:>12,} "
f"{r.get('acc_uninit_reads', 0):>10,} {names.get(a, a)[:90]}")
rows = [(a, r) for a, r in p["sites"].items() if r.get("acc_uninit_reads", 0) > 0]
if rows:
print()
print(f" ACCESS — top {args.top} sites by blocks read before any write (not calloc)")
print(f" {'blocks':>11} {'allocs':>12} site")
for a, r in sorted(rows, key=lambda t: (-t[1]["acc_uninit_reads"], t[0]))[:args.top]:
print(f" {r['acc_uninit_reads']:>11,} {r['allocs']:>12,} {names.get(a, a)[:90]}")
ct = [(k, a, w) for (k, a), w in p["work"].items() if k in CONTAINER_KINDS]
if ct:
print()
# A container reports at every reset: calls are USES, cap/used sum over uses.
# "peak idle" is the largest capacity a single instance left unused.
print(f" CONTAINERS — top {args.top} by largest unused capacity of one instance")
print(f" {'kind':<10} {'uses':>9} {'avg cap MB':>10} {'avg used MB':>11} {'unused%':>7} "
f"{'peak idle MB':>12} {'growths':>9} created at")
for k, a, w in sorted(ct, key=lambda t: (-t[2].get("peak", 0), t[1]))[:args.top]:
uses = max(w["calls"], 1)
print(f" {k[3:]:<10} {w['calls']:>9,} {w['bytes'] / uses / MB:>10,.2f} {w['aux1'] / uses / MB:>11,.2f} "
f"{pct(w['bytes'] - w['aux1'], w['bytes']):>7.1f} {w.get('peak', 0) / MB:>12,.1f} "
f"{w['aux2']:>9,} {names.get(a, a)[:90]}")
def section(title, kinds, sort_key, cols):
rows = [(k, a, w) for (k, a), w in p["work"].items() if k in kinds]
if not rows:
return
print()
print(f" CPU — {title}")
print(" " + " ".join(f"{c[0]:>{c[1]}}" for c in cols) + " site")
for k, a, w in sorted(rows, key=lambda t: (-sort_key(t[2]), t[1]))[:args.top]:
cells = []
for c in cols:
v = c[2](k, w)
cells.append(f"{v:>{c[1]}}" if isinstance(v, str) else f"{v:>{c[1]},.0f}")
print(" " + " ".join(cells) + f" {names.get(a, a)[:90]}")
section("byte work (mem*/str*) by bytes", BYTE_KINDS, lambda w: w["bytes"],
[("kind", 8, lambda k, w: k), ("calls", 12, lambda k, w: w["calls"]),
("MB", 10, lambda k, w: w["bytes"] / MB), ("repeats", 10, lambda k, w: w["repeats"]),
("max call", 10, lambda k, w: w.get("peak", 0))])
section("I/O by calls", IO_KINDS, lambda w: w["calls"],
[("kind", 8, lambda k, w: k), ("calls", 12, lambda k, w: w["calls"]),
("MB", 10, lambda k, w: w["bytes"] / MB), ("tiny", 10, lambda k, w: w["aux1"])])
section("paths opened/statted again", PATH_KINDS, lambda w: w["repeats"],
[("kind", 8, lambda k, w: k), ("calls", 12, lambda k, w: w["calls"]),
("repeats", 10, lambda k, w: w["repeats"]), ("failed", 10, lambda k, w: w["aux1"])])
section("mutex contention", {"mutex"}, lambda w: w["aux1"],
[("calls", 12, lambda k, w: w["calls"]), ("contended", 10, lambda k, w: w["aux1"])])
section("parallel_for imbalance", {"parallel_for"}, lambda w: w["aux1"],
[("calls", 8, lambda k, w: w["calls"]), ("items", 10, lambda k, w: w["bytes"]),
("imbalance", 10, lambda k, w: w["aux1"])])
# imbalance / (work + imbalance): the share of the pool's capacity that sat idle
# while the busiest worker finished (0% = perfectly even).
section("pool balance by work done (per-worker events)", {"pool_ops"}, lambda w: w["aux1"],
[("calls", 8, lambda k, w: w["calls"]), ("events", 14, lambda k, w: w["bytes"]),
("idle%", 6, lambda k, w: f"{pct(w['aux1'], w['bytes'] + w['aux1']):.1f}"),
("workers", 8, lambda k, w: w["aux2"] / max(w["calls"], 1))])
section("hash-table lookups", {"ht_get"}, lambda w: w["repeats"],
[("gets", 12, lambda k, w: w["calls"]), ("misses", 12, lambda k, w: w["aux1"]),
("repeats", 12, lambda k, w: w["repeats"])])
section("dead writes (access lane)", {"dead_write"}, lambda w: w["bytes"],
[("blocks", 10, lambda k, w: w["aux1"]), ("MB", 10, lambda k, w: w["bytes"] / MB)])
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/memwaste-scaling.py
+262
View File
@@ -0,0 +1,262 @@
#!/usr/bin/env python3
"""
memwaste-scaling.py — the CPU scaling lane of the waste sanitizer.
Work that grows faster than the input is the costliest CPU waste there is, and a
single run cannot see it: a function that burns 5 % of the time on the Go corpus
may burn 60 % on a corpus four times larger. This lane indexes the SAME real
source tree replicated K times and 2K times, with clang's execution counters in
every object (vendored ones included), and compares, function by function, how
much work each leg did. Linear work doubles; a function whose counters grow by
more than --threshold (default 2.3, i.e. 15 % above linear) while doing at least
--min-excess extra counter increments is reported as super-linear.
The counters are COUNTS, not time: the verdict is a function of the code and the
input, so it can gate (O9). Wall time is printed as information only.
Replicas are hard links (copies when the file system refuses) under
<out>/leg-<n>/r<i>/, so every replica is a separate directory -- a separate Go
package, a separate module -- and the registry, namespace and resolution paths
see N times the same names: exactly the coupling behind the #1669 class of bugs.
Usage:
scripts/memwaste-scaling.py --corpus ~/perf-bench/go/src/net --replicas 2
scripts/memwaste-scaling.py --corpus DIR --replicas 3 --no-build --json
Build: MEMWASTE_PROFILE=1 into build/memwaste-profile with clang (the driver does
it unless --no-build). llvm-profdata is taken from the directory of the clang
that built the binary, so the profile format always matches.
"""
from __future__ import annotations
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BUILD_DIR = "build/memwaste-profile"
def find_profdata(cc: str) -> str:
cc_path = shutil.which(cc)
if cc_path:
cand = os.path.join(os.path.dirname(os.path.realpath(cc_path)), "llvm-profdata")
if os.access(cand, os.X_OK):
return cand
for name in ("llvm-profdata",) + tuple(f"llvm-profdata-{v}" for v in range(30, 13, -1)):
p = shutil.which(name)
if p:
return p
if shutil.which("xcrun"):
res = subprocess.run(["xcrun", "--find", "llvm-profdata"], capture_output=True, text=True, check=False)
if res.returncode == 0 and res.stdout.strip():
return res.stdout.strip()
sys.exit("memwaste-scaling: llvm-profdata not found next to the compiler or on PATH")
def replicate(src: str, dst: str, copies: int) -> int:
"""Hard-link `copies` replicas of src under dst/r<i>; returns the file count."""
files = 0
for i in range(copies):
base = os.path.join(dst, f"r{i}")
for dirpath, dirnames, filenames in os.walk(src):
dirnames[:] = [d for d in dirnames if d not in (".git", ".hg", ".svn")]
rel = os.path.relpath(dirpath, src)
out_dir = os.path.join(base, rel)
os.makedirs(out_dir, exist_ok=True)
for name in filenames:
s = os.path.join(dirpath, name)
if os.path.islink(s) or not os.path.isfile(s):
continue
d = os.path.join(out_dir, name)
try:
os.link(s, d)
except OSError:
shutil.copy2(s, d)
files += 1
return files
def run_leg(binary: str, profdata: str, corpus: str, work: str, copies: int) -> tuple[dict, dict]:
leg = os.path.join(work, f"leg-{copies}")
tree = os.path.join(leg, "corpus")
os.makedirs(tree, exist_ok=True)
nfiles = replicate(corpus, tree, copies)
raw_dir = os.path.join(leg, "profraw")
os.makedirs(raw_dir, exist_ok=True)
runtime = tempfile.mkdtemp(prefix="cbm-sc.", dir="/tmp")
os.chmod(runtime, 0o700)
env = dict(os.environ)
env.update({
"LLVM_PROFILE_FILE": os.path.join(raw_dir, "cbm-%p.profraw"),
"CBM_PROFILE_DIR": raw_dir, # the index worker names its own: worker-<pid>.profraw
"CBM_CACHE_DIR": os.path.join(leg, "cache"),
"CBM_RUNTIME_DIR": runtime,
"CBM_LOG_LEVEL": "error",
})
env.pop("CBM_MEMWASTE", None) # production code paths: the event layer stays dormant
t0 = time.monotonic()
try:
res = subprocess.run([binary, "cli", "index_repository", "--repo-path", tree, "--mode", "full"],
env=env, capture_output=True, text=True, check=False)
finally:
shutil.rmtree(runtime, ignore_errors=True)
wall = time.monotonic() - t0
if res.returncode != 0:
sys.exit(f"memwaste-scaling: index of leg {copies} failed (rc={res.returncode}):\n{res.stderr[-2000:]}")
raws = sorted(os.path.join(raw_dir, f) for f in os.listdir(raw_dir) if f.endswith(".profraw"))
workers = [r for r in raws if os.path.basename(r).startswith("worker-")]
if not workers:
sys.exit("memwaste-scaling: the index worker wrote no profile -- is the binary built with "
"MEMWASTE_PROFILE=1, and does it call cbm_memev_process_exit before _Exit?")
# The comparison is the WORKER's work: the daemon and CLI processes differ in
# number between legs (and hash the binary at start-up), which is not scaling.
info = {"copies": copies, "files": nfiles, "wall_s": round(wall, 1), "profiles": len(raws)}
return merge_counts(profdata, workers, os.path.join(leg, "worker.profdata")), \
dict(info, other=merge_counts(profdata, [r for r in raws if r not in workers],
os.path.join(leg, "other.profdata")))
def merge_counts(profdata: str, raws: list[str], merged: str) -> dict[str, int]:
if not raws:
return {}
subprocess.run([profdata, "merge", "-o", merged] + raws, check=True)
shown = subprocess.run([profdata, "show", "--all-functions", "--counts", merged],
capture_output=True, text=True, check=True).stdout
return parse_counts(shown)
FUNC_RE = re.compile(r"^ (\S.*):$")
def parse_counts(text: str) -> dict[str, int]:
"""Total counter increments per function from `llvm-profdata show --counts`."""
out: dict[str, int] = {}
name = None
for line in text.splitlines():
m = FUNC_RE.match(line)
if m:
name = m.group(1)
continue
if name is None:
continue
s = line.strip()
if s.startswith("Function count:"):
out[name] = out.get(name, 0) + int(s.split(":", 1)[1])
elif s.startswith("Block counts:"):
vals = s.split(":", 1)[1].strip().strip("[]")
if vals:
out[name] = out.get(name, 0) + sum(int(v) for v in vals.split(",") if v.strip())
return out
def display_name(raw: str) -> str:
# static functions are "file.c;name" (or "file.c:name" in older formats)
for sep in (";", ":"):
if sep in raw:
file_part, fn = raw.rsplit(sep, 1)
return f"{fn} ({os.path.basename(file_part)})"
return raw
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--corpus", required=True, help="a real source tree to replicate")
ap.add_argument("--replicas", type=int, default=2, help="K: legs index K and 2K replicas")
ap.add_argument("--threshold", type=float, default=2.3, help="2K/K counter ratio that is super-linear")
ap.add_argument("--min-excess", type=int, default=10_000_000,
help="ignore functions whose work beyond linear is below this many counter increments")
ap.add_argument("--top", type=int, default=30)
ap.add_argument("--out", help="work directory (default: a new temp dir)")
ap.add_argument("--no-build", action="store_true")
ap.add_argument("--cc", default="clang")
ap.add_argument("--json", action="store_true")
args = ap.parse_args()
corpus = os.path.abspath(os.path.expanduser(args.corpus))
if not os.path.isdir(corpus):
sys.exit(f"memwaste-scaling: {corpus} is not a directory")
binary = os.path.join(ROOT, BUILD_DIR, "codebase-memory-mcp")
if not args.no_build:
cxx = "clang++" if args.cc == "clang" else args.cc.replace("clang", "clang++")
subprocess.run([os.path.join(ROOT, "scripts", "build.sh"), f"BUILD_DIR={BUILD_DIR}",
"MEMWASTE_PROFILE=1", f"CC={args.cc}", f"CXX={cxx}"],
check=True, stdout=subprocess.DEVNULL)
if not os.access(binary, os.X_OK):
sys.exit(f"memwaste-scaling: {binary} is missing (build it, or drop --no-build)")
profdata = find_profdata(args.cc)
work = os.path.abspath(args.out) if args.out else tempfile.mkdtemp(prefix="cbm-scaling.")
os.makedirs(work, exist_ok=True)
for sub in os.listdir(work):
if sub.startswith("leg-"):
shutil.rmtree(os.path.join(work, sub), ignore_errors=True)
k = args.replicas
small, info_k = run_leg(binary, profdata, corpus, work, k)
large, info_2k = run_leg(binary, profdata, corpus, work, 2 * k)
total_k = sum(small.values())
total_2k = sum(large.values())
rows = []
for name, c2 in large.items():
c1 = small.get(name, 0)
excess = c2 - 2 * c1
ratio = (c2 / c1) if c1 else float("inf")
rows.append({"function": name, "k": c1, "2k": c2, "ratio": ratio, "excess": excess})
flagged = [r for r in rows if r["ratio"] > args.threshold and r["excess"] >= args.min_excess]
flagged.sort(key=lambda r: -r["excess"])
heaviest = sorted(rows, key=lambda r: -r["2k"])[: args.top]
for info in (info_k, info_2k):
other = info.pop("other", {}) or {}
info["other_total"] = sum(other.values())
info["other_top"] = sorted(other.items(), key=lambda kv: -kv[1])[:10]
report = {
"corpus": corpus, "legs": [info_k, info_2k], "threshold": args.threshold,
"min_excess": args.min_excess, "total_counts": {"k": total_k, "2k": total_2k},
"total_ratio": (total_2k / total_k) if total_k else None,
"superlinear": flagged,
"work_dir": work,
}
if args.json:
json.dump(report, sys.stdout, indent=1, default=lambda o: None if o == float("inf") else o)
print()
return 1 if flagged else 0
print(f"corpus {corpus}")
for info in (info_k, info_2k):
print(f" leg {info['copies']:>2} replicas: {info['files']:,} files, {info['profiles']} process profile(s), "
f"{info['wall_s']} s wall (information only)")
print(f" worker counter increments: k={total_k:,} 2k={total_2k:,} ratio={report['total_ratio']:.3f}")
for info in (info_k, info_2k):
other = info.get("other") or {}
top = sorted(other.items(), key=lambda kv: -kv[1])[:3]
print(f" other processes at {info['copies']} replicas: {sum(other.values()):,} increments; heaviest: "
+ ", ".join(f"{display_name(n)} {c:,}" for n, c in top))
print()
print(f" SUPER-LINEAR — ratio > {args.threshold} and >= {args.min_excess:,} increments beyond linear")
if not flagged:
print(" none")
else:
print(f" {'ratio':>7} {'excess':>16} {'k':>16} {'2k':>16} function")
for r in flagged[: args.top]:
ratio = "new" if r["ratio"] == float("inf") else f"{r['ratio']:.2f}"
print(f" {ratio:>7} {r['excess']:>16,} {r['k']:>16,} {r['2k']:>16,} {display_name(r['function'])[:100]}")
print()
print(f" HEAVIEST at 2k — top {args.top} by counter increments (share of all work, scaling ratio)")
for r in heaviest:
ratio = "new" if r["ratio"] == float("inf") else f"{r['ratio']:.2f}"
print(f" {100.0 * r['2k'] / total_2k:>6.2f}% x{ratio:>5} {display_name(r['function'])[:100]}")
print()
print(f"work dir: {work}")
return 1 if flagged else 0
if __name__ == "__main__":
sys.exit(main())
scripts/memwaste.sh
Executable
+182
View File
@@ -0,0 +1,182 @@
#!/usr/bin/env bash
# memwaste.sh — run the waste sanitizer on a repository and print the ranked report.
#
# The sanitizer (src/foundation/mem_events.h) measures memory that is CORRECT but
# unnecessary: allocator slack, churn, realloc copying, memory that never reaches
# the memory core. This driver is the one entry point for every venue: a local
# run on a bench corpus, the PR-CI fixture run, the dry-run and release runs.
#
# Usage:
# scripts/memwaste.sh <repo-path> [--lane event|access] [--top N] [--sort METRIC]
# [--out DIR] [--no-build] [--json] [--binary PATH]
#
# --binary runs a kept copy of a flavour binary (a baseline, a replay) instead of
# the one in the build directory; it implies --no-build.
#
# What it does
# 1. builds the `memwaste` flavour (production flags + the event layer) into
# build/memwaste, unless --no-build;
# 2. indexes <repo-path> with CBM_MEMWASTE=1 into a private cache and a private,
# SHORT runtime directory (the coordination socket has a 104-byte path limit);
# 3. prints the symbolised report and writes the raw dump + JSON next to it;
# 4. FAILS when the measurement itself is unsound: a table overflowed, or more
# than 0.1 % of frees were of blocks the layer never saw. A sanitizer that
# silently loses records produces false confidence.
#
# Nothing here reads a clock to decide anything: wall time is printed as
# information only. Gates are counts.
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
REPO=""
TOP=25
SORT=bytes
OUT=""
BUILD=1
JSON=0
LANE=event
BINARY_OVERRIDE=""
while [ $# -gt 0 ]; do
case "$1" in
--binary) BINARY_OVERRIDE="$2"; BUILD=0; shift 2 ;;
--top) TOP="$2"; shift 2 ;;
--sort) SORT="$2"; shift 2 ;;
--out) OUT="$2"; shift 2 ;;
--no-build) BUILD=0; shift ;;
--json) JSON=1; shift ;;
--lane) LANE="$2"; shift 2 ;;
-h | --help) sed -n '2,24p' "$0"; exit 0 ;;
# A typo'd flag must never be read as the repository path — it would index
# nothing and report an empty lane as a clean one.
-*) echo "memwaste.sh: unknown argument $1. Please consult --help." >&2; exit 2 ;;
*) REPO="$1"; shift ;;
esac
done
if [ -z "$REPO" ] || [ ! -d "$REPO" ]; then
echo "memwaste.sh: a repository directory is required. Please consult --help." >&2
exit 2
fi
case "$LANE" in
event) BUILD_ARGS=(BUILD_DIR=build/memwaste MEMWASTE=1 ${CC:+CC=$CC} ${CXX:+CXX=$CXX})
BIN="$ROOT/build/memwaste/codebase-memory-mcp" ;;
access) BUILD_ARGS=(BUILD_DIR=build/memwaste-access MEMWASTE_ACCESS=1 CC=clang CXX=clang++)
BIN="$ROOT/build/memwaste-access/codebase-memory-mcp" ;;
*) echo "memwaste.sh: --lane must be event or access" >&2; exit 2 ;;
esac
if [ "$BUILD" -eq 1 ]; then
"$ROOT/scripts/build.sh" "${BUILD_ARGS[@]}" >/dev/null
fi
if [ -n "$BINARY_OVERRIDE" ]; then
BIN="$BINARY_OVERRIDE"
fi
if [ ! -x "$BIN" ]; then
echo "memwaste.sh: $BIN is missing (build failed, or --no-build without a build)" >&2
exit 2
fi
if [ -z "$OUT" ]; then
OUT="$(mktemp -d "${TMPDIR:-/tmp}/cbm-memwaste.XXXXXX")"
fi
mkdir -p "$OUT"
DUMP="$OUT/memwaste.jsonl"
rm -f "$DUMP"
RUN_BIN="$BIN"
if command -v cygpath >/dev/null 2>&1 && ! command -v winepath >/dev/null 2>&1; then
# Native Windows: the server walks the ancestor chain of its binary, cache and
# runtime directories and refuses any that grant mutation to untrusted SIDs.
# The checkout and msys /tmp both do, so the run happens from a root under
# USERPROFILE stamped to the current user, SYSTEM and Administrators -- by
# SID, the way scripts/memlab.sh does it (#1532).
PROFILE_DIR="$(cygpath -u "$USERPROFILE")"
WORK="$(mktemp -d "$PROFILE_DIR/cbm-mw.XXXXXX")"
WORK_W="$(cygpath -w "$WORK")"
SID_QUERY='[System.Security.Principal.WindowsIdentity]::GetCurrent().User.Value'
ME="$(powershell.exe -NoProfile -NonInteractive -Command "$SID_QUERY" 2>/dev/null | tr -d '\r\n')"
case "$ME" in
S-1-*) ME="*$ME" ;;
*) echo "memwaste.sh: could not resolve the current user's SID" >&2; exit 2 ;;
esac
if ! MSYS2_ARG_CONV_EXCL='*' icacls "$WORK_W" /inheritance:r \
/grant:r "${ME}:(OI)(CI)F" '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' /Q >/dev/null 2>&1; then
echo "memwaste.sh: could not stamp $WORK" >&2
exit 2
fi
mkdir -p "$WORK/cache" "$WORK/rt"
cp "$BIN.exe" "$WORK/codebase-memory-mcp.exe" 2>/dev/null || cp "$BIN" "$WORK/codebase-memory-mcp.exe"
RUN_BIN="$WORK/codebase-memory-mcp.exe"
CACHE_DIR="$WORK_W\\cache"
RUNTIME="$WORK_W\\rt"
DUMP_ENV="$(cygpath -w "$DUMP")"
else
WORK="$(mktemp -d /tmp/cbm-mw.XXXXXX)"
chmod 700 "$WORK"
# A fresh cache every run: an existing index turns --mode full into an
# incremental pass that measures hashing instead of indexing.
rm -rf "$OUT/cache"
mkdir -p "$OUT/cache"
CACHE_DIR="$OUT/cache"
RUNTIME="$WORK"
DUMP_ENV="$DUMP"
fi
# CBM_MEMWASTE_KEEP=1 keeps the private root (binary copy, cache, logs) for inspection.
cleanup() {
if [ "${CBM_MEMWASTE_KEEP:-0}" != 1 ]; then
rm -rf "$WORK"
fi
}
trap cleanup EXIT
INDEX_RC=0
CBM_MEMWASTE=1 CBM_MEMWASTE_OUT="$DUMP_ENV" CBM_CACHE_DIR="$CACHE_DIR" CBM_RUNTIME_DIR="$RUNTIME" \
CBM_LOG_LEVEL=error "$RUN_BIN" cli index_repository --repo-path "$REPO" --mode full \
>"$OUT/index.json" || INDEX_RC=$?
if [ "$INDEX_RC" -ne 0 ]; then
# A failed index is a finding about the flavour, never a silent partial report.
echo "memwaste.sh: the index failed (rc=$INDEX_RC); its logs follow" >&2
for log in "$WORK"/cache/logs/* "$WORK"/cache/logs/.worker-log-* "$OUT"/cache/logs/* "$OUT"/cache/logs/.worker-log-*; do
if [ -f "$log" ]; then
echo "--- $log" >&2
tail -n 60 "$log" >&2
fi
done
exit 1
fi
if [ ! -s "$DUMP" ]; then
echo "memwaste.sh: the sanitizer wrote no dump -- is the flavour built with MEMWASTE=1?" >&2
exit 1
fi
python3 "$ROOT/scripts/memwaste-report.py" "$DUMP" --binary "$BIN" --json >"$OUT/memwaste.json"
if [ "$JSON" -eq 1 ]; then
cat "$OUT/memwaste.json"
else
python3 "$ROOT/scripts/memwaste-report.py" "$DUMP" --binary "$BIN" --top "$TOP" --sort "$SORT"
echo
echo "raw dump: $DUMP"
echo "json: $OUT/memwaste.json"
fi
# Soundness of the measurement itself.
python3 - "$OUT/memwaste.json" <<'PY'
import json, sys
procs = json.load(open(sys.argv[1]))["processes"]
bad = []
for pid, proc in procs.items():
h = proc["header"]
if h.get("site_table_full", 0) or h.get("pointer_table_full", 0):
bad.append(f"process {pid}: a table overflowed (sites={h.get('site_table_full')}, pointers={h.get('pointer_table_full')})")
frees = h.get("frees", 0) + h.get("untracked_frees", 0)
if frees and h.get("untracked_frees", 0) * 1000 > frees:
bad.append(f"process {pid}: {h['untracked_frees']:,} of {frees:,} frees were of blocks the layer never saw (> 0.1 %)")
if bad:
print("memwaste.sh: UNSOUND MEASUREMENT", file=sys.stderr)
for b in bad:
print(" " + b, file=sys.stderr)
sys.exit(1)
PY
scripts/run-test-wave.py
+13 -1
View File
@@ -26,7 +26,19 @@ SUITE_NAME = re.compile(r"^[a-z0-9_]+$")
SUMMARY = re.compile(r"^ (?P<passed>[0-9]+) passed")
FAILED = re.compile(r"(?:^|, )(?P<failed>[0-9]+) failed")
SKIPPED = re.compile(r"(?:^|, )(?P<skipped>[0-9]+) skipped")
SLOW_SUITES = frozenset(("incremental", "store_arch", "daemon_runtime"))
# Suites whose honest runtime does not fit the default per-suite budget, and so
# get --slow-timeout instead. This is a statement about SIZE, never about
# flakiness: every suite here is deterministic and simply long, and a racy suite
# must be made deterministic rather than given more clock.
#
# `cli` joined the list because the classification had gone stale, not because
# anything regressed. It spends 497s of the 900s default on macos-14 -- the
# FASTEST macOS runner -- while the macos-15-intel runner in the same matrix is
# 2.4-3.6x slower on comparable suites (daemon_runtime 842s vs 349s,
# stack_overflow_b 277s vs 76s). 497s at that ratio cannot fit, so the suite was
# killed at 900s and reported as hung. daemon_runtime, at 842s on that same
# runner, survives only because it was already listed here.
SLOW_SUITES = frozenset(("incremental", "store_arch", "daemon_runtime", "cli"))
POLL_SECONDS = 0.05
# WHY: the Windows descendant probe below is a cold `powershell.exe` + CIM
scripts/sanitizer-coverage-map.py
+306
View File
@@ -0,0 +1,306 @@
#!/usr/bin/env python3
"""
sanitizer-coverage-map.py — what our sanitizer lanes instrument, what they
actually execute, and every hole, vendored code included.
A sanitizer only checks code that runs under it. A function no suite reaches is
unchecked in every lane, and until now nobody could name those functions. This
tool answers three questions and keeps the answers from regressing:
static Which translation units does each lane COMPILE with its sanitizer,
per the build's own compile commands (make -n)? Lists every TU no
dynamic lane instruments and every TU the static lanes filter out.
run Build one lane with clang source coverage on top of its sanitizer
flags, run its test runner (suites optional), and export per-function
execution counts (lcov) -- vendored objects included, because the
flags ride on the lane's own sanitizer variable, which the vendored
rules already use.
union Merge the per-lane exports: the functions that ran under NO lane,
grouped by file, vendored code flagged.
ratchet Compare a union against the checked-in baseline: a function that
joins the never-executed list fails; one that leaves it is reported
so the baseline can be lowered.
map Validate and print scripts/sanitizer-map.json: every defect class x
platform cell must be covered, partial, or a named limit with a reason.
`run` executes test suites, so it belongs to the lane's own venue (local CI, the
CI job); `static`, `union`, `ratchet` and `map` run anywhere in seconds.
Usage:
scripts/sanitizer-coverage-map.py static
scripts/sanitizer-coverage-map.py run --lane asan --out build/covmap [SUITE...]
scripts/sanitizer-coverage-map.py union --out build/covmap
scripts/sanitizer-coverage-map.py ratchet --out build/covmap
scripts/sanitizer-coverage-map.py map
"""
from __future__ import annotations
import argparse
import json
import os
import shlex
import shutil
import subprocess
import sys
import tempfile
from collections import defaultdict
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BASELINE = os.path.join(ROOT, "scripts", "sanitizer-coverage-baseline.txt")
MAP_FILE = os.path.join(ROOT, "scripts", "sanitizer-map.json")
COVERAGE_FLAGS = "-fprofile-instr-generate -fcoverage-mapping"
SRC_EXT = (".c", ".cc", ".cpp")
# lane -> how the lane builds and runs. `var` is the make variable carrying the
# lane's sanitizer flags; the coverage flags are appended to it.
LANES = {
"asan": {"target": "test-runner", "var": "SANITIZE",
"value": "-fsanitize=address,undefined -fno-omit-frame-pointer", "env": {}},
"tsan": {"target": "test-runner-tsan", "var": "TSAN_SANITIZE",
"value": "-fsanitize=thread -fno-omit-frame-pointer",
"env": {"TSAN_OPTIONS": "halt_on_error=1:report_thread_leaks=0"}},
"lsan": {"target": "test-runner", "var": "SANITIZE",
"value": "-fsanitize=address,undefined -fno-omit-frame-pointer",
"env": {"ASAN_OPTIONS": "detect_leaks=1:halt_on_error=1"}},
}
DYNAMIC_FLAGS = ("-fsanitize=address", "-fsanitize=thread", "-fsanitize=memory", "-fsanitize=undefined",
"-fsanitize=address,undefined")
def is_vendored(path: str) -> bool:
# grammar_<lang>.c and the runtime/unity wrappers only #include vendored sources
base = os.path.basename(path)
return "vendored/" in path or base.startswith(("grammar_", "ts_runtime")) or base in (
"sqlite3.c", "mimalloc.c", "unixcoder_blob.c")
def dry_commands(target: str, extra: list[str]) -> list[list[str]]:
res = subprocess.run(["make", "-f", "Makefile.cbm", "-n", "-B", target] + extra,
cwd=ROOT, capture_output=True, text=True, check=False)
cmds = []
for line in res.stdout.replace("\\\n", " ").splitlines():
try:
argv = shlex.split(line)
except ValueError:
continue
if argv and any(a.endswith(SRC_EXT) for a in argv) and ("-c" in argv or "-o" in argv):
cmds.append(argv)
return cmds
def make_var(name: str) -> list[str]:
"""The fully expanded value of a Makefile.cbm variable (works with make 3.81: no --eval)."""
with tempfile.NamedTemporaryFile("w", suffix=".mk", delete=False) as mk:
mk.write("print-cbm-var-%:\n\t@echo $($*)\n")
helper = mk.name
try:
res = subprocess.run(["make", "-s", "-f", "Makefile.cbm", "-f", helper, f"print-cbm-var-{name}"],
cwd=ROOT, capture_output=True, text=True, check=False)
finally:
os.unlink(helper)
return res.stdout.split() if res.returncode == 0 else []
def cmd_static(args) -> int:
instrumented: dict[str, set[str]] = defaultdict(set)
compiled: set[str] = set()
for lane, spec in {"asan": LANES["asan"], "tsan": LANES["tsan"]}.items():
bdir = f"build/covmap-static-{lane}"
for argv in dry_commands(f"{bdir}/{spec['target']}", [f"BUILD_DIR={bdir}"]):
flags = " ".join(argv)
for a in argv:
if a.endswith(SRC_EXT) and os.path.isfile(os.path.join(ROOT, a)):
compiled.add(a)
if any(f in flags for f in ("-fsanitize=address", "-fsanitize=thread")):
instrumented[a].add(lane)
uninstrumented = sorted(s for s in compiled if not instrumented.get(s))
lint_srcs = set(make_var("LINT_SRCS"))
static_skipped = sorted(s for s in compiled if s not in lint_srcs)
out = {
"translation_units": len(compiled),
"not_instrumented_by_any_dynamic_lane": uninstrumented,
"not_in_static_lanes": static_skipped,
"per_tu": {s: sorted(instrumented.get(s, ())) for s in sorted(compiled)},
}
if args.json:
json.dump(out, sys.stdout, indent=1)
print()
return 0
print(f"translation units in the test builds: {len(compiled)}")
print(f" instrumented by ASan+UBSan: {sum(1 for v in instrumented.values() if 'asan' in v)}"
f" by TSan: {sum(1 for v in instrumented.values() if 'tsan' in v)}")
print()
print(f" NOT instrumented by any dynamic lane ({len(uninstrumented)}):")
for s in uninstrumented:
print(f" {s}{' [vendored]' if is_vendored(s) else ''}")
print()
print(f" outside the static lanes' source list (cppcheck, clang-analyzer) ({len(static_skipped)}):")
vend = [s for s in static_skipped if is_vendored(s)]
own = [s for s in static_skipped if not is_vendored(s) and not s.startswith("tests/")]
print(f" vendored: {len(vend)} project: {len(own)}")
for s in own:
print(f" {s}")
return 0
def find_llvm_tool(name: str, cc: str) -> str:
cc_path = shutil.which(cc)
if cc_path:
cand = os.path.join(os.path.dirname(os.path.realpath(cc_path)), name)
if os.access(cand, os.X_OK):
return cand
p = shutil.which(name)
if p:
return p
if shutil.which("xcrun"):
res = subprocess.run(["xcrun", "--find", name], capture_output=True, text=True, check=False)
if res.returncode == 0:
return res.stdout.strip()
sys.exit(f"sanitizer-coverage-map: {name} not found next to {cc} or on PATH")
def cmd_run(args) -> int:
spec = LANES[args.lane]
out = os.path.abspath(args.out)
os.makedirs(out, exist_ok=True)
bdir = f"build/covmap-{args.lane}"
cxx = "clang++" if args.cc == "clang" else args.cc.replace("clang", "clang++")
flags = f"{spec['value']} {COVERAGE_FLAGS}"
subprocess.run(["make", "-f", "Makefile.cbm", f"-j{os.cpu_count() or 4}", f"{bdir}/{spec['target']}",
f"BUILD_DIR={bdir}", f"CC={args.cc}", f"CXX={cxx}", f"{spec['var']}={flags}"],
cwd=ROOT, check=True)
raw_dir = os.path.join(out, f"{args.lane}-profraw")
shutil.rmtree(raw_dir, ignore_errors=True)
os.makedirs(raw_dir)
env = dict(os.environ)
env.update(spec["env"])
env["LLVM_PROFILE_FILE"] = os.path.join(raw_dir, "%p-%m.profraw")
runner = os.path.join(ROOT, bdir, spec["target"])
rc = subprocess.run([runner] + args.suites, cwd=ROOT, env=env, check=False).returncode
raws = [os.path.join(raw_dir, f) for f in os.listdir(raw_dir) if f.endswith(".profraw")]
if not raws:
sys.exit("sanitizer-coverage-map: the runner wrote no profile")
profdata = find_llvm_tool("llvm-profdata", args.cc)
cov = find_llvm_tool("llvm-cov", args.cc)
merged = os.path.join(out, f"{args.lane}.profdata")
subprocess.run([profdata, "merge", "-sparse", "-o", merged] + raws, check=True)
with open(os.path.join(out, f"{args.lane}.lcov"), "w") as fh:
subprocess.run([cov, "export", "-format=lcov", "-skip-expansions", f"-instr-profile={merged}", runner],
cwd=ROOT, stdout=fh, check=True)
print(f"{args.lane}: runner rc={rc}, {len(raws)} profile(s) -> {out}/{args.lane}.lcov")
return 0 if rc == 0 else rc
def read_lcov(path: str) -> dict[tuple[str, str], int]:
counts: dict[tuple[str, str], int] = {}
src = None
for line in open(path, encoding="utf-8", errors="replace"):
if line.startswith("SF:"):
src = os.path.relpath(line[3:].strip(), ROOT)
elif line.startswith("FNDA:") and src:
count, name = line[5:].strip().split(",", 1)
key = (src, name)
counts[key] = max(counts.get(key, 0), int(count))
return counts
def cmd_union(args) -> int:
out = os.path.abspath(args.out)
lanes = [f[:-5] for f in sorted(os.listdir(out)) if f.endswith(".lcov")]
if not lanes:
sys.exit(f"sanitizer-coverage-map: no <lane>.lcov under {out} (run `run` first)")
total: dict[tuple[str, str], int] = {}
for lane in lanes:
for key, c in read_lcov(os.path.join(out, f"{lane}.lcov")).items():
total[key] = max(total.get(key, 0), c)
never = sorted(k for k, c in total.items() if c == 0 and not k[0].startswith(("tests/", "..")))
with open(os.path.join(out, "never-executed.txt"), "w") as fh:
for src, fn in never:
fh.write(f"{src}\t{fn}\n")
by_file = defaultdict(int)
for src, _ in never:
by_file[src] += 1
functions = sum(1 for k in total if not k[0].startswith(("tests/", "..")))
print(f"lanes merged: {', '.join(lanes)}")
print(f"functions: {functions:,} never executed under any lane: {len(never):,} "
f"(vendored: {sum(1 for s, _ in never if is_vendored(s)):,})")
for src, n in sorted(by_file.items(), key=lambda kv: (-kv[1], kv[0]))[: args.top]:
print(f" {n:>6} {src}{' [vendored]' if is_vendored(src) else ''}")
print(f"list: {out}/never-executed.txt")
return 0
def cmd_ratchet(args) -> int:
current_path = os.path.join(os.path.abspath(args.out), "never-executed.txt")
current = set(open(current_path).read().splitlines())
if not os.path.exists(BASELINE):
print(f"no baseline yet: copy {current_path} to {BASELINE} to arm the ratchet")
return 0
base = set(l for l in open(BASELINE).read().splitlines() if l and not l.startswith("#"))
new = sorted(current - base)
gone = sorted(base - current)
for line in new:
print(f"NEW never-executed: {line}")
if gone:
print(f"{len(gone)} function(s) now executed: lower the baseline (remove them from {BASELINE})")
return 1 if new else 0
def cmd_map(args) -> int:
data = json.load(open(MAP_FILE))
platforms = data["platforms"]
bad = []
for cls in data["classes"]:
cells = cls["cells"]
for plat in platforms:
cell = cells.get(plat)
if not cell:
bad.append(f"{cls['name']} / {plat}: missing")
elif cell.get("status") not in ("covered", "partial", "limit"):
bad.append(f"{cls['name']} / {plat}: status must be covered, partial or limit")
elif cell["status"] != "covered" and not cell.get("why"):
bad.append(f"{cls['name']} / {plat}: {cell['status']} needs a why")
if not args.quiet:
width = max(len(c["name"]) for c in data["classes"])
print(f"{'defect class':<{width}} " + " ".join(f"{p:<14}" for p in platforms))
for cls in data["classes"]:
row = []
for plat in platforms:
cell = cls["cells"].get(plat, {})
row.append(f"{cell.get('status', '?') + ':' + cell.get('lane', ''):<14}"[:14])
print(f"{cls['name']:<{width}} " + " ".join(row))
print()
for cls in data["classes"]:
for plat in platforms:
cell = cls["cells"].get(plat, {})
if cell.get("status") in ("partial", "limit"):
print(f" {cell['status']:<7} {cls['name']} / {plat}: {cell.get('why', '')}")
for b in bad:
print(f"MAP DEFECT: {b}", file=sys.stderr)
return 1 if bad else 0
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = ap.add_subparsers(dest="cmd", required=True)
s = sub.add_parser("static")
s.add_argument("--json", action="store_true")
r = sub.add_parser("run")
r.add_argument("--lane", choices=sorted(LANES), required=True)
r.add_argument("--out", default="build/covmap")
r.add_argument("--cc", default="clang")
r.add_argument("suites", nargs="*")
u = sub.add_parser("union")
u.add_argument("--out", default="build/covmap")
u.add_argument("--top", type=int, default=40)
t = sub.add_parser("ratchet")
t.add_argument("--out", default="build/covmap")
m = sub.add_parser("map")
m.add_argument("--quiet", action="store_true")
args = ap.parse_args()
return {"static": cmd_static, "run": cmd_run, "union": cmd_union,
"ratchet": cmd_ratchet, "map": cmd_map}[args.cmd](args)
if __name__ == "__main__":
sys.exit(main())
scripts/sanitizer-map.json
+126
View File
@@ -0,0 +1,126 @@
{
"about": "Defect class x platform coverage of the sanitizer lanes. Every cell is covered, partial (with why), or a named limit (with why). Validated by scripts/sanitizer-coverage-map.py map. Which functions each lane actually EXECUTES is the run/union/ratchet part of the same tool.",
"platforms": ["linux-x64", "linux-arm64", "macos-arm64", "windows-x64", "windows-arm64"],
"classes": [
{
"name": "heap/stack memory errors",
"cells": {
"linux-x64": {"status": "covered", "lane": "asan", "venue": "PR CI"},
"linux-arm64": {"status": "covered", "lane": "asan", "venue": "local container"},
"macos-arm64": {"status": "partial", "lane": "asan+diag", "why": "use-after-return and use-after-scope only on the Homebrew-LLVM diag lane, which is local-only"},
"windows-x64": {"status": "covered", "lane": "asan", "venue": "PR CI"},
"windows-arm64": {"status": "limit", "lane": "none", "why": "no ASan runtime for Windows ARM64; the class is covered for Windows by the x64 leg"}
}
},
{
"name": "allocator-path memory errors",
"cells": {
"linux-x64": {"status": "partial", "lane": "asan", "why": "test builds use the C runtime allocator: mimalloc bindings, the --wrap shims and the slab never run sanitized; mimalloc itself is built without ASan"},
"linux-arm64": {"status": "partial", "lane": "asan", "why": "same as linux-x64"},
"macos-arm64": {"status": "partial", "lane": "asan", "why": "same as linux-x64; the zone-observer path runs only in the memwaste flavours"},
"windows-x64": {"status": "partial", "lane": "asan", "why": "same as linux-x64"},
"windows-arm64": {"status": "limit", "lane": "none", "why": "no ASan runtime for Windows ARM64"}
}
},
{
"name": "undefined behaviour",
"cells": {
"linux-x64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},
"linux-arm64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},
"macos-arm64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},
"windows-x64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},
"windows-arm64": {"status": "partial", "lane": "ubsan-trap", "why": "trap mode: a violation stops the run without a message"}
}
},
{
"name": "leaks",
"cells": {
"linux-x64": {"status": "covered", "lane": "lsan", "venue": "PR CI (on by default under ASan)"},
"linux-arm64": {"status": "covered", "lane": "lsan", "venue": "local container"},
"macos-arm64": {"status": "covered", "lane": "test-lsan", "venue": "local (Homebrew LLVM; Apple clang refuses detect_leaks)"},
"windows-x64": {"status": "partial", "lane": "memwaste-event", "why": "no leak sanitizer on Windows; the waste lane's live-per-site report at the last phase and the soak slope stand in"},
"windows-arm64": {"status": "partial", "lane": "memwaste-event", "why": "no leak sanitizer on Windows; the waste lane's live-per-site report at the last phase and the soak slope stand in"}
}
},
{
"name": "uninitialised reads",
"cells": {
"linux-x64": {"status": "partial", "lane": "msan", "why": "CI only; deep-recursion suites need larger thread stacks under MSan"},
"linux-arm64": {"status": "partial", "lane": "msan", "why": "locally red: three deep-recursion suites overflow their thread stacks"},
"macos-arm64": {"status": "partial", "lane": "memwaste-access", "why": "no MSan on macOS; the access lane's read-before-write verdict on a real corpus is the second net, per block, not per byte"},
"windows-x64": {"status": "partial", "lane": "memwaste-access", "why": "no MSan on Windows; the access lane (clang) is the second net"},
"windows-arm64": {"status": "partial", "lane": "memwaste-access", "why": "no MSan on Windows; the access lane (clang) is the second net"}
}
},
{
"name": "data races",
"cells": {
"linux-x64": {"status": "partial", "lane": "tsan", "why": "a subset of suites runs under TSan"},
"linux-arm64": {"status": "partial", "lane": "tsan", "why": "a subset of suites runs under TSan"},
"macos-arm64": {"status": "partial", "lane": "tsan", "why": "a subset of suites runs under TSan"},
"windows-x64": {"status": "limit", "lane": "none", "why": "no TSan on Windows"},
"windows-arm64": {"status": "limit", "lane": "none", "why": "no TSan on Windows"}
}
},
{
"name": "static analysis",
"cells": {
"linux-x64": {"status": "partial", "lane": "cppcheck+lint-mem+codeql", "why": "vendored trees are filtered out of cppcheck and the analyzer gate"},
"linux-arm64": {"status": "partial", "lane": "cppcheck+lint-mem", "why": "vendored trees are filtered out"},
"macos-arm64": {"status": "partial", "lane": "lint-mem", "why": "vendored trees are filtered out"},
"windows-x64": {"status": "limit", "lane": "none", "why": "static lanes are platform-independent source passes; Windows-only code is analysed by CodeQL only"},
"windows-arm64": {"status": "limit", "lane": "none", "why": "static lanes are platform-independent source passes"}
}
},
{
"name": "hostile input",
"cells": {
"linux-x64": {"status": "partial", "lane": "security-fuzz+libfuzzer", "why": "fixed MCP payloads in CI; libFuzzer targets for extractors, Cypher and config editors run in dry-run/nightly"},
"linux-arm64": {"status": "partial", "lane": "libfuzzer", "why": "local only"},
"macos-arm64": {"status": "partial", "lane": "libfuzzer", "why": "local only (Homebrew LLVM ships libFuzzer; Apple clang does not)"},
"windows-x64": {"status": "limit", "lane": "none", "why": "libFuzzer is not built for the MinGW toolchain"},
"windows-arm64": {"status": "limit", "lane": "none", "why": "libFuzzer is not built for the MinGW toolchain"}
}
},
{
"name": "algorithmic blow-up",
"cells": {
"linux-x64": {"status": "covered", "lane": "complexity+memwaste-scaling"},
"linux-arm64": {"status": "covered", "lane": "complexity+memwaste-scaling"},
"macos-arm64": {"status": "covered", "lane": "complexity+memwaste-scaling"},
"windows-x64": {"status": "partial", "lane": "complexity", "why": "the per-function scaling lane needs clang's profile runtime; whole-pipeline counters only"},
"windows-arm64": {"status": "partial", "lane": "complexity", "why": "the per-function scaling lane needs clang's profile runtime; whole-pipeline counters only"}
}
},
{
"name": "memory waste",
"cells": {
"linux-x64": {"status": "covered", "lane": "memwaste-event+access"},
"linux-arm64": {"status": "covered", "lane": "memwaste-event+access"},
"macos-arm64": {"status": "covered", "lane": "memwaste-event+access"},
"windows-x64": {"status": "partial", "lane": "memwaste-event", "why": "libc wrappers cover mem*/str* and FILE I/O only; the access lane needs a clang build"},
"windows-arm64": {"status": "partial", "lane": "memwaste-event", "why": "libc wrappers cover mem*/str* and FILE I/O only"}
}
},
{
"name": "CPU waste",
"cells": {
"linux-x64": {"status": "covered", "lane": "memwaste-cpu+scaling"},
"linux-arm64": {"status": "covered", "lane": "memwaste-cpu+scaling"},
"macos-arm64": {"status": "covered", "lane": "memwaste-cpu+scaling"},
"windows-x64": {"status": "partial", "lane": "memwaste-cpu", "why": "no system-call or pthread lock wrappers on Windows: Win32 file and lock APIs are imports --wrap cannot redirect"},
"windows-arm64": {"status": "partial", "lane": "memwaste-cpu", "why": "no system-call or pthread lock wrappers on Windows: Win32 file and lock APIs are imports --wrap cannot redirect"}
}
},
{
"name": "executed-code coverage of the lanes",
"cells": {
"linux-x64": {"status": "partial", "lane": "sanitizer-coverage-map", "why": "needs a clang job; the gcc legs are not measured"},
"linux-arm64": {"status": "covered", "lane": "sanitizer-coverage-map"},
"macos-arm64": {"status": "covered", "lane": "sanitizer-coverage-map"},
"windows-x64": {"status": "limit", "lane": "none", "why": "no sanitizer lane on Windows executes under clang source coverage yet"},
"windows-arm64": {"status": "limit", "lane": "none", "why": "no sanitizer lane on Windows executes under clang source coverage yet"}
}
}
]
}
scripts/smoke-invariants.sh
+21 -7
View File
@@ -520,10 +520,10 @@ inv_mcp_initialize() {
return 0
}
# ── Invariant 4: tools/list returns all expected tools ─────────────────────
# Cross-check against the canonical 14-tool list (TOOLS[] in src/mcp/mcp.c).
EXPECTED_TOOLS="index_repository search_graph query_graph trace_path get_code_snippet get_graph_schema get_architecture search_code list_projects delete_project index_status detect_changes manage_adr ingest_traces"
EXPECTED_TOOL_COUNT=14
# ── Invariant 4: tools/list returns exactly the expected tools ─────────────
# Cross-check against the canonical registry (TOOLS[] in src/mcp/mcp.c).
EXPECTED_TOOLS="index_repository search_graph query_graph trace_path get_code_snippet get_file_outline get_graph_schema compare_graphs get_architecture search_code list_projects delete_project index_status check_index_coverage detect_changes manage_adr ingest_traces"
EXPECTED_TOOL_COUNT="$(printf '%s\n' $EXPECTED_TOOLS | grep -c .)"
inv_tools_list() {
if ! mcp_alive; then
fail "tools-list" "server not alive"
@@ -573,7 +573,7 @@ print(r.get("nextCursor") or "")' 2>/dev/null)"
fail "tools-list" "got $got_count tools across $page page(s), expected $EXPECTED_TOOL_COUNT; names=[$got_names]"
return
fi
local missing=""
local missing="" extra="" duplicates=""
local t
for t in $EXPECTED_TOOLS; do
case " $got_names " in
@@ -581,10 +581,21 @@ print(r.get("nextCursor") or "")' 2>/dev/null)"
*) missing="$missing $t" ;;
esac
done
for t in $got_names; do
case " $EXPECTED_TOOLS " in
*" $t "*) ;;
*) extra="$extra $t" ;;
esac
done
duplicates="$(printf '%s\n' $got_names | sort | uniq -d | tr '\n' ' ')"
if [ -n "$missing" ]; then
fail "tools-list" "missing tools:$missing"
elif [ -n "$extra" ]; then
fail "tools-list" "unexpected tools:$extra"
elif [ -n "$duplicates" ]; then
fail "tools-list" "duplicate tools: $duplicates"
else
pass "tools-list (all $EXPECTED_TOOL_COUNT tools present)"
pass "tools-list (exactly $EXPECTED_TOOL_COUNT unique tools present)"
fi
}
@@ -626,7 +637,7 @@ inv_every_tool() {
return
fi
# name|minimal-args (JSON object) for the remaining 13 tools.
# name|minimal-args (JSON object) for the remaining 16 tools.
# Args chosen to be minimally valid per TOOLS[] required fields.
local p="$PROJ_NAME"
local -a CALLS
@@ -635,11 +646,14 @@ inv_every_tool() {
"query_graph|{\"project\":\"$p\",\"query\":\"MATCH (n) RETURN n.name LIMIT 5\"}"
"trace_path|{\"project\":\"$p\",\"function_name\":\"compute\",\"direction\":\"both\"}"
"get_code_snippet|{\"project\":\"$p\",\"qualified_name\":\"compute\"}"
"get_file_outline|{\"project\":\"$p\",\"file_path\":\"src/main.py\"}"
"get_graph_schema|{\"project\":\"$p\"}"
"compare_graphs|{\"base_project\":\"$p\",\"target_project\":\"$p\"}"
"get_architecture|{\"project\":\"$p\"}"
"search_code|{\"project\":\"$p\",\"pattern\":\"def \"}"
"list_projects|{}"
"index_status|{\"project\":\"$p\"}"
"check_index_coverage|{\"project\":\"$p\",\"paths\":[\"src/main.py\"]}"
"detect_changes|{\"project\":\"$p\"}"
"manage_adr|{\"project\":\"$p\",\"mode\":\"get\"}"
"ingest_traces|{\"project\":\"$p\",\"traces\":[]}"
scripts/smoke-test.sh
+47 -3
View File
@@ -22,6 +22,8 @@ they stage the release fixture, start the fixture server, and sandbox
HOME/TEMP/agent-config destinations. Called bare, the download/checksum/
install-script phases (12-13) SKIP for lack of a fixture server, and the run
mutates the REAL profile — the venue-parity contract forbids that in any venue.
The daemon runtime and cache are private to the run either way: every product
process is started under a CBM_RUNTIME_DIR/CBM_CACHE_DIR this harness owns.
Arguments:
<binary-path> product binary to smoke
@@ -41,6 +43,20 @@ if [ -n "$SMOKE_MODE" ] && [ "$SMOKE_MODE" != "--agent-config-only" ]; then
fi
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)"
# Every product process below — the phases, the install/update E2E and the
# daemon retirements — must reach a daemon rendezvous this run owns. Only
# CBM_RUNTIME_DIR moves that rendezvous; the wrappers' HOME/TMPDIR/CBM_CACHE_DIR
# sandbox does not, so without this the retirements land on the operator's live
# account daemon (#1691, #1696).
# shellcheck source=test-runtime.sh
source "$REPO_ROOT/scripts/test-runtime.sh"
cbm_test_runtime_init
# Armed here rather than only with the fixture trap below: the fixture mktemp
# and its cygpath conversion sit between the two, and under `set -e` a failure
# there would otherwise leave the private root behind. The fixture trap
# replaces this one and keeps the same cleanup as its first step.
trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
smoke_mktemp_file() {
if [ -n "${SMOKE_TEMP_ROOT:-}" ]; then
mktemp "$SMOKE_TEMP_ROOT/cbm-smoke.XXXXXX"
@@ -100,8 +116,8 @@ copy_smoke_binary() {
cp "$BINARY" "$destination"
}
# Retire the shared account daemon (if one is running) and wait until it
# reports not-running. Install/uninstall flows leave an ephemeral daemon
# Retire this run's private account daemon (if one is running) and wait until
# it reports not-running. Install/uninstall flows leave an ephemeral daemon
# draining asynchronously whose mapped generation backing and open logs
# block rm on Windows (POSIX rm doesn't care) — so every cleanup of a
# fixture HOME that received an install, and the final cache removal, must
@@ -149,7 +165,11 @@ CODEX_LIFECYCLE_HOME=""
if command -v cygpath &>/dev/null; then
TMPDIR=$(cygpath -m "$TMPDIR")
fi
trap 'smoke_rmtree "$TMPDIR" "${DRYRUN_HOME:-}" "${CODEX_LIFECYCLE_HOME:-}"' EXIT
# Runtime cleanup first, so no earlier cleanup step stands between the exit
# and the private daemon's retirement; on Windows that retirement is also what
# unblocks the fixture rm (mapped binary, open logs). smoke_rmtree never fails,
# so the fixture removal still runs after it.
trap 'cbm_test_runtime_cleanup "$BINARY"; smoke_rmtree "$TMPDIR" "${DRYRUN_HOME:-}" "${CODEX_LIFECYCLE_HOME:-}"' EXIT
CLI_STDERR=$(smoke_mktemp_file)
# 10 of the cli call sites assign directly (VAR=$(cli ...)). Under
@@ -174,6 +194,29 @@ cli() {
return "$rc"
}
# A worker failure says "inspect log: <path>" — and in CI that path dies with the
# job's sandbox, so the one artifact naming the cause is the one nobody can open.
# A worker killed by a signal writes no summary of its own either, which is
# exactly the case that most needs the log (PR #2233: "index worker ended with
# killed (exit=-1, signal=9)" on ubuntu-latest, unreproducible on every local
# venue). Print it while it still exists.
smoke_dump_worker_log() {
local log
log=$(sed -n 's/.*inspect log: \([^ ]*\).*/\1/p' "$CLI_STDERR" 2>/dev/null | tail -1)
if [ -z "$log" ] || [ ! -f "$log" ]; then
# The path is only printed for some failures; fall back to the newest log
# the run produced.
log=$(ls -t "${CBM_CACHE_DIR:-$HOME/.cache/codebase-memory-mcp}"/logs/.worker-log-* 2>/dev/null | head -1)
fi
if [ -n "$log" ] && [ -f "$log" ]; then
echo "--- worker log: $log ---"
tail -80 "$log"
echo "--- end worker log ---"
else
echo "--- no worker log found (cache ${CBM_CACHE_DIR:-unset}) ---"
fi
}
echo "=== Phase 1: version ==="
VERSION_STATUS=0
OUTPUT=$("$BINARY" --version 2>&1) || VERSION_STATUS=$?
@@ -334,6 +377,7 @@ GENEOF
if ! RESULT=$(cli index_repository --repo-path "$TMPDIR"); then
echo "FAIL: index_repository (flag form) exited non-zero"
cat "$CLI_STDERR"
smoke_dump_worker_log
exit 1
fi
echo "$RESULT"
scripts/soak-test.sh
+20 -7
View File
@@ -66,9 +66,17 @@ CBM_SOAK_MODE="${CBM_SOAK_MODE:-default}"
RESULTS_DIR="${RESULTS_DIR:-soak-results}"
mkdir -p "$RESULTS_DIR"
# Isolate daemon coordination from interactive CBM sessions and give this run
# a deterministic host-side daemon log. Wine needs a Windows-form cache path
# in the child environment while this Bash harness retains the host path.
# Every product process below must reach a daemon rendezvous this run owns.
# Only CBM_RUNTIME_DIR moves that rendezvous — a private CBM_CACHE_DIR alone
# still shares the operator's account daemon (#1691, #1696).
# shellcheck source=test-runtime.sh
source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
cbm_test_runtime_init
trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
# Give this run a deterministic host-side daemon log. Wine needs a Windows-form
# cache path in the child environment while this Bash harness retains the host
# path.
#
# On native Windows the cache CANNOT live under msys /tmp: the server's
# cache-private executable-identity check walks the cache path's ancestors
@@ -146,7 +154,7 @@ if [[ "$BINARY" == *.exe ]] && command -v cygpath >/dev/null 2>&1 &&
exit 1
fi
else
SOAK_CACHE_DIR_HOST=$(mktemp -d "${TMPDIR:-/tmp}/cbm-soak-cache.XXXXXX")
SOAK_CACHE_DIR_HOST="$CBM_TEST_CACHE_DIR_HOST"
fi
SOAK_CACHE_DIR_VALUE="$SOAK_CACHE_DIR_HOST"
if [[ "$BINARY" == *.exe ]] && command -v winepath >/dev/null 2>&1; then
@@ -206,7 +214,12 @@ soak_cleanup() {
if [ -f "$DAEMON_LOG" ]; then
cp "$DAEMON_LOG" "$RESULTS_DIR/cbm-daemon.log" 2>/dev/null || true
fi
rm -rf -- "$SOAK_PROJECT" "$SOAK_CACHE_DIR_HOST"
rm -rf -- "$SOAK_PROJECT"
# The helper stops this run's private daemon before removing its root and
# leaves the root behind for diagnosis when the daemon will not stop. The
# native-Windows cache sits under SOAK_WIN_ROOT, which goes only after the
# daemon check because the binary it probes with is the copy inside it.
cbm_test_runtime_cleanup "$BINARY"
[ -z "${SOAK_WIN_ROOT:-}" ] || rm -rf -- "$SOAK_WIN_ROOT"
}
@@ -807,7 +820,7 @@ if [ "$SKIP_CRASH" != "--skip-crash-test" ] && [ "$CBM_SOAK_MODE" != "query-leak
SERVER_PID=""
exec 3>&- 4<&-
if ! wait_for_daemon_stop "$DAEMON_STOP_COUNT"; then
echo "FAIL: last-session crash did not stop the shared daemon"
echo "FAIL: last-session crash did not stop the private daemon"
exit 1
fi
@@ -848,7 +861,7 @@ kill "$SERVER_PID" 2>/dev/null || true
wait "$SERVER_PID" 2>/dev/null || true
SERVER_PID=""
if ! wait_for_daemon_stop "$FINAL_DAEMON_STOP_COUNT"; then
echo "FAIL: final frontend shutdown did not stop the shared daemon"
echo "FAIL: final frontend shutdown did not stop the private daemon"
PASS=false
fi
scripts/test-runtime.sh
+10
View File
@@ -105,6 +105,16 @@ _cbm_test_runtime_daemon() {
CBM_CACHE_DIR="$_CBM_TEST_RUNTIME_PRODUCT_CACHE" "$1" daemon "$2"
}
# The supervisor resolves one resource policy per index and hands it to the
# worker in argv; a worker that finds no complete policy refuses to start rather
# than index unbounded. A shell test that spawns `cli --index-worker` itself
# stands in for the supervisor and owes the worker the same object. Mirrors
# cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and
# nothing else.
cbm_test_index_worker_policy_json() {
printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}'
}
cbm_test_runtime_cleanup() {
local binary="${1:-}" root="$_CBM_TEST_RUNTIME_CREATED_ROOT"
local name="${_CBM_TEST_RUNTIME_CREATED_ROOT##*/}" runtime_entry="" active=0
scripts/test.sh
+38
View File
@@ -212,15 +212,27 @@ bash "$ROOT/tests/test_ui_dev_proxy_security.sh"
echo "=== Step 0d: daemon soak recovery contract ==="
bash "$ROOT/tests/test_soak_daemon_recovery_contract.sh"
echo "=== Step 0d2: soak harness runtime isolation contract (#1696) ==="
bash "$ROOT/tests/test_soak_runtime_isolation_contract.sh"
echo "=== Step 0e: Windows launcher bundle contract ==="
bash "$ROOT/tests/test_windows_bundle_contract.sh"
echo "=== Step 0e2: activation refusal diagnostic contract ==="
bash "$ROOT/tests/test_activation_diagnostic_contract.sh"
echo "=== Step 0e3: security gate fail-closed contract ==="
bash "$ROOT/tests/test_security_gate_fail_closed.sh"
echo "=== Step 0f: tree-sitter runtime Makefile dependencies ==="
bash "$ROOT/tests/test_makefile_ts_runtime_dependencies.sh"
echo "=== Step 0g: security fuzz harness self-test ==="
bash "$ROOT/tests/test_security_fuzz_harness.sh"
echo "=== Step 0g2: memlab harness runtime isolation contract (#1696) ==="
bash "$ROOT/tests/test_memlab_runtime_isolation_contract.sh"
echo "=== Step 0h: smoke release-fixture contract ==="
bash "$ROOT/tests/test_smoke_fixture_contract.sh"
@@ -260,6 +272,9 @@ bash "$ROOT/tests/test_release_gate_chain_contract.sh"
echo "=== Step 0t: test runtime isolation contract (#1691) ==="
bash "$ROOT/tests/test_runtime_isolation_contract.sh"
echo "=== Step 0t2: smoke harness runtime isolation contract (#1696) ==="
bash "$ROOT/tests/test_smoke_runtime_isolation_contract.sh"
echo "=== Step 0u: shell line-ending contract ==="
bash "$ROOT/tests/test_shell_line_endings.sh"
@@ -272,6 +287,14 @@ bash "$ROOT/tests/test_language_count_contract.sh"
echo "=== Step 0x: packaging version-metadata contract ==="
bash "$ROOT/tests/test_version_metadata_contract.sh"
# Step 0y: the Windows leg must not decide its verdict from an exit status that
# the ssh/msys2_shell chain can mangle — a channel that turns 0 into 1 can turn
# 1 into 0, and that direction reports a RED Windows leg as green. Runs
# everywhere (it drives synthetic logs, no VM needed) because the guard it pins
# is what every Windows verdict rests on.
echo "=== Step 0y: VM leg verdict contract ==="
bash "$ROOT/tests/test_vm_verdict_contract.sh"
# Verify compiler supports target arch
verify_compiler "$CC"
@@ -312,6 +335,15 @@ make -j"$NPROC" -f Makefile.cbm cbm TEST_SEAMS=1 ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}
WATCHDOG_BINARY="$ROOT/$BUILD_DIR/codebase-memory-mcp"
CBM_TEST_BINARY="$WATCHDOG_BINARY" bash "$ROOT/tests/test_parent_watchdog.sh"
# Step 5a: that watchdog is also the SMALLEST-stack thread in the image, which
# makes it the first casualty when static TLS grows — glibc takes the TLS block
# out of each thread's own stack allocation. Checked here, against the binary
# Step 5 just built, because the failure it prevents surfaces nowhere near its
# cause (PR #2233: a thread-local cache in an extraction file stopped the index
# worker from starting, on x86-64 only).
echo "=== Step 5a: static-TLS budget against the smallest thread stack ==="
bash "$ROOT/tests/test_thread_stack_tls_contract.sh" "$WATCHDOG_BINARY"
# Step 5b: worker-mode parent-death watchdog (#845). A supervised index worker
# (`cli --index-worker …`) whose supervisor dies must self-exit instead of
# indexing on as an orphan. Reuses the prod binary built in Step 5.
@@ -346,6 +378,12 @@ CBM_TEST_BINARY="$WATCHDOG_BINARY" bash "$ROOT/tests/test_worker_error_response.
echo "=== Step 5e: watcher_enabled kill-switch regression (#335) ==="
CBM_TEST_BINARY="$WATCHDOG_BINARY" bash "$ROOT/tests/test_watcher_disabled.sh"
# Step 5f: a supervised worker is scoped to the request the daemon admitted,
# never to the CBM_ALLOWED_ROOT it inherited from the daemon starter's
# environment. Reuses the prod binary built in Step 5.
echo "=== Step 5f: worker request-scope regression ==="
CBM_TEST_BINARY="$WATCHDOG_BINARY" bash "$ROOT/tests/test_worker_session_scope.sh"
# Step 6: security-strings URL allow-list regression. The MSYS2 CLANG64 toolchain
# bakes its package-tracker URL into the static Windows .exe; the binary string
# audit must allow-list it (Windows-only — Linux smoke never saw it).
scripts/vendored-checksums.txt
+2 -2
View File
@@ -5,7 +5,7 @@ c5cfb43042b6b72045f4ba997834d0a7786d2793d91680868b5815b39f14fc78 internal/cbm/v
b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/common/tree_sitter/alloc.h
31e60a1bff6f715afacce03b5b70efe42b58371b4f9595dd4af52a577ff9608c internal/cbm/vendored/common/tree_sitter/array.h
180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/common/tree_sitter/parser.h
00a84162eb34fb58bcdd38c08922d4fea4c87b7c2219dd64b773dd15ba18de71 internal/cbm/vendored/grammars/MANIFEST.md
a66a04ed735fdb3d4a90ee4353b8f0490dc0360da5965b0d13ec5d6766f146fa internal/cbm/vendored/grammars/MANIFEST.md
ad8425038de519f8c4e3e9339feebf99dfad8a6002dcf79227d91402780a32cc internal/cbm/vendored/grammars/ada/LICENSE
02805ec13939b749c891567be36bf024b09034e04c80683a1ae667272458d549 internal/cbm/vendored/grammars/ada/parser.c
115a75d000bef9c70c4de7dfbb7f2a80b90cbb14ba265056dc7abb1ef3b9b6db internal/cbm/vendored/grammars/ada/tree_sitter/alloc.h
@@ -610,7 +610,7 @@ fbce36bb615a015ea545373da62c178fe9e7e3a7b7de656455738e6f671aa652 internal/cbm/v
c4b9482069f61a2a26a590181baa34f08eb6dcc57a138def578e6f9720c35ea5 internal/cbm/vendored/grammars/prisma/tree_sitter/parser.h
3297073719030b7261afd78b7267931ccc03a27f7a65ca6fc644316180f71bf6 internal/cbm/vendored/grammars/properties/LICENSE
98e06c3a7b3043d7641ec0b2c28c9f66e54341e343e1f553f994ae5a6f0c0452 internal/cbm/vendored/grammars/properties/parser.c
3c00e7a8ec38662b05cb00858d2959b123b2b5c783eb3dc25c80b7577b4c27ac internal/cbm/vendored/grammars/properties/scanner.c
a313549615512cf06b2ee55b080bfee1559860aaa1dfcab56e9af6010323aad8 internal/cbm/vendored/grammars/properties/scanner.c
115a75d000bef9c70c4de7dfbb7f2a80b90cbb14ba265056dc7abb1ef3b9b6db internal/cbm/vendored/grammars/properties/tree_sitter/alloc.h
02d139d44303b0b0b85a899839dbc1346c478e7318d1f2971fad2b38fd57cb73 internal/cbm/vendored/grammars/properties/tree_sitter/array.h
c4b9482069f61a2a26a590181baa34f08eb6dcc57a138def578e6f9720c35ea5 internal/cbm/vendored/grammars/properties/tree_sitter/parser.h
server.json
+3 -3
View File
@@ -8,12 +8,12 @@
"source": "github"
},
"websiteUrl": "https://deusdata.github.io/codebase-memory-mcp/",
"version": "0.10.8",
"version": "0.11.0",
"packages": [
{
"registryType": "npm",
"identifier": "codebase-memory-mcp",
"version": "0.10.8",
"version": "0.11.0",
"runtimeHint": "npx",
"transport": {
"type": "stdio"
@@ -22,7 +22,7 @@
{
"registryType": "pypi",
"identifier": "codebase-memory-mcp",
"version": "0.10.8",
"version": "0.11.0",
"runtimeHint": "uvx",
"transport": {
"type": "stdio"
src/cli/activation_transaction.c
+133 -8
View File
@@ -18,6 +18,7 @@
#define WIN32_LEAN_AND_MEAN
#endif
#include <aclapi.h>
#include <ntsecapi.h>
#include <sddl.h>
#include <windows.h>
#else
@@ -424,6 +425,102 @@ static void activation_windows_security_destroy(activation_windows_security_t *s
memset(security, 0, sizeof(*security));
}
/* #1705: THIS machine's built-in Administrator ACCOUNT (RID 500 under the local
* machine account-domain SID, S-1-5-21-<machine>-500) is a trusted owner/grantee,
* mirroring the daemon's win_sid_trusted (src/daemon/ipc.c). It is resolved via
* LSA (the local machine account-domain SID) plus CreateWellKnownSid and compared
* with EqualSid. It is deliberately NOT IsWellKnownSid(WinAccountAdministratorSid)
* and NOT a trailing-RID-500 test: both accept a FOREIGN S-1-5-21-*-500 (a domain
* admin, or another machine's built-in Administrator), opening a cross-machine
* bypass. Resolved once and cached for the process; any LSA or synthesis failure
* caches "none" and grants NO tolerance (fail closed). advapi32 is already loaded
* (this file calls GetSecurityInfo etc.), so the functions are resolved from its
* module handle with no new import. CLI activation is single-threaded by
* contract, so the cache needs no lock. */
typedef NTSTATUS(NTAPI *activation_lsa_open_policy_fn)(PLSA_UNICODE_STRING, PLSA_OBJECT_ATTRIBUTES,
ACCESS_MASK, PLSA_HANDLE);
typedef NTSTATUS(NTAPI *activation_lsa_query_information_policy_fn)(LSA_HANDLE,
POLICY_INFORMATION_CLASS,
PVOID *);
typedef NTSTATUS(NTAPI *activation_lsa_free_memory_fn)(PVOID);
typedef NTSTATUS(NTAPI *activation_lsa_close_fn)(LSA_HANDLE);
typedef BOOL(WINAPI *activation_create_well_known_sid_fn)(WELL_KNOWN_SID_TYPE, PSID, PSID, DWORD *);
static PSID activation_windows_local_admin_sid(void) {
static bool resolved = false;
static PSID cached = NULL;
if (resolved) {
return cached;
}
resolved = true;
HMODULE advapi = GetModuleHandleW(L"advapi32.dll");
if (!advapi) {
return NULL;
}
activation_lsa_open_policy_fn lsa_open =
(activation_lsa_open_policy_fn)(void (*)(void))GetProcAddress(advapi, "LsaOpenPolicy");
activation_lsa_query_information_policy_fn lsa_query =
(activation_lsa_query_information_policy_fn)(void (*)(void))GetProcAddress(
advapi, "LsaQueryInformationPolicy");
activation_lsa_free_memory_fn lsa_free =
(activation_lsa_free_memory_fn)(void (*)(void))GetProcAddress(advapi, "LsaFreeMemory");
activation_lsa_close_fn lsa_close =
(activation_lsa_close_fn)(void (*)(void))GetProcAddress(advapi, "LsaClose");
activation_create_well_known_sid_fn create_sid =
(activation_create_well_known_sid_fn)(void (*)(void))GetProcAddress(advapi,
"CreateWellKnownSid");
if (!lsa_open || !lsa_query || !lsa_free || !lsa_close || !create_sid) {
return NULL;
}
LSA_OBJECT_ATTRIBUTES attributes;
memset(&attributes, 0, sizeof(attributes));
LSA_HANDLE policy = NULL;
/* STATUS_SUCCESS is 0; any other status is treated as failure (fail closed). */
if (lsa_open(NULL, &attributes, POLICY_VIEW_LOCAL_INFORMATION, &policy) != 0 || !policy) {
return NULL;
}
POLICY_ACCOUNT_DOMAIN_INFO *domain = NULL;
if (lsa_query(policy, PolicyAccountDomainInformation, (PVOID *)&domain) == 0 && domain &&
domain->DomainSid && IsValidSid(domain->DomainSid)) {
DWORD needed = 0;
(void)create_sid(WinAccountAdministratorSid, domain->DomainSid, NULL, &needed);
if (needed > 0U) {
PSID admin = malloc(needed);
if (admin &&
create_sid(WinAccountAdministratorSid, domain->DomainSid, admin, &needed) &&
IsValidSid(admin)) {
cached = admin;
} else {
free(admin);
}
}
}
if (domain) {
(void)lsa_free(domain);
}
(void)lsa_close(policy);
return cached;
}
/* #2023/#1686: an owner refusal must name WHICH owner, not just "status -3, os 0".
* The path is already carried by g_activation_refusal_object; this appends the
* offending owner's SID string, mirroring the daemon's ACL/owner diagnostics, so
* the operator sees the exact identity to remove or the directory to move. */
static void activation_windows_note_untrusted_owner(const char *predicate, PSID owner,
DWORD os_error) {
char label[192];
LPSTR owner_text = NULL;
(void)snprintf(
label, sizeof(label), "%s; owner=%s", predicate,
(owner && IsValidSid(owner) && ConvertSidToStringSidA(owner, &owner_text) && owner_text)
? owner_text
: "unresolved-sid");
if (owner_text) {
(void)LocalFree(owner_text);
}
activation_note_refusal(label, os_error);
}
/* Trusted-owner acceptance for SOURCE-side objects: a downloaded release
* bundle is owned by whatever the machine's default-owner policy dictates
* (Administrators on GitHub-runner-class images). Its integrity is enforced
@@ -440,11 +537,13 @@ static bool activation_windows_owner_is_trusted(HANDLE handle) {
PSECURITY_DESCRIPTOR descriptor = NULL;
DWORD result = GetSecurityInfo(handle, SE_FILE_OBJECT, OWNER_SECURITY_INFORMATION, &owner, NULL,
NULL, NULL, &descriptor);
PSID local_admin = activation_windows_local_admin_sid();
bool trusted = result == ERROR_SUCCESS && owner && IsValidSid(owner) &&
(EqualSid(owner, user_sid) || IsWellKnownSid(owner, WinLocalSystemSid) ||
IsWellKnownSid(owner, WinBuiltinAdministratorsSid));
IsWellKnownSid(owner, WinBuiltinAdministratorsSid) ||
(local_admin && EqualSid(owner, local_admin)));
if (!trusted) {
activation_note_refusal("owner-not-trusted", result);
activation_windows_note_untrusted_owner("owner-not-trusted", owner, result);
}
if (descriptor) {
(void)LocalFree(descriptor);
@@ -522,7 +621,7 @@ static bool activation_windows_owner_is_current(HANDLE handle) {
}
}
if (!same) {
activation_note_refusal("owner-not-current-user", result);
activation_windows_note_untrusted_owner("owner-not-current-user", owner, result);
}
if (descriptor) {
(void)LocalFree(descriptor);
@@ -596,10 +695,15 @@ static bool activation_windows_acl_check(HANDLE handle, DWORD tolerated_untruste
PSID sid = (PSID)&ace->SidStart;
size_t sid_capacity = (size_t)header->AceSize - sid_offset;
DWORD sid_length = GetSidLengthRequired(((SID *)sid)->SubAuthorityCount);
PSID local_admin = activation_windows_local_admin_sid();
bool trusted = sid_length <= sid_capacity && IsValidSid(sid) &&
GetLengthSid(sid) == sid_length &&
(EqualSid(sid, user_sid) || IsWellKnownSid(sid, WinLocalSystemSid) ||
IsWellKnownSid(sid, WinBuiltinAdministratorsSid) ||
/* #1705: THIS machine's built-in Administrator (RID-500),
* resolved via LSA — never a foreign S-1-5-21-*-500; same
* tolerance as the daemon's win_sid_trusted. */
(local_admin && EqualSid(sid, local_admin)) ||
/* OWNER RIGHTS modulates whoever owns the object; the
* owner is separately validated in every chain that
* reaches here (same tolerance as the daemon IPC and
@@ -608,9 +712,17 @@ static bool activation_windows_acl_check(HANDLE handle, DWORD tolerated_untruste
if (!trusted) {
char label[128];
LPSTR sid_text = NULL;
(void)snprintf(label, sizeof(label), "acl-grants-cross-account-mutation to %s",
/* #1856: say whether the grant is INHERITED. The remedy differs and
* the wrong one silently does nothing: `icacls <dir> /remove:g <sid>`
* cannot remove an inherited ACE -- that needs `/inheritance:r` --
* and the stock `C:\` ACE for Authenticated Users reaches every new
* child directory exactly this way. A reporter following the
* generic advice sees the command succeed and the refusal persist. */
bool inherited = (header->AceFlags & INHERITED_ACE) != 0;
(void)snprintf(label, sizeof(label), "acl-grants-cross-account-mutation to %s%s",
ConvertSidToStringSidA(sid, &sid_text) && sid_text ? sid_text
: "unparsable-sid");
: "unparsable-sid",
inherited ? " (inherited from a parent directory)" : "");
if (sid_text) {
(void)LocalFree(sid_text);
}
@@ -778,8 +890,16 @@ static bool activation_posix_acl_empty(int descriptor) {
}
static char *activation_posix_walk_path(const char *directory) {
#ifdef __APPLE__
#if defined(__APPLE__) || defined(__linux__)
/* macOS and immutable Linux layouts can expose writable trees through
* root-owned aliases (for example /tmp and /home on Atomic systems).
* Resolve only these trusted system aliases; arbitrary user symlinks must
* still fail the O_NOFOLLOW walk below. */
#ifdef __linux__
static const char *const aliases[] = {"/tmp", "/var", "/home"};
#else
static const char *const aliases[] = {"/tmp", "/var"};
#endif
for (size_t index = 0; index < sizeof(aliases) / sizeof(aliases[0]); index++) {
const char *alias = aliases[index];
size_t alias_length = strlen(alias);
@@ -789,8 +909,13 @@ static char *activation_posix_walk_path(const char *directory) {
}
struct stat alias_status;
char resolved[4096];
if (lstat(alias, &alias_status) != 0 || !S_ISLNK(alias_status.st_mode) ||
alias_status.st_uid != 0 || !realpath(alias, resolved)) {
if (lstat(alias, &alias_status) != 0) {
continue;
}
if (!S_ISLNK(alias_status.st_mode)) {
continue;
}
if (alias_status.st_uid != 0 || !realpath(alias, resolved)) {
return NULL;
}
struct stat resolved_status;
src/cli/cli.c
+587 -27
View File
@@ -8,6 +8,7 @@
#include "cli/agent_profiles.h"
#include "cli/cli.h"
#include "cli/activation_transaction.h"
#include "cli/config_edit_path.h"
#include "cli/config_json_like.h"
#include "cli/config_text_edit.h"
#include "cli/config_toml_edit.h"
@@ -214,6 +215,12 @@ typedef struct {
bool cleanup_ok;
bool original_cache_environment_present;
bool cache_environment_overridden;
/* Scope: what THIS activation replaces decides whether any session must
* be quiesced at all, and the target cache namespace decides WHICH
* cohort. A skipped coordination holds no lock and drains nobody. */
bool quiesce_required;
bool coordination_skipped;
char scope_detail[CBM_SZ_1K];
} cli_activation_production_context_t;
static cbm_cli_activation_ops_t g_cli_activation_test_ops;
@@ -239,7 +246,9 @@ static void cli_activation_diagnostic(const cbm_cli_activation_ops_t *ops, const
"error: activation was refused by a filesystem safety check before any "
"change was made: %s\n"
"error: this is not a session problem. If the flagged directory is one you "
"trust, remove the flagged permission grant (icacls <dir> /remove:g <sid>) "
"trust, remove the flagged permission grant (icacls <dir> /remove:g <sid>; "
"an INHERITED grant needs icacls <dir> /inheritance:r /grant:r "
"\"%%USERNAME%%\":(OI)(CI)F instead, because /remove:g cannot delete one) "
"or use an owner-private directory for --dir/CBM_CACHE_DIR, then retry.",
note);
diagnostic = attributed;
@@ -528,6 +537,91 @@ static void cli_activation_release_cleanup_lease(cli_activation_production_conte
}
}
typedef enum {
CLI_ACTIVATION_SCOPE_ACTIVE = 0,
CLI_ACTIVATION_SCOPE_NOTHING_TO_REPLACE,
CLI_ACTIVATION_SCOPE_FOREIGN_COHORT,
CLI_ACTIVATION_SCOPE_ERROR,
} cli_activation_scope_t;
static void cli_activation_log_guard_decision(const cli_activation_production_context_t *context,
const char *decision, const char *active_cache) {
char clients[32];
(void)snprintf(clients, sizeof(clients), "%llu",
(unsigned long long)context->daemon_result.active_clients);
const char *scope = cbm_daemon_ipc_endpoint_runtime_dir(context->endpoint);
cbm_log_info("activation.guard", "scope", scope ? scope : "", "cache",
context->cache_fingerprint, "clients", clients, "decision", decision,
"active_cache", active_cache ? active_cache : "");
}
/* Whose sessions does this activation have to stop? The rendezvous directory
* is per OS account (service.h), so the host daemon of another HOME /
* CBM_CACHE_DIR meets this activation at the very same endpoint; only the
* cache fingerprint in the cohort identity separates the namespaces. An
* `install` into a sandbox HOME used to drain the host daemon and every MCP
* client behind it although nothing it touched belonged to the host.
*
* Two questions, answered from the activation's own target: does it replace
* anything at all (a --skip-binary install without an index reset publishes
* nothing), and whose cohort is active. Admission with an immediate deadline
* is the cheapest authoritative read of the active lifetime record: OK means
* the cohort is ours or empty, CONFLICT names the active identity (its cache
* fingerprint is filled for every conflict kind), BUSY means another
* activation holds maintenance and the barrier waits for it as before. */
static cli_activation_scope_t cli_activation_resolve_scope(
cli_activation_production_context_t *context) {
const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(context->endpoint);
const char *scope = runtime_dir ? runtime_dir : "";
const char *action = cli_activation_action_text(context->action);
if (!context->quiesce_required) {
(void)snprintf(context->scope_detail, sizeof(context->scope_detail),
"nothing to quiesce: published binary and indexes unchanged; scope=%s",
scope);
cli_activation_log_guard_decision(context, "nothing_to_replace", NULL);
printf("No CBM session needs to stop for %s: the published binary and indexes are "
"unchanged.\n",
action);
(void)fflush(stdout);
return CLI_ACTIVATION_SCOPE_NOTHING_TO_REPLACE;
}
cbm_version_cohort_lease_t *lease = NULL;
cbm_daemon_conflict_t conflict;
memset(&conflict, 0, sizeof(conflict));
cbm_version_cohort_status_t status = cbm_version_cohort_acquire(
context->cohort_manager, &context->identity, cbm_now_ms(), &lease, &conflict);
cli_activation_release_cleanup_lease(context, &lease);
if (lease) {
return CLI_ACTIVATION_SCOPE_ERROR;
}
switch (status) {
case CBM_VERSION_COHORT_OK:
case CBM_VERSION_COHORT_BUSY:
return CLI_ACTIVATION_SCOPE_ACTIVE;
case CBM_VERSION_COHORT_CONFLICT:
break;
default:
return CLI_ACTIVATION_SCOPE_ERROR;
}
if (!conflict.active_cache_fingerprint[0] ||
strcmp(conflict.active_cache_fingerprint, context->cache_fingerprint) == 0) {
/* Same cache namespace, another build or version: that IS the daemon
* this activation replaces. An unreadable active cache stays in scope
* rather than silently exempting a same-namespace daemon. */
return CLI_ACTIVATION_SCOPE_ACTIVE;
}
(void)snprintf(context->scope_detail, sizeof(context->scope_detail),
"active cohort serves another cache namespace (%.12s), this %s targets "
"%.12s; scope=%s; nothing stopped",
conflict.active_cache_fingerprint, action, context->cache_fingerprint, scope);
cli_activation_log_guard_decision(context, "foreign_cohort", conflict.active_cache_fingerprint);
printf("Leaving active CBM sessions untouched: they serve another cache namespace than "
"this %s targets.\n",
action);
(void)fflush(stdout);
return CLI_ACTIVATION_SCOPE_FOREIGN_COHORT;
}
static int cli_activation_production_reserve(void *opaque, cbm_cli_activation_lock_t *lease_out) {
cli_activation_production_context_t *context = opaque;
if (lease_out) {
@@ -536,6 +630,26 @@ static int cli_activation_production_reserve(void *opaque, cbm_cli_activation_lo
if (!context || !context->cohort_manager || !lease_out) {
return CLI_ERR;
}
cli_activation_scope_t scope = cli_activation_resolve_scope(context);
if (scope == CLI_ACTIVATION_SCOPE_ERROR) {
return CLI_ERR;
}
if (scope != CLI_ACTIVATION_SCOPE_ACTIVE) {
/* Nothing in the target namespace is being replaced, or the only
* active cohort serves another namespace: hold no maintenance,
* admission, lifetime or startup lock (each wakes or blocks the other
* namespace's sessions) and send no drain request. */
if (!cli_activation_log_event(context, "quiesce_skipped", context->scope_detail)) {
return CLI_ERR;
}
context->coordination_skipped = true;
context->mutation_authorized = true;
*lease_out = context;
return 1;
}
printf("Stopping active CBM sessions and operations for %s...\n",
cli_activation_action_text(context->action));
(void)fflush(stdout);
cbm_version_cohort_quiesce_result_t quiesce = CBM_VERSION_COHORT_QUIESCE_NOT_NEEDED;
cbm_version_cohort_lease_t *lease = NULL;
context->control_deadline_ms = cli_activation_deadline_after(CLI_ACTIVATION_CONTROL_TIMEOUT_MS);
@@ -582,6 +696,8 @@ static int cli_activation_production_reserve(void *opaque, cbm_cli_activation_lo
}
context->cohort_lease = lease;
cli_activation_log_guard_decision(
context, context->shutdown_requested ? "cohort_drained" : "no_active_cohort", NULL);
if (!cli_activation_log_event(context, "daemon_stopped",
context->shutdown_requested ? "cohort drained"
: "no active cohort")) {
@@ -600,6 +716,13 @@ static void cli_activation_production_release(void *opaque, cbm_cli_activation_l
if (!context) {
return;
}
if (context->coordination_skipped) {
/* Nothing was held: the token is the context itself. */
if (lease != (cbm_cli_activation_lock_t)context) {
context->cleanup_ok = false;
}
return;
}
/* Global release order is the inverse of acquisition: startup first,
* then lifetime/admission/maintenance through the cohort lease. */
if (context->startup_lock) {
@@ -626,9 +749,11 @@ static void cli_activation_production_diagnostic(void *opaque, const char *messa
static bool cli_activation_production_context_init(cli_activation_production_context_t *context,
cbm_daemon_runtime_activation_action_t action,
const char *target_version,
const char *target_build) {
const char *target_build,
bool quiesce_required) {
memset(context, 0, sizeof(*context));
context->action = action;
context->quiesce_required = quiesce_required;
context->target_version = target_version;
context->target_build = target_build;
context->cleanup_ok = true;
@@ -735,23 +860,34 @@ static void cli_activation_production_context_close(cli_activation_production_co
context->original_cache_environment = NULL;
}
static int cli_activation_guard(cbm_daemon_runtime_activation_action_t action,
const char *target_version, const char *target_build,
cbm_cli_activation_mutation_fn mutation, void *mutation_context) {
/* quiesce_required: false when the activation publishes no binary and resets
* no index (a config-only install) — nothing running is then replaced, and
* no session is stopped for it. */
static int cli_activation_guard_scoped(cbm_daemon_runtime_activation_action_t action,
const char *target_version, const char *target_build,
bool quiesce_required,
cbm_cli_activation_mutation_fn mutation,
void *mutation_context) {
if (g_cli_activation_test_ops_set) {
return cbm_cli_activation_guard_with_ops(&g_cli_activation_test_ops, mutation,
mutation_context);
}
cli_activation_production_context_t context;
if (!cli_activation_production_context_init(&context, action, target_version, target_build)) {
if (!cli_activation_production_context_init(&context, action, target_version, target_build,
quiesce_required)) {
cli_activation_production_context_close(&context);
cli_activation_production_diagnostic(NULL, CLI_ACTIVATION_REFUSED_MESSAGE);
/* #1856: this refusal used to print the generic text BARE. Every other
* emitter routes through cli_activation_diagnostic, which appends the
* transaction refusal note or cbm_daemon_ipc_validation_detail() and so
* names the check that actually refused. Here the reader got "Check the
* errors above" with nothing above -- exactly the dead end #1537/#1416
* fixed on the other paths and missed on this one. Context init is
* where the cache, rendezvous and log directories are validated, so it
* is the emitter MOST likely to hold a detail worth showing. */
cli_activation_diagnostic(NULL, CLI_ACTIVATION_REFUSED_MESSAGE);
return CLI_TRUE;
}
printf("Stopping active CBM sessions and operations for %s...\n",
cli_activation_action_text(action));
(void)fflush(stdout);
if (!cli_activation_log_event(&context, "requested", NULL)) {
cli_activation_production_context_close(&context);
(void)fprintf(stderr, "error: activation request could not be recorded safely; "
@@ -793,6 +929,13 @@ static int cli_activation_guard(cbm_daemon_runtime_activation_action_t action,
return rc;
}
static int cli_activation_guard(cbm_daemon_runtime_activation_action_t action,
const char *target_version, const char *target_build,
cbm_cli_activation_mutation_fn mutation, void *mutation_context) {
return cli_activation_guard_scoped(action, target_version, target_build, true, mutation,
mutation_context);
}
/* Tar header field offsets */
#define TAR_NAME_LEN 101 /* filename field: bytes 0-99 + NUL */
#define TAR_SIZE_OFFSET 124 /* octal size field offset */
@@ -1370,8 +1513,8 @@ static const char skill_content[] =
"\n"
"## Edge Types\n"
"CALLS, HTTP_CALLS, ASYNC_CALLS, DATA_FLOWS, IMPORTS, DEFINES, DEFINES_METHOD,\n"
"HANDLES, IMPLEMENTS, OVERRIDE, USAGE, CALL_REFERENCE, CONFIGURES, FILE_CHANGES_WITH,\n"
"SIMILAR_TO, SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER,\n"
"HANDLES, IMPLEMENTS, OVERRIDE, USAGE, CALL_REFERENCE, CONFIGURES, REFERENCES_FILE,\n"
"FILE_CHANGES_WITH, SIMILAR_TO, SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER,\n"
"CONTAINS_PACKAGE\n"
"\n"
"## Cypher Examples (for query_graph)\n"
@@ -2286,6 +2429,140 @@ static int cbm_remove_openclaw_compaction(const char *config_path) {
: CLI_ERR;
}
/* ── OpenHands settings.json mcp_config (#1826) ───────────────
* The mcpServers-style config installed above (cbm_install_editor_mcp into
* ~/.openhands/mcp.json) is not enough: OpenHands only loads a global MCP
* server it finds registered under settings.json -> mcp_config, in its own
* shape ({transport, command, enabled} — no args array). Agent profiles then
* opt in individually via mcp_server_refs (below). */
static size_t cbm_openhands_ownership_fields(cbm_json_like_object_field_t fields[3]) {
fields[0] = (cbm_json_like_object_field_t){
.key = "transport",
.shape = CBM_JSON_LIKE_VALUE_STRING,
.expected_string = "stdio",
.flags = CBM_JSON_LIKE_FIELD_REQUIRED,
};
fields[1] = (cbm_json_like_object_field_t){
.key = "command",
.shape = CBM_JSON_LIKE_VALUE_STRING,
.expected_string = NULL,
.flags = CBM_JSON_LIKE_FIELD_REQUIRED | CBM_JSON_LIKE_FIELD_CAPTURE_STRING,
};
fields[2] = (cbm_json_like_object_field_t){
.key = "enabled",
.shape = CBM_JSON_LIKE_VALUE_LITERAL,
.expected_string = "true",
.flags = CBM_JSON_LIKE_FIELD_REQUIRED,
};
return 3U;
}
static char *cbm_build_openhands_mcp_entry(const char *binary_path) {
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
if (!doc) {
return NULL;
}
yyjson_mut_val *root = yyjson_mut_obj(doc);
bool ok = root && yyjson_mut_obj_add_strcpy(doc, root, "transport", "stdio") &&
yyjson_mut_obj_add_strcpy(doc, root, "command", binary_path) &&
yyjson_mut_obj_add_bool(doc, root, "enabled", true);
char *json = NULL;
if (ok) {
yyjson_mut_doc_set_root(doc, root);
json = yyjson_mut_write(doc, YYJSON_WRITE_NOFLAG, NULL);
}
yyjson_mut_doc_free(doc);
return json;
}
/* Insert or leave alone: an already-owned entry (exact match, or annotated
* with extra keys the client added) needs no write. A same-named entry that
* does not match our shape is left untouched and reported as an error rather
* than overwritten, matching cbm_upsert_json_named_mcp's fail-closed rule for
* every other editor client. */
static int cbm_upsert_openhands_settings_mcp(const char *binary_path, const char *settings_path) {
if (!binary_path || !settings_path) {
return CLI_ERR;
}
static const char *const path[] = {"mcp_config"};
char *document = NULL;
size_t document_length = 0U;
int read_result = cbm_json_like_read_document(settings_path, &document, &document_length);
if (read_result < 0) {
return CLI_ERR;
}
if (read_result == 0) {
cbm_json_like_object_field_t fields[3];
size_t field_count = cbm_openhands_ownership_fields(fields);
char *command = NULL;
int ownership = cbm_json_like_match_object_entry(document, document_length, path, 1U,
CBM_DEFAULT_MCP_SERVER_NAME, fields,
field_count, &command);
free(command);
if (ownership == CBM_JSON_LIKE_OBJECT_MATCH ||
ownership == CBM_JSON_LIKE_OBJECT_MATCH_WITH_EXTRAS) {
free(document);
return CLI_OK;
}
if (ownership != CBM_JSON_LIKE_OBJECT_MISSING) {
free(document);
return CLI_ERR;
}
}
char *entry = cbm_build_openhands_mcp_entry(binary_path);
if (!entry) {
free(document);
return CLI_ERR;
}
int edit_result = cbm_json_like_upsert_entry_if_unchanged(
settings_path, path, 1U, CBM_DEFAULT_MCP_SERVER_NAME, entry,
read_result == 1 ? NULL : document, document_length);
free(entry);
free(document);
return edit_result == 0 ? CLI_OK : CLI_ERR;
}
/* Remove only an entry that is still recognisably ours (an annotated entry is
* left in place and reported, same rule as insertion above); a missing file,
* path, or entry is a successful no-op. */
static int cbm_remove_openhands_settings_mcp(const char *settings_path) {
if (!settings_path) {
return CLI_ERR;
}
static const char *const path[] = {"mcp_config"};
char *document = NULL;
size_t document_length = 0U;
int read_result = cbm_json_like_read_document(settings_path, &document, &document_length);
if (read_result == 1) {
free(document);
return CLI_OK;
}
if (read_result < 0) {
free(document);
return CLI_ERR;
}
cbm_json_like_object_field_t fields[3];
size_t field_count = cbm_openhands_ownership_fields(fields);
char *command = NULL;
int ownership = cbm_json_like_match_object_entry(document, document_length, path, 1U,
CBM_DEFAULT_MCP_SERVER_NAME, fields,
field_count, &command);
free(command);
if (ownership == CBM_JSON_LIKE_OBJECT_MISSING || ownership == CBM_JSON_LIKE_OBJECT_MISMATCH) {
free(document);
return CLI_OK;
}
if (ownership != CBM_JSON_LIKE_OBJECT_MATCH) {
free(document);
return CLI_ERR;
}
int edit_result = cbm_json_like_remove_entry_if_unchanged(
settings_path, path, 1U, CBM_DEFAULT_MCP_SERVER_NAME, document, document_length);
free(document);
return edit_result == 0 ? CLI_OK : CLI_ERR;
}
/* ── VS Code MCP (servers key with type:stdio) ────────────────── */
int cbm_install_vscode_mcp(const char *binary_path, const char *config_path) {
@@ -6965,6 +7242,12 @@ int cbm_remove_indexes(const char *home_dir) {
if (cbm_unlink(path) == 0) {
count++;
}
/* Remove the SQLite sidecars (-wal/-shm/-journal) for both the
* live and staged DBs. Idempotent and ENOENT-tolerant, so it runs
* even when the .db unlink failed -- an orphan -wal can outlive
* its .db. Sidecars are not indexes, so count is unchanged. */
cbm_remove_db_sidecars(path);
cbm_remove_db_sidecars(tmp_path);
}
}
cbm_closedir(d);
@@ -7121,6 +7404,25 @@ bool cbm_config_watcher_enabled(cbm_config_t *cfg) {
return cbm_config_get_bool(cfg, CBM_CONFIG_WATCHER_ENABLED, true);
}
bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy,
char *error, size_t error_size) {
if (!cfg || !policy) {
if (error && error_size > 0) {
(void)snprintf(error, error_size, "index resource configuration is unavailable");
}
return false;
}
cbm_index_policy_init(policy);
for (size_t index = 0; index < cbm_index_policy_key_count(); index++) {
const char *key = cbm_index_policy_key_at(index);
const char *value = cbm_config_get(cfg, key, cbm_index_policy_default_value(key));
if (!cbm_index_policy_set(policy, key, value, error, error_size)) {
return false;
}
}
return true;
}
/* ── Config CLI subcommand ────────────────────────────────────── */
/* THE config-key table. list, get, help, and key validation all read this one
@@ -7145,6 +7447,8 @@ static const config_key_def_t CONFIG_KEYS[] = {
{CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"},
{CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"},
{CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"},
{CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"},
{CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"},
};
/* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing
@@ -7170,6 +7474,31 @@ static bool config_key_is_ui(const char *key) {
return key && (strcmp(key, CBM_CONFIG_UI_ENABLED) == 0 || strcmp(key, CBM_CONFIG_UI_PORT) == 0);
}
static bool config_key_is_index_policy(const char *key) {
return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 ||
strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0);
}
static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) {
cbm_index_resource_policy_t candidate;
cbm_index_policy_init(&candidate);
char error[CLI_BUF_256];
if (!cbm_index_policy_set(&candidate, key, value, error, sizeof(error))) {
(void)fprintf(stderr, "error: %s\n", error);
return CLI_ERR;
}
int rc = cbm_config_set(config, key, value);
if (rc != 0) {
/* The caller suppresses its own message for policy keys because this
* helper names the precise reason. That is only true if the helper
* speaks on every failure it can return: a validated value whose write
* then fails -- a locked or read-only _config.db -- used to exit
* non-zero having printed nothing at all. */
(void)fprintf(stderr, "error: failed to set %s\n", key);
}
return rc;
}
static void config_ui_read(const char *key, char *out, size_t out_sz) {
cbm_ui_config_t ui;
cbm_ui_config_load(&ui);
@@ -7307,10 +7636,16 @@ int cbm_cmd_config(int argc, char **argv) {
rc = CLI_TRUE;
}
} else {
if (cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) == 0) {
int set_rc =
config_key_is_index_policy(argv[CLI_SKIP_ONE])
? config_index_policy_write(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN])
: cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]);
if (set_rc == 0) {
printf("%s = %s\n", argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]);
} else {
(void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]);
if (!config_key_is_index_policy(argv[CLI_SKIP_ONE])) {
(void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]);
}
rc = CLI_TRUE;
}
}
@@ -7843,6 +8178,90 @@ static cbm_install_plan_t *g_install_plan = NULL;
static int g_agent_install_errors = 0;
static int g_agent_uninstall_errors = 0;
/* Every agent configuration uninstall could not clean, kept for the closing
* summary. A cleanup failure no longer stops executable and index removal
* (#1954: one symlinked ~/.cursor/mcp.json left a 300 MB binary plus the whole
* cache behind), so the user needs ONE list of what is still theirs to fix,
* with the observed reason next to each file. */
typedef struct {
char agent[64];
char operation[48];
char path[CLI_BUF_1K];
char reason[160];
char detail[160];
} cbm_agent_config_failure_t;
static cbm_agent_config_failure_t *g_agent_uninstall_failures = NULL;
static int g_agent_uninstall_failure_count = 0;
static int g_agent_uninstall_failure_cap = 0;
static void agent_uninstall_failures_reset(void) {
free(g_agent_uninstall_failures);
g_agent_uninstall_failures = NULL;
g_agent_uninstall_failure_count = 0;
g_agent_uninstall_failure_cap = 0;
}
static void agent_uninstall_failure_record(const char *agent, const char *operation,
const char *path, const char *reason,
const char *detail) {
if (g_agent_uninstall_failure_count >= g_agent_uninstall_failure_cap) {
int ncap = g_agent_uninstall_failure_cap ? g_agent_uninstall_failure_cap * 2 : CLI_BUF_16;
cbm_agent_config_failure_t *grown =
realloc(g_agent_uninstall_failures, (size_t)ncap * sizeof(*grown));
if (!grown) {
return;
}
g_agent_uninstall_failures = grown;
g_agent_uninstall_failure_cap = ncap;
}
cbm_agent_config_failure_t *entry =
&g_agent_uninstall_failures[g_agent_uninstall_failure_count++];
(void)snprintf(entry->agent, sizeof(entry->agent), "%s", agent ? agent : "unknown");
(void)snprintf(entry->operation, sizeof(entry->operation), "%s",
operation ? operation : "unknown");
(void)snprintf(entry->path, sizeof(entry->path), "%s", path ? path : "unknown");
(void)snprintf(entry->reason, sizeof(entry->reason), "%s", reason ? reason : "");
(void)snprintf(entry->detail, sizeof(entry->detail), "%s", detail ? detail : "");
}
/* The agent-configuration writers opt in to following user-owned symlinked
* config files under the user's configuration roots (#1954, decision C);
* every other caller of the config editors keeps refusing links. Cleared by
* the same command when its configuration work is done. */
static void cli_config_follow_begin(const char *home) {
cbm_config_edit_path_follow_clear();
if (home && home[0]) {
(void)cbm_config_edit_path_follow_add_root(home);
}
const char *xdg_config = getenv("XDG_CONFIG_HOME");
if (xdg_config && xdg_config[0]) {
(void)cbm_config_edit_path_follow_add_root(xdg_config);
}
}
/* The closing list of what uninstall could not clean. Printed AFTER the
* executable and the indexes are gone, so nothing in it is a reason to keep
* the installation around — each line is one file the user removes an entry
* from by hand. */
static void agent_uninstall_failures_report(bool dry_run) {
if (g_agent_uninstall_failure_count == 0) {
return;
}
(void)fprintf(stderr, "\nerror: uninstall %s with %d agent configuration(s) left uncleaned:\n",
dry_run ? "dry-run finished" : "finished", g_agent_uninstall_failure_count);
for (int i = 0; i < g_agent_uninstall_failure_count; i++) {
const cbm_agent_config_failure_t *entry = &g_agent_uninstall_failures[i];
(void)fprintf(stderr, " %s (%s): %s", entry->agent, entry->operation, entry->path);
if (entry->reason[0]) {
(void)fprintf(stderr, " reason=%s", entry->reason);
}
(void)fputs(entry->detail, stderr);
(void)fputc('\n', stderr);
}
(void)fputs("Remove the codebase-memory-mcp entries from these files by hand.\n", stderr);
}
static void plan_record(const char *agent, const char *kind, const char *path) {
if (!g_install_plan || !path || !path[0]) {
return;
@@ -7898,6 +8317,14 @@ static void describe_agent_config_target(const char *path, char *out, size_t out
: info.is_directory ? "directory"
: info.is_regular ? "regular file"
: "special file";
/* A refused symlink names the rule that refused it (#1954): the user
* then knows whether to fix ownership, the target, or the parent. */
char refusal[160];
if (info.is_symlink && cbm_config_edit_path_refusal(path, refusal, sizeof(refusal))) {
(void)snprintf(out, out_size, " (target: symlink, %lld bytes; not followed: %s)",
(long long)info.size, refusal);
return;
}
(void)snprintf(out, out_size, " (target: %s, %lld bytes)", kind, (long long)info.size);
}
@@ -7915,6 +8342,9 @@ static void record_agent_config_error_with_reason(bool uninstalling, const char
}
(void)fputs(detail, stderr);
(void)fputc('\n', stderr);
if (uninstalling) {
agent_uninstall_failure_record(agent, operation, path, reason, detail);
}
}
static void record_agent_config_error(bool uninstalling, const char *agent, const char *operation,
@@ -9355,6 +9785,47 @@ static void uninstall_vscode_profile_configs(const char *code_user, const char *
cbm_closedir(directory);
}
static bool cbm_filename_has_suffix(const char *name, const char *suffix) {
size_t name_len = strlen(name);
size_t suffix_len = strlen(suffix);
return name_len >= suffix_len && strcmp(name + (name_len - suffix_len), suffix) == 0;
}
/* Register or unregister our server against every existing OpenHands agent
* profile's mcp_server_refs array (#1826). A missing agent-profiles/
* directory is a silent no-op in both directions — install must never invent
* it, and uninstall has nothing to undo there. Only *.json entries are
* touched; a profile directory may hold arbitrary notes alongside profiles. */
static void openhands_update_profile_refs(const char *profiles_dir, bool installing, bool dry_run) {
cbm_dir_t *d = cbm_opendir(profiles_dir);
if (!d) {
return;
}
cbm_dirent_t *ent;
while ((ent = cbm_readdir(d)) != NULL) {
if (strcmp(ent->name, ".") == 0 || strcmp(ent->name, "..") == 0 ||
!cbm_filename_has_suffix(ent->name, ".json")) {
continue;
}
char profile_path[CLI_BUF_1K];
snprintf(profile_path, sizeof(profile_path), "%s/%s", profiles_dir, ent->name);
struct stat state;
if (stat(profile_path, &state) != 0 || !S_ISREG(state.st_mode) || dry_run) {
continue;
}
int result = installing ? cbm_json_like_add_unique_string(profile_path, "mcp_server_refs",
CBM_DEFAULT_MCP_SERVER_NAME)
: cbm_json_like_remove_string(profile_path, "mcp_server_refs",
CBM_DEFAULT_MCP_SERVER_NAME);
if (result != CLI_OK) {
record_agent_config_error(
!installing, "OpenHands",
installing ? "profile_refs_install" : "profile_refs_uninstall", profile_path);
}
}
cbm_closedir(d);
}
/* Install MCP configs for editor-based agents (Zed, KiloCode, VS Code, OpenClaw). */
static void install_editor_agent_configs(const cbm_detected_agents_t *agents, const char *home,
const char *binary_path, bool force, bool dry_run) {
@@ -9617,11 +10088,32 @@ static void install_additional_agent_configs(const cbm_detected_agents_t *agents
if (agents->openhands) {
char cp[CLI_BUF_1K];
char skills_dir[CLI_BUF_1K];
char settings_path[CLI_BUF_1K];
char profiles_dir[CLI_BUF_1K];
snprintf(cp, sizeof(cp), "%s/.openhands/mcp.json", home);
snprintf(skills_dir, sizeof(skills_dir), "%s/.agents/skills", home);
snprintf(settings_path, sizeof(settings_path), "%s/.openhands/settings.json", home);
snprintf(profiles_dir, sizeof(profiles_dir), "%s/.openhands/agent-profiles", home);
install_generic_agent_config("OpenHands", binary_path, cp, NULL, dry_run,
cbm_install_editor_mcp);
install_agent_skill("OpenHands", skills_dir, force, dry_run);
/* #1826: the mcpServers-shaped mcp.json above is not enough — OpenHands
* only loads a server registered under settings.json -> mcp_config, and
* only for agent profiles that reference it. agent-profiles/ is never
* invented; a missing directory means nothing to register into yet. */
if (g_install_plan) {
plan_record("OpenHands", "mcp_config", settings_path);
} else {
if (!dry_run) {
if (!prepare_config_parent(settings_path) ||
cbm_upsert_openhands_settings_mcp(binary_path, settings_path) != CLI_OK) {
record_agent_config_error(false, "OpenHands", "settings_mcp_install",
settings_path);
}
}
printf(" settings mcp_config: %s\n", settings_path);
openhands_update_profile_refs(profiles_dir, true, dry_run);
}
}
if (agents->augment) {
char cp[CLI_BUF_1K];
@@ -9887,12 +10379,24 @@ static void install_additional_agent_configs(const cbm_detected_agents_t *agents
}
}
static int cbm_install_agent_configs_in_scope(const char *home, const char *binary_path, bool force,
bool dry_run, cbm_detected_agents_t *agents_in);
int cbm_install_agent_configs(const char *home, const char *binary_path, bool force, bool dry_run) {
g_agent_install_errors = 0;
cbm_detected_agents_t agents = cbm_detect_agents(home);
if (g_client_selection && !cli_clients_apply_selection(g_client_selection, &agents)) {
return CLI_ERR;
}
cli_config_follow_begin(home);
int result = cbm_install_agent_configs_in_scope(home, binary_path, force, dry_run, &agents);
cbm_config_edit_path_follow_clear();
return result;
}
static int cbm_install_agent_configs_in_scope(const char *home, const char *binary_path, bool force,
bool dry_run, cbm_detected_agents_t *agents_in) {
cbm_detected_agents_t agents = *agents_in;
if (!g_install_plan) {
print_detected_agents(&agents, home);
}
@@ -10883,11 +11387,16 @@ int cbm_cmd_install(int argc, char **argv) {
.force = force,
.dry_run = dry_run,
};
int activation_rc =
dry_run ? cli_install_activate(&activation)
: cli_activation_guard(CBM_DAEMON_RUNTIME_ACTIVATION_INSTALL, CBM_VERSION,
has_binary_validator ? binary_validator.fingerprint : NULL,
cli_install_activate, &activation);
/* What this install replaces decides whether any session must stop: with
* the published binary untouched (--skip-binary, or an externally managed
* binary) and no index reset, agent configs are refreshed while every
* session stays up. */
bool quiesce_required = has_binary_validator || delete_indexes;
int activation_rc = dry_run ? cli_install_activate(&activation)
: cli_activation_guard_scoped(
CBM_DAEMON_RUNTIME_ACTIVATION_INSTALL, CBM_VERSION,
has_binary_validator ? binary_validator.fingerprint : NULL,
quiesce_required, cli_install_activate, &activation);
if (activation.binary_transaction) {
(void)cli_activation_transaction_abort(&activation.binary_transaction);
}
@@ -11877,11 +12386,21 @@ static void uninstall_additional_agents(const cbm_detected_agents_t *agents, con
if (agents->openhands) {
char cp[CLI_BUF_1K];
char skills_dir[CLI_BUF_1K];
char settings_path[CLI_BUF_1K];
char profiles_dir[CLI_BUF_1K];
snprintf(cp, sizeof(cp), "%s/.openhands/mcp.json", home);
snprintf(skills_dir, sizeof(skills_dir), "%s/.agents/skills", home);
snprintf(settings_path, sizeof(settings_path), "%s/.openhands/settings.json", home);
snprintf(profiles_dir, sizeof(profiles_dir), "%s/.openhands/agent-profiles", home);
uninstall_agent_mcp_instr((mcp_uninstall_args_t){"OpenHands", cp, NULL}, dry_run,
cbm_remove_editor_mcp_owned);
printf(" removed %d skill(s)\n", cbm_remove_skills(skills_dir, dry_run));
/* #1826 counterpart: undo the settings.json registration and every
* agent profile's mcp_server_refs entry the install above added. */
if (!dry_run && cbm_remove_openhands_settings_mcp(settings_path) != CLI_OK) {
record_agent_config_error(true, "OpenHands", "settings_mcp_uninstall", settings_path);
}
openhands_update_profile_refs(profiles_dir, false, dry_run);
}
if (agents->augment) {
char cp[CLI_BUF_1K];
@@ -12192,6 +12711,7 @@ static int cli_uninstall_activate(void *opaque) {
return CLI_TRUE;
}
cli_config_follow_begin(activation->home);
if (activation->agents.claude_code) {
uninstall_claude_code(activation->home, activation->bin_path, activation->dry_run);
}
@@ -12199,14 +12719,14 @@ static int cli_uninstall_activate(void *opaque) {
uninstall_editor_agents(&activation->agents, activation->home, activation->dry_run);
uninstall_additional_agents(&activation->agents, activation->home, activation->dry_run);
uninstall_agent_client_registry(activation->home, activation->dry_run);
cbm_config_edit_path_follow_clear();
if (g_agent_uninstall_errors != 0) {
cli_activation_transaction_abort_or_fail_stop(&activation->binary_transaction,
"uninstall_transaction_config_cleanup_abort");
(void)fprintf(stderr, "error: one or more agent cleanup operations failed; executable "
"and index removal were not started\n");
return CLI_ACTIVATION_PARTIAL;
}
/* Agent-config failures are collected, never a gate: an entry the editors
* refuse to touch (a symlinked config, a foreign file, a malformed
* document) is the user's to fix by hand, and leaving a 300 MB executable
* plus every index behind because of it is the data-loss shape of #1954.
* The indexes and the executable go now; the failures are listed at the
* end and decide the exit code. */
#ifdef _WIN32
/* #2117: install registers the install directory in the persistent
@@ -12344,6 +12864,7 @@ int cbm_cmd_uninstall(int argc, char **argv) {
printf("codebase-memory-mcp uninstall\n\n");
g_agent_uninstall_errors = 0;
agent_uninstall_failures_reset();
cbm_detected_agents_t agents = cbm_detect_agents(home);
/* Confirm index removal outside the startup lock, but defer the mutation
@@ -12413,6 +12934,19 @@ int cbm_cmd_uninstall(int argc, char **argv) {
(void)cli_activation_transaction_abort(&activation.binary_transaction);
}
if (activation_rc != CLI_OK) {
agent_uninstall_failures_reset();
return CLI_TRUE;
}
if (g_agent_uninstall_errors != 0) {
agent_uninstall_failures_report(dry_run);
agent_uninstall_failures_reset();
printf("\nUninstall finished with errors; the files listed above still hold "
"codebase-memory-mcp entries. Please restart your coding-agent sessions "
"to properly take this into account.\n");
if (dry_run) {
printf("(dry-run — no files were modified)\n");
}
return CLI_TRUE;
}
@@ -12421,7 +12955,7 @@ int cbm_cmd_uninstall(int argc, char **argv) {
if (dry_run) {
printf("(dry-run — no files were modified)\n");
}
return g_agent_uninstall_errors == 0 ? 0 : CLI_TRUE;
return 0;
}
/* ── Subcommand: update ───────────────────────────────────────── */
@@ -13218,6 +13752,32 @@ static void cli_add_typed(yyjson_mut_doc *out, yyjson_mut_val *obj, const char *
arr = yyjson_mut_arr(out);
yyjson_mut_obj_add(obj, yyjson_mut_strcpy(out, key), arr);
}
/* The help prints `--paths <array>`, and a caller who writes the
* array literally — `--paths '["lib","t"]'` — used to get ONE element
* holding that literal text (2026-09-16 probe: check_index_coverage
* reported the fake path `["lib","t"]`). A value that parses as a JSON
* array contributes its elements; anything else is one element. */
if (have_value && value && value[0] == '[') {
yyjson_doc *lit = yyjson_read(value, strlen(value), 0);
yyjson_val *lit_root = lit ? yyjson_doc_get_root(lit) : NULL;
if (lit_root && yyjson_is_arr(lit_root)) {
size_t idx;
size_t max;
yyjson_val *elem;
yyjson_arr_foreach(lit_root, idx, max, elem) {
if (yyjson_is_str(elem)) {
yyjson_mut_arr_add_strcpy(out, arr, yyjson_get_str(elem));
} else {
yyjson_mut_arr_add_val(arr, yyjson_val_mut_copy(out, elem));
}
}
yyjson_doc_free(lit);
return;
}
if (lit) {
yyjson_doc_free(lit);
}
}
yyjson_mut_arr_add_strcpy(out, arr, have_value ? value : "");
return;
}
src/cli/cli.h
+19
View File
@@ -13,6 +13,8 @@
#include <stddef.h>
#include <stdint.h>
#include "foundation/index_policy.h"
typedef struct cbm_mcp_server cbm_mcp_server_t;
/* ── Version ──────────────────────────────────────────────────── */
@@ -25,6 +27,18 @@ const char *cbm_cli_get_version(void);
/* ── CLI tool arguments (flags / --args-file / --help) ────────── */
/* Top-level `cli --help` text printed by run_cli() in src/main.c.
* Documents tool-level --format without adding a session-wide flag (#2102). */
#define CBM_CLI_USAGE \
"Usage: codebase-memory-mcp cli [--quiet] [--progress] [--verbose] [--json] <tool_name> " \
"[json_args]\n" \
" --quiet Show errors only; cannot combine with --progress or outer --verbose\n" \
" --progress Show lifecycle progress even when stderr is redirected\n" \
" --verbose Include informational logs (preserves CBM_LOG_LEVEL=debug)\n" \
" --json Print the raw MCP result envelope\n" \
" Tools that accept format support --format tree|json (default: tree).\n" \
" --format json prints payload JSON; outer --json prints the full MCP envelope.\n"
/* Convert `--flag value` / `--flag=value` / bare-boolean `--flag` arguments for
* a tool into a JSON arguments object string, using the tool's input_schema to
* type values (string/integer/boolean) and to collect repeated flags into
@@ -420,6 +434,11 @@ int cbm_config_set(cbm_config_t *cfg, const char *key, const char *value);
/* Delete a config key. Returns 0 on success. */
int cbm_config_delete(cbm_config_t *cfg, const char *key);
/* Load and validate the operator-controlled discovery policy. Invalid stored
* values fail closed instead of silently disabling a guard. */
bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy,
char *error, size_t error_size);
/* Well-known config keys */
#define CBM_CONFIG_AUTO_INDEX "auto_index"
#define CBM_CONFIG_AUTO_INDEX_LIMIT "auto_index_limit"
src/cli/client_adapter.c
+166 -5
View File
@@ -7,6 +7,7 @@
#include "foundation/constants.h"
#include "mcp/mcp.h"
#include "yyjson/yyjson.h"
#include <stdbool.h>
#include <stdio.h>
@@ -138,6 +139,167 @@ static void emit_header(adapter_sb_t *sb, const char *client) {
"// are overwritten; edit outside it, or remove the markers to take ownership.\n");
}
static void append_typebox_type(adapter_sb_t *sb, yyjson_val *val, const char *description,
bool include_description) {
yyjson_val *type_val = yyjson_obj_get(val, "type");
yyjson_val *enum_val = yyjson_obj_get(val, "enum");
yyjson_val *items_val = yyjson_obj_get(val, "items");
const char *type_str = yyjson_get_str(type_val);
if (type_str && strcmp(type_str, "string") == 0 && enum_val && yyjson_is_arr(enum_val) &&
yyjson_arr_size(enum_val) > 0) {
sb_append(sb, "Type.Union([");
size_t enum_idx = 0;
size_t enum_max = yyjson_arr_size(enum_val);
bool first_enum = true;
for (enum_idx = 0; enum_idx < enum_max; enum_idx++) {
yyjson_val *enum_item = yyjson_arr_get(enum_val, enum_idx);
const char *enum_str = yyjson_get_str(enum_item);
if (!enum_str) {
continue;
}
if (!first_enum) {
sb_append(sb, ", ");
}
first_enum = false;
sb_append(sb, "Type.Literal(");
sb_append_js_string(sb, enum_str);
sb_append(sb, ")");
}
if (first_enum) {
sb_append(sb, "Type.String()");
} else {
sb_append(sb, "]");
if (include_description && description) {
sb_append(sb, ", { description: ");
sb_append_js_string(sb, description);
sb_append(sb, " }");
}
sb_append(sb, ")");
return;
}
sb_append(sb, ")");
return;
}
const char *typebox_name = "Any";
if (type_str && strcmp(type_str, "string") == 0) {
typebox_name = "String";
} else if (type_str && strcmp(type_str, "integer") == 0) {
typebox_name = "Integer";
} else if (type_str && strcmp(type_str, "number") == 0) {
typebox_name = "Number";
} else if (type_str && strcmp(type_str, "boolean") == 0) {
typebox_name = "Boolean";
}
if (type_str && strcmp(type_str, "array") == 0) {
sb_append(sb, "Type.Array(");
if (items_val) {
append_typebox_type(sb, items_val, NULL, false);
} else {
sb_append(sb, "Type.Any()");
}
if (include_description && description) {
sb_append(sb, ", { description: ");
sb_append_js_string(sb, description);
sb_append(sb, " }");
}
sb_append(sb, ")");
return;
}
sb_append(sb, "Type.");
sb_append(sb, typebox_name);
sb_append(sb, "(");
if (include_description && description) {
sb_append(sb, "{ description: ");
sb_append_js_string(sb, description);
sb_append(sb, " }");
}
sb_append(sb, ")");
}
static bool typebox_property_is_required(yyjson_val *required, const char *prop_name) {
if (!required || !yyjson_is_arr(required)) {
return false;
}
size_t idx = 0;
size_t max = yyjson_arr_size(required);
for (idx = 0; idx < max; idx++) {
const char *required_name = yyjson_get_str(yyjson_arr_get(required, idx));
if (required_name && strcmp(required_name, prop_name) == 0) {
return true;
}
}
return false;
}
static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_str) {
if (!schema_str) {
sb_append(sb, "Type.Object({})");
return;
}
/* Parse the JSON schema to extract properties and convert to TypeBox */
yyjson_doc *doc = yyjson_read(schema_str, strlen(schema_str), 0);
if (!doc) {
sb_append(sb, "Type.Object({})");
return;
}
yyjson_val *root = yyjson_doc_get_root(doc);
if (!root) {
sb_append(sb, "Type.Object({})");
yyjson_doc_free(doc);
return;
}
yyjson_val *properties = yyjson_obj_get(root, "properties");
yyjson_val *required = yyjson_obj_get(root, "required");
if (!properties) {
sb_append(sb, "Type.Object({})");
yyjson_doc_free(doc);
return;
}
sb_append(sb, "Type.Object({\n");
bool first_prop = true;
size_t idx = 0, max = 0;
yyjson_val *key = NULL, *val = NULL;
yyjson_obj_foreach(properties, idx, max, key, val) {
const char *prop_name = yyjson_get_str(key);
if (!prop_name) {
continue;
}
if (!first_prop) {
sb_append(sb, ",\n");
}
first_prop = false;
sb_append(sb, " ");
sb_append(sb, prop_name);
sb_append(sb, ": ");
yyjson_val *description_val = yyjson_obj_get(val, "description");
const char *description = yyjson_get_str(description_val);
bool is_required = typebox_property_is_required(required, prop_name);
if (!is_required) {
sb_append(sb, "Type.Optional(");
}
append_typebox_type(sb, val, description, true);
if (!is_required) {
sb_append(sb, ")");
}
}
sb_append(sb, "\n })");
yyjson_doc_free(doc);
}
char *cbm_client_adapter_pi(const char *binary_path) {
if (!binary_path || !binary_path[0]) {
return NULL;
@@ -157,7 +319,8 @@ char *cbm_client_adapter_pi(const char *binary_path) {
* (pods CLI) or an old AgentTool arity cannot be mistaken for this file. */
sb_append(&sb, "// Target: @earendil-works/pi-coding-agent >= 0.74.0 (verified 0.84.2)\n"
"// ToolDefinition.execute(toolCallId, params, signal, onUpdate, ctx)\n");
sb_append(&sb, "import { spawn } from 'node:child_process';\n\n");
sb_append(&sb, "import { spawn } from 'node:child_process';\n");
sb_append(&sb, "import { Type } from 'typebox';\n\n");
sb_append(&sb, "const BIN = '");
sb_append(&sb, bin);
sb_append(&sb, "';\n\n");
@@ -224,10 +387,8 @@ char *cbm_client_adapter_pi(const char *binary_path) {
sb_append(&sb, ",\n description: ");
sb_append_js_string(&sb, description ? description : "");
sb_append(&sb, ",\n parameters: ");
/* input_schema is compact JSON, which is a valid JavaScript object
* literal; embedding it directly keeps the generated module free of a
* JSON.parse indirection and of any escaping drift. */
sb_append(&sb, schema ? schema : "{}");
/* Convert JSON schema to TypeBox format */
convert_json_schema_to_typebox(&sb, schema);
/* 0.84.2 calls execute(toolCallId, params, signal, onUpdate, ctx).
* The previous (args, ctx) shape bound the call id as the MCP args. */
sb_append(&sb, ",\n execute: async (toolCallId, params, signal, _onUpdate, ctx) => {\n");
src/cli/config_edit_path.c
+457
View File
@@ -0,0 +1,457 @@
/*
* config_edit_path.c — follow a config symlink only when the agent-config
* writer opted in, the invoking user owns both the link and its regular-file
* target, and the target's parent directory can be pinned. See the header
* for the decision, the sequence and its boundaries (#1954).
*/
#include "cli/config_edit_path.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifndef _WIN32
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#endif
enum { EDIT_PATH_MAX_ROOTS = 4 };
static int edit_path_copy(const char *source, char *out, size_t out_size) {
int written = snprintf(out, out_size, "%s", source);
return written >= 0 && (size_t)written < out_size ? 0 : -1;
}
static void edit_path_target_reset(cbm_config_edit_target_t *target) {
target->status = CBM_CONFIG_EDIT_PATH_DIRECT;
target->dirfd = -1;
target->fd = -1;
target->base[0] = '\0';
target->path[0] = '\0';
}
#ifndef _WIN32
/* Only the POSIX symlink-follow resolver records a refusal reason; the Windows
* build never follows, so guard this with the same condition as its callers to
* avoid -Werror,-Wunused-function (#2110 CI). */
static void edit_path_note(char *reason, size_t reason_size, const char *text) {
if (reason && reason_size > 0U) {
(void)snprintf(reason, reason_size, "%s", text);
}
}
#endif /* !_WIN32 */
void cbm_config_edit_target_close(cbm_config_edit_target_t *target) {
if (!target) {
return;
}
int saved_errno = errno;
#ifndef _WIN32
if (target->fd >= 0) {
(void)close(target->fd);
}
if (target->dirfd >= 0) {
(void)close(target->dirfd);
}
#endif
target->fd = -1;
target->dirfd = -1;
errno = saved_errno;
}
#ifndef _WIN32
/* The opted-in follow-roots are per-thread (test isolation), but the storage
* is a heap buffer behind a thread-local POINTER, not a thread-local array.
* A `_Thread_local char[4][4096]` sits in the binary's static TLS block, and
* glibc carves that block out of every thread's stack allocation — including
* the parent-death watchdog's deliberately tiny 64 KiB stack (main.c). The
* 16 KiB array pushed the watchdog's pthread_create over the static-TLS +
* guard budget, so it failed with EINVAL on glibc/x86-64 (green on macOS,
* whose TLS is not stack-carved): #2110 CI. The pointer keeps the static TLS
* block at 8 bytes; the buffer is allocated on first add and freed by clear. */
static CBM_TLS char (*edit_path_roots)[CBM_CONFIG_EDIT_PATH_MAX] = NULL;
static CBM_TLS size_t edit_path_root_count = 0U;
#ifdef CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API
static CBM_TLS int edit_path_uid_override_set = 0;
static CBM_TLS uid_t edit_path_uid_override = 0;
static CBM_TLS int edit_path_identity_skew = 0;
void cbm_config_edit_path_set_invoking_uid_for_test(unsigned uid, int set) {
edit_path_uid_override_set = set != 0;
edit_path_uid_override = (uid_t)uid;
}
void cbm_config_edit_path_set_identity_skew_for_test(int set) {
edit_path_identity_skew = set != 0;
}
#endif
static uid_t edit_path_invoking_uid(void) {
#ifdef CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API
if (edit_path_uid_override_set) {
return edit_path_uid_override;
}
#endif
return geteuid();
}
int cbm_config_edit_path_follow_add_root(const char *root) {
if (!root || !root[0] || edit_path_root_count >= EDIT_PATH_MAX_ROOTS) {
return -1;
}
char canonical[CBM_CONFIG_EDIT_PATH_MAX];
if (!cbm_canonical_path(root, canonical, sizeof(canonical))) {
return -1;
}
if (!edit_path_roots) {
edit_path_roots = calloc(EDIT_PATH_MAX_ROOTS, sizeof(*edit_path_roots));
if (!edit_path_roots) {
return -1;
}
}
if (edit_path_copy(canonical, edit_path_roots[edit_path_root_count],
sizeof(edit_path_roots[0])) != 0) {
return -1;
}
edit_path_root_count++;
return 0;
}
void cbm_config_edit_path_follow_clear(void) {
edit_path_root_count = 0U;
free(edit_path_roots);
edit_path_roots = NULL;
}
static int edit_path_under_root(const char *canonical) {
for (size_t i = 0U; i < edit_path_root_count; i++) {
const char *root = edit_path_roots[i];
size_t root_length = strlen(root);
if (strcmp(canonical, root) == 0) {
return 1;
}
if (strncmp(canonical, root, root_length) == 0 &&
(canonical[root_length] == '/' || root[root_length - 1U] == '/')) {
return 1;
}
}
return 0;
}
/* Splits a canonical absolute path into its parent and its final name. */
static int edit_path_split(const char *canonical, char *parent, size_t parent_size, char *base,
size_t base_size) {
const char *slash = strrchr(canonical, '/');
if (!slash || slash[1] == '\0') {
return -1;
}
size_t parent_length = slash == canonical ? 1U : (size_t)(slash - canonical);
if (parent_length >= parent_size || edit_path_copy(slash + 1, base, base_size) != 0) {
return -1;
}
memcpy(parent, canonical, parent_length);
parent[parent_length] = '\0';
return 0;
}
/* The link's OWN directory must sit under a registered root: that is what
* makes the follow an opt-in of the agent-configuration writers rather than
* a property of every editor caller. */
static int edit_path_link_dir_under_root(const char *path) {
if (edit_path_root_count == 0U) {
return 0;
}
char parent[CBM_CONFIG_EDIT_PATH_MAX];
if (edit_path_copy(path, parent, sizeof(parent)) != 0) {
return 0;
}
char *slash = strrchr(parent, '/');
if (!slash) {
parent[0] = '.';
parent[1] = '\0';
} else if (slash == parent) {
parent[1] = '\0';
} else {
*slash = '\0';
}
char canonical[CBM_CONFIG_EDIT_PATH_MAX];
if (!cbm_canonical_path(parent, canonical, sizeof(canonical))) {
return 0;
}
return edit_path_under_root(canonical);
}
static void edit_path_note_owner(char *reason, size_t reason_size, const char *what,
unsigned long owner, unsigned long self) {
if (reason && reason_size > 0U) {
(void)snprintf(reason, reason_size, "%s owned by uid %lu, not the invoking user (uid %lu)",
what, owner, self);
}
}
/* One classification serves both the resolver and the diagnostic, so the
* reason a user reads is exactly the rule that refused the link. On REFUSED
* every descriptor opened along the way is closed again. */
static int edit_path_classify(const char *path, cbm_config_edit_target_t *target, char *reason,
size_t reason_size) {
edit_path_target_reset(target);
if (reason && reason_size > 0U) {
reason[0] = '\0';
}
if (!path || !path[0]) {
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
struct stat link_state;
if (lstat(path, &link_state) != 0 || !S_ISLNK(link_state.st_mode)) {
return edit_path_copy(path, target->path, sizeof(target->path)) == 0
? CBM_CONFIG_EDIT_PATH_DIRECT
: CBM_CONFIG_EDIT_PATH_REFUSED;
}
if (!edit_path_link_dir_under_root(path)) {
edit_path_note(reason, reason_size,
edit_path_root_count == 0U
? "symlinks are not followed by this command"
: "symlink outside the opted-in configuration roots");
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
uid_t self = edit_path_invoking_uid();
if (self == 0 && link_state.st_uid != 0) {
if (reason && reason_size > 0U) {
(void)snprintf(reason, reason_size,
"symlink owned by uid %lu is not followed by a root process",
(unsigned long)link_state.st_uid);
}
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
if (link_state.st_uid != self) {
edit_path_note_owner(reason, reason_size, "symlink", (unsigned long)link_state.st_uid,
(unsigned long)self);
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
char resolved[CBM_CONFIG_EDIT_PATH_MAX];
struct stat target_state;
if (!cbm_canonical_path(path, resolved, sizeof(resolved)) ||
stat(resolved, &target_state) != 0) {
edit_path_note(reason, reason_size, "dangling symlink");
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
if (!S_ISREG(target_state.st_mode)) {
edit_path_note(reason, reason_size,
S_ISDIR(target_state.st_mode) ? "symlink to a directory"
: "symlink to a special file");
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
if (target_state.st_uid != self) {
edit_path_note_owner(reason, reason_size, "symlink target",
(unsigned long)target_state.st_uid, (unsigned long)self);
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
#if CBM_CONFIG_EDIT_TARGET_UNDER_ROOT
if (!edit_path_under_root(resolved)) {
edit_path_note(reason, reason_size, "symlink target outside the configuration roots");
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
#endif
char parent[CBM_CONFIG_EDIT_PATH_MAX];
if (edit_path_split(resolved, parent, sizeof(parent), target->base, sizeof(target->base)) !=
0 ||
edit_path_copy(resolved, target->path, sizeof(target->path)) != 0) {
edit_path_note(reason, reason_size, "symlink target path is too long");
edit_path_target_reset(target);
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
int dir_flags = O_RDONLY | O_NOFOLLOW;
#ifdef O_DIRECTORY
dir_flags |= O_DIRECTORY;
#endif
#ifdef O_CLOEXEC
dir_flags |= O_CLOEXEC;
#endif
target->dirfd = open(parent, dir_flags);
struct stat dir_state;
if (target->dirfd < 0 || fstat(target->dirfd, &dir_state) != 0 || !S_ISDIR(dir_state.st_mode)) {
edit_path_note(reason, reason_size,
"parent directory of the symlink target cannot be opened");
cbm_config_edit_target_close(target);
edit_path_target_reset(target);
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
if (dir_state.st_uid != self && dir_state.st_uid != 0) {
if (reason && reason_size > 0U) {
(void)snprintf(reason, reason_size,
"parent directory of the symlink target is owned by uid %lu",
(unsigned long)dir_state.st_uid);
}
cbm_config_edit_target_close(target);
edit_path_target_reset(target);
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
if ((dir_state.st_mode & S_IWOTH) != 0 && (dir_state.st_mode & S_ISVTX) == 0) {
edit_path_note(reason, reason_size,
"parent directory of the symlink target is writable by others");
cbm_config_edit_target_close(target);
edit_path_target_reset(target);
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
int file_flags = O_RDONLY | O_NOFOLLOW | O_NONBLOCK;
#ifdef O_CLOEXEC
file_flags |= O_CLOEXEC;
#endif
target->fd = openat(target->dirfd, target->base, file_flags);
struct stat opened_state;
if (target->fd < 0 || fstat(target->fd, &opened_state) != 0) {
edit_path_note(reason, reason_size, "symlink target cannot be opened beside its parent");
cbm_config_edit_target_close(target);
edit_path_target_reset(target);
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
const char *mismatch = NULL;
if (!S_ISREG(opened_state.st_mode)) {
mismatch = "symlink target is not a regular file";
} else if (opened_state.st_uid != self) {
mismatch = "symlink target changed owner while it was being checked";
} else if (opened_state.st_nlink != 1U) {
mismatch = "symlink target is hard-linked";
} else if (opened_state.st_dev != target_state.st_dev ||
opened_state.st_ino != target_state.st_ino) {
mismatch = "symlink target changed while it was being checked";
}
#ifdef CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API
if (!mismatch && edit_path_identity_skew) {
mismatch = "symlink target changed while it was being checked";
}
#endif
if (mismatch) {
edit_path_note(reason, reason_size, mismatch);
cbm_config_edit_target_close(target);
edit_path_target_reset(target);
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
target->status = CBM_CONFIG_EDIT_PATH_FOLLOWED;
return CBM_CONFIG_EDIT_PATH_FOLLOWED;
}
int cbm_config_edit_target_create_temp(const cbm_config_edit_target_t *target, const char *name,
unsigned mode) {
if (!target || target->status != CBM_CONFIG_EDIT_PATH_FOLLOWED || target->dirfd < 0 || !name ||
!name[0] || strchr(name, '/')) {
errno = EINVAL;
return -1;
}
int flags = O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW;
#ifdef O_CLOEXEC
flags |= O_CLOEXEC;
#endif
return openat(target->dirfd, name, flags, (mode_t)mode);
}
int cbm_config_edit_target_commit(const cbm_config_edit_target_t *target, const char *name) {
if (!target || target->status != CBM_CONFIG_EDIT_PATH_FOLLOWED || target->dirfd < 0 || !name ||
!name[0] || strchr(name, '/')) {
errno = EINVAL;
return -1;
}
if (renameat(target->dirfd, name, target->dirfd, target->base) != 0) {
return -1;
}
return fsync(target->dirfd) == 0 ? 0 : -1;
}
int cbm_config_edit_target_unlink(const cbm_config_edit_target_t *target, const char *name) {
if (!target || target->status != CBM_CONFIG_EDIT_PATH_FOLLOWED || target->dirfd < 0 || !name ||
!name[0] || strchr(name, '/')) {
errno = EINVAL;
return -1;
}
return unlinkat(target->dirfd, name, 0);
}
#else /* _WIN32 */
#ifdef CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API
void cbm_config_edit_path_set_invoking_uid_for_test(unsigned uid, int set) {
(void)uid;
(void)set;
}
void cbm_config_edit_path_set_identity_skew_for_test(int set) {
(void)set;
}
#endif
int cbm_config_edit_path_follow_add_root(const char *root) {
(void)root;
return 0;
}
void cbm_config_edit_path_follow_clear(void) {}
/* Reparse points are never followed by the editors on Windows; the path is
* handed back unchanged and FILE_FLAG_OPEN_REPARSE_POINT keeps refusing. */
static int edit_path_classify(const char *path, cbm_config_edit_target_t *target, char *reason,
size_t reason_size) {
edit_path_target_reset(target);
if (reason && reason_size > 0U) {
reason[0] = '\0';
}
if (!path || !path[0]) {
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
return edit_path_copy(path, target->path, sizeof(target->path)) == 0
? CBM_CONFIG_EDIT_PATH_DIRECT
: CBM_CONFIG_EDIT_PATH_REFUSED;
}
int cbm_config_edit_target_create_temp(const cbm_config_edit_target_t *target, const char *name,
unsigned mode) {
(void)target;
(void)name;
(void)mode;
errno = EINVAL;
return -1;
}
int cbm_config_edit_target_commit(const cbm_config_edit_target_t *target, const char *name) {
(void)target;
(void)name;
errno = EINVAL;
return -1;
}
int cbm_config_edit_target_unlink(const cbm_config_edit_target_t *target, const char *name) {
(void)target;
(void)name;
errno = EINVAL;
return -1;
}
#endif /* _WIN32 */
int cbm_config_edit_target_open(const char *path, cbm_config_edit_target_t *target) {
if (!target) {
return CBM_CONFIG_EDIT_PATH_REFUSED;
}
int status = edit_path_classify(path, target, NULL, 0U);
target->status = status;
return status;
}
int cbm_config_edit_path_refusal(const char *path, char *out, size_t out_size) {
if (!out || out_size == 0U) {
return 0;
}
cbm_config_edit_target_t target;
(void)edit_path_classify(path, &target, out, out_size);
cbm_config_edit_target_close(&target);
return out[0] != '\0' ? 1 : 0;
}
src/cli/config_edit_path.h
+109
View File
@@ -0,0 +1,109 @@
/*
* config_edit_path.h — the path a configuration editor may operate on when
* the path it was given is a symlink.
*
* Every config editor (JSON-like, TOML, YAML, text) opens its file with
* O_NOFOLLOW and refuses symlinks: a link planted at ~/.cursor/mcp.json could
* otherwise point a privileged writer at any file its planter can name. Users,
* however, keep exactly those files as symlinks into a dotfiles checkout, and
* a blanket refusal turned `uninstall` into data loss (#1954).
*
* Decision C (2026-09-07): a symlink the INVOKING user owns, resolving to a
* regular file the same user owns, is followed — but only as an explicit
* OPT-IN of the agent-configuration writers, scoped to configuration roots
* (HOME / XDG dirs) they register with cbm_config_edit_path_follow_add_root().
* With no root registered the editors behave exactly as before; a path whose
* link lives outside every registered root is never followed, so repository
* paths stay refused whatever the caller.
*
* Race-safe sequence for a followed link (POSIX):
* lstat(link) → link owner == invoking uid (a root process follows only
* root-owned links) → realpath → open the target's PARENT DIRECTORY as a
* dirfd (O_DIRECTORY|O_NOFOLLOW; owned by the invoking user or root, not
* writable by others unless sticky) → openat(dirfd, base, O_NOFOLLOW) →
* fstat: regular file, owned by the invoking user, link count 1, and the
* same (st_dev, st_ino) as stat(realpath).
* Everything else is refused with an observable reason
* (cbm_config_edit_path_refusal): another owner, a root process over a
* non-root link, a dangling link, a link to a directory or special file, a
* hard-linked or swapped target, a foreign or world-writable parent. The
* editor then reads through the validated descriptor, stages its temp file
* with openat(dirfd, …, O_CREAT|O_EXCL) and commits with renameat on the same
* dirfd, so the link itself is never replaced. Windows never follows reparse
* points here: the editors keep FILE_FLAG_OPEN_REPARSE_POINT and the helper
* hands the path back unchanged.
*
* The check runs at every open and before every commit, never cached: a link
* swapped mid-edit resolves to a different identity, which also fails the
* editors' own pre-publish snapshot comparison on the resolved path.
*/
#ifndef CBM_CONFIG_EDIT_PATH_H
#define CBM_CONFIG_EDIT_PATH_H
#include <stddef.h>
enum {
/* Callers pass a buffer of at least this size (cbm_canonical_path's floor). */
CBM_CONFIG_EDIT_PATH_MAX = 4096,
CBM_CONFIG_EDIT_NAME_MAX = 256,
};
/* Pending user decision (#1954): must the resolved TARGET also stay under a
* registered root (1), or may it live anywhere the invoking user owns, such
* as a dotfiles checkout on an external volume (0, the default)? One switch. */
#ifndef CBM_CONFIG_EDIT_TARGET_UNDER_ROOT
#define CBM_CONFIG_EDIT_TARGET_UNDER_ROOT 0
#endif
typedef enum {
CBM_CONFIG_EDIT_PATH_REFUSED = -1, /* a link that must not be followed */
CBM_CONFIG_EDIT_PATH_DIRECT = 0, /* not a link (or absent): path is used as given */
CBM_CONFIG_EDIT_PATH_FOLLOWED = 1, /* user-owned link inside a root: the target is used */
} cbm_config_edit_path_status_t;
typedef struct {
int status; /* cbm_config_edit_path_status_t */
int dirfd; /* FOLLOWED: the target's parent directory, pinned; else -1 */
int fd; /* FOLLOWED: the validated target, O_RDONLY|O_NOFOLLOW; else -1 */
char base[CBM_CONFIG_EDIT_NAME_MAX]; /* FOLLOWED: the target's name inside dirfd */
char path[CBM_CONFIG_EDIT_PATH_MAX]; /* the path the editor operates on */
} cbm_config_edit_target_t;
/* Opt-in scope (per thread): links under these canonical roots may be
* followed. Up to four roots; returns 0 when registered. Clear when the
* agent-configuration work is done. */
int cbm_config_edit_path_follow_add_root(const char *root);
void cbm_config_edit_path_follow_clear(void);
/* Classifies path and, for a followed link, pins its parent directory and
* validates the target. Returns the status; on REFUSED the target holds no
* descriptors and an empty path, and the editor fails closed as before.
* Always pair with cbm_config_edit_target_close(). */
int cbm_config_edit_target_open(const char *path, cbm_config_edit_target_t *target);
void cbm_config_edit_target_close(cbm_config_edit_target_t *target);
/* FOLLOWED targets only (return -1 with errno = EINVAL otherwise):
* create a temp file `name` beside the target (openat O_CREAT|O_EXCL|
* O_NOFOLLOW; EEXIST when the name is taken), publish it over the target with
* renameat on the pinned directory (which is then fsync'ed), or discard it. */
int cbm_config_edit_target_create_temp(const cbm_config_edit_target_t *target, const char *name,
unsigned mode);
int cbm_config_edit_target_commit(const cbm_config_edit_target_t *target, const char *name);
int cbm_config_edit_target_unlink(const cbm_config_edit_target_t *target, const char *name);
/* Observed-fact reason a symlink at path is refused, for diagnostics
* ("symlink owned by uid 0, not the invoking user (uid 501)", "dangling
* symlink", …). Writes "" and returns 0 when path is not a refused link, 1
* when a reason was written. Never reports errno — it may be stale (#1537). */
int cbm_config_edit_path_refusal(const char *path, char *out, size_t out_size);
#ifdef CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API
/* Test seams (POSIX; no-ops on Windows). A test cannot create a link owned by
* another user without root, so the foreign-owner and root-run refusals move
* the observer instead of the file; the identity skew makes the validated
* descriptor disagree with stat(realpath), the swapped-target refusal. */
void cbm_config_edit_path_set_invoking_uid_for_test(unsigned uid, int set);
void cbm_config_edit_path_set_identity_skew_for_test(int set);
#endif
#endif /* CBM_CONFIG_EDIT_PATH_H */
src/cli/config_json_like.c
+125 -18
View File
@@ -8,6 +8,7 @@
*/
#include "cli/config_json_like.h"
#include "cli/config_edit_path.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
@@ -1223,8 +1224,19 @@ static int jl_make_insertion(const char *text, size_t length, size_t object_star
return 0;
}
if (object->close_pos == gap_start ||
!jl_is_space((unsigned char)text[object->close_pos - 1U])) {
/* Zero-width insertion right at close_pos: whatever byte already sits at
* close_pos - 1 is untouched and stays in the output. When the original
* had a real gap there (close_pos != gap_start, e.g. "[ \"a\" ]"), that
* preserved byte already supplies a separator on the leading side, so
* mirroring it on the trailing side (space before the bracket) matches
* the array's own loose style. The fully tight case ("[\"a\"]", nothing
* at all between the last value and the bracket) has no such byte to
* lean on: manufacture ONE space so `,new` reads `, new` — the
* comma-spacing convention every other insertion path uses — and add
* nothing after, so `new]` stays `new]` rather than gaining a trailing
* space the original never had (#1826 byte-for-byte round-trip). */
bool tight = object->close_pos == gap_start;
if (tight || !jl_is_space((unsigned char)text[object->close_pos - 1U])) {
if (jl_buffer_char(insertion, ' ') != 0) {
return -1;
}
@@ -1235,7 +1247,7 @@ static int jl_make_insertion(const char *text, size_t length, size_t object_star
if (object->trailing_comma && jl_buffer_char(insertion, ',') != 0) {
return -1;
}
return jl_buffer_char(insertion, ' ');
return tight ? 0 : jl_buffer_char(insertion, ' ');
}
static int jl_apply_edits(const char *source, size_t source_length, jl_edit_t *edits,
@@ -1429,11 +1441,24 @@ static int jl_read_file(const char *path, char **content_out, size_t *length_out
#ifdef O_CLOEXEC
flags |= O_CLOEXEC;
#endif
int descriptor = open(path, flags);
/* A symlink the invoking user owns, inside an opted-in configuration
* root, is read through the descriptor the helper validated on the
* target's pinned parent directory (#1954); everything else is opened by
* name with O_NOFOLLOW exactly as before. */
cbm_config_edit_target_t target;
if (cbm_config_edit_target_open(path, &target) < 0) {
return -1;
}
int descriptor = target.fd;
target.fd = -1;
if (target.status != CBM_CONFIG_EDIT_PATH_FOLLOWED) {
descriptor = open(target.path, flags);
}
cbm_config_edit_target_close(&target);
if (descriptor < 0) {
if (errno == ENOENT) {
struct stat path_state;
if (lstat(path, &path_state) == 0 || errno != ENOENT) {
if (lstat(target.path, &path_state) == 0 || errno != ENOENT) {
return -1;
}
*missing_out = true;
@@ -1598,10 +1623,31 @@ static int jl_replace_atomic(const char *temp_path, const char *path, bool desti
#endif
}
static int jl_write_atomic(const char *path, const char *content, size_t length,
const char *expected_content, size_t expected_length,
const jl_file_snapshot_t *expected_snapshot) {
if (jl_ensure_parent(path) != 0) {
static const char *jl_temp_name(const char *temp_path) {
const char *slash = strrchr(temp_path, '/');
return slash ? slash + 1 : temp_path;
}
/* Drop a staged temp file: through the pinned parent for a followed link,
* by name otherwise. */
static void jl_discard_temp(const cbm_config_edit_target_t *target, const char *temp_path) {
if (target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED) {
(void)cbm_config_edit_target_unlink(target, jl_temp_name(temp_path));
} else {
(void)cbm_unlink(temp_path);
}
}
/* For a followed link (#1954) the temp file is created with openat() beside
* the target and published with renameat() on the pinned parent, so the link
* itself is never replaced; every pre-publish comparison runs against the
* resolved path. A direct path keeps the by-name sequence unchanged. */
static int jl_write_atomic_at(const cbm_config_edit_target_t *target, const char *content,
size_t length, const char *expected_content, size_t expected_length,
const jl_file_snapshot_t *expected_snapshot) {
const char *path = target->path;
bool followed = target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED;
if (!followed && jl_ensure_parent(path) != 0) {
return -1;
}
size_t path_length = strlen(path);
@@ -1636,13 +1682,15 @@ static int jl_write_atomic(const char *path, const char *content, size_t length,
#ifdef O_CLOEXEC
flags |= O_CLOEXEC;
#endif
int descriptor = open(temp_path, flags, 0600);
int descriptor =
followed ? cbm_config_edit_target_create_temp(target, jl_temp_name(temp_path), 0600U)
: open(temp_path, flags, 0600);
if (descriptor >= 0) {
file = fdopen(descriptor, "wb");
if (!file) {
int saved_error = errno;
close(descriptor);
(void)cbm_unlink(temp_path);
jl_discard_temp(target, temp_path);
errno = saved_error;
}
}
@@ -1685,7 +1733,7 @@ static int jl_write_atomic(const char *path, const char *content, size_t length,
failed = true;
}
if (failed) {
cbm_unlink(temp_path);
jl_discard_temp(target, temp_path);
free(temp_path);
return -1;
}
@@ -1697,7 +1745,7 @@ static int jl_write_atomic(const char *path, const char *content, size_t length,
temp_missing || temp_length != length ||
(length != 0U && memcmp(temp_content, content, length) != 0)) {
free(temp_content);
cbm_unlink(temp_path);
jl_discard_temp(target, temp_path);
free(temp_path);
return -1;
}
@@ -1708,7 +1756,7 @@ static int jl_write_atomic(const char *path, const char *content, size_t length,
}
#endif
if (jl_snapshot_matches_path(path, expected_content, expected_length, expected_snapshot) != 0) {
cbm_unlink(temp_path);
jl_discard_temp(target, temp_path);
free(temp_path);
return -1;
}
@@ -1719,8 +1767,9 @@ static int jl_write_atomic(const char *path, const char *content, size_t length,
#endif
if (jl_snapshot_matches_path(path, expected_content, expected_length, expected_snapshot) != 0 ||
jl_snapshot_matches_path(temp_path, content, length, &temp_snapshot) != 0 ||
jl_replace_atomic(temp_path, path, expected_snapshot->exists) != 0) {
cbm_unlink(temp_path);
(followed ? cbm_config_edit_target_commit(target, jl_temp_name(temp_path))
: jl_replace_atomic(temp_path, path, expected_snapshot->exists)) != 0) {
jl_discard_temp(target, temp_path);
free(temp_path);
return -1;
}
@@ -1728,6 +1777,19 @@ static int jl_write_atomic(const char *path, const char *content, size_t length,
return 0;
}
static int jl_write_atomic(const char *requested_path, const char *content, size_t length,
const char *expected_content, size_t expected_length,
const jl_file_snapshot_t *expected_snapshot) {
cbm_config_edit_target_t target;
if (cbm_config_edit_target_open(requested_path, &target) < 0) {
return -1;
}
int result = jl_write_atomic_at(&target, content, length, expected_content, expected_length,
expected_snapshot);
cbm_config_edit_target_close(&target);
return result;
}
static int jl_decode_utf8(const unsigned char *text, size_t remaining, uint32_t *codepoint,
size_t *byte_count) {
if (remaining == 0U) {
@@ -2411,6 +2473,13 @@ int cbm_json_like_remove_entry_if_unchanged(const char *file_path, const char *c
expected_length);
}
/* True when [start, end) is the bare 4-byte literal token null (never a
* quoted "null" string, which is 6 bytes with the quotes) — the documented
* OpenHands "no list yet" shape for mcp_server_refs (#1826). */
static bool jl_is_null_literal(const char *text, size_t start, size_t end) {
return end - start == 4U && memcmp(text + start, "null", 4U) == 0;
}
int cbm_json_like_add_unique_string_at_path(const char *file_path, const char *const *object_path,
size_t path_len, const char *array_key,
const char *string_value) {
@@ -2502,6 +2571,24 @@ int cbm_json_like_add_unique_string_at_path(const char *file_path, const char *c
result = jl_insert_member(source, source_length, object_start, &object, object_path,
path_len, SIZE_MAX, array_key, array_json.data, array_json.length,
&updated, &updated_length);
} else if (jl_is_null_literal(source, object.match.value_start, object.match.value_end)) {
/* OpenHands profiles ship `"mcp_server_refs": null` as the documented
* "no list yet" shape (#1826). Splice the built one-element array over
* the bare null token in place, preserving every other byte (trailing
* comments included) the way the object-member and array-element
* edits below already do for their own value spans. */
size_t head = object.match.value_start;
size_t tail = object.match.value_end;
updated_length = head + array_json.length + (source_length - tail);
updated = (char *)malloc(updated_length + 1U);
if (!updated) {
result = -1;
} else {
memcpy(updated, source, head);
memcpy(updated + head, array_json.data, array_json.length);
memcpy(updated + head + array_json.length, source + tail, source_length - tail);
updated[updated_length] = '\0';
}
} else if (source[object.match.value_start] != '[') {
result = -1;
} else {
@@ -2764,6 +2851,24 @@ static int jl_decode_field_string(const char *text, size_t start, size_t end,
}
return jl_decode_string_value(text, start, end, value_out) == 0 ? 0 : 1;
}
if (shape == CBM_JSON_LIKE_VALUE_LITERAL) {
/* The value's token boundaries already exclude surrounding trivia
* (jl_parse_value advances pos past exactly the token). Copy the raw
* bytes verbatim so a quoted "true" never equals the bare literal
* true, and the caller's expected_string comparison decides match. */
if (start >= end) {
return 1;
}
size_t length = end - start;
char *copy = (char *)malloc(length + 1U);
if (!copy) {
return 1;
}
memcpy(copy, text + start, length);
copy[length] = '\0';
*value_out = copy;
return 0;
}
if (shape != CBM_JSON_LIKE_VALUE_SINGLE_STRING_ARRAY || start >= end || text[start] != '[') {
return 1;
}
@@ -2832,11 +2937,13 @@ int cbm_json_like_match_object_entry(const char *document, size_t document_lengt
size_t capture_count = 0U;
for (size_t i = 0U; i < field_count; ++i) {
if (!fields[i].key || fields[i].key[0] == '\0' ||
fields[i].shape > CBM_JSON_LIKE_VALUE_SINGLE_STRING_ARRAY ||
fields[i].shape > CBM_JSON_LIKE_VALUE_LITERAL ||
(fields[i].flags &
~(CBM_JSON_LIKE_FIELD_REQUIRED | CBM_JSON_LIKE_FIELD_CAPTURE_STRING)) != 0U ||
((fields[i].flags & CBM_JSON_LIKE_FIELD_CAPTURE_STRING) != 0U &&
fields[i].shape == CBM_JSON_LIKE_VALUE_EMPTY_ARRAY)) {
(fields[i].shape == CBM_JSON_LIKE_VALUE_EMPTY_ARRAY ||
fields[i].shape == CBM_JSON_LIKE_VALUE_LITERAL)) ||
(fields[i].shape == CBM_JSON_LIKE_VALUE_LITERAL && !fields[i].expected_string)) {
return -1;
}
capture_count += (fields[i].flags & CBM_JSON_LIKE_FIELD_CAPTURE_STRING) != 0U ? 1U : 0U;
src/cli/config_json_like.h
+5
View File
@@ -47,6 +47,11 @@ typedef enum {
CBM_JSON_LIKE_VALUE_STRING,
CBM_JSON_LIKE_VALUE_EMPTY_ARRAY,
CBM_JSON_LIKE_VALUE_SINGLE_STRING_ARRAY,
/* An exact bare token (true/false/null/a number) matched byte-for-byte
* against expected_string. Requires expected_string and may not be
* combined with CBM_JSON_LIKE_FIELD_CAPTURE_STRING — a fixed token has
* nothing meaningful to capture for the caller. */
CBM_JSON_LIKE_VALUE_LITERAL,
} cbm_json_like_value_shape_t;
enum {
src/cli/config_text_edit.c
+65 -13
View File
@@ -3,6 +3,7 @@
*/
#include "cli/config_text_edit.h"
#include "cli/config_edit_path.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
@@ -442,13 +443,26 @@ static int text_read_file(const char *path, char **data_out, size_t *len_out,
#ifdef O_CLOEXEC
flags |= O_CLOEXEC;
#endif
int descriptor = open(path, flags);
/* A symlink the invoking user owns, inside an opted-in configuration
* root, is read through the descriptor the helper validated on the
* target's pinned parent directory (#1954); everything else is opened by
* name with O_NOFOLLOW exactly as before. */
cbm_config_edit_target_t target;
if (cbm_config_edit_target_open(path, &target) < 0) {
return TEXT_ERROR;
}
int descriptor = target.fd;
target.fd = -1;
if (target.status != CBM_CONFIG_EDIT_PATH_FOLLOWED) {
descriptor = open(target.path, flags);
}
cbm_config_edit_target_close(&target);
if (descriptor < 0) {
if (errno != ENOENT) {
return TEXT_ERROR;
}
struct stat path_state;
if (lstat(path, &path_state) == 0 || errno != ENOENT) {
if (lstat(target.path, &path_state) == 0 || errno != ENOENT) {
return TEXT_ERROR;
}
char *empty = (char *)calloc(1U, 1U);
@@ -589,14 +603,35 @@ static int text_replace_file(const char *temp_path, const char *path, int destin
#endif
}
static int text_write_atomic_mode(const char *path, const char *new_data, size_t new_len,
const char *old_data, size_t old_len,
const text_file_snapshot_t *snapshot, int override_mode,
unsigned int requested_mode) {
static const char *text_temp_name(const char *temp_path) {
const char *slash = strrchr(temp_path, '/');
return slash ? slash + 1 : temp_path;
}
/* Drop a staged temp file: through the pinned parent for a followed link,
* by name otherwise. */
static void text_discard_temp(const cbm_config_edit_target_t *target, const char *temp_path) {
if (target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED) {
(void)cbm_config_edit_target_unlink(target, text_temp_name(temp_path));
} else {
(void)cbm_unlink(temp_path);
}
}
/* For a followed link (#1954) the temp file is created with openat() beside
* the target and published with renameat() on the pinned parent, so the link
* itself is never replaced; every pre-publish comparison runs against the
* resolved path. A direct path keeps the by-name sequence unchanged. */
static int text_write_atomic_mode_at(const cbm_config_edit_target_t *target, const char *new_data,
size_t new_len, const char *old_data, size_t old_len,
const text_file_snapshot_t *snapshot, int override_mode,
unsigned int requested_mode) {
if (new_len > TEXT_MAX_BYTES || old_len > TEXT_MAX_BYTES ||
!text_requested_mode_valid(override_mode, requested_mode)) {
return TEXT_ERROR;
}
const char *path = target->path;
bool followed = target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED;
int content_same =
new_len == old_len && (new_len == 0U || memcmp(new_data, old_data, new_len) == 0);
#ifndef _WIN32
@@ -643,13 +678,15 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t
#ifdef O_CLOEXEC
flags |= O_CLOEXEC;
#endif
int descriptor = open(temp_path, flags, 0600);
int descriptor =
followed ? cbm_config_edit_target_create_temp(target, text_temp_name(temp_path), 0600U)
: open(temp_path, flags, 0600);
if (descriptor >= 0) {
file = text_fdopen(descriptor, "wb");
if (!file) {
int saved_error = errno;
text_close(descriptor);
(void)cbm_unlink(temp_path);
text_discard_temp(target, temp_path);
errno = saved_error;
}
}
@@ -702,7 +739,7 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t
failed = 1;
}
if (failed) {
(void)cbm_unlink(temp_path);
text_discard_temp(target, temp_path);
free(temp_path);
return TEXT_ERROR;
}
@@ -718,7 +755,7 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t
!text_snapshot_publication_equal(&trusted_temp_snapshot, &temp_snapshot) ||
temp_len != new_len || (new_len != 0U && memcmp(temp_data, new_data, new_len) != 0)) {
free(temp_data);
(void)cbm_unlink(temp_path);
text_discard_temp(target, temp_path);
free(temp_path);
return TEXT_ERROR;
}
@@ -730,7 +767,7 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t
}
#endif
if (text_snapshot_matches_path(path, old_data, old_len, snapshot) != TEXT_OK) {
(void)cbm_unlink(temp_path);
text_discard_temp(target, temp_path);
free(temp_path);
return TEXT_ERROR;
}
@@ -741,8 +778,9 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t
#endif
if (text_snapshot_matches_path(path, old_data, old_len, snapshot) != TEXT_OK ||
text_snapshot_matches_path(temp_path, new_data, new_len, &temp_snapshot) != TEXT_OK ||
text_replace_file(temp_path, path, snapshot->exists) != TEXT_OK) {
(void)cbm_unlink(temp_path);
(followed ? cbm_config_edit_target_commit(target, text_temp_name(temp_path))
: text_replace_file(temp_path, path, snapshot->exists)) != TEXT_OK) {
text_discard_temp(target, temp_path);
free(temp_path);
return TEXT_ERROR;
}
@@ -750,6 +788,20 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t
return TEXT_OK;
}
static int text_write_atomic_mode(const char *requested_path, const char *new_data, size_t new_len,
const char *old_data, size_t old_len,
const text_file_snapshot_t *snapshot, int override_mode,
unsigned int requested_mode) {
cbm_config_edit_target_t target;
if (cbm_config_edit_target_open(requested_path, &target) < 0) {
return TEXT_ERROR;
}
int result = text_write_atomic_mode_at(&target, new_data, new_len, old_data, old_len, snapshot,
override_mode, requested_mode);
cbm_config_edit_target_close(&target);
return result;
}
static int text_write_atomic(const char *path, const char *new_data, size_t new_len,
const char *old_data, size_t old_len,
const text_file_snapshot_t *snapshot) {
src/cli/config_toml_edit.c
+80 -16
View File
@@ -3,6 +3,7 @@
*/
#include "cli/config_toml_edit.h"
#include "cli/config_edit_path.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
@@ -465,13 +466,26 @@ static int toml_read_file(const char *path, char **out_data, size_t *out_len,
#ifdef O_CLOEXEC
flags |= O_CLOEXEC;
#endif
int fd = open(path, flags);
/* A symlink the invoking user owns, inside an opted-in configuration
* root, is read through the descriptor the helper validated on the
* target's pinned parent directory (#1954); everything else is opened by
* name with O_NOFOLLOW exactly as before. */
cbm_config_edit_target_t target;
if (cbm_config_edit_target_open(path, &target) < 0) {
return TOML_EDIT_ERR;
}
int fd = target.fd;
target.fd = -1;
if (target.status != CBM_CONFIG_EDIT_PATH_FOLLOWED) {
fd = open(target.path, flags);
}
cbm_config_edit_target_close(&target);
if (fd < 0) {
if (errno != ENOENT) {
return TOML_EDIT_ERR;
}
struct stat path_state;
if (lstat(path, &path_state) == 0 || errno != ENOENT) {
if (lstat(target.path, &path_state) == 0 || errno != ENOENT) {
return TOML_EDIT_ERR;
}
char *empty = (char *)malloc(1U);
@@ -612,28 +626,65 @@ static int toml_replace_atomic(const char *temp_path, const char *path, int exis
#endif
}
static int toml_write_atomic(const char *path, const char *old_data, size_t old_len,
const char *new_data, size_t new_len,
const toml_file_snapshot_t *snapshot) {
static const char *toml_temp_name(const char *temp_path) {
const char *slash = strrchr(temp_path, '/');
return slash ? slash + 1 : temp_path;
}
/* Drop a staged temp file: through the pinned parent for a followed link,
* by name otherwise. */
static void toml_discard_temp(const cbm_config_edit_target_t *target, const char *temp_path) {
if (target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED) {
(void)cbm_config_edit_target_unlink(target, toml_temp_name(temp_path));
} else {
(void)cbm_unlink(temp_path);
}
}
/* For a followed link (#1954) the temp file is created with openat() beside
* the target and published with renameat() on the pinned parent, so the link
* itself is never replaced; every pre-publish comparison runs against the
* resolved path. A direct path keeps the mkstemp + rename sequence unchanged. */
static int toml_write_atomic_at(const cbm_config_edit_target_t *target, const char *old_data,
size_t old_len, const char *new_data, size_t new_len,
const toml_file_snapshot_t *snapshot) {
if (old_len > TOML_EDIT_MAX_BYTES || new_len > TOML_EDIT_MAX_BYTES) {
return TOML_EDIT_ERR;
}
const char *path = target->path;
bool followed = target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED;
if (old_len == new_len && (old_len == 0 || memcmp(old_data, new_data, old_len) == 0)) {
return TOML_EDIT_OK;
}
size_t path_len = strlen(path);
static const char suffix[] = ".XXXXXX";
if (path_len > SIZE_MAX - sizeof(suffix)) {
enum { TOML_TEMP_SUFFIX_SPACE = 48, TOML_TEMP_ATTEMPTS = 64 };
if (path_len > SIZE_MAX - TOML_TEMP_SUFFIX_SPACE) {
return TOML_EDIT_ERR;
}
char *temp_path = (char *)malloc(path_len + sizeof(suffix));
size_t temp_capacity = path_len + TOML_TEMP_SUFFIX_SPACE;
char *temp_path = (char *)malloc(temp_capacity);
if (!temp_path) {
return TOML_EDIT_ERR;
}
memcpy(temp_path, path, path_len);
memcpy(temp_path + path_len, suffix, sizeof(suffix));
int fd = cbm_mkstemp(temp_path);
int fd = -1;
if (followed) {
for (unsigned attempt = 0U; attempt < TOML_TEMP_ATTEMPTS && fd < 0; attempt++) {
int written = snprintf(temp_path, temp_capacity, "%s.cbm-toml-%u.tmp", path, attempt);
if (written < 0 || (size_t)written >= temp_capacity) {
break;
}
fd = cbm_config_edit_target_create_temp(target, toml_temp_name(temp_path), 0600U);
if (fd < 0 && errno != EEXIST) {
break;
}
}
} else {
memcpy(temp_path, path, path_len);
memcpy(temp_path + path_len, suffix, sizeof(suffix));
fd = cbm_mkstemp(temp_path);
}
if (fd < 0) {
free(temp_path);
return TOML_EDIT_ERR;
@@ -641,7 +692,7 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_
FILE *file = toml_fdopen(fd, "wb");
if (!file) {
(void)toml_close(fd);
(void)cbm_unlink(temp_path);
toml_discard_temp(target, temp_path);
free(temp_path);
return TOML_EDIT_ERR;
}
@@ -667,7 +718,7 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_
failed = 1;
}
if (failed) {
(void)cbm_unlink(temp_path);
toml_discard_temp(target, temp_path);
free(temp_path);
return TOML_EDIT_ERR;
}
@@ -678,7 +729,7 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_
!temp_snapshot.exists || temp_len != new_len ||
(new_len != 0U && memcmp(temp_data, new_data, new_len) != 0)) {
free(temp_data);
(void)cbm_unlink(temp_path);
toml_discard_temp(target, temp_path);
free(temp_path);
return TOML_EDIT_ERR;
}
@@ -689,7 +740,7 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_
}
#endif
if (toml_snapshot_matches_path(path, old_data, old_len, snapshot) != TOML_EDIT_OK) {
(void)cbm_unlink(temp_path);
toml_discard_temp(target, temp_path);
free(temp_path);
return TOML_EDIT_ERR;
}
@@ -700,8 +751,9 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_
#endif
if (toml_snapshot_matches_path(path, old_data, old_len, snapshot) != TOML_EDIT_OK ||
toml_snapshot_matches_path(temp_path, new_data, new_len, &temp_snapshot) != TOML_EDIT_OK ||
toml_replace_atomic(temp_path, path, snapshot->exists) != TOML_EDIT_OK) {
(void)cbm_unlink(temp_path);
(followed ? cbm_config_edit_target_commit(target, toml_temp_name(temp_path))
: toml_replace_atomic(temp_path, path, snapshot->exists)) != TOML_EDIT_OK) {
toml_discard_temp(target, temp_path);
free(temp_path);
return TOML_EDIT_ERR;
}
@@ -709,6 +761,18 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_
return TOML_EDIT_OK;
}
static int toml_write_atomic(const char *requested_path, const char *old_data, size_t old_len,
const char *new_data, size_t new_len,
const toml_file_snapshot_t *snapshot) {
cbm_config_edit_target_t target;
if (cbm_config_edit_target_open(requested_path, &target) < 0) {
return TOML_EDIT_ERR;
}
int result = toml_write_atomic_at(&target, old_data, old_len, new_data, new_len, snapshot);
cbm_config_edit_target_close(&target);
return result;
}
#ifdef CBM_TOML_EDIT_ENABLE_TEST_API
void cbm_toml_set_precommit_hook_for_testing(cbm_toml_precommit_test_hook_t hook, void *context) {
toml_precommit_test_hook = hook;
src/cli/config_yaml_edit.c
+63 -12
View File
@@ -7,6 +7,7 @@
*/
#include "cli/config_yaml_edit.h"
#include "cli/config_edit_path.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
@@ -694,13 +695,26 @@ static int yaml_read_file(const char *path, char **out_data, size_t *out_len,
#ifdef O_CLOEXEC
flags |= O_CLOEXEC;
#endif
int descriptor = open(path, flags);
/* A symlink the invoking user owns, inside an opted-in configuration
* root, is read through the descriptor the helper validated on the
* target's pinned parent directory (#1954); everything else is opened by
* name with O_NOFOLLOW exactly as before. */
cbm_config_edit_target_t target;
if (cbm_config_edit_target_open(path, &target) < 0) {
return YAML_ERROR;
}
int descriptor = target.fd;
target.fd = -1;
if (target.status != CBM_CONFIG_EDIT_PATH_FOLLOWED) {
descriptor = open(target.path, flags);
}
cbm_config_edit_target_close(&target);
if (descriptor < 0) {
if (errno != ENOENT) {
return YAML_ERROR;
}
struct stat path_state;
if (lstat(path, &path_state) == 0 || errno != ENOENT) {
if (lstat(target.path, &path_state) == 0 || errno != ENOENT) {
return YAML_ERROR;
}
char *empty = (char *)calloc(YAML_UNIT, YAML_UNIT);
@@ -869,9 +883,30 @@ static int yaml_replace_file(const char *temp_path, const char *path, bool desti
#endif
}
static int yaml_write_atomic(const char *path, const char *data, size_t len,
const char *expected_data, size_t expected_len,
const yaml_file_snapshot_t *expected_snapshot) {
static const char *yaml_temp_name(const char *temp_path) {
const char *slash = strrchr(temp_path, '/');
return slash ? slash + 1 : temp_path;
}
/* Drop a staged temp file: through the pinned parent for a followed link,
* by name otherwise. */
static void yaml_discard_temp(const cbm_config_edit_target_t *target, const char *temp_path) {
if (target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED) {
(void)cbm_config_edit_target_unlink(target, yaml_temp_name(temp_path));
} else {
(void)cbm_unlink(temp_path);
}
}
/* For a followed link (#1954) the temp file is created with openat() beside
* the target and published with renameat() on the pinned parent, so the link
* itself is never replaced; every pre-publish comparison runs against the
* resolved path. A direct path keeps the by-name sequence unchanged. */
static int yaml_write_atomic_at(const cbm_config_edit_target_t *target, const char *data,
size_t len, const char *expected_data, size_t expected_len,
const yaml_file_snapshot_t *expected_snapshot) {
const char *path = target->path;
bool followed = target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED;
size_t path_len = 0U;
if (yaml_bounded_strlen(path, YAML_OUTPUT_MAX, &path_len) != 0 ||
path_len > SIZE_MAX - YAML_TMP_SUFFIX_MAX - YAML_UNIT) {
@@ -901,12 +936,14 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len,
#ifdef O_CLOEXEC
flags |= O_CLOEXEC;
#endif
int descriptor = open(temp_path, flags, YAML_NEW_FILE_MODE);
int descriptor = followed ? cbm_config_edit_target_create_temp(
target, yaml_temp_name(temp_path), YAML_NEW_FILE_MODE)
: open(temp_path, flags, YAML_NEW_FILE_MODE);
if (descriptor >= 0) {
file = fdopen(descriptor, "wb");
if (!file) {
(void)close(descriptor);
(void)cbm_unlink(temp_path);
yaml_discard_temp(target, temp_path);
free(temp_path);
return YAML_ERROR;
}
@@ -947,7 +984,7 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len,
failed = true;
}
if (failed) {
(void)cbm_unlink(temp_path);
yaml_discard_temp(target, temp_path);
free(temp_path);
return YAML_ERROR;
}
@@ -958,7 +995,7 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len,
!temp_snapshot.exists || temp_len != len ||
(len != 0U && memcmp(temp_data, data, len) != 0)) {
free(temp_data);
(void)cbm_unlink(temp_path);
yaml_discard_temp(target, temp_path);
free(temp_path);
return YAML_ERROR;
}
@@ -970,7 +1007,7 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len,
}
#endif
if (yaml_snapshot_matches_path(path, expected_data, expected_len, expected_snapshot) != 0) {
(void)cbm_unlink(temp_path);
yaml_discard_temp(target, temp_path);
free(temp_path);
return YAML_ERROR;
}
@@ -981,8 +1018,9 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len,
#endif
if (yaml_snapshot_matches_path(path, expected_data, expected_len, expected_snapshot) != 0 ||
yaml_snapshot_matches_path(temp_path, data, len, &temp_snapshot) != 0 ||
yaml_replace_file(temp_path, path, expected_snapshot->exists) != 0) {
(void)cbm_unlink(temp_path);
(followed ? cbm_config_edit_target_commit(target, yaml_temp_name(temp_path))
: yaml_replace_file(temp_path, path, expected_snapshot->exists)) != 0) {
yaml_discard_temp(target, temp_path);
free(temp_path);
return YAML_ERROR;
}
@@ -990,6 +1028,19 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len,
return 0;
}
static int yaml_write_atomic(const char *requested_path, const char *data, size_t len,
const char *expected_data, size_t expected_len,
const yaml_file_snapshot_t *expected_snapshot) {
cbm_config_edit_target_t target;
if (cbm_config_edit_target_open(requested_path, &target) < 0) {
return YAML_ERROR;
}
int result =
yaml_write_atomic_at(&target, data, len, expected_data, expected_len, expected_snapshot);
cbm_config_edit_target_close(&target);
return result;
}
#ifdef CBM_YAML_ENABLE_TEST_API
void cbm_yaml_set_precommit_hook_for_testing(cbm_yaml_precommit_test_hook_t hook, void *context) {
yaml_precommit_test_hook = hook;
src/cypher/cypher.c
+134 -6
View File
@@ -1830,20 +1830,27 @@ tail:
}
}
/* Optional SKIP */
/* Optional SKIP. A non-numeric operand is a loud parse error: expect() has
* already filled p->error, and dropping the clause instead would leave
* r->skip at its "none" default while the query still reports success. */
if (match(p, TOK_SKIP)) {
const cbm_token_t *num = expect(p, TOK_NUMBER);
if (num) {
r->skip = (int)strtol(num->text, NULL, CBM_DECIMAL_BASE);
if (!num) {
free_return_clause(r);
return CBM_NOT_FOUND;
}
r->skip = (int)strtol(num->text, NULL, CBM_DECIMAL_BASE);
}
/* Optional LIMIT */
/* Optional LIMIT. Same rule: a dropped LIMIT is the failure mode #1334
* banned - the caller would silently receive the whole result set. */
if (match(p, TOK_LIMIT)) {
const cbm_token_t *num = expect(p, TOK_NUMBER);
if (num) {
r->limit = (int)strtol(num->text, NULL, CBM_DECIMAL_BASE);
if (!num) {
free_return_clause(r);
return CBM_NOT_FOUND;
}
r->limit = (int)strtol(num->text, NULL, CBM_DECIMAL_BASE);
}
*out = r;
@@ -4450,11 +4457,33 @@ static void ret_agg_init_group(ret_agg_entry_t *entry, const char *key, int item
}
/* Accumulate a binding into RETURN aggregation */
static bool binding_has_value(binding_t *b, const char *var, const char *prop) {
if (!var || strcmp(var, "*") == 0) {
return true;
}
char full[CBM_SZ_256];
if (prop) {
snprintf(full, sizeof(full), "%s.%s", var, prop);
} else {
snprintf(full, sizeof(full), "%s", var);
}
for (int i = 0; i < b->var_count; i++) {
if (strcmp(b->var_names[i], full) == 0) {
return true;
}
}
return binding_get_edge(b, var) != NULL || binding_get(b, var) != NULL;
}
static void ret_agg_accumulate(ret_agg_entry_t *entry, cbm_return_clause_t *ret, binding_t *b) {
for (int ci = 0; ci < ret->count; ci++) {
if (!is_aggregate_func(ret->items[ci].func)) {
continue;
}
if (strcmp(ret->items[ci].func, "COUNT") == 0 &&
!binding_has_value(b, ret->items[ci].variable, ret->items[ci].property)) {
continue;
}
entry->counts[ci]++;
const char *raw = binding_get_virtual(b, ret->items[ci].variable, ret->items[ci].property);
double dv = strtod(raw, NULL);
@@ -4985,8 +5014,107 @@ static void execute_return_clause(cbm_query_t *q, cbm_return_clause_t *ret, bind
limit_is_engine_budget);
}
/* ── Planner: seed a single-hop pattern from its selective end ──────────────
*
* The executor scans pattern node 0 and expands from there. Written as
* `MATCH (a)-[:CALLS]->(b) WHERE b.name = '_printk'`, that scans every node
* in the store before the filter on `b` can discard any — on an 8.5 M node
* kernel graph the query hit the execution-time limit while the anchored
* spelling `MATCH (b:Function {name:'_printk'})<-[:CALLS]-(a)` answered in
* seconds (2026-09-16 probe). A reader should not have to know which end to
* write first: when the far node carries an equality on name/qualified_name,
* an inline property, or a label and the near node carries less, the pattern
* is walked from the far end with the relationship direction inverted. The
* result set is identical; only the enumeration order changes. Restricted to
* one single-relationship pattern with an explicit RETURN, and never to an
* OPTIONAL MATCH, whose anchor is part of its meaning. */
static int cypher_cond_selectivity(const cbm_condition_t *c, const char *var) {
if (!c || !c->variable || !var || strcmp(c->variable, var) != 0 || c->negated) {
return 0;
}
if (!c->op || strcmp(c->op, "=") != 0 || !c->property) {
return 0;
}
return (strcmp(c->property, "name") == 0 || strcmp(c->property, "qualified_name") == 0) ? 3 : 2;
}
static int cypher_expr_selectivity(const cbm_expr_t *e, const char *var) {
if (!e) {
return 0;
}
if (e->type == EXPR_CONDITION) {
return cypher_cond_selectivity(&e->cond, var);
}
if (e->type == EXPR_AND) {
int l = cypher_expr_selectivity(e->left, var);
int r = cypher_expr_selectivity(e->right, var);
return l > r ? l : r;
}
return 0; /* OR / NOT / XOR: no single equality to seed from */
}
static int cypher_node_selectivity(const cbm_node_pattern_t *n, const cbm_where_clause_t *w) {
int s = 0;
for (int i = 0; i < n->prop_count; i++) {
const char *k = n->props[i].key;
int v = (k && (strcmp(k, "name") == 0 || strcmp(k, "qualified_name") == 0)) ? 3 : 2;
if (v > s) {
s = v;
}
}
if (w && n->variable) {
int v = 0;
if (w->root) {
v = cypher_expr_selectivity(w->root, n->variable);
} else if (!w->op || strcmp(w->op, "AND") == 0) {
for (int i = 0; i < w->count; i++) {
int c = cypher_cond_selectivity(&w->conditions[i], n->variable);
if (c > v) {
v = c;
}
}
}
if (v > s) {
s = v;
}
}
if (s == 0 && n->label) {
s = 1;
}
return s;
}
static void cypher_plan_seed_from_selective_end(cbm_query_t *q) {
if (!q || q->pattern_count != 1 || !q->ret || (q->pattern_optional && q->pattern_optional[0])) {
return;
}
cbm_pattern_t *p = &q->patterns[0];
if (p->rel_count != 1 || p->node_count != 2) {
return;
}
if (cypher_node_selectivity(&p->nodes[1], q->where) <=
cypher_node_selectivity(&p->nodes[0], q->where)) {
return;
}
cbm_node_pattern_t tmp = p->nodes[0];
p->nodes[0] = p->nodes[1];
p->nodes[1] = tmp;
cbm_rel_pattern_t *r = &p->rels[0];
const char *inverted = NULL;
if (r->direction && strcmp(r->direction, "outbound") == 0) {
inverted = "inbound";
} else if (r->direction && strcmp(r->direction, "inbound") == 0) {
inverted = "outbound";
}
if (inverted) {
safe_str_free(&r->direction);
r->direction = heap_strdup(inverted);
}
}
static int execute_single(cbm_store_t *store, cbm_query_t *q, const char *project, int max_rows,
result_builder_t *rb) {
cypher_plan_seed_from_selective_end(q);
cbm_pattern_t *pat0 = &q->patterns[0];
/* Step 1: Scan initial nodes */
src/daemon/application.c
+29 -6
View File
@@ -208,7 +208,8 @@ static bool application_unique_recovery_file(char out[APPLICATION_PATH_CAP], con
static bool application_update_reap(cbm_daemon_application_t *application, bool wait,
uint32_t timeout_ms);
static void *application_job_thread(void *opaque);
static char *application_auto_index_args(const char *root_path);
static char *application_auto_index_args(cbm_daemon_application_t *application,
const char *root_path);
static cbm_daemon_application_job_t *application_job_subscribe_locked(
cbm_daemon_application_t *application, const char *project_key, const char *root_path,
const char *args_json, application_job_subscribe_status_t *status_out);
@@ -1455,7 +1456,7 @@ static void application_auto_index_retry_pending_locked(cbm_daemon_application_t
application_refresh_watch_locked(session);
continue;
}
char *args = application_auto_index_args(root_path);
char *args = application_auto_index_args(application, root_path);
if (!args) {
continue;
}
@@ -2005,7 +2006,27 @@ static bool application_update_reap(cbm_daemon_application_t *application, bool
}
}
static char *application_auto_index_args(const char *root_path) {
static bool application_index_args_add_policy(cbm_daemon_application_t *application,
yyjson_mut_doc *document, yyjson_mut_val *root) {
cbm_config_t *owned_config = NULL;
cbm_config_t *config = application ? application->config : NULL;
if (!config) {
owned_config = cbm_config_open(cbm_resolve_cache_dir());
config = owned_config;
}
cbm_index_resource_policy_t policy;
char error[CBM_SZ_256] = {0};
bool loaded = cbm_config_load_index_policy(config, &policy, error, sizeof(error));
cbm_config_close(owned_config);
if (!loaded) {
cbm_log_error("daemon.index.policy", "error", error);
return false;
}
return cbm_mcp_index_policy_add_to_args(document, root, &policy);
}
static char *application_auto_index_args(cbm_daemon_application_t *application,
const char *root_path) {
yyjson_mut_doc *document = yyjson_mut_doc_new(NULL);
yyjson_mut_val *root = document ? yyjson_mut_obj(document) : NULL;
if (!document || !root) {
@@ -2013,7 +2034,8 @@ static char *application_auto_index_args(const char *root_path) {
return NULL;
}
yyjson_mut_doc_set_root(document, root);
char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path)
char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) &&
application_index_args_add_policy(application, document, root)
? yyjson_mut_write(document, 0, NULL)
: NULL;
yyjson_mut_doc_free(document);
@@ -2060,7 +2082,7 @@ static void application_background_initialize_impl(cbm_daemon_application_sessio
files);
}
bool args_required = auto_index_candidate && within_auto_index_limit;
char *args = args_required ? application_auto_index_args(root_path) : NULL;
char *args = args_required ? application_auto_index_args(application, root_path) : NULL;
application_jobs_reap_completed(application);
cbm_mutex_lock(&application->mutex);
if (application->stopping || application_request_cancelled_locked(session)) {
@@ -3458,7 +3480,8 @@ static int application_background_index(cbm_daemon_application_t *application,
return -1;
}
yyjson_mut_doc_set_root(document, root);
bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root);
bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) &&
application_index_args_add_policy(application, document, root);
char *default_project = cbm_project_name_from_path(canonical_root);
bool custom_project =
project_name[0] && (!default_project || strcmp(default_project, project_name) != 0);
src/daemon/bootstrap.c
+338 -2
View File
@@ -6,13 +6,17 @@
#include "daemon/ipc.h"
#include "daemon/service.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include "foundation/log.h"
#include "foundation/platform.h"
#include <errno.h>
#include <limits.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#ifdef _WIN32
#ifndef WIN32_LEAN_AND_MEAN
@@ -38,6 +42,14 @@ extern char **environ;
enum {
BOOTSTRAP_RETRY_NS = 1000000,
/* #1828: how often a waiting client re-reads the start-failure log
* (the wait loop itself ticks every millisecond). */
BOOTSTRAP_START_FAILURE_CHECK_MS = 50,
/* A record this many seconds OLDER than the client's own spawn instant is
* still accepted: a sibling client's attempt that just failed is the same
* evidence, and second-granularity clocks need the slack anyway. */
BOOTSTRAP_START_FAILURE_SKEW_S = 2,
BOOTSTRAP_START_FAILURE_LOG_CAP = 65536,
BOOTSTRAP_COORDINATION_CLEANUP_MS = 500,
BOOTSTRAP_PATH_CAP = 4096,
};
@@ -292,6 +304,270 @@ static _Noreturn void bootstrap_cleanup_fail_stop(const char *component) {
#endif
}
static const char BOOTSTRAP_START_FAILURE_LOG_NAME[] = "cbm-daemon.start-failures.log";
bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity) {
if (!out || capacity == 0) {
return false;
}
out[0] = '\0';
const char *cache = cbm_resolve_cache_dir();
if (!cache || !cache[0]) {
return false;
}
int written = snprintf(out, capacity, "%s/logs", cache);
if (written <= 0 || (size_t)written >= capacity) {
out[0] = '\0';
return false;
}
return true;
}
/* One record field: the line format is tab-separated, so a tab or newline in
* a value (a path, in theory) becomes '?' rather than a parser ambiguity. */
static void bootstrap_record_field(char *out, size_t capacity, const char *value) {
size_t used = 0;
for (const char *cursor = value ? value : ""; *cursor && used + 1 < capacity; cursor++) {
unsigned char ch = (unsigned char)*cursor;
out[used++] = ch == '\t' || ch == '\n' || ch == '\r' ? '?' : (char)ch;
}
out[used] = '\0';
}
static uint64_t bootstrap_current_pid(void) {
#ifdef _WIN32
return (uint64_t)GetCurrentProcessId();
#else
return (uint64_t)getpid();
#endif
}
bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
const char *component) {
if (!log_directory || !log_directory[0] || !endpoint || !component || !component[0]) {
return false;
}
const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint);
if (!runtime_dir || !runtime_dir[0]) {
return false;
}
cbm_daemon_ipc_listen_failure_t detail;
if (!cbm_daemon_ipc_listen_failure_detail(&detail)) {
memset(&detail, 0, sizeof(detail));
}
char safe_runtime_dir[BOOTSTRAP_PATH_CAP];
char safe_component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP];
char safe_stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP];
char safe_path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP];
bootstrap_record_field(safe_runtime_dir, sizeof(safe_runtime_dir), runtime_dir);
bootstrap_record_field(safe_component, sizeof(safe_component), component);
bootstrap_record_field(safe_stage, sizeof(safe_stage), detail.stage);
bootstrap_record_field(safe_path, sizeof(safe_path), detail.path);
FILE *stream = cbm_daemon_ipc_private_log_open(log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME,
BOOTSTRAP_START_FAILURE_LOG_CAP);
if (!stream) {
return false;
}
int written =
fprintf(stream, "v1\t%llu\t%llu\t%s\t%s\t%s\t%d\t%s\n", (unsigned long long)time(NULL),
(unsigned long long)bootstrap_current_pid(), safe_runtime_dir, safe_component,
safe_stage, detail.errno_value, safe_path);
bool ok = written > 0 && fflush(stream) == 0;
if (fclose(stream) != 0) {
ok = false;
}
return ok;
}
/* Parse one "v1\tts\tpid\truntime_dir\tcomponent\tstage\terrno\tpath" line
* (line is NUL-terminated, newline already stripped, and is modified). */
static bool bootstrap_start_failure_parse(char *line, char **runtime_dir_out,
cbm_daemon_bootstrap_start_failure_t *out) {
char *fields[8];
size_t count = 0;
char *cursor = line;
while (count < 8) {
fields[count++] = cursor;
char *tab = strchr(cursor, '\t');
if (!tab) {
break;
}
*tab = '\0';
cursor = tab + 1;
}
if (count != 8 || strcmp(fields[0], "v1") != 0) {
return false;
}
memset(out, 0, sizeof(*out));
out->recorded_at_s = strtoull(fields[1], NULL, 10);
out->pid = strtoull(fields[2], NULL, 10);
*runtime_dir_out = fields[3];
(void)snprintf(out->component, sizeof(out->component), "%s", fields[4]);
(void)snprintf(out->stage, sizeof(out->stage), "%s", fields[5]);
out->errno_value = (int)strtol(fields[6], NULL, 10);
(void)snprintf(out->path, sizeof(out->path), "%s", fields[7]);
return true;
}
static FILE *bootstrap_start_failure_open(const char *log_directory, int *status_out) {
char path[BOOTSTRAP_PATH_CAP];
int written =
snprintf(path, sizeof(path), "%s/%s", log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME);
if (written <= 0 || written >= (int)sizeof(path)) {
*status_out = -1;
return NULL;
}
#ifndef _WIN32
struct stat status;
if (lstat(path, &status) != 0) {
*status_out = errno == ENOENT ? 0 : -1;
return NULL;
}
if (!S_ISREG(status.st_mode) || status.st_uid != geteuid() || status.st_nlink != 1) {
*status_out = -1;
return NULL;
}
#endif
/* The directory is the trust boundary (owner-only, no symlinked
* ancestry); the same validation guards cbm-daemon.log itself. */
if (!cbm_daemon_ipc_private_directory_secure(log_directory)) {
*status_out = -1;
return NULL;
}
FILE *file = cbm_fopen(path, "rb");
if (!file) {
*status_out = errno == ENOENT ? 0 : -1;
return NULL;
}
*status_out = 1;
return file;
}
int cbm_daemon_bootstrap_start_failure_read(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure) {
if (!out_failure) {
return -1;
}
memset(out_failure, 0, sizeof(*out_failure));
if (!log_directory || !log_directory[0] || !endpoint) {
return -1;
}
const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint);
if (!runtime_dir || !runtime_dir[0]) {
return -1;
}
int status = 0;
FILE *file = bootstrap_start_failure_open(log_directory, &status);
if (!file) {
return status;
}
char *buffer = malloc(BOOTSTRAP_START_FAILURE_LOG_CAP + 1U);
if (!buffer) {
(void)fclose(file);
return -1;
}
size_t used = fread(buffer, 1, BOOTSTRAP_START_FAILURE_LOG_CAP, file);
bool read_ok = !ferror(file);
(void)fclose(file);
buffer[used] = '\0';
int found = 0;
if (read_ok) {
char *line = buffer;
while (line && *line) {
char *end = strchr(line, '\n');
if (!end) {
break; /* a partial trailing line is a record still being written */
}
*end = '\0';
char *record_runtime_dir = NULL;
cbm_daemon_bootstrap_start_failure_t candidate;
if (bootstrap_start_failure_parse(line, &record_runtime_dir, &candidate) &&
strcmp(record_runtime_dir, runtime_dir) == 0 &&
candidate.recorded_at_s >= not_before_s) {
*out_failure = candidate; /* later lines are newer */
found = 1;
}
line = end + 1;
}
} else {
found = -1;
}
free(buffer);
return found;
}
void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure,
const char *log_directory, char *out,
size_t capacity) {
if (!out || capacity == 0) {
return;
}
if (!failure) {
out[0] = '\0';
return;
}
const char *what = failure->stage[0] ? failure->stage
: failure->component[0] ? failure->component
: "startup";
char where[BOOTSTRAP_PATH_CAP];
if (log_directory && log_directory[0]) {
(void)snprintf(where, sizeof(where), "see %s/cbm-daemon.log", log_directory);
} else {
(void)snprintf(where, sizeof(where), "see the daemon log");
}
if (failure->errno_value != 0 && failure->path[0]) {
(void)snprintf(out, capacity,
"CBM daemon failed to start: %s failed with %s (%s) at %s; %s", what,
cbm_errno_name(failure->errno_value), strerror(failure->errno_value),
failure->path, where);
} else if (failure->errno_value != 0) {
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed with %s (%s); %s",
what, cbm_errno_name(failure->errno_value), strerror(failure->errno_value),
where);
} else if (failure->path[0]) {
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed at %s; %s", what,
failure->path, where);
} else {
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed; %s", what, where);
}
}
/* After this attempt spawned a daemon: did that daemon (or a sibling attempt
* moments earlier) record a start failure? A found record ends the wait with
* its cause -- respawning a deterministically failing daemon until the
* deadline only produced "active or starting" (#1828). */
static bool bootstrap_start_failure_detected(const cbm_daemon_bootstrap_config_t *config,
const cbm_daemon_bootstrap_ops_t *ops,
uint64_t spawn_wall_s, bool force,
uint64_t *next_check_ms,
cbm_daemon_bootstrap_result_t *result_out) {
if (spawn_wall_s == 0 || !ops->start_failure_probe) {
return false;
}
uint64_t now_ms = cbm_now_ms();
if (!force && now_ms < *next_check_ms) {
return false;
}
*next_check_ms = now_ms + BOOTSTRAP_START_FAILURE_CHECK_MS;
uint64_t not_before_s = spawn_wall_s > BOOTSTRAP_START_FAILURE_SKEW_S
? spawn_wall_s - BOOTSTRAP_START_FAILURE_SKEW_S
: 0;
cbm_daemon_bootstrap_start_failure_t failure;
if (ops->start_failure_probe(ops->context, config->endpoint, not_before_s, &failure) != 1) {
return false;
}
char logs[BOOTSTRAP_PATH_CAP];
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) {
logs[0] = '\0';
}
cbm_daemon_bootstrap_start_failure_format(&failure, logs, result_out->message,
sizeof(result_out->message));
return true;
}
static void bootstrap_pause(uint64_t deadline) {
uint64_t now = cbm_now_ms();
if (now >= deadline) {
@@ -458,7 +734,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
bool lock_acquired = false;
bool generation_observed = probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL;
uint64_t spawn_wall_s = 0;
uint64_t next_failure_check_ms = 0;
bool start_failed = false;
while (cbm_now_ms() < deadline) {
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false,
&next_failure_check_ms, result_out)) {
start_failed = true;
break;
}
if (!bootstrap_probe_is_waitable(probe)) {
break;
}
@@ -508,11 +792,20 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
break;
}
/* Never relaunch a daemon whose predecessor already recorded why it
* could not start; the record is written before that daemon lets go
* of its lifetime reservation, so this unconditional read sees it. */
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, true,
&next_failure_check_ms, result_out)) {
start_failed = true;
break;
}
cbm_daemon_bootstrap_launch_spec_t spec;
bool spec_ready =
config->spawn_permanent
? cbm_daemon_bootstrap_launch_spec_init_permanent(config->executable_path, &spec)
: cbm_daemon_bootstrap_launch_spec_init(config->executable_path, &spec);
spawn_wall_s = (uint64_t)time(NULL);
if (!spec_ready || !ops->startup_lock_prepare_handoff(ops->context, startup_lock) ||
!ops->spawn_daemon(ops->context, &spec)) {
probe = CBM_DAEMON_BOOTSTRAP_PROBE_ERROR;
@@ -537,7 +830,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
if (!bootstrap_probe_is_waitable(probe)) {
break;
}
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false,
&next_failure_check_ms, result_out)) {
start_failed = true;
break;
}
} while (cbm_now_ms() < deadline);
if (start_failed) {
break;
}
if (!lock_acquired) {
continue;
}
@@ -555,7 +856,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
}
result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED;
if (muted_holder_pid != 0) {
if (start_failed) {
/* result_out->message already names the recorded cause. */
} else if (muted_holder_pid != 0) {
/* The one diagnostic the 2026-08-29 zombie recovery had to assemble by
* hand from process, pipe, and log correlation: name the pid that
* holds the endpoint without answering, and say what to do with it. */
@@ -773,6 +1076,17 @@ static bool bootstrap_production_unlock(void *context, cbm_daemon_bootstrap_lock
return released;
}
static int bootstrap_production_start_failure_probe(
void *context, const cbm_daemon_ipc_endpoint_t *endpoint, uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure) {
(void)context;
char logs[BOOTSTRAP_PATH_CAP];
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) {
return -1;
}
return cbm_daemon_bootstrap_start_failure_read(logs, endpoint, not_before_s, out_failure);
}
static bool bootstrap_production_handoff(void *context, cbm_daemon_bootstrap_lock_t lock) {
(void)context;
return cbm_daemon_ipc_startup_lock_prepare_handoff((cbm_daemon_ipc_startup_lock_t *)lock);
@@ -1041,10 +1355,26 @@ static void bootstrap_production_diagnostic(void *context, const char *message)
(void)fflush(stderr);
}
#ifdef CBM_ENABLE_TEST_SEAMS
static cbm_daemon_bootstrap_spawn_fn g_bootstrap_spawn_override_for_test;
static void *g_bootstrap_spawn_override_context_for_test;
void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn,
void *context) {
g_bootstrap_spawn_override_context_for_test = context;
g_bootstrap_spawn_override_for_test = spawn;
}
static bool bootstrap_seam_spawn(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec) {
(void)context;
return g_bootstrap_spawn_override_for_test(g_bootstrap_spawn_override_context_for_test, spec);
}
#endif
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
const cbm_daemon_bootstrap_config_t *config, cbm_daemon_bootstrap_result_t *result_out) {
bootstrap_production_context_t context = {0};
const cbm_daemon_bootstrap_ops_t ops = {
cbm_daemon_bootstrap_ops_t ops = {
.context = &context,
.cohort_acquire = bootstrap_production_cohort_acquire,
.cohort_release = bootstrap_production_cohort_release,
@@ -1054,6 +1384,12 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
.startup_lock_release = bootstrap_production_unlock,
.spawn_daemon = bootstrap_production_spawn,
.visible_diagnostic = bootstrap_production_diagnostic,
.start_failure_probe = bootstrap_production_start_failure_probe,
};
#ifdef CBM_ENABLE_TEST_SEAMS
if (g_bootstrap_spawn_override_for_test) {
ops.spawn_daemon = bootstrap_seam_spawn;
}
#endif
return cbm_daemon_bootstrap_execute_with_ops(config, &ops, result_out);
}
src/daemon/bootstrap.h
+58
View File
@@ -119,6 +119,47 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect
typedef void *cbm_daemon_bootstrap_lock_t;
typedef void *cbm_daemon_bootstrap_cohort_t;
/* Durable daemon start-failure record (#1828). A detached daemon has no
* stderr and no channel back to the client that launched it; when its
* listener publication fails (a full /tmp, for instance) the client used to
* respawn it for the whole startup deadline and then report "active or
* starting". The host now appends one record per failed start to
* <cache>/logs/cbm-daemon.start-failures.log -- the same owner-only log
* directory as cbm-daemon.log, deliberately NOT the runtime directory: that
* one is exactly what a full /tmp cannot write, and a new artifact there
* would entangle stale-generation cleanup. A waiting client reads the newest
* record for its endpoint that is not older than its own spawn and fails
* immediately with the cause. Diagnostic only: policy never reads it. */
enum { CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP = 32 };
typedef struct {
uint64_t recorded_at_s; /* wall clock, seconds */
uint64_t pid;
char component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP]; /* host component that refused */
char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; /* IPC stage, "" when none */
int errno_value; /* 0 when unknown */
char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; /* "" when none */
} cbm_daemon_bootstrap_start_failure_t;
/* <cache>/logs, the owner-only directory that holds every daemon log. */
bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity);
/* Append one record for this process's most recent listener failure detail
* (cbm_daemon_ipc_listen_failure_detail) under `component`. */
bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
const char *component);
/* Newest record for `endpoint` recorded at or after not_before_s. Returns 1
* and fills out_failure, 0 when none matches, -1 when the log directory or
* file cannot be validated. */
int cbm_daemon_bootstrap_start_failure_read(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure);
/* The client-facing sentence: "CBM daemon failed to start: <stage> failed
* with ENOSPC (No space left on device) at <path>; see <log>". */
void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure,
const char *log_directory, char *out,
size_t capacity);
/* Injectable OS/runtime boundary used by the deterministic unit contract.
* Production callers use cbm_daemon_bootstrap_execute(), whose built-in
* operations delegate to daemon IPC/runtime and write visible diagnostics to
@@ -149,6 +190,12 @@ typedef struct {
bool (*startup_lock_release)(void *context, cbm_daemon_bootstrap_lock_t *lock_io);
bool (*spawn_daemon)(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec);
void (*visible_diagnostic)(void *context, const char *message);
/* Optional. After this attempt spawned a daemon, report a start-failure
* record for the endpoint not older than not_before_s (1 found, 0 none,
* -1 unreadable). A found record ends the wait immediately (#1828). */
int (*start_failure_probe)(void *context, const cbm_daemon_ipc_endpoint_t *endpoint,
uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure);
} cbm_daemon_bootstrap_ops_t;
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
@@ -156,6 +203,17 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
/* Test seam for the same state machine. All callbacks are synchronous and
* borrowed for the duration of the call. */
#ifdef CBM_ENABLE_TEST_SEAMS
/* Replace ONLY the production spawn while every other production operation
* (cohort, probe, startup lock, handoff) stays real. A test forks a real
* daemon host in place of exec'ing the product binary, so the complete
* client/daemon rendezvous runs in one deterministic process tree (#1828). */
typedef bool (*cbm_daemon_bootstrap_spawn_fn)(void *context,
const cbm_daemon_bootstrap_launch_spec_t *spec);
void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn,
void *context);
#endif
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops,
cbm_daemon_bootstrap_result_t *result_out);
src/daemon/daemon.c
+23 -1
View File
@@ -47,6 +47,11 @@ struct cbm_daemon_coordinator {
size_t client_count;
/* See cbm_daemon_coordinator_set_permanent. */
bool permanent;
/* See cbm_daemon_coordinator_set_linger. Distinct from `permanent`: a
* transient, runtime-driven hold that keeps an ephemeral coordinator
* admitting while cohort participants are still mid-bootstrap (cold-storm
* race, 2026-09). Clearing it while idle retires the coordinator at once. */
bool linger;
size_t job_count;
size_t watch_count;
size_t callback_count;
@@ -291,7 +296,7 @@ static void release_client_locked(cbm_daemon_coordinator_t *coordinator,
release_client_resources_locked(coordinator, client->id, batch);
free(client);
coordinator->client_count--;
if (coordinator->client_count == 0 && !coordinator->permanent) {
if (coordinator->client_count == 0 && !coordinator->permanent && !coordinator->linger) {
coordinator->state = CBM_DAEMON_COORDINATOR_STOPPING;
}
}
@@ -329,6 +334,23 @@ void cbm_daemon_coordinator_set_permanent(cbm_daemon_coordinator_t *coordinator,
cbm_mutex_unlock(&coordinator->mutex);
}
void cbm_daemon_coordinator_set_linger(cbm_daemon_coordinator_t *coordinator, bool linger) {
if (!coordinator) {
return;
}
cbm_mutex_lock(&coordinator->mutex);
coordinator->linger = linger;
/* Clearing the hold while the coordinator is already idle is the retirement
* signal: an ephemeral generation whose cohort participants have all
* connected and left (or whose linger window elapsed) transitions to
* STOPPING now, exactly as the last-client disconnect would have. */
if (!linger && !coordinator->permanent && coordinator->client_count == 0 &&
coordinator->state == CBM_DAEMON_COORDINATOR_RUNNING) {
coordinator->state = CBM_DAEMON_COORDINATOR_STOPPING;
}
cbm_mutex_unlock(&coordinator->mutex);
}
cbm_daemon_coordinator_t *cbm_daemon_coordinator_new(uint64_t lease_timeout_ms) {
cbm_daemon_coordinator_t *coordinator = calloc(1, sizeof(*coordinator));
if (!coordinator) {
src/daemon/daemon.h
+8
View File
@@ -74,6 +74,14 @@ cbm_daemon_coordinator_t *cbm_daemon_coordinator_new(uint64_t lease_timeout_ms);
* self-transitions to STOPPING when its client count reaches zero; only the
* explicit stop/drain paths end it. */
void cbm_daemon_coordinator_set_permanent(cbm_daemon_coordinator_t *coordinator, bool permanent);
/* Transient, runtime-driven hold (distinct from `permanent`) that keeps an
* ephemeral coordinator RUNNING — still admitting new clients — after its last
* client disconnects, while cohort participants are still mid-bootstrap racing
* connect() (cold-storm race, 2026-09). Setting it before the last disconnect
* suppresses the self-transition to STOPPING; clearing it while the coordinator
* is already idle transitions to STOPPING at once, so retirement stays prompt
* and bounded. Has no effect on a permanent coordinator. */
void cbm_daemon_coordinator_set_linger(cbm_daemon_coordinator_t *coordinator, bool linger);
/* The caller must first quiesce coordinator calls and hook invocations. */
void cbm_daemon_coordinator_free(cbm_daemon_coordinator_t *coordinator);
src/daemon/host.c
+24 -1
View File
@@ -5,6 +5,7 @@
#include "daemon/host_internal.h"
#include "daemon/application.h"
#include "daemon/bootstrap.h"
#include "daemon/runtime.h"
#include "daemon/project_lock.h"
#include "daemon/version_cohort.h"
@@ -146,6 +147,18 @@ static bool host_log_open(char conflict_log_out[HOST_PATH_CAP]) {
return true;
}
/* #1828: a detached daemon has no stderr; its start failure must reach the
* client that is waiting for it, or that client burns its whole deadline and
* reports the opposite of the truth. */
static void host_start_failure_record(const cbm_daemon_ipc_endpoint_t *endpoint,
const char *component) {
char logs[HOST_PATH_CAP];
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs)) ||
!cbm_daemon_bootstrap_start_failure_record(logs, endpoint, component)) {
cbm_log_error("daemon.start_failure_record_failed", "component", component);
}
}
static void host_log_close(void) {
cbm_log_set_sink(NULL);
if (g_host_log_mutex_initialized) {
@@ -891,6 +904,10 @@ static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service,
return cbm_daemon_runtime_service_stop(service, HOST_RUNTIME_SHUTDOWN_MS);
}
host_http_reconcile_at(host, cbm_now_ms(), false);
/* Retire an ephemeral generation that lingered for cold-storm cohort
* participants once they drain, or once its bounded linger elapses.
* A no-op unless a cohort-participant hook armed a linger. */
cbm_daemon_runtime_service_reconcile_lifetime(service);
(void)cbm_daemon_runtime_service_wait_exited(service, HOST_WAIT_TICK_MS);
}
}
@@ -1051,10 +1068,16 @@ int cbm_daemon_host_run(const cbm_daemon_host_config_t *config) {
};
cbm_daemon_runtime_service_t *service =
cbm_daemon_runtime_service_start_reserved(&runtime_config, &lifetime_reservation);
if (!service) {
cbm_log_error("daemon.start_failed", "component", "runtime");
/* Recorded while the lifetime reservation is still held: a client
* that watches this generation vanish finds the cause already there
* and never launches a doomed replacement. */
host_start_failure_record(config->endpoint, "runtime");
}
cbm_daemon_ipc_lifetime_reservation_release(lifetime_reservation);
lifetime_reservation = NULL;
if (!service) {
cbm_log_error("daemon.start_failed", "component", "runtime");
host_state_free(&host);
host_log_close();
host_participant_guard_close(&participant_guard);
src/daemon/ipc.c
+384 -36
View File
@@ -9,6 +9,7 @@
#include "foundation/compat_thread.h"
#include "foundation/log.h"
#include "foundation/macos_acl.h"
#include "foundation/platform.h"
#include "foundation/private_file_lock_internal.h"
#include "foundation/sha256.h"
#include "foundation/secure_random.h"
@@ -53,6 +54,60 @@ const char *cbm_daemon_ipc_validation_detail(void) {
return ipc_validation_detail_buffer;
}
static cbm_daemon_ipc_listen_failure_t ipc_listen_failure;
/* The failure recorder and its reset are only reached from the POSIX socket
* listener (all callers live in the `#ifndef _WIN32` block below); the Windows
* named-pipe listener does its own reporting. Guarding them keeps the always-
* compiled struct + accessor cross-platform while avoiding -Werror,
* -Wunused-function on the Windows build (#1828 CI). */
#ifndef _WIN32
static void ipc_listen_failure_reset(void) {
memset(&ipc_listen_failure, 0, sizeof(ipc_listen_failure));
}
/* Record and log one listener failure. `name` is an artifact inside the
* endpoint's runtime directory, "" for the directory itself, or NULL when
* the stage has no path. The log line names the errno symbolically and the
* full path, so `daemon.ipc.listen_failed stage=pending_publication` alone
* (the whole trace #1828's reporter had) can no longer happen. */
static void ipc_listen_failed(const char *runtime_dir, const char *stage, int errno_value,
const char *name) {
ipc_listen_failure_reset();
(void)snprintf(ipc_listen_failure.stage, sizeof(ipc_listen_failure.stage), "%s",
stage ? stage : "");
ipc_listen_failure.errno_value = errno_value;
if (runtime_dir && name) {
int written = name[0] ? snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path),
"%s/%s", runtime_dir, name)
: snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path),
"%s", runtime_dir);
if (written <= 0 || written >= (int)sizeof(ipc_listen_failure.path)) {
ipc_listen_failure.path[0] = '\0';
}
}
const char *errno_name = errno_value != 0 ? cbm_errno_name(errno_value) : NULL;
if (errno_name && ipc_listen_failure.path[0]) {
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name, "path",
ipc_listen_failure.path);
} else if (errno_name) {
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name);
} else if (ipc_listen_failure.path[0]) {
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "path", ipc_listen_failure.path);
} else {
cbm_log_error("daemon.ipc.listen_failed", "stage", stage);
}
}
#endif /* !_WIN32 */
bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out) {
if (!out) {
return false;
}
*out = ipc_listen_failure;
return ipc_listen_failure.stage[0] != '\0';
}
static bool instance_key_valid(const char *key) {
if (!key) {
return false;
@@ -122,6 +177,19 @@ void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned
memory_order_release);
}
#ifndef _WIN32
static atomic_int g_posix_record_write_failure_errno_for_test;
#endif
void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value) {
#ifndef _WIN32
atomic_store_explicit(&g_posix_record_write_failure_errno_for_test, errno_value,
memory_order_release);
#else
(void)errno_value;
#endif
}
#ifdef _WIN32
static cbm_daemon_ipc_startup_gate_fn g_startup_gate_for_test;
static void *g_startup_gate_context_for_test;
@@ -146,6 +214,31 @@ static void ipc_startup_gate_run(void) {
}
#endif
#ifdef _WIN32
static cbm_daemon_ipc_win_directory_create_hook_fn g_win_directory_create_hook_for_test;
static void *g_win_directory_create_hook_context_for_test;
#endif
void cbm_daemon_ipc_win_directory_create_hook_set_for_test(
cbm_daemon_ipc_win_directory_create_hook_fn hook, void *context) {
#ifdef _WIN32
g_win_directory_create_hook_context_for_test = context;
g_win_directory_create_hook_for_test = hook;
#else
(void)hook;
(void)context;
#endif
}
#ifdef _WIN32
static void ipc_win_directory_create_hook_run(const wchar_t *path) {
cbm_daemon_ipc_win_directory_create_hook_fn hook = g_win_directory_create_hook_for_test;
if (hook) {
hook(path, g_win_directory_create_hook_context_for_test);
}
}
#endif
bool cbm_daemon_ipc_windows_legacy_names(const char *canonical_runtime_parent,
const char *instance_key,
char pipe_out[CBM_DAEMON_IPC_WINDOWS_NAME_CAP],
@@ -1395,9 +1488,21 @@ static char *private_log_directory_path_copy(const char *directory_path) {
*
* The overflow owner is tolerated for ANCESTORS ONLY, and ONLY when
* /proc/self/uid_map is a single-uid map whose sole inside id is our euid. In
* that shape no in-namespace principal can be the overflow owner, so an
* overflow-owned ancestor is exactly as safe as a root-owned one on the host:
* nobody reachable can have created it or can mutate it. A multi-uid map
* that shape no IN-NAMESPACE principal can be the overflow owner.
*
* Be precise about what that does and does not buy, because an earlier version
* of this comment overstated it. The overflow uid is what EVERY unmapped host
* uid maps to, not only host root (user_namespaces(7)), so "overflow-owned"
* does NOT prove "created by root". On a shared host, a directory owned by
* another local user is indistinguishable from root-owned /tmp once you are
* inside the namespace -- and there is no in-namespace discriminator that could
* tell them apart, which is precisely why the tolerance is scoped the way it
* is rather than made smarter. The real guarantee is narrower and still
* sufficient: no principal REACHABLE FROM INSIDE the namespace can create or
* mutate such an ancestor, and the private directory itself is never tolerated
* as overflow (see below), so a hostile host-side owner of an ancestor is
* bounded to denial of service and socket-path control. It cannot reach the
* leaf, which stays 0700 and euid-rechecked. A multi-uid map
* (rootless podman with subuids) still shows host root as overflow and is
* refused — conservative scope, not the safety argument. There is deliberately
* no environment escape hatch.
@@ -1477,7 +1582,20 @@ static bool posix_read_small_proc_file(const char *path, char *buffer, size_t ca
return true;
}
#ifdef CBM_ENABLE_TEST_SEAMS
/* Counts real derivations. A cache here was a security hazard once (see the
* note above posix_ancestor_overflow_uid); the contract test asserts this
* climbs on EVERY call so re-introducing one fails loudly. */
static unsigned g_posix_overflow_compute_count;
unsigned cbm_daemon_ipc_posix_overflow_compute_count_for_test(void) {
return g_posix_overflow_compute_count;
}
#endif
static uid_t posix_compute_ancestor_overflow_uid(void) {
#ifdef CBM_ENABLE_TEST_SEAMS
g_posix_overflow_compute_count++;
#endif
char map[256];
if (!posix_read_small_proc_file("/proc/self/uid_map", map, sizeof(map)) ||
!posix_uid_map_is_single_uid(map, geteuid())) {
@@ -1496,15 +1614,27 @@ static uid_t posix_compute_ancestor_overflow_uid(void) {
return (uid_t)overflow;
}
static uid_t g_posix_overflow_cached = POSIX_NO_OVERFLOW_UID;
static pthread_once_t g_posix_overflow_once = PTHREAD_ONCE_INIT;
static void posix_overflow_init_once(void) {
g_posix_overflow_cached = posix_compute_ancestor_overflow_uid();
}
#endif /* __linux__ */
/* The overflow uid tolerated for ancestors in this process, or
* POSIX_NO_OVERFLOW_UID when none. Derived once from immutable /proc state. */
/* The overflow uid tolerated for ancestors, or POSIX_NO_OVERFLOW_UID when none.
*
* DERIVED FRESH ON EVERY CALL, deliberately. This used to memoise via
* pthread_once behind a comment claiming "immutable /proc state". That claim
* was false in both halves: unshare(CLONE_NEWUSER) rewrites
* /proc/self/uid_map, and pthread_once state survives a forked child already
* marked done -- so a process that forked and then changed namespace kept the
* parent answer and refused a directory it should have accepted.
* (Spelled without the call syntax on purpose: scripts/security-audit.sh
* blocks that literal in src/, and an allow-list entry to let a COMMENT pass
* would weaken a real check on a real file.) It happened to be
* harmless because every caller today runs in a freshly exec'd process, but
* that made a security decision depend on an invariant nothing enforced, and
* the next fork-without-exec caller would have silently inherited a stale
* verdict.
*
* The cost of not caching is two small /proc reads per ancestor check, against
* an openat + fstat + fchmod + ACL check per path component in the same walk.
* Do not re-introduce a cache here; the contract test counts derivations. */
static uid_t posix_ancestor_overflow_uid(void) {
#ifdef CBM_ENABLE_TEST_SEAMS
if (g_posix_overflow_override_active) {
@@ -1512,8 +1642,7 @@ static uid_t posix_ancestor_overflow_uid(void) {
}
#endif
#if defined(__linux__)
(void)pthread_once(&g_posix_overflow_once, posix_overflow_init_once);
return g_posix_overflow_cached;
return posix_compute_ancestor_overflow_uid();
#else
return POSIX_NO_OVERFLOW_UID;
#endif
@@ -1999,6 +2128,29 @@ static bool posix_fd_write_all(int fd, const uint8_t *buffer, size_t length) {
return true;
}
/* The failing step of the most recent record publication in this process:
* its errno (0 when a validation predicate, not a syscall, refused) and the
* artifact name it was operating on. Read by the listener right after a
* publish returns false. */
static int posix_publish_failure_errno;
static char posix_publish_failure_name[NAME_MAX + 1];
static void posix_publish_failure_note(int errno_value, const char *name) {
posix_publish_failure_errno = errno_value;
(void)snprintf(posix_publish_failure_name, sizeof(posix_publish_failure_name), "%s",
name ? name : "");
}
static bool posix_record_fd_write(int fd, const uint8_t *buffer, size_t length) {
int injected =
atomic_load_explicit(&g_posix_record_write_failure_errno_for_test, memory_order_acquire);
if (injected != 0) {
errno = injected;
return false;
}
return posix_fd_write_all(fd, buffer, length);
}
static bool posix_fd_pread_all(int fd, uint8_t *buffer, size_t length) {
size_t offset = 0;
while (offset < length) {
@@ -2353,50 +2505,75 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin
return false;
}
struct stat existing;
errno = 0;
if (fstatat(endpoint->dir_fd, record_name, &existing, AT_SYMLINK_NOFOLLOW) == 0 ||
errno != ENOENT) {
posix_publish_failure_note(errno == 0 ? EEXIST : errno, record_name);
return false;
}
uint8_t record[POSIX_SOCKET_RECORD_SIZE];
if (!posix_socket_record_encode(magic, source, record)) {
posix_publish_failure_note(0, record_name);
return false;
}
char temp_name[NAME_MAX + 1];
if (!posix_socket_record_temp_name(record_name, temp_name)) {
posix_publish_failure_note(ENAMETOOLONG, record_name);
return false;
}
int fd = openat(endpoint->dir_fd, temp_name,
O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0600);
if (fd < 0) {
posix_publish_failure_note(errno, temp_name);
return false;
}
struct stat created = {0};
bool temp_exists = true;
bool stable_linked = false;
errno = 0;
bool ok = fd_set_cloexec(fd) && fchmod(fd, 0600) == 0 &&
private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) &&
posix_fd_write_all(fd, record, sizeof(record)) && fsync(fd) == 0 &&
posix_record_fd_write(fd, record, sizeof(record)) && fsync(fd) == 0 &&
private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) &&
created.st_size == (off_t)POSIX_SOCKET_RECORD_SIZE &&
endpoint_runtime_still_valid(endpoint);
if (!ok) {
posix_publish_failure_note(errno, temp_name);
}
if (ok) {
posix_record_publication_stage_reached(magic, false);
errno = 0;
ok =
posix_linkat_no_follow(endpoint->dir_fd, temp_name, endpoint->dir_fd, record_name) == 0;
stable_linked = ok;
if (!ok) {
posix_publish_failure_note(errno, record_name);
}
}
if (ok) {
posix_record_publication_stage_reached(magic, true);
errno = 0;
ok = posix_path_unlink_regular_if_matches(endpoint->dir_fd, temp_name, created.st_dev,
created.st_ino, 2);
temp_exists = !ok;
if (!ok) {
posix_publish_failure_note(errno, temp_name);
}
}
if (ok) {
errno = 0;
ok = posix_directory_sync(endpoint->dir_fd);
if (!ok) {
posix_publish_failure_note(errno, "");
}
}
errno = 0;
if (close(fd) != 0) {
if (ok) {
posix_publish_failure_note(errno, temp_name);
}
ok = false;
}
@@ -2408,6 +2585,9 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin
posix_socket_identity_equal(&published_record.identity, &source->identity) &&
strcmp(published_record.anchor_name, source->anchor_name) == 0 &&
published_status.st_dev == created.st_dev && published_status.st_ino == created.st_ino;
if (!ok) {
posix_publish_failure_note(0, record_name);
}
}
if (!ok) {
if (stable_linked) {
@@ -2723,8 +2903,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
cbm_daemon_ipc_lifetime_reservation_t **reservation_io) {
cbm_daemon_ipc_lifetime_reservation_t *lifetime_reservation =
reservation_io ? *reservation_io : NULL;
ipc_listen_failure_reset();
posix_publish_failure_note(0, NULL);
const char *runtime_dir = endpoint ? endpoint->runtime_dir : NULL;
if (!lifetime_reservation_matches_endpoint(endpoint, lifetime_reservation)) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "reservation_validation");
ipc_listen_failed(runtime_dir, "reservation_validation", 0, NULL);
return NULL;
}
/* Stale removal happens only under the startup lock, before a daemon host
@@ -2734,7 +2917,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
char pending_temp_name[NAME_MAX + 1];
if (!posix_socket_record_temp_name(endpoint->socket_identity_name, identity_temp_name) ||
!posix_socket_record_temp_name(endpoint->socket_pending_name, pending_temp_name)) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "temp_names");
ipc_listen_failed(runtime_dir, "temp_names", ENAMETOOLONG, endpoint->socket_pending_name);
return NULL;
}
struct stat existing;
@@ -2743,27 +2926,32 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
endpoint->socket_pending_name, identity_temp_name, pending_temp_name,
};
bool namespace_absent = true;
const char *occupied_name = "";
int occupied_errno = 0;
for (size_t index = 0; index < sizeof(required_absent) / sizeof(required_absent[0]); index++) {
errno = 0;
if (fstatat(endpoint->dir_fd, required_absent[index], &existing, AT_SYMLINK_NOFOLLOW) ==
0 ||
errno != ENOENT) {
namespace_absent = false;
occupied_name = required_absent[index];
occupied_errno = errno == 0 ? EEXIST : errno;
break;
}
}
if (!endpoint_runtime_still_valid(endpoint) || !namespace_absent) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "namespace_validation");
ipc_listen_failed(runtime_dir, "namespace_validation", occupied_errno, occupied_name);
return NULL;
}
int fd = local_socket_new();
if (fd < 0) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_creation");
ipc_listen_failed(runtime_dir, "socket_creation", errno, NULL);
return NULL;
}
cbm_daemon_ipc_listener_t *listener = calloc(1, sizeof(*listener));
if (!listener) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_allocation");
ipc_listen_failed(runtime_dir, "listener_allocation", ENOMEM, NULL);
(void)close(fd);
return NULL;
}
@@ -2781,7 +2969,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
if (listener->dir_fd < 0 || !fd_set_cloexec(listener->dir_fd) || !listener->runtime_dir ||
!listener->address || !listener->socket_name || !listener->socket_anchor_name ||
!listener->socket_identity_name || !listener->socket_pending_name) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_initialization");
ipc_listen_failed(runtime_dir, "listener_initialization",
listener->dir_fd < 0 ? errno : ENOMEM, NULL);
if (listener->dir_fd >= 0) {
(void)close(listener->dir_fd);
}
@@ -2799,20 +2988,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
struct sockaddr_un address;
socklen_t address_length;
if (!unix_address_set(&address, endpoint->socket_anchor_address, &address_length)) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_address");
ipc_listen_failed(runtime_dir, "socket_address", ENAMETOOLONG,
endpoint->socket_anchor_name);
cbm_daemon_ipc_listener_close(listener);
return NULL;
}
if (bind(fd, (const struct sockaddr *)&address, address_length) != 0) {
int bind_error = errno;
char error_text[32];
(void)snprintf(error_text, sizeof(error_text), "%d", bind_error);
cbm_log_error("daemon.ipc.listen_failed", "bind_errno", error_text);
ipc_listen_failed(runtime_dir, "socket_bind", errno, endpoint->socket_anchor_name);
cbm_daemon_ipc_listener_close(listener);
return NULL;
}
struct stat bound_status;
errno = 0;
bool bound_path_ok = fstatat(endpoint->dir_fd, endpoint->socket_anchor_name, &bound_status,
AT_SYMLINK_NOFOLLOW) == 0 &&
S_ISSOCK(bound_status.st_mode) && bound_status.st_uid == geteuid();
@@ -2832,11 +3020,13 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
anchor_status.st_ino != bound_status.st_ino ||
!posix_socket_identity_from_stat(&anchor_status, &anchor_identity) ||
!posix_directory_sync(endpoint->dir_fd)) {
int security_errno = errno;
if (bound_path_ok) {
posix_bound_socket_unlink_if_matches(endpoint->dir_fd, endpoint->socket_anchor_name,
bound_status.st_dev, bound_status.st_ino);
}
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_security");
ipc_listen_failed(runtime_dir, "socket_security", security_errno,
endpoint->socket_anchor_name);
cbm_daemon_ipc_listener_close(listener);
return NULL;
}
@@ -2853,16 +3043,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
endpoint, endpoint->socket_pending_name, POSIX_SOCKET_PENDING_MAGIC, &pending,
&pending_status.st_dev, &pending_status.st_ino);
if (!pending_published) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_publication");
ipc_listen_failed(runtime_dir, "pending_publication", posix_publish_failure_errno,
posix_publish_failure_name);
posix_publication_abort(endpoint, &anchor_identity, false, NULL, false, NULL);
cbm_daemon_ipc_listener_close(listener);
return NULL;
}
posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_PENDING_DURABLE);
errno = 0;
bool stable_linked = posix_linkat_no_follow(endpoint->dir_fd, endpoint->socket_anchor_name,
endpoint->dir_fd, endpoint->socket_name) == 0 &&
posix_directory_sync(endpoint->dir_fd);
int stable_errno = stable_linked ? 0 : errno;
posix_socket_identity_t stable_identity = {0};
posix_socket_identity_t committed_identity = {0};
struct stat stable_status = {0};
@@ -2877,7 +3070,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
posix_socket_identity_equal(&stable_identity, &committed_identity) &&
posix_socket_inode_equal(&anchor_identity, &committed_identity);
if (!stable_valid) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "stable_publication");
ipc_listen_failed(runtime_dir, "stable_publication", stable_errno, endpoint->socket_name);
posix_publication_abort(endpoint, &anchor_identity, pending_published, &pending_status,
false, NULL);
cbm_daemon_ipc_listener_close(listener);
@@ -2896,7 +3089,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
endpoint, endpoint->socket_identity_name, POSIX_SOCKET_MARKER_MAGIC, &marker,
&marker_status.st_dev, &marker_status.st_ino);
if (!marker_published) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "marker_publication");
ipc_listen_failed(runtime_dir, "marker_publication", posix_publish_failure_errno,
posix_publish_failure_name);
posix_publication_abort(endpoint, &committed_identity, pending_published, &pending_status,
false, NULL);
cbm_daemon_ipc_listener_close(listener);
@@ -2906,10 +3100,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
listener->identity_inode = marker_status.st_ino;
posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_MARKER_DURABLE);
errno = 0;
if (!posix_path_unlink_regular_if_matches(endpoint->dir_fd, endpoint->socket_pending_name,
pending_status.st_dev, pending_status.st_ino, 1) ||
!posix_directory_sync(endpoint->dir_fd)) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_removal");
ipc_listen_failed(runtime_dir, "pending_removal", errno, endpoint->socket_pending_name);
posix_publication_abort(endpoint, &committed_identity, true, &pending_status,
marker_published, &marker_status);
cbm_daemon_ipc_listener_close(listener);
@@ -3561,6 +3756,7 @@ bool cbm_daemon_ipc_local_transition_release(cbm_daemon_ipc_local_transition_t *
#include <windows.h>
#include <aclapi.h>
#include <sddl.h>
#include <ntsecapi.h>
#include <fcntl.h>
#include <io.h>
#include <shlobj.h>
@@ -3823,6 +4019,16 @@ static char *wide_to_utf8(const wchar_t *value) {
return utf8;
}
/* Record "<path>: <message>" as the validation detail, with the wide path
* rendered as UTF-8. One place owns the conversion buffer, so every refusal on
* the directory walk can name its component without each call site carrying
* its own allocation. */
static void ipc_validation_detail_set_for_path(const wchar_t *path, const char *message) {
char *path_utf8 = wide_to_utf8(path);
ipc_validation_detail_set("%s: %s", path_utf8 ? path_utf8 : "<path>", message);
free(path_utf8);
}
static wchar_t *wide_copy(const wchar_t *value) {
if (!value) {
return NULL;
@@ -4218,17 +4424,120 @@ static bool win_sid_is_trusted_installer(const uint8_t *sid, size_t sid_length)
return true;
}
/* #1705: the built-in Administrator ACCOUNT of THIS machine — RID 500 under the
* local machine's own account-domain SID (S-1-5-21-<machine>-500) — is a
* legitimate owner/grantee of directories an elevated install created, even when
* that account has been renamed or is disabled. It is resolved by asking LSA for
* the local machine account-domain SID and synthesizing its RID-500 SID with
* CreateWellKnownSid, then compared with EqualSid.
*
* It is deliberately NOT tested with IsWellKnownSid(sid, WinAccountAdministratorSid)
* and NOT by matching a trailing RID of 500: BOTH of those accept ANY domain's
* -500 — a domain administrator, or another machine's built-in Administrator —
* which is exactly the cross-machine trust escalation this must never open. Only
* THIS machine's -500 is trusted.
*
* Resolved once per process and cached; any LSA or synthesis failure leaves the
* cache NULL and therefore grants NO tolerance at all (fail closed). advapi32 is
* reached through the already-loaded module handle in win_security_t and the
* function pointers are resolved dynamically, matching this file's SID-API style
* and adding no static import. */
typedef NTSTATUS(NTAPI *lsa_open_policy_fn)(PLSA_UNICODE_STRING, PLSA_OBJECT_ATTRIBUTES,
ACCESS_MASK, PLSA_HANDLE);
typedef NTSTATUS(NTAPI *lsa_query_information_policy_fn)(LSA_HANDLE, POLICY_INFORMATION_CLASS,
PVOID *);
typedef NTSTATUS(NTAPI *lsa_free_memory_fn)(PVOID);
typedef NTSTATUS(NTAPI *lsa_close_fn)(LSA_HANDLE);
typedef BOOL(WINAPI *create_well_known_sid_fn)(WELL_KNOWN_SID_TYPE, PSID, PSID, DWORD *);
static INIT_ONCE g_local_admin_sid_once = INIT_ONCE_STATIC_INIT;
static PSID g_local_admin_sid = NULL; /* process-lifetime cache; NULL => no tolerance */
static BOOL CALLBACK win_resolve_local_admin_sid(PINIT_ONCE once, PVOID parameter, PVOID *context) {
(void)once;
(void)context;
win_security_t *security = (win_security_t *)parameter;
if (!security || !security->advapi) {
return TRUE; /* ran once; cache stays NULL (fail closed) */
}
HMODULE advapi = security->advapi;
lsa_open_policy_fn lsa_open =
(lsa_open_policy_fn)(void (*)(void))GetProcAddress(advapi, "LsaOpenPolicy");
lsa_query_information_policy_fn lsa_query = (lsa_query_information_policy_fn)(void (*)(
void))GetProcAddress(advapi, "LsaQueryInformationPolicy");
lsa_free_memory_fn lsa_free =
(lsa_free_memory_fn)(void (*)(void))GetProcAddress(advapi, "LsaFreeMemory");
lsa_close_fn lsa_close = (lsa_close_fn)(void (*)(void))GetProcAddress(advapi, "LsaClose");
create_well_known_sid_fn create_sid =
(create_well_known_sid_fn)(void (*)(void))GetProcAddress(advapi, "CreateWellKnownSid");
if (!lsa_open || !lsa_query || !lsa_free || !lsa_close || !create_sid) {
return TRUE;
}
LSA_OBJECT_ATTRIBUTES attributes;
memset(&attributes, 0, sizeof(attributes));
LSA_HANDLE policy = NULL;
/* STATUS_SUCCESS is 0; any other status (including informational positives) is
* treated as failure, keeping the outcome fail-closed. */
if (lsa_open(NULL, &attributes, POLICY_VIEW_LOCAL_INFORMATION, &policy) != 0 || !policy) {
return TRUE;
}
POLICY_ACCOUNT_DOMAIN_INFO *domain = NULL;
if (lsa_query(policy, PolicyAccountDomainInformation, (PVOID *)&domain) == 0 && domain &&
domain->DomainSid && security->is_valid_sid(domain->DomainSid)) {
DWORD needed = 0;
(void)create_sid(WinAccountAdministratorSid, domain->DomainSid, NULL, &needed);
if (needed > 0U) {
PSID resolved = malloc(needed);
if (resolved &&
create_sid(WinAccountAdministratorSid, domain->DomainSid, resolved, &needed) &&
security->is_valid_sid(resolved)) {
g_local_admin_sid = resolved;
} else {
free(resolved);
}
}
}
if (domain) {
(void)lsa_free(domain);
}
(void)lsa_close(policy);
return TRUE;
}
static PSID win_local_admin_sid(win_security_t *security) {
if (!security || !security->advapi) {
return NULL;
}
(void)InitOnceExecuteOnce(&g_local_admin_sid_once, win_resolve_local_admin_sid, (PVOID)security,
NULL);
return g_local_admin_sid;
}
static bool win_sid_trusted(win_security_t *security, PSID sid) {
if (!security || !sid || !security->is_valid_sid(sid)) {
return false;
}
DWORD sid_length = security->get_length_sid(sid);
PSID local_admin = win_local_admin_sid(security);
return (sid_length > 0U && security->equal_sid(sid, security->user_sid)) ||
security->is_well_known_sid(sid, WinLocalSystemSid) ||
security->is_well_known_sid(sid, WinBuiltinAdministratorsSid) ||
(local_admin && security->equal_sid(sid, local_admin)) ||
win_sid_is_trusted_installer((const uint8_t *)sid, (size_t)sid_length);
}
#ifdef CBM_ENABLE_TEST_SEAMS
bool cbm_daemon_ipc_win_sid_trusted_for_testing(void *sid) {
win_security_t security;
if (!win_security_init(&security)) {
return false;
}
bool trusted = win_sid_trusted(&security, (PSID)sid);
win_security_destroy(&security);
return trusted;
}
#endif
/* AppContainer identities: package SIDs (S-1-15-2-*) and capability SIDs
* (S-1-15-3-*), under the APP_PACKAGE identifier authority (15).
*
@@ -4587,13 +4896,28 @@ static bool win_runtime_directory_secure(const wchar_t *runtime_dir) {
return false;
}
bool created = CreateDirectoryW(runtime_dir, &security.directory_attributes) != 0;
if (!created && GetLastError() != ERROR_ALREADY_EXISTS) {
DWORD create_error = created ? ERROR_SUCCESS : GetLastError();
if (!created && create_error != ERROR_ALREADY_EXISTS) {
char message[96];
(void)snprintf(message, sizeof(message),
"could not create the directory (Windows error %lu)",
(unsigned long)create_error);
ipc_validation_detail_set_for_path(runtime_dir, message);
win_security_destroy(&security);
return false;
}
DWORD attributes = GetFileAttributesW(runtime_dir);
if (attributes == INVALID_FILE_ATTRIBUTES || (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0 ||
(attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) {
/* What already exists at this path is adopted only if it is a plain
* directory. Say which rule refused it: the walk in front of this
* function tolerates losing a creation race, so this check is what
* stands between that tolerance and a planted file or junction. */
const char *rule = attributes == INVALID_FILE_ATTRIBUTES ? "cannot be inspected"
: (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0
? "exists but is not a directory"
: "is a reparse point (junction or symlink)";
ipc_validation_detail_set_for_path(runtime_dir, rule);
win_security_destroy(&security);
return false;
}
@@ -4747,8 +5071,35 @@ static bool win_private_directory_tree_secure(const wchar_t *directory_path) {
DWORD attributes = GetFileAttributesW(path);
if (attributes == INVALID_FILE_ATTRIBUTES) {
DWORD error = GetLastError();
ok = (error == ERROR_FILE_NOT_FOUND || error == ERROR_PATH_NOT_FOUND) &&
CreateDirectoryW(path, &security.directory_attributes) != 0;
bool absent = error == ERROR_FILE_NOT_FOUND || error == ERROR_PATH_NOT_FOUND;
if (absent) {
ipc_win_directory_create_hook_run(path);
/* Several processes first-starting together all observe
* this component absent; one CreateDirectoryW wins and
* the rest get ERROR_ALREADY_EXISTS. Losing that race is
* not a failure -- the directory this process wanted now
* exists, exactly as if it had been there all along. It
* is NOT trusted for that: an ancestor still goes through
* win_directory_component_secure() below, and the final
* component through win_runtime_directory_secure(), which
* refuses a non-directory or reparse point and enforces
* owner and DACL. The POSIX walk tolerates EEXIST at the
* same point for the same reason. */
if (CreateDirectoryW(path, &security.directory_attributes) == 0) {
error = GetLastError();
absent = error == ERROR_ALREADY_EXISTS;
}
}
ok = absent;
if (!ok) {
/* Never a bare "(endpoint)": name the component and the
* Windows error that refused it. */
char message[96];
(void)snprintf(message, sizeof(message),
"could not create or inspect the directory (Windows error %lu)",
(unsigned long)error);
ipc_validation_detail_set_for_path(path, message);
}
}
/* Ancestors are observe-only and must already be secure. The
* final current-user directory is intentionally handled below by
@@ -4760,10 +5111,7 @@ static bool win_private_directory_tree_secure(const wchar_t *directory_path) {
* at this walk position; the helper set the inner rule. */
char inner[384];
(void)snprintf(inner, sizeof(inner), "%s", ipc_validation_detail_buffer);
char *component_utf8 = wide_to_utf8(path);
ipc_validation_detail_set(
"%s: %s", component_utf8 ? component_utf8 : "<component>", inner);
free(component_utf8);
ipc_validation_detail_set_for_path(path, inner);
}
}
}
src/daemon/ipc.h
+32
View File
@@ -91,9 +91,35 @@ bool cbm_daemon_ipc_private_directory_secure(const char *directory_path);
* validation refusal (empty string when none). Diagnostic only — callers
* append it to their error messages; policy decisions never read it. */
const char *cbm_daemon_ipc_validation_detail(void);
/* This process's most recent listener-publication failure: the stage that
* refused, the errno the failing step reported (0 when the step reported
* none), and the artifact path it was operating on ("" when no path applies).
* Every `daemon.ipc.listen_failed` log line carries the same three fields.
* Reset at the start of each listen attempt. Returns false while no failure
* has been recorded. Diagnostic only: the daemon host copies it into its
* durable start-failure record so a waiting client can name the cause instead
* of burning its full startup deadline (#1828). */
enum {
CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP = 32,
CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP = 4096
};
typedef struct {
char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP];
int errno_value;
char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP];
} cbm_daemon_ipc_listen_failure_t;
bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out);
#ifdef CBM_ENABLE_TEST_SEAMS
/* #1537: seed the detail so a test can prove the CLI refusal surfaces it. */
void cbm_daemon_ipc_set_validation_detail_for_testing(const char *detail);
#ifdef _WIN32
/* #1705: run the daemon's directory-owner/ACE trust predicate against an
* arbitrary SID, so a test can assert THIS machine's built-in Administrator
* (RID-500) is trusted while a foreign S-1-5-21-*-500 is not. Returns false on
* any setup failure. Windows only. */
bool cbm_daemon_ipc_win_sid_trusted_for_testing(void *sid);
#endif
#ifndef _WIN32
/* #1830 seams (POSIX). Override the single-uid user-namespace overflow uid that
* ancestors may be owned by (active=false restores the real /proc-derived
@@ -105,6 +131,12 @@ bool cbm_daemon_ipc_posix_uid_map_is_single_uid_for_test(const char *uid_map, un
bool cbm_daemon_ipc_posix_ancestor_stat_ok_for_test(unsigned long owner, unsigned int mode,
unsigned long euid, bool overflow_active,
unsigned long overflow_uid);
#if defined(__linux__)
/* Number of REAL overflow-uid derivations so far. The value is meaningless on
* its own; the point is that it must rise on every ancestor check, proving no
* cache has crept back in. */
unsigned cbm_daemon_ipc_posix_overflow_compute_count_for_test(void);
#endif
#endif
#endif
src/daemon/ipc_internal.h
+16
View File
@@ -95,6 +95,11 @@ typedef void (*cbm_daemon_ipc_posix_publication_hook_fn)(
void cbm_daemon_ipc_posix_publication_hook_set_for_test(
cbm_daemon_ipc_posix_publication_hook_fn hook, void *context);
void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned int count);
/* #1828: while non-zero, every POSIX socket-record publication write fails
* with this errno. This is the deterministic stand-in for a full runtime
* filesystem (tmpfs ENOSPC): the socket binds, the record file is created,
* and only its data write is refused -- exactly the reporter's failure shape. */
void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value);
/* Deterministic-interleaving seam: fires on the Windows startup path once the
* startup lock is held, before the rendezvous handoff. A test parks here to
@@ -105,4 +110,15 @@ void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned
typedef void (*cbm_daemon_ipc_startup_gate_fn)(void *context);
void cbm_daemon_ipc_startup_gate_set_for_test(cbm_daemon_ipc_startup_gate_fn gate, void *context);
/* Deterministic-interleaving seam for the Windows private-directory walk:
* fires after a path component was observed ABSENT and before this process
* tries to create it. A test plays the concurrent process that wins that
* creation, which pins the cold-start race (N processes first-starting
* against a runtime directory that does not exist yet) by construction
* instead of by thread timing. `path` is the NUL-terminated component path at
* the walk position. No-op off Windows. */
typedef void (*cbm_daemon_ipc_win_directory_create_hook_fn)(const wchar_t *path, void *context);
void cbm_daemon_ipc_win_directory_create_hook_set_for_test(
cbm_daemon_ipc_win_directory_create_hook_fn hook, void *context);
#endif /* CBM_DAEMON_IPC_INTERNAL_H */
src/daemon/runtime.c
+136 -2
View File
@@ -49,6 +49,16 @@ void cbm_daemon_runtime_set_containment_hook_for_testing(
cbm_daemon_runtime_containment_hook_t hook) {
atomic_store(&runtime_containment_hook_seam, hook);
}
/* Cold-storm ephemeral-linger seam (2026-09). Overrides the bounded linger
* window that ephemeral last-committed-client retirement grants while cohort
* participants are still mid-bootstrap, so a test can drive both the linger and
* its expiry backstop in test time. UINT32_MAX leaves the production constant;
* any other value (0 = expire immediately) overrides. */
static _Atomic uint32_t runtime_ephemeral_linger_timeout_seam = UINT32_MAX;
void cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(uint32_t timeout_ms) {
atomic_store(&runtime_ephemeral_linger_timeout_seam, timeout_ms);
}
#endif
#ifdef _WIN32
@@ -95,6 +105,14 @@ enum {
* wedged behind it, and the dead generation held the endpoint pipes and
* the UI port for nine hours with nothing logged. */
RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS = 30000,
/* Cold-storm race (2026-09): when the final committed client of an
* ephemeral generation disconnects while cohort participants are still
* admitted but mid-bootstrap (racing connect()), the generation lingers
* this long for them to connect instead of retiring out from under them.
* Mirrors the host initial-client window; the linger is always bounded so
* a participant that never connects cannot wedge the generation (the
* 900s host_serving hang). */
RUNTIME_EPHEMERAL_LINGER_MS = 10000,
RUNTIME_PATH_CAP = 4096,
RENDEZVOUS_REQUEST_ABI_OFFSET = 0,
@@ -231,6 +249,12 @@ struct cbm_daemon_runtime_service {
/* Owned only by the convenience start() path. start_reserved() callers
* retain their externally managed participant guard. */
cbm_daemon_ipc_participant_guard_t *owned_participant_guard;
/* Cold-storm ephemeral-retirement gate (2026-09). When the final committed
* client disconnects while cohort participants are still admitted but not
* yet committed, the generation lingers until this bounded deadline instead
* of retiring; reconcile_lifetime retires it once that window elapses with
* no new client committing. Zero means no linger is armed. */
uint64_t ephemeral_linger_deadline_ms;
};
struct cbm_daemon_runtime_worker {
@@ -289,6 +313,16 @@ static uint64_t runtime_deadline_after(uint32_t timeout_ms) {
return now_ms + (uint64_t)timeout_ms;
}
static uint32_t runtime_ephemeral_linger_timeout_ms(void) {
#ifdef CBM_ENABLE_TEST_SEAMS
uint32_t seam = atomic_load(&runtime_ephemeral_linger_timeout_seam);
if (seam != UINT32_MAX) {
return seam;
}
#endif
return RUNTIME_EPHEMERAL_LINGER_MS;
}
static void runtime_wait_tick(uint64_t deadline_ms) {
uint64_t now_ms = cbm_now_ms();
if (now_ms >= deadline_ms) {
@@ -1174,10 +1208,32 @@ static void runtime_service_interrupt_connections(cbm_daemon_runtime_service_t *
runtime_service_interrupt_connections_except(service, NULL, false);
}
/* A cold-storm racer is a connection that has been ACCEPTED but has not yet
* passed HELLO admission (in_use with a live connection, not yet admitted, not
* tearing down) -- a one-shot client still mid-bootstrap. This deliberately
* does NOT count an already-admitted provisional session (HELLO done, app
* session opening): a provisional coordinator client must not keep a retiring
* generation alive (see runtime_worker_disconnect below and the
* daemon_runtime_final_disconnect_rejects_blocked_provisional_session guard).
* Caller holds service->mutex. `except` excludes the departing worker. */
static bool runtime_has_pending_hello_peer_locked(const cbm_daemon_runtime_service_t *service,
const cbm_daemon_runtime_worker_t *except) {
for (size_t i = 0; i < service->worker_capacity; i++) {
const cbm_daemon_runtime_worker_t *worker = &service->workers[i];
if (worker != except && worker->in_use && worker->connection && !worker->admitted &&
!atomic_load_explicit(&worker->disconnecting, memory_order_acquire)) {
return true;
}
}
return false;
}
static void runtime_worker_disconnect(cbm_daemon_runtime_worker_t *worker) {
cbm_daemon_runtime_service_t *service = worker->service;
cbm_daemon_client_id_t client_id = CBM_DAEMON_CLIENT_ID_INVALID;
uint64_t shutdown_deadline = runtime_deadline_after(service->shutdown_timeout_ms);
bool last_committed_left = false;
bool linger_armed = false;
atomic_store_explicit(&worker->disconnecting, true, memory_order_release);
cbm_mutex_lock(&service->mutex);
if (worker->admitted) {
@@ -1195,14 +1251,47 @@ static void runtime_worker_disconnect(cbm_daemon_runtime_worker_t *worker) {
* alive after the final fully committed frontend disconnects.
* A permanent generation (`daemon start`) deliberately survives
* this: only the stop/drain ops or a process kill end it. */
runtime_service_begin_stopping_locked(service, shutdown_deadline, false,
"last_committed_client_disconnected");
last_committed_left = true;
/* Cold-storm race (2026-09): if another connection is already
* accepted and still mid-HELLO (not yet admitted) when the last
* committed client leaves, it is a one-shot peer racing to commit —
* retiring now strands it on a STOPPING daemon and forces a full
* cold respawn ("cold-storm client failed (racing daemon spawn)").
* Linger a bounded window for it; the accept loop retires the
* generation once that racer drains without committing or the window
* elapses, and a racer that does commit clears the linger
* (runtime_worker_commit_admission). This deliberately lingers ONLY
* for a pre-HELLO racer, never for an already-admitted provisional
* session (HELLO done, app session opening): the contract that a
* provisional coordinator client cannot keep a retiring generation
* alive is preserved (the reject-blocked-provisional-session guard).
* With no such racer — the common single-client case — retire
* immediately, unchanged. (A cross-process cohort peer count is
* unavailable: advisory locks expose presence, not a holder count,
* and Windows LockFileEx has no non-owning probe.) */
if (runtime_has_pending_hello_peer_locked(service, worker)) {
service->ephemeral_linger_deadline_ms =
runtime_deadline_after(runtime_ephemeral_linger_timeout_ms());
linger_armed = true;
} else {
service->ephemeral_linger_deadline_ms = 0;
runtime_service_begin_stopping_locked(service, shutdown_deadline, false,
"last_committed_client_disconnected");
}
}
}
cbm_mutex_unlock(&service->mutex);
if (client_id == CBM_DAEMON_CLIENT_ID_INVALID) {
return;
}
/* Set the coordinator hold BEFORE releasing this client so its last-client
* self-transition to STOPPING is suppressed while a racing peer is still
* mid-HELLO; when no peer is present the hold stays clear so the release
* retires the coordinator normally. last_committed_left implies a committed (hence
* admitted) client, so client_id is always valid here. */
if (last_committed_left) {
cbm_daemon_coordinator_set_linger(service->coordinator, linger_armed);
}
(void)cbm_daemon_client_disconnected(service->coordinator, client_id, cbm_now_ms());
if (worker->application_session_opened && !worker->application_cancelled) {
worker->application_cancelled = true;
@@ -1250,8 +1339,18 @@ static bool runtime_worker_commit_admission(cbm_daemon_runtime_worker_t *worker)
if (committed) {
worker->admission_committed = true;
service->committed_clients++;
/* A freshly committed client ends any last-client linger window; the
* next drop to zero re-arms it against the participants outstanding
* then. */
service->ephemeral_linger_deadline_ms = 0;
}
cbm_mutex_unlock(&service->mutex);
if (committed) {
/* Mirror the cleared linger onto the coordinator. client_count is
* already nonzero (admission incremented it), so this only resets the
* hold for the next idle window — it never retires a live coordinator. */
cbm_daemon_coordinator_set_linger(service->coordinator, false);
}
return committed;
}
@@ -2183,6 +2282,11 @@ static void *runtime_accept_loop(void *opaque) {
continue;
}
/* Self-retire a generation lingering for a cold-storm peer once the peer
* drains or the bounded window elapses, without an external driver. A
* no-op unless a last-committed-client linger is armed. */
cbm_daemon_runtime_service_reconcile_lifetime(service);
cbm_daemon_ipc_connection_t *connection = NULL;
int accepted =
cbm_daemon_ipc_accept(service->listener, RUNTIME_ACCEPT_POLL_MS, &connection);
@@ -2474,6 +2578,36 @@ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_
return count;
}
void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service) {
if (!service) {
return;
}
bool retired = false;
cbm_mutex_lock(&service->mutex);
if (service->state == CBM_DAEMON_RUNTIME_SERVICE_RUNNING && !service->permanent &&
service->committed_clients == 0 && service->ephemeral_linger_deadline_ms != 0) {
bool peers_drained = !runtime_has_pending_hello_peer_locked(service, NULL);
bool window_elapsed = cbm_now_ms() >= service->ephemeral_linger_deadline_ms;
if (peers_drained || window_elapsed) {
/* The racing peer drained without committing (retire now), or never
* committed within the bounded window (the backstop that rules out
* an unbounded idle hang — the host_serving 900s mode). */
service->ephemeral_linger_deadline_ms = 0;
runtime_service_begin_stopping_locked(
service, runtime_deadline_after(service->shutdown_timeout_ms), false,
peers_drained ? "ephemeral_peer_drained" : "ephemeral_linger_expired");
retired = true;
}
}
cbm_mutex_unlock(&service->mutex);
if (retired) {
/* Release the coordinator hold now that the linger has resolved. With
* no client left this transitions the coordinator to STOPPING, so the
* service drains and exits cleanly rather than only on the deadline. */
cbm_daemon_coordinator_set_linger(service->coordinator, false);
}
}
size_t cbm_daemon_runtime_service_job_subscribers(cbm_daemon_runtime_service_t *service,
const char *project_key) {
return service ? cbm_daemon_job_subscribers(service->coordinator, project_key) : 0;
src/daemon/runtime.h
+13
View File
@@ -329,6 +329,13 @@ uint64_t cbm_daemon_runtime_service_clients_admitted_total(cbm_daemon_runtime_se
/* Includes accepted connections still waiting for HELLO. Never exceeds the
* configured max_clients; over-cap peers receive REJECTED before close. */
size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_t *service);
/* Re-evaluate an ephemeral generation that is lingering after its last committed
* client disconnected (the bounded cold-storm debounce). Retires it once the
* linger window elapses with no new client committing; the bound also rules out
* an unbounded idle hang. The host lifetime loop calls this every tick; it is a
* no-op unless a last-committed-client linger is armed. */
void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service);
size_t cbm_daemon_runtime_service_job_subscribers(cbm_daemon_runtime_service_t *service,
const char *project_key);
uint64_t cbm_daemon_runtime_service_client_process_id(cbm_daemon_runtime_service_t *service,
@@ -441,6 +448,12 @@ typedef void (*cbm_daemon_runtime_containment_hook_t)(const char *component);
void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms);
void cbm_daemon_runtime_set_containment_hook_for_testing(
cbm_daemon_runtime_containment_hook_t hook);
/* Cold-storm ephemeral-linger seam (2026-09). Overrides the bounded linger
* window the last-committed-client retirement grants while cohort participants
* are mid-bootstrap, so a test drives both the linger and its expiry backstop
* in test time. UINT32_MAX restores the production constant; any other value
* (0 = expire immediately) overrides. Process-global; reset it after use. */
void cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(uint32_t timeout_ms);
#endif
#endif /* CBM_DAEMON_RUNTIME_H */

Some files were not shown because too many files have changed in this diff Show More