diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 000000000..5c2a1ea68 --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,27 @@ +# CodeQL scope: the repository's first-party source. +# +# Read by .github/workflows/codeql.yml (every scan, full and PR) and by +# scripts/ci/select-lanes.sh (which changed paths select which language's PR +# scan). Keep the lists plain: one path prefix per line, no globs. +# +# paths / paths-ignore narrow what the INTERPRETED languages extract +# (javascript-typescript, python, actions). For c-cpp they have no effect: +# CodeQL extracts exactly what the traced build compiles. A PR's c-cpp scan +# keeps vendored C out by compiling it before tracing starts (except vendored +# files the PR changes); the full scan on main, the dry run and the release +# traces everything, vendored C included. +name: cbm first-party source +paths: + - src + - internal + - graph-ui + - pkg + - scripts + - tests + - .github +paths-ignore: + - internal/cbm/vendored + - graph-ui/vendor + - graph-ui/fixtures + - graph-ui/verification + - tests/fixtures diff --git a/.github/workflows/_lint.yml b/.github/workflows/_lint.yml index 6fcda15e4..741e3c438 100644 --- a/.github/workflows/_lint.yml +++ b/.github/workflows/_lint.yml @@ -1,5 +1,5 @@ # Reusable: lint only (cppcheck + clang-format). -# Security-static and CodeQL gate are separate — see _security.yml. +# Security-static is separate — see _security.yml; CodeQL — see codeql.yml. name: Lint & Security on: diff --git a/.github/workflows/_security.yml b/.github/workflows/_security.yml index 666573471..40514091a 100644 --- a/.github/workflows/_security.yml +++ b/.github/workflows/_security.yml @@ -1,4 +1,6 @@ -# Reusable: security-static + CodeQL gate. +# Reusable: security-static + license gate. +# CodeQL is not here: PRs do not run it. It runs on push to main, weekly, and +# inside dry-run.yml / release.yml, which call codeql.yml directly. # Runs independently from lint/test/build — does not block the main pipeline. # Affects overall workflow success status. name: Security Gate @@ -52,96 +54,3 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: scripts/audit-license-provenance.py - codeql-gate: - if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'codeql-gate') }} - runs-on: ubuntu-latest - timeout-minutes: 240 - # Without security-events:read this job inherits the workflow's - # contents:read, the code-scanning alert API answers 403, and the gate - # below reported "0 alerts" on every run it has ever made. actions:read is - # what the CodeQL-run poll above needs for the same reason. - permissions: - contents: read - security-events: read - actions: read - steps: - - name: Wait for CodeQL on current commit (max 150 min) - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # On pull_request events github.sha is the synthetic merge commit; - # CodeQL runs are recorded against the PR head SHA. - CURRENT_SHA="${{ github.event.pull_request.head.sha || github.sha }}" - echo "Waiting for CodeQL to complete on $CURRENT_SHA..." - # 300 attempts x 30s = 150 min. The previous budget was 90 x 30s = 45 min, - # which is shorter than CodeQL actually takes on this repository: a measured - # run on PR #1426 (head 7b72652a) completed with conclusion=success after - # 124 min, 7 minutes AFTER this gate had already given up. That marked at - # least five contributor PRs red for a scan that passed. The job's own - # timeout-minutes is 240, so 150 still leaves headroom. - for attempt in $(seq 1 300); do - LATEST=$(gh api "repos/${{ github.repository }}/actions/workflows/codeql.yml/runs?head_sha=$CURRENT_SHA&per_page=1" \ - --jq '.workflow_runs[] | "\(.conclusion) \(.status)"' 2>/dev/null | head -1 || echo "") - if [ -z "$LATEST" ]; then - echo " $attempt/300: no run yet..."; sleep 30; continue - fi - CONCLUSION=$(echo "$LATEST" | cut -d' ' -f1) - STATUS=$(echo "$LATEST" | cut -d' ' -f2) - if [ "$STATUS" = "completed" ] && [ "$CONCLUSION" = "success" ]; then - echo "=== CodeQL passed ==="; exit 0 - elif [ "$STATUS" = "completed" ]; then - echo "BLOCKED: CodeQL $CONCLUSION"; exit 1 - fi - echo " $attempt/300: $STATUS..."; sleep 30 - done - echo "BLOCKED: CodeQL timeout"; exit 1 - - - name: Check for open code scanning alerts - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # FAIL CLOSED. The previous form was `gh api ... 2>/dev/null || echo "0"`, - # which looks defensive and is the opposite. gh writes the API error - # BODY to stdout, so a 403 made ALERTS the string - # {"message":"Resource not accessible by integration",...,"status":"403"}0 - # every `[` comparison then failed as non-integer, the `if` took its - # false branch, and the step printed "CodeQL gate passed (0 alerts)" - # and exited 0. That is exactly how a security gate should NEVER fail: - # silently, in the direction of "everything is fine". An unreadable - # alert list is now an error, not a zero. - # NOTE: this runs in a command substitution, so it must `return`, not - # `exit` -- an `exit` here would only end the subshell and let the - # caller continue with an empty count, recreating the very bug this - # replaces. The callers below check the status explicitly. - count_open_alerts() { - local body status - body=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' \ - --jq 'length' 2>/dev/null) - status=$? - if [ $status -ne 0 ]; then - echo "BLOCKED: cannot read code scanning alerts (gh exit $status)." >&2 - echo " Refusing to treat an unreadable alert list as zero." >&2 - echo " Most likely cause: the job lacks security-events:read." >&2 - return 1 - fi - case "$body" in - ''|*[!0-9]*) - echo "BLOCKED: alert count was not a number: ${body:-}" >&2 - return 1 - ;; - esac - printf '%s' "$body" - } - echo "Waiting 60s for alert API to settle..." - sleep 60 - ALERTS=$(count_open_alerts) || exit 1 - sleep 15 - ALERTS2=$(count_open_alerts) || exit 1 - [ "$ALERTS" -lt "$ALERTS2" ] && ALERTS=$ALERTS2 - if [ "$ALERTS" -gt 0 ]; then - echo "BLOCKED: $ALERTS open alert(s)" - gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' \ - --jq '.[] | " #\(.number) [\(.rule.security_severity_level // .rule.severity)] \(.rule.id) — \(.most_recent_instance.location.path):\(.most_recent_instance.location.start_line)"' 2>/dev/null || true - exit 1 - fi - echo "=== CodeQL gate passed (0 alerts) ===" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 621d407fc..f5cfb59d6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,47 +1,194 @@ name: CodeQL SAST +# One analysis per language: c-cpp, javascript-typescript (graph-ui, pkg/npm), +# python, actions. Scope: .github/codeql/codeql-config.yml (first-party source). +# +# FULL scan -- push to main, weekly, dry run, release: every language; c-cpp +# traces the whole production build, vendored C included. +# PR scan -- pr.yml's `codeql` job (light: true), needed by ci-ok. One leg per +# language whose scope the PR changes (codeql-* lanes of +# scripts/ci/select-lanes.sh; an undeterminable diff or a change to the +# workflows, the build or the CodeQL machinery selects every language). +# Each leg analyses its language's WHOLE first-party scope, so cross-file +# data flow is kept; c-cpp compiles the vendored objects before tracing +# starts, except vendored files the PR changes. Diff-informed analysis then +# restricts the REPORTED alerts to the changed lines; what is extracted and +# analysed is never narrowed by the diff. +# There is no pull_request trigger and no job anywhere that polls for a CodeQL +# run: the former PR gate waited up to 150 min per PR for a full analysis and, +# with ~30 PRs pushed at once, starved PR CI for hours. on: push: branches: [main] - pull_request: - branches: [main] + schedule: + - cron: "23 5 * * 1" + workflow_dispatch: + inputs: + light: + description: "PR-style c-cpp build (vendored objects compiled untraced) -- to time it" + type: boolean + default: false + workflow_call: + inputs: + light: + description: "PR scan: vendored C compiled untraced except changed files; diff-informed results" + type: boolean + default: false + matrix: + description: 'The legs to run, {"include":[...]} (pr.yml: the selected codeql-* lanes); empty = every language' + type: string + default: "" permissions: contents: read -# One analysis per pull request and one per pushed branch. A newer push to main -# cancels the analysis of an older main commit; only the newest commit's -# results matter for main, and a merge-heavy day otherwise queues one 40-100 -# minute analysis per merge ahead of every pull request run. -concurrency: - group: codeql-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - jobs: analyze: runs-on: ubuntu-latest - # CodeQL needs security-events: write to upload results; scoped to this job so - # the workflow's top-level token stays read-only (Scorecard TokenPermissions). + strategy: + fail-fast: false + # The same legs (and keys) everywhere, so a PR's analysis and main's + # share one code-scanning configuration per language. + matrix: ${{ fromJSON(inputs.matrix || '{"include":[{"language":"c-cpp","build-mode":"manual","lane":"codeql-cpp"},{"language":"javascript-typescript","build-mode":"none","lane":"codeql-js"},{"language":"python","build-mode":"none","lane":"codeql-py"},{"language":"actions","build-mode":"none","lane":"codeql-actions"}]}') }} + # Superseded runs are always cancelled: a newer push to main cancels the + # analysis of an older main commit (batched merges -> only the newest main + # commit is scanned), a newer push to a PR cancels that PR's scan. Every + # other trigger gets a group of its own and is never cancelled: a dry run + # or a release tests ONE commit, and two dispatched together must not + # cancel each other's gate. Per language, so legs never cancel each other. + # In a called workflow `github` is the CALLER's context: event_name is + # pull_request for pr.yml, github.workflow names the dry run or release. + concurrency: + group: codeql-${{ matrix.language }}-${{ github.event_name == 'push' && github.ref || github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || format('{0}-{1}', github.workflow, github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} + # CodeQL needs security-events: write to upload results (and read for the + # alert check); scoped to this job so the workflow's top-level token stays + # read-only (Scorecard TokenPermissions). permissions: security-events: write actions: read contents: read + env: + # One code-scanning configuration for every trigger. The action derives + # the key from the RUN's workflow file, which for a called workflow is + # the caller: pr.yml / dry-run.yml / release.yml would each open a + # configuration of their own, and on main an alert fixed by a later push + # would stay open in a stale one. The value is the key the push-to-main + # analysis has always used (the action documents this variable as + # settable for non-standard setups). + CODEQL_ACTION_ANALYSIS_KEY: .github/workflows/codeql.yml:analyze + # Diff-informed analysis: codeql-action v4.38.1 (CLI 2.27.0) reports a + # pull request's alerts only on its changed lines when the + # diff_informed_queries feature is on (default on; this variable forces + # it, src/feature-flags.ts). It applies only when the event payload is a + # pull request -- pr.yml's call -- and is inert on main, dry run, release. + CODEQL_ACTION_DIFF_INFORMED_QUERIES: ${{ inputs.light && 'true' || '' }} + # No compiler cache: a cache hit skips the compiler, and CodeQL extracts + # only what a traced compiler invocation actually compiles. + CBM_NO_CCACHE: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The PR scan reads the merge commit's first parent (the base) to + # find the vendored files the PR changes. + fetch-depth: ${{ inputs.light && 2 || 1 }} - name: Install build dependencies + if: ${{ matrix.language == 'c-cpp' }} run: sudo apt-get update && sudo apt-get install -y zlib1g-dev + # PR c-cpp: compile the third-party objects (tree-sitter grammars, + # sqlite, mimalloc, lz4, zstd, the tree-sitter runtime: ~1.3 GB of + # vendored C against ~45 MB first-party) BEFORE tracing starts, so the + # traced build below compiles first-party code only ... + - name: Build third-party objects (PR scan, untraced) + if: ${{ matrix.language == 'c-cpp' && inputs.light }} + run: scripts/build.sh --vendored-only + + # ... except the vendored files this PR changes: those are rebuilt + # under tracing (an undeterminable diff leaves the list empty; the scan + # still covers all first-party C). + - name: Vendored files this PR changes (traced too) + if: ${{ matrix.language == 'c-cpp' && inputs.light }} + run: scripts/ci/codeql-retrace-vendored.sh "$RUNNER_TEMP/codeql-retrace.txt" + - name: Initialize CodeQL uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: - languages: c-cpp - build-mode: manual + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + config-file: ./.github/codeql/codeql-config.yml - - name: Build for CodeQL analysis + - name: Build for CodeQL analysis (full) + if: ${{ matrix.language == 'c-cpp' && !inputs.light }} run: scripts/build.sh + - name: Build for CodeQL analysis (PR scan, first-party + changed vendored) + if: ${{ matrix.language == 'c-cpp' && inputs.light }} + run: scripts/build.sh --reuse-vendored --retrace-from "$RUNNER_TEMP/codeql-retrace.txt" + - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: - category: "/language:c-cpp" + category: "/language:${{ matrix.language }}" + + # Alert policy: the default branch carries ZERO open code-scanning + # alerts. Any open alert on the analysed ref fails this leg: on main the + # commit turns red and GitHub notifies whoever triggered the run; the + # dry run and the release cannot pass; a PR (refs/pull//merge: what + # its changed lines introduce) fails ci-ok. With a zero baseline "any + # open alert" is exactly "a new alert"; a dismissal (with a reason, in + # the Security tab) holds on every ref. Every leg checks after its own + # upload, so each language's alerts are seen by at least its own leg. + # Moved from _security.yml's former PR-CI codeql-gate job. + - name: Check for open code scanning alerts + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ALERT_REF: ${{ github.ref }} + run: | + # FAIL CLOSED. The previous form was `gh api ... 2>/dev/null || echo "0"`, + # which looks defensive and is the opposite. gh writes the API error + # BODY to stdout, so a 403 made ALERTS the string + # {"message":"Resource not accessible by integration",...,"status":"403"}0 + # every `[` comparison then failed as non-integer, the `if` took its + # false branch, and the step printed "CodeQL gate passed (0 alerts)" + # and exited 0. That is exactly how a security gate should NEVER fail: + # silently, in the direction of "everything is fine". An unreadable + # alert list is now an error, not a zero. + # NOTE: this runs in a command substitution, so it must `return`, not + # `exit` -- an `exit` here would only end the subshell and let the + # caller continue with an empty count, recreating the very bug this + # replaces. The callers below check the status explicitly. + count_open_alerts() { + local body status + body=$(gh api "repos/${{ github.repository }}/code-scanning/alerts?state=open&ref=$ALERT_REF" \ + --jq 'length' 2>/dev/null) + status=$? + if [ $status -ne 0 ]; then + echo "BLOCKED: cannot read code scanning alerts (gh exit $status)." >&2 + echo " Refusing to treat an unreadable alert list as zero." >&2 + echo " Most likely cause: the job lacks security-events:read." >&2 + return 1 + fi + case "$body" in + ''|*[!0-9]*) + echo "BLOCKED: alert count was not a number: ${body:-}" >&2 + return 1 + ;; + esac + printf '%s' "$body" + } + echo "Waiting 60s for alert API to settle..." + sleep 60 + ALERTS=$(count_open_alerts) || exit 1 + sleep 15 + ALERTS2=$(count_open_alerts) || exit 1 + [ "$ALERTS" -lt "$ALERTS2" ] && ALERTS=$ALERTS2 + if [ "$ALERTS" -gt 0 ]; then + echo "::error title=Open code scanning alerts::$ALERTS open alert(s) on $ALERT_REF -- fix, or dismiss with a reason in the Security tab" + echo "BLOCKED: $ALERTS open alert(s)" + gh api "repos/${{ github.repository }}/code-scanning/alerts?state=open&ref=$ALERT_REF" \ + --jq '.[] | " #\(.number) [\(.rule.security_severity_level // .rule.severity)] \(.rule.id) — \(.most_recent_instance.location.path):\(.most_recent_instance.location.start_line)"' 2>/dev/null || true + exit 1 + fi + echo "=== CodeQL gate passed (0 alerts) ===" diff --git a/.github/workflows/dry-run.yml b/.github/workflows/dry-run.yml index 600dbad04..463424092 100644 --- a/.github/workflows/dry-run.yml +++ b/.github/workflows/dry-run.yml @@ -2,9 +2,10 @@ # Each step can be skipped for faster iteration. # # Pipeline: lint → test → build + candidate scan/select → smoke/soak -# Security: security-static + codeql-gate (independent island) +# Security: security-static + license-gate (independent island) +# CodeQL: codeql.yml full scan of this commit + alert check (independent island) # -# Security does NOT block lint/test/build/smoke/soak. All jobs must pass +# Security and CodeQL do NOT block lint/test/build/smoke/soak. All jobs must pass # for the overall workflow to be green. # # Candidate VirusTotal scanning and tuple-local selection are part of the @@ -62,15 +63,24 @@ jobs: # ── Security (independent island — does not block main pipeline) ── security: uses: ./.github/workflows/_security.yml - # The called job requests security-events:read to count code-scanning - # alerts; a reusable workflow cannot request more than its caller grants, - # so withholding this here fails the whole run at startup. + # No job in _security.yml reads code scanning or the Actions API any more + # (the CodeQL gate moved to codeql.yml), so the token stays read-only. permissions: contents: read - security-events: read - actions: read secrets: inherit + # ── CodeQL: full scan of this exact commit + the open-alert check ── + # Every language, vendored C included (PRs run the narrower PR scan). + # codeql.yml uploads results and reads the alerts: a called workflow + # cannot request more than its caller grants, so withholding either + # permission fails the whole run at startup. + codeql: + uses: ./.github/workflows/codeql.yml + permissions: + contents: read + security-events: write + actions: read + # ── Lint (cppcheck + clang-format) ──────────────────────────── lint: if: ${{ inputs.skip_lint != true }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 815146253..81a838e80 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -26,13 +26,10 @@ jobs: security: needs: [changes] uses: ./.github/workflows/_security.yml - # The called job requests security-events:read to count code-scanning - # alerts; a reusable workflow cannot request more than its caller grants, - # so withholding this here fails the whole run at startup. + # No job in _security.yml reads code scanning or the Actions API any more + # (the CodeQL gate moved to codeql.yml), so the token stays read-only. permissions: contents: read - security-events: read - actions: read secrets: inherit with: lanes: ${{ needs.changes.outputs.lanes }} @@ -128,6 +125,7 @@ jobs: full: ${{ steps.select.outputs.full }} lanes: ${{ steps.select.outputs.lanes }} smoke: ${{ steps.select.outputs.smoke }} + codeql: ${{ steps.select.outputs.codeql }} steps: - name: List the changed files env: @@ -180,6 +178,10 @@ jobs: SMOKE='[{"os":"ubuntu-latest","lane":"smoke-ubuntu"},{"os":"macos-14","lane":"smoke-mac"},{"os":"windows-latest","lane":"smoke-win"}]' LANES=$(sed -n 's/^lanes=//p' "$RUNNER_TEMP/selection.txt") echo "smoke={\"include\":$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$SMOKE")}" >> "$GITHUB_OUTPUT" + # The CodeQL legs, tagged the same way (codeql.yml's full matrix has + # the same entries, so PR and main share a configuration). + CODEQL='[{"language":"c-cpp","build-mode":"manual","lane":"codeql-cpp"},{"language":"javascript-typescript","build-mode":"none","lane":"codeql-js"},{"language":"python","build-mode":"none","lane":"codeql-py"},{"language":"actions","build-mode":"none","lane":"codeql-actions"}]' + echo "codeql={\"include\":$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$CODEQL")}" >> "$GITHUB_OUTPUT" # ── Light smoke: build the PRODUCTION binary and run the core smoke on the # three RELIABLE NATIVE platforms. Catches built-binary regressions at PR @@ -361,13 +363,34 @@ jobs: - name: Contract steps only run: scripts/test.sh --contracts-only + # ── CodeQL PR scan (lanes codeql-cpp / -js / -py / -actions): one leg per + # language whose first-party scope this PR changes; each analyses that + # language's whole first-party scope, c-cpp with the vendored objects + # compiled before tracing (except vendored files the PR changes), and + # reports alerts on the changed lines (diff-informed). A job of this run, + # needed by ci-ok -- nothing waits for a separate CodeQL run. Push to + # main, dry run and release run the full scan (codeql.yml). ── + codeql: + needs: [changes] + if: ${{ !cancelled() && needs.changes.result == 'success' && (contains(fromJSON(needs.changes.outputs.lanes), 'codeql-cpp') || contains(fromJSON(needs.changes.outputs.lanes), 'codeql-js') || contains(fromJSON(needs.changes.outputs.lanes), 'codeql-py') || contains(fromJSON(needs.changes.outputs.lanes), 'codeql-actions')) }} + uses: ./.github/workflows/codeql.yml + # codeql.yml uploads results and reads alerts: a called workflow cannot + # request more than its caller grants (startup failure otherwise). + permissions: + contents: read + security-events: write + actions: read + with: + light: true + matrix: ${{ needs.changes.outputs.codeql }} + ci-ok: # The one required context (besides dco) — fails unless every PR stage # succeeded, so matrix renames can never silently deadlock merges, and # every lane the selector chose ran green: `skipped` is OK only for a lane # nobody selected (a selected lane that skipped is a silent gate loss). # memwaste and the test-impact shadow are report-only and not needed here. - needs: [changes, security, lint, test, pr-smoke, contracts] + needs: [changes, security, lint, test, pr-smoke, contracts, codeql] if: ${{ always() }} runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 134b425e2..707c69924 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,8 @@ # Release pipeline: lint → test → build → smoke/soak → draft → verify → publish # -# Security (security-static + codeql-gate) starts immediately and runs in -# parallel with lint/test/build/smoke/soak. It only blocks the final verify +# Security (security-static + license-gate) and CodeQL (codeql.yml: full scan +# of this commit + the open-alert check) start immediately and run in +# parallel with lint/test/build/smoke/soak. They only block the final verify # step — everything else proceeds independently. name: Release @@ -44,15 +45,24 @@ jobs: # ── Security (starts immediately, blocks only verify) ─────────── security: uses: ./.github/workflows/_security.yml - # The called job requests security-events:read to count code-scanning - # alerts; a reusable workflow cannot request more than its caller grants, - # so withholding this here fails the whole run at startup. + # No job in _security.yml reads code scanning or the Actions API any more + # (the CodeQL gate moved to codeql.yml), so the token stays read-only. permissions: contents: read - security-events: read - actions: read secrets: inherit + # ── CodeQL (starts immediately, blocks only verify) ────────────── + # Full scan of this exact commit (every language, vendored C included) + + # the open-alert check. codeql.yml uploads results and reads the alerts: + # a called workflow cannot request more than its caller grants, so + # withholding either permission fails the whole run at startup. + codeql: + uses: ./.github/workflows/codeql.yml + permissions: + contents: read + security-events: write + actions: read + # ── 0. Preflight: refuse malformed dispatch inputs ────────────── # The tag is inputs.version VERBATIM. A bare "0.10.7" publishes a release the # installers can never resolve (they fetch releases/download/v/...), @@ -312,12 +322,12 @@ jobs: body: ${{ inputs.release_notes || '' }} generate_release_notes: ${{ inputs.release_notes == '' }} - # ── 7. Verify + Publish (requires security gate) ─────────────── + # ── 7. Verify + Publish (requires the security and CodeQL gates) ─ verify: - needs: [release-draft, security] + needs: [release-draft, security, codeql] # Explicit condition: a deliberately skipped test ancestor must not skip # the publish chain; the draft itself proves smoke and soak both succeeded. - if: ${{ !cancelled() && !failure() && needs.security.result == 'success' && needs.release-draft.result == 'success' }} + if: ${{ !cancelled() && !failure() && needs.security.result == 'success' && needs.codeql.result == 'success' && needs.release-draft.result == 'success' }} runs-on: ubuntu-latest permissions: contents: write diff --git a/MAINTAINERS.md b/MAINTAINERS.md index d749762b6..6c4b07368 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -52,7 +52,7 @@ Operational authority is stricter than code review authority. | Operation | Current authority | Project rule | | --- | --- | --- | -| PR validation (`pr.yml`, DCO, CodeQL) | Automatic | Anyone may trigger it by opening or updating a PR. Required checks must pass. | +| PR validation (`pr.yml`, DCO, PR CodeQL scan) | Automatic | Anyone may trigger it by opening or updating a PR. Required checks must pass. | | Dry run (`dry-run.yml`) | `@DeusData` | Delegated release operators may run dry runs after promotion. Dry-run delegation does not imply release authority. | | Smoke/soak/repro manual runs | `@DeusData` | Area reviewers may operate these runs when delegated for diagnosis. Results are advisory. | | Release workflow (`release.yml`) | `@DeusData` only | Owner-only until a release operator is explicitly promoted. Publishing, replacing releases, and tag movement remain owner-gated. | diff --git a/Makefile.cbm b/Makefile.cbm index 6d19f430a..3b93146f1 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -537,6 +537,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_compile_commands.c \ src/pipeline/pass_infrascan.c \ src/pipeline/pass_k8s.c \ + src/pipeline/helm_template.c \ src/pipeline/pass_operator.c \ src/pipeline/pass_shell.c \ src/pipeline/pass_terraform.c \ @@ -1298,6 +1299,16 @@ $(BUILD_DIR)/prod_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c $(BUILD_DIR)/.build-co OBJS_VENDORED_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LSP_OBJ_PROD) $(PP_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ) +# The part of OBJS_VENDORED_PROD that holds no first-party code: everything but +# prod_lsp_all.o (first-party LSP unity) and prod_preprocessor.o (first-party +# code around vendored simplecpp). CodeQL's light PR scan builds these BEFORE +# it starts tracing (scripts/build.sh --vendored-only), so the traced build +# (--reuse-vendored) compiles first-party code only: the grammars alone are +# most of the 1.3 GB of vendored C, against ~45 MB of first-party code. +OBJS_THIRD_PARTY_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ) +.PHONY: cbm-third-party +cbm-third-party: $(OBJS_THIRD_PARTY_PROD) + MAIN_SRC = src/main.c # Rebuild when the build CONFIGURATION changes, not only when sources do. The diff --git a/README.md b/README.md index 590c9eef9..7cdac83bf 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -# codebase-memory-mcp: local MCP server for code search, lower token usage and visualization +# codebase-memory-mcp: local code intelligence for AI agents via MCP [![GitHub Release](https://img.shields.io/github/v/release/DeusData/codebase-memory-mcp?style=flat&color=blue)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE) [![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/release.yml?branch=main&label=main%20CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/release.yml) -[![C tests](https://img.shields.io/badge/C_tests-9388_cases-blue)](#tests) +[![Native and frontend tests](https://img.shields.io/badge/tests-native_%2B_frontend-blue)](#tests) [![MCP tools](https://img.shields.io/badge/MCP_tools-17-blue)](#mcp-tools) [![Languages](https://img.shields.io/badge/languages-162-orange)](https://github.com/DeusData/codebase-memory-mcp) [![Hybrid LSP](https://img.shields.io/badge/Hybrid_LSP-type_resolution-blue)](#hybrid-lsp) @@ -14,33 +14,28 @@ [![SLSA 3](https://slsa.dev/images/gh-badge-level3.svg)](https://slsa.dev) [![VirusTotal](https://img.shields.io/badge/VirusTotal-scanned_every_release-brightgreen?logo=virustotal)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) [![arXiv](https://img.shields.io/badge/arXiv-2603.27277-b31b1b?logo=arxiv)](https://arxiv.org/abs/2603.27277) +[![Join the community on Discord](https://img.shields.io/badge/Discord-Join_the_community-5865F2?logo=discord&logoColor=white)](https://discord.gg/KguGXxg2X) -**Search your code, follow call graphs, and explore software architecture in your browser.** codebase-memory-mcp is a local [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server that gives Claude Code, Codex, and other coding agents a persistent code knowledge graph. It indexes functions, classes, imports, HTTP routes, and their relationships across [162 languages](#language-support). [Hybrid LSP](#hybrid-lsp) adds semantic type resolution for **Python, TypeScript, JavaScript, JSX/TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl**. +**Understand your repository, from source code to the systems around it.** codebase-memory-mcp is an MCP server that connects code, documentation, configuration, and infrastructure in a persistent local knowledge graph. Use it to find an unfamiliar feature, follow a call across services, trace a deployment back to its source, or assess which parts of a project a change could affect. -**About 120× fewer tokens** in the project's reported five-query comparison: ~3,400 vs ~412,000 with file-by-file exploration. Published backend benchmarks also report **Linux kernel indexing in ~3 minutes** (28M lines of code) and **Cypher relationship queries under 1 ms** on an Apple M3 Pro. [Benchmark details](#performance). +Give your coding agent structural context through [Model Context Protocol](https://modelcontextprotocol.io/) (MCP), or query the graph directly from the CLI. **The MCP server is the core of the project.** Its optional visual companion, **CBM Lens, is in open beta (v0.0.1)**, with a source reader, architecture maps, the Galaxy graph, coverage diagnostics, architecture decisions, and local code chat. -**CBM Lens Beta helps you understand unfamiliar code and stay in control as it grows.** Use it to get up to speed in a new codebase, tackle rescue engineering on an inherited project, or make sense of 50,000 lines of AI-generated code from a single day. Its IDE-style source reader, local code chat, 3D architecture map, and Galaxy graph let you see how the code fits together and follow dependencies and call paths back to the source. Use that context to review changes, investigate fragile areas, and keep code quality high. Architecture decisions stay alongside the project's index. +The native C backend exposes **17 MCP tools**, parses [languages and file formats](#language-support) with vendored grammars, and adds [Hybrid LSP](#hybrid-lsp) type resolution for supported languages. The installer supports **45 automatic or conditional client integrations**, including client configuration, skills, agent profiles, and applicable hooks. Indexing and graph queries need no hosted service or API key. Focused graph queries can reduce repeated file reads and token use; see the [measured results](#performance). -The native C backend exposes 17 MCP tools and supports 45 automatic or conditional client integrations. Indexing and graph queries need no hosted service or API key. - -> **CBM Lens Beta is new on `main`:** it arrived with [PR #2068](https://github.com/DeusData/codebase-memory-mcp/pull/2068) and is not part of the current v0.11.0 release, whose installer still ships the previous graph UI. To run the interface shown below now, [build from source](#build-from-source). - -[Quick start](#quick-start) · [Explore CBM Lens](#cbm-lens-beta-code-visualization-and-source-navigation) · [Chat with your codebase locally](#chat-with-your-codebase-locally) · [MCP tools](#mcp-tools) · [Build from source](#build-from-source) - -> **Research:** The [preprint](https://arxiv.org/abs/2603.27277) compares agents across 31 repositories, reporting **10× lower token use** and **2.1× fewer tool calls**, with answer-quality scores of 83% for CBM versus 92% for file-by-file exploration. These backend results are separate from the five-query comparison above and from CBM Lens. +[Quick start](#quick-start) · [Capabilities](#features) · [Documentation](#documentation-connected-to-code) · [Infrastructure](#infrastructure-and-deployment) · [Smart CI](#smart-ci-and-test-impact) · [Explore CBM Lens](#cbm-lens-beta-code-visualization-and-source-navigation) · [Agent setup](#agent-setup-skills-profiles-and-hooks) · [MCP tools](#mcp-tools) > **Local processing:** Source analysis and graph storage run on your machine. CBM Lens can optionally download a model for browser AI after you enable it; inference stays on the device. The installer writes agent configuration for the clients you select. See [Security](#security) for release verification and [SECURITY.md](SECURITY.md) for reporting issues. -**Architecture overview.** Folders become nested platforms, files and symbols become selectable blocks, and arrows show indexed relationships. The recording zooms into the map of this repository, selects `src/pipeline`, opens the area, selects `artifact.c` and opens its source. Height can show source size; hotspot gravity reflects static fan-in. [Still image](docs/images/lens-architecture.png). +**See the graph with CBM Lens (open beta v0.0.1).** Folders become nested platforms, files and symbols become selectable blocks, and arrows show indexed relationships. The recording opens `src/foundation` in the map of this repository, selects `arena.c` and opens its source. Height can show source size; hotspot gravity reflects static fan-in. [Still image](docs/images/lens-architecture.png).

- CBM Lens Beta architecture walkthrough: zoom into the source folders of this repository, open src/pipeline, select artifact.c and open its source + CBM Lens Beta architecture walkthrough: open src/foundation, select arena.c and read its source beside the map

-**Galaxy.** Explore the code knowledge graph in 3D. The recording searches `cbm_pipeline_run`, follows its outgoing calls one layer further, switches to the hierarchy and shows the path to `cbm_store_check_integrity`. Edge pulses show relationship direction; they do not represent live execution. [Current still: two dependency layers around `cbm_max_file_bytes`](docs/images/lens-galaxy.jpg). +**Galaxy.** Explore the code knowledge graph in 3D. The recording follows the outgoing relationships of `cbm_arena_alloc` one layer further, switches to the hierarchy and shows the two-hop call path to `cbm_alloc`. Edge pulses show relationship direction; they do not represent live execution. [Still: two dependency layers around `cbm_max_file_bytes`](docs/images/lens-galaxy.jpg).

- Code knowledge graph in CBM Lens Beta Galaxy: search a function, expand its outgoing calls, switch to the hierarchy and follow a path + Code knowledge graph in CBM Lens Beta Galaxy: expand cbm_arena_alloc, switch to the hierarchy and follow its call path through arena_grow to cbm_alloc

**Explore.** Browse files in an IDE-style workspace, read syntax-highlighted source in Monaco, and keep the code graph alongside it. File impact shows what depends on the open file, and Code details shows the callers and calls of the symbol under the caret. Select code to [ask the local chat about it](#chat-with-your-codebase-locally). @@ -58,27 +53,27 @@ The native C backend exposes 17 MCP tools and supports 45 automatic or condition -These captures show CBM Lens Beta on 12–13 October 2026, using public source from this repository. See [capture details](docs/images/README.md) for each capture's build, indexed scope and recording setup. +The high-resolution GIFs show CBM Lens Beta on 20 October 2026; the still images were captured on 12–13 October. All show public source from this repository. See [capture details](docs/images/README.md) for each capture's build, indexed scope and recording setup. ## Why codebase-memory-mcp -- **Indexing performance**: Linux kernel (28M LOC, 75K files) in 3 minutes. RAM-first pipeline: LZ4 compression, in-memory SQLite, fused Aho-Corasick pattern matching. Memory released after indexing. -- **Plug and play**: native executable plus authenticated release-owned assets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (arm64/amd64). The native install needs no Docker, language runtime, or API keys. Download → `install` → restart agent → done. -- **162 languages**: vendored tree-sitter grammars compiled into the binary. No separate parser installation. -- **About 120× fewer tokens in the reported five-query comparison**: ~3,400 tokens vs ~412,000 via file-by-file search. See [Performance](#performance) for the scope and sources. -- **45 supported automatic/conditional client surfaces**: `install` configures detected clients and safely activates conditional clients only when their documented platform, marker, or explicit existing config path is present. See [Multi-Agent Support](#multi-agent-support) for the complete matrix and manual/UI-only boundaries. -- **CBM Lens Beta**: read source, chat about selected code with a local model, explore architecture and call graphs, inspect coverage gaps, and edit architecture decision records in your browser. -- **Infrastructure-as-code indexing**: Dockerfiles, container images, docker compose, Kubernetes manifests, and Kustomize overlays indexed as graph nodes with cross-references. `Resource` nodes for K8s kinds and compose services, `Image` nodes joining the images that are built and run, `Module` nodes for Kustomize overlays with `IMPORTS` edges to referenced resources; a Service walks to the program its container runs. -- **17 MCP tools**: search, trace, architecture, impact analysis, targeted index-coverage checks, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. +- **Find the right code with context**: combine symbol, text, and semantic search with callers, dependencies, source snippets, and file outlines. Targeted queries and compact, paged results can reduce token use and repeated file reads; see [Performance](#performance) for scoped measurements. +- **Connect documentation to implementation**: index document sections, PDF text pages, architecture decisions, and supported references into code. Find the explanation alongside the symbol. +- **Follow the system beyond source files**: trace supported relationships through Docker, Kubernetes, Helm, Kustomize, Terraform/OpenTofu, Terragrunt, Ansible, scripts, Make, and CI configuration. +- **Understand a change before running it**: inspect affected symbols, compare indexed snapshots, and find related tests. [Experimental Smart CI](#smart-ci-and-test-impact) combines static relationships and per-test coverage for supported C/C++ test harnesses, falling back to broader runs when evidence is insufficient. +- **Explore visually with CBM Lens Beta**: read source, follow call paths, inspect architecture and coverage gaps, manage architecture decisions, and ask a local model about selected code. +- **Equip your coding agents**: `install` sets up supported client configuration, durable guidance, skills, Scout/Verify/Auditor profiles, and applicable context hooks. Support varies by client; see the [integration matrix](#multi-agent-support). +- **Keep context local and reusable**: indexes persist between sessions, eligible active MCP sessions can keep them synchronized, and optional graph artifacts let a team share a baseline. +- **Run a native tool on your machine**: standard releases include CBM Lens for macOS, Linux, and Windows. The native runtime needs no Docker or language runtime; parser grammars are included. ## Quick Start -**Install the latest release** (macOS / Linux). For CBM Lens Beta shown above, [build from source](#build-from-source) until the next release. +**Install the latest release** (macOS / Linux): ```bash curl -fsSL https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.sh | bash ``` -The released graph UI is included in the standard installer. +The standard native release includes CBM Lens. Start it after installation with `codebase-memory-mcp daemon start --open --port=9749`. The [Nix packages](#nix-flake) and [source builds](#build-from-source) also offer builds without the UI. **Windows** (PowerShell): ```powershell @@ -136,7 +131,19 @@ Restart your coding agent. Say **"Index this project"**: done. The `install` command automatically strips macOS quarantine attributes and ad-hoc signs the binary; no manual `xattr`/`codesign` needed. -The `install` command auto-detects installed coding agents and configures their documented MCP entries plus durable instructions, skills, and lifecycle hooks where supported. +### Agent setup: skills, profiles, and hooks + +Installation also equips supported coding agents to use the graph in daily work. The native `install` command detects clients and installs the integration pieces their documented formats support: + +| Component | What it adds | +|-----------|--------------| +| **MCP configuration** | Connects the client to the local server and its tools. | +| **Instructions and skills** | Explains graph-first exploration, index freshness, coverage checks, and when to read source. | +| **Scout, Verify, and Auditor profiles** | Offers fast provisional lookup, task-directed verification, or bounded full verification, where custom agents are supported. | +| **Session and compaction hooks** | Restores graph context at supported lifecycle events. | +| **Search and read hooks** | Adds relevant graph context to supported searches and flags recorded coverage gaps after supported file reads. | + +Hook augmentation is context-only and fail-open: it does not replace or block the requested tool call. Features vary by client and platform; some clients use durable instructions or a parent-agent handoff instead of hooks or direct subagent MCP access. Codex users review and trust installed hooks through `/hooks`. See [Multi-Agent Support](#multi-agent-support) and [Sessions, compaction, and subagents](#sessions-compaction-and-subagents) for exact support and activation requirements. ### Session Coordination Daemon @@ -154,18 +161,18 @@ Thin frontends still write immediate startup and session-specific errors to thei All active CBM processes must run the exact same version, executable build, coordination ABI, and canonical cache root. Equivalent `CBM_CACHE_DIR` aliases resolve to the same root; a genuinely different root is rejected while any CBM process is active. MCP servers, hooks, one-shot CLI commands, temporary index workers, and the daemon share a crash-safe OS admission barrier; starting an ordinary conflicting process fails before doing work and records an explicit conflict in `${CBM_CACHE_DIR}/logs/daemon-conflicts.ndjson`. -The native `install`, `update`, and `uninstall` commands are the deliberate exception to that conflict rule. Download, verification, and private same-filesystem staging happen first so a bad candidate never disrupts active work. Activation then publishes account-wide maintenance intent, asks the daemon and every temporary local operation to cancel, and waits to a finite deadline for all coordinated CBM processes to exit. It holds the admission and lifetime barriers exclusively while changing the active binary, configuration, PATH, or indexes. New CBM work cannot enter during this window. Activation progress and results are recorded in `${CBM_CACHE_DIR}/logs/activation-events.ndjson`, and a successful command tells you to restart open coding-agent sessions so they launch the activated build. +The native `install` and `uninstall` commands are the deliberate exception to that conflict rule. For installation, download, verification, and private same-filesystem staging finish before activation so a bad candidate never disrupts active work. The native `update` command prints the installer command that performs the update. Activation then publishes account-wide maintenance intent, asks the daemon and every temporary local operation to cancel, and waits to a finite deadline for all coordinated CBM processes to exit. It holds the admission and lifetime barriers exclusively while changing the active binary, configuration, PATH, or indexes. New CBM work cannot enter during this window. Activation progress and results are recorded in `${CBM_CACHE_DIR}/logs/activation-events.ndjson`, and a successful command tells you to restart open coding-agent sessions so they launch the activated build. Package-manager setup (npm, PyPI, or Go) verifies and caches the native executable for that package. It does not replace the active native installation and therefore does not stop running CBM sessions. When that cached binary is executed, it still enters the same exact-build admission barrier. The shell and PowerShell installers invoke the verified candidate's native `install` command, so they do receive the full account-wide activation guarantee. -The ordinary `cli` mode runs one command locally, but it is not disconnected from the daemon above: it connects to the shared coordination daemon (starting one if none is already running) for that same admission barrier plus per-project locks on graph mutations. Its connection is a `cli_session`: it never registers with the background watcher and never starts a UI, and it holds the admission lease only for the command's lifetime. If the command's own connection is what started the daemon, that daemon exits again once the command closes and no other session is attached; if a daemon was already running, the command simply joins and leaves it exactly as found. While the command is running, a temporary monitor separately lets activation cancel that operation and its supervised worker safely; the monitor exits with the command and never becomes a standing daemon. See [CLI Mode](#cli-mode) for details. +The ordinary `cli` mode runs one command locally, but it is not disconnected from the daemon above: it connects to the shared coordination daemon (starting one if none is already running) for that same admission barrier plus per-project locks on graph mutations. Its connection is a `cli_session`: it does not register a project watcher or open a browser, and it holds the admission lease only for the command's lifetime. The shared daemon may serve Lens according to its UI configuration and available assets. If the command's own connection is what started the daemon, that daemon exits again once the command closes and no other session is attached; if a daemon was already running, the command simply joins and leaves it exactly as found. While the command is running, a temporary monitor separately lets activation cancel that operation and its supervised worker safely; the monitor exits with the command and never becomes a standing daemon. See [CLI Mode](#cli-mode) for details. ### CBM Lens Beta: code visualization and source navigation -CBM Lens Beta is the browser interface built into the binary. It builds on [CodeAtlasWeb](graph-ui/README.md) by [Bernhard Jackiewicz](https://github.com/BernhardJackiewicz) (design and tracking in #1964). It runs on loopback; the only request that leaves your machine is the optional model download from Hugging Face, and only after you start it. Each panel names where its facts come from, and what the index did not record is shown as a gap. Start the local server: +**CBM Lens is an optional visual companion to the MCP server, currently in open beta (v0.0.1).** Standard native releases include its browser interface, so you can launch Lens directly after installation; UI-enabled source builds include it too. It builds on [CodeAtlasWeb](graph-ui/README.md) by [Bernhard Jackiewicz](https://github.com/BernhardJackiewicz) (design and tracking in #1964). Lens is served on loopback. Browser AI downloads its optional model only after you enable it; inference stays on-device. Each panel names where its facts come from, and what the index did not record is shown as a gap. Start the local server: ```bash -./build/c/codebase-memory-mcp daemon start --open --port=9749 +codebase-memory-mcp daemon start --open --port=9749 ``` The browser opens at `http://127.0.0.1:9749`. A permanent daemon keeps the UI available after the command exits. Use `daemon status` to inspect it and `daemon stop` when you are finished. For an MCP session, `--ui=true --port=9749` enables the same shared UI. Active clients must use the same executable build and cache directory. @@ -209,9 +216,9 @@ Open **Chat** with the panel button at the top right to ask about the code you a Questions about callers and callees, such as "Who calls cbm_pipeline_run?", are answered from the loaded graph without the model, and the answer says so. When you select a symbol, the chat lists its indexed facts and the lines of its code, and the model adds one sentence; that sentence is left out when it claims something the source does not show. Text written by the model is labelled as such. -The recording shows a question about callers answered from the indexed graph, next to the automatic explanation of the selected symbol. [Still image](docs/images/lens-chat.png). +The recording asks who calls `arena_grow` and what it calls, with both answers listed directly from the indexed graph beside the selected symbol. [Still image of a model explanation and graph answer](docs/images/lens-chat.png). -

Chat with your codebase locally in CBM Lens Beta: a selected symbol, its listed facts and a caller question answered from the indexed graph

+

Chat with your codebase locally in CBM Lens Beta: callers and callees of arena_grow answered directly from the indexed graph

The chat history is kept for 24 hours in this browser, per project. Each new message uses the source currently open in the reader; earlier source snapshots are not silently accumulated, and messages about another file or selection are not sent along. If a prompt is too long for the model, older messages are left out first; if it still does not fit, CBM Lens asks you to select less code instead of cutting the code. This chat explains the context you give it; it cannot edit files or run tools. Check its answers against the source. The small default model words things loosely, especially in languages other than English. @@ -232,7 +239,7 @@ The Agents workspace and activity overlays are hidden in normal builds. To inclu VITE_EXPERIMENTAL_AGENTS=true scripts/build.sh --with-ui ``` -Activity comes from explicitly configured tool hooks. The bundled adapter supports Claude Code PostToolUse; other clients need their own adapter. Recorded events go to the local daemon, and the reader must enable activity before the browser polls them. This does not automatically observe every agent. See the [frontend setup guide](graph-ui/README.md#die-agentenebene-was-live-arbeitende-agenten-auf-dem-graphen-zeigen). +Activity comes from explicitly configured tool hooks. The bundled adapter supports Claude Code PostToolUse; other clients need their own adapter. Recorded events go to the local daemon, and the reader must enable activity before the browser polls them. This does not automatically observe every agent. See the [frontend setup guide](graph-ui/README.md#experimental-agent-activity). @@ -246,7 +253,7 @@ Enable automatic indexing on MCP session start: codebase-memory-mcp config set auto_index true ``` -When enabled, new projects are indexed automatically on first connection. Previously-indexed projects are registered with the background watcher for ongoing git-based change detection. Configurable file limit: `config set auto_index_limit 50000`. +`auto_index` is off by default. When enabled, new projects are indexed automatically on first connection. The configurable file limit is `config set auto_index_limit 50000`. Previously indexed projects can be watched independently through `auto_watch` (on by default), subject to the session rules below. The watcher follows MCP sessions: it watches the project each open MCP session is rooted in (the client's working directory) and stops when the last session for that project closes. A repository indexed with `cli index_repository`, or indexed from a session rooted somewhere else, is not watched; re-run `index_repository` after changing it. `index_status` reports this in its `watch` object: `watched`, and either the poll cadence and last scan time or the `reason` it is not watched. @@ -267,7 +274,7 @@ bash "/install.sh" ```powershell # Windows -powershell -ExecutionPolicy Bypass -File "\install.ps1" +powershell -File "\install.ps1" ``` The install script is placed next to the binary at install time, so the printed path resolves beside the executable. It is idempotent, so re-running it *is* the update: it stops the daemon, retires the running binary, installs the new one, and cleans up. @@ -299,9 +306,10 @@ The install script placed beside the binary is **reported, not deleted**: uninst ### Graph & analysis - **Architecture overview**: `get_architecture` returns a compact summary by default. Request specific `aspects` for routes, hotspots, dependencies, clusters, or cycles. `system_structure` provides the projection used by CBM Lens and must be requested on its own. -- **Architecture Decision Records**: `manage_adr` persists architectural decisions across sessions +- **Architecture Decision Records**: `manage_adr` persists the project's decision record across sessions; indexing also extracts supported ADR metadata and supersession from documentation files - **Louvain community detection**: Discovers functional modules by clustering call edges -- **Git diff impact mapping**: `detect_changes` maps uncommitted changes to affected symbols with risk classification +- **Change impact**: `detect_changes` maps uncommitted changes to affected symbols with risk classification and can report test impact with `scope: "tests"`; `compare_graphs` compares indexed snapshots +- **Runtime observations**: `ingest_traces` adds observed call relationships from separately collected traces alongside static analysis; static call paths alone do not show what ran - **Call graph**: Resolves function calls across files and packages (import-aware, type-inferred) - **Dead code candidates**: find functions with no indexed callers, excluding entry points. Check dynamic dispatch, coverage gaps, and framework conventions before removing code. - **Cypher-like queries**: `MATCH (f:Function)-[:CALLS]->(g) WHERE f.name = 'main' RETURN g.name` @@ -312,6 +320,27 @@ The install script placed beside the binary is **reported, not deleted**: uninst - **Structural search** (`search_graph`): regex name patterns, label filters, min/max degree, file scoping - **Code search** (`search_code`): graph-augmented grep over indexed files only +### Documentation connected to code + +Find the explanation behind a symbol and follow supported references back to its implementation. Documentation participates in the graph alongside source files: + +- **Markdown, reStructuredText, and AsciiDoc** become navigable document sections; **PDFs with text layers** become searchable text pages. PDF extraction does not perform OCR. +- **Document-to-code references** and supported language doc-comment references create `MENTIONS` relationships when the target can be resolved. +- **Architecture decision records** contribute metadata such as status, deciders, and supersession, so decisions can be explored with the code they describe. +- **Supported diagrams and OpenAPI descriptions** contribute structured entities. Extraction and reference resolution differ by format; not every textual mention becomes a graph edge. + +Use graph and text search to locate the relevant section, follow its recorded relationships, and open the source evidence. Check `check_index_coverage` before treating a missing result as proof that the repository has no such documentation or reference. + +### Infrastructure and deployment + +Ask **"What deploys this service?"**, **"Which image runs it?"**, or **"Where does this configuration reach code?"** CBM connects supported infrastructure forms to the source they build, configure, and run: + +- **Containers and clusters**: Dockerfiles, Compose services, Kubernetes resources, Helm charts and releases, and Kustomize overlays and patches. +- **Provisioning and configuration**: Terraform/OpenTofu resources and modules, tfvars overrides, Terragrunt dependencies, Ansible tasks and handlers, and supported configuration keys and environment-variable access. +- **Build and delivery**: shell commands, Make targets, and supported CI steps that build images, invoke repository scripts, or deploy resources. + +For resolvable inputs, a trace can follow **Service → workload → image → Dockerfile → program**, or follow deployment and configuration relationships. Use `trace_path` modes for topology, deployment, and run paths. These are static relationships from supported source forms; dynamic values and unresolved templates can leave gaps. See [Edge types](#edge-types-selected) for the exact relationship rules and limitations. + ### Cross-service linking - **HTTP** route ↔ call-site matching with confidence scoring - **gRPC, GraphQL, tRPC** service detection with protobuf Route extraction @@ -323,6 +352,21 @@ The install script placed beside the binary is **reported, not deleted**: uninst - **Run status**: `index_status` (and the UI layout next to `linked_projects`) carries a `cross_repo` object, `{"status":"never_run"}` until `index_repository(mode="cross-repo-intelligence")` has run with that project as the source, then `"ran"` with `last_run_at`, `outcome`, `targets`, `projects_scanned`, `total_cross_edges` and `skipped_projects`. That is how you tell "never linked" apart from "linked, nothing matched". The status is stored in the project's own index next to its `CROSS_*` edges, so a reindex that rebuilds the index drops both at once and the status goes back to `never_run`. - **Skipped stores**: `target_projects: ["*"]` skips an index built before the #768 schema change rather than failing the whole run. The response lists each one as `skipped_projects: [{"name", "reason": "pre_768_schema", "hint": "reindex this project"}]`. +### Smart CI and test impact + +Use the graph to explain which tests a change reaches, and use that evidence to focus supported CI runs. **Smart CI is experimental** and has two layers: + +| Layer | What it does | What you supply | +|-------|--------------|-----------------| +| **CI job selection** | Chooses jobs from the changed-file list. | Repository-specific path and job rules; this repository's selector is an example. | +| **Test selection** | Combines static reachability, changed tests, and admitted per-test coverage to select suites or tests. | A supported C/C++ test harness, configuration, a coverage map, and a runner that accepts named selections. | + +`codebase-memory-mcp test-impact select` produces a selection with reasons; `test-impact publish` produces the graph-and-coverage bundle used as its baseline. For agent-assisted investigation, `detect_changes` with `scope: "tests"` exposes test impact through MCP. + +**Narrowing currently supports C/C++ family code with this repository's test conventions or declared equivalents.** It is not general test selection for every language CBM indexes. Missing, stale, incompatible, or incomplete evidence broadens the selection or returns `run_all`; static reachability without admitted coverage never omits a suite. Begin with report-only shadow mode and validate your harness before enabling gating. + +In this repository, test narrowing is wired to the Linux x86_64 test leg when `TEST_IMPACT_MODE=gate`; release and dry-run workflows retain full execution. See the [Smart CI guide](docs/SMART_CI.md) for supported conventions, coverage admission, setup, version requirements, and fallback behavior. + ### Edge types (selected) - `CALLS`: a callable is invoked at the source site - `CALL_REFERENCE`: a callable is used at a supported reference site (for example, a direct value argument) and resolves to one exact target @@ -333,6 +377,7 @@ The install script placed beside the binary is **reported, not deleted**: uninst - `INFRA_MAPS`: a Kubernetes Service to each workload whose pod labels carry every `key: value` of the Service's selector, within one namespace - `CONNECTS_TO`: a Kubernetes workload or compose service to the Service its containers' environment addresses (`PRODUCT_CATALOG_SERVICE_ADDR: productcatalogservice:3550`, a URL's host, `..svc...`, a bare name on an `*_ADDR` / `*_HOST` / `*_URL` / ... key; compose: the project's `.env` interpolated): a Kubernetes Service of the same bundle and namespace, or a compose service of the same file. The edge keeps `via: env`, `key` and `container`, never the value - `USAGE` with `ref: image`: a Kubernetes workload (any pod template: containers, initContainers, CronJobs, custom resources) or a docker compose service to the `Image` it runs +- Helm templates are rendered for their images: with the chart's values (the `Chart` `DEPLOYS` them, `default_of`: the values keys read) and with each release's (a Flux HelmRelease, a kustomize `helmCharts` entry, a Terraform `helm_release`: `override_of`). An image the render cannot know is left out, never guessed: after a change of values it does not run exactly, a `tpl` or include it cannot look into, a release value it cannot read, or a work budget it runs past. Known limit: each template file renders on its own, while Helm renders a chart's files with one shared `.Values` (a subchart's is its parent's `.Values.`), so a chart that changes `.Values` in one file and reads it in another can still show a wrong image - `DEPLOYS`: what puts manifests, kustomizations or charts on a cluster, to what it puts there: a Flux Kustomization or Argo Application to the kustomizations / manifests / chart of its path, a Flux HelmRelease to its chart (`spec.chart` or `chartRef` → OCIRepository), a shell command or Make recipe (`kubectl apply -f/-k`, `kustomize build | kubectl apply`, `helm install/upgrade/template`), a Go installer loading a chart with the Helm SDK, a Terraform `helm_release`, a Terraform `kubernetes_*` resource the object it declares (a `Resource` like a manifest's, `ecosystem: terraform`, when its name is a literal), a kustomization inflating a chart (`helmCharts`; its values file an `IMPORTS`). A chart outside the repository is an external `Chart`, keyed by its publisher when its repository URL is stated (`__chart__/`, the same node in every project, the URL in `repository`), else one per project and name; the edge says `via` (`flux`, `argo`, `helmrelease`, `kubectl`, `kustomize`, `helm`, `helm-sdk`, `terraform`, `helmcharts`, ...) - `DEPENDS_ON`: a Helm chart to the charts it declares (the local chart for a resolvable `file://` repository, otherwise an external `Chart`; the edge keeps `repository`), and a `go.mod` / `requirements.txt` file to an external `Package`; a Dockerfile to the `Image`s it builds on (`via: from`, `copy-from`); an `Image` to the Dockerfile it is built from (`via: build`; `tool`: `skaffold`, `compose`, `docker-build` for `docker` / `podman` / `nerdctl` / `buildah` build commands in scripts, Make and CI); a compose service to the services its `depends_on` / `links` name - `CONFIGURES`: code reading an environment variable to the shared `EnvVar` of that name (`strategy: env_access`); a Kubernetes container, compose service or Dockerfile (`ENV` of the image) setting one to that container's own `EnvVar` (`strategy: env_set`, with `source`: `value`, `configMapKeyRef`, `secretKeyRef`, `fieldRef`, ..., `compose`, `dockerfile`; values are never stored); and that container's `EnvVar` to the code reading the name inside the files its image is built from (`strategy: env_read`, with `dockerfile`): the image's files are the sources its Dockerfile's `COPY` / `ADD` put into it, less the context's `.dockerignore` @@ -423,15 +468,15 @@ Published backend benchmarks on Apple M3 Pro. These are separate from the CBM Le | Dead code detection | ~150ms | Full graph scan with degree filtering | | Trace call path (depth=5) | <10ms | BFS traversal | -**RAM-first pipeline**: All indexing runs in memory (LZ4 HC compressed read, in-memory SQLite, single dump at end). Memory is released back to the OS after indexing completes. +**Memory-aware pipeline**: Indexing keeps hot data in memory and can spill extraction results to disk when needed. Memory is released back to the OS after indexing completes. -**Token efficiency**: The project's [original README report](https://github.com/DeusData/codebase-memory-mcp/blob/4ff6feeab639c4668da0f491930326ed5ec63093/README.md#performance) lists ~3,400 tokens for five structural queries via codebase-memory-mcp versus ~412,000 via file-by-file exploration. That is approximately **120× fewer tokens**, or a **99.2% reduction**, for that comparison. The separate [31-repository study](https://arxiv.org/html/2603.27277v1#S4.SS1) reports a 10× token reduction. Its [system-performance measurements](https://arxiv.org/html/2603.27277v1#S4.SS3) include Linux kernel indexing in about 3 minutes and Cypher relationship queries under 1 ms on an Apple M3 Pro. +**More focused context, fewer tokens**: The project's [31-repository preprint](https://arxiv.org/html/2603.27277v1#S4.SS1) reports **10× lower token use** and **2.1× fewer tool calls** compared with file-by-file exploration. These March 2026 backend measurements cover the study's workloads and agent setup; the paper documents the full evaluation. An earlier [five-query comparison](https://github.com/DeusData/codebase-memory-mcp/blob/4ff6feeab639c4668da0f491930326ed5ec63093/README.md#performance) reported ~3,400 versus ~412,000 tokens for structural queries. The study's [system-performance measurements](https://arxiv.org/html/2603.27277v1#S4.SS3) include the Linux indexing and Cypher timings above. -To measure comparable quality, latency, and agent-efficiency metrics on your own workload, see [Measuring quality, latency, and agent savings](docs/MEASURING_SAVINGS.md). Exact reproduction of the figures above requires the original inputs and raw artifacts. +Explore the [measurement guide](docs/MEASURING_SAVINGS.md) to evaluate quality, latency and token use on your own workload. ## Troubleshooting & Diagnostics -codebase-memory-mcp runs **100% locally and collects no telemetry**: your code, queries, environment, and usage never leave your machine. That privacy guarantee also means that when you hit something we can't reproduce on our side (a slow memory climb over hours, a performance regression, a leak that only appears after days of real use), **we have no data at all unless you choose to send it.** Here is how to capture it yourself. +codebase-memory-mcp **indexes and queries locally and collects no telemetry**. When you hit something we can't reproduce on our side (a slow memory climb over hours, a performance regression, a leak that only appears after days of real use), **we have no diagnostic data unless you choose to send it.** Here is how to capture it yourself. ### Capture a diagnostics log @@ -570,7 +615,7 @@ Counts on `main` at `4d9592b5`, where CBM Lens arrived. Native and Go counts use | npm package wrapper | 24 tests | Counted in the source | | Python package wrapper | 43 tests | Counted in the source | -Run the test suite (9,388 tests across 189 suites): +Run the native test suite: ```bash scripts/test.sh # full: clean sanitizer build + all suites + guards @@ -626,7 +671,7 @@ Restart your agent. Verify with `/mcp`; you should see `codebase-memory-mcp` wit ## Multi-Agent Support -`install` configures 45 client surfaces: 39 detected automatically and 6 +`install` configures **45 supported automatic/conditional client surfaces**: 39 detected automatically and 6 conditional or explicit. “Conditional” means the installer writes only when the documented platform or an explicit, already-existing config path proves the target is active. It never flips experimental feature flags, enables plugins, @@ -820,7 +865,7 @@ codebase-memory-mcp cli search_graph --project my-project --label Function --for codebase-memory-mcp cli list_projects --format json --detail stats | jq '.projects[].name' ``` -JSON arguments can also be piped on stdin, for tools that take arguments. A tool whose input schema declares none; `list_projects`; never reads stdin, so it stays responsive when it inherits a pipe the caller never closes (the default for `child_process.spawn` and similar wrappers). Inline JSON remains accepted for backward compatibility but is deprecated in favor of flags, `--args-file`, or stdin. +Tools requiring arguments accept JSON piped on stdin. Tools with only optional arguments, such as `list_projects`, accept flags, `--args-file`, or redirected JSON files without waiting on an inherited pipe. Inline JSON remains accepted for backward compatibility but is deprecated in favor of flags, `--args-file`, or stdin. ## MCP Tools @@ -829,7 +874,7 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool | Tool | Description | |------|-------------| | `index_repository` | Index a repository into the graph. Auto-sync keeps it fresh only while it is the project of an open MCP session; repos indexed from the CLI or from another project's session need a re-run (`index_status` shows `watch`). Waits for the whole index by default; pass `async: true` to start it in the daemon and return at once, then poll with `status: true` (see below). | -| `list_projects` | List all indexed projects with node/edge counts. | +| `list_projects` | List indexed projects with paging; use `detail: "stats"` for node/edge counts and database sizes. | | `delete_project` | Remove a project and all its graph data. | | `index_status` | Check indexing status of a project. | | `check_index_coverage` | Check whether exact paths or a scope are indexed and fresh. A clean result means no recorded gap, not proof of completeness. | @@ -945,7 +990,7 @@ is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). | `CBM_DOWNLOAD_URL` | *(GitHub releases)* | Override the download URL for updates. Used for testing or self-hosted deployments. | | `CBM_LOG_LEVEL` | role-aware | Set the minimum log level. Thin MCP/CLI/hook frontends default to `warn`; the detached daemon and its supervised index workers default to `info` so lifecycle and liveness records remain available. Accepted values (case-insensitive): `debug`, `info`, `warn`, `error`, `none`; or their numeric equivalents `0` to `4`. A physical worker retains INFO liveness records even under a stricter override because its private log drives the supervisor's no-progress timeout. Frontend messages go to that session's stderr; detached daemon events go to `${CBM_CACHE_DIR}/logs/cbm-daemon.log`. Stdout is reserved for MCP JSON-RPC. | | `CBM_WORKERS` | *(detected)* | Override the parallel-indexing worker count returned by `cbm_default_worker_count`. Useful inside containers where `sysconf(_SC_NPROCESSORS_ONLN)` reports host CPUs rather than the cgroup's effective quota. Range 1 to 256; invalid values are ignored with a warning. | -| `CBM_MEM_BUDGET_MB` | *(detected)* | Override the in-memory graph budget with an explicit cap in MiB, taking precedence over the `ram_fraction × total_RAM` default. Useful on bare-metal hosts without a cgroup limit, or to pin a budget *below* the cgroup limit so headroom is left for sibling processes. Must be a positive integer; it is clamped to detected total RAM (logged as `mem.budget.clamped`), and non-numeric or non-positive values are ignored with a warning (`mem.budget.env.invalid`). | +| `CBM_MEM_BUDGET_MB` | *(detected)* | Override the in-memory graph budget with an explicit cap in MiB, taking precedence over the derived default (`ram_fraction × total_RAM`, adjusted for available memory and daemon worker caps). Useful on bare-metal hosts without a cgroup limit, or to pin a budget *below* the cgroup limit so headroom is left for sibling processes. Must be a positive integer; it is clamped to detected total RAM (logged as `mem.budget.clamped`), and non-numeric or non-positive values are ignored with a warning (`mem.budget.env.invalid`). | | `CBM_DUMP_VERIFY_MIN_RATIO` | `0.5` | After indexing, compare persisted SQLite node count to the in-memory dump count. When persisted nodes fall below this fraction of committed nodes (and committed > 50), `index_repository` returns `status:"degraded"` instead of silent `indexed`. Range 0 to 1; set `0` to disable. Invalid values are ignored with a warning. | Environment used by daemon-owned components (such as diagnostics, daemon logging, and process-wide indexing resource limits) is captured from the first daemon-backed session that starts the daemon. Later sessions join that process and cannot replace those values. To change them, close all daemon-backed sessions, update the relevant agent configurations consistently, and restart a session. `CBM_ALLOWED_ROOT` remains session-specific, a conflicting `CBM_CACHE_DIR` is rejected, and a one-shot CLI command is not exempt from the rule above: it connects to the coordination daemon like any other session, starting one if none is running. Its own environment becomes the captured daemon-owned environment only when its invocation is the one that starts the daemon; joining an already-running daemon, it inherits that daemon's already-captured values instead. @@ -975,9 +1020,9 @@ Need the full config-file reference? See [docs/CONFIGURATION.md](docs/CONFIGURAT Each entry maps an extension (which **must** start with `.`) to a language name. Language names are matched **case-insensitively**. Accepted values (aliases in parentheses) are: -`bash` (`sh`), `c`, `c++` (`cpp`), `c#` (`csharp`), `clojure`, `cmake`, `cobol`, `common lisp` (`commonlisp`, `lisp`), `css`, `cuda`, `dart`, `dockerfile`, `elixir`, `elm`, `emacs lisp` (`emacslisp`), `erlang`, `f#` (`fsharp`), `form`, `fortran`, `glsl`, `go`, `graphql`, `groovy`, `haskell`, `hcl` (`terraform`), `html`, `ini`, `java`, `javascript`, `json`, `julia`, `kotlin`, `lean`, `lua`, `magma`, `makefile`, `markdown`, `matlab`, `meson`, `nix`, `objective-c` (`objc`), `ocaml`, `perl`, `php`, `protobuf`, `python`, `r`, `ruby`, `rust`, `scala`, `scss`, `sql`, `svelte`, `swift`, `toml`, `tsx`, `typescript`, `verilog`, `vimscript`, `vue`, `wolfram`, `xml`, `yaml`, `zig`. +`arkts`, `asciidoc`, `bash` (`sh`), `c`, `c++` (`cpp`), `c#` (`csharp`), `chialisp`, `clojure`, `cmake`, `cobol`, `common lisp` (`commonlisp`, `lisp`), `css`, `cuda`, `dart`, `diagram`, `dockerfile`, `elixir`, `elm`, `emacs lisp` (`emacslisp`), `erlang`, `f#` (`fsharp`), `form`, `fortran`, `glsl`, `go`, `graphql`, `groovy`, `haskell`, `hcl` (`terraform`), `html`, `ini`, `java`, `javascript`, `jinja2` (`jinja`), `json`, `julia`, `kotlin`, `lean`, `lua`, `magma`, `makefile`, `markdown`, `matlab`, `meson`, `mojo`, `nix`, `objective-c` (`objc`), `ocaml`, `openapi`, `pdf`, `perl`, `php`, `plsql`, `protobuf`, `python`, `r`, `ruby`, `rust`, `scala`, `scss`, `sql`, `svelte`, `swift`, `toml`, `tsx`, `typescript`, `verilog`, `vimscript`, `vue`, `wolfram`, `xml`, `yaml`, `zig`. -Project config overrides global for conflicting extensions. An entry whose language name is unknown, or whose extension does not start with `.`, is skipped and a warning is logged to stderr (shown at the default `info` log level). Missing config files are ignored. +Project config overrides global for conflicting extensions. An entry whose language name is unknown, or whose extension does not start with `.`, is skipped and logged as a warning. Missing config files are ignored. ## Persistence @@ -1067,7 +1112,7 @@ Every release is verified through a multi-layer pipeline before publication: - **Sigstore cosign**: keyless signatures on all artifacts; bundles included in every release - **SHA-256 checksums**: `checksums.txt` published with every release; verified by both install scripts before extraction - **CodeQL SAST**: blocks release pipeline if any open alerts remain -- **No language-runtime dependency chain**: the native backend uses libraries vendored at compile time, and the frontend is embedded by the UI build +- **No language-runtime dependency chain**: the native backend needs no language-runtime environment; UI builds embed the frontend ### VirusTotal release scans @@ -1086,6 +1131,10 @@ The [v0.11.0 release notes](https://github.com/DeusData/codebase-memory-mcp/rele The release also publishes [candidate scan results](https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.11.0/virustotal-candidate-results.tsv), [binary provenance](https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.11.0/release-candidates.tsv), and [selection records](https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.11.0/release-selection.tsv). For subsequent versions, use the scan evidence linked from the [latest release](https://github.com/DeusData/codebase-memory-mcp/releases/latest). +## Help shape what comes next + +The next idea could be yours. We invite the OSS community to bring ideas, tricky integration questions, and feedback to [our Discord community](https://discord.gg/KguGXxg2X). When a direct conversation works better, reach out to **DeusData** or the project's co-developers there. Let's build what comes next together. + ## License MIT diff --git a/SECURITY.md b/SECURITY.md index 48da1d1a1..ec9fea712 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -111,7 +111,7 @@ This project implements multiple layers of security verification. Every release - **All dangerous function calls** require a reviewed entry in `scripts/security-allowlist.txt` - **Time-bomb pattern detection** — scans for `time()`/`sleep()` near dangerous calls (could indicate delayed activation) - **MCP tool handler file read audit** — tracks file read count in `mcp.c` against an expected maximum (detects added file reads that could exfiltrate data through tool responses) -- **CodeQL SAST** — static application security testing on every push (taint analysis, CWE detection, data flow tracking). Any open alert blocks the release. +- **CodeQL SAST** — static application security testing of the C/C++, JavaScript/TypeScript, Python and GitHub Actions code (taint analysis, CWE detection, data flow tracking): a full scan on every push to main, weekly, and in every dry run and release; a first-party scan of the languages a pull request changes, with alerts reported on its changed lines. Any open alert fails the run and blocks the release. - **Fuzz testing** — random/mutated inputs to MCP server and Cypher parser (60 seconds per build). Catches crashes, segfaults, and memory errors that structured tests miss. - **Native antivirus scanning** on every platform (any detection fails the build): - **Windows**: Windows Defender with ML heuristics — the same engine end users run diff --git a/docs/images/README.md b/docs/images/README.md index 1c1ecc96d..89f811c37 100644 --- a/docs/images/README.md +++ b/docs/images/README.md @@ -1,49 +1,53 @@ # CBM Lens Beta README captures -These are screenshots and browser recordings of the application, captured on 12–13 October 2026. The application data and text were not retouched. Bernhard Jackiewicz captured the recordings and PNG stills on 12 October. The Explorer and Galaxy JPEG stills were refreshed on 13 October. One addition is drawn on top of the recordings: a small mouse pointer and click ring, because a headless browser shows no pointer of its own. +These are screenshots and browser recordings of the real application using public source from this repository. The application data and text were not retouched. The GIFs were refreshed on 20 October 2026; the still images preserve the earlier captures from 12–13 October. -## Original captures: 12 October +## High-resolution recordings: 20 October - Repository: [DeusData/codebase-memory-mcp](https://github.com/DeusData/codebase-memory-mcp) -- Branch: `main`, after [PR #2068](https://github.com/DeusData/codebase-memory-mcp/pull/2068) brought CBM Lens Beta in -- Commit: `4d9592b5` -- Build: `scripts/build.sh --with-ui` -- Browser: headless Chromium (Playwright) on macOS, English locale, viewport 1440 × 900, device scale factor 1 -- Agents: standard build, experimental agent activity disabled +- Source commit: `4cf44147e639ae070ee44d9230244bfb9d9676e0` +- Build: `make -f Makefile.cbm cbm-with-ui`, with the standard frontend and experimental agent activity disabled +- Capture: built-in browser screenshots on macOS, English locale, native viewport **1600 × 1000**, **12 frames per second** +- Encoding: JPEG screenshot frames converted to GIF with one 256-color palette per recording, no dithering, infinite looping; no upscaling, sharpening or pointer overlays -The `v0.0.1` label in the application is the frontend package version. It is not the backend release version. +The `v0.0.1` label is the frontend package version, not the backend release version. Standard native releases include Lens; using a source build for these recordings is a capture-provenance detail, not an installation requirement. -### Data shown +### Data and interaction -The project `codebase-memory-mcp` is an export of the same public commit (`git archive`), indexed in `full` mode into an isolated local cache. No private repository was used. Because the export has no `.git` folder, Explore reports "Git unavailable" in its file impact line. +The project `codebase-memory-mcp` is a clean `git archive` export of the source commit, indexed in `full` mode into an isolated local cache. No private repository was used. Because the export has no `.git` folder, the interface reports "Git unavailable" where applicable. -The index reported 75,037 nodes and 361,349 edges. These are properties of this source scope, not a benchmark. The interface keeps its coverage and display-limit notices. +The index reported 81,190 nodes and 411,072 edges. These counts describe this source scope, not a benchmark or proof of complete repository coverage. The interface retains its coverage and display-limit notices. The clips join recorded interaction segments with pauses shortened between them; their displayed application states and answers are unchanged. Directional pulses represent indexed relationships, not observed execution. -## Refreshed stills: 13 October +| Recording | Duration | What it shows | +|-----------|----------|---------------| +| [lens-overview.gif](lens-overview.gif) | 12.5 seconds | Architecture overview, opening `src/foundation`, browsing its files, selecting `arena.c` and reading its source beside the map. | +| [lens-galaxy.gif](lens-galaxy.gif) | 12 seconds | Expanding outgoing relationships around `cbm_arena_alloc`, switching to the hierarchy and tracing the two-hop call path `cbm_arena_alloc → arena_grow → cbm_alloc`. | +| [lens-chat.gif](lens-chat.gif) | 11.5 seconds | `arena_grow` selected in Galaxy, with caller and callee questions answered directly from the indexed graph. | + +All three GIFs are 1600 × 1000. Their frames and playback duration were decoded and checked after export. README and website use these same files. + +### Local chat check + +The optional Qwen2.5 Coder 0.5B model was downloaded and loaded through the application's own setup. Automatic explanations were disabled for this recording. Both displayed answers are deterministic graph answers, labelled "Listed from the indexed graph; not generated by the model." The clip shows one incoming `CALLS` relationship from `cbm_arena_alloc` and two outgoing `CALLS` relationships to `cbm_alloc` and `cbm_free`, alongside the other indexed relationship types. It does not demonstrate model-generated prose. + +## Original still images: 12 October + +Bernhard Jackiewicz captured the PNG stills on 12 October 2026, from commit `4d9592b5` on `main` after [PR #2068](https://github.com/DeusData/codebase-memory-mcp/pull/2068). The build used `scripts/build.sh --with-ui`; headless Chromium on macOS used English locale, a 1440 × 900 viewport and device scale factor 1. Experimental agent activity was disabled. + +The indexed project was a public `git archive` export of that commit in `full` mode, with 75,037 nodes and 361,349 edges. The PNG stills are 1440 × 900, except the chat still at 1440 × 1200 so the explanation and answer fit together. These are historical application views, also used as reduced-motion alternatives on the website; they are not extracted from the new GIFs. + +| Still | What it shows | +|-------|---------------| +| [lens-architecture.png](lens-architecture.png) | The whole repository on the architecture map. | +| [lens-chat.png](lens-chat.png) | A local model explanation of `cbm_pipeline_pass_doclinks` and a caller answer listed from the graph. | +| [lens-system-structure.png](lens-system-structure.png) | Component groups and their dependencies in System structure, with `src` selected. | +| [lens-behavior.png](lens-behavior.png) | Possible direct calls of `compileCards`, with its declaration. | + +## Refreshed still images: 13 October `lens-explore.jpg` and `lens-galaxy.jpg` are unedited browser captures at 1280 × 720 from the locally served CBM Lens Beta UI. The running binary reports `dev`; its exact build commit is unavailable, so these captures do not certify an exact source revision. The local project is named `cbm-triage-main-20260919`; the screenshots show public CBM source and indexed relationships. -- **Explore:** `src/foundation/limits.c`, its compact file impact summary and direct dependencies (24 nodes, 26 edges). The UI's freshness, sampling and coverage caveats remain visible. -- **Galaxy:** `cbm_max_file_bytes`, both directions, all edge types, two layers (160 nodes, 218 edges). +- **[Explore](lens-explore.jpg):** `src/foundation/limits.c`, its compact file impact summary and direct dependencies (24 nodes, 26 edges). The UI's freshness, sampling and coverage notices remain visible. +- **[Galaxy](lens-galaxy.jpg):** `cbm_max_file_bytes`, both directions, all edge types, two layers (160 nodes, 218 edges). -These counts describe the displayed index snapshot, not a benchmark or proof of complete repository coverage. The original recordings and other stills are unchanged. - -## Media - -| File | What it shows | -|------|---------------| -| [lens-overview.gif](lens-overview.gif) | Architecture overview of the repository: zoom and a small orbit, selecting `src/pipeline`, opening that area, selecting `artifact.c` and opening its source. | -| [lens-architecture.png](lens-architecture.png) | Still image of the whole repository on the architecture map. | -| [lens-galaxy.gif](lens-galaxy.gif) | Galaxy: searching `cbm_pipeline_run`, following its outgoing calls one layer further, the hierarchy view and the path to `cbm_store_check_integrity`. | -| [lens-galaxy.jpg](lens-galaxy.jpg) | Two dependency layers around `cbm_max_file_bytes`, with direction and edge-type controls. | -| [lens-explore.jpg](lens-explore.jpg) | `src/foundation/limits.c` in the reader with its file impact summary, file tree and direct graph dependencies. | -| [lens-chat.gif](lens-chat.gif) | The chat with the local Qwen2.5 Coder 0.5B model: the explanation of `cbm_pipeline_pass_doclinks` and "Who calls cbm_pipeline_pass_doclinks?" answered from the graph. | -| [lens-chat.png](lens-chat.png) | Still image of the same explanation and listed answer. | -| [lens-system-structure.png](lens-system-structure.png) | Component groups and their dependencies in System structure, `src` selected. | -| [lens-behavior.png](lens-behavior.png) | Possible direct calls of `compileCards`, with its declaration. | - -The recordings were captured as browser video, then encoded as GIF at 960 pixels wide and 10 frames per second with a palette per recording. While a search is typed, the Galaxy briefly shows "No matching indexed nodes." until the index search answers; those frames (under one second) were cut. The original PNG stills are 1440 × 900, the chat still 1440 × 1200 so that the explanation and the answer fit together. The refreshed JPEG stills are 1280 × 720. Directional pulses represent indexed relationships, not observed execution. - -## Local chat check - -The chat recording uses the actual browser model, downloaded through the application's own setup. The explanation lists the indexed facts and the declaration of the selected function; the one sentence after them was written by the model and is labelled as such. The caller answer is listed from the indexed graph without the model. +These counts describe the displayed index snapshot, not a benchmark or proof of complete repository coverage. diff --git a/docs/images/lens-chat.gif b/docs/images/lens-chat.gif index 1eaaecb86..946f777e7 100644 Binary files a/docs/images/lens-chat.gif and b/docs/images/lens-chat.gif differ diff --git a/docs/images/lens-galaxy.gif b/docs/images/lens-galaxy.gif index 1aa8899e3..3c906eea1 100644 Binary files a/docs/images/lens-galaxy.gif and b/docs/images/lens-galaxy.gif differ diff --git a/docs/images/lens-overview.gif b/docs/images/lens-overview.gif index 733833553..058397d1e 100644 Binary files a/docs/images/lens-overview.gif and b/docs/images/lens-overview.gif differ diff --git a/docs/index.html b/docs/index.html index e39ded17e..fac7e3b5b 100644 --- a/docs/index.html +++ b/docs/index.html @@ -3,838 +3,504 @@ - codebase-memory-mcp — Code Intelligence Knowledge Graph for AI Coding Agents - - + Codebase Memory — Local Code Intelligence & MCP + - - - - - + + + - - - - - - - + + + + + + - - - - + + + + - - - - - - + + - - - - - - - -
-
-

codebase-memory-mcp

-

by DeusData

-

- The fastest, most efficient code intelligence engine for AI coding agents. It indexes any - repository into a persistent knowledge graph — full-indexing an average repo in seconds and the - Linux kernel in 3 minutes — so your agent answers structural questions with ~120x fewer tokens. - Tree-sitter parsing across 162 languages, Hybrid LSP type resolution, native executable with verified assets. -

-
-
~120x
fewer tokens
-
158
languages
-
3 min
Linux kernel index
-
44
client surfaces
+ +
+
- -
- 3D knowledge-graph visualization of the codebase-memory-mcp graph showing thousands of nodes and edges -

Built-in 3D graph visualization (UI variant) — explore your knowledge graph at localhost:9749.

-
- - - -
-

What is codebase-memory-mcp?

-

- codebase-memory-mcp is an open-source Model Context - Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, - classes, call chains, HTTP routes, and cross-service links. Instead of reading files one at a time, - an AI coding agent queries the graph — answering structural questions with roughly 120x fewer - tokens. It parses 162 languages and ships as a native executable with authenticated release-owned assets; - the native install needs no language runtime. -

-

- It is a structural-analysis backend, not a chatbot: there is no embedded LLM and no API key. Your - MCP client (Claude Code, or any MCP-compatible agent) is the intelligence layer; codebase-memory-mcp - builds and serves the graph. All processing happens locally — your code never leaves your machine. -

-
- -
-

Why do AI agents waste tokens exploring code?

-

- AI coding agents explore codebases by reading files one at a time. Every structural question - triggers a cascade of grep → read file → grep again → read more files. The cost compounds fast. -

-

- Across five structural questions about a real codebase, file-by-file search consumed - ~412,000 tokens; the same questions answered from the - knowledge graph took ~3,400 tokens — a ~120x reduction. -

-

- The win is not about fitting the context window. It is cost (at $3–15 per million tokens, exploration - adds up), latency (sub-millisecond graph queries versus seconds of file reading), and accuracy - (less noise means better answers and no "lost in the middle" problem). -

-

Source: project benchmark, 5 structural queries — see the - full benchmark report.

-
- -
-

How do I install codebase-memory-mcp?

-

- Install with a single command, then tell your agent to index the project. The native runtime set contains - an executable with verified assets for macOS, Linux, and Windows — no Docker, language runtime, or API key. -

-
- # 1. One-line install (macOS / Linux). Add --ui for the 3D graph UI.
- curl -fsSL https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.sh | bash

- # 2. The installer configures detected clients; explicit flags cover conditional clients.

- # 3. Restart your agent, then say:
- "Index this project" + +
+
+
+
+
+

Local MCP server · Open source

+

Understand the
whole repository.

+

Codebase Memory (CBM) is an open-source server for the Model Context Protocol (MCP). Give your AI coding agent a persistent knowledge graph of code, documentation and infrastructure. Search symbols, trace relationships and retrieve source evidence through 17 MCP tools.

+

Focused graph queries help reduce repeated code reading and token use. See the evidence →

+
+ Install the MCP server + Explore the MCP tools +
+

macOS · Linux · Windows  /  Native C · MIT licensed

+
+
+ For your agent

17 MCP tools, source evidence and coverage checks, with client-specific skills, profiles and hooks.

+ For your workflow

Native CLI access, persistent indexes and optional visual exploration in Lens.

+
-

- One command configures 45 automatic/conditional client surfaces. Detected automatically (39): - Claude Code, Codex CLI, Gemini CLI, Zed, OpenCode, Antigravity, Aider, KiloCode, VS Code, Cursor, - Windsurf, Augment / Auggie, OpenClaw, Kiro, Junie, Hermes, OpenHands, Cline, Warp, Qwen Code, - GitHub Copilot CLI, Factory Droid, Crush, Goose, Mistral Vibe, Grok Build, Qoder CLI, Kimi Code CLI, - GitLab Duo CLI, Rovo Dev CLI, Amp, Devin CLI / Local, Tabnine, Amazon Q Developer IDE, - CodeBuddy Code CLI, IBM Bob Shell, Pochi, Pi, and Oh My Pi (omp). -

-

- Conditional or explicit (6): Continue / cn, Visual Studio, TRAE, Roo Code, IBM Bob IDE, - and Sourcegraph Cody. Manual, UI, cloud, or repository-managed (not counted): Qodo, - Warp MCP, JetBrains AI/ACP, GitHub Copilot coding agent, Jules, CodeRabbit, Replit, BLACKBOX AI, - Plandex, and SWE-agent. Warp is counted above for its detected skill installation; its MCP connection - remains manual. Windows users run install.ps1. Also available via npm, - pip, Homebrew, Scoop, AUR, and go install. -

-

- Lifecycle installation follows documented context contracts: Qoder uses SessionStart, - SubagentStart, and post-Read coverage, including its documented Windows - PowerShell executor. Kimi uses UserPromptSubmit; - on macOS/Linux, GitLab Duo gets a fail-open user SessionStart, while Devin gets - UserPromptSubmit, PostCompaction, and a deduplicated SessionStart - when Claude does not already provide it. GitLab Duo, Devin, and Factory hooks are withheld on - Windows where no deterministic shell/executor contract is documented. Cline's auto-activating file hooks - are withheld because their context output is not reliably consumed; CodeBuddy's beta hooks are not - auto-installed; Junie's EAP - SessionStart output is documented as ignored; and Cursor context hooks are withheld because - the documented events cannot safely provide race-free MCP context to read-only subagents. -

-

- Documented custom-agent formats receive three exact-owned evidence tiers: Scout for narrow provisional - discovery, Verify as the task-directed default, and Auditor for bounded, paginated, current-generation - verification. Every direct tier calls check_index_coverage for cited paths and relevant scopes, - then reads flagged ranges or skipped files directly. Kiro and Junie use positive-allowlist - --tool-profile scout and --tool-profile analysis server surfaces; - Junie selects dedicated named aliases because its subagent schema filters by server. - Qoder combines named-server selection with exact tier-specific MCP tool IDs. Factory uses - exact registered tool IDs without its additive whole-server mcpServers field. - A foreign Junie alias is preserved and causes the installed profiles to fail closed to parent handoff. - Cursor, Rovo, Augment, Pochi, and Cline use explicit - parent handoff where direct child MCP is unavailable or unsafe; Pochi is limited to readFile. - Neither IBM Bob surface receives an invented hook or custom agent. Amazon Q Developer - IDE defaults to ~/.aws/amazonq/default.json while preserving either existing documented - alternative. -

-
- -
-

What is Hybrid LSP?

-

- Hybrid LSP is semantic type resolution beyond tree-sitter. Tree-sitter alone produces a syntactic - AST — it handles naming, structure, and call sites, but it cannot tell that - user.profile.display_name() resolves to Profile.display_name declared - three modules away, because it does not track imports, generics, inheritance, or stdlib types. -

-

- codebase-memory-mcp ships a lightweight C implementation of language type-resolution - algorithms, structurally inspired by and compatible with major language servers — - tsserver/typescript-go, pyright, gopls, Roslyn, Eclipse JDT, and rust-analyzer — - embedded directly into the native executable. There is no language-server process, no - per-project setup, and no API key. This layer runs alongside tree-sitter on every parse and refines - CALLS, USAGE, and RESOLVED_CALLS edges with type information, - so the graph mirrors what an IDE "Go to Definition" would resolve. -

-

Languages with full Hybrid LSP

- - - - - - - - - - - - - -
LanguageWhat it resolves
PythonImports and dotted submodule walks, dataclasses, Self return types, generics, @property, match/case patterns, SQLAlchemy 2.0 Mapped[T], Pydantic models, typing annotations, async/await, isinstance/walrus narrowing, and common stdlib.
TypeScript / JavaScript / JSX / TSXGenerics, JSX component dispatch, JSDoc inference for plain JS, .d.ts declarations, module re-exports, and method chaining via return-type propagation across a shared cross-file registry.
PHPNamespaces, traits, late-static-binding, PHPDoc inference, parameter binding, and return-type inference.
C#Global usings, file-scoped namespaces, records (incl. C# 12 primary constructors), LINQ method syntax, async Task<T>/ValueTask<T> unwrap, generic methods, var inference, and common BCL stdlib.
GoPre-built per-package cross-file registry, generics, embedded structs, interface satisfaction, and package-aware import resolution.
C / C++Shared cross-language registry: macros, typedef chains, and header-vs-source linking on the C side; templates, namespaces, auto inference, and class-hierarchy method resolution on the C++ side.
Java (new in v0.8.0)Imports (single-type, on-demand, static), class hierarchies with this/super dispatch, generics, annotations, overload matching by arity and parameter types, lambdas and method references bound to functional interfaces, and common JDK stdlib.
Kotlin (new in v0.8.0)Imports and same-package resolution, classes / objects / companion objects, extension functions, data classes, nullable-type unwrapping, scope functions (let/apply/run/also/with), infix calls, and common stdlib.
Rust (new in v0.8.0)use declarations and module paths, impl blocks and trait methods, struct fields, generics with trait bounds, operator-trait desugaring, derive-macro method synthesis, UFCS static paths, and common std prelude.
-

- The two-layer pipeline runs a fast syntactic tree-sitter pass for every one of the 162 languages, - then a type-aware Hybrid LSP pass on top for the families above. Languages without a Hybrid LSP pass - yet fall back to textual resolution, so you always get an answer. -

-
- - -
-

How much does the knowledge graph save?

-

- Each common structural question costs hundreds of tokens against the graph versus tens of thousands - via file-by-file search. Totals across five queries: ~3,400 vs ~412,000 tokens. -

- - - - - - - - - - - - -
Question typeGraphFile-by-fileSavings
Find function by pattern~200~45,000225x
Trace call chain (depth 3)~800~120,000150x
Dead code detection~500~85,000170x
List all routes~400~62,000155x
Architecture overview~1,500~100,00067x
Total~3,400~412,000~121x
-

- A separate evaluation across 31 real-world repositories, described in the preprint, reported 83% answer quality, - 10x fewer tokens, and 2.1x fewer tool calls versus file-by-file exploration. -

-

Source: “Codebase-Memory: Tree-Sitter-Based Knowledge - Graphs for LLM Code Exploration via MCP”, arXiv:2603.27277 — and the - project benchmark report.

+
+
+

Beyond file search

Find the relationship.
Then read the evidence.

+

A function rarely tells the whole story. Follow its callers, find the decision behind it, and trace the infrastructure that runs it.

+
+
+
+ 01 / CODE

What calls this?

+

Explore symbols, call paths, types, dependencies and routes. Move across files, services and indexed repositories.

+
  • Structural, full-text and local semantic search
  • Hybrid LSP type resolution for supported languages
  • HTTP, gRPC, GraphQL, tRPC and async links
+ Explore code intelligence → +
+
+ 02 / DOCUMENTATION

Why does it work this way?

+

Bring documentation into the graph. Search sections, follow resolved code references, and keep architecture decisions close to their implementation.

+
  • Markdown, reStructuredText and AsciiDoc sections
  • Searchable pages from PDFs with text layers
  • ADRs, supersession and supported doc comments
+ Documentation support → +
+
+ 03 / INFRASTRUCTURE

What builds and deploys it?

+

Connect supported infrastructure and configuration to source. Trace the path from a service to the program inside its container.

+
  • Docker, Compose, Kubernetes, Helm and Kustomize
  • Terraform / OpenTofu, Terragrunt and Ansible
  • Shell, Make and supported CI steps
+ Infrastructure support → +
+
+
+ A deployment pathServiceWorkloadImageDockerfileProgram +
+

Relationships come from supported source forms. Dynamic values and unresolved references can leave gaps; PDF extraction requires a text layer and does not perform OCR.

-
-

How fast is it?

-

- Indexing is RAM-first (LZ4 compression, in-memory SQLite, single dump at end) and memory is released - to the OS afterward. Queries run in under a millisecond. -

- - - - - - - - - -
OperationTimeNotes
Linux kernel full index3 min28M LOC, 75K files → 4.81M nodes, 7.72M edges
Django full index~6 s49K nodes, 196K edges
Cypher query<1 msRelationship traversal
Name search (regex)<10 msSQL LIKE pre-filtering
Trace call path (depth 5)<10 msBFS traversal
-

Source: project Performance benchmarks, measured on Apple M3 Pro.

+
+
+

Code intelligence

Ask better questions
of your repository.

+

Give your coding agent structural context through 17 MCP tools. Query the same graph from the CLI and open source evidence whenever a result needs verification.

+
+ +

Follow actual relationships

Tree-sitter parsing and Hybrid LSP resolution build call and dependency relationships for supported languages. Trace callers, callees, data flow and supported cross-service links; query more complex patterns with Cypher.

Language and framework coverage varies. Check the support matrix →

+

Understand change

Map a diff to affected symbols, investigate the blast radius, compare graph snapshots and inspect related tests. Near-clone and semantic relationships help surface related implementations worth reviewing.

detect_changescompare_graphsSIMILAR_TO
+

Connect the wider system

Link multiple repositories in the same store. Explore service relationships across HTTP, gRPC, GraphQL, tRPC and supported async patterns, or ingest runtime traces to add observed relationships.

Cross-repo intelligenceingest_tracesget_architecture
+

Know what the index knows

Inspect index status and recorded coverage before making a claim. Coverage diagnostics identify skipped, partial or stale evidence so you can read the relevant source directly. Preserve architecture decisions with ADR management.

check_index_coverageindex_statusmanage_adr
+
Explore all 17 MCP tools
index_repositorysearch_graphquery_graphtrace_pathget_code_snippetget_file_outlineget_graph_schemacompare_graphsget_architecturesearch_codelist_projectsdelete_projectindex_statuscheck_index_coveragedetect_changesmanage_adringest_traces

Tool reference, inputs and examples →

-
-

Features

-
-
-

162 languages

-

Python, Go, JS, TS, TSX, Rust, Java, C++, C#, C, PHP, Ruby, Kotlin, Scala, Zig, Elixir, Haskell, OCaml, Swift, Dart, Lean 4, and many more via vendored tree-sitter grammars compiled into the binary.

-
-
-

Hybrid LSP type resolution

-

Language-server-grade type inference for Python, TS/JS, PHP, C#, Go, C/C++, Java, Kotlin, and Rust — embedded in the binary, no server process or per-project setup.

-
-
-

Pure C, no language runtime

-

A native executable with authenticated release-owned assets for macOS, Linux, and Windows. The native install needs no Docker, language runtime, or API keys; fully static Linux builds are available in the -portable archives.

-
-
-

Call-graph tracing

-

Trace callers and callees across files and packages with import-aware, type-inferred resolution. BFS traversal up to depth 5.

-
-
-

Dead-code detection

-

Find functions with zero callers, with smart filtering that excludes entry points like route handlers, main(), and framework decorators.

-
-
-

Cross-service linking

-

Matches REST routes to HTTP call sites across services with confidence scoring — and detects gRPC, GraphQL, and tRPC services plus pub/sub channels (EMITS/LISTENS_ON for Socket.IO, EventEmitter, and message buses) and async queue dispatch.

-
-
-

Infrastructure-as-code indexing

-

Dockerfiles, Kubernetes manifests, and Kustomize overlays become graph nodes with cross-references to the resources they configure.

-
-
-

Auto-sync

-

A background watcher detects changes in the project your MCP session is working in and re-indexes it incrementally. No manual reindex after editing files there.

-
-
-

Team-shared graph artifact

-

Commit one zstd-compressed snapshot (.codebase-memory/graph.db.zst); teammates bootstrap from it and skip the full reindex.

-
-
-

3D graph visualization

-

The optional UI variant adds one verified, content-addressed frontend pack and serves an interactive 3D graph at localhost:9749 to explore nodes, edges, and clusters visually.

-
-
-

17 MCP tools

-

search_graph, trace_path, detect_changes, query_graph (Cypher), get_architecture, get_code_snippet, check_index_coverage, manage_adr, and 9 more.

-
-
-

Cypher graph queries

-

Run read-only Cypher-style queries against the graph for multi-hop patterns that structured search can't express.

-
-
-

Semantic code search

-

Find code by meaning, not just name, via semantic_query vector search — powered by nomic-embed-code embeddings baked into the binary. No API key, fully local.

-
-
-

Clone & similarity detection

-

SIMILAR_TO edges (MinHash + LSH) surface near-duplicate code; SEMANTICALLY_RELATED edges link conceptually similar functions across the graph.

-
-
-

Cross-repo intelligence

-

Index multiple repositories in one store and link them with CROSS_* edges. A multi-galaxy 3D layout and cross-repo architecture summary span the whole fleet.

-
-
-

Data-flow tracing

-

DATA_FLOWS edges follow values from argument to parameter, with field-access chains — trace how data moves, not just who calls whom.

-
-
-

Change-impact analysis

-

detect_changes maps an uncommitted git diff to affected symbols and their blast radius, with risk classification — see what a change touches before you ship it.

-
-
-

Architecture Decision Records

-

manage_adr persists architectural decisions alongside the graph, so design rationale survives across sessions and teammates.

+
+
+

Smart CI Experimental

Turn change impact
into test selection.

Combine static reachability, changed tests and admitted per-test coverage to explain which tests a change reaches. Use the result to focus supported CI runs.

Job selection uses repository-specific path rules. Test narrowing currently supports C/C++ with supported TEST/SUITE-style harnesses or declared equivalents.

Start in shadow mode and validate your harness before enabling gating. This repository narrows its Linux x86_64 PR unit-test leg; release and dry-run workflows retain full execution.

Read the Smart CI setup guide →
+
+
FROM CHANGE TO EXPLAINED SELECTION
+
01
Establish a baseline

Publish the graph and compatible per-test coverage with test-impact publish.

+
02
Trace the change

Combine changed tests, static relationships and admitted coverage.

+
03
Select with reasons

test-impact select explains its selection for a runner that accepts named tests or suites.

+
↳
Broaden when evidence is insufficient

Missing, stale or incomplete evidence broadens the selection or returns run_all. Static reachability alone never omits a suite.

-
-

What are the recent releases?

-

- The latest release notes are loaded live from GitHub. Each entry links to its full changelog. -

-
-

Loading recent releases from GitHub…

+
+

Agent setup

Context that fits
your coding workflow.

The installer configures 45 automatic/conditional client surfaces. MCP connections, instructions, skills, profiles and hooks are installed according to each client's supported capabilities.

+
+
01 / CONNECTSet up your clients

Configure supported integrations for clients such as Claude Code, Codex CLI, Gemini CLI, Cursor, Zed and OpenCode.

+
02 / GUIDEGive agents a working method

Skills and instructions guide graph queries, coverage checks and source verification. Supported custom-agent formats get Scout, Verify and Auditor profiles.

+
03 / MAINTAIN CONTEXTUse supported hooks

Client-specific lifecycle and read/search hooks help surface relevant graph context. Hook and subagent capabilities differ by client and platform.

-

- View all releases on GitHub → -

+
See the supported clients and integration boundaries +

Detected automatically (39): Claude Code, Codex CLI, Gemini CLI, Zed, OpenCode, Antigravity, Aider, KiloCode, VS Code, Cursor, Windsurf, Augment / Auggie, OpenClaw, Kiro, Junie, Hermes, OpenHands, Cline, Warp, Qwen Code, GitHub Copilot CLI, Factory Droid, Crush, Goose, Mistral Vibe, Grok Build, Qoder CLI, Kimi Code CLI, GitLab Duo CLI, Rovo Dev CLI, Amp, Devin CLI / Local, Tabnine, Amazon Q Developer IDE, CodeBuddy Code CLI, IBM Bob Shell, Pochi, Pi, and Oh My Pi (omp).

+

Conditional or explicit (6): Continue / cn, Visual Studio, TRAE, Roo Code, IBM Bob IDE, and Sourcegraph Cody.

+

These are client surfaces, not a promise of identical MCP, hook or subagent support. For example, Pi receives instructions and a skill; Warp's detected skill installation is counted while its MCP connection remains manual. Other manual, UI, cloud and repository-managed integrations are documented separately.

+

Review the client matrix and setup details →

+
-
-

Frequently asked questions

+
+
+

CBM Lens · Open beta v0.0.1

A visual companion
to the MCP server.

+

An optional way to explore the same graph your agent queries. Included in standard native releases, with source navigation, architecture maps and the Galaxy view.

+
+
+
Lens source reader beside the file impact summary and graph dependencies for limits.c
Source navigation with indexed relationships and impact context.
+
+

Read code in context

Browse files and symbols, inspect callers and dependencies, and keep the relevant source beside the graph.

+

See the shape of the system

Explore architecture, system structure and the Galaxy graph. Inspect coverage diagnostics and architecture decisions from the same workspace.

+
+
+
+
CBM Lens Galaxy walkthrough: expand cbm_arena_alloc and follow its call path through arena_grow to cbm_alloc
Follow the connectionsExpand relationships and trace a call path.
+
CBM Lens system structure showing source groups and their dependencies
Inspect system structureSee source groups and dependencies.
+
CBM Lens local chat walkthrough: callers and callees of arena_grow answered directly from the indexed graph
Ask about selected codeCaller and callee answers from the indexed graph.
+
+

Lens is served on loopback. Browser AI downloads its optional model after you enable it. Its explanations should be checked against the displayed source.

+ Read the Lens guide and watch the walkthroughs → +
-

Does codebase-memory-mcp send my code anywhere?

-

No. All indexing and querying happen 100% locally. There is no embedded LLM and no API key. Release - archives are signed and checksummed. Before testing, three behaviourally identical executable candidates - (unstripped, debug-stripped, stripped) for every platform and link mode are submitted to VirusTotal; the selected candidate is packaged with its - SHA-256 unchanged. Release notes link every measured result. Publication permits only the - documented single-Microsoft !ml tolerance.

+
+

Get started

Install. Index. Explore.

Install the MCP server and configure your coding agent. The native runtime needs no Docker or language runtime; indexing and graph queries need no API key. Standard releases also include the optional Lens open beta.

+
+
TERMINAL · macOS / Linux
# Install the native release and agent integrations
+curl -fsSL https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.sh | bash
 
-        
Does it support semantic or natural-language code search?

-        
Yes. Alongside structural and full-text search, search_graph's semantic_query
-           parameter runs vector search over the whole graph, powered by nomic-embed-code embeddings compiled
-           into the binary — so it finds publish when you search send. No API key, no
-           Ollama, no Docker; the embeddings run on-device. The indexer also builds SEMANTICALLY_RELATED
-           edges between similar functions and SIMILAR_TO edges for near-clone detection.

+# Restart your agent, then ask:
+"Index this project"
 
-        
Do I need Docker or a runtime?

-        
The native install needs no language runtime. It is a native executable with authenticated release-owned
-           assets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (arm64/amd64); npm, PyPI, and Go wrapper
-           installations naturally use their package runtime.

+# Optional: open CBM Lens (open beta)
+codebase-memory-mcp daemon start --open
Inspect the installer ↗Windows / PowerShell ↗Other installation methods ↗
+

Your machine. Your index.

Source analysis and graph storage run locally. Native releases contain the executable and authenticated release-owned assets, including the Lens frontend and agent integration templates.

Use native packages or the npm, PyPI and Go wrappers. Package wrappers use their respective runtimes. Nix and source builds also offer builds without the UI.

Choose the clients you want to configure, restart them after installation, then build your first index.

Download the latest release
+
+
-

How does it stay up to date as I edit code?

-

A background watcher detects file changes and re-indexes incrementally — typically a sub-millisecond - no-op when nothing changed. You only run a manual index for the first build or after a large - git pull.

+
+

Performance and research

More useful context.
Less repeated work.

Reuse the graph to find relevant symbols, follow relationships and retrieve focused source snippets. Compact, paged results help keep repeated repository exploration out of your agent’s context window.

Our March 2026 preprint reports 10× lower token use and 2.1× fewer tool calls across 31 repositories compared with file-by-file exploration.

These measurements cover the study’s backend workloads from March 2026. See the paper for the evaluation setup and full results.

Token-use results ↗Indexing benchmarks ↗Research paper ↗
+
-

Why is there no built-in LLM?

-

Other code-graph tools embed an LLM to translate natural language into graph queries, which means - extra API keys and cost. With MCP, the agent you are already talking to is the query translator — - codebase-memory-mcp just builds and serves the graph.

+
+

Common questions

Is Codebase Memory
right for your workflow?

+
When should I use Codebase Memory?

Choose CBM when your coding agent needs a persistent local graph of code, docs and infrastructure. The MCP server brings structural search, source evidence, agent integrations and experimental test selection into one open-source package. The CLI provides direct access, and CBM Lens adds an optional visual workspace in open beta. Native releases run on macOS, Linux and Windows without Docker or a language runtime. Explore the capabilities and Smart CI’s current scope.

+
How does Codebase Memory reduce AI token use?

Your agent can query indexed relationships and retrieve the relevant symbols or source snippets instead of repeatedly exploring whole files. Compact results and pagination control how much context each query returns. The benefit depends on the task; explore the published token-use measurements.

+
Does Codebase Memory send my source code anywhere?

The backend indexes source and stores and queries its graph locally. Your coding agent has its own data-handling settings. Lens can optionally download a browser model after you enable it, with inference on-device. Installers and update checks use the network to obtain releases and assets.

+
How does the index stay current?

Incremental indexing updates changed files. With auto-watch enabled, an eligible active MCP session can watch an already indexed project. Automatic first indexing is a separate opt-in setting. The local UI daemon is not a promise of a permanent background watcher.

+
How much language coverage is available?

Bundled tree-sitter grammars cover a broad range of languages and formats. Supported language families also have Hybrid LSP semantic resolution. Extraction depth, type resolution and framework relationships vary; the language support matrix describes the details.

+
Does Smart CI select tests for every indexed language?

No. Test narrowing currently supports C/C++ family code with supported test-harness conventions or declared equivalents. It requires configuration, admitted coverage and a compatible runner. Missing or incompatible evidence broadens execution or returns run_all. See the Smart CI guide.

+
How do I verify the release?

Use the signed release artifacts and checksums, and review the release's verification information. The security policy explains artifact verification, antivirus reports and vulnerability reporting.

+
Is it free and open source?

Yes. Codebase Memory is MIT licensed. Source, release binaries, detailed documentation and contribution guidelines are available on GitHub.

+
+
-

Is it free and open source?

-

Yes. It is MIT licensed. The full source, signed release binaries, and checksums are on - GitHub.

+
+

Project updates

What's shipping.

Recent releases from GitHub, with the implementation details and fixes behind each version.

+

Loading recent releases from GitHub…

+ + View all releases on GitHub →
- - + - diff --git a/docs/llms.txt b/docs/llms.txt index 0d0c5a49c..83829939f 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -1,26 +1,49 @@ # codebase-memory-mcp -> An open-source Model Context Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. AI coding agents query the graph instead of reading files one by one, answering structural questions with roughly 120x fewer tokens (~3,400 vs ~412,000 across five structural queries). Parses 162 languages via vendored tree-sitter grammars, with Hybrid LSP semantic type resolution for Python, TypeScript/JavaScript, PHP, C#, Go, C/C++, Java, Kotlin, and Rust. It also generates semantic graph edges (SEMANTICALLY_RELATED for vocabulary-mismatch matches, SIMILAR_TO for near-clone detection) and supports semantic vector search via bundled nomic-embed-code embeddings compiled into the native executable — no API key, no Ollama, no Docker. Ships a native executable with an authenticated integration asset; the UI variant adds one content-addressed frontend pack. All processing is local — the embeddings run on-device; there is no embedded LLM and no API key. +> Codebase Memory is an open-source local MCP server for AI coding agents. It connects code, documentation, configuration, and infrastructure in a persistent knowledge graph, with 17 MCP tools and native CLI access. Search for a feature, follow callers and dependencies, trace services and deployments, and inspect change impact. Indexing, graph queries, and semantic embeddings run locally without a hosted service or API key. CBM Lens is the optional visual companion, currently in open beta (v0.0.1); its optional browser model supports on-device code chat after an opt-in download. -## Key facts +## Capabilities + +- Code exploration: symbol and structural search, BM25 text search, semantic search, source snippets and file outlines, call paths, data flow, architecture summaries, hotspots, cycles, and coverage diagnostics. The server registers 17 MCP tools. +- Semantic and similarity relationships: bundled embeddings support natural-language discovery; SEMANTICALLY_RELATED edges connect vocabulary-mismatch matches, and SIMILAR_TO edges expose near-clone candidates. +- Languages and formats: vendored parsers cover source, documentation, and configuration formats. Hybrid LSP adds semantic type resolution for Python, TypeScript/JavaScript/JSX/TSX, PHP, C#, Go, C/C++, Java, Kotlin, Rust, and Perl. Support and resolution depth differ by language; use the README language matrix and check_index_coverage for the indexed project. +- Documentation connected to code: Markdown, reStructuredText, and AsciiDoc sections; PDF text pages (no OCR); supported document and language doc-comment references to code; ADR metadata and supersession. Supported diagrams and OpenAPI descriptions contribute entities, with format-specific reference-resolution limits. +- Infrastructure and deployment: Docker/Compose, Kubernetes, Helm, Kustomize, Terraform/OpenTofu, Terragrunt, Ansible, shell, Make, and supported CI relationships. Trace deployments, images, entrypoints, configuration keys, and environment-variable access back to source when inputs can be resolved statically. +- Cross-service and cross-repository relationships: supported HTTP, asynchronous messaging, gRPC, GraphQL, and tRPC links. Cross-repo analysis explicitly links projects indexed in the same store; missing links do not prove that no runtime relationship exists. +- Change impact: detect_changes maps diffs to affected symbols and investigates related tests with scope: "tests"; compare_graphs compares indexed snapshots. +- Smart CI (experimental): repository-specific changed-file rules select CI jobs; test-impact select combines static reachability, changed tests, and admitted per-test coverage to select tests. Narrowing supports C/C++ family code with supported test-harness conventions and an explicitly configured runner. Missing or doubtful evidence broadens the selection or runs all tests. Other indexed languages are not automatically supported for test narrowing. test-impact publish creates the baseline graph-and-coverage bundle. Start in shadow mode; consult the guide for version requirements. +- Architecture decisions and runtime evidence: manage_adr stores the project's decision record; ingest_traces adds separately collected observed call relationships alongside static analysis. +- CBM Lens open beta (v0.0.1): optional visual companion to the MCP server, with a local browser interface with source exploration, Galaxy graph navigation, architecture and behavior views, ADR editing, index-coverage diagnostics, daemon/system information, and optional local code chat. Static graph animations are not live execution traces. Activity overlays are experimental source-build features, not a standard-release promise. +- Agent integrations: 45 automatic or conditional client surfaces. Installation configures supported MCP entries, durable instructions, skills, Scout/Verify/Auditor profiles, and applicable session, compaction, search, and read hooks. Coverage and activation vary by client and platform. Hooks provide context without blocking or replacing the user's tool call; some clients require trust or use parent-agent handoff instead. +- Persistent operation: a per-account daemon coordinates sessions and indexing. Existing indexes can stay synchronized during eligible active MCP sessions; initial auto-indexing is separately opt-in. Optional team graph artifacts share a baseline while each user keeps a local query database. +- Token-efficient retrieval: targeted graph queries, focused source snippets, and compact, paged results can reduce token use and repeated file reads during code exploration. +- Distribution: native C backend, MIT license, standard native releases for macOS, Linux, and Windows include CBM Lens; Linux portable archives provide static builds. Nix and development source builds can omit the UI. Package-manager options and release verification are documented in the README. + +## Client coverage -- License: MIT, open source. -- Languages: 158 (158 vendored tree-sitter grammars compiled into the binary). -- Hybrid LSP type resolution: 9 language families (Python, TypeScript/JavaScript/JSX/TSX, PHP, C#, Go, C/C++, Java, Kotlin, Rust) — a lightweight C implementation of language type-resolution algorithms, structurally inspired by and compatible with major language servers including tsserver, pyright, gopls, Roslyn, Eclipse JDT, and rust-analyzer. -- MCP tools: 15 (search_graph incl. semantic_query vector search, trace_path (alias: trace_call_path), check_index_coverage, query_graph (Cypher), detect_changes, get_architecture, get_code_snippet, manage_adr, and more). -- Semantic search: natural-language code discovery via bundled nomic-embed-code embeddings (768-dim, compiled into the binary); 11-signal combined scoring; fully local, no API key. -- Semantic & similarity edges: SEMANTICALLY_RELATED (vocabulary-mismatch matches) and SIMILAR_TO (MinHash + LSH near-clone / duplicate detection). -- Cross-repo intelligence: CROSS_* edges link nodes across multiple repos indexed in one store; multi-galaxy 3D layout and cross-repo architecture summary. -- Cross-service linking: HTTP route ↔ call-site matching, plus gRPC/GraphQL/tRPC detection and pub/sub channels (EMITS/LISTENS_ON for Socket.IO, EventEmitter, generic buses). - Supported agents: 45 automatic/conditional client surfaces (39 automatically detected + 6 conditional/explicit): Claude Code, Codex CLI, Gemini CLI, Zed, OpenCode, Antigravity, Aider, KiloCode, VS Code, Cursor, Windsurf, Augment / Auggie, OpenClaw, Kiro, Junie, Hermes, OpenHands, Cline, Warp, Qwen Code, GitHub Copilot CLI, Factory Droid, Crush, Goose, Mistral Vibe, Grok Build, Qoder CLI, Kimi Code CLI, GitLab Duo CLI, Rovo Dev CLI, Amp, Devin CLI / Local, Tabnine, Continue / cn, Visual Studio, TRAE, Roo Code, Amazon Q Developer IDE, CodeBuddy Code CLI, IBM Bob IDE, IBM Bob Shell, Pochi, Pi, Oh My Pi (omp), and Sourcegraph Cody. -- Agent profiles: documented custom-agent formats receive Scout (fast/provisional), Verify (default/task-directed), and Auditor (bounded/full verification) definitions. Every direct tier checks exact path/scope coverage with check_index_coverage and falls back to source for flagged gaps; unsafe child-MCP formats use explicit parent handoff. Kiro and Junie use positive-allowlist Scout/Analysis server profiles (7/11 tools); Qoder combines named-server selection with exact tier-specific MCP tool IDs, while Factory uses exact registered IDs without additive whole-server exposure. Foreign Junie aliases are preserved and force parent handoff. -- Performance: Linux kernel (28M LOC, 75K files) full index in 3 minutes → 4.81M nodes, 7.72M edges; Cypher queries in under 1ms. -- Distribution: verified runtime set (native executable plus authenticated release-owned assets); fully static Linux builds are available in the `-portable` archives. Also available through npm, PyPI, Homebrew, Scoop, AUR, and `go install`. + +## Measurement and limits + +A March 2026 preprint across 31 real-world repositories reports 10× lower token use and 2.1× fewer tool calls compared with file-by-file exploration. These historical backend measurements cover the study’s workloads and agent setup. The paper documents the full evaluation; the README Performance section links to the research and earlier measurements. + +Graph evidence depends on index generation, freshness, and exact path/scope coverage. Partial, skipped, excluded, stale, or unknown coverage requires source verification, especially for absence claims. Infrastructure relationships are static, and test-selection support is narrower than language-indexing support. ## Links - Homepage: https://deusdata.github.io/codebase-memory-mcp/ -- Source code (GitHub): https://github.com/DeusData/codebase-memory-mcp +- Source and feature reference: https://github.com/DeusData/codebase-memory-mcp - Latest release: https://github.com/DeusData/codebase-memory-mcp/releases/latest +- Quick start and installation: https://github.com/DeusData/codebase-memory-mcp#quick-start +- Documentation indexing: https://github.com/DeusData/codebase-memory-mcp#documentation-connected-to-code +- Infrastructure and deployment: https://github.com/DeusData/codebase-memory-mcp#infrastructure-and-deployment +- CBM Lens: https://github.com/DeusData/codebase-memory-mcp#cbm-lens-beta-code-visualization-and-source-navigation +- Agent integrations: https://github.com/DeusData/codebase-memory-mcp#multi-agent-support +- Smart CI guide: https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/SMART_CI.md +- Configuration: https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/CONFIGURATION.md +- MCP tools: https://github.com/DeusData/codebase-memory-mcp#mcp-tools +- Token savings and evidence: https://deusdata.github.io/codebase-memory-mcp/#token-savings +- Performance and research: https://github.com/DeusData/codebase-memory-mcp#performance - Benchmark report: https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/BENCHMARK.md - Research paper: https://arxiv.org/abs/2603.27277 +- Security and release verification: https://github.com/DeusData/codebase-memory-mcp#security diff --git a/docs/sitemap.xml b/docs/sitemap.xml index 086be8523..550a3cabe 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -2,7 +2,6 @@ https://deusdata.github.io/codebase-memory-mcp/ - 2026-06-12 - 1.0 + 2026-10-20 diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 25aaed2cd..ade41fea5 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -734,7 +734,9 @@ typedef struct { * from the URL `name`; a HelmRelease whose chartRef names it deploys the * chart there. * CBM_K8S_FACT_IMAGE: object def_index runs its container `container` - * (NULL: unnamed) from image `name` (a literal reference); via "container". + * (NULL: unnamed) from image `name` (a literal reference, or a Helm + * template's placeholder `<...>`, which names none); via "container"; + * `line` the image's line, start_line / end_line the container's. * CBM_K8S_FACT_ENV: object def_index sets environment variable `name` in * its container `container` (NULL: unnamed) at `line`; via = where the * value comes from ("value", or the valueFrom kind) -- never the value. @@ -773,6 +775,8 @@ typedef struct { const char *url; /* DEPLOY (kustomize): the chart repository */ const char *release; /* DEPLOY (kustomize): the release name */ int line; + int start_line; /* IMAGE: the container mapping's first line */ + int end_line; /* IMAGE: its last line */ } CBMK8sFact; typedef struct { diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index ddad886af..55596732d 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -416,6 +416,33 @@ static TSNode resolve_lean_func_name(TSNode node, TSNode name) { return cbm_find_child_by_kind(node, "ident"); } +// Haskell infix-form definition head (`a <+> b`, ``x `plus` y``): the defined +// name is the operator, not the left operand that is the head's first child. A +// backticked `infix_id` yields its inner variable (`plus`); a symbolic `operator` +// is returned as-is and parenthesised by cbm_func_name_node_text, matching the +// prefix form `(<+>) a b = ...` (#2440). +static TSNode resolve_haskell_infix_head_name(TSNode head) { + enum { INFIX_MIN_CHILDREN = 3, INFIX_OP_IDX = 1 }; + TSNode op = ts_node_child_by_field_name(head, TS_FIELD("operator")); + if (ts_node_is_null(op) && ts_node_child_count(head) >= INFIX_MIN_CHILDREN) { + op = ts_node_child(head, INFIX_OP_IDX); + } + if (!ts_node_is_null(op)) { + const char *ok = ts_node_type(op); + if (strcmp(ok, "operator") == 0) { + return op; + } + if (strcmp(ok, "infix_id") == 0) { + TSNode v = cbm_find_child_by_kind(op, "variable"); + if (!ts_node_is_null(v)) { + return v; + } + } + } + TSNode null_node = {0}; + return null_node; +} + // Haskell: resolve function name from first named child (variable/name). static TSNode resolve_haskell_func_name(TSNode node) { if (ts_node_named_child_count(node) > 0) { @@ -424,6 +451,9 @@ static TSNode resolve_haskell_func_name(TSNode node) { if (strcmp(hk, "variable") == 0 || strcmp(hk, "name") == 0) { return head; } + if (strcmp(hk, "infix") == 0) { + return resolve_haskell_infix_head_name(head); + } if (ts_node_named_child_count(head) > 0) { TSNode v = ts_node_named_child(head, 0); const char *vk = ts_node_type(v); diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index b11436091..c19debd80 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -484,12 +484,32 @@ static void parse_python_imports(CBMExtractCtx *ctx) { ts_tree_cursor_delete(&cursor); } +/* A proven same-module function (its OWN_DEF fact and definition), and + * whether a top-level call of it leaves the namespace intact (see + * py_ns_allowed_call). */ +typedef enum { + PY_NS_CALL_UNCHECKED = 0, + PY_NS_CALL_ALLOWED = 1, + PY_NS_CALL_EFFECT = 2, +} PyNsCallVerdict; + +typedef struct { + int fact; + TSNode node; + PyNsCallVerdict verdict; + const char **shadows; /* submodules the body imports into the package */ + int shadow_count; +} PyNsFunction; + /* Ordered Python namespace facts are a separate content-only pass. The older * import records keep their existing graph/heuristic meaning. */ typedef struct { CBMExtractCtx *ctx; CBMPyNamespaceFacts facts; CBMPyNamespaceFailure failure; + PyNsFunction *functions; /* in fact order; scratch, never part of the result */ + int function_count; + int function_cap; } PyNamespaceBuilder; #ifdef CBM_ENABLE_TEST_SEAMS @@ -516,6 +536,18 @@ static void *py_ns_alloc(PyNamespaceBuilder *b, size_t size) { return p; } +/* Builder bookkeeping nothing in the result points at: the extraction scratch + * when the context has one. */ +static void *py_ns_scratch_alloc(PyNamespaceBuilder *b, size_t size) { + if (b->failure) + return NULL; + CBMArena *arena = b->ctx->scratch ? b->ctx->scratch : b->ctx->arena; + void *p = cbm_arena_alloc(arena, size); + if (!p) + b->failure = CBM_PY_NS_FAILURE_ALLOCATION; + return p; +} + static const char *py_ns_copy(PyNamespaceBuilder *b, const char *text, size_t len) { if (len == SIZE_MAX) { b->failure = CBM_PY_NS_FAILURE_LIMIT; @@ -1024,6 +1056,477 @@ static bool py_ns_future_import(PyNamespaceBuilder *b, TSNode node) { return ok && emitted > 0; } +/* ── A top-level call the namespace survives ────────────────────────────── + * Any top-level call may rebind any name of the module, so it withdraws the + * whole namespace (UNKNOWN, reason EFFECT). sqlalchemy's sql/__init__.py ends + * with `__go(locals())`, which withdrew all of its 108 re-exports. A call is + * allowed when its callee provably cannot rebind the caller's names: + * - the callee is the latest binding of its name, a proven (undecorated) + * same-module function, with no star import or unknown effect since; + * - every argument is a safe expression or a bare `locals()` / `globals()` + * the module does not rebind; + * - the body, nested scopes included, has no `global` / `nonlocal` and never + * names a parameter, a function or class this module defined earlier (it + * could rebind through `global`), the module's own name, or a name that + * reaches a namespace (py_ns_namespace_names). + * The body may import: each submodule of this package it imports becomes a + * package attribute, so in a package `__init__` those names are shadowed at + * the call. Every other call stays a whole-namespace effect. */ + +static const char *const py_ns_namespace_names[] = { + "globals", "vars", "locals", "exec", "eval", "setattr", "delattr", + "__import__", "sys", "importlib", "__name__", "__dict__", "__globals__", "__builtins__", + "builtins", "__self__", "inspect", "f_globals", "f_locals", "f_back", "gc", +}; + +typedef struct { + const char *text; + size_t len; +} PyNsSlice; + +typedef struct { + PyNamespaceBuilder *b; + PyNsSlice own; + bool package; + PyNsSlice *params; + int param_count; + const char **defs; /* earlier same-module functions and classes, sorted */ + int def_count; + const char **shadows; + int shadow_count; + int shadow_cap; +} PyNsCallCheck; + +static bool py_ns_text_is(const char *text, size_t len, const char *s) { + return strlen(s) == len && memcmp(text, s, len) == 0; +} + +/* strcmp of a NUL-terminated string against a slice. */ +static int py_ns_slice_cmp(const char *s, const char *text, size_t len) { + int r = strncmp(s, text, len); + if (r != 0) + return r; + return s[len] ? 1 : 0; +} + +static int py_ns_cmp_cstr(const void *a, const void *b) { + return strcmp(*(const char *const *)a, *(const char *const *)b); +} + +/* `pkg/sub/__init__.py` -> "sub", a package; `pkg/mod.py` -> "mod". */ +static bool py_ns_own_name(const char *rel_path, PyNsSlice *own, bool *package) { + if (!rel_path) + return false; + const char *slash = strrchr(rel_path, '/'); + const char *file = slash ? slash + 1 : rel_path; + const char *ext = strrchr(file, '.'); + if (!ext || ext == file) + return false; + size_t stem = (size_t)(ext - file); + *package = py_ns_text_is(file, stem, "__init__"); + if (!*package) { + *own = (PyNsSlice){file, stem}; + } else { + if (!slash) + return false; + const char *dir = slash; + while (dir > rel_path && dir[-1] != '/') + dir--; + *own = (PyNsSlice){dir, (size_t)(slash - dir)}; + } + return py_ns_identifier_bytes(own->text, own->len, false); +} + +static bool py_ns_reaches_namespace(const char *text, size_t len) { + for (size_t i = 0; i < sizeof(py_ns_namespace_names) / sizeof(py_ns_namespace_names[0]); i++) { + if (py_ns_text_is(text, len, py_ns_namespace_names[i])) + return true; + } + return false; +} + +/* A name of the body (not a module path) the call cannot allow. */ +static bool py_ns_forbidden_name(const PyNsCallCheck *c, const char *text, size_t len) { + if (py_ns_reaches_namespace(text, len) || + (len == c->own.len && memcmp(text, c->own.text, len) == 0)) + return true; + for (int i = 0; i < c->param_count; i++) { + if (len == c->params[i].len && memcmp(text, c->params[i].text, len) == 0) + return true; + } + int lo = 0, hi = c->def_count - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + int r = py_ns_slice_cmp(c->defs[mid], text, len); + if (r == 0) + return true; + if (r < 0) + lo = mid + 1; + else + hi = mid - 1; + } + return false; +} + +static bool py_ns_node_forbidden(const PyNsCallCheck *c, TSNode node) { + const char *text = NULL; + size_t len = 0; + return ts_node_is_null(node) || !py_ns_slice(c->b, node, &text, &len) || + py_ns_forbidden_name(c, text, len); +} + +static bool py_ns_add_shadow(PyNsCallCheck *c, const char *text, size_t len) { + if (!py_ns_identifier_bytes(text, len, false)) + return false; + if (c->shadow_count == c->shadow_cap) { + int cap = c->shadow_cap ? c->shadow_cap * 2 : 8; + const char **items = py_ns_scratch_alloc(c->b, (size_t)cap * sizeof(*items)); + if (!items) + return false; + if (c->shadow_count) + memcpy(items, c->shadows, (size_t)c->shadow_count * sizeof(*items)); + c->shadows = items; + c->shadow_cap = cap; + } + const char *name = py_ns_copy(c->b, text, len); + if (!name) + return false; + c->shadows[c->shadow_count++] = name; + return true; +} + +/* The segments of a dotted module path: none may reach a namespace. Sets + * *own_at to the index of the segment naming this module (-1: none) and + * *next to the slice after it (empty when it is the last segment). */ +static bool py_ns_check_path(const PyNsCallCheck *c, const char *text, size_t len, int *own_at, + PyNsSlice *next) { + *own_at = -1; + *next = (PyNsSlice){NULL, 0}; + int index = 0; + size_t start = 0; + while (start <= len && len > 0) { + size_t end = start; + while (end < len && text[end] != '.') + end++; + const char *seg = text + start; + size_t seg_len = end - start; + if (py_ns_reaches_namespace(seg, seg_len)) + return false; + if (*own_at >= 0 && *own_at == index - 1) + *next = (PyNsSlice){seg, seg_len}; + if (*own_at < 0 && seg_len == c->own.len && memcmp(seg, c->own.text, seg_len) == 0) + *own_at = index; + index++; + start = end + 1; + } + return true; +} + +/* The relative level of a `from` module path: its leading dots. */ +static size_t py_ns_leading_dots(const char *text, size_t len) { + const char *end = text + len; + const char *p = text; + while (p != end && *p == '.') + p++; + return (size_t)(p - text); +} + +/* An import in the callee's body. When its module is this package, the + * submodule it installs as a package attribute is shadowed: the first + * segment after the package, or every imported member when the import names + * the package itself. A relative import of a package `__init__` is relative + * to the package (level 1) or to its parent (level 2, whose first segment + * then names this package or a sibling). Deeper relative imports that name + * this module cannot be placed and are refused. The names an import binds are + * checked like every name of the body; this module's own name is allowed in a + * module path only. */ +static bool py_ns_check_import(PyNsCallCheck *c, TSNode node, bool from) { + TSNode module = from ? ts_node_child_by_field_name(node, TS_FIELD("module_name")) : (TSNode){0}; + bool into_package = false; /* the import's module is this package */ + PyNsSlice sub = {NULL, 0}; /* the submodule after the package, if any */ + if (from) { + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(c->b, module, &text, &len)) + return false; + size_t level = py_ns_leading_dots(text, len); + PyNsSlice rest = {text + level, len - level}; + if (rest.len && !py_ns_identifier_bytes(rest.text, rest.len, true)) + return false; + int own_at = -1; + PyNsSlice next; + if (!py_ns_check_path(c, rest.text, rest.len, &own_at, &next)) + return false; + if (c->package && level == 1) { + /* `from . import x` names the package; `from .x.y import z` installs x */ + const char *dot = rest.len ? memchr(rest.text, '.', rest.len) : NULL; + into_package = rest.len == 0; + sub = (PyNsSlice){rest.text, dot ? (size_t)(dot - rest.text) : rest.len}; + } else if (c->package && (level == 0 || (level == 2 && own_at == 0)) && own_at >= 0) { + /* level 2: the parent's child of this name is this package; a + * later segment of the same name is a package further down */ + if (next.len) + sub = next; + else + into_package = true; + } else if (c->package && level > 2 && own_at >= 0) { + return false; + } + if (sub.len && !py_ns_add_shadow(c, sub.text, sub.len)) + return false; + } + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child) || ts_node_eq(child, module) || + strcmp(ts_node_type(child), "comment") == 0) + continue; + const char *kind = ts_node_type(child); + bool alias = strcmp(kind, "aliased_import") == 0; + if (!alias && strcmp(kind, "dotted_name") != 0) { + ok = false; /* a star import, or a form this check does not read */ + break; + } + TSNode name = alias ? ts_node_child_by_field_name(child, TS_FIELD("name")) : child; + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(c->b, name, &text, &len)) { + ok = false; + break; + } + if (alias && + py_ns_node_forbidden(c, ts_node_child_by_field_name(child, TS_FIELD("alias")))) { + ok = false; + break; + } + if (from) { + /* a member is a name the import binds (or a submodule) */ + if (py_ns_forbidden_name(c, text, len) || + (into_package && !py_ns_add_shadow(c, text, len))) { + ok = false; + break; + } + continue; + } + int own_at = -1; + PyNsSlice next; + if (!py_ns_identifier_bytes(text, len, true) || + !py_ns_check_path(c, text, len, &own_at, &next) || + (alias && own_at >= 0 && !next.len)) { + ok = false; /* `import pkg.own as alias` hands out this module */ + break; + } + if (c->package && own_at >= 0 && next.len && + !py_ns_add_shadow(c, next.text, next.len)) { + ok = false; + break; + } + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return ok; +} + +static bool py_ns_collect_params(PyNsCallCheck *c, TSNode def) { + TSNode params = ts_node_child_by_field_name(def, TS_FIELD("parameters")); + if (ts_node_is_null(params)) + return false; + uint32_t count = ts_node_named_child_count(params); + c->params = count ? py_ns_scratch_alloc(c->b, (size_t)count * sizeof(*c->params)) : NULL; + if (count && !c->params) + return false; + for (uint32_t i = 0; i < count; i++) { + TSNode p = ts_node_named_child(params, i); + const char *kind = ts_node_type(p); + if (strcmp(kind, "comment") == 0 || strcmp(kind, "keyword_separator") == 0 || + strcmp(kind, "positional_separator") == 0) + continue; + TSNode name = p; + if (strcmp(kind, "default_parameter") == 0 || + strcmp(kind, "typed_default_parameter") == 0) { + name = ts_node_child_by_field_name(p, TS_FIELD("name")); + } else if (strcmp(kind, "typed_parameter") == 0 || + strcmp(kind, "list_splat_pattern") == 0 || + strcmp(kind, "dictionary_splat_pattern") == 0) { + name = ts_node_named_child(p, 0); + const char *inner = ts_node_is_null(name) ? "" : ts_node_type(name); + if (strcmp(inner, "list_splat_pattern") == 0 || + strcmp(inner, "dictionary_splat_pattern") == 0) + name = ts_node_named_child(name, 0); + } else if (strcmp(kind, "identifier") != 0) { + return false; + } + PyNsSlice s; + if (ts_node_is_null(name) || strcmp(ts_node_type(name), "identifier") != 0 || + !py_ns_slice(c->b, name, &s.text, &s.len)) + return false; + c->params[c->param_count++] = s; + } + return true; +} + +static bool py_ns_collect_defs(PyNsCallCheck *c, int before) { + c->defs = before ? py_ns_scratch_alloc(c->b, (size_t)before * sizeof(*c->defs)) : NULL; + if (before && !c->defs) + return false; + for (int i = 0; i < before; i++) { + const CBMPyNamespaceFact *f = &c->b->facts.items[i]; + bool def = f->kind == CBM_PY_NS_OWN_DEF || + (f->kind == CBM_PY_NS_SHADOW && (f->reason == CBM_PY_NS_REASON_DECORATED || + f->reason == CBM_PY_NS_REASON_UNPROVEN_CLASS)); + if (def && f->local_name) + c->defs[c->def_count++] = f->local_name; + } + if (c->def_count > 1) + qsort(c->defs, (size_t)c->def_count, sizeof(*c->defs), py_ns_cmp_cstr); + return true; +} + +/* Walk the callee's body (iteratively: nesting must not exhaust the stack). */ +static bool py_ns_check_body(PyNsCallCheck *c, TSNode body) { + TSTreeCursor cursor = ts_tree_cursor_new(body); + bool ok = true; + bool done = false; + while (ok && !done && !c->b->failure) { + TSNode node = ts_tree_cursor_current_node(&cursor); + const char *kind = ts_node_type(node); + bool from = strcmp(kind, "import_from_statement") == 0; + bool is_import = from || strcmp(kind, "import_statement") == 0; + if (strcmp(kind, "global_statement") == 0 || strcmp(kind, "nonlocal_statement") == 0 || + strcmp(kind, "future_import_statement") == 0) { + ok = false; + } else if (is_import) { + ok = py_ns_check_import(c, node, from); + } else if (strcmp(kind, "identifier") == 0) { + ok = !py_ns_node_forbidden(c, node); + } + if (!ok) + break; + if (!is_import && ts_tree_cursor_goto_first_child(&cursor)) + continue; + while (!ts_tree_cursor_goto_next_sibling(&cursor)) { + if (!ts_tree_cursor_goto_parent(&cursor)) { + done = true; + break; + } + } + } + ts_tree_cursor_delete(&cursor); + return ok && !c->b->failure; +} + +static PyNsCallVerdict py_ns_check_function(PyNamespaceBuilder *b, PyNsFunction *fn) { + PyNsCallCheck c = {.b = b}; + TSNode body = ts_node_child_by_field_name(fn->node, TS_FIELD("body")); + if (ts_node_is_null(body) || !py_ns_own_name(b->ctx->rel_path, &c.own, &c.package) || + !py_ns_collect_params(&c, fn->node) || !py_ns_collect_defs(&c, fn->fact) || + !py_ns_check_body(&c, body)) + return PY_NS_CALL_EFFECT; + if (!c.package) + c.shadow_count = 0; /* a plain module's imports install nothing in it */ + fn->shadows = c.shadows; + fn->shadow_count = c.shadow_count; + return PY_NS_CALL_ALLOWED; +} + +static void py_ns_note_function(PyNamespaceBuilder *b, int fact, TSNode node) { + if (b->function_count == b->function_cap) { + int cap = b->function_cap ? b->function_cap * 2 : 16; + PyNsFunction *items = py_ns_scratch_alloc(b, (size_t)cap * sizeof(*items)); + if (!items) + return; + if (b->function_count) + memcpy(items, b->functions, (size_t)b->function_count * sizeof(*items)); + b->functions = items; + b->function_cap = cap; + } + b->functions[b->function_count++] = (PyNsFunction){.fact = fact, .node = node}; +} + +enum { PY_NS_UNBOUND = -1, PY_NS_BARRIER = -2 }; + +/* The index of the latest fact binding `name`: PY_NS_UNBOUND when none did, + * PY_NS_BARRIER when a star import or an unknown effect may have since. */ +static int py_ns_latest_binding(const PyNamespaceBuilder *b, const char *name, size_t len) { + for (int i = b->facts.count - 1; i >= 0; i--) { + const CBMPyNamespaceFact *f = &b->facts.items[i]; + if (f->kind == CBM_PY_NS_UNKNOWN || f->kind == CBM_PY_NS_IMPORT_STAR) + return PY_NS_BARRIER; + if (f->local_name && py_ns_slice_cmp(f->local_name, name, len) == 0) + return i; + } + return PY_NS_UNBOUND; +} + +/* Safe expressions, or a bare `locals()` / `globals()` of the builtins (the + * body never names the parameter that receives it). */ +static bool py_ns_call_arguments_safe(PyNamespaceBuilder *b, TSNode args) { + if (ts_node_is_null(args) || strcmp(ts_node_type(args), "argument_list") != 0) + return false; + uint32_t count = ts_node_named_child_count(args); + for (uint32_t i = 0; i < count; i++) { + TSNode arg = ts_node_named_child(args, i); + if (py_ns_safe_expr(b, arg, 0)) + continue; + if (strcmp(ts_node_type(arg), "call") != 0) + return false; + TSNode fn = ts_node_child_by_field_name(arg, TS_FIELD("function")); + TSNode inner = ts_node_child_by_field_name(arg, TS_FIELD("arguments")); + const char *text = NULL; + size_t len = 0; + if (ts_node_is_null(fn) || strcmp(ts_node_type(fn), "identifier") != 0 || + !py_ns_slice(b, fn, &text, &len) || + !(py_ns_text_is(text, len, "locals") || py_ns_text_is(text, len, "globals")) || + ts_node_is_null(inner) || ts_node_named_child_count(inner) != 0 || + py_ns_latest_binding(b, text, len) != PY_NS_UNBOUND) + return false; + } + return true; +} + +/* True when the top-level call `call` cannot rebind a name of this module's + * namespace; the submodules its callee imports into the package are then + * shadowed here. False when it stays a whole-namespace effect. */ +static bool py_ns_allowed_call(PyNamespaceBuilder *b, TSNode call) { + TSNode callee = ts_node_child_by_field_name(call, TS_FIELD("function")); + const char *name = NULL; + size_t len = 0; + if (ts_node_is_null(callee) || strcmp(ts_node_type(callee), "identifier") != 0 || + !py_ns_slice(b, callee, &name, &len) || + !py_ns_call_arguments_safe(b, ts_node_child_by_field_name(call, TS_FIELD("arguments")))) + return false; + int at = py_ns_latest_binding(b, name, len); + if (at < 0 || b->facts.items[at].kind != CBM_PY_NS_OWN_DEF || + b->facts.items[at].def_kind != CBM_PY_NS_DEF_FUNCTION) + return false; + int lo = 0, hi = b->function_count - 1; + PyNsFunction *fn = NULL; + while (lo <= hi && !fn) { + int mid = lo + (hi - lo) / 2; + if (b->functions[mid].fact == at) + fn = &b->functions[mid]; + else if (b->functions[mid].fact < at) + lo = mid + 1; + else + hi = mid - 1; + } + if (!fn) + return false; + if (fn->verdict == PY_NS_CALL_UNCHECKED) + fn->verdict = py_ns_check_function(b, fn); + if (fn->verdict != PY_NS_CALL_ALLOWED || b->failure) + return false; + for (int i = 0; i < fn->shadow_count; i++) { + if (!py_ns_push(b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, + .local_name = fn->shadows[i], + .reason = CBM_PY_NS_REASON_VALUE})) + return false; + } + return true; +} + static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { const char *kind = ts_node_type(node); if (strcmp(kind, "comment") == 0 || strcmp(kind, "pass_statement") == 0) @@ -1050,16 +1553,20 @@ static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { if (!py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT)) return false; } - return py_ns_push( - b, proven ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_OWN_DEF, - .local_name = name, - .def_kind = function ? CBM_PY_NS_DEF_FUNCTION - : CBM_PY_NS_DEF_CLASS} - : (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, - .local_name = name, - .reason = decorated - ? CBM_PY_NS_REASON_DECORATED - : CBM_PY_NS_REASON_UNPROVEN_CLASS}); + if (!py_ns_push( + b, proven ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_OWN_DEF, + .local_name = name, + .def_kind = function ? CBM_PY_NS_DEF_FUNCTION + : CBM_PY_NS_DEF_CLASS} + : (CBMPyNamespaceFact){ + .kind = CBM_PY_NS_SHADOW, + .local_name = name, + .reason = decorated ? CBM_PY_NS_REASON_DECORATED + : CBM_PY_NS_REASON_UNPROVEN_CLASS})) + return false; + if (proven && function) + py_ns_note_function(b, b->facts.count - 1, def); + return !b->failure; } } if (strcmp(kind, "delete_statement") == 0) @@ -1069,7 +1576,9 @@ static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { const char *ek = ts_node_type(expr); if (strcmp(ek, "assignment") == 0 || strcmp(ek, "augmented_assignment") == 0) return py_ns_assignment(b, expr); - return py_ns_safe_expr(b, expr, 0) || py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT); + if (py_ns_safe_expr(b, expr, 0) || (strcmp(ek, "call") == 0 && py_ns_allowed_call(b, expr))) + return true; + return py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT); } if (strcmp(kind, "if_statement") == 0 && py_ns_is_type_checking_if(b, node)) return py_ns_type_checking_block(b, node); diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index a3572763e..ce41a5690 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -1487,12 +1487,18 @@ static void k8s_container(k8s_walk_t *w, const char *path, TSNode container) { const char *cname = k8s_text(w, container, "name"); cname = cname && cname[0] && !strstr(cname, "{{") ? cname : NULL; const char *image = k8s_text(w, container, "image"); - if (k8s_image_literal(image)) { + /* a Helm template's image is its skeleton placeholder: pass_k8s.c renders it */ + if (k8s_image_literal(image) || (image && strchr(image, '<'))) { + TSNode iv = k8s_child(w, container, "image"); CBMK8sFact f = {.fact = CBM_K8S_FACT_IMAGE, .def_index = w->def_index, .name = image, .container = cname, - .via = "container"}; + .via = "container", + .line = + ts_node_is_null(iv) ? 0 : (int)ts_node_start_point(iv).row + SKIP_ONE, + .start_line = (int)ts_node_start_point(container).row + SKIP_ONE, + .end_line = (int)ts_node_end_point(container).row + SKIP_ONE}; cbm_k8sfact_push(&w->ctx->result->k8s_facts, w->ctx->arena, f); } k8s_iter_t env = k8s_iter(k8s_child(w, container, "env")); diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index 0a770804f..9e310bc6c 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -1213,6 +1213,14 @@ char *cbm_func_name_node_text(CBMArena *a, TSNode name_node, const char *source, if (text && lang == CBM_LANG_NIX) { cbm_nix_strip_attr_quotes(text); } + /* Haskell infix-form operator definition: `a <+> b = …` resolves to the bare + * `operator` node `<+>`, while the prefix form `(<+>) a b = …` is named by its + * `prefix_id` text `(<+>)`. Parenthesise here, not in the resolver, so both + * forms mint the same name and the def name and the call-scope QN, which both + * route through this function, cannot disagree (#2440). */ + if (text && lang == CBM_LANG_HASKELL && strcmp(ts_node_type(name_node), "operator") == 0) { + return cbm_arena_sprintf(a, "(%s)", text); + } return text; } diff --git a/internal/cbm/lsp/ts_lsp.c b/internal/cbm/lsp/ts_lsp.c index 7a7a43edc..8ee5e6642 100644 --- a/internal/cbm/lsp/ts_lsp.c +++ b/internal/cbm/lsp/ts_lsp.c @@ -59,6 +59,10 @@ void cbm_ts_full_registry_builds_reset(void) { * constructs it does not model. -1 = unlimited (no entry point armed it). */ static _Thread_local long g_ts_type_budget = -1; static _Thread_local bool g_ts_type_budget_warned; +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int g_ts_max_member_lookup_steps; +static _Thread_local int g_ts_max_method_lookup_steps; +#endif static void ts_type_budget_reset(size_t source_len) { const char *e = cbm_runtime_getenv("CBM_TS_TYPE_BUDGET"); @@ -170,6 +174,12 @@ long cbm_ts_lsp_test_budget_remaining(void) { bool cbm_ts_lsp_test_budget_warned(void) { return g_ts_type_budget_warned; } +int cbm_ts_lsp_test_max_member_lookup_steps(void) { + return g_ts_max_member_lookup_steps; +} +int cbm_ts_lsp_test_max_method_lookup_steps(void) { + return g_ts_max_method_lookup_steps; +} #endif #define TS_LSP_MAX_EVAL_DEPTH 64 @@ -190,6 +200,10 @@ static const CBMType *type_of_identifier(TSLSPContext *ctx, const char *name); static const CBMType *lookup_member_type(TSLSPContext *ctx, const CBMType *recv, const char *name); static const CBMRegisteredFunc *lookup_method(TSLSPContext *ctx, const CBMType *recv, const char *method_name); +typedef struct TSLookupState TSLookupState; +static const CBMRegisteredFunc *lookup_method_at(TSLSPContext *ctx, const CBMType *recv, + const char *method_name, int depth, + TSLookupState *state); static char *node_text(TSLSPContext *ctx, TSNode node); // Collect a node's children into an arena array via a single O(n) cursor pass. @@ -1569,28 +1583,77 @@ static const char *builtin_wrapper_class(const char *builtin_name) { } // Look up a property `name` on a receiver type. Returns the property type or UNKNOWN. -static const CBMType *lookup_member_type_inner(TSLSPContext *ctx, const CBMType *recv, - const char *name); - #define TS_LSP_MAX_MEMBER_DEPTH 64 +#define TS_LSP_INLINE_VISITED_TYPES 16 -/* Depth-guarded entry: member lookup recurses through wrapper classes, union - * members and registered-type expansion; cyclic type graphs in real-world TS - * (microsoft/TypeScript reallyLargeFile.ts) recursed without bound — SIGBUS - * stack overflow under endless lookup_member_type frames. Past the cap the - * member resolves as unknown — graceful degradation, not a crash. */ -static const CBMType *lookup_member_type(TSLSPContext *ctx, const CBMType *recv, const char *name) { - if (!ctx || ctx->member_depth >= TS_LSP_MAX_MEMBER_DEPTH) - return cbm_type_unknown(); - ctx->member_depth++; - const CBMType *r = lookup_member_type_inner(ctx, recv, name); - ctx->member_depth--; - return r; +typedef struct { + const char *qn; + int depth; +} TSVisitedType; + +struct TSLookupState { + TSVisitedType inline_visited[TS_LSP_INLINE_VISITED_TYPES]; + TSVisitedType *visited; + int visited_count; + int visited_capacity; +#ifdef CBM_ENABLE_TEST_SEAMS + int steps; +#endif +}; + +/* A named type needs another visit only when reached along a shallower path. + * Grow the visited table rather than silently dropping entries at a fixed cap: + * the depth bound then limits each named type to at most 64 admitted visits. */ +static bool ts_lookup_visit_named(CBMArena *arena, TSLookupState *state, const char *qn, + int depth) { + for (int i = 0; i < state->visited_count; i++) { + if (strcmp(state->visited[i].qn, qn) == 0) { + if (state->visited[i].depth <= depth) + return false; + state->visited[i].depth = depth; + return true; + } + } + if (!state->visited) { + state->visited = state->inline_visited; + state->visited_capacity = TS_LSP_INLINE_VISITED_TYPES; + } else if (state->visited_count == state->visited_capacity) { + int capacity = state->visited_capacity * 2; + TSVisitedType *larger = + (TSVisitedType *)cbm_arena_alloc(arena, (size_t)capacity * sizeof(*larger)); + if (!larger) { + fprintf(stderr, "[tslsp] lookup visited allocation failed\n"); + return false; + } + memcpy(larger, state->visited, (size_t)state->visited_count * sizeof(*larger)); + state->visited = larger; + state->visited_capacity = capacity; + } + state->visited[state->visited_count++] = (TSVisitedType){.qn = qn, .depth = depth}; + return true; } -static const CBMType *lookup_member_type_inner(TSLSPContext *ctx, const CBMType *recv, - const char *name) { - if (!ctx || !recv || !name) +static const CBMType *lookup_member_type_at(TSLSPContext *ctx, const CBMType *recv, + const char *name, int depth, TSLookupState *state); + +/* Depth and shallower-only named-type visits bound work on cyclic inheritance. */ +static const CBMType *lookup_member_type(TSLSPContext *ctx, const CBMType *recv, const char *name) { + TSLookupState state = {0}; + const CBMType *result = lookup_member_type_at(ctx, recv, name, 0, &state); +#ifdef CBM_ENABLE_TEST_SEAMS + if (state.steps > g_ts_max_member_lookup_steps) + g_ts_max_member_lookup_steps = state.steps; +#endif + return result; +} + +static const CBMType *lookup_member_type_at(TSLSPContext *ctx, const CBMType *recv, + const char *name, int depth, TSLookupState *state) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (state) + state->steps++; +#endif + if (!ctx || !recv || !name || !state || depth >= TS_LSP_MAX_MEMBER_DEPTH) return cbm_type_unknown(); const CBMType *base = simplify_type(ctx, recv); if (!base) @@ -1604,7 +1667,7 @@ static const CBMType *lookup_member_type_inner(TSLSPContext *ctx, const CBMType const char *wrap = builtin_wrapper_class(base->data.builtin.name); if (wrap) { const CBMType *wrapped = cbm_type_named(ctx->arena, wrap); - return lookup_member_type(ctx, wrapped, name); + return lookup_member_type_at(ctx, wrapped, name, depth + 1, state); } return cbm_type_unknown(); } @@ -1659,7 +1722,10 @@ static const CBMType *lookup_member_type_inner(TSLSPContext *ctx, const CBMType // first hit instead of building a union of results.) if (base->data.union_type.members) { for (int i = 0; i < base->data.union_type.count; i++) { - const CBMType *m = lookup_member_type(ctx, base->data.union_type.members[i], name); + if (depth + 1 >= TS_LSP_MAX_MEMBER_DEPTH) + break; + const CBMType *m = lookup_member_type_at(ctx, base->data.union_type.members[i], + name, depth + 1, state); if (!cbm_type_is_unknown(m)) return m; } @@ -1670,7 +1736,10 @@ static const CBMType *lookup_member_type_inner(TSLSPContext *ctx, const CBMType if (base->kind == CBM_TYPE_INTERSECTION) { if (base->data.union_type.members) { for (int i = 0; i < base->data.union_type.count; i++) { - const CBMType *m = lookup_member_type(ctx, base->data.union_type.members[i], name); + if (depth + 1 >= TS_LSP_MAX_MEMBER_DEPTH) + break; + const CBMType *m = lookup_member_type_at(ctx, base->data.union_type.members[i], + name, depth + 1, state); if (!cbm_type_is_unknown(m)) return m; } @@ -1682,6 +1751,10 @@ static const CBMType *lookup_member_type_inner(TSLSPContext *ctx, const CBMType return cbm_type_unknown(); const char *recv_qn = base->data.named.qualified_name; + if (!recv_qn) + return cbm_type_unknown(); + if (!ts_lookup_visit_named(ctx->arena, state, recv_qn, depth)) + return cbm_type_unknown(); const CBMRegisteredType *rt = cbm_registry_resolve_alias(ctx->registry, recv_qn); if (!rt) { rt = cbm_registry_lookup_type(ctx->registry, recv_qn); @@ -1711,8 +1784,10 @@ static const CBMType *lookup_member_type_inner(TSLSPContext *ctx, const CBMType // Walk extends/implements. if (rt->embedded_types) { for (int i = 0; rt->embedded_types[i]; i++) { + if (depth + 1 >= TS_LSP_MAX_MEMBER_DEPTH) + break; const CBMType *parent = cbm_type_named(ctx->arena, rt->embedded_types[i]); - const CBMType *m = lookup_member_type(ctx, parent, name); + const CBMType *m = lookup_member_type_at(ctx, parent, name, depth + 1, state); if (!cbm_type_is_unknown(m)) return m; } @@ -1795,9 +1870,6 @@ static const CBMType *eval_indexed_access(TSLSPContext *ctx, const CBMType *obj, return cbm_type_unknown(); } -static const CBMRegisteredFunc *lookup_method_inner(TSLSPContext *ctx, const CBMType *recv, - const char *method_name); - /* Depth-guarded entry, same contract as lookup_member_type: the method walk * recurses through wrapper classes, union members and the extends/implements * chain. A cyclic chain (#1743: the Effect idiom `class Service extends @@ -1806,18 +1878,24 @@ static const CBMRegisteredFunc *lookup_method_inner(TSLSPContext *ctx, const CBM * cap the method is simply not found. */ static const CBMRegisteredFunc *lookup_method(TSLSPContext *ctx, const CBMType *recv, const char *method_name) { - if (!ctx || ctx->method_depth >= TS_LSP_MAX_MEMBER_DEPTH) - return NULL; - ctx->method_depth++; - const CBMRegisteredFunc *f = lookup_method_inner(ctx, recv, method_name); - ctx->method_depth--; - return f; + TSLookupState state = {0}; + const CBMRegisteredFunc *result = lookup_method_at(ctx, recv, method_name, 0, &state); +#ifdef CBM_ENABLE_TEST_SEAMS + if (state.steps > g_ts_max_method_lookup_steps) + g_ts_max_method_lookup_steps = state.steps; +#endif + return result; } -// Look up a method on a receiver type — returns the registered func. -static const CBMRegisteredFunc *lookup_method_inner(TSLSPContext *ctx, const CBMType *recv, - const char *method_name) { - if (!ctx || !recv || !method_name) +/* Depth and shallower-only named-type visits bound cyclic inheritance work. */ +static const CBMRegisteredFunc *lookup_method_at(TSLSPContext *ctx, const CBMType *recv, + const char *method_name, int depth, + TSLookupState *state) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (state) + state->steps++; +#endif + if (!ctx || !recv || !method_name || !state || depth >= TS_LSP_MAX_MEMBER_DEPTH) return NULL; const CBMType *base = simplify_type(ctx, recv); if (!base) @@ -1831,7 +1909,7 @@ static const CBMRegisteredFunc *lookup_method_inner(TSLSPContext *ctx, const CBM const char *wrap = builtin_wrapper_class(base->data.builtin.name); if (wrap) { const CBMType *wrapped = cbm_type_named(ctx->arena, wrap); - return lookup_method(ctx, wrapped, method_name); + return lookup_method_at(ctx, wrapped, method_name, depth + 1, state); } return NULL; } @@ -1860,8 +1938,10 @@ static const CBMRegisteredFunc *lookup_method_inner(TSLSPContext *ctx, const CBM if (base->kind == CBM_TYPE_UNION || base->kind == CBM_TYPE_INTERSECTION) { if (base->data.union_type.members) { for (int i = 0; i < base->data.union_type.count; i++) { - const CBMRegisteredFunc *f = - lookup_method(ctx, base->data.union_type.members[i], method_name); + if (depth + 1 >= TS_LSP_MAX_MEMBER_DEPTH) + break; + const CBMRegisteredFunc *f = lookup_method_at(ctx, base->data.union_type.members[i], + method_name, depth + 1, state); if (f) return f; } @@ -1872,6 +1952,11 @@ static const CBMRegisteredFunc *lookup_method_inner(TSLSPContext *ctx, const CBM if (base->kind != CBM_TYPE_NAMED) return NULL; const char *recv_qn = base->data.named.qualified_name; + if (!recv_qn) + return NULL; + + if (!ts_lookup_visit_named(ctx->arena, state, recv_qn, depth)) + return NULL; const CBMRegisteredFunc *f = cbm_registry_lookup_method_aliased(ctx->registry, recv_qn, method_name); @@ -1882,8 +1967,11 @@ static const CBMRegisteredFunc *lookup_method_inner(TSLSPContext *ctx, const CBM const CBMRegisteredType *rt = cbm_registry_lookup_type(ctx->registry, recv_qn); if (rt && rt->embedded_types) { for (int i = 0; rt->embedded_types[i]; i++) { + if (depth + 1 >= TS_LSP_MAX_MEMBER_DEPTH) + break; const CBMType *parent = cbm_type_named(ctx->arena, rt->embedded_types[i]); - const CBMRegisteredFunc *pf = lookup_method(ctx, parent, method_name); + const CBMRegisteredFunc *pf = + lookup_method_at(ctx, parent, method_name, depth + 1, state); if (pf) return pf; } @@ -3757,6 +3845,30 @@ static void process_function_body(TSLSPContext *ctx, TSNode func_node, const cha ctx->current_scope = saved_scope; } +static void process_top_level_variable_arrows(TSLSPContext *ctx, TSNode declaration) { + if (!ctx || ts_node_is_null(declaration) || !ctx->module_qn) + return; + + uint32_t count = ts_node_named_child_count(declaration); + for (uint32_t i = 0; i < count; i++) { + TSNode declarator = ts_node_named_child(declaration, i); + if (!node_kind_is(declarator, "variable_declarator")) + continue; + + TSNode name = ts_node_child_by_field_name(declarator, "name", TS_LSP_FIELD_LEN("name")); + TSNode value = ts_node_child_by_field_name(declarator, "value", TS_LSP_FIELD_LEN("value")); + if (!node_kind_is(name, "identifier") || !node_kind_is(value, "arrow_function")) + continue; + + char *function_name = node_text(ctx, name); + if (!function_name) + continue; + const char *function_qn = + cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->module_qn, function_name); + process_function_body(ctx, value, function_qn, NULL); + } +} + void ts_lsp_process_file(TSLSPContext *ctx, TSNode root) { if (!ctx || ts_node_is_null(root)) return; @@ -3782,7 +3894,9 @@ void ts_lsp_process_file(TSLSPContext *ctx, TSNode root) { TSNode child = kids[i]; const char *kind = ts_node_type(child); - if (strcmp(kind, "function_declaration") == 0) { + if (strcmp(kind, "lexical_declaration") == 0 || strcmp(kind, "variable_declaration") == 0) { + process_top_level_variable_arrows(ctx, child); + } else if (strcmp(kind, "function_declaration") == 0) { TSNode name = ts_node_child_by_field_name(child, "name", TS_LSP_FIELD_LEN("name")); if (ts_node_is_null(name) || !ctx->module_qn) continue; @@ -3821,6 +3935,9 @@ void ts_lsp_process_file(TSLSPContext *ctx, TSNode root) { continue; const char *fqn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->module_qn, fn); process_function_body(ctx, inner, fqn, NULL); + } else if (strcmp(ik, "lexical_declaration") == 0 || + strcmp(ik, "variable_declaration") == 0) { + process_top_level_variable_arrows(ctx, inner); } else if (strcmp(ik, "class_declaration") == 0 || strcmp(ik, "internal_module") == 0) { process_node(ctx, inner); @@ -3838,6 +3955,10 @@ void ts_lsp_init(TSLSPContext *ctx, CBMArena *arena, const char *source, int sou bool jsx_mode, bool dts_mode, CBMResolvedCallArray *out) { if (!ctx) return; +#ifdef CBM_ENABLE_TEST_SEAMS + g_ts_max_member_lookup_steps = 0; + g_ts_max_method_lookup_steps = 0; +#endif memset(ctx, 0, sizeof(TSLSPContext)); ctx->arena = arena; ctx->source = source; diff --git a/internal/cbm/lsp/ts_lsp.h b/internal/cbm/lsp/ts_lsp.h index 5fc368f39..1096bea90 100644 --- a/internal/cbm/lsp/ts_lsp.h +++ b/internal/cbm/lsp/ts_lsp.h @@ -65,13 +65,6 @@ typedef struct { // first completed eval; see TsEvalMemo in ts_lsp.c). Kills the exponential // re-evaluation of shared subexpressions under overload resolution. struct TsEvalMemo *eval_memo; - // Recursion guard for lookup_member_type: cyclic type graphs (mutually - // recursive unions/wrappers across registered types) otherwise recurse - // without bound — stack overflow on real repos. - int member_depth; - // Same guard for lookup_method: its extends/implements walk recursed - // without bound on a cyclic heritage chain (#1743). - int method_depth; } TSLSPContext; #ifdef CBM_ENABLE_TEST_SEAMS @@ -79,6 +72,10 @@ typedef struct { // for the calling thread, valid after a cbm_run_ts_lsp on the same thread. long cbm_ts_lsp_test_budget_remaining(void); bool cbm_ts_lsp_test_budget_warned(void); +// Maximum recursive calls in one member/method lookup during the last TS LSP run, +// including attempts rejected by the depth or visited-type guard. +int cbm_ts_lsp_test_max_member_lookup_steps(void); +int cbm_ts_lsp_test_max_method_lookup_steps(void); #endif // --- Initialization --- diff --git a/pkg/npm/README.md b/pkg/npm/README.md index 63dc88f0b..bce4d5217 100644 --- a/pkg/npm/README.md +++ b/pkg/npm/README.md @@ -5,9 +5,9 @@ [![License](https://img.shields.io/badge/license-MIT-green)](https://github.com/DeusData/codebase-memory-mcp/blob/main/LICENSE) [![Platform](https://img.shields.io/badge/macOS_%7C_Linux_%7C_Windows-supported-lightgrey)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) -**The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. This npm wrapper downloads, verifies, and caches the selected native runtime set: the executable, its authenticated integration asset, and—when requested—the content-addressed UI pack. +**Local code intelligence for developers and AI agents.** CBM connects source code, documentation and infrastructure in a persistent knowledge graph. Explore through MCP, the CLI or CBM Lens to find code, trace dependencies and deployments, and understand what a change affects. -High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 162 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 17 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 45 automatic/conditional client surfaces. +This npm wrapper downloads, verifies and caches the native executable, its authenticated integration asset and the content-addressed Lens frontend pack. Node.js manages installation and launch; the graph is processed locally without a hosted service or API key. The installer offers 45 client integration targets, with capabilities that vary by client. ## Installation @@ -25,14 +25,16 @@ Then configure your coding agents: codebase-memory-mcp install ``` -Restart your agent. Say **"Index this project"** — done. +Restart your agent and ask it to **"Index this project"**. Use `codebase-memory-mcp daemon start --open` to open CBM Lens. Supported integrations also install graph instructions, skills, agent profiles and context hooks; consult the full client matrix for differences. ## Why codebase-memory-mcp -- **Extreme indexing speed** — Linux kernel (28M LOC, 75K files) in 3 minutes. RAM-first pipeline with LZ4 compression and in-memory SQLite. +- **Connected repository knowledge** — source symbols and calls, documentation sections and references, configuration, infrastructure and cross-service links. - **Plug and play** — verified native runtime sets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (arm64/amd64). No Docker or API keys; Node.js owns package download, cache repair, and launch. -- **162 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks. -- **120x fewer tokens** — 5 structural queries: ~3,400 tokens vs ~412,000 via file-by-file search. +- **Broad language and format support** — bundled parsers, with deeper semantic resolution for supported language families. See the [language reference](https://github.com/DeusData/codebase-memory-mcp#language-support) for coverage. +- **Focused context** — retrieve symbols, relationships and bounded source excerpts instead of repeatedly reading whole files. Savings depend on the task and workflow. +- **CBM Lens** — explore source, architecture, call paths, decisions and index coverage in a local browser workspace. +- **Smart CI** — experimental change-aware test selection for supported C/C++ test runners, using graph relationships and admitted per-test coverage. Missing evidence broadens execution. See the [Smart CI guide](https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/SMART_CI.md). - **45 supported automatic/conditional client surfaces** — `install` configures the appropriate MCP, durable-context, and documented hook surfaces without widening client permissions. - **Detected automatically (39)** — Claude Code, Codex CLI, Gemini CLI, Zed, OpenCode, Antigravity, Aider, KiloCode, VS Code, Cursor, Windsurf, Augment / Auggie, OpenClaw, Kiro, Junie, Hermes, OpenHands, Cline, Warp, Qwen Code, GitHub Copilot CLI, Factory Droid, Crush, Goose, Mistral Vibe, Grok Build, Qoder CLI, Kimi Code CLI, GitLab Duo CLI, Rovo Dev CLI, Amp, Devin CLI / Local, Tabnine, Amazon Q Developer IDE, CodeBuddy Code CLI, IBM Bob Shell, Pochi, Pi, and Oh My Pi (omp). - **Conditional or explicit (6)** — Continue / cn, Visual Studio, TRAE, Roo Code, IBM Bob IDE, and Sourcegraph Cody. Bob IDE is touched only when `~/.bob/mcp.json` already exists. @@ -68,30 +70,25 @@ Every MCP tool is also available directly from the command line: ```bash codebase-memory-mcp cli index_repository '{"repo_path": "/path/to/repo"}' -codebase-memory-mcp cli search_graph '{"name_pattern": ".*Handler.*", "label": "Function"}' -codebase-memory-mcp cli trace_call_path '{"function_name": "main", "direction": "both"}' -codebase-memory-mcp cli get_architecture '{}' +codebase-memory-mcp cli list_projects '{}' +# Replace PROJECT with the indexed project name returned above. +codebase-memory-mcp cli search_graph '{"project":"PROJECT","name_pattern":".*Handler.*","label":"Function"}' +codebase-memory-mcp cli trace_path '{"project":"PROJECT","function_name":"main","direction":"both"}' +codebase-memory-mcp cli get_architecture '{"project":"PROJECT"}' ``` ## MCP Tools | Category | Tools | |----------|-------| -| **Indexing** | `index_repository`, `list_projects`, `delete_project`, `index_status` | -| **Querying** | `search_graph`, `trace_call_path`, `detect_changes`, `query_graph` | -| **Analysis** | `get_architecture`, `get_graph_schema`, `get_code_snippet`, `search_code` | -| **Advanced** | `manage_adr`, `ingest_traces` | +| **Indexing and coverage** | `index_repository`, `list_projects`, `delete_project`, `index_status`, `check_index_coverage` | +| **Search and source** | `search_graph`, `search_code`, `get_code_snippet`, `get_file_outline` | +| **Relationships and changes** | `trace_path`, `query_graph`, `detect_changes`, `compare_graphs` | +| **Architecture and evidence** | `get_architecture`, `get_graph_schema`, `manage_adr`, `ingest_traces` | ## Performance -Benchmarked on Apple M3 Pro: - -| Operation | Time | -|-----------|------| -| Linux kernel full index (28M LOC, 75K files) | 3 min | -| Django full index | ~6s | -| Cypher query | <1ms | -| Trace call path (depth=5) | <10ms | +Performance and context savings depend on the repository, query, machine and agent workflow. See the [scoped benchmark results](https://github.com/DeusData/codebase-memory-mcp#performance) and [methodology](https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/BENCHMARK.md) for recorded measurements and limitations. ## Full Documentation diff --git a/scripts/build.sh b/scripts/build.sh index a71029b93..ad646c5de 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -32,6 +32,15 @@ Options: --with-ui Build the web UI as a content-addressed sidecar (needs node). --version V Stamp the version string (release venue passes the tag). --arch ARCH Force target arch (arm64 | x86_64), e.g. under Rosetta. + --vendored-only Build only the third-party objects (make cbm-third-party) + into a clean BUILD_DIR, no binary. + --reuse-vendored Do NOT clean BUILD_DIR: build the binary on top of a + --vendored-only run with the same arguments, compiling + first-party code only. The pair exists for CodeQL's light + PR scan (codeql.yml): the first runs before tracing starts. + --retrace-from F With --reuse-vendored: F lists vendored paths (one per + line) whose objects must be rebuilt anyway (make -W), so a + vendored file the PR changes is compiled under tracing too. -h, --help This text. Make passthrough (VAR=VAL, forwarded verbatim): @@ -79,6 +88,8 @@ source "$ROOT/scripts/path-safety.sh" # containerized legs can build in their own directory instead of deleting and # clobbering the host's native build/c artifacts. WITH_UI=false +PHASE=all +RETRACE_FROM="" VERSION="" BUILD_DIR="build/c" EXTRA_MAKE_ARGS=() @@ -94,7 +105,13 @@ for arg in "$@"; do --with-ui) WITH_UI=true ;; - --version) + --vendored-only) + PHASE=vendored + ;; + --reuse-vendored) + PHASE=reuse + ;; + --version|--retrace-from) prev_arg="$arg" continue ;; @@ -122,6 +139,8 @@ for arg in "$@"; do # a usage error, not a silent make argument. if [[ "${prev_arg:-}" == "--version" ]]; then VERSION="$arg" + elif [[ "${prev_arg:-}" == "--retrace-from" ]]; then + RETRACE_FROM="$arg" else echo "build.sh: unexpected argument '$arg'. Please consult --help." >&2 exit 2 @@ -144,15 +163,47 @@ echo " ui=$WITH_UI version=${VERSION:-dev}" # Verify compiler supports target arch verify_compiler "$CC" -# Step 1: Clean C build artifacts only (not node_modules — npm ci handles that) -cbm_remove_build_dir "$ROOT" "$BUILD_DIR" +if [[ "$PHASE" != all ]] && $WITH_UI; then + echo "build.sh: --vendored-only / --reuse-vendored do not combine with --with-ui. Please consult --help." >&2 + exit 2 +fi + +# --retrace-from: each listed path becomes `make -W ` (treated as changed +# now -- the retrace helper supplies direct compilation entrypoints, so nested +# includes do not depend on Makefile wildcard coverage; no timestamp race). +RETRACE_ARGS=() +if [[ -n "$RETRACE_FROM" ]]; then + if [[ "$PHASE" != reuse || ! -f "$RETRACE_FROM" ]]; then + echo "build.sh: --retrace-from needs --reuse-vendored and an existing list file. Please consult --help." >&2 + exit 2 + fi + while IFS= read -r retrace_path || [[ -n "$retrace_path" ]]; do + [[ -n "$retrace_path" ]] && RETRACE_ARGS+=(-W "$retrace_path") + done < "$RETRACE_FROM" +fi + +# Step 1: Clean C build artifacts only (not node_modules — npm ci handles that). +# --reuse-vendored keeps the third-party objects of the --vendored-only run. +if [[ "$PHASE" == reuse ]]; then + if [[ ! -d "$BUILD_DIR" ]]; then + echo "build.sh: --reuse-vendored needs a --vendored-only run in $BUILD_DIR first." >&2 + exit 2 + fi +else + cbm_remove_build_dir "$ROOT" "$BUILD_DIR" +fi # Step 2: Build (Makefile applies $ARCHFLAGS for the target arch on macOS) -if $WITH_UI; then +if [[ "$PHASE" == vendored ]]; then + make -j"$NPROC" -f Makefile.cbm cbm-third-party \ + CFLAGS_EXTRA="$CFLAGS_EXTRA" "${EXTRA_MAKE_ARGS[@]+"${EXTRA_MAKE_ARGS[@]}"}" + echo "=== Third-party objects built in ${BUILD_DIR} (no binary) ===" + exit 0 +elif $WITH_UI; then make -j"$NPROC" -f Makefile.cbm cbm-with-ui \ CFLAGS_EXTRA="$CFLAGS_EXTRA" "${EXTRA_MAKE_ARGS[@]+"${EXTRA_MAKE_ARGS[@]}"}" else - make -j"$NPROC" -f Makefile.cbm cbm \ + make -j"$NPROC" -f Makefile.cbm cbm "${RETRACE_ARGS[@]+"${RETRACE_ARGS[@]}"}" \ CFLAGS_EXTRA="$CFLAGS_EXTRA" "${EXTRA_MAKE_ARGS[@]+"${EXTRA_MAKE_ARGS[@]}"}" fi diff --git a/scripts/ci/codeql-retrace-vendored.sh b/scripts/ci/codeql-retrace-vendored.sh new file mode 100755 index 000000000..51e33402e --- /dev/null +++ b/scripts/ci/codeql-retrace-vendored.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# codeql-retrace-vendored.sh — prebuilt C/C++ translation units to retrace, +# for the PR c-cpp CodeQL scan (.github/workflows/codeql.yml). +# +# A PR's c-cpp scan compiles the third-party objects BEFORE CodeQL starts +# tracing (scripts/build.sh --vendored-only), so vendored C stays out of the +# analysis, while ALL first-party C is traced -- except the vendored files the +# PR itself changes, including grammar/runtime wrappers. This lists their +# compilation entrypoints; scripts/build.sh --reuse-vendored --retrace-from +# then rebuilds their objects under tracing. Use direct prerequisites +# for make -W: its shallow wildcard dependencies miss nested headers/sources, +# and compiler include paths can contain aliases such as ../unix/prim.c. +# +# The changed files come from the PR merge commit (HEAD^1 is the base it was +# merged onto, as in pr.yml's file-list fallback; needs fetch-depth 2) unless +# --changed FILE gives them. An undeterminable diff (no pull request, no +# parent, git failing, an unreadable FILE) is NOT an error and never narrows +# the scan: the list is empty and the scan still covers all first-party C. +# +# Usage: scripts/ci/codeql-retrace-vendored.sh [--changed FILE] OUT +# OUT is written with the entrypoints to retrace, one per line (may be +# empty). The scope test is scripts/ci/select-lanes.sh --codeql-scope c-cpp. +# Exit: 0 = list written · 2 = usage error. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +CHANGED_FILE="" +OUT="" +while [ $# -gt 0 ]; do + case "$1" in + -h | --help) + sed -n '2,22p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + --changed) + CHANGED_FILE="${2:-}" + [ -n "$CHANGED_FILE" ] || { + echo "codeql-retrace-vendored.sh: --changed needs a file. Please consult --help." >&2 + exit 2 + } + shift + ;; + -*) + echo "codeql-retrace-vendored.sh: unknown option '$1'. Please consult --help." >&2 + exit 2 + ;; + *) + if [ -n "$OUT" ]; then + echo "codeql-retrace-vendored.sh: more than one output file. Please consult --help." >&2 + exit 2 + fi + OUT="$1" + ;; + esac + shift +done +if [ -z "$OUT" ]; then + echo "codeql-retrace-vendored.sh: no output file. Please consult --help." >&2 + exit 2 +fi + +changed="" +determined=false +if [ -n "$CHANGED_FILE" ]; then + if [ -r "$CHANGED_FILE" ]; then + changed=$(cat "$CHANGED_FILE") + determined=true + fi +elif [ "${GITHUB_EVENT_NAME:-}" = pull_request ]; then + if changed=$(git -C "$ROOT" diff --name-only --no-renames HEAD^1 HEAD 2>/dev/null); then + determined=true + fi +fi + +: >"$OUT" +if ! $determined; then + echo "diff undetermined: no vendored file to retrace; the scan covers all first-party C" + exit 0 +fi +printf '%s\n' "$changed" | + bash "$ROOT/scripts/ci/select-lanes.sh" --codeql-scope c-cpp | + while IFS= read -r path; do + entrypoints=() + case "$path" in + internal/cbm/grammar_*.c | internal/cbm/ts_runtime.c) + entrypoints=("$path") + ;; + vendored/mimalloc/*) entrypoints=(vendored/mimalloc/src/static.c) ;; + vendored/sqlite3/*) entrypoints=(vendored/sqlite3/sqlite3.c) ;; + vendored/tre/*) entrypoints=(vendored/tre/tre_all.c) ;; + internal/cbm/vendored/lz4/*) + entrypoints=(internal/cbm/vendored/lz4/lz4.c internal/cbm/vendored/lz4/lz4hc.c) + ;; + internal/cbm/vendored/zstd/*) entrypoints=(internal/cbm/vendored/zstd/zstd.c) ;; + internal/cbm/vendored/ts_runtime/*) entrypoints=(internal/cbm/ts_runtime.c) ;; + internal/cbm/vendored/grammars/*) + family="${path#internal/cbm/vendored/grammars/}" + wrapper="internal/cbm/grammar_${family%%/*}.c" + if [ -f "$ROOT/$wrapper" ]; then + entrypoints=("$wrapper") + else + # Shared or unfamiliar grammar code can have many consumers. + entrypoints=("$ROOT"/internal/cbm/grammar_*.c) + fi + ;; + internal/cbm/vendored/common/*) + entrypoints=("$ROOT"/internal/cbm/grammar_*.c) + ;; + internal/cbm/vendored/* | vendored/*) + # Other vendored code is compiled with the first-party build. + entrypoints=("$path") + ;; + esac + for entrypoint in "${entrypoints[@]+"${entrypoints[@]}"}"; do + entrypoint="${entrypoint#"$ROOT"/}" + if [ -f "$ROOT/$entrypoint" ]; then printf '%s\n' "$entrypoint"; fi + done + done | LC_ALL=C sort -u >"$OUT" +count=$(wc -l <"$OUT" | tr -d ' ') +echo "prebuilt translation units to retrace: $count" +cat "$OUT" diff --git a/scripts/ci/require-all-green.sh b/scripts/ci/require-all-green.sh index 027118287..149eef2da 100755 --- a/scripts/ci/require-all-green.sh +++ b/scripts/ci/require-all-green.sh @@ -66,12 +66,13 @@ if 'LANES' in os.environ: single = { 'lint / lint': 'lint', 'lint / lint-mem': 'lint-mem', 'security / security-static': 'security-static', - 'security / license-gate': 'license-gate', 'security / codeql-gate': 'codeql-gate', + 'security / license-gate': 'license-gate', 'test / test-diag': 'diag', 'test / test-msan': 'msan', 'test / test-lsan-macos': 'lsan-macos', 'test / test-windows-guards': 'windows-guards', 'test / shard-completeness': 'shard-completeness', 'contracts': 'contracts', } - matrix = ('test / test-unix (', 'test / test-windows (', 'test / test-tsan (', 'pr-smoke (') + matrix = ('test / test-unix (', 'test / test-windows (', 'test / test-tsan (', 'pr-smoke (', + 'codeql / analyze (') def lane_of(name): if name in single: return single[name] diff --git a/scripts/ci/select-lanes.sh b/scripts/ci/select-lanes.sh index aaabdb107..18f72c578 100755 --- a/scripts/ci/select-lanes.sh +++ b/scripts/ci/select-lanes.sh @@ -30,13 +30,19 @@ # replay's single process). # scripts/ci/select-lanes.sh --list-lanes # Every lane name, one per line (the universe ci-ok checks against). +# scripts/ci/select-lanes.sh --codeql-scope LANGUAGE [FILE] +# The paths of FILE (default stdin) in that CodeQL language's PR +# scope (c-cpp, javascript-typescript, python, actions), one per line. +# The codeql-* lanes are selected by these scopes; the interpreted +# ones follow .github/codeql/codeql-config.yml (paths/paths-ignore). # Exit: 0 = decided · 2 = usage error. set -euo pipefail -usage() { sed -n '2,33p' "$0" | sed 's/^# \{0,1\}//'; } +usage() { sed -n '2,38p' "$0" | sed 's/^# \{0,1\}//'; } FORMAT=json MODE="select" +CODEQL_LANGUAGE="" INPUT="" EXPECT="" while [ $# -gt 0 ]; do @@ -56,6 +62,11 @@ while [ $# -gt 0 ]; do [ $# -gt 1 ] && shift ;; --list-lanes) MODE="list" ;; + --codeql-scope) + MODE="codeql-scope" + CODEQL_LANGUAGE="${2:-}" + [ $# -gt 1 ] && shift + ;; --expect-files) # An empty value (a workflow whose count went missing) must fail # loudly, never quietly skip the truncation check. @@ -101,20 +112,29 @@ fi # The heredoc below is the program, so stdin input is staged in a file first. STAGED="" -if [ "$MODE" = select ] && [ -z "$INPUT" ]; then +case "$MODE:$CODEQL_LANGUAGE" in +codeql-scope:c-cpp | codeql-scope:javascript-typescript | codeql-scope:python | codeql-scope:actions | select:* | batch:* | list:*) ;; +*) + echo "select-lanes.sh: --codeql-scope takes c-cpp, javascript-typescript, python or actions. Please consult --help." >&2 + exit 2 + ;; +esac +if { [ "$MODE" = select ] || [ "$MODE" = codeql-scope ]; } && [ -z "$INPUT" ]; then STAGED=$(mktemp "${TMPDIR:-/tmp}/cbm-select-lanes.XXXXXX") trap 'rm -f "$STAGED"' EXIT cat >"$STAGED" INPUT="$STAGED" fi -python3 - "$MODE" "$FORMAT" "$INPUT" "$EXPECT" <<'PY' +CODEQL_CONFIG="$(cd "$(dirname "$0")/../.." && pwd)/.github/codeql/codeql-config.yml" +python3 - "$MODE" "$FORMAT" "$INPUT" "$EXPECT" "$CODEQL_CONFIG" "$CODEQL_LANGUAGE" <<'PY' import json import re import sys mode, fmt, source = sys.argv[1:4] expect = int(sys.argv[4]) if sys.argv[4] else None +codeql_config, codeql_language = sys.argv[5:7] # ── Path -> category (first matching rule wins; order is part of the rule) ── GATE_WORKFLOWS = {"pr.yml", "_test.yml", "_lint.yml", "_security.yml", @@ -192,6 +212,8 @@ def category(path): return "grammar" if path.startswith(".github/workflows/"): return "ci-gate" if base in GATE_WORKFLOWS else "ci-other" + if path.startswith(".github/codeql/"): + return "ci-gate" if path.startswith(".github/"): if path.endswith(".md") or ("ISSUE_TEMPLATE" in path and path.endswith((".yml", ".yaml"))): return "docs" @@ -293,11 +315,16 @@ def tier_of(cats): # ── Tier -> lanes (R1) ── -FULL = {"codeql-gate", "diag", "license-gate", "lint", "lint-mem", "lsan-macos", +# The PR CodeQL scans, one lane per language (pr.yml's `codeql` matrix). Each +# is selected by a change in its scope (codeql_scope below) and, like every +# lane, by any full run: an undeterminable diff scans every language. +CODEQL_LANES = {"c-cpp": "codeql-cpp", "javascript-typescript": "codeql-js", + "python": "codeql-py", "actions": "codeql-actions"} +FULL = {"diag", "license-gate", "lint", "lint-mem", "lsan-macos", "memwaste", "msan", "pkg-wrappers", "security-static", "shard-completeness", "smoke-mac", "smoke-ubuntu", "smoke-win", "tsan-arm", "tsan-mac", "tsan-x86", "unix-arm64", "unix-macos-intel", "unix-macos14", "unix-x86", "windows", - "windows-guards"} + "windows-guards"} | set(CODEQL_LANES.values()) UNIVERSE = FULL | {"contracts"} # contracts: the T0a/T0b job; every test leg runs them too LINT = {"lint", "lint-mem"} CORE_LEGS = {"unix-x86", "unix-macos14", "windows", "shard-completeness"} @@ -308,9 +335,9 @@ R1 = { # The contract steps police test-infrastructure/ and lint config too. "T0b-nonproduct": LINT | {"contracts"}, "T1-tests": LINT | CORE_LEGS, - "T2-extract": LINT | CORE_LEGS | {"codeql-gate", "tsan-x86", "smoke-ubuntu"}, - "T3-mid": LINT | CORE_LEGS | {"codeql-gate", "tsan-x86", "smoke-ubuntu"}, - "T4-platform": FULL - {"msan", "diag", "memwaste"}, + "T2-extract": LINT | CORE_LEGS | {"tsan-x86", "smoke-ubuntu"}, + "T3-mid": LINT | CORE_LEGS | {"tsan-x86", "smoke-ubuntu"}, + "T4-platform": FULL - {"msan", "diag", "memwaste"} - set(CODEQL_LANES.values()), "T5-core": FULL, } # Category add-ons: (categories, lanes). pkg adds nothing to a docs-only set @@ -328,6 +355,73 @@ HARNESS = {"scripts/test.sh", "scripts/run-tests-parallel.sh", "scripts/run-test "tests/test_main.c", "tests/test_framework.h"} +# ── CodeQL PR scope (the codeql-* lanes) ── +# c-cpp: any C/C++ the production build can compile -- first-party AND +# vendored (a vendored-only change is scanned: codeql.yml traces the vendored +# files a PR changes). Tests are not in the analysed build. +# Interpreted languages: their extensions inside the CodeQL config's +# paths minus paths-ignore (.github/codeql/codeql-config.yml, read below). +# A change to the CodeQL machinery itself scans every language (fail closed); +# those paths are full runs anyway, this keeps that true if a tier moves. +CODEQL_C_EXT = (".c", ".h", ".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx", ".inc") +CODEQL_C_ROOTS = ("src/", "internal/", "vendored/") +CODEQL_EXT = {"javascript-typescript": (".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx", + ".mts", ".cts", ".vue", ".html", ".htm"), + "python": (".py",)} +CODEQL_FAIL_CLOSED_PREFIXES = (".github/workflows/", ".github/codeql/") +CODEQL_FAIL_CLOSED = {"Makefile.cbm", "scripts/build.sh", "scripts/ci/select-lanes.sh", + "scripts/ci/codeql-retrace-vendored.sh"} + + +def config_lists(path): + """paths / paths-ignore of the CodeQL config: `key:` then ` - item` lines. + None when the file is unreadable -- every interpreted scope then matches + by extension alone (fail closed).""" + try: + lines = open(path, encoding="utf-8").read().splitlines() + except OSError: + return None + lists, key = {"paths": [], "paths-ignore": []}, None + for line in lines: + if not line.strip() or line.lstrip().startswith("#"): + continue + m = re.match(r"^([A-Za-z-]+):\s*$", line) + if m: + key = m.group(1) if m.group(1) in lists else None + continue + m = re.match(r"^\s+-\s+['\"]?([^'\"#]+?)['\"]?\s*$", line) + if m and key: + lists[key].append(m.group(1).rstrip("/")) + elif not line.startswith((" ", "\t")): + key = None + return lists + + +CODEQL_PATHS = config_lists(codeql_config) + + +def under(path, prefixes): + return any(path == p or path.startswith(p + "/") for p in prefixes) + + +def codeql_fail_closed(path): + return path in CODEQL_FAIL_CLOSED or path.startswith(CODEQL_FAIL_CLOSED_PREFIXES) + + +def codeql_scope(language, path): + if codeql_fail_closed(path): + return True + if language == "c-cpp": + return path.startswith(CODEQL_C_ROOTS) and path.endswith(CODEQL_C_EXT) + if CODEQL_PATHS is not None and (not under(path, CODEQL_PATHS["paths"]) + or under(path, CODEQL_PATHS["paths-ignore"])): + return False + if language == "actions": + return path.startswith(".github/workflows/") or \ + path.rsplit("/", 1)[-1] in ("action.yml", "action.yaml") + return path.endswith(CODEQL_EXT[language]) + + def triggers(path, cat): found = [] if path == "scripts/ci/select-lanes.sh": @@ -380,6 +474,12 @@ def select(paths, expect=None): lanes = lanes | extra if lanes & TEST_LEGS: lanes = lanes | {"shard-completeness"} + for language, lane in sorted(CODEQL_LANES.items()): + hits = [p for p in paths if codeql_scope(language, p)] + if hits and lane not in lanes: + more = f" (+{len(hits) - 1} more)" if len(hits) > 1 else "" + reasons.append(f"codeql {language}: {hits[0]}{more}") + lanes = lanes | {lane} for trigger, hits in by_trigger.items(): more = f" (+{len(hits) - 1} more)" if len(hits) > 1 else "" reasons.append(f"full: {trigger}: {hits[0]}{more}") @@ -407,6 +507,10 @@ def read_paths(path): if mode == "list": print("\n".join(sorted(UNIVERSE))) +elif mode == "codeql-scope": + scoped = sorted({p for p in read_paths(source) if p and codeql_scope(codeql_language, p)}) + if scoped: + print("\n".join(scoped)) elif mode == "batch": with open(source, encoding="utf-8") as fh: for line in fh: diff --git a/scripts/package-release.sh b/scripts/package-release.sh index ff399c76d..aefaf20b3 100755 --- a/scripts/package-release.sh +++ b/scripts/package-release.sh @@ -293,7 +293,7 @@ build_mcpb_bundle() { "name": "codebase-memory-mcp", "display_name": "Codebase Memory", "version": "$mcpb_version", - "description": "Codebase knowledge graph for AI agents — 162 languages, sub-ms queries, 99% fewer tokens.", + "description": "Local code intelligence connecting code, docs and infrastructure for developers and AI agents.", "author": { "name": "DeusData", "url": "https://github.com/DeusData" }, "repository": { "type": "git", "url": "https://github.com/DeusData/codebase-memory-mcp" }, "homepage": "https://deusdata.github.io/codebase-memory-mcp/", diff --git a/scripts/test-impact/fixtures/2026-09-real-failures.tsv b/scripts/test-impact/fixtures/2026-09-real-failures.tsv index 6b6c92cc5..ce944f6a6 100644 --- a/scripts/test-impact/fixtures/2026-09-real-failures.tsv +++ b/scripts/test-impact/fixtures/2026-09-real-failures.tsv @@ -2,12 +2,14 @@ # pushes, from the lane analysis ground truth: which lane caught each. The # replay asserts the selector selects that lane for that push. # Columns (TAB): run_id, attempt, pr, head_sha, lane, classification. +# The three codeql-action-bump failures were caught by the codeql-gate lane; +# CodeQL now runs inside PR CI as the codeql-* lanes, so their lane is codeql-cpp. 34263688699 1 2115 013e1f7ec7d4b981a1cd28bf35d5767c195f8447 unix-arm64 real: cli suite hang on slow runners (arm64, then intel) 34288397854 1 2115 92cb3b9a4bebbc79bd283e447104669ca068ea64 unix-arm64 real: cli suite hang on slow runners (arm64, then intel) 34649406520 1 2115 78a6d19f1eaaa73054639ab0f14c372c53cd1bb3 unix-arm64 real: cli suite hang on slow runners (arm64, then intel) 34649406520 2 2115 78a6d19f1eaaa73054639ab0f14c372c53cd1bb3 unix-arm64 real: cli suite hang on slow runners (arm64, then intel) -34676336284 1 2185 eb5e34cfc141454e849daf413f648e73f74c455a codeql-gate real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze) -34676429093 1 2187 cea35fcc0520a0ba95139bd0210052f09b9cfa20 codeql-gate real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze) +34676336284 1 2185 eb5e34cfc141454e849daf413f648e73f74c455a codeql-cpp real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze) +34676429093 1 2187 cea35fcc0520a0ba95139bd0210052f09b9cfa20 codeql-cpp real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze) 34696991657 1 2115 9a460b389ce3002ac164605c3627bb86aa3b9be3 unix-macos-intel real: cli suite hang on slow runners (arm64, then intel) 34805101784 1 2202 7b77fd4834afe355cc4c7326ac3a55f0502f7264 lsan-macos real: macOS-only LSan leaks (memory-core) 34805101784 1 2202 7b77fd4834afe355cc4c7326ac3a55f0502f7264 msan real?: MSan stack overflow (memory-core, branch-specific) @@ -23,4 +25,4 @@ 35483768695 1 2236 d2dd3b347bbdcc695f911e80b1b7f7a32d790e5a smoke-mac real: cache CLI broken on macOS/Windows 35483768695 1 2236 d2dd3b347bbdcc695f911e80b1b7f7a32d790e5a smoke-win real: cache CLI broken on macOS/Windows 35493947102 1 2236 2caafc0c41bd083092afeb692824e718a8e1e007 smoke-mac real: cache CLI broken on macOS/Windows -36242675954 1 2363 363575af920b05f911fc76ca964dcfa7ef1c62bb codeql-gate real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze) +36242675954 1 2363 363575af920b05f911fc76ca964dcfa7ef1c62bb codeql-cpp real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze) diff --git a/scripts/test.sh b/scripts/test.sh index a10a7b330..7f4fea6e8 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -383,6 +383,8 @@ bash "$ROOT/tests/test_setup_scripts_contract.sh" # every September 2026 PR push and real failure against it. echo "=== Step 0z: PR lane selector contract ===" bash "$ROOT/tests/test_select_lanes.sh" +echo "=== Step 0z1: CodeQL retraces affected prebuilt objects ===" +bash "$ROOT/tests/test_codeql_retrace.sh" echo "=== Step 0z2: lane selector history replay ===" bash "$ROOT/scripts/test-impact/replay-selector.sh" echo "=== Step 0z3: lane-aware aggregate gates (ci-ok, shard union) ===" diff --git a/server.json b/server.json index 91a46a646..2313e40ab 100644 --- a/server.json +++ b/server.json @@ -2,7 +2,7 @@ "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", "name": "io.github.DeusData/codebase-memory-mcp", "title": "Codebase Memory", - "description": "Codebase knowledge graph for AI agents \u2014 162 languages, sub-ms queries, 99% fewer tokens.", + "description": "Local code intelligence connecting code, docs and infrastructure for developers and AI agents.", "repository": { "url": "https://github.com/DeusData/codebase-memory-mcp", "source": "github" diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index 68feb6102..83a404276 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -435,29 +435,51 @@ char *cbm_client_adapter_opencode(const char *binary_path) { sb_append(&sb, "import { spawn } from 'node:child_process';\n\n"); sb_append(&sb, "const BIN = '"); sb_append(&sb, bin); - sb_append(&sb, "';\n\n"); + sb_append(&sb, "';\n" + "const AUGMENT_TIMEOUT_MS = 10000;\n\n"); /* hook-augment requires hook_event_name; its default dialect accepts * Grep/Glob under PreToolUse, Read under PostToolUse, and the * SessionStart lifecycle event — and emits the Claude JSON envelope, so * the plugin unwraps additionalContext instead of pasting raw JSON into - * the tool output. Every failure path resolves to '' (fail open). */ + * the tool output. Every failure path resolves to '' (fail open) — + * including a child that never exits: augment() is awaited inline in + * tool.execute.after, so a hung hook-augment would stall every + * grep/glob/read completion. The timer SIGKILLs the child and resolves + * ''; the promise keeps the first resolution, so the kill-triggered + * 'close' event cannot override it. Stream errors (EPIPE on a write + * still buffered when the child dies, spawn ENOENT) fire on child.stdin, + * not on the child, and would crash the host as an unhandled 'error' + * event — hence the stdin listener and the guarded end(). */ sb_append(&sb, "function augment(payload) {\n" " return new Promise((resolve) => {\n" " const child = spawn(BIN, ['hook-augment'], {\n" " stdio: ['pipe', 'pipe', 'ignore'],\n" " env: { ...process.env, CBM_LOG_LEVEL: 'error' },\n" " });\n" + " // Stream errors fire on the stream, not the child.\n" + " child.stdin.on('error', () => {});\n" + " const timer = setTimeout(() => {\n" + " child.kill('SIGKILL');\n" + " child.stdout.destroy();\n" + " resolve('');\n" + " }, AUGMENT_TIMEOUT_MS);\n" " let out = '';\n" " child.stdout.on('data', (d) => (out += d.toString()));\n" - " child.on('error', () => resolve(''));\n" + " child.on('error', () => {\n" + " clearTimeout(timer);\n" + " resolve('');\n" + " });\n" " child.on('close', () => {\n" + " clearTimeout(timer);\n" " try {\n" " const ctx = JSON.parse(out)?.hookSpecificOutput?.additionalContext;\n" " resolve(typeof ctx === 'string' ? ctx : '');\n" " } catch { resolve(''); }\n" " });\n" - " child.stdin.end(JSON.stringify(payload));\n" + " try {\n" + " child.stdin.end(JSON.stringify(payload));\n" + " } catch { resolve(''); }\n" " });\n" "}\n\n"); diff --git a/src/cypher/cypher.c b/src/cypher/cypher.c index 8eaae7a26..c86be43bf 100644 --- a/src/cypher/cypher.c +++ b/src/cypher/cypher.c @@ -2899,12 +2899,23 @@ static cyp_partial_t partial_or(cyp_partial_t l, cyp_partial_t r) { : CYP_PARTIAL_UNKNOWN; } -/* A condition whose alias is not bound yet is UNKNOWN; a literal-only LHS - * (func condition with no variable arg) has nothing to wait for. */ +static bool partial_alias_unbound(binding_t *b, const char *alias) { + return alias && !binding_get(b, alias) && !binding_get_edge(b, alias); +} + +/* A condition is UNKNOWN while any alias it reads is not bound yet: its own + * variable, and every function argument's -- a func condition is filed under + * its first argument's alias, so the b in coalesce(a.x, b.name) is only seen + * here. A literal-only LHS has nothing to wait for. */ static cyp_partial_t eval_condition_partial(const cbm_condition_t *c, binding_t *b) { - if (c->variable && !binding_get(b, c->variable) && !binding_get_edge(b, c->variable)) { + if (partial_alias_unbound(b, c->variable)) { return CYP_PARTIAL_UNKNOWN; } + for (int i = 0; i < c->arg_count; i++) { + if (partial_alias_unbound(b, c->args[i].variable)) { + return CYP_PARTIAL_UNKNOWN; + } + } return eval_condition(c, b) ? CYP_PARTIAL_TRUE : CYP_PARTIAL_FALSE; } diff --git a/src/daemon/application.c b/src/daemon/application.c index 22b6f0d1a..32b407743 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -449,8 +449,8 @@ static bool application_regular_db_exists(const char *project) { if (written <= 0 || (size_t)written >= sizeof(path)) { return false; } - struct stat status; - return stat(path, &status) == 0 && S_ISREG(status.st_mode); + cbm_path_info_t status; + return cbm_path_info_utf8(path, &status) == CBM_PATH_INFO_OK && status.is_regular; } static bool application_canonical_directory_exists(const char *path) { diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index a0c1ddbcd..2afe49976 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -31,6 +31,8 @@ enum { CBM_DAEMON_IPC_RETRY_INTERVAL_MS = 10, CBM_DAEMON_IPC_COORDINATION_CLEANUP_MS = 500, CBM_DAEMON_IPC_WINDOWS_RENDEZVOUS_RETRY_MS = 250, + CBM_DAEMON_IPC_RELEASE_ERROR = -1, + CBM_DAEMON_IPC_RELEASE_COMPLETE = 1, }; /* Last private-namespace validation refusal, set at the exact failing check. @@ -3988,19 +3990,19 @@ bool cbm_daemon_ipc_local_transition_begin_work(cbm_daemon_ipc_local_transition_ return true; } -bool cbm_daemon_ipc_local_transition_release(cbm_daemon_ipc_local_transition_t **transition_io) { +int cbm_daemon_ipc_local_transition_try_release(cbm_daemon_ipc_local_transition_t **transition_io) { if (!transition_io) { - return false; + return CBM_DAEMON_IPC_RELEASE_ERROR; } cbm_daemon_ipc_local_transition_t *transition = *transition_io; if (!transition) { - return true; + return CBM_DAEMON_IPC_RELEASE_COMPLETE; } if (transition->legacy_fd < 0 || !transition->legacy_process_entry || transition->owner_pid != getpid() || (!transition->work_begun && (transition->startup_v2_fd < 0 || !transition->startup_v2_process_entry))) { - return false; + return CBM_DAEMON_IPC_RELEASE_ERROR; } posix_named_lock_release(transition->legacy_fd, transition->legacy_process_entry); if (!transition->work_begun) { @@ -4008,7 +4010,7 @@ bool cbm_daemon_ipc_local_transition_release(cbm_daemon_ipc_local_transition_t * } free(transition); *transition_io = NULL; - return true; + return CBM_DAEMON_IPC_RELEASE_COMPLETE; } #else /* _WIN32 */ @@ -6203,35 +6205,39 @@ static void win_participant_group_release_complete(cbm_private_file_lock_t **gro } } -static bool win_participant_state_release(const cbm_daemon_ipc_endpoint_t *endpoint, - cbm_private_lock_directory_t *directory, - cbm_private_file_lock_t **startup_v2_io, - win_legacy_mutex_guard_t **legacy_guard_io, - cbm_private_file_lock_t **group_io, HANDLE *sentinel_io) { +static int win_participant_state_release(const cbm_daemon_ipc_endpoint_t *endpoint, + cbm_private_lock_directory_t *directory, + cbm_private_file_lock_t **startup_v2_io, + win_legacy_mutex_guard_t **legacy_guard_io, + cbm_private_file_lock_t **group_io, HANDLE *sentinel_io) { if (!endpoint || !directory || !startup_v2_io || !legacy_guard_io || !group_io || !sentinel_io) { - return false; + return CBM_DAEMON_IPC_RELEASE_ERROR; } if (!*startup_v2_io) { int startup = win_startup_v2_try_acquire(directory, startup_v2_io); if (startup != 1) { - return false; + return startup; } } if (!*legacy_guard_io) { int legacy = win_legacy_mutex_guard_try_acquire(endpoint, legacy_guard_io); if (legacy != 1) { - (void)cbm_private_file_lock_release(startup_v2_io); - return false; + if (cbm_private_file_lock_release(startup_v2_io) != CBM_PRIVATE_FILE_LOCK_OK) { + return CBM_DAEMON_IPC_RELEASE_ERROR; + } + return legacy; } } if (!win_participant_group_release_locked(group_io, sentinel_io)) { - return false; + return CBM_DAEMON_IPC_RELEASE_ERROR; } if (!win_legacy_mutex_guard_release(legacy_guard_io)) { - return false; + return CBM_DAEMON_IPC_RELEASE_ERROR; } - return cbm_private_file_lock_release(startup_v2_io) == CBM_PRIVATE_FILE_LOCK_OK; + return cbm_private_file_lock_release(startup_v2_io) == CBM_PRIVATE_FILE_LOCK_OK + ? CBM_DAEMON_IPC_RELEASE_COMPLETE + : CBM_DAEMON_IPC_RELEASE_ERROR; } cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( @@ -7007,9 +7013,9 @@ bool cbm_daemon_ipc_startup_lock_release(cbm_daemon_ipc_startup_lock_t **lock_io if (!lock) { return true; } - if (!win_participant_state_release(lock->endpoint, lock->directory, &lock->startup_v2_lock, - &lock->legacy_guard, &lock->group_lock, - &lock->legacy_sentinel)) { + if (win_participant_state_release(lock->endpoint, lock->directory, &lock->startup_v2_lock, + &lock->legacy_guard, &lock->group_lock, + &lock->legacy_sentinel) != CBM_DAEMON_IPC_RELEASE_COMPLETE) { return false; } cbm_private_lock_directory_close(lock->directory); @@ -7072,9 +7078,10 @@ bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t if (!guard) { return true; } - if (!win_participant_state_release( - guard->endpoint, guard->directory, &guard->teardown_startup_v2_lock, - &guard->teardown_legacy_guard, &guard->group_lock, &guard->legacy_sentinel)) { + if (win_participant_state_release(guard->endpoint, guard->directory, + &guard->teardown_startup_v2_lock, + &guard->teardown_legacy_guard, &guard->group_lock, + &guard->legacy_sentinel) != CBM_DAEMON_IPC_RELEASE_COMPLETE) { return false; } cbm_private_lock_directory_close(guard->directory); @@ -7169,39 +7176,45 @@ bool cbm_daemon_ipc_local_transition_begin_work(cbm_daemon_ipc_local_transition_ return true; } -bool cbm_daemon_ipc_local_transition_release(cbm_daemon_ipc_local_transition_t **transition_io) { +int cbm_daemon_ipc_local_transition_try_release(cbm_daemon_ipc_local_transition_t **transition_io) { if (!transition_io) { - return false; + return CBM_DAEMON_IPC_RELEASE_ERROR; } cbm_daemon_ipc_local_transition_t *transition = *transition_io; if (!transition) { - return true; + return CBM_DAEMON_IPC_RELEASE_COMPLETE; } if (!transition->endpoint || !transition->directory) { - return false; + return CBM_DAEMON_IPC_RELEASE_ERROR; } if (transition->group_lock || transition->legacy_sentinel != INVALID_HANDLE_VALUE || transition->teardown_legacy_guard) { - if (!win_participant_state_release(transition->endpoint, transition->directory, - &transition->startup_v2_lock, - &transition->teardown_legacy_guard, - &transition->group_lock, &transition->legacy_sentinel)) { - return false; + int released = win_participant_state_release( + transition->endpoint, transition->directory, &transition->startup_v2_lock, + &transition->teardown_legacy_guard, &transition->group_lock, + &transition->legacy_sentinel); + if (released != CBM_DAEMON_IPC_RELEASE_COMPLETE) { + return released; } } else if (transition->startup_v2_lock && cbm_private_file_lock_release(&transition->startup_v2_lock) != CBM_PRIVATE_FILE_LOCK_OK) { - return false; + return CBM_DAEMON_IPC_RELEASE_ERROR; } cbm_private_lock_directory_close(transition->directory); transition->directory = NULL; free(transition); *transition_io = NULL; - return true; + return CBM_DAEMON_IPC_RELEASE_COMPLETE; } #endif /* _WIN32 */ +bool cbm_daemon_ipc_local_transition_release(cbm_daemon_ipc_local_transition_t **transition_io) { + return cbm_daemon_ipc_local_transition_try_release(transition_io) == + CBM_DAEMON_IPC_RELEASE_COMPLETE; +} + bool cbm_daemon_ipc_send_frame(cbm_daemon_ipc_connection_t *connection, cbm_daemon_frame_type_t type, uint16_t flags, const void *payload, uint32_t length) { diff --git a/src/daemon/ipc.h b/src/daemon/ipc.h index f8195f3d5..4646618f5 100644 --- a/src/daemon/ipc.h +++ b/src/daemon/ipc.h @@ -320,7 +320,13 @@ bool cbm_daemon_ipc_local_transition_begin_work(cbm_daemon_ipc_local_transition_ * Windows teardown reacquires startup-v2 and the legacy mutex, releases group * SH, closes this participant's sentinel last, then releases mutex and v2. * Success consumes and clears *transition_io; a retryable native cleanup - * failure returns false and retains it. */ + * failure or startup-gate contention returns false and retains it. */ bool cbm_daemon_ipc_local_transition_release(cbm_daemon_ipc_local_transition_t **transition_io); +/* Nonblocking release: 1 consumes and clears *transition_io (also when already + * empty), 0 retains it while another startup owns a required gate, and -1 + * retains it on an unsafe/native I/O failure. A NULL transition_io is invalid. + * Callers can give healthy contention its own wait policy without extending + * the deadline for repeated native cleanup failures. */ +int cbm_daemon_ipc_local_transition_try_release(cbm_daemon_ipc_local_transition_t **transition_io); #endif /* CBM_DAEMON_IPC_H */ diff --git a/src/discover/discover.c b/src/discover/discover.c index 134dd18c8..aaa604771 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -1197,7 +1197,12 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram struct stat st; bool is_symlink = false; if (safe_stat(abs_path, &st, &is_symlink) != 0) { - if (out->count_only) { + /* A count aborts when it cannot observe an entry, because its caller + * (auto-index admission) treats an unreliable count as a refusal. A + * symlink is not that case: it is skipped deliberately, on every walk. + * Failing here made one symlink anywhere in a tree refuse the whole + * repo, and the refusal surfaced as "unsafe_or_unavailable_path". */ + if (out->count_only && !is_symlink) { out->failed = true; } else if (is_symlink) { /* Deliberately not indexed (#963): record so callers can diff --git a/src/discover/userconfig.c b/src/discover/userconfig.c index 7e14734a2..905209a0b 100644 --- a/src/discover/userconfig.c +++ b/src/discover/userconfig.c @@ -168,6 +168,8 @@ static const lang_name_entry_t LANG_NAME_TABLE[] = { {"form", CBM_LANG_FORM}, {"magma", CBM_LANG_MAGMA}, {"wolfram", CBM_LANG_WOLFRAM}, + {"jinja2", CBM_LANG_JINJA2}, + {"jinja", CBM_LANG_JINJA2}, }; #define LANG_NAME_TABLE_SIZE (sizeof(LANG_NAME_TABLE) / sizeof(LANG_NAME_TABLE[0])) diff --git a/src/foundation/workspace.c b/src/foundation/workspace.c index eab52d48f..952e9aba0 100644 --- a/src/foundation/workspace.c +++ b/src/foundation/workspace.c @@ -257,7 +257,16 @@ static bool ws_prefix_equal(const char *a, const char *b, size_t len) { } /* True when b is a or lives under a. Compares on a separator boundary so - * "/a/bc" is not treated as living under "/a/b". */ + * "/a/bc" is not treated as living under "/a/b". + * + * '/' and '\' are equivalent here, as they are everywhere else in this module + * (ws_is_sep). The Windows sensitive-grant escape hatch (#1718) records the + * grant from cbm_canonical_path — backslashes — while the indexer candidate + * arrives normalized to '/'. A byte-exact comparison made the grant never + * match the candidate on Windows, so --approve-sensitive could not lift a + * "Program Files" refusal. Comparing on the separator-equivalence boundary + * keeps the containment semantics (a separator byte is never "part of a + * component name") while erasing the platform spelling difference. */ static bool ws_is_ancestor_or_equal(const char *a, const char *b) { if (!a || !b || !a[0] || !b[0]) { return false; diff --git a/src/main.c b/src/main.c index 54bad875e..3e658c801 100644 --- a/src/main.c +++ b/src/main.c @@ -1645,20 +1645,24 @@ static bool main_project_lock_manager_close(cbm_project_lock_manager_t **manager } static bool main_local_transition_close(cbm_daemon_ipc_local_transition_t **transition) { - uint64_t deadline = main_deadline_after(MAIN_COORDINATION_CLEANUP_MS); + uint64_t ceiling = main_deadline_after(MAIN_STARTUP_CONTENTION_CEILING_MS); + uint64_t error_deadline = main_deadline_after(MAIN_COORDINATION_CLEANUP_MS); while (transition && *transition) { - /* A failed release always RETAINS the transition and is retriable by - * contract: the Windows release transition must briefly try-hold the - * shared startup-v2/legacy gates, so concurrent one-shot teardowns - * legitimately collide and succeed on retry. Success consumes the - * transition; only never-finishing cleanup is a failure, and the - * deadline escalation below owns that. */ - (void)cbm_daemon_ipc_local_transition_release(transition); + /* Windows teardown reacquires the startup-v2/legacy gates before + * releasing its participant claim. A live startup owner can hold them + * beyond the native cleanup deadline. Retain protection and use the + * acquisition backstop for contention; repeated native errors still + * get only the short cleanup deadline. Success consumes the handle. */ + int status = cbm_daemon_ipc_local_transition_try_release(transition); if (*transition) { - if (cbm_now_ms() >= deadline) { + uint64_t now = cbm_now_ms(); + if (now >= ceiling || (status < 0 && now >= error_deadline)) { main_coordination_cleanup_fail_stop("local_transition_cleanup"); } - cbm_usleep(1000); + if (status == 0) { + error_deadline = main_deadline_after(MAIN_COORDINATION_CLEANUP_MS); + } + cbm_usleep(status == 0 ? 10000 : 1000); } } return true; diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index f6e43b317..5bd02e4e3 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -21029,7 +21029,9 @@ char *cbm_mcp_server_handle(cbm_mcp_server_t *srv, const char *line) { cbm_clock_gettime(CLOCK_MONOTONIC, &t1); long long dur_us = ((long long)(t1.tv_sec - req_t0.tv_sec) * MCP_S_TO_US) + ((long long)(t1.tv_nsec - req_t0.tv_nsec) / MCP_MS_TO_US); - cbm_log_mcp_request(req.method, NULL, true, dur_us); + /* server/discover is the modern connect probe. -32601 stays so the + * client falls back to initialize; that miss is not an error (#2433). */ + cbm_log_mcp_request(req.method, NULL, strcmp(req.method, "server/discover") != 0, dur_us); cbm_mcp_server_request_scope_end(srv); cbm_jsonrpc_request_free(&req); return err; diff --git a/src/pipeline/fqn.c b/src/pipeline/fqn.c index f4603dc80..f7b5b5efb 100644 --- a/src/pipeline/fqn.c +++ b/src/pipeline/fqn.c @@ -329,6 +329,9 @@ static char *resolve_js_relative(char *buf, size_t buf_size, const char *module_ continue; } if (seg_len == FQN_DOTDOT_LEN && seg_start[0] == '.' && seg_start[FQN_SEP_LEN] == '.') { + if (!buf[0]) { + return NULL; + } path_pop_segment(buf); continue; } diff --git a/src/pipeline/helm_template.c b/src/pipeline/helm_template.c new file mode 100644 index 000000000..7c9a5d894 --- /dev/null +++ b/src/pipeline/helm_template.c @@ -0,0 +1,3737 @@ +/* + * helm_template.c -- Helm values and the rendering of a chart template's + * image references (see helm_template.h). + * + * Values: YAML -> JSON as Helm does it (sigs.k8s.io/yaml over go-yaml v2): + * YAML 1.1 booleans (yes/no/on/off), `~` / null / empty a null, every number + * a float64 -- a float prints like Go's %v (1.10 -> 1.1, 20231010 -> + * 2.023101e+07), the way Helm renders it. --set values keep int64. + * + * Rendering: Go text/template over the vendored gotmpl grammar -- text with + * {{- / -}} trimming; if / else if / else, with, range (lists, maps by sorted + * key, else, break / continue), variables (:= / =, `$` the template's data), + * define / template / block / include / tpl, pipelines (the piped value is + * the last argument); and the Sprig functions image expressions use. A missing + * map key renders "" (Helm drops ""); a field of nil is an error. + * An error, a function not modelled, .Release.Name / .Capabilities / lookup, + * a define not found -- each is UNKNOWN, which spreads to what reads it; a + * condition or range that is UNKNOWN runs no branch. Nesting of include / + * template stops where Helm's does (1000), an error there too. + */ +#include "pipeline/helm_template.h" + +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/json_writer.h" /* cbm_json_writer_allocator */ +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "cbm.h" +#include "lang_specs.h" + +#include +#include +#include +#include +#include +#include + +enum { HELM_MAX_NESTING = 1000 }; /* Helm engine's recursionMaxNums */ +enum { HELM_MAX_SET_INDEX = 65536 }; /* Helm strvals' MaxIndex for `--set a[n]` */ + +/* A render doing more than this -- steps (an action, an evaluation, a range + * iteration), bytes written to any sink, or memory it takes (the arena's + * growth: captures, list copies, strings) -- is one Helm never finishes + * either (a range over a huge int, defines that call each other in a fan): + * it stops, and every probe of the template is UNKNOWN. Counted, never timed; + * far above any real chart (the 20 infra corpora and the helm oracle charts + * never reach it). */ +enum { HELM_MAX_STEPS = 10000000 }; +static const size_t HELM_MAX_OUTPUT = (size_t)CBM_SZ_64 * CBM_SZ_1K * CBM_SZ_1K; /* 64 MiB */ +static const size_t HELM_MAX_MEMORY = (size_t)CBM_SZ_256 * CBM_SZ_1K * CBM_SZ_1K; /* 256 MiB */ + +/* ── Values: YAML -> JSON ─────────────────────────────────────────── */ + +typedef struct { + const char *name; + size_t len; + yyjson_mut_val *val; + long decodes; /* the nodes its value took to decode, its aliases expanded */ +} hy_anchor_t; + +typedef struct { + yyjson_mut_doc *doc; + const char *src; + hy_anchor_t *anchors; + int nanchors; + int cap; + bool unknown; + long decodes; /* nodes decoded, alias expansions included (go-yaml decodeCount) */ + long alias_decodes; /* of those, the ones an alias expanded (aliasCount) */ +} hy_t; + +/* go-yaml v2 (decode.go): a document whose decodes come mostly from alias + * expansion -- a billion laughs -- is an error, so Helm reads no values. The + * share allowed falls from 99% below 400,000 decodes to 10% above 4,000,000. */ +enum { + HY_ALIAS_MIN_ALIASES = 100, + HY_ALIAS_MIN_DECODES = 1000, + HY_ALIAS_RANGE_LOW = 400000, + HY_ALIAS_RANGE_HIGH = 4000000 +}; +static const double HY_ALIAS_RATIO_SMALL = 0.99; +static const double HY_ALIAS_RATIO_LARGE = 0.10; + +static bool hy_excessive_aliasing(const hy_t *y) { + if (y->alias_decodes <= HY_ALIAS_MIN_ALIASES || y->decodes <= HY_ALIAS_MIN_DECODES) { + return false; + } + double allowed = HY_ALIAS_RATIO_SMALL; + if (y->decodes >= HY_ALIAS_RANGE_HIGH) { + allowed = HY_ALIAS_RATIO_LARGE; + } else if (y->decodes > HY_ALIAS_RANGE_LOW) { + allowed = HY_ALIAS_RATIO_SMALL - (HY_ALIAS_RATIO_SMALL - HY_ALIAS_RATIO_LARGE) * + ((double)(y->decodes - HY_ALIAS_RANGE_LOW) / + (double)(HY_ALIAS_RANGE_HIGH - HY_ALIAS_RANGE_LOW)); + } + return (double)y->alias_decodes / (double)y->decodes > allowed; +} + +static bool hy_type(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +static const char *hy_text(const hy_t *y, TSNode n, size_t *len) { + uint32_t b = ts_node_start_byte(n); + *len = ts_node_end_byte(n) - b; + return y->src + b; +} + +static void hy_anchor(hy_t *y, const char *name, size_t len, yyjson_mut_val *v, long decodes) { + if (y->nanchors == y->cap) { + int cap = y->cap ? y->cap * PAIR_LEN : CBM_SZ_16; + hy_anchor_t *g = cbm_realloc(CBM_MEM_CLASS_OTHER, y->anchors, (size_t)cap * sizeof(*g)); + if (!g) { + y->unknown = true; + return; + } + y->anchors = g; + y->cap = cap; + } + y->anchors[y->nanchors++] = (hy_anchor_t){name, len, v, decodes}; +} + +/* An alias: its anchor's value decoded again (a copy), as go-yaml counts it. */ +static yyjson_mut_val *hy_alias(hy_t *y, const char *name, size_t len) { + for (int i = y->nanchors - SKIP_ONE; i >= 0; i--) { + if (y->anchors[i].len == len && memcmp(y->anchors[i].name, name, len) == 0) { + y->decodes += y->anchors[i].decodes; + y->alias_decodes += y->anchors[i].decodes; + if (hy_excessive_aliasing(y)) { + y->unknown = true; + return NULL; + } + return yyjson_mut_val_mut_copy(y->doc, y->anchors[i].val); + } + } + y->unknown = true; + return NULL; +} + +static bool hy_in(const char *s, size_t n, const char *const *set) { + for (; *set; set++) { + if (strlen(*set) == n && memcmp(*set, s, n) == 0) { + return true; + } + } + return false; +} + +/* A float of go-yaml v2's form: [-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)? */ +static bool hy_float_form(const char *s, size_t n) { + size_t i = 0; + if (i < n && (s[i] == '-' || s[i] == '+')) { + i++; + } + size_t digits = 0; + while (i < n && isdigit((unsigned char)s[i])) { + i++; + digits++; + } + if (i < n && s[i] == '.') { + i++; + size_t frac = 0; + while (i < n && isdigit((unsigned char)s[i])) { + i++; + frac++; + } + if (digits == 0 && frac == 0) { + return false; + } + } else if (digits == 0) { + return false; + } + if (i < n && (s[i] == 'e' || s[i] == 'E')) { + i++; + if (i < n && (s[i] == '-' || s[i] == '+')) { + i++; + } + size_t ed = 0; + while (i < n && isdigit((unsigned char)s[i])) { + i++; + ed++; + } + if (ed == 0) { + return false; + } + } + return i == n; +} + +/* An integer the way go-yaml v2 reads one: strconv.ParseInt(s, 0, 64) after + * dropping `_` -- decimal, 0x hex, 0b binary, a leading 0 octal. */ +static bool hy_int_form(const char *s, size_t n, int64_t *out) { + char buf[CBM_SZ_64]; + size_t k = 0; + for (size_t i = 0; i < n && k + SKIP_ONE < sizeof(buf); i++) { + if (s[i] != '_') { + buf[k++] = s[i]; + } + } + if (k == 0 || k + SKIP_ONE >= sizeof(buf)) { + return false; + } + buf[k] = '\0'; + const char *p = buf; + bool neg = false; + if (*p == '-' || *p == '+') { + neg = *p == '-'; + p++; + } + int base = 10; + if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) { + base = 16; + p += PAIR_LEN; + } else if (p[0] == '0' && (p[1] == 'b' || p[1] == 'B')) { + base = 2; + p += PAIR_LEN; + } else if (p[0] == '0' && p[1]) { + base = 8; + p++; + } + if (!*p) { + return false; + } + char *end = NULL; + long long v = strtoll(p, &end, base); + if (!end || *end) { + return false; + } + *out = neg ? -(int64_t)v : (int64_t)v; + return true; +} + +static yyjson_mut_val *hy_plain(hy_t *y, const char *s, size_t n) { + static const char *const NULLS[] = {"", "~", "null", "Null", "NULL", NULL}; + static const char *const TRUES[] = {"y", "Y", "yes", "Yes", "YES", "true", + "True", "TRUE", "on", "On", "ON", NULL}; + static const char *const FALSES[] = {"n", "N", "no", "No", "NO", "false", + "False", "FALSE", "off", "Off", "OFF", NULL}; + static const char *const SPECIAL[] = {".inf", ".Inf", ".INF", "+.inf", "+.Inf", + "+.INF", "-.inf", "-.Inf", "-.INF", ".nan", + ".NaN", ".NAN", NULL}; + if (hy_in(s, n, NULLS)) { + return yyjson_mut_null(y->doc); + } + if (hy_in(s, n, TRUES)) { + return yyjson_mut_true(y->doc); + } + if (hy_in(s, n, FALSES)) { + return yyjson_mut_false(y->doc); + } + if (hy_in(s, n, SPECIAL)) { + y->unknown = true; /* JSON holds no inf / nan: Helm fails here */ + return NULL; + } + char c = n ? s[0] : '\0'; + if (isdigit((unsigned char)c) || c == '-' || c == '+' || c == '.') { + int64_t iv = 0; + if (hy_int_form(s, n, &iv)) { + return yyjson_mut_real(y->doc, (double)iv); + } + if (hy_float_form(s, n)) { + char buf[CBM_SZ_64]; + snprintf(buf, sizeof(buf), "%.*s", (int)n, s); + return yyjson_mut_real(y->doc, strtod(buf, NULL)); + } + } + return yyjson_mut_strncpy(y->doc, s, n); +} + +static int hy_hex(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + c = (char)tolower((unsigned char)c); + return c >= 'a' && c <= 'f' ? c - 'a' + 10 : -SKIP_ONE; +} + +static size_t hy_utf8(unsigned long cp, char *out) { + if (cp < 0x80) { + out[0] = (char)cp; + return SKIP_ONE; + } + if (cp < 0x800) { + out[0] = (char)(0xC0 | (cp >> 6)); + out[1] = (char)(0x80 | (cp & 0x3F)); + return PAIR_LEN; + } + if (cp < 0x10000) { + out[0] = (char)(0xE0 | (cp >> 12)); + out[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[2] = (char)(0x80 | (cp & 0x3F)); + return 3; + } + out[0] = (char)(0xF0 | (cp >> 18)); + out[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); + out[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[3] = (char)(0x80 | (cp & 0x3F)); + return 4; +} + +/* Fold a quoted scalar's line breaks: a break and the blanks around it + * become one space, an empty line a newline. */ +static size_t hy_fold(char *buf, size_t n) { + size_t k = 0; + for (size_t i = 0; i < n;) { + if (buf[i] == '\n') { + while (k > 0 && (buf[k - SKIP_ONE] == ' ' || buf[k - SKIP_ONE] == '\t')) { + k--; + } + size_t breaks = 0; + while (i < n && (buf[i] == '\n' || buf[i] == ' ' || buf[i] == '\t' || buf[i] == '\r')) { + breaks += buf[i] == '\n'; + i++; + } + if (breaks > SKIP_ONE) { + for (size_t b = SKIP_ONE; b < breaks; b++) { + buf[k++] = '\n'; + } + } else { + buf[k++] = ' '; + } + continue; + } + buf[k++] = buf[i++]; + } + return k; +} + +static yyjson_mut_val *hy_double(hy_t *y, const char *s, size_t n) { + if (n < PAIR_LEN) { + y->unknown = true; + return NULL; + } + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, n * 4 + SKIP_ONE); + if (!buf) { + y->unknown = true; + return NULL; + } + size_t k = 0; + for (size_t i = SKIP_ONE; i + SKIP_ONE < n; i++) { + char c = s[i]; + if (c != '\\') { + buf[k++] = c; + continue; + } + if (++i + SKIP_ONE >= n + SKIP_ONE) { + break; + } + c = s[i]; + static const char ESC_IN[] = "0abtnvfre \"/\\N_LP\t"; + static const char ESC_OUT[] = "\0\a\b\t\n\v\f\r\x1b \"/\\"; + const char *e = strchr(ESC_IN, c); + if (c == '\n') { + while (i + SKIP_ONE < n && (s[i + SKIP_ONE] == ' ' || s[i + SKIP_ONE] == '\t')) { + i++; + } + } else if (e && (size_t)(e - ESC_IN) < sizeof(ESC_OUT) - SKIP_ONE) { + buf[k++] = ESC_OUT[e - ESC_IN]; + } else if (c == 'x' || c == 'u' || c == 'U') { + int digits = c == 'x' ? PAIR_LEN : c == 'u' ? 4 : 8; + uint32_t cp = 0; + for (int d = 0; d < digits; d++) { + int h = i + SKIP_ONE < n ? hy_hex(s[i + SKIP_ONE]) : -SKIP_ONE; + if (h < 0) { + y->unknown = true; + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return NULL; + } + cp = cp * 16 + (uint32_t)h; + i++; + } + k += hy_utf8(cp, buf + k); + } else { + y->unknown = true; + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return NULL; + } + } + k = hy_fold(buf, k); + yyjson_mut_val *v = yyjson_mut_strncpy(y->doc, buf, k); + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return v; +} + +static yyjson_mut_val *hy_single(hy_t *y, const char *s, size_t n) { + if (n < PAIR_LEN) { + y->unknown = true; + return NULL; + } + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, n + SKIP_ONE); + if (!buf) { + y->unknown = true; + return NULL; + } + size_t k = 0; + for (size_t i = SKIP_ONE; i + SKIP_ONE < n; i++) { + buf[k++] = s[i]; + if (s[i] == '\'' && i + PAIR_LEN < n && s[i + SKIP_ONE] == '\'') { + i++; + } + } + k = hy_fold(buf, k); + yyjson_mut_val *v = yyjson_mut_strncpy(y->doc, buf, k); + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return v; +} + +/* A block scalar (| or >, chomping - / +, an explicit indentation digit). */ +static yyjson_mut_val *hy_block(hy_t *y, const char *s, size_t n) { + size_t i = 0; + bool folded = n > 0 && s[0] == '>'; + char chomp = 'c'; + int explicit_indent = 0; + for (i = SKIP_ONE; i < n && s[i] != '\n' && s[i] != '#'; i++) { + if (s[i] == '-' || s[i] == '+') { + chomp = s[i]; + } else if (isdigit((unsigned char)s[i])) { + explicit_indent = s[i] - '0'; + } + } + while (i < n && s[i] != '\n') { + i++; + } + i += i < n ? SKIP_ONE : 0; + /* the content's indentation: the explicit one, else its first non-empty line's */ + size_t indent = 0; + if (explicit_indent > 0) { + /* relative to the parent's: the line indentation of the scalar's own start */ + const char *ls = s; + while (ls > y->src && ls[-1] != '\n') { + ls--; + } + size_t parent = 0; + while (ls + parent < s && ls[parent] == ' ') { + parent++; + } + indent = parent + (size_t)explicit_indent; + } else { + for (size_t j = i; j < n;) { + size_t sp = 0; + while (j + sp < n && s[j + sp] == ' ') { + sp++; + } + if (j + sp < n && s[j + sp] != '\n') { + indent = sp; + break; + } + j += sp + SKIP_ONE; + } + } + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, n + PAIR_LEN); + if (!buf) { + y->unknown = true; + return NULL; + } + size_t k = 0; + bool prev_more = false; /* the previous line was more indented (folded keeps its break) */ + bool first = true; + while (i < n) { + size_t sp = 0; + while (i + sp < n && s[i + sp] == ' ' && sp < indent) { + sp++; + } + size_t e = i + sp; + while (e < n && s[e] != '\n') { + e++; + } + bool empty = e == i + sp; + bool more = !empty && (s[i + sp] == ' ' || s[i + sp] == '\t'); + if (folded && !first && k > 0 && buf[k - SKIP_ONE] == '\n' && !empty && !more && + !prev_more) { + /* a break between two plain lines folds into a space ... */ + size_t back = k - SKIP_ONE; + bool only = back == 0 || buf[back - SKIP_ONE] != '\n'; + if (only) { + buf[back] = ' '; + } else { + /* ... one of several breaks is dropped */ + k = back; + } + } + memcpy(buf + k, s + i + sp, e - (i + sp)); + k += e - (i + sp); + buf[k++] = '\n'; + prev_more = more; + first = empty && first; + i = e + SKIP_ONE; + } + if (chomp != '+') { + while (k > 0 && buf[k - SKIP_ONE] == '\n') { + k--; + } + if (chomp == 'c' && k > 0) { + buf[k++] = '\n'; + } + } + yyjson_mut_val *v = yyjson_mut_strncpy(y->doc, buf, k); + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return v; +} + +static yyjson_mut_val *hy_node(hy_t *y, TSNode n); + +/* A mapping key as JSON: its scalar's text (sigs.k8s.io/yaml stringifies). */ +static yyjson_mut_val *hy_key(hy_t *y, TSNode k) { + yyjson_mut_val *v = ts_node_is_null(k) ? yyjson_mut_null(y->doc) : hy_node(y, k); + if (!v) { + return NULL; + } + if (yyjson_mut_is_str(v)) { + return v; + } + if (yyjson_mut_is_null(v)) { + return yyjson_mut_strcpy(y->doc, "null"); + } + if (yyjson_mut_is_bool(v)) { + return yyjson_mut_strcpy(y->doc, yyjson_mut_get_bool(v) ? "true" : "false"); + } + if (yyjson_mut_is_real(v)) { + /* go-yaml v2 keeps an integer key an int: its decimal digits */ + double d = yyjson_mut_get_real(v); + char buf[CBM_SZ_64]; + if (d == floor(d) && fabs(d) < 1e18) { + snprintf(buf, sizeof(buf), "%lld", (long long)d); + } else { + snprintf(buf, sizeof(buf), "%.17g", d); + } + return yyjson_mut_strcpy(y->doc, buf); + } + y->unknown = true; /* a mapping / sequence as a key */ + return NULL; +} + +/* YAML merge key: `<<: *a` / `<<: [*a, *b]` -- entries not already there. */ +static void hy_merge(yyjson_mut_val *obj, yyjson_mut_val *src) { + if (yyjson_mut_is_arr(src)) { + size_t idx, max; + yyjson_mut_val *it; + yyjson_mut_arr_foreach(src, idx, max, it) { + hy_merge(obj, it); + } + return; + } + if (!yyjson_mut_is_obj(src)) { + return; + } + size_t idx, max; + yyjson_mut_val *k, *v; + yyjson_mut_obj_foreach(src, idx, max, k, v) { + if (!yyjson_mut_obj_getn(obj, yyjson_mut_get_str(k), yyjson_mut_get_len(k))) { + yyjson_mut_obj_add(obj, k, v); + } + } +} + +static yyjson_mut_val *hy_mapping(hy_t *y, TSNode n, const char *pair_type) { + yyjson_mut_val *obj = yyjson_mut_obj(y->doc); + yyjson_mut_val *merges = NULL; + for (uint32_t i = 0; obj && i < ts_node_named_child_count(n); i++) { + TSNode p = ts_node_named_child(n, i); + if (hy_type(p, "comment")) { + continue; + } + TSNode kn = p; + TSNode vn = {0}; + if (hy_type(p, pair_type)) { + kn = ts_node_child_by_field_name(p, "key", (uint32_t)strlen("key")); + vn = ts_node_child_by_field_name(p, "value", (uint32_t)strlen("value")); + } else if (!hy_type(p, "flow_node")) { + y->unknown = true; + return NULL; + } + size_t kl = 0; + const char *kt = ts_node_is_null(kn) ? "" : hy_text(y, kn, &kl); + yyjson_mut_val *val = ts_node_is_null(vn) ? yyjson_mut_null(y->doc) : hy_node(y, vn); + if (!val || y->unknown) { + y->unknown = true; + return NULL; + } + if (kl == PAIR_LEN && memcmp(kt, "<<", PAIR_LEN) == 0) { + merges = merges ? merges : yyjson_mut_arr(y->doc); + yyjson_mut_arr_append(merges, val); + continue; + } + yyjson_mut_val *key = hy_key(y, kn); + if (!key) { + y->unknown = true; + return NULL; + } + (void)yyjson_mut_obj_put(obj, key, val); /* a duplicate key: the last one */ + } + if (obj && merges) { + hy_merge(obj, merges); + } + return obj; +} + +static yyjson_mut_val *hy_sequence(hy_t *y, TSNode n) { + yyjson_mut_val *arr = yyjson_mut_arr(y->doc); + for (uint32_t i = 0; arr && i < ts_node_named_child_count(n); i++) { + TSNode it = ts_node_named_child(n, i); + if (hy_type(it, "comment")) { + continue; + } + TSNode content = it; + if (hy_type(it, "block_sequence_item")) { + content = ts_node_named_child_count(it) > 0 ? ts_node_named_child(it, 0) : (TSNode){0}; + while (!ts_node_is_null(content) && hy_type(content, "comment")) { + content = ts_node_next_named_sibling(content); + } + } + yyjson_mut_val *v = ts_node_is_null(content) ? yyjson_mut_null(y->doc) + : hy_type(content, "flow_pair") ? NULL + : hy_node(y, content); + if (hy_type(content, "flow_pair")) { + /* [a: b] -- a single-pair map */ + v = yyjson_mut_obj(y->doc); + TSNode kn = ts_node_child_by_field_name(content, "key", (uint32_t)strlen("key")); + TSNode vn = ts_node_child_by_field_name(content, "value", (uint32_t)strlen("value")); + yyjson_mut_val *key = hy_key(y, kn); + yyjson_mut_val *val = ts_node_is_null(vn) ? yyjson_mut_null(y->doc) : hy_node(y, vn); + if (!key || !val) { + y->unknown = true; + return NULL; + } + yyjson_mut_obj_add(v, key, val); + } + if (!v || y->unknown) { + y->unknown = true; + return NULL; + } + yyjson_mut_arr_append(arr, v); + } + return arr; +} + +static yyjson_mut_val *hy_node(hy_t *y, TSNode n) { + if (ts_node_is_null(n) || ts_node_has_error(n) || y->unknown) { + y->unknown = true; + return NULL; + } + const char *t = ts_node_type(n); + if (strcmp(t, "block_node") == 0 || strcmp(t, "flow_node") == 0) { + TSNode content = {0}; + const char *anchor = NULL; + size_t al = 0; + bool str_tag = false; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (hy_type(c, "anchor")) { + TSNode nm = ts_node_named_child(c, 0); + anchor = ts_node_is_null(nm) ? NULL : hy_text(y, nm, &al); + } else if (hy_type(c, "tag")) { + size_t tl = 0; + const char *tt = hy_text(y, c, &tl); + str_tag = tl == strlen("!!str") && memcmp(tt, "!!str", tl) == 0; + } else if (!hy_type(c, "comment")) { + content = c; + } + } + yyjson_mut_val *v = NULL; + long before = y->decodes; + if (ts_node_is_null(content)) { + y->decodes++; + v = yyjson_mut_null(y->doc); + } else if (str_tag && hy_type(content, "plain_scalar")) { + size_t l = 0; + const char *s = hy_text(y, content, &l); + y->decodes++; + v = yyjson_mut_strncpy(y->doc, s, l); + } else { + v = hy_node(y, content); + } + if (v && anchor) { + hy_anchor(y, anchor, al, v, y->decodes - before); + } + return v; + } + y->decodes++; /* a mapping, sequence, scalar or alias node */ + size_t l = 0; + const char *s = hy_text(y, n, &l); + if (strcmp(t, "block_mapping") == 0) { + return hy_mapping(y, n, "block_mapping_pair"); + } + if (strcmp(t, "flow_mapping") == 0) { + return hy_mapping(y, n, "flow_pair"); + } + if (strcmp(t, "block_sequence") == 0 || strcmp(t, "flow_sequence") == 0) { + return hy_sequence(y, n); + } + if (strcmp(t, "plain_scalar") == 0) { + return hy_plain(y, s, l); + } + if (strcmp(t, "double_quote_scalar") == 0) { + return hy_double(y, s, l); + } + if (strcmp(t, "single_quote_scalar") == 0) { + return hy_single(y, s, l); + } + if (strcmp(t, "block_scalar") == 0) { + return hy_block(y, s, l); + } + if (strcmp(t, "alias") == 0) { + TSNode nm = ts_node_named_child(n, 0); + size_t nl = 0; + const char *name = ts_node_is_null(nm) ? NULL : hy_text(y, nm, &nl); + return name ? hy_alias(y, name, nl) : NULL; + } + y->unknown = true; + return NULL; +} + +static void hy_done(hy_t *y) { + cbm_free(CBM_MEM_CLASS_OTHER, y->anchors); + y->anchors = NULL; +} + +yyjson_mut_val *cbm_helm_values_node(yyjson_mut_doc *doc, const char *src, TSNode node) { + hy_t y = {.doc = doc, .src = src}; + yyjson_mut_val *v = hy_node(&y, node); + hy_done(&y); + return y.unknown ? NULL : v; +} + +yyjson_mut_val *cbm_helm_values_parse(yyjson_mut_doc *doc, const char *src, size_t len) { + TSParser *parser = ts_parser_new(); + if (!parser || !ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_YAML))) { + if (parser) { + ts_parser_delete(parser); + } + return NULL; + } + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len); + yyjson_mut_val *out = NULL; + if (tree) { + TSNode root = ts_tree_root_node(tree); + TSNode docn = {0}; + for (uint32_t i = 0; i < ts_node_named_child_count(root); i++) { + if (hy_type(ts_node_named_child(root, i), "document")) { + docn = ts_node_named_child(root, i); + break; + } + } + TSNode content = {0}; + for (uint32_t i = 0; !ts_node_is_null(docn) && i < ts_node_named_child_count(docn); i++) { + TSNode c = ts_node_named_child(docn, i); + if (!hy_type(c, "comment")) { + content = c; + break; + } + } + if (ts_node_has_error(root) && (ts_node_is_null(docn) || ts_node_has_error(docn))) { + out = NULL; + } else if (ts_node_is_null(content)) { + out = yyjson_mut_obj(doc); + } else { + out = cbm_helm_values_node(doc, src, content); + if (out && yyjson_mut_is_null(out)) { + out = yyjson_mut_obj(doc); + } + } + ts_tree_delete(tree); + } + ts_parser_delete(parser); + return out; +} + +static yyjson_mut_val *hc_merge(yyjson_mut_doc *doc, yyjson_mut_val *dst, yyjson_mut_val *over) { + if (!yyjson_mut_is_obj(dst) || !yyjson_mut_is_obj(over)) { + return over ? yyjson_mut_val_mut_copy(doc, over) : dst; + } + size_t idx, max; + yyjson_mut_val *k, *v; + yyjson_mut_obj_foreach(over, idx, max, k, v) { + const char *ks = yyjson_mut_get_str(k); + size_t kl = yyjson_mut_get_len(k); + if (yyjson_mut_is_null(v)) { + (void)yyjson_mut_obj_remove_keyn(dst, ks, kl); + continue; + } + yyjson_mut_val *cur = yyjson_mut_obj_getn(dst, ks, kl); + yyjson_mut_val *merged = cur && yyjson_mut_is_obj(cur) && yyjson_mut_is_obj(v) + ? hc_merge(doc, cur, v) + : yyjson_mut_val_mut_copy(doc, v); + if (merged != cur) { + (void)yyjson_mut_obj_remove_keyn(dst, ks, kl); + yyjson_mut_obj_add(dst, yyjson_mut_strncpy(doc, ks, kl), merged); + } + } + return dst; +} + +yyjson_mut_val *cbm_helm_values_coalesce(yyjson_mut_doc *doc, yyjson_mut_val *base, + yyjson_mut_val *over) { + yyjson_mut_val *out = base ? yyjson_mut_val_mut_copy(doc, base) : yyjson_mut_obj(doc); + return over ? hc_merge(doc, out, over) : out; +} + +/* Helm strvals' typedVal. */ +static yyjson_mut_val *hs_typed(yyjson_mut_doc *doc, const char *v, bool as_string) { + if (as_string) { + return yyjson_mut_strcpy(doc, v); + } + if (strcasecmp(v, "true") == 0) { + return yyjson_mut_true(doc); + } + if (strcasecmp(v, "false") == 0) { + return yyjson_mut_false(doc); + } + if (strcasecmp(v, "null") == 0) { + return yyjson_mut_null(doc); + } + if (strcmp(v, "0") == 0) { + return yyjson_mut_sint(doc, 0); + } + if (v[0] && v[0] != '0') { + char *end = NULL; + long long iv = strtoll(v, &end, 10); + if (end && !*end && (isdigit((unsigned char)v[0]) || v[0] == '-' || v[0] == '+')) { + return yyjson_mut_sint(doc, iv); + } + } + return yyjson_mut_strcpy(doc, v); +} + +bool cbm_helm_values_set(yyjson_mut_doc *doc, yyjson_mut_val *root, const char *name, + const char *value, bool as_string) { + if (!yyjson_mut_is_obj(root) || !name || !name[0] || !value) { + return false; + } + yyjson_mut_val *cur = root; + const char *p = name; + char key[CBM_SZ_512]; + while (*p) { + size_t k = 0; + while (*p && *p != '.' && *p != '[' && k + SKIP_ONE < sizeof(key)) { + if (*p == '\\' && p[1]) { + p++; + } + key[k++] = *p++; + } + key[k] = '\0'; + if (*p && *p != '.' && *p != '[') { + return false; /* a key longer than the buffer: never split into two */ + } + long index = -SKIP_ONE; + if (*p == '[') { + char *end = NULL; + index = strtol(p + SKIP_ONE, &end, 10); + /* strvals: a negative index or one past MaxIndex is an error */ + if (!end || *end != ']' || index < 0 || index > HELM_MAX_SET_INDEX) { + return false; + } + p = end + SKIP_ONE; + } + bool last = !*p || (*p == '.' && !p[1]); + if (*p == '.') { + p++; + } + if (k == 0 || !yyjson_mut_is_obj(cur)) { + return false; + } + yyjson_mut_val *child = yyjson_mut_obj_get(cur, key); + if (index >= 0) { + if (!yyjson_mut_is_arr(child)) { + child = yyjson_mut_arr(doc); + (void)yyjson_mut_obj_remove_key(cur, key); + yyjson_mut_obj_add(cur, yyjson_mut_strcpy(doc, key), child); + } + while (yyjson_mut_arr_size(child) <= (size_t)index) { + yyjson_mut_arr_append(child, yyjson_mut_null(doc)); + } + if (last) { + (void)yyjson_mut_arr_replace(child, (size_t)index, hs_typed(doc, value, as_string)); + return true; + } + yyjson_mut_val *el = yyjson_mut_arr_get(child, (size_t)index); + if (!yyjson_mut_is_obj(el)) { + el = yyjson_mut_obj(doc); + (void)yyjson_mut_arr_replace(child, (size_t)index, el); + } + cur = el; + continue; + } + if (last) { + (void)yyjson_mut_obj_remove_key(cur, key); + yyjson_mut_obj_add(cur, yyjson_mut_strcpy(doc, key), hs_typed(doc, value, as_string)); + return true; + } + if (!yyjson_mut_is_obj(child)) { + child = yyjson_mut_obj(doc); + (void)yyjson_mut_obj_remove_key(cur, key); + yyjson_mut_obj_add(cur, yyjson_mut_strcpy(doc, key), child); + } + cur = child; + } + return false; +} + +/* ── Defines ──────────────────────────────────────────────────────── */ + +typedef struct { + char *rel; + char *src; + size_t len; + TSTree *tree; +} hd_file_t; + +typedef struct { + const hd_file_t *file; + TSNode node; /* the define_action / block_action */ + int depth; +} hd_def_t; + +struct cbm_helm_defines { + hd_file_t **files; /* each allocated once: defines point into them */ + int nfiles; + int cap; + CBMHashTable *names; /* name -> hd_def_t* */ + CBMHashTable *by_rel; /* path -> hd_file_t* */ +}; + +cbm_helm_defines_t *cbm_helm_defines_new(void) { + cbm_helm_defines_t *d = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*d)); + if (d) { + d->names = cbm_ht_create(0); + d->by_rel = cbm_ht_create(0); + if (!d->names || !d->by_rel) { + cbm_helm_defines_free(d); + return NULL; + } + } + return d; +} + +static int hd_depth(const char *rel) { + int n = 0; + for (const char *p = rel; *p; p++) { + n += *p == '/'; + } + return n; +} + +/* A Go string literal's value ("..." with escapes, `...` raw); NULL: invalid. */ +static char *hg_string(const char *s, size_t n, size_t *out_len); + +static void hd_scan(cbm_helm_defines_t *d, hd_file_t *f, TSNode n, int depth) { + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + const char *t = ts_node_type(c); + bool def = strcmp(t, "define_action") == 0; + bool block = strcmp(t, "block_action") == 0; + if (def || block) { + TSNode nm = ts_node_child_by_field_name(c, "name", (uint32_t)strlen("name")); + size_t nl = 0; + char *name = ts_node_is_null(nm) + ? NULL + : hg_string(f->src + ts_node_start_byte(nm), + ts_node_end_byte(nm) - ts_node_start_byte(nm), &nl); + if (name) { + hd_def_t *old = cbm_ht_get(d->names, name); + bool wins = !old || depth < old->depth || + (depth == old->depth && strcmp(f->rel, old->file->rel) < 0); + hd_def_t *nd = wins ? cbm_alloc(CBM_MEM_CLASS_OTHER, sizeof(*nd)) : NULL; + if (nd) { + *nd = (hd_def_t){f, c, depth}; + if (old) { + const char *key = cbm_ht_get_key(d->names, name); + cbm_ht_set(d->names, key, nd); + cbm_free(CBM_MEM_CLASS_OTHER, old); + cbm_free(CBM_MEM_CLASS_OTHER, name); + } else { + cbm_ht_set(d->names, name, nd); + } + name = NULL; + } + cbm_free(CBM_MEM_CLASS_OTHER, name); + } + } + if (block || strcmp(t, "if_action") == 0 || strcmp(t, "with_action") == 0 || + strcmp(t, "range_action") == 0) { + hd_scan(d, f, c, depth); /* a define is top-level; a block nests */ + } + } +} + +void cbm_helm_defines_add(cbm_helm_defines_t *d, const char *rel, const char *src, size_t len) { + if (!d || !src) { + return; + } + if (d->nfiles == d->cap) { + int cap = d->cap ? d->cap * PAIR_LEN : CBM_SZ_16; + hd_file_t **g = cbm_realloc(CBM_MEM_CLASS_OTHER, d->files, (size_t)cap * sizeof(*g)); + if (!g) { + return; + } + d->files = g; + d->cap = cap; + } + hd_file_t *f = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*f)); + if (!f) { + return; + } + f->rel = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, rel); + f->src = cbm_alloc(CBM_MEM_CLASS_OTHER, len + SKIP_ONE); + if (!f->rel || !f->src) { + cbm_free(CBM_MEM_CLASS_OTHER, f->rel); + cbm_free(CBM_MEM_CLASS_OTHER, f->src); + cbm_free(CBM_MEM_CLASS_OTHER, f); + return; + } + memcpy(f->src, src, len); + f->src[len] = '\0'; + f->len = len; + TSParser *parser = ts_parser_new(); + if (parser && ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_GOTEMPLATE))) { + f->tree = ts_parser_parse_string(parser, NULL, f->src, (uint32_t)len); + } + if (parser) { + ts_parser_delete(parser); + } + d->files[d->nfiles++] = f; + cbm_ht_set(d->by_rel, f->rel, f); + if (f->tree) { + hd_scan(d, f, ts_tree_root_node(f->tree), hd_depth(rel)); + } +} + +static void hd_free_def(const char *key, void *value, void *ud) { + (void)ud; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); + cbm_free(CBM_MEM_CLASS_OTHER, value); +} + +void cbm_helm_defines_free(cbm_helm_defines_t *d) { + if (!d) { + return; + } + if (d->names) { + cbm_ht_foreach(d->names, hd_free_def, NULL); + cbm_ht_free(d->names); + } + if (d->by_rel) { + cbm_ht_free(d->by_rel); /* its keys are the files' own */ + } + for (int i = 0; i < d->nfiles; i++) { + if (d->files[i]->tree) { + ts_tree_delete(d->files[i]->tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, d->files[i]->rel); + cbm_free(CBM_MEM_CLASS_OTHER, d->files[i]->src); + cbm_free(CBM_MEM_CLASS_OTHER, d->files[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, d->files); + cbm_free(CBM_MEM_CLASS_OTHER, d); +} + +/* Template file `rel` of the set; NULL: none. */ +static const hd_file_t *hd_file_find(const cbm_helm_defines_t *d, const char *rel) { + return d && d->by_rel ? cbm_ht_get(d->by_rel, rel) : NULL; +} + +static const hd_def_t *hd_find(const cbm_helm_defines_t *d, const char *name) { + return d && d->names ? cbm_ht_get(d->names, name) : NULL; +} + +/* ── Go literals and formatting ───────────────────────────────────── */ + +/* The value of Go string literal s ("..." or `...`) into out (capacity n); + * its length, or (size_t)-1 when it is no valid literal. */ +static size_t hg_unquote(const char *s, size_t n, char *out) { + if (n >= PAIR_LEN && s[0] == '`' && s[n - SKIP_ONE] == '`') { + memcpy(out, s + SKIP_ONE, n - PAIR_LEN); + return n - PAIR_LEN; + } + if (n < PAIR_LEN || s[0] != '"' || s[n - SKIP_ONE] != '"') { + return (size_t)-1; + } + size_t k = 0; + for (size_t i = SKIP_ONE; i + SKIP_ONE < n; i++) { + char c = s[i]; + if (c != '\\') { + out[k++] = c; + continue; + } + if (++i + SKIP_ONE >= n) { + return (size_t)-1; + } + c = s[i]; + static const char IN[] = "abfnrtv\\\"'"; + static const char OUT[] = "\a\b\f\n\r\t\v\\\"'"; + const char *e = strchr(IN, c); + if (e && c) { + out[k++] = OUT[e - IN]; + } else if (c == 'x' || c == 'u' || c == 'U') { + int digits = c == 'x' ? PAIR_LEN : c == 'u' ? 4 : 8; + uint32_t cp = 0; + for (int d = 0; d < digits; d++) { + int h = i + SKIP_ONE + SKIP_ONE < n ? hy_hex(s[i + SKIP_ONE]) : -SKIP_ONE; + if (h < 0) { + return (size_t)-1; + } + cp = cp * 16 + (uint32_t)h; + i++; + } + if (c == 'x') { + out[k++] = (char)cp; + } else { + k += hy_utf8(cp, out + k); + } + } else if (c >= '0' && c <= '7') { + uint32_t v = 0; + for (int d = 0; d < 3 && i + SKIP_ONE < n && s[i] >= '0' && s[i] <= '7'; d++, i++) { + v = v * 8 + (uint32_t)(s[i] - '0'); + } + i--; + out[k++] = (char)v; + } else { + return (size_t)-1; + } + } + return k; +} + +static char *hg_string(const char *s, size_t n, size_t *out_len) { + char *out = cbm_alloc(CBM_MEM_CLASS_OTHER, n * 4 + SKIP_ONE); + if (!out) { + return NULL; + } + size_t k = hg_unquote(s, n, out); + if (k == (size_t)-1) { + cbm_free(CBM_MEM_CLASS_OTHER, out); + return NULL; + } + out[k] = '\0'; + *out_len = k; + return out; +} + +/* float64 as Go's %v writes it (strconv 'g', shortest). */ +static void hg_float(double f, char *out, size_t sz) { + if (f == 0) { + snprintf(out, sz, signbit(f) ? "-0" : "0"); + return; + } + char e[CBM_SZ_64]; + int prec = 0; + for (prec = 0; prec < 17; prec++) { + snprintf(e, sizeof(e), "%.*e", prec, f); + if (strtod(e, NULL) == f) { + break; + } + } + /* e: [-]d[.ddd]e(+|-)XX */ + bool neg = e[0] == '-'; + const char *p = e + (neg ? SKIP_ONE : 0); + char digs[CBM_SZ_32]; + int nd = 0; + for (; *p && *p != 'e'; p++) { + if (isdigit((unsigned char)*p) && nd + SKIP_ONE < (int)sizeof(digs)) { + digs[nd++] = *p; + } + } + while (nd > SKIP_ONE && digs[nd - SKIP_ONE] == '0') { + nd--; + } + digs[nd] = '\0'; + int exp = *p == 'e' ? atoi(p + SKIP_ONE) : 0; + int dp = exp + SKIP_ONE; + size_t k = 0; + if (neg) { + out[k++] = '-'; + } + if (exp < -4 || exp >= 6) { + out[k++] = digs[0]; + if (nd > SKIP_ONE) { + out[k++] = '.'; + for (int i = SKIP_ONE; i < nd && k + 8 < sz; i++) { + out[k++] = digs[i]; + } + } + snprintf(out + k, sz - k, "e%c%02d", exp < 0 ? '-' : '+', exp < 0 ? -exp : exp); + return; + } + if (dp <= 0) { + out[k++] = '0'; + out[k++] = '.'; + for (int i = 0; i < -dp && k + 4 < sz; i++) { + out[k++] = '0'; + } + for (int i = 0; i < nd && k + 4 < sz; i++) { + out[k++] = digs[i]; + } + } else { + for (int i = 0; i < dp && k + 4 < sz; i++) { + out[k++] = i < nd ? digs[i] : '0'; + } + if (nd > dp) { + out[k++] = '.'; + for (int i = dp; i < nd && k + 4 < sz; i++) { + out[k++] = digs[i]; + } + } + } + out[k] = '\0'; +} + +bool cbm_helm_scalar_text(yyjson_mut_val *v, char *out, size_t sz) { + if (yyjson_mut_is_str(v)) { + snprintf(out, sz, "%s", yyjson_mut_get_str(v)); + } else if (yyjson_mut_is_real(v)) { + hg_float(yyjson_mut_get_real(v), out, sz); + } else if (yyjson_mut_is_int(v)) { + snprintf(out, sz, "%lld", + yyjson_mut_is_sint(v) ? (long long)yyjson_mut_get_sint(v) + : (long long)yyjson_mut_get_uint(v)); + } else if (yyjson_mut_is_bool(v)) { + snprintf(out, sz, "%s", yyjson_mut_get_bool(v) ? "true" : "false"); + } else { + return false; + } + return true; +} + +/* ── Runtime values ───────────────────────────────────────────────── */ + +typedef enum { + HV_UNK = 0, + HV_NIL, + HV_BOOL, + HV_INT, + HV_FLOAT, + HV_STR, + HV_JSON, + HV_MAP, + HV_LIST +} hv_kind_t; + +typedef struct hv hv_t; + +typedef struct { + const char **keys; + hv_t *vals; + int n; + bool strict; /* a struct (.Chart, .Release): a field not held is UNKNOWN */ +} hv_map_t; + +typedef struct { + hv_t *items; + int n; +} hv_list_t; + +struct hv { + hv_kind_t k; + bool b; + int64_t i; + double f; + const char *s; + size_t sn; + yyjson_mut_val *j; /* HV_JSON: an object or array of the values */ + hv_map_t *m; + hv_list_t *l; + const char *path; /* the values path it holds; NULL: none */ +}; + +static const hv_t HV_UNKNOWN = {.k = HV_UNK}; + +static hv_t hv_nil(void) { + return (hv_t){.k = HV_NIL}; +} + +static hv_t hv_bool(bool b) { + return (hv_t){.k = HV_BOOL, .b = b}; +} + +static hv_t hv_int(int64_t i) { + return (hv_t){.k = HV_INT, .i = i}; +} + +static hv_t hv_float(double f) { + return (hv_t){.k = HV_FLOAT, .f = f}; +} + +static hv_t hv_str(const char *s, size_t n) { + return (hv_t){.k = HV_STR, .s = s, .sn = n}; +} + +static hv_t hv_json(yyjson_mut_val *v, const char *path) { + hv_t out = {.path = path}; + if (!v || yyjson_mut_is_null(v)) { + out.k = HV_NIL; + } else if (yyjson_mut_is_bool(v)) { + out.k = HV_BOOL; + out.b = yyjson_mut_get_bool(v); + } else if (yyjson_mut_is_int(v)) { + out.k = HV_INT; + out.i = yyjson_mut_is_sint(v) ? yyjson_mut_get_sint(v) : (int64_t)yyjson_mut_get_uint(v); + } else if (yyjson_mut_is_real(v)) { + out.k = HV_FLOAT; + out.f = yyjson_mut_get_real(v); + } else if (yyjson_mut_is_str(v)) { + out.k = HV_STR; + out.s = yyjson_mut_get_str(v); + out.sn = yyjson_mut_get_len(v); + } else if (yyjson_mut_is_obj(v) || yyjson_mut_is_arr(v)) { + out.k = HV_JSON; + out.j = v; + } + return out; +} + +static bool hv_is_map(const hv_t *v) { + return v->k == HV_MAP || (v->k == HV_JSON && yyjson_mut_is_obj(v->j)); +} + +static bool hv_is_list(const hv_t *v) { + return v->k == HV_LIST || (v->k == HV_JSON && yyjson_mut_is_arr(v->j)); +} + +static int hv_len(const hv_t *v) { + switch (v->k) { + case HV_STR: + return (int)v->sn; + case HV_MAP: + return v->m->n; + case HV_LIST: + return v->l->n; + case HV_JSON: + return (int)(yyjson_mut_is_obj(v->j) ? yyjson_mut_obj_size(v->j) + : yyjson_mut_arr_size(v->j)); + default: + return -SKIP_ONE; + } +} + +/* Go template truth (if / with / and / or / not). */ +static bool hv_truth(const hv_t *v) { + switch (v->k) { + case HV_BOOL: + return v->b; + case HV_INT: + return v->i != 0; + case HV_FLOAT: + return v->f != 0; + case HV_STR: + case HV_MAP: + case HV_LIST: + case HV_JSON: + return hv_len(v) > 0; + default: + return false; + } +} + +/* ── Renderer state ───────────────────────────────────────────────── */ + +typedef struct { + const char *name; + size_t nl; + hv_t v; +} hvar_t; + +typedef struct { + char *buf; + size_t n; + size_t cap; + bool unk; +} hsink_t; + +typedef struct { + const char *src; /* the text these nodes are of */ + hv_t dot; + int var_base; /* the template's first variable (`$`) */ + hsink_t *out; /* NULL: the template's own output, which the probes read */ +} hframe_t; + +typedef struct { + CBMArena *a; + yyjson_mut_doc *scratch; + const cbm_helm_defines_t *defs; + hvar_t *vars; + int nvars; + int vcap; + int nesting; + bool brk; + bool cont; + long steps; + size_t written; /* bytes appended to any sink */ + size_t arena0; /* the arena's total when the render began */ + bool stop; /* over HELM_MAX_STEPS / _OUTPUT / _MEMORY: the render is UNKNOWN */ + const char *why; /* which budget stopped it */ + const char *rel; /* the template's path (logs) */ + /* maps a set / unset / merge changed in place (not modelled): what they + * hold is UNKNOWN from then on -- by address ("%p" keys in the arena), + * and for a merge of values also by values path (everything below it) */ + CBMHashTable *poison_maps; + CBMHashTable *poison_paths; + /* the values path of each values map changed (a set or a merge): a merge + * above one of them reaches what it put there */ + const char **changes; + int nchanges; + int chcap; + /* a change of maps not run exactly: every capture ending later is UNKNOWN */ + bool tainted; + CBMHashTable *mutates; /* define name -> it may change a map (he_define_may_mutate) */ + CBMHashTable *file_mutates; /* template file -> the same (he_file_may_mutate) */ + const char *base_path; /* .Template.BasePath; NULL: not known */ + const char *chart_dir; /* where the chart's template files are (meta.dir) */ + long busy_hits; /* defines met again while being scanned */ + /* probes of the top-level template */ + const char *top_src; + size_t top_len; + cbm_helm_probe_t *probes; + int nprobes; + const uint32_t *lines; /* line start offsets */ + int nlines; + int capture; /* the probe capturing; -1: none */ + hsink_t cap; + const char **ckeys; + int nck; + int ckcap; + CBMHashTable *ckset; /* ckeys as a set */ + TSParser *parser; + TSTree **trees; + int ntrees; + int tcap; +} hst_t; + +/* Is the render over its memory budget (the arena grown since it began)? */ +static bool hst_over_memory(const hst_t *st) { + return cbm_arena_total(st->a) - st->arena0 > HELM_MAX_MEMORY; +} + +/* The render stops: budget `why` is spent. */ +static void hst_trip(hst_t *st, const char *why) { + if (!st->stop) { + st->stop = true; + st->why = why; + } +} + +/* One step of work; false once the render is over its budget. */ +static bool hst_step(hst_t *st) { + if (!st->stop && ++st->steps > HELM_MAX_STEPS) { + hst_trip(st, "steps"); + } else if (!st->stop && hst_over_memory(st)) { + hst_trip(st, "memory"); + } + return !st->stop; +} + +/* n more bytes of output; false (and the render stopped) past the budget. */ +static bool hst_write(hst_t *st, size_t n) { + if (!st->stop && n > HELM_MAX_OUTPUT - st->written) { + hst_trip(st, "output"); + } else if (!st->stop && hst_over_memory(st)) { + hst_trip(st, "memory"); + } + if (st->stop) { + return false; + } + st->written += n; + return true; +} + +/* A change of maps the render does not run exactly (see he_call): from here + * on no capture is known. */ +static void hst_taint(hst_t *st, const char *cause) { + if (!st->tainted) { + st->tainted = true; + cbm_log_debug("helm.render.taint", "file", st->rel ? st->rel : "", "cause", cause); + } +} + +static void hs_append(hst_t *st, hsink_t *s, const char *p, size_t n) { + if (s->unk || n == 0) { + return; + } + if (!hst_write(st, n)) { + s->unk = true; + return; + } + if (s->n + n + SKIP_ONE > s->cap) { + size_t cap = (s->cap ? s->cap : CBM_SZ_64) * PAIR_LEN; + while (cap < s->n + n + SKIP_ONE) { + cap *= PAIR_LEN; + } + char *g = cbm_arena_alloc(st->a, cap); + if (!g) { + s->unk = true; + return; + } + if (s->n) { + memcpy(g, s->buf, s->n); + } + s->buf = g; + s->cap = cap; + } + memcpy(s->buf + s->n, p, n); + s->n += n; + s->buf[s->n] = '\0'; +} + +static const char *hs_str(hst_t *st, const hsink_t *s) { + return s->n ? s->buf : cbm_arena_strdup(st->a, ""); +} + +static void hst_record(hst_t *st, const char *path) { + if (st->capture < 0 || !path) { + return; + } + if (!st->ckset) { + st->ckset = cbm_ht_create(0); + } + if (!st->ckset) { + st->cap.unk = true; /* a key read not kept: the capture's mark is not whole */ + return; + } + if (cbm_ht_get(st->ckset, path)) { + return; + } + if (st->nck == st->ckcap) { + int cap = st->ckcap ? st->ckcap * PAIR_LEN : CBM_SZ_16; + const char **g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); + if (!g) { + st->cap.unk = true; + return; + } + if (st->nck) { + memcpy(g, st->ckeys, (size_t)st->nck * sizeof(*g)); + } + st->ckeys = g; + st->ckcap = cap; + } + cbm_ht_set(st->ckset, path, (void *)(uintptr_t)SKIP_ONE); + st->ckeys[st->nck++] = path; +} + +/* A values path joined with key `k` (its own `.` escaped). */ +static const char *hst_path(hst_t *st, const char *path, const char *k, size_t kl) { + if (!path) { + return NULL; + } + size_t dots = 0; + for (size_t i = 0; i < kl; i++) { + dots += k[i] == '.'; + } + size_t pl = strlen(path); + char *out = cbm_arena_alloc(st->a, pl + kl + dots + PAIR_LEN); + if (!out) { + return NULL; + } + size_t o = 0; + if (pl) { + memcpy(out, path, pl); + o = pl; + out[o++] = '.'; + } + for (size_t i = 0; i < kl; i++) { + if (k[i] == '.') { + out[o++] = '\\'; + } + out[o++] = k[i]; + } + out[o] = '\0'; + return out; +} + +static bool hst_var_push(hst_t *st, const char *name, size_t nl, hv_t v) { + if (st->nvars == st->vcap) { + int cap = st->vcap ? st->vcap * PAIR_LEN : CBM_SZ_32; + hvar_t *g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); + if (!g) { + return false; + } + if (st->nvars) { + memcpy(g, st->vars, (size_t)st->nvars * sizeof(*g)); + } + st->vars = g; + st->vcap = cap; + } + st->vars[st->nvars++] = (hvar_t){name, nl, v}; + return true; +} + +static hvar_t *hst_var(hst_t *st, const hframe_t *fr, const char *name, size_t nl) { + for (int i = st->nvars - SKIP_ONE; i >= fr->var_base; i--) { + if (st->vars[i].nl == nl && memcmp(st->vars[i].name, name, nl) == 0) { + return &st->vars[i]; + } + } + return NULL; +} + +/* Has a set / unset / merge changed map v (or, for a merge, a map holding + * it)? Then what v holds is UNKNOWN. */ +static bool hv_poisoned(const hst_t *st, const hv_t *v) { + const void *map = v->k == HV_MAP ? (const void *)v->m + : v->k == HV_JSON ? (const void *)v->j + : NULL; + if (!map) { + return false; + } + char key[CBM_SZ_32]; + snprintf(key, sizeof(key), "%p", map); + if (st->poison_maps && cbm_ht_get(st->poison_maps, key)) { + return true; + } + if (!st->poison_paths || !v->path) { + return false; + } + /* a merged values path covers what is below it: each prefix of v's path + * that ends at a key ("", "a", "a.b", "a.b[0]" ...; an escaped `\.` is no + * end) is looked up */ + size_t pl = strlen(v->path); + char small[CBM_SZ_256]; + char *buf = pl < sizeof(small) ? small : cbm_alloc(CBM_MEM_CLASS_OTHER, pl + SKIP_ONE); + bool hit = false; + for (size_t i = 0; buf && i <= pl && !hit; i++) { + bool end = i == pl || ((v->path[i] == '.' || v->path[i] == '[') && + !(i > 0 && v->path[i - SKIP_ONE] == '\\')); + if (!end) { + continue; + } + memcpy(buf, v->path, i); + buf[i] = '\0'; + hit = cbm_ht_get(st->poison_paths, buf) != NULL; + } + if (buf != small) { + cbm_free(CBM_MEM_CLASS_OTHER, buf); + } + return hit || !buf; +} + +/* Key `k` into set *set (made when NULL; the key copied into the arena). */ +static bool hst_set_add(hst_t *st, CBMHashTable **set, const char *k) { + if (!*set) { + *set = cbm_ht_create(0); + } + if (!*set) { + return false; + } + if (cbm_ht_get(*set, k)) { + return true; + } + const char *dup = cbm_arena_strdup(st->a, k); + if (!dup) { + return false; + } + cbm_ht_set(*set, dup, (void *)(uintptr_t)SKIP_ONE); + return true; +} + +static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth); + +/* Map v changed in place; false when v is no map, when (`deep`, a merge) + * a map within it cannot be poisoned whole, or with no room to say so. A + * merge may change every map within it too -- a map the template made (dict) + * holds others by reference, so those are poisoned as well. */ +static bool hst_poison(hst_t *st, const hv_t *v, bool deep) { + return hst_poison_at(st, v, deep, 0); +} + +/* Has a values map at path p or below it been changed? */ +static bool hst_changed_below(hst_t *st, const char *p) { + size_t lp = strlen(p); + for (int i = 0; i < st->nchanges; i++) { + const char *c = st->changes[i]; + if (!hst_step(st) || + (strncmp(c, p, lp) == 0 && (c[lp] == '\0' || c[lp] == '.' || c[lp] == '['))) { + return true; + } + } + return false; +} + +/* Values map at path p changed in place; false with no room to say so. */ +static bool hst_record_change(hst_t *st, const char *p) { + if (st->nchanges == st->chcap) { + int cap = st->chcap ? st->chcap * PAIR_LEN : CBM_SZ_16; + const char **g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); + if (!g) { + return false; + } + if (st->nchanges) { + memcpy(g, st->changes, (size_t)st->nchanges * sizeof(*g)); + } + st->changes = g; + st->chcap = cap; + } + st->changes[st->nchanges++] = p; + return true; +} + +static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth) { + if (depth > HELM_MAX_NESTING || !hst_step(st)) { + return false; + } + const void *map = v->k == HV_MAP ? (const void *)v->m + : v->k == HV_JSON && yyjson_mut_is_obj(v->j) ? (const void *)v->j + : NULL; + if (!map) { + return false; + } + /* a deep change reaches every map within: one not known, one a change + * already made unknown, the values root (every path below it), a map not + * of the values (whose maps no path covers), or a values map below which + * a change was made (it may hold another's map now) cannot be poisoned + * whole */ + if (deep && (hv_poisoned(st, v) || (v->path && !v->path[0]) || (v->k == HV_JSON && !v->path) || + (v->path && hst_changed_below(st, v->path)))) { + return false; + } + if (deep && v->k == HV_MAP) { + for (int i = 0; i < v->m->n; i++) { + const hv_t *x = &v->m->vals[i]; + if (x->k == HV_UNK || (hv_is_map(x) && !hst_poison_at(st, x, deep, depth + SKIP_ONE))) { + return false; + } + } + } + char key[CBM_SZ_32]; + snprintf(key, sizeof(key), "%p", map); + if (!hst_set_add(st, &st->poison_maps, key) || (v->path && !hst_record_change(st, v->path))) { + return false; + } + return !(deep && v->path) || hst_set_add(st, &st->poison_paths, v->path); +} + +static bool hn_is(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +static const char *hn_text(const hframe_t *fr, TSNode n, size_t *len) { + uint32_t b = ts_node_start_byte(n); + *len = ts_node_end_byte(n) - b; + return fr->src + b; +} + +static TSNode hn_field(TSNode n, const char *f) { + return ts_node_child_by_field_name(n, f, (uint32_t)strlen(f)); +} + +/* The name of a variable node ("" for `$`). */ +static const char *hn_var_name(const hframe_t *fr, TSNode var, size_t *nl) { + TSNode nm = hn_field(var, "name"); + if (ts_node_is_null(nm)) { + *nl = 0; + return ""; + } + return hn_text(fr, nm, nl); +} + +/* ── Printing ─────────────────────────────────────────────────────── */ + +/* Go's %v of a basic value into the arena; NULL for one not modelled. */ +static const char *hv_fmt(hst_t *st, const hv_t *v, size_t *len) { + char buf[CBM_SZ_64]; + switch (v->k) { + case HV_STR: + *len = v->sn; + return v->s; + case HV_BOOL: + *len = v->b ? 4 : 5; + return v->b ? "true" : "false"; + case HV_INT: + snprintf(buf, sizeof(buf), "%lld", (long long)v->i); + break; + case HV_FLOAT: + hg_float(v->f, buf, sizeof(buf)); + break; + case HV_NIL: + *len = 5; + return ""; + default: + return NULL; + } + char *out = cbm_arena_strdup(st->a, buf); + *len = out ? strlen(out) : 0; + return out; +} + +/* Sprig's strval. */ +static bool hv_strval(hst_t *st, const hv_t *v, hv_t *out) { + size_t n = 0; + const char *s = hv_fmt(st, v, &n); + if (!s) { + return false; + } + *out = hv_str(s, n); + return true; +} + +/* ── Output and probes ────────────────────────────────────────────── */ + +/* A capture's scalar with its YAML quoting removed; NULL: none / invalid. */ +static const char *hst_scalar(hst_t *st, const char *s, size_t n) { + while (n && (*s == ' ' || *s == '\t')) { + s++; + n--; + } + while (n && (s[n - SKIP_ONE] == ' ' || s[n - SKIP_ONE] == '\t' || s[n - SKIP_ONE] == '\r')) { + n--; + } + if (n == 0) { + return NULL; + } + if (s[0] == '"' || s[0] == '\'') { + /* the closing quote: \" escapes one in "..", '' one in '..' */ + size_t e = SKIP_ONE; + for (; e < n; e++) { + if (s[0] == '"' && s[e] == '\\') { + e++; + } else if (s[0] == '\'' && s[e] == '\'' && e + SKIP_ONE < n && + s[e + SKIP_ONE] == '\'') { + e++; + } else if (s[e] == s[0]) { + break; + } + } + if (e >= n) { + return NULL; + } + for (size_t r = e + SKIP_ONE; r < n; r++) { + if (s[r] == '#') { + break; + } + if (s[r] != ' ' && s[r] != '\t') { + return NULL; + } + } + yyjson_mut_val *v = cbm_helm_values_parse(st->scratch, s, e + SKIP_ONE); + return v && yyjson_mut_is_str(v) ? cbm_arena_strdup(st->a, yyjson_mut_get_str(v)) : NULL; + } + for (size_t i = 0; i < n; i++) { + if (s[i] == '#' && i > 0 && (s[i - SKIP_ONE] == ' ' || s[i - SKIP_ONE] == '\t')) { + n = i; + break; + } + if (s[i] == ':' && (i + SKIP_ONE == n || s[i + SKIP_ONE] == ' ')) { + return NULL; /* a mapping, no scalar */ + } + } + while (n && (s[n - SKIP_ONE] == ' ' || s[n - SKIP_ONE] == '\t')) { + n--; + } + return n ? cbm_arena_strndup(st->a, s, n) : NULL; +} + +/* strcmp order of string s against k + "." (lk = strlen(k)): 0 for each s + * that starts with it. */ +static int hst_cmp_below(const char *s, const char *k, size_t lk) { + int r = strncmp(s, k, lk); + if (r != 0) { + return r; + } + unsigned char c = (unsigned char)s[lk]; + return c < '.' ? -SKIP_ONE : c > '.' ? SKIP_ONE : 0; +} + +/* Does sorted (n keys, strcmp order) hold a key below k (k + "." ...)? */ +static bool hst_has_below(const char *const *sorted, int n, const char *k) { + size_t lk = strlen(k); + int lo = 0; + int hi = n; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + if (hst_cmp_below(sorted[mid], k, lk) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo < n && hst_cmp_below(sorted[lo], k, lk) == 0; +} + +static int hst_str_cmp(const void *x, const void *y) { + return strcmp(*(const char *const *)x, *(const char *const *)y); +} + +/* The leaves the capture read as a JSON array in the order read (NULL: none, + * or *whole false: not built in full). A path that is the prefix of another + * (at a `.`) is dropped. One sort and one allocation: k keys cost O(k log k) + * steps and their bytes of memory. */ +static const char *hst_capture_keys(hst_t *st, bool *whole) { + *whole = true; + int n = st->nck; + if (n == 0) { + return NULL; + } + const char **sorted = cbm_arena_alloc(st->a, (size_t)n * sizeof(*sorted)); + bool *keep = cbm_arena_alloc(st->a, (size_t)n * sizeof(*keep)); + if (!sorted || !keep) { + *whole = false; + return NULL; + } + memcpy(sorted, st->ckeys, (size_t)n * sizeof(*sorted)); + qsort(sorted, (size_t)n, sizeof(*sorted), hst_str_cmp); + size_t total = PAIR_LEN; /* [ ] */ + int kept = 0; + for (int i = 0; i < n; i++) { + if (!hst_step(st)) { + *whole = false; + return NULL; + } + const char *k = st->ckeys[i]; + keep[i] = k[0] && !hst_has_below(sorted, n, k); + if (!keep[i]) { + continue; + } + total += PAIR_LEN + (kept ? SKIP_ONE : 0); /* "" and , */ + for (const char *c = k; *c; c++) { + total += *c == '"' || *c == '\\' ? PAIR_LEN : SKIP_ONE; + } + kept++; + } + if (kept == 0) { + return NULL; + } + char *json = cbm_arena_alloc(st->a, total + SKIP_ONE); + if (!json) { + *whole = false; + return NULL; + } + size_t o = 0; + json[o++] = '['; + for (int i = 0; i < n; i++) { + if (!keep[i]) { + continue; + } + if (o > SKIP_ONE) { + json[o++] = ','; + } + json[o++] = '"'; + for (const char *c = st->ckeys[i]; *c; c++) { + if (*c == '"' || *c == '\\') { + json[o++] = '\\'; + } + json[o++] = *c; + } + json[o++] = '"'; + } + json[o++] = ']'; + json[o] = '\0'; + return json; +} + +/* The capture (if any) ends: its image and the keys it read join its probe. */ +static void hst_capture_end(hst_t *st) { + if (st->capture < 0) { + return; + } + cbm_helm_probe_t *p = &st->probes[st->capture]; + if (p->n == p->cap) { + /* the captures of a probe grow by doubling: n captures cost O(n) */ + int cap = p->cap ? p->cap * PAIR_LEN : CBM_SZ_4; + const char **gi = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*gi)); + const char **gk = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*gk)); + if (gi && gk) { + if (p->n) { + memcpy(gi, p->image, (size_t)p->n * sizeof(*gi)); + memcpy(gk, p->keys, (size_t)p->n * sizeof(*gk)); + } + p->image = gi; + p->keys = gk; + p->cap = cap; + } + } + if (p->n < p->cap) { + bool whole = true; + const char *keys = hst_capture_keys(st, &whole); + p->image[p->n] = st->cap.unk || st->tainted || !whole + ? NULL /* a mark not written in full: no image, never one unmarked */ + : hst_scalar(st, st->cap.buf ? st->cap.buf : "", st->cap.n); + p->keys[p->n] = whole ? keys : NULL; + p->n++; + } + /* the keys this capture recorded, one by one: a clear walks the set's + * whole capacity, which one huge capture leaves behind for every later + * one */ + for (int i = 0; st->ckset && i < st->nck; i++) { + (void)cbm_ht_delete(st->ckset, st->ckeys[i]); + } + st->capture = -SKIP_ONE; + st->cap = (hsink_t){0}; + st->nck = 0; +} + +/* Output of the top-level template: into the capture until its newline. */ +static void hst_out(hst_t *st, hframe_t *fr, const char *p, size_t n) { + if (fr->out) { + hs_append(st, fr->out, p, n); + return; + } + if (st->capture < 0) { + return; + } + const char *nl = memchr(p, '\n', n); + hs_append(st, &st->cap, p, nl ? (size_t)(nl - p) : n); + if (nl) { + hst_capture_end(st); + } +} + +static void hst_out_unknown(hst_t *st, hframe_t *fr) { + if (fr->out) { + fr->out->unk = true; + } else if (st->capture >= 0) { + st->cap.unk = true; + } +} + +/* Output of a value as text/template prints it (nil: "", as Helm). */ +static void hst_out_value(hst_t *st, hframe_t *fr, const hv_t *v) { + if (v->k == HV_NIL) { + return; + } + size_t n = 0; + const char *s = hv_fmt(st, v, &n); + if (!s) { + hst_out_unknown(st, fr); + return; + } + hst_out(st, fr, s, n); +} + +static bool hn_ws(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f'; +} + +/* A text node, trimmed by an adjacent `-}}` / `{{-`; at the top level each + * probe whose `image:` key it holds starts its capture after the key. */ +static void hst_text(hst_t *st, hframe_t *fr, TSNode n) { + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + /* the markers are lexical -- `{{- ` / ` -}}` beside the text, whichever + * action (of whatever nesting) they open or close */ + if (b >= 4 && strncmp(fr->src + b - 3, "-}}", 3) == 0 && hn_ws(fr->src[b - 4])) { + while (b < e && hn_ws(fr->src[b])) { + b++; + } + } + if (strncmp(fr->src + e, "{{-", 3) == 0 && hn_ws(fr->src[e + 3])) { + while (e > b && hn_ws(fr->src[e - SKIP_ONE])) { + e--; + } + } + if (fr->out || fr->src != st->top_src) { + hst_out(st, fr, fr->src + b, e - b); + return; + } + uint32_t at = b; + for (;;) { + /* the next probe key in [at, e) */ + uint32_t key = e; + int which = -SKIP_ONE; + for (int i = 0; i < st->nprobes; i++) { + int line = st->probes[i].line; + if (line < SKIP_ONE || line > st->nlines) { + continue; + } + uint32_t k = st->lines[line - SKIP_ONE]; + while (k < st->top_len && (st->top_src[k] == ' ' || st->top_src[k] == '\t')) { + k++; + } + if (k < st->top_len && st->top_src[k] == '-' && k + SKIP_ONE < st->top_len && + st->top_src[k + SKIP_ONE] == ' ') { + k++; + while (k < st->top_len && st->top_src[k] == ' ') { + k++; + } + } + if (k >= at && k < key && k + strlen("image:") <= e && + memcmp(st->top_src + k, "image:", strlen("image:")) == 0) { + key = k; + which = i; + } + } + if (which < 0) { + hst_out(st, fr, fr->src + at, e - at); + return; + } + uint32_t after = key + (uint32_t)strlen("image:"); + hst_out(st, fr, fr->src + at, after - at); + hst_capture_end(st); + st->capture = which; + st->cap = (hsink_t){0}; + st->nck = 0; + at = after; + } +} + +/* ── Evaluation ───────────────────────────────────────────────────── */ + +static hv_t he_eval(hst_t *st, hframe_t *fr, TSNode n); +static void he_items(hst_t *st, hframe_t *fr, TSNode parent, const char *field); +static hv_t he_render_define(hst_t *st, const char *name, hv_t arg, hsink_t *into); + +/* Field `name` of value v. */ +static hv_t he_field(hst_t *st, const hv_t *v, const char *name, size_t nl) { + if (hv_poisoned(st, v)) { + return HV_UNKNOWN; + } + if (v->k == HV_MAP) { + for (int i = 0; i < v->m->n; i++) { + if (strlen(v->m->keys[i]) == nl && memcmp(v->m->keys[i], name, nl) == 0) { + hst_record(st, v->m->vals[i].path); + return v->m->vals[i]; + } + } + return v->m->strict ? HV_UNKNOWN : hv_nil(); + } + if (v->k == HV_JSON && yyjson_mut_is_obj(v->j)) { + const char *path = hst_path(st, v->path, name, nl); + yyjson_mut_val *c = yyjson_mut_obj_getn(v->j, name, nl); + hst_record(st, path); + return hv_json(c, path); + } + return HV_UNKNOWN; /* a field of nil, of a scalar, of a list: an error */ +} + +static hv_t he_index1(hst_t *st, const hv_t *v, const hv_t *k) { + if (hv_poisoned(st, v)) { + return HV_UNKNOWN; + } + if (hv_is_map(v)) { + if (k->k != HV_STR) { + return HV_UNKNOWN; + } + return he_field(st, v, k->s, k->sn); + } + if (hv_is_list(v)) { + if (k->k != HV_INT) { + return HV_UNKNOWN; + } + if (v->k == HV_LIST) { + return k->i >= 0 && k->i < v->l->n ? v->l->items[k->i] : HV_UNKNOWN; + } + yyjson_mut_val *c = k->i >= 0 ? yyjson_mut_arr_get(v->j, (size_t)k->i) : NULL; + if (!c) { + return HV_UNKNOWN; + } + const char *path = + v->path ? cbm_arena_sprintf(st->a, "%s[%lld]", v->path, (long long)k->i) : NULL; + hst_record(st, path); + return hv_json(c, path); + } + return v->k == HV_NIL ? hv_nil() : HV_UNKNOWN; +} + +/* Sprig empty. */ +static bool hv_empty(const hv_t *v) { + return !hv_truth(v); +} + +static bool hv_eq(const hv_t *a, const hv_t *b, bool *ok) { + *ok = true; + if (a->k == HV_STR && b->k == HV_STR) { + return a->sn == b->sn && memcmp(a->s, b->s, a->sn) == 0; + } + if (a->k == HV_BOOL && b->k == HV_BOOL) { + return a->b == b->b; + } + if (a->k == HV_INT && b->k == HV_INT) { + return a->i == b->i; + } + if (a->k == HV_FLOAT && b->k == HV_FLOAT) { + return a->f == b->f; + } + *ok = false; /* incompatible kinds: an error in Go */ + return false; +} + +static hv_t hv_mklist(hst_t *st, const hv_t *items, int n) { + hv_list_t *l = cbm_arena_alloc(st->a, sizeof(*l)); + hv_t *v = n ? cbm_arena_alloc(st->a, (size_t)n * sizeof(*v)) : NULL; + if (!l || (n && !v)) { + return HV_UNKNOWN; + } + if (n) { + memcpy(v, items, (size_t)n * sizeof(*v)); + } + *l = (hv_list_t){v, n}; + return (hv_t){.k = HV_LIST, .l = l}; +} + +/* The items of a list value into an arena array; -1 when it is none. */ +static int hv_items(hst_t *st, const hv_t *v, hv_t **out) { + if (v->k == HV_LIST) { + *out = v->l->items; + return v->l->n; + } + if (v->k == HV_JSON && yyjson_mut_is_arr(v->j)) { + int n = (int)yyjson_mut_arr_size(v->j); + hv_t *items = n ? cbm_arena_alloc(st->a, (size_t)n * sizeof(*items)) : NULL; + if (n && !items) { + return -SKIP_ONE; + } + size_t idx, max; + yyjson_mut_val *it; + yyjson_mut_arr_foreach(v->j, idx, max, it) { + const char *path = v->path ? cbm_arena_sprintf(st->a, "%s[%zu]", v->path, idx) : NULL; + items[idx] = hv_json(it, path); + } + *out = items; + return n; + } + if (v->k == HV_NIL) { + *out = NULL; + return 0; + } + return -SKIP_ONE; +} + +static int he_key_cmp(const void *x, const void *y) { + return strcmp(*(const char *const *)x, *(const char *const *)y); +} + +/* A map's keys, sorted (Go's order for range), into an arena array. */ +static int hv_keys(hst_t *st, const hv_t *v, const char ***out) { + int n = hv_len(v); + const char **keys = n > 0 ? cbm_arena_alloc(st->a, (size_t)n * sizeof(*keys)) : NULL; + if (n > 0 && !keys) { + return -SKIP_ONE; + } + if (v->k == HV_MAP) { + for (int i = 0; i < n; i++) { + keys[i] = v->m->keys[i]; + } + } else { + size_t idx, max; + yyjson_mut_val *k, *val; + yyjson_mut_obj_foreach(v->j, idx, max, k, val) { + (void)val; + keys[idx] = yyjson_mut_get_str(k); + } + } + if (n > SKIP_ONE) { + qsort(keys, (size_t)n, sizeof(*keys), he_key_cmp); + } + *out = keys; + return n; +} + +static bool hv_to_int(const hv_t *v, int64_t *out) { + switch (v->k) { + case HV_INT: + *out = v->i; + return true; + case HV_FLOAT: + *out = (int64_t)v->f; + return true; + case HV_BOOL: + *out = v->b ? SKIP_ONE : 0; + return true; + case HV_NIL: + *out = 0; + return true; + case HV_STR: { + char buf[CBM_SZ_64]; + if (v->sn == 0 || v->sn >= sizeof(buf)) { + *out = 0; + return v->sn == 0; + } + memcpy(buf, v->s, v->sn); + buf[v->sn] = '\0'; + char *end = NULL; + long long i = strtoll(buf, &end, 0); + if (end && !*end) { + *out = i; + return true; + } + double d = strtod(buf, &end); + *out = end && !*end ? (int64_t)d : 0; + return true; + } + default: + return false; + } +} + +/* Go's %q of s. */ +static const char *hg_quote(hst_t *st, const char *s, size_t n, char q) { + char *out = cbm_arena_alloc(st->a, n * 4 + 3); + if (!out) { + return NULL; + } + size_t k = 0; + out[k++] = q; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (q == '"' && (c == '"' || c == '\\')) { + out[k++] = '\\'; + out[k++] = (char)c; + } else if (q == '"' && c < 0x20) { + static const char HEX[] = "0123456789abcdef"; + const char *named = c == '\n' ? "\\n" : c == '\t' ? "\\t" : c == '\r' ? "\\r" : NULL; + if (named) { + out[k++] = named[0]; + out[k++] = named[1]; + } else { + out[k++] = '\\'; + out[k++] = 'x'; + out[k++] = HEX[c >> 4]; + out[k++] = HEX[c & 0xF]; + } + } else { + out[k++] = (char)c; + } + } + out[k++] = q; + out[k] = '\0'; + return out; +} + +/* fmt.Sprintf for the verbs %s %v %d %q %% without flags; NULL otherwise. */ +static const char *he_printf(hst_t *st, const hv_t *args, int n) { + if (n < SKIP_ONE || args[0].k != HV_STR) { + return NULL; + } + hsink_t out = {0}; + int ai = SKIP_ONE; + const char *f = args[0].s; + size_t fl = args[0].sn; + for (size_t i = 0; i < fl; i++) { + if (f[i] != '%') { + hs_append(st, &out, f + i, SKIP_ONE); + continue; + } + if (++i >= fl) { + return NULL; + } + char verb = f[i]; + if (verb == '%') { + hs_append(st, &out, "%", SKIP_ONE); + continue; + } + if (ai >= n) { + return NULL; + } + const hv_t *a = &args[ai++]; + size_t sl = 0; + const char *s = NULL; + if (verb == 's' && a->k == HV_STR) { + s = a->s; + sl = a->sn; + } else if (verb == 'v') { + s = hv_fmt(st, a, &sl); + } else if (verb == 'd' && a->k == HV_INT) { + s = hv_fmt(st, a, &sl); + } else if (verb == 'q' && a->k == HV_STR) { + s = hg_quote(st, a->s, a->sn, '"'); + sl = s ? strlen(s) : 0; + } + if (!s) { + return NULL; /* %!s(float64=...) and the like: no image */ + } + hs_append(st, &out, s, sl); + } + if (ai != n || out.unk) { + return NULL; + } + return hs_str(st, &out); +} + +static hv_t he_strfn(hst_t *st, const char *fn, const hv_t *a, int n) { + /* Sprig string functions take the subject last. */ + hv_t s; + if (n < SKIP_ONE || !hv_strval(st, &a[n - SKIP_ONE], &s)) { + return HV_UNKNOWN; + } + if (strcmp(fn, "trim") == 0 && n == SKIP_ONE) { + size_t b = 0; + size_t e = s.sn; + while (b < e && hn_ws(s.s[b])) { + b++; + } + while (e > b && hn_ws(s.s[e - SKIP_ONE])) { + e--; + } + return hv_str(s.s + b, e - b); + } + if ((strcmp(fn, "lower") == 0 || strcmp(fn, "upper") == 0) && n == SKIP_ONE) { + char *o = cbm_arena_strndup(st->a, s.s, s.sn); + for (size_t i = 0; o && i < s.sn; i++) { + o[i] = + (char)(fn[0] == 'l' ? tolower((unsigned char)o[i]) : toupper((unsigned char)o[i])); + } + return o ? hv_str(o, s.sn) : HV_UNKNOWN; + } + if (n != PAIR_LEN && strcmp(fn, "replace") != 0) { + return HV_UNKNOWN; + } + hv_t p; + if (!hv_strval(st, &a[0], &p)) { + return HV_UNKNOWN; + } + if (strcmp(fn, "trimPrefix") == 0) { + bool has = p.sn <= s.sn && memcmp(s.s, p.s, p.sn) == 0; + return has ? hv_str(s.s + p.sn, s.sn - p.sn) : s; + } + if (strcmp(fn, "trimSuffix") == 0) { + bool has = p.sn <= s.sn && memcmp(s.s + s.sn - p.sn, p.s, p.sn) == 0; + return has ? hv_str(s.s, s.sn - p.sn) : s; + } + if (strcmp(fn, "trimAll") == 0) { + size_t b = 0; + size_t e = s.sn; + while (b < e && memchr(p.s, s.s[b], p.sn)) { + b++; + } + while (e > b && memchr(p.s, s.s[e - SKIP_ONE], p.sn)) { + e--; + } + return hv_str(s.s + b, e - b); + } + if (strcmp(fn, "contains") == 0 || strcmp(fn, "hasPrefix") == 0 || + strcmp(fn, "hasSuffix") == 0) { + if (p.sn > s.sn) { + return hv_bool(false); + } + if (fn[0] == 'c') { + char *hs = cbm_arena_strndup(st->a, s.s, s.sn); + char *ns = cbm_arena_strndup(st->a, p.s, p.sn); + return hs && ns ? hv_bool(strstr(hs, ns) != NULL) : HV_UNKNOWN; + } + const char *at = fn[3] == 'P' ? s.s : s.s + s.sn - p.sn; + return hv_bool(memcmp(at, p.s, p.sn) == 0); + } + if (strcmp(fn, "trunc") == 0) { + int64_t c = 0; + if (!hv_to_int(&a[0], &c)) { + return HV_UNKNOWN; + } + if (c >= 0) { + return hv_str(s.s, (size_t)c < s.sn ? (size_t)c : s.sn); + } + size_t keep = (size_t)(-c) < s.sn ? (size_t)(-c) : s.sn; + return hv_str(s.s + s.sn - keep, keep); + } + if (strcmp(fn, "replace") == 0 && n == 3) { + hv_t to; + if (!hv_strval(st, &a[SKIP_ONE], &to) || p.sn == 0) { + return HV_UNKNOWN; + } + hsink_t out = {0}; + for (size_t i = 0; i < s.sn;) { + if (i + p.sn <= s.sn && memcmp(s.s + i, p.s, p.sn) == 0) { + hs_append(st, &out, to.s, to.sn); + i += p.sn; + } else { + hs_append(st, &out, s.s + i, SKIP_ONE); + i++; + } + } + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + return HV_UNKNOWN; +} + +/* Sprig functions that change their argument in place: set / unset / merge + * a map, sortAlpha a []string (splitList, keys, regexSplit give one; a list + * is never poisoned, so it always counts as a change not run). Every other + * list function copies its argument (push, append, prepend, reverse, uniq, + * without, compact, rest, initial, concat, chunk) or reads it (first, last, + * has, slice -- which shares it but changes nothing). */ +static bool he_mutator(const char *s, size_t n) { + static const char *const MUTATORS[] = { + "set", "unset", "merge", "mergeOverwrite", "mustMerge", "mustMergeOverwrite", + "sortAlpha", NULL}; + return hy_in(s, n, MUTATORS); +} + +static bool he_define_may_mutate(hst_t *st, const char *name, int depth); + +/* The define a call / action names with a string literal into the arena; + * NULL (and *computed) when the name is no literal. */ +static const char *he_called_define(hst_t *st, const char *src, TSNode c, bool *computed) { + TSNode nm = {0}; + *computed = false; + if (hn_is(c, "function_call")) { + TSNode fn = hn_field(c, "function"); + TSNode args = hn_field(c, "arguments"); + uint32_t fb = ts_node_is_null(fn) ? 0 : ts_node_start_byte(fn); + size_t fl = ts_node_is_null(fn) ? 0 : ts_node_end_byte(fn) - fb; + if (fl != strlen("include") || strncmp(src + fb, "include", fl) != 0) { + return NULL; + } + nm = ts_node_is_null(args) ? (TSNode){0} : ts_node_named_child(args, 0); + } else { + nm = hn_field(c, "name"); + } + if (!hn_is(nm, "interpreted_string_literal") && !hn_is(nm, "raw_string_literal")) { + *computed = true; + return NULL; + } + uint32_t b = ts_node_start_byte(nm); + size_t l = ts_node_end_byte(nm) - b; + char *name = cbm_arena_alloc(st->a, l * 4 + SKIP_ONE); + size_t k = name ? hg_unquote(src + b, l, name) : (size_t)-1; + if (k == (size_t)-1) { + *computed = true; + return NULL; + } + name[k] = '\0'; + return name; +} + +/* The text of call node c's function (a function_call's, or a bare + * identifier's); NULL when it has none. */ +static const char *he_fn_text(const char *src, TSNode c, size_t *n) { + TSNode fn = hn_is(c, "identifier") ? c : hn_field(c, "function"); + if (ts_node_is_null(fn)) { + return NULL; + } + uint32_t b = ts_node_start_byte(fn); + *n = ts_node_end_byte(fn) - b; + return src + b; +} + +static bool he_is_fn(const char *s, size_t n, const char *name) { + return n == strlen(name) && memcmp(s, name, n) == 0; +} + +/* May running node n (of text src) change a map in place? It may when it + * holds a set / unset / merge (whatever the target), a tpl (its text is not + * looked into), or an include / template of a name not known or of one that + * may (he_define_may_mutate). A variable's name ($set) is no call. */ +static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth) { + if (depth > HELM_MAX_NESTING || ts_node_has_error(n)) { + return true; /* a form the grammar rejects (Go may not) may hide a change */ + } + TSTreeCursor cur = ts_tree_cursor_new(n); + bool found = false; + bool more = true; + while (more && !found && hst_step(st)) { + TSNode c = ts_tree_cursor_current_node(&cur); + const char *t = ts_node_type(c); + size_t fl = 0; + const char *fs = NULL; + bool call = strcmp(t, "function_call") == 0 || + (strcmp(t, "identifier") == 0 && hn_is(ts_node_parent(c), "chained_pipeline")); + if (call && (fs = he_fn_text(src, c, &fl)) != NULL && + (he_mutator(fs, fl) || he_is_fn(fs, fl, "tpl"))) { + found = true; + } else if (strcmp(t, "function_call") == 0 || strcmp(t, "template_action") == 0 || + strcmp(t, "block_action") == 0) { + bool computed = false; + const char *name = he_called_define(st, src, c, &computed); + found = computed || (name && he_define_may_mutate(st, name, depth + SKIP_ONE)); + } + if (found || (strcmp(t, "variable") != 0 && ts_tree_cursor_goto_first_child(&cur))) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); + return found || st->stop; +} + +/* Memo states of he_define_may_mutate / he_file_may_mutate. */ +enum { HM_NO = 1, HM_YES = 2, HM_BUSY = 3, HM_STALE = 4 }; + +/* The chart's template file an include names -- Helm names each one + * `<.Template.BasePath>/` -- or NULL: none of this chart's. */ +static const hd_file_t *he_template_file(hst_t *st, const char *name) { + if (!st->base_path || !st->chart_dir) { + return NULL; + } + size_t bl = strlen(st->base_path); + if (strncmp(name, st->base_path, bl) != 0 || name[bl] != '/') { + return NULL; + } + const char *rel = cbm_arena_sprintf(st->a, "%s%stemplates/%s", st->chart_dir, + st->chart_dir[0] ? "/" : "", name + bl + SKIP_ONE); + return rel ? hd_file_find(st->defs, rel) : NULL; +} + +/* May template file f, included by its name and so run whole, change a map + * in place (he_may_mutate over all of it)? Memoized; a file reached again + * while it is being scanned -- one including itself -- may. A no that leaned + * on a define still being scanned is not kept. */ +static bool he_file_may_mutate(hst_t *st, const hd_file_t *f, int depth) { + if (!f->tree) { + return true; + } + if (!st->file_mutates) { + st->file_mutates = cbm_ht_create(0); + } + if (!st->file_mutates) { + return true; + } + uintptr_t memo = (uintptr_t)cbm_ht_get(st->file_mutates, f->rel); + if (memo) { + return memo != HM_NO; + } + cbm_ht_set(st->file_mutates, f->rel, (void *)(uintptr_t)HM_BUSY); + long hits = st->busy_hits; + bool m = he_may_mutate(st, f->src, ts_tree_root_node(f->tree), depth); + if (m || st->busy_hits == hits) { + cbm_ht_set(st->file_mutates, f->rel, (void *)(uintptr_t)(m ? HM_YES : HM_NO)); + } else { + (void)cbm_ht_delete(st->file_mutates, f->rel); + } + return m; +} + +/* Memoized per define. A define reached again while it is being scanned + * counts as no for now; a no that leaned on such a one is not kept (STALE: + * scanned again when next asked), so defines including each other never fix + * a wrong no -- a yes is final. */ +static bool he_define_may_mutate(hst_t *st, const char *name, int depth) { + if (!st->mutates) { + st->mutates = cbm_ht_create(0); + } + uintptr_t memo = st->mutates ? (uintptr_t)cbm_ht_get(st->mutates, name) : 0; + if (memo == HM_YES || memo == HM_NO) { + return memo == HM_YES; + } + const hd_def_t *d = hd_find(st->defs, name); + if (!d) { + /* a template file of the chart by its name (the checksum idiom) is + * looked into; a name not the chart's (a library's) may. Kept by + * name once final, so an include in a loop asks once */ + const hd_file_t *f = he_template_file(st, name); + bool m = f ? he_file_may_mutate(st, f, depth) : true; + bool final = + m || (st->file_mutates && (uintptr_t)cbm_ht_get(st->file_mutates, f->rel) == HM_NO); + const char *key = final && st->mutates ? cbm_arena_strdup(st->a, name) : NULL; + if (key) { + cbm_ht_set(st->mutates, key, (void *)(uintptr_t)(m ? HM_YES : HM_NO)); + } + return m; + } + if (memo == HM_BUSY) { + st->busy_hits++; + return false; + } + const char *key = memo ? cbm_ht_get_key(st->mutates, name) : cbm_arena_strdup(st->a, name); + if (st->mutates && key) { + cbm_ht_set(st->mutates, key, (void *)(uintptr_t)HM_BUSY); + } + long hits = st->busy_hits; + bool m = he_may_mutate(st, d->file->src, d->node, depth); + bool leaned = st->busy_hits > hits; /* on a define still being scanned */ + if (st->mutates && key) { + uintptr_t res = m ? HM_YES : leaned ? HM_STALE : HM_NO; + cbm_ht_set(st->mutates, key, (void *)res); + } + return m; +} + +static hv_t he_apply_fn(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) { + for (int i = 0; i < n; i++) { + if (a[i].k == HV_UNK || hv_poisoned(st, &a[i])) { + /* an include not run (an argument UNKNOWN or poisoned) that may + * change maps, or a tpl not run: what it would change is missed */ + bool runs_code = strcmp(fn, "include") == 0 || strcmp(fn, "tpl") == 0; + const char *dn = strcmp(fn, "include") == 0 && a[0].k == HV_STR + ? cbm_arena_strndup(st->a, a[0].s, a[0].sn) + : NULL; + if (runs_code && (!dn || he_define_may_mutate(st, dn, st->nesting))) { + hst_taint(st, fn); + } + return HV_UNKNOWN; + } + } + if (strcmp(fn, "include") == 0 && n == PAIR_LEN && a[0].k == HV_STR) { + hsink_t out = {0}; + char *name = cbm_arena_strndup(st->a, a[0].s, a[0].sn); + hv_t r = name ? he_render_define(st, name, a[1], &out) : HV_UNKNOWN; + return r.k == HV_UNK || out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if (strcmp(fn, "tpl") == 0 && n == PAIR_LEN && a[0].k == HV_STR) { + if (!st->parser) { + hst_taint(st, "tpl"); + return HV_UNKNOWN; + } + char *src = cbm_arena_strndup(st->a, a[0].s, a[0].sn); + TSTree *t = src ? ts_parser_parse_string(st->parser, NULL, src, (uint32_t)a[0].sn) : NULL; + if (!t) { + return HV_UNKNOWN; + } + if (st->ntrees == st->tcap) { + int cap = st->tcap ? st->tcap * PAIR_LEN : CBM_SZ_16; + TSTree **g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); + if (!g) { + ts_tree_delete(t); + return HV_UNKNOWN; + } + if (st->ntrees) { + memcpy(g, st->trees, (size_t)st->ntrees * sizeof(*g)); + } + st->trees = g; + st->tcap = cap; + } + st->trees[st->ntrees++] = t; + TSNode root = ts_tree_root_node(t); + if (ts_node_has_error(root) || ++st->nesting > HELM_MAX_NESTING) { + st->nesting -= ts_node_has_error(root) ? 0 : SKIP_ONE; + hst_taint(st, "tpl"); + return HV_UNKNOWN; + } + hsink_t out = {0}; + int mark = st->nvars; + hframe_t sub = {.src = src, .dot = a[1], .var_base = st->nvars, .out = &out}; + hst_var_push(st, "", 0, a[1]); + he_items(st, &sub, root, NULL); + st->nvars = mark; + st->nesting--; + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if (strcmp(fn, "default") == 0 && n >= SKIP_ONE) { + return n == SKIP_ONE || hv_empty(&a[SKIP_ONE]) ? a[0] : a[SKIP_ONE]; + } + if (strcmp(fn, "coalesce") == 0) { + for (int i = 0; i < n; i++) { + if (!hv_empty(&a[i])) { + return a[i]; + } + } + return hv_nil(); + } + if (strcmp(fn, "ternary") == 0 && n == 3) { + return a[2].k == HV_BOOL ? (a[2].b ? a[0] : a[SKIP_ONE]) : HV_UNKNOWN; + } + if (strcmp(fn, "empty") == 0 && n == SKIP_ONE) { + return hv_bool(hv_empty(&a[0])); + } + if (strcmp(fn, "not") == 0 && n == SKIP_ONE) { + return hv_bool(!hv_truth(&a[0])); + } + if ((strcmp(fn, "eq") == 0 && n >= PAIR_LEN) || (strcmp(fn, "ne") == 0 && n == PAIR_LEN)) { + bool any = false; + for (int i = SKIP_ONE; i < n; i++) { + bool ok = false; + any = any || hv_eq(&a[0], &a[i], &ok); + if (!ok) { + return HV_UNKNOWN; + } + } + return hv_bool(fn[0] == 'e' ? any : !any); + } + if (strcmp(fn, "required") == 0 && n == PAIR_LEN) { + bool missing = a[1].k == HV_NIL || (a[1].k == HV_STR && a[1].sn == 0); + return missing ? HV_UNKNOWN : a[1]; + } + if (strcmp(fn, "printf") == 0) { + const char *s = he_printf(st, a, n); + return s ? hv_str(s, strlen(s)) : HV_UNKNOWN; + } + if (strcmp(fn, "print") == 0 || strcmp(fn, "cat") == 0) { + hsink_t out = {0}; + bool prev_str = true; + int written = 0; + for (int i = 0; i < n; i++) { + if (fn[0] == 'c' && a[i].k == HV_NIL) { + continue; + } + size_t sl = 0; + const char *s = hv_fmt(st, &a[i], &sl); + if (!s) { + return HV_UNKNOWN; + } + bool is_str = a[i].k == HV_STR; + /* fmt.Sprint spaces two operands neither of which is a string; cat always */ + if (written && (fn[0] == 'c' || (!is_str && !prev_str))) { + hs_append(st, &out, " ", SKIP_ONE); + } + hs_append(st, &out, s, sl); + prev_str = is_str; + written++; + } + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if (strcmp(fn, "quote") == 0 || strcmp(fn, "squote") == 0) { + hsink_t out = {0}; + int written = 0; + for (int i = 0; i < n; i++) { + hv_t s; + if (a[i].k == HV_NIL) { + continue; + } + if (!hv_strval(st, &a[i], &s)) { + return HV_UNKNOWN; + } + const char *q = fn[0] == 'q' ? hg_quote(st, s.s, s.sn, '"') + : cbm_arena_sprintf(st->a, "'%.*s'", (int)s.sn, s.s); + if (!q) { + return HV_UNKNOWN; + } + if (written++) { + hs_append(st, &out, " ", SKIP_ONE); + } + hs_append(st, &out, q, strlen(q)); + } + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if (strcmp(fn, "toString") == 0 && n == SKIP_ONE) { + hv_t s; + return hv_strval(st, &a[0], &s) ? s : HV_UNKNOWN; + } + if ((strcmp(fn, "int") == 0 || strcmp(fn, "int64") == 0 || strcmp(fn, "toInt") == 0) && + n == SKIP_ONE) { + int64_t i = 0; + return hv_to_int(&a[0], &i) ? hv_int(i) : HV_UNKNOWN; + } + if (strcmp(fn, "float64") == 0 && n == SKIP_ONE) { + int64_t i = 0; + return a[0].k == HV_FLOAT ? a[0] : hv_to_int(&a[0], &i) ? hv_float((double)i) : HV_UNKNOWN; + } + if (strcmp(fn, "len") == 0 && n == SKIP_ONE) { + int l = hv_len(&a[0]); + return l >= 0 ? hv_int(l) : HV_UNKNOWN; + } + if (strcmp(fn, "dict") == 0) { + hv_map_t *m = cbm_arena_calloc(st->a, sizeof(*m)); + int pairs = (n + SKIP_ONE) / PAIR_LEN; + if (!m) { + return HV_UNKNOWN; + } + m->keys = pairs ? cbm_arena_alloc(st->a, (size_t)pairs * sizeof(*m->keys)) : NULL; + m->vals = pairs ? cbm_arena_alloc(st->a, (size_t)pairs * sizeof(*m->vals)) : NULL; + if (pairs && (!m->keys || !m->vals)) { + return HV_UNKNOWN; + } + for (int i = 0; i + SKIP_ONE <= n; i += PAIR_LEN) { + hv_t k; + if (!hv_strval(st, &a[i], &k)) { + return HV_UNKNOWN; + } + const char *key = cbm_arena_strndup(st->a, k.s, k.sn); + hv_t val = i + SKIP_ONE < n ? a[i + SKIP_ONE] : hv_str("", 0); + int at = m->n; + for (int j = 0; j < m->n; j++) { + if (strcmp(m->keys[j], key) == 0) { + at = j; + } + } + m->keys[at] = key; + m->vals[at] = val; + m->n += at == m->n; + } + return (hv_t){.k = HV_MAP, .m = m}; + } + if (strcmp(fn, "list") == 0) { + return hv_mklist(st, a, n); + } + if (strcmp(fn, "get") == 0 && n == PAIR_LEN) { + if (!hv_is_map(&a[0]) || a[1].k != HV_STR) { + return HV_UNKNOWN; + } + hv_t v = he_field(st, &a[0], a[1].s, a[1].sn); + return v.k == HV_NIL ? hv_str("", 0) : v; + } + if (strcmp(fn, "hasKey") == 0 && n == PAIR_LEN) { + if (!hv_is_map(&a[0]) || a[1].k != HV_STR) { + return HV_UNKNOWN; + } + if (a[0].k == HV_MAP) { + for (int i = 0; i < a[0].m->n; i++) { + if (strlen(a[0].m->keys[i]) == a[1].sn && + memcmp(a[0].m->keys[i], a[1].s, a[1].sn) == 0) { + return hv_bool(true); + } + } + return hv_bool(false); + } + hst_record(st, hst_path(st, a[0].path, a[1].s, a[1].sn)); + return hv_bool(yyjson_mut_obj_getn(a[0].j, a[1].s, a[1].sn) != NULL); + } + if (strcmp(fn, "index") == 0 && n >= SKIP_ONE) { + hv_t v = a[0]; + for (int i = SKIP_ONE; i < n && v.k != HV_UNK; i++) { + v = he_index1(st, &v, &a[i]); + } + return v; + } + if ((strcmp(fn, "first") == 0 || strcmp(fn, "last") == 0) && n == SKIP_ONE) { + hv_t *items = NULL; + int c = hv_items(st, &a[0], &items); + if (c < 0) { + return HV_UNKNOWN; + } + return c == 0 ? hv_nil() : items[fn[0] == 'f' ? 0 : c - SKIP_ONE]; + } + if (strcmp(fn, "join") == 0 && n == PAIR_LEN) { + hv_t *items = NULL; + int c = hv_items(st, &a[1], &items); + hv_t sep; + if (c < 0 || !hv_strval(st, &a[0], &sep)) { + return HV_UNKNOWN; + } + hsink_t out = {0}; + for (int i = 0; i < c; i++) { + hv_t s; + if (!hv_strval(st, &items[i], &s)) { + return HV_UNKNOWN; + } + if (i) { + hs_append(st, &out, sep.s, sep.sn); + } + hs_append(st, &out, s.s, s.sn); + } + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if (strcmp(fn, "splitList") == 0 && n == PAIR_LEN && a[0].k == HV_STR && a[1].k == HV_STR && + a[0].sn > 0) { + int cap = SKIP_ONE; + for (size_t i = 0; i + a[0].sn <= a[1].sn; i++) { + cap += memcmp(a[1].s + i, a[0].s, a[0].sn) == 0; + } + hv_t *items = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*items)); + if (!items) { + return HV_UNKNOWN; + } + int c = 0; + size_t from = 0; + for (size_t i = 0; i + a[0].sn <= a[1].sn;) { + if (memcmp(a[1].s + i, a[0].s, a[0].sn) == 0) { + items[c++] = hv_str(a[1].s + from, i - from); + i += a[0].sn; + from = i; + } else { + i++; + } + } + items[c++] = hv_str(a[1].s + from, a[1].sn - from); + return hv_mklist(st, items, c); + } + if ((strcmp(fn, "nindent") == 0 || strcmp(fn, "indent") == 0) && n == PAIR_LEN) { + int64_t w = 0; + hv_t s; + if (!hv_to_int(&a[0], &w) || w < 0 || !hv_strval(st, &a[1], &s)) { + return HV_UNKNOWN; + } + /* every line gets w spaces: a width whose output cannot fit stops here */ + size_t lines = SKIP_ONE; + for (size_t i = 0; i < s.sn; i++) { + lines += s.s[i] == '\n'; + } + if ((uint64_t)w > HELM_MAX_OUTPUT / lines) { + hst_trip(st, "output"); + return HV_UNKNOWN; + } + hsink_t out = {0}; + if (fn[0] == 'n') { + hs_append(st, &out, "\n", SKIP_ONE); + } + for (int64_t i = 0; i < w && !out.unk; i++) { + hs_append(st, &out, " ", SKIP_ONE); + } + for (size_t i = 0; i < s.sn && !out.unk; i++) { + hs_append(st, &out, s.s + i, SKIP_ONE); + for (int64_t k = 0; s.s[i] == '\n' && k < w && !out.unk; k++) { + hs_append(st, &out, " ", SKIP_ONE); + } + } + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if ((strcmp(fn, "kindIs") == 0 && n == PAIR_LEN) || (strcmp(fn, "kindOf") == 0 && n == 1)) { + const hv_t *v = &a[n - SKIP_ONE]; + const char *kind = v->k == HV_NIL ? "invalid" + : v->k == HV_BOOL ? "bool" + : v->k == HV_INT ? (v->path ? "int64" : "int") + : v->k == HV_FLOAT ? "float64" + : v->k == HV_STR ? "string" + : hv_is_map(v) ? "map" + : hv_is_list(v) ? "slice" + : NULL; + if (!kind) { + return HV_UNKNOWN; + } + if (fn[4] == 'O') { + return hv_str(kind, strlen(kind)); + } + return a[0].k == HV_STR + ? hv_bool(strlen(kind) == a[0].sn && memcmp(kind, a[0].s, a[0].sn) == 0) + : HV_UNKNOWN; + } + if (strcmp(fn, "dig") == 0 && n >= 3) { + hv_t v = a[n - SKIP_ONE]; + for (int i = 0; i < n - PAIR_LEN; i++) { + if (!hv_is_map(&v) || a[i].k != HV_STR) { + return HV_UNKNOWN; + } + v = he_field(st, &v, a[i].s, a[i].sn); + if (v.k == HV_NIL) { + return a[n - PAIR_LEN]; + } + } + return v; + } + if (strcmp(fn, "trim") == 0 || strcmp(fn, "trimPrefix") == 0 || strcmp(fn, "trimSuffix") == 0 || + strcmp(fn, "trimAll") == 0 || strcmp(fn, "lower") == 0 || strcmp(fn, "upper") == 0 || + strcmp(fn, "replace") == 0 || strcmp(fn, "trunc") == 0 || strcmp(fn, "contains") == 0 || + strcmp(fn, "hasPrefix") == 0 || strcmp(fn, "hasSuffix") == 0) { + return he_strfn(st, fn, a, n); + } + (void)fr; + return HV_UNKNOWN; /* not modelled (toYaml, lookup, regex, semver, merge, ...) */ +} + +/* A function call node -- `fn args...` or a bare `fn` -- with `piped` (may + * be NULL) as its last argument. and / or evaluate lazily. */ +static hv_t he_call(hst_t *st, hframe_t *fr, TSNode call, const hv_t *piped) { + TSNode fnode = hn_is(call, "identifier") ? call : hn_field(call, "function"); + if (ts_node_is_null(fnode)) { + return HV_UNKNOWN; + } + size_t fl = 0; + const char *ft = hn_text(fr, fnode, &fl); + char fn[CBM_SZ_64]; + if (fl >= sizeof(fn)) { + return HV_UNKNOWN; + } + memcpy(fn, ft, fl); + fn[fl] = '\0'; + TSNode args = hn_is(call, "identifier") ? (TSNode){0} : hn_field(call, "arguments"); + uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); + if (he_mutator(fn, fl)) { + /* set / unset / merge change a map in place -- not run: the map is + * poisoned, what it holds UNKNOWN from here on (after a merge, every + * map within it too). That is exact only for a known map (a merge: + * whose every map within is known, see hst_poison); any other target + * -- one not known, whatever gives it -- leaves every image after it + * UNKNOWN. */ + TSNode first = na ? ts_node_named_child(args, 0) : (TSNode){0}; + hv_t target = na ? he_eval(st, fr, first) : piped ? *piped : HV_UNKNOWN; + for (uint32_t i = SKIP_ONE; i < na; i++) { + (void)he_eval(st, fr, ts_node_named_child(args, i)); /* what it runs */ + } + if (!hv_is_map(&target) || !hst_poison(st, &target, fn[0] == 'm')) { + hst_taint(st, fn); + } + return HV_UNKNOWN; /* the changed map */ + } + bool lazy = strcmp(fn, "and") == 0 || strcmp(fn, "or") == 0; + if (lazy) { + hv_t v = HV_UNKNOWN; + uint32_t total = na + (piped ? SKIP_ONE : 0); + for (uint32_t i = 0; i < total; i++) { + v = i < na ? he_eval(st, fr, ts_node_named_child(args, i)) : *piped; + if (v.k == HV_UNK || hv_poisoned(st, &v)) { + /* its truth is not known (a poisoned map's either): whether + * the operands after it run is not, nor a change they make */ + for (uint32_t j = i + SKIP_ONE; j < na; j++) { + if (he_may_mutate(st, fr->src, ts_node_named_child(args, j), st->nesting)) { + hst_taint(st, fn); + break; + } + } + return HV_UNKNOWN; + } + bool t = hv_truth(&v); + if ((fn[0] == 'a' && !t) || (fn[0] == 'o' && t)) { + return v; + } + } + return total ? v : HV_UNKNOWN; + } + int n = (int)na + (piped ? SKIP_ONE : 0); + hv_t *a = n ? cbm_arena_alloc(st->a, (size_t)n * sizeof(*a)) : NULL; + if (n && !a) { + return HV_UNKNOWN; + } + for (uint32_t i = 0; i < na; i++) { + a[i] = he_eval(st, fr, ts_node_named_child(args, i)); + } + if (piped) { + a[na] = *piped; + } + return he_apply_fn(st, fr, fn, a, n); +} + +static hv_t he_literal_int(hframe_t *fr, TSNode n) { + size_t l = 0; + const char *t = hn_text(fr, n, &l); + char buf[CBM_SZ_64]; + size_t k = 0; + for (size_t i = 0; i < l && k + SKIP_ONE < sizeof(buf); i++) { + if (t[i] != '_') { + buf[k++] = t[i]; + } + } + buf[k] = '\0'; + int base = 10; + const char *p = buf; + if (p[0] == '0' && (p[1] == 'o' || p[1] == 'O')) { + base = 8; + p += PAIR_LEN; + } else if (p[0] == '0' && (p[1] == 'b' || p[1] == 'B')) { + base = 2; + p += PAIR_LEN; + } else if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) { + base = 16; + p += PAIR_LEN; + } else if (p[0] == '0' && p[1]) { + base = 8; + p++; + } + char *end = NULL; + long long v = strtoll(p, &end, base); + return end && !*end ? hv_int(v) : HV_UNKNOWN; +} + +static hv_t he_eval(hst_t *st, hframe_t *fr, TSNode n) { + if (ts_node_is_null(n) || ts_node_has_error(n) || ts_node_is_missing(n) || !hst_step(st)) { + return HV_UNKNOWN; + } + const char *t = ts_node_type(n); + size_t l = 0; + if (strcmp(t, "dot") == 0) { + return fr->dot; + } + if (strcmp(t, "field") == 0) { + TSNode nm = hn_field(n, "name"); + if (ts_node_is_null(nm)) { + return HV_UNKNOWN; + } + const char *name = hn_text(fr, nm, &l); + return he_field(st, &fr->dot, name, l); + } + if (strcmp(t, "variable") == 0) { + const char *name = hn_var_name(fr, n, &l); + hvar_t *v = hst_var(st, fr, name, l); + return v ? v->v : HV_UNKNOWN; + } + if (strcmp(t, "selector_expression") == 0) { + TSNode op = hn_field(n, "operand"); + TSNode fd = hn_field(n, "field"); + if (ts_node_is_null(fd)) { + return HV_UNKNOWN; + } + hv_t base = he_eval(st, fr, op); + if (base.k == HV_UNK) { + return HV_UNKNOWN; + } + const char *name = hn_text(fr, fd, &l); + return he_field(st, &base, name, l); + } + if (strcmp(t, "chained_pipeline") == 0) { + uint32_t c = ts_node_named_child_count(n); + if (c == 0) { + return HV_UNKNOWN; + } + hv_t v = he_eval(st, fr, ts_node_named_child(n, 0)); + for (uint32_t i = SKIP_ONE; i < c && v.k != HV_UNK; i++) { + TSNode stage = ts_node_named_child(n, i); + if (!hn_is(stage, "function_call") && !hn_is(stage, "identifier")) { + return HV_UNKNOWN; + } + v = he_call(st, fr, stage, &v); + } + return v; + } + if (strcmp(t, "function_call") == 0 || strcmp(t, "identifier") == 0) { + return he_call(st, fr, n, NULL); + } + if (strcmp(t, "parenthesized_pipeline") == 0) { + return ts_node_named_child_count(n) == SKIP_ONE ? he_eval(st, fr, ts_node_named_child(n, 0)) + : HV_UNKNOWN; + } + if (strcmp(t, "interpreted_string_literal") == 0 || strcmp(t, "raw_string_literal") == 0) { + const char *s = hn_text(fr, n, &l); + char *out = cbm_arena_alloc(st->a, l * 4 + SKIP_ONE); + size_t k = out ? hg_unquote(s, l, out) : (size_t)-1; + if (k == (size_t)-1) { + return HV_UNKNOWN; + } + out[k] = '\0'; + return hv_str(out, k); + } + if (strcmp(t, "int_literal") == 0) { + return he_literal_int(fr, n); + } + if (strcmp(t, "float_literal") == 0) { + const char *s = hn_text(fr, n, &l); + char *buf = cbm_arena_strndup(st->a, s, l); + return buf ? hv_float(strtod(buf, NULL)) : HV_UNKNOWN; + } + if (strcmp(t, "true") == 0 || strcmp(t, "false") == 0) { + return hv_bool(t[0] == 't'); + } + if (strcmp(t, "nil") == 0) { + return hv_nil(); + } + if (strcmp(t, "variable_definition") == 0 || strcmp(t, "assignment") == 0) { + TSNode var = hn_field(n, "variable"); + hv_t v = he_eval(st, fr, hn_field(n, "value")); + const char *name = ts_node_is_null(var) ? NULL : hn_var_name(fr, var, &l); + if (!name) { + return HV_UNKNOWN; + } + if (t[0] == 'v') { + hst_var_push(st, name, l, v); + } else { + hvar_t *cur = hst_var(st, fr, name, l); + if (!cur) { + return HV_UNKNOWN; + } + cur->v = v; + } + return v; + } + return HV_UNKNOWN; /* method calls, rune / imaginary literals, ... */ +} + +/* ── Execution ────────────────────────────────────────────────────── */ + +static void he_item(hst_t *st, hframe_t *fr, TSNode c); + +/* The named children of `parent` in field `field` (NULL: every one). */ +static void he_items(hst_t *st, hframe_t *fr, TSNode parent, const char *field) { + uint32_t cc = ts_node_child_count(parent); + for (uint32_t i = 0; i < cc && !st->brk && !st->cont && hst_step(st); i++) { + TSNode c = ts_node_child(parent, i); + if (!ts_node_is_named(c)) { + continue; + } + const char *f = ts_node_field_name_for_child(parent, i); + if (field ? !(f && strcmp(f, field) == 0) : f != NULL) { + continue; + } + he_item(st, fr, c); + } +} + +/* How a part not run would leave the loop around it. */ +enum { HL_NONE = 0, HL_CONTINUE = 1, HL_BREAK = 2 }; + +/* The variables of the frames around n that n assigns (`$x = ...`) become + * UNKNOWN; returns how n would leave the loop around it (HL_*) -- by a break + * / continue not inside a range of n's own (`in_range`: n is a range's + * body). */ +static int he_scan_skipped(hst_t *st, hframe_t *fr, TSNode n, bool in_range) { + TSTreeCursor cur = ts_tree_cursor_new(n); + int leave = HL_NONE; + bool more = true; + while (more && hst_step(st)) { + TSNode c = ts_tree_cursor_current_node(&cur); + const char *t = ts_node_type(c); + if (strcmp(t, "assignment") == 0) { + TSNode var = hn_field(c, "variable"); + size_t l = 0; + const char *name = ts_node_is_null(var) ? NULL : hn_var_name(fr, var, &l); + hvar_t *v = name ? hst_var(st, fr, name, l) : NULL; + if (v) { + v->v = HV_UNKNOWN; + } + } else if (strcmp(t, "break_action") == 0 || strcmp(t, "continue_action") == 0) { + /* a range around it inside n (n itself included) is the one it leaves */ + bool own = in_range || hn_is(n, "range_action"); + for (TSNode p = c; !own && !ts_node_eq(p, n) && !ts_node_is_null(p);) { + p = ts_node_parent(p); + own = hn_is(p, "range_action"); + } + if (!own) { + leave |= t[0] == 'b' ? HL_BREAK : HL_CONTINUE; + } + } + if (ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); + return leave; +} + +/* The children of n but what was run -- its collection, its conditions up + * to `ran` (the one run; an else if / else with after it would be) -- each: + * the variables it would assign UNKNOWN, and a change of maps it may make + * (he_may_mutate) leaves every image after it UNKNOWN; returns how it would + * leave the loop. */ +static int he_spoil_parts(hst_t *st, hframe_t *fr, TSNode n, TSNode ran, const char *only, + bool in_range) { + int leave = HL_NONE; + bool mutates = false; + bool after = false; /* past `ran` */ + uint32_t cc = ts_node_child_count(n); + for (uint32_t i = 0; i < cc; i++) { + TSNode c = ts_node_child(n, i); + const char *f = ts_node_field_name_for_child(n, i); + bool cond = f && strcmp(f, "condition") == 0; + bool was_run = cond && !after; + after = after || (cond && ts_node_eq(c, ran)); + if (!ts_node_is_named(c) || hn_is(c, "range_variable_definition") || was_run || + (f && strcmp(f, "range") == 0) || (only && !(f && strcmp(f, only) == 0))) { + continue; + } + leave |= he_scan_skipped(st, fr, c, in_range); + mutates = mutates || he_may_mutate(st, fr->src, c, st->nesting); + } + if (mutates) { + hst_taint(st, "skipped"); + } + return leave; +} + +/* The bodies of if / with / range n, not run because its condition `ran` or + * its collection (a null `ran`) is UNKNOWN (that one was run): their output, + * the variables they -- and the conditions after `ran` -- would assign and a + * change they would make are UNKNOWN. A break / continue in them may or may + * not leave the loop around: what the rest of that loop's body would assign + * or change is UNKNOWN too, and a break leaves it now (its later iterations + * are not known), a continue this iteration. */ +static void he_skipped(hst_t *st, hframe_t *fr, TSNode n, TSNode ran) { + hst_out_unknown(st, fr); + bool range = hn_is(n, "range_action"); + int leave = he_spoil_parts(st, fr, n, ran, NULL, range); + if (leave == HL_NONE) { + return; + } + TSNode loop = ts_node_parent(n); + for (int depth = 0; + !ts_node_is_null(loop) && !hn_is(loop, "range_action") && depth < HELM_MAX_NESTING; + depth++) { + loop = ts_node_parent(loop); + } + if (hn_is(loop, "range_action")) { + (void)he_spoil_parts(st, fr, loop, (TSNode){0}, "body", true); + } + if (leave & HL_BREAK) { + st->brk = true; + } else { + st->cont = true; + } +} + +/* if / else if / else, and with / else with: the first branch whose + * condition holds runs; an UNKNOWN condition runs none. */ +static void he_branches(hst_t *st, hframe_t *fr, TSNode n, bool with) { + uint32_t cc = ts_node_child_count(n); + bool taken = false; + bool stop = false; + bool active = false; + int mark = st->nvars; + hframe_t sub = *fr; + for (uint32_t i = 0; i < cc && !st->brk && !st->cont; i++) { + TSNode c = ts_node_child(n, i); + if (!ts_node_is_named(c)) { + continue; + } + const char *f = ts_node_field_name_for_child(n, i); + if (!f) { + continue; + } + if (strcmp(f, "condition") == 0) { + if (taken || stop) { + active = false; + continue; + } + hv_t v = he_eval(st, fr, c); + if (v.k == HV_UNK || hv_poisoned(st, &v)) { + stop = true; + active = false; + he_skipped(st, fr, n, c); /* which branch renders is unknown */ + continue; + } + active = hv_truth(&v); + taken = active; + sub = *fr; + if (with && active) { + sub.dot = v; + } + } else if (strcmp(f, "consequence") == 0 || strcmp(f, "option") == 0) { + if (active) { + he_item(st, &sub, c); + } + } else if (strcmp(f, "alternative") == 0) { + if (!taken && !stop) { + he_item(st, fr, c); + } + } + } + st->nvars = mark; +} + +static void he_range(hst_t *st, hframe_t *fr, TSNode n) { + TSNode def = {0}; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + if (hn_is(ts_node_named_child(n, i), "range_variable_definition")) { + def = ts_node_named_child(n, i); + } + } + TSNode expr = ts_node_is_null(def) ? hn_field(n, "range") : hn_field(def, "range"); + TSNode ivar = ts_node_is_null(def) ? (TSNode){0} : hn_field(def, "index"); + TSNode evar = ts_node_is_null(def) ? (TSNode){0} : hn_field(def, "element"); + if (!ts_node_is_null(ivar) && ts_node_is_null(evar)) { + evar = ivar; /* range $x := ...: the element */ + ivar = (TSNode){0}; + } + hv_t coll = he_eval(st, fr, expr); + if (coll.k == HV_UNK || hv_poisoned(st, &coll)) { + he_skipped(st, fr, n, (TSNode){0}); + return; + } + hv_t *items = NULL; + const char **keys = NULL; + int64_t count = 0; + bool is_map = hv_is_map(&coll); + if (is_map) { + count = hv_keys(st, &coll, &keys); + } else if (coll.k == HV_INT) { + count = coll.i > 0 ? coll.i : 0; + } else { + count = hv_items(st, &coll, &items); + } + if (count < 0) { + hst_out_unknown(st, fr); + return; + } + size_t il = 0; + size_t el = 0; + const char *iname = ts_node_is_null(ivar) ? NULL : hn_var_name(fr, ivar, &il); + const char *ename = ts_node_is_null(evar) ? NULL : hn_var_name(fr, evar, &el); + for (int64_t k = 0; k < count && hst_step(st); k++) { + int mark = st->nvars; + hv_t elem; + hv_t index; + if (is_map) { + elem = he_field(st, &coll, keys[k], strlen(keys[k])); + index = hv_str(keys[k], strlen(keys[k])); + } else if (coll.k == HV_INT) { + elem = hv_int(k); + index = hv_int(k); + } else { + elem = items[k]; + index = hv_int(k); + } + if (iname) { + hst_var_push(st, iname, il, index); + } + if (ename) { + hst_var_push(st, ename, el, elem); + } + hframe_t sub = *fr; + sub.dot = elem; + he_items(st, &sub, n, "body"); + st->nvars = mark; + st->cont = false; + if (st->brk) { + st->brk = false; + break; + } + } + if (count == 0) { + he_items(st, fr, n, "alternative"); + } +} + +/* `template "name" arg` / a block: the define rendered into the output. */ +static void he_template(hst_t *st, hframe_t *fr, TSNode n) { + TSNode nm = hn_field(n, "name"); + TSNode arg = hn_field(n, "argument"); + size_t l = 0; + const char *s = ts_node_is_null(nm) ? NULL : hn_text(fr, nm, &l); + char *name = s ? cbm_arena_alloc(st->a, l * 4 + SKIP_ONE) : NULL; + size_t k = name ? hg_unquote(s, l, name) : (size_t)-1; + if (k == (size_t)-1) { + hst_out_unknown(st, fr); + return; + } + name[k] = '\0'; + hv_t a = ts_node_is_null(arg) ? hv_nil() : he_eval(st, fr, arg); + if (a.k == HV_UNK || hv_poisoned(st, &a)) { + hst_out_unknown(st, fr); + /* the define not run: a change of maps it may make is missed */ + if (he_define_may_mutate(st, name, st->nesting)) { + hst_taint(st, "template"); + } + return; + } + if (fr->out) { + if (he_render_define(st, name, a, fr->out).k == HV_UNK) { + fr->out->unk = true; + } + return; + } + /* at the top level the define's text is output that probes may read */ + hsink_t out = {0}; + if (he_render_define(st, name, a, &out).k == HV_UNK || out.unk) { + hst_out_unknown(st, fr); + return; + } + hst_out(st, fr, out.buf ? out.buf : "", out.n); +} + +static void he_item(hst_t *st, hframe_t *fr, TSNode c) { + const char *t = ts_node_type(c); + if (strcmp(t, "text") == 0) { + hst_text(st, fr, c); + } else if (strcmp(t, "comment") == 0 || strcmp(t, "define_action") == 0) { + return; + } else if (strcmp(t, "if_action") == 0) { + he_branches(st, fr, c, false); + } else if (strcmp(t, "with_action") == 0) { + he_branches(st, fr, c, true); + } else if (strcmp(t, "range_action") == 0) { + he_range(st, fr, c); + } else if (strcmp(t, "template_action") == 0 || strcmp(t, "block_action") == 0) { + he_template(st, fr, c); + } else if (strcmp(t, "break_action") == 0) { + st->brk = true; + } else if (strcmp(t, "continue_action") == 0) { + st->cont = true; + } else if (strcmp(t, "variable_definition") == 0 || strcmp(t, "assignment") == 0) { + (void)he_eval(st, fr, c); + } else { + hv_t v = he_eval(st, fr, c); + if (v.k == HV_UNK) { + hst_out_unknown(st, fr); + } else { + hst_out_value(st, fr, &v); + } + } +} + +/* Define `name` rendered with dot / $ = arg into `into`. */ +static hv_t he_render_define(hst_t *st, const char *name, hv_t arg, hsink_t *into) { + const hd_def_t *d = hd_find(st->defs, name); + if (!d) { + /* not run -- a template file of the chart by its name (the checksum + * idiom) or a name not the chart's: its output is UNKNOWN, and a + * change it may make (he_define_may_mutate) is missed */ + if (he_define_may_mutate(st, name, st->nesting)) { + hst_taint(st, "include"); + } + return HV_UNKNOWN; + } + if (++st->nesting > HELM_MAX_NESTING) { + st->nesting--; + hst_taint(st, "include"); /* nested too deep: not run */ + return HV_UNKNOWN; + } + if (ts_node_has_error(d->node)) { + hst_taint(st, "parse"); /* a form the grammar rejects may hide a change */ + } + int mark = st->nvars; + hframe_t sub = {.src = d->file->src, .dot = arg, .var_base = st->nvars, .out = into}; + hst_var_push(st, "", 0, arg); + bool brk = st->brk; + bool cont = st->cont; + st->brk = false; + st->cont = false; + he_items(st, &sub, d->node, "body"); + st->brk = brk; + st->cont = cont; + st->nvars = mark; + st->nesting--; + return into->unk ? HV_UNKNOWN : hv_nil(); +} + +/* ── Entry ────────────────────────────────────────────────────────── */ + +static hv_map_t *hv_struct(hst_t *st, int n) { + hv_map_t *m = cbm_arena_calloc(st->a, sizeof(*m)); + if (m) { + m->keys = cbm_arena_alloc(st->a, (size_t)n * sizeof(*m->keys)); + m->vals = cbm_arena_alloc(st->a, (size_t)n * sizeof(*m->vals)); + m->strict = true; + if (!m->keys || !m->vals) { + return NULL; + } + } + return m; +} + +static void hv_struct_set(hv_map_t *m, const char *k, hv_t v) { + m->keys[m->n] = k; + m->vals[m->n++] = v; +} + +static hv_t hv_cstr(const char *s) { + return s ? hv_str(s, strlen(s)) : hv_nil(); +} + +/* The probes' captures, made in the render's own arena, copied into a: the + * caller keeps a render's results, never its work. An image whose mark is not + * copied is not kept either. */ +static void helm_keep_results(CBMArena *a, cbm_helm_probe_t *probes, int nprobes) { + for (int i = 0; i < nprobes; i++) { + cbm_helm_probe_t *p = &probes[i]; + int n = p->n; + const char **img = n ? cbm_arena_alloc(a, (size_t)n * sizeof(*img)) : NULL; + const char **keys = n ? cbm_arena_alloc(a, (size_t)n * sizeof(*keys)) : NULL; + if (!img || !keys) { + n = 0; + img = NULL; + keys = NULL; + } + for (int c = 0; c < n; c++) { + const char *k = p->keys[c] ? cbm_arena_strdup(a, p->keys[c]) : NULL; + img[c] = p->image[c] && (k || !p->keys[c]) ? cbm_arena_strdup(a, p->image[c]) : NULL; + keys[c] = k; + } + p->image = img; + p->keys = keys; + p->n = n; + p->cap = n; + } +} + +bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, + yyjson_mut_val *values, const cbm_helm_chart_meta_t *chart, + const cbm_helm_defines_t *defs, cbm_helm_probe_t *probes, int nprobes) { + for (int i = 0; i < nprobes; i++) { + probes[i].n = 0; + probes[i].image = NULL; + probes[i].keys = NULL; + probes[i].cap = 0; + } + if (!src || nprobes == 0) { + return true; + } + /* the render's work lives in its own arena, freed when it ends: the + * memory budget bounds one render, and no render keeps another's */ + CBMArena work; + cbm_arena_init_lazy(&work, CBM_SZ_4K); + hst_t st = {.a = &work, + .defs = defs, + .top_src = src, + .top_len = len, + .probes = probes, + .nprobes = nprobes, + .rel = rel, + .base_path = chart ? chart->base_path : NULL, + .chart_dir = chart ? chart->dir : NULL, + .capture = -SKIP_ONE, + .arena0 = cbm_arena_total(&work)}; + st.scratch = yyjson_mut_doc_new(cbm_json_writer_allocator()); + st.parser = ts_parser_new(); + if (!st.scratch || !st.parser || + !ts_parser_set_language(st.parser, cbm_ts_language(CBM_LANG_GOTEMPLATE))) { + if (st.parser) { + ts_parser_delete(st.parser); + } + if (st.scratch) { + yyjson_mut_doc_free(st.scratch); + } + cbm_arena_destroy(&work); + return true; + } + int nl = SKIP_ONE; + for (size_t i = 0; i < len; i++) { + nl += src[i] == '\n'; + } + uint32_t *lines = cbm_arena_alloc(&work, (size_t)nl * sizeof(*lines)); + TSTree *tree = lines ? ts_parser_parse_string(st.parser, NULL, src, (uint32_t)len) : NULL; + if (tree) { + /* a template that does not parse may hide a change (Go may take a + * form the grammar rejects): no image of it is known */ + if (ts_node_has_error(ts_tree_root_node(tree))) { + hst_taint(&st, "parse"); + } + int k = 0; + lines[k++] = 0; + for (size_t i = 0; i < len; i++) { + if (src[i] == '\n' && k < nl) { + lines[k++] = (uint32_t)(i + SKIP_ONE); + } + } + st.lines = lines; + st.nlines = k; + yyjson_mut_val *vals = values ? values : yyjson_mut_obj(st.scratch); + hv_map_t *ch = hv_struct(&st, 3); + hv_map_t *rl = hv_struct(&st, SKIP_ONE); + hv_map_t *tp = st.base_path ? hv_struct(&st, SKIP_ONE) : NULL; + hv_map_t *root = hv_struct(&st, PAIR_LEN + PAIR_LEN); + if (ch && rl && root) { + hv_struct_set(ch, "Name", hv_cstr(chart ? chart->name : NULL)); + hv_struct_set(ch, "Version", hv_cstr(chart ? chart->version : NULL)); + hv_struct_set(ch, "AppVersion", hv_cstr(chart ? chart->app_version : NULL)); + hv_struct_set(rl, "Service", hv_cstr("Helm")); + hv_struct_set(root, "Values", hv_json(vals, "")); + hv_struct_set(root, "Chart", (hv_t){.k = HV_MAP, .m = ch}); + hv_struct_set(root, "Release", (hv_t){.k = HV_MAP, .m = rl}); + if (tp) { + /* .Template.Name is not modelled: UNKNOWN */ + hv_struct_set(tp, "BasePath", hv_cstr(st.base_path)); + hv_struct_set(root, "Template", (hv_t){.k = HV_MAP, .m = tp}); + } + hv_t dot = {.k = HV_MAP, .m = root}; + hframe_t fr = {.src = src, .dot = dot, .var_base = 0, .out = NULL}; + hst_var_push(&st, "", 0, dot); + he_items(&st, &fr, ts_tree_root_node(tree), NULL); + hst_capture_end(&st); + } + ts_tree_delete(tree); + } + for (int i = 0; i < st.ntrees; i++) { + ts_tree_delete(st.trees[i]); + } + ts_parser_delete(st.parser); + yyjson_mut_doc_free(st.scratch); + if (st.mutates) { + cbm_ht_free(st.mutates); + } + if (st.file_mutates) { + cbm_ht_free(st.file_mutates); + } + if (st.poison_maps) { + cbm_ht_free(st.poison_maps); + } + if (st.poison_paths) { + cbm_ht_free(st.poison_paths); + } + if (st.ckset) { + cbm_ht_free(st.ckset); + } + if (st.stop) { + /* Helm would not finish this template: none of its images is known, + * also those captured before the budget ran out */ + for (int i = 0; i < nprobes; i++) { + for (int c = 0; c < probes[i].n; c++) { + probes[i].image[c] = NULL; + probes[i].keys[c] = NULL; + } + } + cbm_log_warn("helm.render.budget", "file", rel ? rel : "", "reason", + st.why ? st.why : "memory"); + } + helm_keep_results(a, probes, nprobes); + cbm_arena_destroy(&work); + return !st.stop; +} diff --git a/src/pipeline/helm_template.h b/src/pipeline/helm_template.h new file mode 100644 index 000000000..e2136b5e8 --- /dev/null +++ b/src/pipeline/helm_template.h @@ -0,0 +1,93 @@ +/* + * helm_template.h -- Helm values and the rendering of a chart template's + * image references (pass_k8s.c). Internal to the pipeline. + * + * Values are JSON (yyjson mutable values), as Helm holds them after its + * YAML -> JSON step: go-yaml v2 scalars (YAML 1.1 booleans), every number a + * float64. A template renders with Go text/template semantics and the part + * of Sprig image expressions use; anything else -- a function not modelled, + * .Release / .Capabilities / lookup, a missing helper, a parse error on the + * way -- is UNKNOWN, and an image that reads an UNKNOWN is no image. Nothing + * is guessed: a value is what Helm would render, or none. + */ +#ifndef CBM_HELM_TEMPLATE_H +#define CBM_HELM_TEMPLATE_H + +#include "foundation/arena.h" + +#include "tree_sitter/api.h" +#include + +#include +#include + +/* The first document of YAML text src as a value of doc (Helm's scalar + * types); NULL when it does not parse. An empty document is an empty map. */ +yyjson_mut_val *cbm_helm_values_parse(yyjson_mut_doc *doc, const char *src, size_t len); + +/* The YAML node `node` of text `src` (already parsed, e.g. a HelmRelease's + * spec.values) as a value of doc; NULL when it holds an unknown part. */ +yyjson_mut_val *cbm_helm_values_node(yyjson_mut_doc *doc, const char *src, TSNode node); + +/* Helm's coalesce: a deep copy of `base` with `over` merged in -- maps merge + * key by key, any other value replaces, a null deletes. Either may be NULL. */ +yyjson_mut_val *cbm_helm_values_coalesce(yyjson_mut_doc *doc, yyjson_mut_val *base, + yyjson_mut_val *over); + +/* Helm --set NAME=VALUE into map `root`: NAME's `.` separates keys (`\.` + * escapes one), `[n]` indexes a list; VALUE is typed like --set (int, bool, + * null, else string) unless `as_string`. False when NAME is malformed. */ +bool cbm_helm_values_set(yyjson_mut_doc *doc, yyjson_mut_val *root, const char *name, + const char *value, bool as_string); + +/* A values scalar as a template prints it (a number like Go's %v: 1.10 -> + * 1.1) into out; false for a map, a list or null. */ +bool cbm_helm_scalar_text(yyjson_mut_val *v, char *out, size_t sz); + +/* A chart's metadata, as its templates see it under .Chart -- and where its + * templates are: .Template.BasePath as Helm sets it (`/templates`, + * the chart path its root's name and `/charts/` per subchart + * below it; NULL: not known) and the chart's directory as its template files + * are added to the defines ("" at the repository root). */ +typedef struct { + const char *name; + const char *version; + const char *app_version; + const char *base_path; + const char *dir; +} cbm_helm_chart_meta_t; + +/* The named templates (`define` / `block`) of one chart family -- a chart and + * the charts below it share one namespace; for one name the define of the + * file nearest the family root wins (Helm loads deeper files first), then + * the alphabetically first path. */ +typedef struct cbm_helm_defines cbm_helm_defines_t; + +cbm_helm_defines_t *cbm_helm_defines_new(void); +/* Register the defines of template file `rel` (text src, len). The text is + * copied; its parse lives as long as the set. */ +void cbm_helm_defines_add(cbm_helm_defines_t *d, const char *rel, const char *src, size_t len); +void cbm_helm_defines_free(cbm_helm_defines_t *d); + +/* A probe: the `image:` key on a template line. Each time rendering passes + * that line it captures the scalar written after the key. */ +typedef struct { + int line; /* 1-based line of the key in the template */ + int n; /* captures */ + int cap; /* room in image / keys (the render's own) */ + const char **image; /* the rendered scalar, YAML quoting removed; NULL: UNKNOWN */ + const char **keys; /* JSON array of the values paths it read; NULL: none */ +} cbm_helm_probe_t; + +/* Render template `rel` (text src, len) with `values` (its .Values; NULL: + * an empty map), `chart` and the family's defines, filling `probes` (strings + * in arena a, which gets only these: the render works in an arena of its own). + * Values paths are dot-joined keys, a key's own `.` escaped (`\.`), as --set + * writes them. False when the render ran past its step, output or memory + * budget -- a template Helm would not finish either (logged): then every + * capture is UNKNOWN. */ +bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, + yyjson_mut_val *values, const cbm_helm_chart_meta_t *chart, + const cbm_helm_defines_t *defs, cbm_helm_probe_t *probes, int nprobes); + +#endif /* CBM_HELM_TEMPLATE_H */ diff --git a/src/pipeline/lsp_resolve.h b/src/pipeline/lsp_resolve.h index 647b3f801..419f29e26 100644 --- a/src/pipeline/lsp_resolve.h +++ b/src/pipeline/lsp_resolve.h @@ -414,6 +414,43 @@ static inline bool cbm_pipeline_invocation_leaf_matches(const CBMResolvedCall *r call_leaf && strcmp(cbm_lsp_bare_segment(resolved->reason), call_leaf) == 0; } +/* #606: a sub-floor `php_method_typed_unindexed` row still proves the receiver + * is not a project class, so a weak short-name fallback must not bind it. A + * project class with the same simple name keeps the fallback, since the PHP + * LSP does not follow every cross-file type. */ +static inline bool cbm_pipeline_php_receiver_type_unindexed(const CBMResolvedCallArray *arr, + const CBMCall *call, + const cbm_registry_t *registry) { + if (!arr || !call || !call->enclosing_func_qn || !call->callee_name) { + return false; + } + for (int i = 0; i < arr->count; i++) { + const CBMResolvedCall *rc = &arr->items[i]; + if (rc->kind != CBM_RESOLVED_INVOCATION || !rc->caller_qn || !rc->callee_qn || + !rc->strategy || strcmp(rc->strategy, "php_method_typed_unindexed") != 0 || + strcmp(rc->caller_qn, call->enclosing_func_qn) != 0) { + continue; + } + int site_rank = cbm_pipeline_invocation_site_rank(rc, call); + if (site_rank == 0 || !cbm_pipeline_invocation_leaf_matches(rc, call, site_rank)) { + continue; + } + const char *method_dot = strrchr(rc->callee_qn, '.'); + if (!method_dot || method_dot == rc->callee_qn) { + continue; + } + char class_qn[CBM_SZ_512]; + size_t class_len = (size_t)(method_dot - rc->callee_qn); + if (class_len >= sizeof(class_qn)) { + continue; + } + memcpy(class_qn, rc->callee_qn, class_len); + class_qn[class_len] = '\0'; + return !cbm_registry_has_type_named(registry, cbm_lsp_bare_segment(class_qn)); + } + return false; +} + /* Look up the highest-confidence LSP-resolved call entry whose caller QN * matches the textual call's enclosing function and whose callee QN * short-name matches the textual callee. Returns a pointer into `arr` diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index f80be332e..61a72f141 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -1170,7 +1170,10 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM ctx->gbuf, ctx->project_name, rel)) || cbm_suppress_external_import_shadow(call->callee_name, res.strategy, imports, imp_keys, imp_count, cbm_pipeline_get_pkgmap(), - cbm_pipeline_get_nsmap()); + cbm_pipeline_get_nsmap()) || + /* MUST match pass_parallel.c exactly. */ + (lang == CBM_LANG_PHP && cbm_suppress_weak_member_match(true, true, res.strategy) && + cbm_pipeline_php_receiver_type_unindexed(lsp_calls, call, ctx->registry)); /* Service-pattern HTTP/ASYNC calls to an EXTERNAL client library (e.g. * `requests.get("/api/orders/{id}")`) resolve to a QN containing the library diff --git a/src/pipeline/pass_ci.c b/src/pipeline/pass_ci.c index 2fc9e0912..486b1a5b2 100644 --- a/src/pipeline/pass_ci.c +++ b/src/pipeline/pass_ci.c @@ -25,6 +25,7 @@ */ #include "foundation/constants.h" #include "foundation/compat.h" +#include "foundation/str_util.h" /* cbm_json_escape */ #include "pipeline/pipeline_internal.h" #include "cbm.h" #include "helpers.h" /* cbm_env_address */ @@ -783,9 +784,33 @@ static const char *ci_env_lookup(const ci_env_t *env, const char *name, size_t n return NULL; } +/* JSON array `list` (NULL: empty) with identifier `name` appended once. */ +static const char *ci_names_add(CBMArena *a, const char *list, const char *name, size_t nl) { + if (!list) { + return cbm_arena_sprintf(a, "[\"%.*s\"]", (int)nl, name); + } + char quoted[CBM_SZ_256]; + snprintf(quoted, sizeof(quoted), "\"%.*s\"", (int)nl, name); + if (strstr(list, quoted)) { + return list; + } + return cbm_arena_sprintf(a, "%.*s,%s]", (int)(strlen(list) - SKIP_ONE), list, quoted); +} + +static const char *ci_interp_marks(CBMArena *a, const char *s, const ci_env_t *env, + const char **dflt_of, const char **ovr_of); + /* `s` with ${X} / ${X:-d} / ${X-d} / $X expanded from env; NULL when a * variable has no value (and no default). */ static const char *ci_interp(CBMArena *a, const char *s, const ci_env_t *env) { + return ci_interp_marks(a, s, env, NULL, NULL); +} + +/* ci_interp; `dflt_of` (may be NULL) gains, as a JSON array, the variables + * whose inline default was used, `ovr_of` those with an inline default that + * env set instead. */ +static const char *ci_interp_marks(CBMArena *a, const char *s, const ci_env_t *env, + const char **dflt_of, const char **ovr_of) { if (!s || !strchr(s, '$')) { return s; } @@ -839,6 +864,11 @@ static const char *ci_interp(CBMArena *a, const char *s, const ci_env_t *env) { } use = dflt; ul = dl; + if (dflt_of) { + *dflt_of = ci_names_add(a, *dflt_of, name, nl); + } + } else if (dflt && ovr_of) { + *ovr_of = ci_names_add(a, *ovr_of, name, nl); } if (len + ul + SKIP_ONE > cap) { return NULL; @@ -1054,7 +1084,53 @@ static int ci_compose_env(const ci_t *c, TSNode svc, const ci_env_t *dotenv, con return k; } +/* A compose build's `args` (a mapping, or a list of "K=V" items), their + * values interpolated, as a JSON object; NULL: none. An argument without a + * value (the environment's), with an unknown one or with a secret (by name or + * value -- never kept in the graph, never an image) is left out. */ +static const char *ci_compose_build_args(const ci_t *c, TSNode build, const ci_env_t *env) { + TSNode args = ts_node_is_null(build) ? build : ci_get(c, build, "args"); + if (ts_node_is_null(args)) { + return NULL; + } + int n = 0; + TSNode *items = ci_items(c, args, &n); + ci_pair_t *pairs = items ? NULL : ci_pairs(c, args, &n); + const char *json = NULL; + for (int i = 0; i < n; i++) { + const char *key = NULL; + const char *raw = NULL; + if (items) { + const char *s = ci_scalar(c, items[i], NULL); + const char *eq = s ? strchr(s, '=') : NULL; + key = eq ? cbm_arena_strndup(c->a, s, (size_t)(eq - s)) : NULL; + raw = eq ? eq + SKIP_ONE : NULL; + } else { + key = pairs[i].key; + raw = ts_node_is_null(pairs[i].value) ? NULL : ci_scalar(c, pairs[i].value, NULL); + } + const char *value = raw ? ci_interp(c->a, raw, env) : NULL; + if (!key || !key[0] || !value || cbm_is_secret_binding(key, value)) { + continue; + } + /* JSON escaping writes at most 6 bytes a byte: nothing is cut */ + int kn = (int)(strlen(key) * 6 + SKIP_ONE); + int vn = (int)(strlen(value) * 6 + SKIP_ONE); + char *ek = cbm_arena_alloc(c->a, (size_t)kn); + char *ev = cbm_arena_alloc(c->a, (size_t)vn); + if (!ek || !ev) { + return NULL; + } + cbm_json_escape(ek, kn, key); + cbm_json_escape(ev, vn, value); + json = cbm_arena_sprintf(c->a, "%s%s\"%s\":\"%s\"", json ? json : "{", json ? "," : "", ek, + ev); + } + return json ? cbm_arena_sprintf(c->a, "%s}", json) : NULL; +} + static void ci_compose(const ci_t *c, TSNode root, const char *dir, const ci_env_t *env) { + static const ci_env_t no_env = {0}; int ns = 0; ci_pair_t *svcs = ci_pairs(c, ci_get(c, root, "services"), &ns); for (int i = 0; i < ns; i++) { @@ -1064,16 +1140,27 @@ static void ci_compose(const ci_t *c, TSNode root, const char *dir, const ci_env } int row = 0; const char *raw = ci_scalar(c, ci_get(c, svc, "image"), &row); - const char *image = raw ? ci_interp(c->a, raw, env) : NULL; - const char *df = ci_compose_dockerfile(c, ci_get(c, svc, "build"), dir, env); + const char *dflt_of = NULL; + const char *ovr_of = NULL; + const char *image = raw ? ci_interp_marks(c->a, raw, env, &dflt_of, &ovr_of) : NULL; + /* With .env replacing a default: what the defaults alone would name. */ + const char *default_image = image && ovr_of ? ci_interp(c->a, raw, &no_env) : NULL; + TSNode build = ci_get(c, svc, "build"); + const char *df = ci_compose_dockerfile(c, build, dir, env); if (c->site_fn && image && df) { - cbm_build_site_t site = { - .via = "compose", .image = image, .dockerfile = df, .line = row + SKIP_ONE}; + cbm_build_site_t site = {.via = "compose", + .image = image, + .dockerfile = df, + .build_args = ci_compose_build_args(c, build, env), + .line = row + SKIP_ONE}; c->site_fn(&site, c->ud); } if (c->service_fn) { cbm_compose_service_t s = {.name = svcs[i].key, .image = image, + .image_default_of = image ? dflt_of : NULL, + .image_override_of = image ? ovr_of : NULL, + .default_image = default_image, .dockerfile = df, .line = (int)ts_node_start_point(svc).row, .end_line = (int)ts_node_end_point(svc).row + SKIP_ONE}; diff --git a/src/pipeline/pass_docker.c b/src/pipeline/pass_docker.c index f5f303866..5d2c2840d 100644 --- a/src/pipeline/pass_docker.c +++ b/src/pipeline/pass_docker.c @@ -108,12 +108,31 @@ typedef struct { CBMHashTable *dirs; /* every directory holding a file (not the root) */ } dk_index_t; +/* Where a value came from: the ARGs whose default (`dflt`) or build argument + * (`ovr`) supplied part of it, as '\x1f'-separated names; NULL: none. */ +typedef struct { + const char *dflt; + const char *ovr; +} dk_prov_t; + typedef struct dk_var { const char *name; const char *value; /* NULL: declared without a value */ + bool arg; /* an ARG: a build argument can set it */ + dk_prov_t prov; struct dk_var *next; } dk_var_t; +/* An image reference of FROM / COPY --from, linked once the file is read. */ +typedef struct { + int64_t img; + const char *ref; + const char *via; + int stage; + int line; + dk_prov_t prov; +} dk_ref_t; + typedef struct { cbm_pipeline_ctx_t *ctx; CBMArena *a; @@ -131,6 +150,14 @@ typedef struct { dk_entry_t entry; /* the current stage's ENTRYPOINT */ dk_entry_t cmd; /* the current stage's CMD */ const dk_index_t *ix; + dk_ref_t *refs; /* FROM / COPY --from references, in file order */ + int nrefs; + int refs_cap; + /* A build's view (dk_build_args): its build arguments set the ARGs they + * name, and the references they change link from the Image it builds. */ + const dk_var_t *build_args; + int64_t built; /* 0: the Dockerfile's own view */ + const char *build_props; /* the build's tool / file / build_line members */ } dk_t; static bool dk_type(TSNode n, const char *t) { @@ -172,14 +199,72 @@ static const dk_var_t *dk_find(const dk_var_t *v, const char *name, size_t nl) { return NULL; } -static void dk_set(dk_t *d, dk_var_t **list, const char *name, const char *value) { +static void dk_set(dk_t *d, dk_var_t **list, const char *name, const char *value, bool arg, + dk_prov_t prov) { dk_var_t *v = cbm_arena_alloc(d->a, sizeof(*v)); if (v) { - *v = (dk_var_t){.name = name, .value = value, .next = *list}; + *v = (dk_var_t){.name = name, .value = value, .arg = arg, .prov = prov, .next = *list}; *list = v; } } +enum { DK_NAME_SEP = '\x1f' }; + +static bool dk_names_has(const char *list, const char *name, size_t nl) { + for (const char *p = list; p && *p;) { + size_t l = strcspn(p, "\x1f"); + if (l == nl && strncmp(p, name, nl) == 0) { + return true; + } + p += l; + p += *p ? SKIP_ONE : 0; + } + return false; +} + +/* `list` with `name` appended unless it holds it already. */ +static const char *dk_names_add(dk_t *d, const char *list, const char *name, size_t nl) { + if (dk_names_has(list, name, nl)) { + return list; + } + return list && list[0] ? cbm_arena_sprintf(d->a, "%s%c%.*s", list, DK_NAME_SEP, (int)nl, name) + : cbm_arena_strndup(d->a, name, nl); +} + +static const char *dk_names_union(dk_t *d, const char *a, const char *b) { + for (const char *p = b; p && *p;) { + size_t l = strcspn(p, "\x1f"); + a = dk_names_add(d, a, p, l); + p += l; + p += *p ? SKIP_ONE : 0; + } + return a; +} + +static void dk_prov_merge(dk_t *d, dk_prov_t *into, dk_prov_t from) { + into->dflt = dk_names_union(d, into->dflt, from.dflt); + into->ovr = dk_names_union(d, into->ovr, from.ovr); +} + +/* `,"key":["A","B"]` for a non-empty name list, else "". */ +static const char *dk_names_member(dk_t *d, const char *key, const char *list) { + if (!list || !list[0]) { + return ""; + } + const char *out = cbm_arena_sprintf(d->a, ",\"%s\":[", key); + for (const char *p = list; out && *p;) { + size_t l = strcspn(p, "\x1f"); + char esc[CBM_SZ_256]; + char name[CBM_SZ_256]; + snprintf(name, sizeof(name), "%.*s", (int)l, p); + cbm_json_escape(esc, (int)sizeof(esc), name); + out = cbm_arena_sprintf(d->a, "%s%s\"%s\"", out, p == list ? "" : ",", esc); + p += l; + p += *p ? SKIP_ONE : 0; + } + return out ? cbm_arena_sprintf(d->a, "%s]", out) : ""; +} + /* ASCII case-insensitive equality (stage names, `scratch`). */ static bool dk_ieq(const char *a, const char *b) { for (; *a && *b; a++, b++) { @@ -195,8 +280,10 @@ static bool dk_name_char(char c) { } /* `text` with $X / ${X} / ${X:-d} / ${X-d} expanded from `vars`; NULL when a - * reference has no value. */ -static const char *dk_expand(dk_t *d, const dk_var_t *vars, const char *text) { + * reference has no value. `prov` (may be NULL) gains the ARGs whose default + * or build argument supplied a value -- an inline `${X:-d}` default counts + * only for a declared ARG X, the one thing a build argument can set. */ +static const char *dk_expand(dk_t *d, const dk_var_t *vars, const char *text, dk_prov_t *prov) { size_t cap = strlen(text) + SKIP_ONE; size_t len = 0; char *out = cbm_arena_alloc(d->a, cap); @@ -249,6 +336,11 @@ static const char *dk_expand(dk_t *d, const dk_var_t *vars, const char *text) { return NULL; } const char *from = val ? val : dflt; + if (prov && v && val) { + dk_prov_merge(d, prov, v->prov); + } else if (prov && v && v->arg) { + prov->dflt = dk_names_add(d, prov->dflt, name, nl); + } if (len + vl + SKIP_ONE > cap) { size_t ncap = (len + vl + SKIP_ONE) * PAIR_LEN; char *grown = cbm_arena_alloc(d->a, ncap); @@ -370,6 +462,139 @@ static int dk_stage_of(const dk_t *d, const char *ref) { return found; } +/* A path component: [a-z0-9]+ joined by `.`, `_`, `__` or `-`+. */ +static bool dk_ref_component(const char *s, size_t n) { + if (n == 0 || !isalnum((unsigned char)s[0]) || !isalnum((unsigned char)s[n - SKIP_ONE])) { + return false; + } + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (isupper(c)) { + return false; + } + if (isalnum(c)) { + continue; + } + size_t run = SKIP_ONE; + while (i + run < n && s[i + run] == c) { + run++; + } + if (!((c == '.' && run == SKIP_ONE) || (c == '_' && run <= PAIR_LEN) || c == '-')) { + return false; + } + i += run - SKIP_ONE; + } + return true; +} + +/* A registry host: DNS labels (`.`-joined), or an IPv6 literal, with an + * optional :port. */ +static bool dk_ref_domain(const char *s, size_t n) { + const char *colon = memchr(s, ':', n); + if (n && s[0] == '[') { + const char *close = memchr(s, ']', n); + colon = close && (size_t)(close - s) + SKIP_ONE < n ? close + SKIP_ONE : NULL; + if (!close) { + return false; + } + } + size_t hl = colon ? (size_t)(colon - s) : n; + if (hl == 0) { + return false; + } + if (s[0] != '[') { + for (size_t i = 0; i < hl; i++) { + unsigned char c = (unsigned char)s[i]; + if (!isalnum(c) && c != '.' && c != '-') { + return false; + } + } + } + for (size_t i = hl + SKIP_ONE; colon && i < n; i++) { + if (!isdigit((unsigned char)s[i])) { + return false; + } + } + return !colon || hl + SKIP_ONE < n; +} + +bool cbm_image_ref_valid(const char *ref) { + if (!ref || !ref[0]) { + return false; + } + size_t n = strlen(ref); + const char *at = strchr(ref, '@'); + if (at) { + const char *colon = strchr(at, ':'); + if (!colon || colon == at + SKIP_ONE || strlen(colon + SKIP_ONE) < CBM_SZ_32) { + return false; + } + for (const char *p = at + SKIP_ONE; p < colon; p++) { + if (!isalnum((unsigned char)*p) && !strchr("+.-_", *p)) { + return false; + } + } + for (const char *p = colon + SKIP_ONE; *p; p++) { + if (!isxdigit((unsigned char)*p)) { + return false; + } + } + n = (size_t)(at - ref); + } + /* the tag: after the last ':' that follows the last '/' */ + size_t name_end = n; + for (size_t i = n; i > 0; i--) { + if (ref[i - SKIP_ONE] == '/') { + break; + } + if (ref[i - SKIP_ONE] == ':') { + name_end = i - SKIP_ONE; + break; + } + } + if (name_end < n) { + size_t tl = n - name_end - SKIP_ONE; + const char *tag = ref + name_end + SKIP_ONE; + if (tl == 0 || tl > CBM_SZ_128 || !(isalnum((unsigned char)tag[0]) || tag[0] == '_')) { + return false; + } + for (size_t i = 0; i < tl; i++) { + if (!isalnum((unsigned char)tag[i]) && !strchr("_.-", tag[i])) { + return false; + } + } + } + const char *p = ref; + const char *end = ref + name_end; + const char *slash = memchr(p, '/', (size_t)(end - p)); + if (slash) { + size_t fl = (size_t)(slash - p); + bool host = memchr(p, '.', fl) || memchr(p, ':', fl) || + (fl == strlen("localhost") && memcmp(p, "localhost", fl) == 0); + if (host) { + if (!dk_ref_domain(p, fl)) { + return false; + } + p = slash + SKIP_ONE; + } + } + if (p >= end) { + return false; + } + while (p < end) { + const char *s = memchr(p, '/', (size_t)(end - p)); + size_t cl = s ? (size_t)(s - p) : (size_t)(end - p); + if (!dk_ref_component(p, cl)) { + return false; + } + p += cl + (s ? SKIP_ONE : 0); + if (s && p >= end) { + return false; + } + } + return true; +} + int64_t cbm_pipeline_image_node(cbm_pipeline_ctx_t *ctx, const char *ref) { char repo[CBM_SZ_512]; char reg[CBM_SZ_512]; @@ -438,19 +663,101 @@ int64_t cbm_pipeline_env_setter(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t * return id; } -/* DEPENDS_ON from the Dockerfile to the Image of resolved reference `ref`. */ -static void dk_link(dk_t *d, const char *ref, const char *via, int line) { +/* Record the Image of resolved reference `ref`, linked by dk_refs_link. */ +static void dk_link(dk_t *d, const char *ref, const char *via, int line, dk_prov_t prov) { int64_t img = cbm_pipeline_image_node(d->ctx, ref); if (img <= 0) { return; } - char esc_ref[CBM_SZ_1K]; - cbm_json_escape(esc_ref, (int)sizeof(esc_ref), ref); - char props[CBM_SZ_2K]; - snprintf(props, sizeof(props), "{\"via\":\"%s\",\"image\":\"%s\",\"stage\":%d,\"line\":%d}", - via, esc_ref, d->stage, line); - cbm_gbuf_insert_edge(d->ctx->gbuf, d->module, img, "DEPENDS_ON", props); - d->edges++; + if (d->nrefs == d->refs_cap) { + int cap = d->refs_cap ? d->refs_cap * PAIR_LEN : DK_MAX_STAGES; + dk_ref_t *grown = cbm_arena_alloc(d->a, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + if (d->nrefs) { + memcpy(grown, d->refs, (size_t)d->nrefs * sizeof(*grown)); + } + d->refs = grown; + d->refs_cap = cap; + } + d->refs[d->nrefs++] = (dk_ref_t){img, ref, via, d->stage, line, prov}; +} + +/* The names a default supplies to the references of Image `img`: their union, + * or NULL when any of them names it without a default (a literal reference + * wins -- the edges to one Image are one edge, so every one of them carries + * the same mark). */ +static const char *dk_refs_defaults(dk_t *d, int64_t img) { + const char *names = NULL; + for (int i = 0; i < d->nrefs; i++) { + if (d->refs[i].img != img) { + continue; + } + if (!d->refs[i].prov.dflt) { + return NULL; + } + names = dk_names_union(d, names, d->refs[i].prov.dflt); + } + return names; +} + +/* The Dockerfile's own view: DEPENDS_ON from its Module to each Image FROM / + * COPY --from names {via, image, stage, line[, default_of: the ARGs whose + * defaults named it]}. */ +static void dk_refs_link(dk_t *d) { + for (int i = 0; i < d->nrefs; i++) { + const dk_ref_t *r = &d->refs[i]; + char esc_ref[CBM_SZ_1K]; + cbm_json_escape(esc_ref, (int)sizeof(esc_ref), r->ref); + const char *dflt = dk_names_member(d, "default_of", dk_refs_defaults(d, r->img)); + const char *props = + cbm_arena_sprintf(d->a, "{\"via\":\"%s\",\"image\":\"%s\",\"stage\":%d,\"line\":%d%s}", + r->via, esc_ref, r->stage, r->line, dflt); + if (props) { + cbm_gbuf_insert_edge(d->ctx->gbuf, d->module, r->img, "DEPENDS_ON", props); + d->edges++; + } + } +} + +/* A build's view: each Image a reference the build's arguments changed names + * -- the first reference to it, its build arguments unioned -- + * DEPENDS_ON from the Image the build makes {via, image, override_of[, + * default_of], dockerfile, stage, line, tool, file, build_line}. A reference + * no build argument reached is the Dockerfile's own edge already. */ +static void dk_refs_link_build(dk_t *d) { + for (int i = 0; i < d->nrefs; i++) { + const dk_ref_t *r = &d->refs[i]; + bool first = r->prov.ovr != NULL && r->img != d->built; + for (int j = 0; first && j < i; j++) { + first = !(d->refs[j].img == r->img && d->refs[j].prov.ovr); + } + if (!first) { + continue; + } + dk_prov_t all = {0}; + for (int j = i; j < d->nrefs; j++) { + if (d->refs[j].img == r->img && d->refs[j].prov.ovr) { + dk_prov_merge(d, &all, d->refs[j].prov); + } + } + char esc_ref[CBM_SZ_1K]; + char esc_file[CBM_SZ_1K]; + cbm_json_escape(esc_ref, (int)sizeof(esc_ref), r->ref); + cbm_json_escape(esc_file, (int)sizeof(esc_file), d->file); + const char *props = cbm_arena_sprintf( + d->a, + "{\"via\":\"%s\",\"image\":\"%s\"%s%s,\"dockerfile\":\"%s\",\"stage\":%d," + "\"line\":%d,%s}", + r->via, esc_ref, dk_names_member(d, "override_of", all.ovr), + dk_names_member(d, "default_of", all.dflt), esc_file, r->stage, r->line, + d->build_props); + if (props) { + cbm_gbuf_insert_edge(d->ctx->gbuf, d->built, r->img, "DEPENDS_ON", props); + d->edges++; + } + } } static int dk_line(TSNode n) { @@ -467,19 +774,33 @@ static void dk_arg(dk_t *d, TSNode ins) { if (!name) { return; } - if (d->stage < 0) { - dk_set(d, &d->global, name, ts_node_is_null(df) ? NULL : dk_value(d, df)); + size_t nl = strlen(name); + dk_var_t **list = d->stage < 0 ? &d->global : &d->local; + /* A build argument sets the ARG it names, in every stage declaring it, + * whatever its default. */ + const dk_var_t *set = dk_find(d->build_args, name, nl); + if (set) { + dk_set(d, list, name, set->value, true, + (dk_prov_t){.ovr = dk_names_add(d, NULL, name, nl)}); return; } const char *value = NULL; + dk_prov_t prov = {0}; if (!ts_node_is_null(df)) { + /* A default may name ARGs declared before it (ARG BASE=golang:${GO}). */ const char *raw = dk_value(d, df); - value = raw ? dk_expand(d, d->local, raw) : NULL; - } else { - const dk_var_t *g = dk_find(d->global, name, strlen(name)); - value = g ? g->value : NULL; + value = raw ? dk_expand(d, *list, raw, &prov) : NULL; + if (value) { + prov.dflt = dk_names_add(d, prov.dflt, name, nl); + } + } else if (d->stage >= 0) { + const dk_var_t *g = dk_find(d->global, name, nl); + if (g) { + value = g->value; + prov = g->prov; + } } - dk_set(d, &d->local, name, value); + dk_set(d, list, name, value, true, prov); } static void dk_env(dk_t *d, TSNode ins) { @@ -498,7 +819,9 @@ static void dk_env(dk_t *d, TSNode ins) { } const char *raw = ts_node_is_null(vl) ? NULL : dk_value(d, vl); const char *name = dk_text(d, nm); - dk_set(d, &d->local, name, raw ? dk_expand(d, d->local, raw) : NULL); + dk_prov_t prov = {0}; + const char *value = raw ? dk_expand(d, d->local, raw, &prov) : NULL; + dk_set(d, &d->local, name, value, false, prov); dk_envkey_t *k = d->stage < DK_MAX_STAGES ? cbm_arena_alloc(d->a, sizeof(*k)) : NULL; if (k && name) { *k = (dk_envkey_t){name, dk_line(pair), d->st[d->stage].envs}; @@ -515,7 +838,9 @@ static void dk_from(dk_t *d, TSNode ins) { spec = ts_node_named_child(ins, i); } } - const char *ref = ts_node_is_null(spec) ? NULL : dk_expand(d, d->global, dk_text(d, spec)); + dk_prov_t prov = {0}; + const char *ref = + ts_node_is_null(spec) ? NULL : dk_expand(d, d->global, dk_text(d, spec), &prov); d->stage++; d->local = NULL; d->workdir = "/"; @@ -523,7 +848,7 @@ static void dk_from(dk_t *d, TSNode ins) { d->cmd.set = false; bool stage = ref && dk_is_stage(d, ref); if (ref && !stage) { - dk_link(d, ref, "from", dk_line(ins)); + dk_link(d, ref, "from", dk_line(ins), prov); } if (d->stage < DK_MAX_STAGES) { d->st[d->stage].parent = stage ? dk_stage_of(d, ref) : -1; @@ -605,7 +930,7 @@ static int dk_copy_json(dk_t *d, const char *p, const char **out, int max) { if (yyjson_is_arr(arr)) { for (yyjson_arr_iter_init(arr, &it); n < max && (v = yyjson_arr_iter_next(&it));) { const char *s = yyjson_get_str(v); - out[n++] = s ? dk_expand(d, d->local, cbm_arena_strdup(d->a, s)) : NULL; + out[n++] = s ? dk_expand(d, d->local, cbm_arena_strdup(d->a, s), NULL) : NULL; } } if (doc) { @@ -639,11 +964,12 @@ static void dk_copy(dk_t *d, TSNode ins) { if (l > sizeof(FROM) - SKIP_ONE && strncmp(p, FROM, sizeof(FROM) - SKIP_ONE) == 0) { const char *v = cbm_arena_strndup(d->a, p + sizeof(FROM) - SKIP_ONE, l - (sizeof(FROM) - SKIP_ONE)); - const char *ref = v ? dk_expand(d, d->local, v) : NULL; + dk_prov_t prov = {0}; + const char *ref = v ? dk_expand(d, d->local, v, &prov) : NULL; c->has_from = true; c->from = ref; if (ref && ref[0] && !dk_is_stage(d, ref)) { - dk_link(d, ref, "copy-from", dk_line(ins)); + dk_link(d, ref, "copy-from", dk_line(ins), prov); } } p += l; @@ -652,7 +978,7 @@ static void dk_copy(dk_t *d, TSNode ins) { while (!json && *p && nw < DK_MAX_WORDS) { size_t l = strcspn(p, " \t"); const char *w = cbm_arena_strndup(d->a, p, l); - words[nw++] = w && strncmp(w, "<<", PAIR_LEN) != 0 ? dk_expand(d, d->local, w) : NULL; + words[nw++] = w && strncmp(w, "<<", PAIR_LEN) != 0 ? dk_expand(d, d->local, w, NULL) : NULL; p += l; p += strspn(p, " \t"); } @@ -1023,8 +1349,8 @@ static void dk_go_builds(dk_t *d, const char **w, int n) { if (!b) { return; } - *b = (dk_build_t){.out = out ? dk_expand(d, d->local, out) : NULL, - .pkg = dk_expand(d, d->local, pkg ? pkg : "."), + *b = (dk_build_t){.out = out ? dk_expand(d, d->local, out, NULL) : NULL, + .pkg = dk_expand(d, d->local, pkg ? pkg : ".", NULL), .workdir = d->workdir, .next = d->st[d->stage].builds}; d->st[d->stage].builds = b; @@ -1066,7 +1392,7 @@ static void dk_workdir(dk_t *d, TSNode ins) { continue; } const char *raw = dk_text(d, pn); - const char *v = raw ? dk_expand(d, d->local, raw) : NULL; + const char *v = raw ? dk_expand(d, d->local, raw, NULL) : NULL; const char *w = v ? v : raw; if (w) { d->workdir = dk_absolute(d, w, d->workdir); @@ -1431,11 +1757,31 @@ static void dk_file(dk_t *d, TSNode root) { dk_entry(d, ins, &d->cmd); } } + dk_refs_link(d); dk_spawn(d); dk_envs(d); dk_image_set(d); } +/* A build's view of the Dockerfile: only what decides its image references + * (ARG, ENV, FROM, COPY --from) is read, under the build's arguments. */ +static void dk_file_build(dk_t *d, TSNode root) { + for (uint32_t i = 0; i < ts_node_named_child_count(root); i++) { + TSNode ins = ts_node_named_child(root, i); + if (dk_type(ins, "arg_instruction")) { + dk_arg(d, ins); + } else if (dk_type(ins, "env_instruction")) { + dk_env(d, ins); + } else if (dk_type(ins, "from_instruction")) { + dk_from(d, ins); + } else if ((dk_type(ins, "copy_instruction") || dk_type(ins, "add_instruction")) && + d->stage >= 0) { + dk_copy(d, ins); + } + } + dk_refs_link_build(d); +} + static char *dk_read(const char *path, size_t *len) { FILE *f = cbm_fopen(path, "rb"); if (!f) { @@ -1545,12 +1891,15 @@ static void dk_build_site(const cbm_build_site_t *s, void *ud) { char esc_img[CBM_SZ_1K]; cbm_json_escape(esc_file, (int)sizeof(esc_file), b->file); cbm_json_escape(esc_img, (int)sizeof(esc_img), s->image); - char props[CBM_SZ_4K]; - snprintf(props, sizeof(props), - "{\"via\":\"build\",\"tool\":\"%s\",\"file\":\"%s\",\"image\":\"%s\",\"line\":%d}", - s->via, esc_file, esc_img, s->line); - cbm_gbuf_insert_edge(b->ctx->gbuf, img, mod->id, "DEPENDS_ON", props); - b->edges++; + const char *props = cbm_arena_sprintf( + b->a, + "{\"via\":\"build\",\"tool\":\"%s\",\"file\":\"%s\",\"image\":\"%s\",\"line\":%d%s%s}", + s->via, esc_file, esc_img, s->line, s->build_args ? ",\"build_args\":" : "", + s->build_args ? s->build_args : ""); + if (props) { + cbm_gbuf_insert_edge(b->ctx->gbuf, img, mod->id, "DEPENDS_ON", props); + b->edges++; + } } typedef struct { @@ -1599,6 +1948,41 @@ static void dk_compose_connects(dk_compose_t *c, const cbm_compose_service_t *s, * DEFINES; it DEPENDS_ON the services it depends on / links, USAGE * {ref:image} its image, DEPENDS_ON {via:build} the Dockerfile it builds and * CONNECTS_TO the services its environment addresses. */ +/* The image a service runs: USAGE {ref: image, image[, default_of]} -- the + * variables whose inline default named it. When the project's .env replaced + * a default, the edge carries {override_of, override_source: ".env"}, and + * the image the defaults alone name is kept: on the same Image as + * `default_image`, else its own USAGE {ref: image_default, default_of}, a + * reference the service does not run. */ +static void dk_compose_image(dk_compose_t *c, const cbm_compose_service_t *s, int64_t self) { + int64_t img = cbm_pipeline_image_node(c->ctx, s->image); + int64_t dimg = s->default_image ? cbm_pipeline_image_node(c->ctx, s->default_image) : 0; + char esc[CBM_SZ_1K]; + char desc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), s->image); + cbm_json_escape(desc, (int)sizeof(desc), s->default_image ? s->default_image : ""); + const char *props = + cbm_arena_sprintf(c->a, "{\"ref\":\"image\",\"image\":\"%s\"%s%s%s%s%s%s%s}", esc, + s->image_default_of ? ",\"default_of\":" : "", + s->image_default_of ? s->image_default_of : "", + s->image_override_of ? ",\"override_of\":" : "", + s->image_override_of ? s->image_override_of : "", + s->image_override_of ? ",\"override_source\":\".env\"" : "", + dimg > 0 && dimg == img ? ",\"default_image\":\"" : "", + dimg > 0 && dimg == img ? cbm_arena_sprintf(c->a, "%s\"", desc) : ""); + if (props) { + dk_compose_edge(c, self, img, "USAGE", props); + } + if (dimg > 0 && dimg != img && s->image_override_of) { + const char *dprops = cbm_arena_sprintf( + c->a, "{\"ref\":\"image_default\",\"image\":\"%s\",\"default_of\":%s}", desc, + s->image_override_of); + if (dprops) { + dk_compose_edge(c, self, dimg, "USAGE", dprops); + } + } +} + static void dk_compose_service(const cbm_compose_service_t *s, void *ud) { dk_compose_t *c = ud; const char *qn = dk_compose_qn(c, s->name); @@ -1635,11 +2019,7 @@ static void dk_compose_service(const cbm_compose_service_t *s, void *ud) { } dk_compose_connects(c, s, self->id); if (s->image) { - char esc[CBM_SZ_1K]; - cbm_json_escape(esc, (int)sizeof(esc), s->image); - char props[CBM_SZ_2K]; - snprintf(props, sizeof(props), "{\"ref\":\"image\",\"image\":\"%s\"}", esc); - dk_compose_edge(c, self->id, cbm_pipeline_image_node(c->ctx, s->image), "USAGE", props); + dk_compose_image(c, s, self->id); } if (s->dockerfile) { const char *mqn = cbm_fqn_module(c->a, c->ctx->project_name, s->dockerfile); @@ -1706,6 +2086,152 @@ static int dk_link_builds(cbm_pipeline_ctx_t *ctx, CBMArena *a) { return b.edges; } +static bool dk_prop(const char *props, const char *key, char *out, size_t out_sz); +static void *dk_grow(CBMArena *a, void *v, int n, int *cap, size_t size); + +/* A build that passes build arguments (compose build.args, docker build + * --build-arg; on its build edge as `build_args`). */ +typedef struct { + int64_t img; + int64_t module; + const char *dockerfile; + const dk_var_t *args; + const char *props; /* "tool":..,"file":..,"build_line":N */ +} dk_argbuild_t; + +/* The build arguments of build edge properties `props` (NULL: none). */ +static const dk_var_t *dk_build_args_of(CBMArena *a, const char *props) { + if (!props || !strstr(props, "\"build_args\"")) { + return NULL; + } + yyjson_doc *doc = yyjson_read(props, strlen(props), 0); + yyjson_val *args = doc ? yyjson_obj_get(yyjson_doc_get_root(doc), "build_args") : NULL; + dk_var_t *list = NULL; + yyjson_obj_iter it; + yyjson_val *k; + if (yyjson_is_obj(args)) { + yyjson_obj_iter_init(args, &it); + while ((k = yyjson_obj_iter_next(&it))) { + const char *v = yyjson_get_str(yyjson_obj_iter_get_val(k)); + dk_var_t *var = v ? cbm_arena_alloc(a, sizeof(*var)) : NULL; + if (var) { + *var = (dk_var_t){.name = cbm_arena_strdup(a, yyjson_get_str(k)), + .value = cbm_arena_strdup(a, v), + .arg = true, + .next = list}; + list = var; + } + } + } + if (doc) { + yyjson_doc_free(doc); + } + return list; +} + +/* Every build edge (Image -DEPENDS_ON {via: build}-> Dockerfile) that passes + * build arguments, into out (arena); returns how many. */ +static int dk_argbuilds(cbm_pipeline_ctx_t *ctx, CBMArena *a, dk_argbuild_t **out) { + const cbm_gbuf_node_t **imgs = NULL; + int ni = 0; + *out = NULL; + if (cbm_gbuf_find_by_label(ctx->gbuf, "Image", &imgs, &ni) != 0) { + return 0; + } + int n = 0; + int cap = 0; + for (int i = 0; i < ni; i++) { + const cbm_gbuf_edge_t **es = NULL; + int ne = 0; + if (cbm_gbuf_find_edges_by_source_type(ctx->gbuf, imgs[i]->id, "DEPENDS_ON", &es, &ne) != + 0) { + continue; + } + for (int j = 0; j < ne; j++) { + const char *p = es[j]->properties_json; + char via[CBM_SZ_32]; + const dk_var_t *args = NULL; + if (!dk_prop(p, "via", via, sizeof(via)) || strcmp(via, "build") != 0 || + !(args = dk_build_args_of(a, p))) { + continue; + } + const cbm_gbuf_node_t *mod = cbm_gbuf_find_by_id(ctx->gbuf, es[j]->target_id); + char tool[CBM_SZ_64]; + char file[CBM_SZ_1K]; + char esc[CBM_SZ_2K]; + if (!mod || !mod->file_path || !dk_prop(p, "tool", tool, sizeof(tool)) || + !dk_prop(p, "file", file, sizeof(file))) { + continue; + } + const char *line = strstr(p, "\"line\":"); + if (n == cap) { + *out = dk_grow(a, *out, n, &cap, sizeof(**out)); + if (!*out) { + return 0; + } + } + cbm_json_escape(esc, (int)sizeof(esc), file); + (*out)[n++] = (dk_argbuild_t){ + .img = imgs[i]->id, + .module = mod->id, + .dockerfile = cbm_arena_strdup(a, mod->file_path), + .args = args, + .props = cbm_arena_sprintf(a, "\"tool\":\"%s\",\"file\":\"%s\",\"build_line\":%d", + tool, esc, line ? atoi(line + strlen("\"line\":")) : 0)}; + } + } + return n; +} + +/* Each build passing build arguments reads its Dockerfile in its own view + * (dk_file_build): what those arguments change, it builds FROM. Image-sourced + * DEPENDS_ON edges, recomputed whole every run (CBM_BUILD_EDGE_SOURCE). */ +static int dk_link_build_args(cbm_pipeline_ctx_t *ctx, CBMArena *a) { + dk_argbuild_t *bs = NULL; + int n = ctx->repo_path ? dk_argbuilds(ctx, a, &bs) : 0; + if (n == 0) { + return 0; + } + TSParser *parser = ts_parser_new(); + if (!parser || !ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_DOCKERFILE))) { + if (parser) { + ts_parser_delete(parser); + } + return 0; + } + int edges = 0; + for (int i = 0; i < n; i++) { + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, bs[i].dockerfile); + size_t len = 0; + char *src = dk_read(path, &len); + if (!src) { + continue; + } + dk_uncomment(src, len); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len); + if (tree) { + dk_t d = {.ctx = ctx, + .a = a, + .src = src, + .file = bs[i].dockerfile, + .module = bs[i].module, + .stage = -1, + .dir = "", + .workdir = "/", + .build_args = bs[i].args, + .built = bs[i].img, + .build_props = bs[i].props}; + dk_file_build(&d, ts_tree_root_node(tree)); + edges += d.edges; + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + ts_parser_delete(parser); + return edges; +} + static bool dk_node_before(const cbm_gbuf_node_t *a, const cbm_gbuf_node_t *b) { int c = strcmp(a->file_path, b->file_path); return c < 0 || (c == 0 && a->start_line < b->start_line); @@ -1891,17 +2417,19 @@ static void dk_hub_push(CBMArena *a, dk_hub_edges_t *l, const cbm_gbuf_edge_t *e * USAGE {ref:image}, or DEPENDS_ON {via:from|copy-from}? */ static bool dk_hub_ref(const char *type, const char *p) { char v[CBM_SZ_32]; - if (strcmp(type, "USAGE") == 0) { + if (strcmp(type, "USAGE") == 0 || strcmp(type, "DEPLOYS") == 0) { return dk_prop(p, "ref", v, sizeof(v)) && strcmp(v, "image") == 0; } return dk_prop(p, "via", v, sizeof(v)) && (strcmp(v, "from") == 0 || strcmp(v, "copy-from") == 0); } -/* The edges of Image `img`: build edges out (DEPENDS_ON {via:build}) and - * references in, each with the tag of its `image` reference. */ +/* The edges of Image `img`: build edges out (DEPENDS_ON {via:build}), the + * base images a build's arguments chose out (DEPENDS_ON {via: from | + * copy-from, build_line}: they follow their build) and references in, each + * with the tag of its `image` reference. */ static void dk_hub_collect(cbm_pipeline_ctx_t *ctx, CBMArena *a, int64_t img, - dk_hub_edges_t *builds, dk_hub_edges_t *refs) { + dk_hub_edges_t *builds, dk_hub_edges_t *refs, dk_hub_edges_t *follows) { const cbm_gbuf_edge_t **es = NULL; int n = 0; char ref[CBM_SZ_1K]; @@ -1909,8 +2437,13 @@ static void dk_hub_collect(cbm_pipeline_ctx_t *ctx, CBMArena *a, int64_t img, if (cbm_gbuf_find_edges_by_source_type(ctx->gbuf, img, "DEPENDS_ON", &es, &n) == 0) { for (int i = 0; i < n; i++) { char via[CBM_SZ_32]; - if (!dk_prop(es[i]->properties_json, "via", via, sizeof(via)) || - strcmp(via, "build") != 0) { + if (!dk_prop(es[i]->properties_json, "via", via, sizeof(via))) { + continue; + } + if (strcmp(via, "build") != 0) { + if (strstr(es[i]->properties_json, "\"build_line\":")) { + dk_hub_push(a, follows, es[i], ""); + } continue; } tag[0] = '\0'; @@ -1921,7 +2454,7 @@ static void dk_hub_collect(cbm_pipeline_ctx_t *ctx, CBMArena *a, int64_t img, } atomic_fetch_add_explicit(&g_dk_hub_edges, (uint64_t)n, memory_order_relaxed); } - static const char *const REF_TYPES[] = {"USAGE", "DEPENDS_ON"}; + static const char *const REF_TYPES[] = {"USAGE", "DEPENDS_ON", "DEPLOYS"}; for (size_t t = 0; t < sizeof(REF_TYPES) / sizeof(REF_TYPES[0]); t++) { if (cbm_gbuf_find_edges_by_target_type(ctx->gbuf, img, REF_TYPES[t], &es, &n) != 0) { continue; @@ -2122,6 +2655,27 @@ static int dk_hub_move(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_hub_edge_t return SKIP_ONE; } +static int dk_prop_int(const char *props, const char *key) { + char pat[CBM_SZ_64]; + snprintf(pat, sizeof(pat), "\"%s\":", key); + const char *p = props ? strstr(props, pat) : NULL; + return p ? atoi(p + strlen(pat)) : -SKIP_ONE; +} + +/* Does base-image edge `f` come from build edge `b`: one source, and its + * tool / file / build_line those of the build? */ +static bool dk_hub_follows(const dk_hub_edge_t *f, const dk_hub_edge_t *b) { + char ft[CBM_SZ_64]; + char bt[CBM_SZ_64]; + char ff[CBM_SZ_1K]; + char bf[CBM_SZ_1K]; + return f->src == b->src && dk_prop(f->props, "tool", ft, sizeof(ft)) && + dk_prop(b->props, "tool", bt, sizeof(bt)) && strcmp(ft, bt) == 0 && + dk_prop(f->props, "file", ff, sizeof(ff)) && dk_prop(b->props, "file", bf, sizeof(bf)) && + strcmp(ff, bf) == 0 && + dk_prop_int(f->props, "build_line") == dk_prop_int(b->props, "line"); +} + /* One repository's Images (imgs[0..n)): split it by component when it is * built from several Dockerfiles, else keep (or bring back) everything on * the repository Image. Returns the edges moved. */ @@ -2129,10 +2683,11 @@ static int dk_hub_repo(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_img_t *img dk_drop_t *drop) { dk_hub_edges_t builds = {0}; dk_hub_edges_t refs = {0}; + dk_hub_edges_t follows = {0}; bool has_sub = false; for (int i = 0; i < n; i++) { has_sub = has_sub || strcmp(imgs[i].name, imgs[i].repo) != 0; - dk_hub_collect(ctx, a, imgs[i].id, &builds, &refs); + dk_hub_collect(ctx, a, imgs[i].id, &builds, &refs, &follows); } const char **comps = NULL; size_t pre = 0; @@ -2158,6 +2713,11 @@ static int dk_hub_repo(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_img_t *img int k = builds.v[i].comp; int64_t src = sub && k >= 0 && sub[k] > 0 ? sub[k] : repo_img; moved += dk_hub_move(ctx, a, &builds.v[i], src, builds.v[i].tgt, drop); + for (int f = 0; f < follows.n; f++) { + if (dk_hub_follows(&follows.v[f], &builds.v[i])) { + moved += dk_hub_move(ctx, a, &follows.v[f], src, follows.v[f].tgt, drop); + } + } } for (int i = 0; i < refs.n; i++) { int k = sub ? dk_hub_match(refs.v[i].tag, comps, nc) : -1; @@ -2274,6 +2834,7 @@ void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fi cbm_ht_free(ix.dirs); } edges += dk_link_builds(ctx, &arena); + edges += dk_link_build_args(ctx, &arena); edges += dk_image_hubs(ctx, &arena); cbm_arena_destroy(&arena); if (edges > 0) { diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index a9e92acc7..034e8984c 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -20,10 +20,12 @@ #include "foundation/compat_fs.h" #include "foundation/limits.h" #include "cbm.h" -#include "helpers.h" /* cbm_kind_in_set_free_cache */ -#include "foundation/slab_alloc.h" /* cbm_slab_destroy_thread */ -#include "foundation/platform.h" /* cbm_default_worker_count */ -#include "foundation/str_util.h" /* cbm_json_escape */ +#include "helpers.h" /* cbm_kind_in_set_free_cache */ +#include "foundation/slab_alloc.h" /* cbm_slab_destroy_thread */ +#include "foundation/platform.h" /* cbm_default_worker_count */ +#include "foundation/str_util.h" /* cbm_json_escape */ +#include "foundation/json_writer.h" /* cbm_json_writer_allocator */ +#include "pipeline/helm_template.h" #include "pipeline/worker_pool.h" #include @@ -1103,9 +1105,7 @@ static void k8s_bundles_free(k8s_bundles_t *b) { } /* A Helm chart's directory bundles the plain manifests below it. */ -static void k8s_chart_dir_add(k8s_bundles_t *b, const char *chart_yaml) { - char dir[CBM_SZ_1K]; - k8s_dirname(chart_yaml, dir, sizeof(dir)); +static void k8s_chart_dir_register(k8s_bundles_t *b, const char *dir) { if (b->charts && !cbm_ht_get(b->charts, dir)) { char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir); if (owned) { @@ -1114,6 +1114,12 @@ static void k8s_chart_dir_add(k8s_bundles_t *b, const char *chart_yaml) { } } +static void k8s_chart_dir_add(k8s_bundles_t *b, const char *chart_yaml) { + char dir[CBM_SZ_1K]; + k8s_dirname(chart_yaml, dir, sizeof(dir)); + k8s_chart_dir_register(b, dir); +} + /* Build the kustomize bundles (call once every manifest and kustomization is * known and the IMPORTS edges exist). */ static void k8s_bundles_build(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundles_t *b) { @@ -1285,6 +1291,9 @@ typedef struct { int64_t node; char *image; char *container; /* NULL: unnamed */ + int line; /* the image's line */ + int start_line; /* the container's lines */ + int end_line; } k8s_img_t; /* A patch a kustomization states (extract_k8s.c PATCH fact): what it names @@ -1482,8 +1491,12 @@ static void k8s_facts_keep(k8s_facts_t *f, const CBMFileResult *res, const int64 } else if (x->fact == CBM_K8S_FACT_IMAGE) { if (x->name && k8s_grow((void **)&f->imgs, &f->img_cap, f->img_n, sizeof(*f->imgs))) { f->imgs[f->img_n++] = (k8s_img_t){ - node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), - x->container ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->container) : NULL}; + node, + cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), + x->container ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->container) : NULL, + x->line, + x->start_line, + x->end_line}; } } else if (x->fact == CBM_K8S_FACT_ENV) { if (x->name && k8s_grow((void **)&f->envs, &f->env_cap, f->env_n, sizeof(*f->envs))) { @@ -2772,6 +2785,27 @@ static void k8s_link_local_deps(cbm_pipeline_ctx_t *ctx, k8s_local_deps_t *deps) * `file://` dependency (once all charts are read), otherwise to the external * Chart node of the dependency's repository and name, the repository also on * the edge. */ +/* The Chart node `name` defined by chart file `rel_path` (a Chart.yaml, a + * werf.yaml), its File node DEFINES it; 0 on failure. */ +static int64_t k8s_chart_upsert(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *name, + const char *props) { + char *chart_qn = cbm_infra_qn(ctx->project_name, rel_path, "helm-chart", NULL); + if (!chart_qn) { + return 0; + } + int64_t chart_id = + cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, chart_qn, rel_path, SKIP_ONE, 0, props); + free(chart_qn); + + char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel_path, "__file__"); + const cbm_gbuf_node_t *file_node = file_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, file_qn) : NULL; + if (file_node && chart_id > 0) { + cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, chart_id, "DEFINES", "{}"); + } + free(file_qn); + return chart_id > 0 ? chart_id : 0; +} + static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, k8s_local_deps_t *local_deps) { cbm_helm_chart_t hc; @@ -2781,24 +2815,14 @@ static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, con } const char *cname = hc.chart_name[0] ? hc.chart_name : k8s_basename(rel_path); - char *chart_qn = cbm_infra_qn(ctx->project_name, rel_path, "helm-chart", NULL); - if (!chart_qn) { + int64_t chart_id = k8s_chart_upsert(ctx, rel_path, cname, "{\"source\":\"helm\"}"); + if (chart_id <= 0) { cbm_helm_chart_free(&hc); return; } - int64_t chart_id = cbm_gbuf_upsert_node(ctx->gbuf, "Chart", cname, chart_qn, rel_path, SKIP_ONE, - 0, "{\"source\":\"helm\"}"); - free(chart_qn); - - char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel_path, "__file__"); - const cbm_gbuf_node_t *file_node = file_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, file_qn) : NULL; - if (file_node && chart_id > 0) { - cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, chart_id, "DEFINES", "{}"); - } - free(file_qn); int dep_edges = 0; - for (int i = 0; i < hc.dep_count && chart_id > 0; i++) { + for (int i = 0; i < hc.dep_count; i++) { const cbm_helm_dep_t *dep = &hc.deps[i]; if (k8s_is_file_repository(dep->repository)) { k8s_local_deps_push(local_deps, chart_id, rel_path, ctx->project_name, dep); @@ -2812,6 +2836,106 @@ static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, con cbm_log_info("pass.k8s.helm", "file", rel_path, "deps", itoa_k8s(dep_edges)); } +/* ── werf chart handler ──────────────────────────────────────────── */ + +/* werf (werf.io) deploys the Helm chart in the `.helm` directory beside its + * werf.yaml; that chart needs no Chart.yaml, werf names it after its project. + * A `.helm` with a Chart.yaml is an ordinary chart. A custom + * deploy.helmChartDir is not followed. */ +static bool is_werf_file(const char *base) { + return strcmp(base, "werf.yaml") == 0 || strcmp(base, "werf.yml") == 0; +} + +bool cbm_werf_helm_dir(const char *repo_path, const char *dir) { + const char *base = k8s_basename(dir); + if (!repo_path || strcmp(base, ".helm") != 0) { + return false; + } + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", repo_path, dir); + if (!cbm_is_dir(path)) { + return false; + } + static const char *const charts[] = {"Chart.yaml", "Chart.yml"}; + for (size_t i = 0; i < sizeof(charts) / sizeof(charts[0]); i++) { + snprintf(path, sizeof(path), "%s/%s/%s", repo_path, dir, charts[i]); + if (cbm_file_exists(path)) { + return false; + } + } + size_t parent = (size_t)(base - dir); + static const char *const werfs[] = {"werf.yaml", "werf.yml"}; + for (size_t i = 0; i < sizeof(werfs) / sizeof(werfs[0]); i++) { + snprintf(path, sizeof(path), "%s/%.*s%s", repo_path, (int)parent, dir, werfs[i]); + if (cbm_file_exists(path)) { + return true; + } + } + return false; +} + +/* The literal top-level `project:` of a werf.yaml's first document into out, + * "" when there is none or it is templated. */ +static void k8s_werf_project(const char *src, char *out, size_t out_sz) { + out[0] = '\0'; + bool seen_key = false; + for (const char *p = src; p && *p;) { + const char *eol = strchr(p, '\n'); + size_t len = eol ? (size_t)(eol - p) : strlen(p); + if (len >= K8S_SLEN("---") && strncmp(p, "---", K8S_SLEN("---")) == 0 && seen_key) { + return; + } + if (isalpha((unsigned char)p[0])) { + seen_key = true; + } + if (len > K8S_SLEN("project:") && strncmp(p, "project:", K8S_SLEN("project:")) == 0) { + /* the value: [b, e) of the line, a comment and quotes removed */ + size_t b = K8S_SLEN("project:"); + size_t e = len; + while (b < e && (p[b] == ' ' || p[b] == '\t')) { + b++; + } + for (size_t i = b; i < e; i++) { + if (p[i] == '#') { + e = i; + } + } + while (e > b && isspace((unsigned char)p[e - SKIP_ONE])) { + e--; + } + if (e >= b + PAIR_LEN && (p[b] == '"' || p[b] == '\'') && p[e - SKIP_ONE] == p[b]) { + b++; + e--; + } + if (e > b && e - b < out_sz && !memchr(p + b, '{', e - b)) { + memcpy(out, p + b, e - b); + out[e - b] = '\0'; + } + return; + } + p = eol ? eol + SKIP_ONE : NULL; + } +} + +/* A werf.yaml beside a `.helm` chart dir: a Chart named after its project + * {source: werf}, defined by werf.yaml, and the chart dir into `chart_dir`. + * False when it has no such chart. */ +static bool handle_werf_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, + char *chart_dir, size_t chart_dir_sz) { + char dir[CBM_SZ_1K]; + k8s_dirname(rel_path, dir, sizeof(dir)); + snprintf(chart_dir, chart_dir_sz, "%s%s.helm", dir, dir[0] ? "/" : ""); + if (!cbm_werf_helm_dir(ctx->repo_path, chart_dir)) { + return false; + } + char project[CBM_HELM_NAME_MAX]; + k8s_werf_project(source, project, sizeof(project)); + const char *cname = project[0] ? project : k8s_basename(rel_path); + int64_t chart_id = k8s_chart_upsert(ctx, rel_path, cname, "{\"source\":\"werf\"}"); + cbm_log_info("pass.k8s.werf", "file", rel_path, "chart", cname); + return chart_id > 0; +} + /* ── Dependency-manifest handler (go.mod / requirements.txt) ──────── */ static bool is_gomod_file(const char *base) { @@ -2969,7 +3093,7 @@ static void handle_dep_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c * the sequential loop did. Sequentially the pass re-read and re-parsed every * YAML manifest on one thread -- 1.1 s of the Go corpus (6,275 manifests; * profile + waste sanitizer, 2026-09-17). */ -typedef enum { K8S_SKIP = 0, K8S_DEP, K8S_HELM, K8S_MANIFEST, K8S_KUSTOMIZE } k8s_kind_t; +typedef enum { K8S_SKIP = 0, K8S_DEP, K8S_HELM, K8S_WERF, K8S_MANIFEST, K8S_KUSTOMIZE } k8s_kind_t; typedef struct { k8s_kind_t kind; @@ -3153,6 +3277,8 @@ static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { } if (is_helm_chart_file(base)) { out->kind = K8S_HELM; + } else if (is_werf_file(base)) { + out->kind = K8S_WERF; } else if (cbm_is_k8s_manifest(base, out->source)) { /* Always re-extract with CBM_LANG_K8S regardless of any cached * result: cached results were produced during the parallel YAML @@ -3610,7 +3736,8 @@ static void k8s_deploys_edge(k8s_dep_link_t *l, int64_t src, int64_t tgt, const } } -/* The Chart node of chart dir `dir`, 0 when it is none. */ +/* The Chart node of chart dir `dir`, 0 when it is none: its Chart.yaml's, or + * for a werf `.helm` dir the one its werf.yaml defines. */ static int64_t k8s_chart_node(k8s_dep_link_t *l, const char *dir) { static const char *const names[] = {"Chart.yaml", "Chart.yml"}; for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { @@ -3621,6 +3748,19 @@ static int64_t k8s_chart_node(k8s_dep_link_t *l, const char *dir) { return id; } } + const char *base = k8s_basename(dir); + if (strcmp(base, ".helm") != 0) { + return 0; + } + static const char *const werfs[] = {"werf.yaml", "werf.yml"}; + for (size_t i = 0; i < sizeof(werfs) / sizeof(werfs[0]); i++) { + char path[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%.*s%s", (int)(base - dir), dir, werfs[i]); + int64_t id = k8s_first_id(l->charts, path); + if (id > 0) { + return id; + } + } return 0; } @@ -3661,8 +3801,10 @@ static bool k8s_chart_values_file(const char *rel, const char *dir) { /* A chart deploys what its templates render: Chart -DEPLOYS {via: chart}-> * each Resource of /templates/ -- its nearest enclosing chart, so a - * subchart's objects are the subchart's. Its values files (values.yaml, - * values-.yaml beside Chart.yaml) CONFIGURE {strategy: helm_values} it. */ + * subchart's objects are the subchart's -- and each CRD of /crds/, + * which Helm installs first ({via: chart, phase: crds}). Its values files + * (values.yaml, values-.yaml beside Chart.yaml) CONFIGURE {strategy: + * helm_values} it. */ static void k8s_link_chart_contents(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b) { if (!b->charts || cbm_ht_count(b->charts) == 0) { return; @@ -3691,9 +3833,14 @@ static void k8s_link_chart_contents(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b) { } if (k == 0) { char tpl[CBM_SZ_1K]; + char crd[CBM_SZ_1K]; snprintf(tpl, sizeof(tpl), "%s%stemplates/", dir, dir[0] ? "/" : ""); + snprintf(crd, sizeof(crd), "%s%scrds/", dir, dir[0] ? "/" : ""); if (strncmp(fp, tpl, strlen(tpl)) == 0) { k8s_deploys_edge(&l, chart, nodes[i]->id, "{\"via\":\"chart\"}"); + } else if (strncmp(fp, crd, strlen(crd)) == 0) { + k8s_deploys_edge(&l, chart, nodes[i]->id, + "{\"via\":\"chart\",\"phase\":\"crds\"}"); } } else if (k8s_chart_values_file(fp, dir)) { cbm_gbuf_insert_edge(ctx->gbuf, nodes[i]->id, chart, "CONFIGURES", @@ -3709,19 +3856,1188 @@ static void k8s_link_chart_contents(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b) { } } -/* Each Image a chart's objects name literally (their USAGE {ref: image}, after +/* ── Helm template images (PHASE 2 P2.5) ───────────────────────────── + * + * A chart template's container image is a placeholder in the skeleton + * (`<.Values.image.repository>`): it is rendered here (helm_template.c) from + * the chart's own values.yaml, its Chart.yaml (.Chart) and the defines of its + * chart family -- what `helm template` writes with the chart's defaults. The + * object then USES the Image {ref: image, default_of: the values keys read}, + * an image nothing else in the repository changes but a release can. A + * parent chart that changes it for its subchart (its `:` values, its + * `global:`) DEPLOYS the Image it makes {via: chart, ref: image, + * override_of}. A value that does not render (helm_template.h: UNKNOWN) or + * is no image reference names none. */ + +typedef struct { + int64_t node; + char *file; + char *container; /* NULL: unnamed */ + int start_line; + int end_line; +} k8s_timg_t; + +typedef struct { + char *dir; + int64_t chart; + cbm_helm_chart_meta_t meta; + yyjson_mut_val *chart_yaml; /* NULL: none */ + yyjson_mut_val *values; /* NULL: values.yaml does not parse (nothing renders) */ +} k8s_hchart_t; + +/* A template file of a chart family (its defines). */ +typedef struct { + char *file; + char *family; +} k8s_htpl_t; + +/* One rendered image of an object (or a chart / release). */ +typedef struct { + int64_t src; + int64_t img; + const char *ref; + const char *container; + const char *keys; /* JSON array; NULL: none */ +} k8s_himg_t; + +typedef struct { + k8s_himg_t *v; + int n; + int cap; + CBMHashTable *seen; /* "src:img" of each v (keys cbm_alloc'd): one look-up a push */ +} k8s_himgs_t; + +typedef struct { + yyjson_mut_doc *doc; + CBMArena a; + CBMHashTable *charts; /* chart dir -> k8s_hchart_t* */ + CBMHashTable *defines; /* family dir -> cbm_helm_defines_t* */ + k8s_htpl_t *tpls; + int ntpl; + bool tpls_read; + k8s_timg_t *t; + int nt; + int tcap; + k8s_himgs_t dflt; /* the containers' images with the defaults */ + int edges; +} k8s_helm_t; + +/* The templated images of the facts, kept past them. */ +static void k8s_helm_collect(cbm_pipeline_ctx_t *ctx, const k8s_facts_t *f, k8s_helm_t *h) { + for (int i = 0; i < f->img_n; i++) { + const k8s_img_t *im = &f->imgs[i]; + const cbm_gbuf_node_t *n = + strchr(im->image, '<') ? cbm_gbuf_find_by_id(ctx->gbuf, im->node) : NULL; + if (!n || !n->file_path || im->start_line <= 0) { + continue; + } + if (!k8s_grow((void **)&h->t, &h->tcap, h->nt, sizeof(*h->t))) { + return; + } + h->t[h->nt++] = (k8s_timg_t){im->node, cbm_arena_strdup(&h->a, n->file_path), + im->container ? cbm_arena_strdup(&h->a, im->container) : NULL, + im->start_line, im->end_line}; + } +} + +/* Repository file `rel`, read only when discovery took it in: a values file, + * a Chart.yaml, a kustomize valuesFile named by path is never a symlink or + * under one -- nothing is read from outside the repository. */ +static char *k8s_repo_read(cbm_pipeline_ctx_t *ctx, const char *rel, int *len) { + if (!cbm_pipeline_file_node(ctx->gbuf, ctx->project_name, rel)) { + return NULL; + } + char path[CBM_SZ_4K]; + int n = snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, rel); + if (n <= 0 || (size_t)n >= sizeof(path)) { + return NULL; + } + char *buf = k8s_read_file(path, len); + cbm_path_info_t info; + if (!buf && cbm_path_info_utf8(path, &info) == CBM_PATH_INFO_OK && info.is_regular && + !info.is_symlink && info.size == 0) { + /* an empty file is read: empty text (padded like every read) */ + enum { CBM_TS_EMPTY_PAD = 16 }; + buf = cbm_calloc(CBM_MEM_CLASS_OTHER, CBM_TS_EMPTY_PAD); + *len = 0; + } + return buf; +} + +/* Is repository path `rel` there, or can that not be told (a link is + * reported, never followed; only a path proven absent is not there)? */ +static bool k8s_repo_has(cbm_pipeline_ctx_t *ctx, const char *rel) { + char path[CBM_SZ_4K]; + int n = snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, rel); + cbm_path_info_t info; + return n > 0 && (size_t)n < sizeof(path) && + cbm_path_info_utf8(path, &info) != CBM_PATH_INFO_ABSENT; +} + +/* A file of `dir` (one of `names`), parsed as values; *found when one exists. + * *unread when one is there but is not read (a symlink, too large): what it + * holds is unknown, which is not the same as no file. */ +static yyjson_mut_val *k8s_helm_yaml(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, const char *dir, + const char *const *names, bool *found, bool *unread) { + *found = false; + *unread = false; + for (int i = 0; names[i]; i++) { + char rel[CBM_SZ_1K]; + snprintf(rel, sizeof(rel), "%s%s%s", dir, dir[0] ? "/" : "", names[i]); + int len = 0; + char *src = k8s_repo_read(ctx, rel, &len); + if (!src) { + if (k8s_repo_has(ctx, rel)) { + *found = true; + *unread = true; + return NULL; + } + continue; + } + *found = true; + yyjson_mut_val *v = cbm_helm_values_parse(h->doc, src, (size_t)len); + cbm_free(CBM_MEM_CLASS_OTHER, src); + return v; + } + return NULL; +} + +static const char *k8s_helm_meta_str(yyjson_mut_val *obj, const char *key) { + yyjson_mut_val *v = yyjson_mut_is_obj(obj) ? yyjson_mut_obj_get(obj, key) : NULL; + return yyjson_mut_is_str(v) ? yyjson_mut_get_str(v) : NULL; +} + +/* The chart of chart dir `dir` with its metadata and default values. */ +static k8s_hchart_t *k8s_helm_chart(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_dep_link_t *l, + const char *dir) { + k8s_hchart_t *c = cbm_ht_get(h->charts, dir); + if (c) { + return c; + } + c = cbm_arena_calloc(&h->a, sizeof(*c)); + if (!c) { + return NULL; + } + c->dir = cbm_arena_strdup(&h->a, dir); + c->chart = k8s_chart_node(l, dir); + static const char *const CHART[] = {"Chart.yaml", "Chart.yml", NULL}; + static const char *const VALUES[] = {"values.yaml", "values.yml", NULL}; + bool found = false; + bool chart_unread = false; + bool values_unread = false; + c->chart_yaml = k8s_helm_yaml(ctx, h, dir, CHART, &found, &chart_unread); + yyjson_mut_val *vals = k8s_helm_yaml(ctx, h, dir, VALUES, &found, &values_unread); + /* a Chart.yaml or values file there but not read: the chart's values are + * unknown (it renders nothing), never empty */ + c->values = chart_unread || values_unread ? NULL : found ? vals : yyjson_mut_obj(h->doc); + const cbm_gbuf_node_t *cn = c->chart > 0 ? cbm_gbuf_find_by_id(ctx->gbuf, c->chart) : NULL; + const char *name = k8s_helm_meta_str(c->chart_yaml, "name"); + c->meta.name = name ? name : cn && cn->name ? cbm_arena_strdup(&h->a, cn->name) : NULL; + c->meta.version = k8s_helm_meta_str(c->chart_yaml, "version"); + c->meta.app_version = k8s_helm_meta_str(c->chart_yaml, "appVersion"); + cbm_ht_set(h->charts, c->dir, c); + return c; +} + +static bool k8s_helm_tpl_ext(const char *p) { + static const char *const EXT[] = {".yaml", ".yml", ".tpl", ".gotmpl", ".txt", ".json"}; + for (size_t i = 0; i < sizeof(EXT) / sizeof(EXT[0]); i++) { + if (k8s_has_suffix(p, EXT[i])) { + return true; + } + } + return false; +} + +/* The defines of chart family `family`: every template (/templates/) + * of its charts, read from disk. */ +static const cbm_helm_defines_t *k8s_helm_defines(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, + k8s_helm_t *h, const char *family) { + cbm_helm_defines_t *d = cbm_ht_get(h->defines, family); + if (d) { + return d; + } + if (!h->tpls_read) { + h->tpls_read = true; + const cbm_gbuf_node_t **files = NULL; + int n = 0; + if (cbm_gbuf_find_by_label(ctx->gbuf, "File", &files, &n) == 0 && n > 0) { + h->tpls = cbm_arena_alloc(&h->a, (size_t)n * sizeof(*h->tpls)); + for (int i = 0; h->tpls && i < n; i++) { + const char *fp = files[i]->file_path; + char dir[CBM_SZ_1K]; + char fam[CBM_SZ_1K]; + char tdir[CBM_SZ_1K]; + if (!fp || !k8s_helm_tpl_ext(fp) || !k8s_chart_of(b, fp, dir, sizeof(dir))) { + continue; + } + snprintf(tdir, sizeof(tdir), "%s%stemplates/", dir, dir[0] ? "/" : ""); + if (strncmp(fp, tdir, strlen(tdir)) != 0) { + continue; + } + k8s_chart_family(b, dir, fam, sizeof(fam)); + h->tpls[h->ntpl++] = + (k8s_htpl_t){cbm_arena_strdup(&h->a, fp), cbm_arena_strdup(&h->a, fam)}; + } + } + } + d = cbm_helm_defines_new(); + if (!d) { + return NULL; + } + for (int i = 0; i < h->ntpl; i++) { + if (strcmp(h->tpls[i].family, family) != 0) { + continue; + } + int len = 0; + char *src = k8s_repo_read(ctx, h->tpls[i].file, &len); + if (src) { + cbm_helm_defines_add(d, h->tpls[i].file, src, (size_t)len); + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + } + cbm_ht_set(h->defines, cbm_arena_strdup(&h->a, family), d); + return d; +} + +static void k8s_helm_free_defines(const char *key, void *value, void *ud) { + (void)key; + (void)ud; + cbm_helm_defines_free(value); +} + +static int k8s_timg_cmp(const void *x, const void *y) { + const k8s_timg_t *a = x; + const k8s_timg_t *b = y; + int c = strcmp(a->file, b->file); + return c ? c : a->start_line - b->start_line; +} + +/* Does template line `line` (1-based) of src hold an `image:` key? */ +static bool k8s_helm_image_line(const char *src, size_t len, const uint32_t *starts, int line) { + size_t k = starts[line - SKIP_ONE]; + while (k < len && (src[k] == ' ' || src[k] == '\t')) { + k++; + } + if (k + SKIP_ONE < len && src[k] == '-' && src[k + SKIP_ONE] == ' ') { + k++; + while (k < len && src[k] == ' ') { + k++; + } + } + return k + strlen("image:") <= len && memcmp(src + k, "image:", strlen("image:")) == 0; +} + +/* The probes of a template's templated images [from, to): each `image:` + * line inside a container (both branches of an if / else among them). */ +static int k8s_helm_probes(k8s_helm_t *h, const k8s_timg_t *t, int from, int to, const char *src, + size_t len, cbm_helm_probe_t **out) { + int nl = SKIP_ONE; + for (size_t i = 0; i < len; i++) { + nl += src[i] == '\n'; + } + uint32_t *starts = cbm_arena_alloc(&h->a, (size_t)nl * sizeof(*starts)); + if (!starts) { + return 0; + } + int k = 0; + starts[k++] = 0; + for (size_t i = 0; i < len && k < nl; i++) { + if (src[i] == '\n') { + starts[k++] = (uint32_t)(i + SKIP_ONE); + } + } + int cap = 0; + for (int i = from; i < to; i++) { + cap += t[i].end_line - t[i].start_line + SKIP_ONE; + } + cbm_helm_probe_t *p = cap ? cbm_arena_calloc(&h->a, (size_t)cap * sizeof(*p)) : NULL; + int np = 0; + for (int i = from; p && i < to; i++) { + for (int line = t[i].start_line; line <= t[i].end_line && line <= k; line++) { + bool dup = false; + for (int j = 0; j < np && !dup; j++) { + dup = p[j].line == line; + } + if (!dup && k8s_helm_image_line(src, len, starts, line)) { + p[np++].line = line; + } + } + } + *out = p; + return np; +} + +static yyjson_mut_val *k8s_helm_dig(yyjson_mut_val *v, const char *path) { + const char *p = path; + while (v && *p) { + const char *dot = strchr(p, '.'); + size_t kl = dot ? (size_t)(dot - p) : strlen(p); + v = yyjson_mut_is_obj(v) ? yyjson_mut_obj_getn(v, p, kl) : NULL; + p += kl + (dot ? SKIP_ONE : 0); + } + return v; +} + +/* Is values key path `key` (dot-joined, `\.` escaped) set in `v`? */ +static bool k8s_helm_sets(k8s_helm_t *h, yyjson_mut_val *v, const char *key) { + char *part = cbm_arena_alloc(&h->a, strlen(key) + SKIP_ONE); /* a key is never cut */ + const char *p = key; + while (v && part && *p) { + size_t k = 0; + while (*p && *p != '.' && *p != '[') { + if (*p == '\\' && p[1]) { + p++; + } + part[k++] = *p++; + } + part[k] = '\0'; + v = yyjson_mut_is_obj(v) ? yyjson_mut_obj_get(v, part) : NULL; + while (v && *p == '[') { + long idx = strtol(p + SKIP_ONE, (char **)&p, 10); + p += *p == ']' ? SKIP_ONE : 0; + v = yyjson_mut_is_arr(v) ? yyjson_mut_arr_get(v, (size_t)idx) : NULL; + } + if (*p == '.') { + p++; + } + } + return v != NULL; +} + +/* The values keys of JSON array `keys` that `overlay` sets (relative to + * the chart), prefixed by `prefix` -- as a JSON array; NULL: none. */ +static const char *k8s_helm_overridden(k8s_helm_t *h, const char *keys, yyjson_mut_val *overlay, + yyjson_mut_val *globals, const char *prefix) { + if (!keys) { + return NULL; + } + yyjson_doc *doc = yyjson_read(keys, strlen(keys), 0); + yyjson_val *arr = doc ? yyjson_doc_get_root(doc) : NULL; + const char *out = NULL; + if (yyjson_is_arr(arr)) { + size_t idx, max; + yyjson_val *k; + yyjson_arr_foreach(arr, idx, max, k) { + const char *key = yyjson_get_str(k); + bool global = key && strncmp(key, "global.", strlen("global.")) == 0; + bool set = key && (global ? k8s_helm_sets(h, globals, key + strlen("global.")) + : k8s_helm_sets(h, overlay, key)); + if (!set) { + continue; + } + /* JSON escaping writes at most 6 bytes a byte */ + int en = (int)(strlen(key) * 6 + SKIP_ONE); + char *esc = cbm_arena_alloc(&h->a, (size_t)en); + if (!esc) { + out = NULL; + break; + } + cbm_json_escape(esc, en, key); + out = cbm_arena_sprintf(&h->a, "%s%s\"%s%s%s\"", out ? out : "[", out ? "," : "", + global || !prefix ? "" : prefix, global || !prefix ? "" : ".", + esc); + } + } + if (doc) { + yyjson_doc_free(doc); + } + return out ? cbm_arena_sprintf(&h->a, "%s]", out) : NULL; +} + +static void k8s_himg_push(k8s_himgs_t *l, k8s_himg_t x) { + char key[CBM_SZ_64]; + snprintf(key, sizeof(key), "%lld:%lld", (long long)x.src, (long long)x.img); + if (!l->seen) { + l->seen = cbm_ht_create(0); + } + if (!l->seen || cbm_ht_get(l->seen, key)) { + return; /* the first (in file order) of the same Image stands for it */ + } + size_t kl = strlen(key) + SKIP_ONE; + char *k = cbm_alloc(CBM_MEM_CLASS_OTHER, kl); + if (!k || !k8s_grow((void **)&l->v, &l->cap, l->n, sizeof(*l->v))) { + cbm_free(CBM_MEM_CLASS_OTHER, k); + return; + } + memcpy(k, key, kl); + cbm_ht_set(l->seen, k, (void *)(uintptr_t)SKIP_ONE); + l->v[l->n++] = x; +} + +static void k8s_himg_key_free(const char *key, void *value, void *ud) { + (void)value; + (void)ud; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +static void k8s_himgs_free(k8s_himgs_t *l) { + if (l->seen) { + cbm_ht_foreach(l->seen, k8s_himg_key_free, NULL); + cbm_ht_free(l->seen); + } + cbm_free(CBM_MEM_CLASS_OTHER, l->v); + *l = (k8s_himgs_t){0}; +} + +/* Does `src` USE Image `img` already by a literal reference? */ +static bool k8s_helm_literal_use(cbm_pipeline_ctx_t *ctx, int64_t src, int64_t img) { + const cbm_gbuf_edge_t **es = NULL; + int n = 0; + if (cbm_gbuf_find_edges_by_source_type(ctx->gbuf, src, "USAGE", &es, &n) != 0) { + return false; + } + for (int i = 0; i < n; i++) { + if (es[i]->target_id == img && es[i]->properties_json && + strstr(es[i]->properties_json, "\"ref\":\"image\"")) { + return true; + } + } + return false; +} + +/* The captures of probes within [first, last] lines, as images of `src`. */ +static void k8s_helm_take(cbm_pipeline_ctx_t *ctx, const cbm_helm_probe_t *p, int np, int first, + int last, int64_t src, const char *container, k8s_himgs_t *out) { + for (int i = 0; i < np; i++) { + if (p[i].line < first || p[i].line > last) { + continue; + } + for (int c = 0; c < p[i].n; c++) { + const char *ref = p[i].image[c]; + int64_t img = ref && cbm_image_ref_valid(ref) ? cbm_pipeline_image_node(ctx, ref) : 0; + if (img > 0) { + k8s_himg_push(out, (k8s_himg_t){src, img, ref, container, p[i].keys[c]}); + } + } + } +} + +/* The images of templated containers [from, to) of one template, rendered + * with `values` (NULL: the chart's own). */ +static void k8s_helm_render(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, + const k8s_hchart_t *c, const char *as_name, const char *full, + const k8s_timg_t *t, int from, int to, yyjson_mut_val *values, + const char *src, size_t len, cbm_helm_probe_t *probes, int np) { + char fam[CBM_SZ_1K]; + k8s_chart_family(b, c->dir, fam, sizeof(fam)); + const cbm_helm_defines_t *defs = k8s_helm_defines(ctx, b, h, fam); + /* a subchart a dependency aliases is .Chart.Name = its alias (Helm's + * getAliasDependency) */ + cbm_helm_chart_meta_t meta = c->meta; + meta.name = as_name ? as_name : meta.name; + /* .Template.BasePath: the chart's path (full) and templates/, as Helm */ + meta.base_path = full ? cbm_arena_sprintf(&h->a, "%s/templates", full) : NULL; + meta.dir = c->dir; + cbm_helm_render(&h->a, t[from].file, src, len, values ? values : c->values, &meta, defs, probes, + np); + (void)to; +} + +static void k8s_helm_emit_uses(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, const k8s_himgs_t *l) { + for (int i = 0; i < l->n; i++) { + const k8s_himg_t *x = &l->v[i]; + if (k8s_helm_literal_use(ctx, x->src, x->img)) { + continue; /* a literal reference of the object wins */ + } + char esc[CBM_SZ_1K]; + char cesc[CBM_SZ_512]; + cbm_json_escape(esc, (int)sizeof(esc), x->ref); + cbm_json_escape(cesc, (int)sizeof(cesc), x->container ? x->container : "-"); + const char *props = cbm_arena_sprintf( + &h->a, "{\"ref\":\"image\",\"image\":\"%s\",\"container\":\"%s\"%s%s}", esc, cesc, + x->keys ? ",\"default_of\":" : "", x->keys ? x->keys : ""); + if (props) { + cbm_gbuf_insert_edge(ctx->gbuf, x->src, x->img, "USAGE", props); + h->edges++; + } + } +} + +/* Chart `from` -DEPLOYS {via, ref: image, image, override_of}-> the images an + * overlay changed: rendered there, absent from the defaults' images. */ +static void k8s_helm_emit_overrides(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, int64_t from, + const char *via, const k8s_himgs_t *dflt, + const k8s_himgs_t *got, yyjson_mut_val *overlay, + yyjson_mut_val *globals, const char *prefix, + k8s_himgs_t *emitted) { + for (int i = 0; i < got->n; i++) { + const k8s_himg_t *x = &got->v[i]; + bool same = false; + for (int j = 0; j < dflt->n && !same; j++) { + same = dflt->v[j].src == x->src && dflt->v[j].img == x->img && + strcmp(dflt->v[j].ref, x->ref) == 0; + } + const char *ovr = k8s_helm_overridden(h, x->keys, overlay, globals, prefix); + if (same || !ovr || from <= 0) { + continue; + } + k8s_himg_t e = {from, x->img, x->ref, NULL, ovr}; + int before = emitted->n; + k8s_himg_push(emitted, e); + if (emitted->n == before) { + continue; + } + char esc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), x->ref); + const char *props = cbm_arena_sprintf( + &h->a, "{\"via\":\"%s\",\"ref\":\"image\",\"image\":\"%s\",\"override_of\":%s}", via, + esc, ovr); + if (props) { + cbm_gbuf_insert_edge(ctx->gbuf, from, x->img, "DEPLOYS", props); + h->edges++; + } + } +} + +/* The chart dirs enclosing chart dir `dir`, nearest first (arena). */ +static int k8s_helm_ancestors(k8s_helm_t *h, k8s_bundles_t *b, const char *dir, const char ***out) { + int n = 0; + int cap = 0; + const char **v = NULL; + char cur[CBM_SZ_1K]; + snprintf(cur, sizeof(cur), "%s", dir); + while (cur[0]) { + char up[CBM_SZ_1K]; + if (!k8s_chart_of(b, cur, up, sizeof(up))) { + break; + } + if (n == cap) { + cap = cap ? cap * PAIR_LEN : CBM_SZ_16; + const char **g = cbm_arena_alloc(&h->a, (size_t)cap * sizeof(*g)); + if (!g) { + break; + } + if (n) { + memcpy(g, v, (size_t)n * sizeof(*g)); + } + v = g; + } + v[n++] = cbm_arena_strdup(&h->a, up); + snprintf(cur, sizeof(cur), "%s", up); + } + *out = v; + return n; +} + +/* The dependency of `parent`'s Chart.yaml naming `sub` (by name), NULL: none. */ +static yyjson_mut_val *k8s_helm_dep(const k8s_hchart_t *parent, const k8s_hchart_t *sub) { + yyjson_mut_val *deps = yyjson_mut_is_obj(parent->chart_yaml) + ? yyjson_mut_obj_get(parent->chart_yaml, "dependencies") + : NULL; + if (yyjson_mut_is_arr(deps) && sub->meta.name) { + size_t idx, max; + yyjson_mut_val *d; + yyjson_mut_arr_foreach(deps, idx, max, d) { + const char *name = k8s_helm_meta_str(d, "name"); + if (name && strcmp(name, sub->meta.name) == 0) { + return d; + } + } + } + return NULL; +} + +/* Is the subchart a dependency disables in the parent's values `vals` (its + * `condition`: the first of its comma-separated paths that holds a bool)? */ +static bool k8s_helm_disabled(k8s_helm_t *h, yyjson_mut_val *dep, yyjson_mut_val *vals) { + const char *cond = dep ? k8s_helm_meta_str(dep, "condition") : NULL; + for (const char *p = cond; p && *p;) { + const char *comma = strchr(p, ','); + size_t l = comma ? (size_t)(comma - p) : strlen(p); + char *path = cbm_arena_strndup(&h->a, p, l); + yyjson_mut_val *v = path ? k8s_helm_dig(vals, path) : NULL; + if (yyjson_mut_is_bool(v)) { + return !yyjson_mut_get_bool(v); + } + p += l + (comma ? SKIP_ONE : 0); + } + return false; +} + +/* The values chart dir `dir` renders with when chart dir `top` deploys it + * with values `top_vals` (NULL: top's defaults): down the charts between + * them, each subchart's defaults with its parent's `:` values and + * `global:` coalesced over them (Helm's coalesce; a subchart always has a + * `global`). *path gets dir's key path from top ("" for top itself), *as_name + * the name dir's chart is included under (its alias; NULL for top), *full its + * chart path as Helm's ChartFullPath (top's name, then `/charts/` + * per chart down; NULL: not known). NULL: a dependency condition disables it, + * or a values file does not parse. */ +static yyjson_mut_val *k8s_helm_context(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, + k8s_dep_link_t *l, const char *dir, const char *top, + yyjson_mut_val *top_vals, const char **path, + const char **as_name, const char **full) { + const char **anc = NULL; + int na = k8s_helm_ancestors(h, b, dir, &anc); + int ti = -SKIP_ONE; /* top's index among the ancestors; na: dir itself */ + if (strcmp(dir, top) == 0) { + ti = na; + } else { + for (int i = 0; i < na && ti < 0; i++) { + ti = strcmp(anc[i], top) == 0 ? i : ti; + } + } + *path = ""; + *as_name = NULL; + *full = NULL; + if (ti < 0) { + return NULL; + } + k8s_hchart_t *cur_chart = k8s_helm_chart(ctx, h, l, top); + if (!cur_chart || !cur_chart->values) { + return NULL; + } + *full = cur_chart->meta.name; + yyjson_mut_val *cur = top_vals ? cbm_helm_values_coalesce(h->doc, cur_chart->values, top_vals) + : cur_chart->values; + for (int i = (ti == na ? na : ti) - SKIP_ONE; i >= -SKIP_ONE && ti != na; i--) { + k8s_hchart_t *child = k8s_helm_chart(ctx, h, l, i >= 0 ? anc[i] : dir); + if (!child || !child->values) { + return NULL; + } + yyjson_mut_val *dep = k8s_helm_dep(cur_chart, child); + if (k8s_helm_disabled(h, dep, cur)) { + return NULL; + } + const char *alias = dep ? k8s_helm_meta_str(dep, "alias") : NULL; + alias = alias ? alias : child->meta.name; + if (!alias) { + return NULL; + } + *path = (*path)[0] ? cbm_arena_sprintf(&h->a, "%s.%s", *path, alias) + : cbm_arena_strdup(&h->a, alias); + *as_name = alias; + *full = *full ? cbm_arena_sprintf(&h->a, "%s/charts/%s", *full, alias) : NULL; + yyjson_mut_val *sub = yyjson_mut_is_obj(cur) ? yyjson_mut_obj_get(cur, alias) : NULL; + yyjson_mut_val *next = cbm_helm_values_coalesce(h->doc, child->values, sub); + yyjson_mut_val *pg = yyjson_mut_is_obj(cur) ? yyjson_mut_obj_get(cur, "global") : NULL; + yyjson_mut_val *cg = next ? yyjson_mut_obj_get(next, "global") : NULL; + yyjson_mut_val *g = cbm_helm_values_coalesce(h->doc, cg, pg); /* the parent's win */ + if (!next || !g) { + return NULL; + } + (void)yyjson_mut_obj_remove_key(next, "global"); + yyjson_mut_obj_add(next, yyjson_mut_strcpy(h->doc, "global"), g); + cur = next; + cur_chart = child; + } + return cur; +} + +/* Render the templated containers [from, to) of one template with `vals` + * and take each container's images. */ +static void k8s_helm_images_of(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, + const k8s_hchart_t *c, const char *as_name, const char *full, + int from, int to, yyjson_mut_val *vals, const char *src, size_t len, + cbm_helm_probe_t *probes, int np, k8s_himgs_t *out) { + k8s_helm_render(ctx, h, b, c, as_name, full, h->t, from, to, vals, src, len, probes, np); + for (int i = from; i < to; i++) { + k8s_helm_take(ctx, probes, np, h->t[i].start_line, h->t[i].end_line, h->t[i].node, + h->t[i].container, out); + } +} + +/* Every templated container's images with its chart family's defaults -- + * what `helm template ` renders. */ +static void k8s_helm_link_defaults(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8s_helm_t *h) { + if (h->nt == 0 || !ctx->repo_path) { + return; + } + qsort(h->t, (size_t)h->nt, sizeof(*h->t), k8s_timg_cmp); + k8s_dep_link_t l = {.ctx = ctx, .charts = k8s_index_by_file(ctx, "Chart")}; + k8s_himgs_t uses = {0}; + for (int from = 0; from < h->nt;) { + int to = from + SKIP_ONE; + while (to < h->nt && strcmp(h->t[to].file, h->t[from].file) == 0) { + to++; + } + char dir[CBM_SZ_1K]; + char fam[CBM_SZ_1K]; + k8s_hchart_t *c = k8s_chart_of(b, h->t[from].file, dir, sizeof(dir)) + ? k8s_helm_chart(ctx, h, &l, dir) + : NULL; + const char *path = NULL; + const char *as_name = NULL; + const char *full = NULL; + yyjson_mut_val *vals = NULL; + if (c) { + k8s_chart_family(b, c->dir, fam, sizeof(fam)); + vals = k8s_helm_context(ctx, h, b, &l, c->dir, fam, NULL, &path, &as_name, &full); + } + int len = 0; + char *src = vals ? k8s_repo_read(ctx, h->t[from].file, &len) : NULL; + cbm_helm_probe_t *probes = NULL; + int np = src ? k8s_helm_probes(h, h->t, from, to, src, (size_t)len, &probes) : 0; + if (np > 0) { + k8s_helm_images_of(ctx, h, b, c, as_name, full, from, to, vals, src, (size_t)len, + probes, np, &uses); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + from = to; + } + k8s_helm_emit_uses(ctx, h, &uses); + h->dflt = uses; /* the release views compare against these */ + k8s_free_index(l.charts); +} + +/* ── Release images ─────────────────────────────────────────────────── + * + * A release (a Flux HelmRelease's spec.values, a kustomize helmCharts + * entry's valuesFile + valuesInline, a Terraform helm_release's values + + * set) renders its chart with its values over the defaults. Of a chart in + * the repository the templates render again: each image that differs from + * the defaults' is the release's own -- release -DEPLOYS {via, ref: image, + * image, override_of: the keys it set that the image read}-> Image. A chart + * outside the repository has no templates here: its images are read from the + * release values by convention -- a mapping with `repository` (and `tag` / + * `digest` / `registry`) whose reference names a registry host; one without + * a host names none (no host is invented) -- {inferred: values, values_key}. */ + +/* The chart dir of local Chart node `cn` (a werf chart: its `.helm`). */ +static bool k8s_helm_chart_dir(const cbm_gbuf_node_t *cn, char *out, size_t sz) { + if (!cn->file_path) { + return false; + } + k8s_dirname(cn->file_path, out, sz); + const char *base = k8s_basename(cn->file_path); + if (strcmp(base, "werf.yaml") == 0 || strcmp(base, "werf.yml") == 0) { + size_t l = strlen(out); + snprintf(out + l, sz - l, "%s.helm", l ? "/" : ""); + } + return true; +} + +/* A local chart's images rendered with release values `rv`, against the + * defaults. */ +static void k8s_helm_release_local(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8s_helm_t *h, + k8s_dep_link_t *l, int64_t src, const char *via, const char *top, + yyjson_mut_val *rv, k8s_himgs_t *emitted) { + size_t tl = strlen(top); + for (int from = 0; from < h->nt;) { + int to = from + SKIP_ONE; + while (to < h->nt && strcmp(h->t[to].file, h->t[from].file) == 0) { + to++; + } + char dir[CBM_SZ_1K]; + bool in = + k8s_chart_of(b, h->t[from].file, dir, sizeof(dir)) && + (strcmp(dir, top) == 0 || (tl == 0 || (strncmp(dir, top, tl) == 0 && dir[tl] == '/'))); + k8s_hchart_t *c = in ? k8s_helm_chart(ctx, h, l, dir) : NULL; + const char *path = NULL; + const char *as_name = NULL; + const char *full = NULL; + yyjson_mut_val *vals = + c ? k8s_helm_context(ctx, h, b, l, dir, top, rv, &path, &as_name, &full) : NULL; + int len = 0; + char *srcs = vals ? k8s_repo_read(ctx, h->t[from].file, &len) : NULL; + cbm_helm_probe_t *probes = NULL; + int np = srcs ? k8s_helm_probes(h, h->t, from, to, srcs, (size_t)len, &probes) : 0; + if (np > 0) { + k8s_himgs_t got = {0}; + k8s_helm_images_of(ctx, h, b, c, as_name, full, from, to, vals, srcs, (size_t)len, + probes, np, &got); + yyjson_mut_val *overlay = path && path[0] ? k8s_helm_dig(rv, path) : rv; + yyjson_mut_val *globals = yyjson_mut_obj_get(rv, "global"); + k8s_helm_emit_overrides(ctx, h, src, via, &h->dflt, &got, overlay, globals, + path && path[0] ? path : NULL, emitted); + k8s_himgs_free(&got); + } + cbm_free(CBM_MEM_CLASS_OTHER, srcs); + from = to; + } +} + +/* Does image reference `ref` name a registry host (its first path part has + * a `.` or `:`, or is localhost)? */ +static bool k8s_helm_has_host(const char *ref) { + const char *slash = strchr(ref, '/'); + if (!slash) { + return false; + } + size_t fl = (size_t)(slash - ref); + return memchr(ref, '.', fl) || memchr(ref, ':', fl) || + (fl == strlen("localhost") && strncmp(ref, "localhost", fl) == 0); +} + +/* The images release values `v` name by convention (see above). */ +static void k8s_helm_convention(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, int64_t src, + const char *via, yyjson_mut_val *v, const char *path, + k8s_himgs_t *emitted) { + size_t idx, max; + if (yyjson_mut_is_arr(v)) { + yyjson_mut_val *it; + yyjson_mut_arr_foreach(v, idx, max, it) { + k8s_helm_convention(ctx, h, src, via, it, + cbm_arena_sprintf(&h->a, "%s[%zu]", path ? path : "", idx), + emitted); + } + return; + } + if (!yyjson_mut_is_obj(v)) { + return; + } + yyjson_mut_val *repo = yyjson_mut_obj_get(v, "repository"); + if (yyjson_mut_is_str(repo) && yyjson_mut_get_len(repo) > 0) { + char reg[CBM_SZ_512] = ""; + char tag[CBM_SZ_256] = ""; + char dig[CBM_SZ_256] = ""; + yyjson_mut_val *r = yyjson_mut_obj_get(v, "registry"); + yyjson_mut_val *tg = yyjson_mut_obj_get(v, "tag"); + yyjson_mut_val *dg = yyjson_mut_obj_get(v, "digest"); + bool ok = (!r || yyjson_mut_is_null(r) || cbm_helm_scalar_text(r, reg, sizeof(reg))) && + (!tg || yyjson_mut_is_null(tg) || cbm_helm_scalar_text(tg, tag, sizeof(tag))) && + (!dg || yyjson_mut_is_null(dg) || cbm_helm_scalar_text(dg, dig, sizeof(dig))); + const char *ref = ok ? cbm_arena_sprintf(&h->a, "%s%s%s%s%s", reg, reg[0] ? "/" : "", + yyjson_mut_get_str(repo), + dig[0] ? "@" + : tag[0] ? ":" + : "", + dig[0] ? dig : tag) + : NULL; + int64_t img = ref && k8s_helm_has_host(ref) && cbm_image_ref_valid(ref) + ? cbm_pipeline_image_node(ctx, ref) + : 0; + int before = emitted->n; + if (img > 0) { + k8s_himg_push(emitted, (k8s_himg_t){src, img, ref, NULL, NULL}); + } + if (emitted->n > before) { + char esc[CBM_SZ_1K]; + char kesc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), ref); + cbm_json_escape(kesc, (int)sizeof(kesc), path ? path : ""); + const char *props = cbm_arena_sprintf( + &h->a, + "{\"via\":\"%s\",\"ref\":\"image\",\"image\":\"%s\",\"inferred\":\"values\"," + "\"values_key\":\"%s\"}", + via, esc, kesc); + if (props) { + cbm_gbuf_insert_edge(ctx->gbuf, src, img, "DEPLOYS", props); + h->edges++; + } + } + } + yyjson_mut_val *k; + yyjson_mut_val *child; + yyjson_mut_obj_foreach(v, idx, max, k, child) { + if (!yyjson_mut_is_obj(child) && !yyjson_mut_is_arr(child)) { + continue; + } + const char *key = yyjson_mut_get_str(k); + const char *cp = path && path[0] ? cbm_arena_sprintf(&h->a, "%s.%s", path, key) + : cbm_arena_strdup(&h->a, key); + k8s_helm_convention(ctx, h, src, via, child, cp, emitted); + } +} + +/* One release of Chart node `chart` with values `rv`. */ +static void k8s_helm_release(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8s_helm_t *h, + k8s_dep_link_t *l, int64_t src, const char *via, int64_t chart, + yyjson_mut_val *rv) { + const cbm_gbuf_node_t *cn = chart > 0 ? cbm_gbuf_find_by_id(ctx->gbuf, chart) : NULL; + if (!cn || !yyjson_mut_is_obj(rv) || src <= 0) { + return; + } + k8s_himgs_t emitted = {0}; + char dir[CBM_SZ_1K]; + if (cbm_pipeline_chart_is_external(cn)) { + k8s_helm_convention(ctx, h, src, via, rv, NULL, &emitted); + } else if (k8s_helm_chart_dir(cn, dir, sizeof(dir)) && b->charts && + cbm_ht_get(b->charts, dir)) { + k8s_helm_release_local(ctx, b, h, l, src, via, dir, rv, &emitted); + } + k8s_himgs_free(&emitted); +} + +/* A node's document (lines start_line..end_line of its file) as values. */ +static yyjson_mut_val *k8s_helm_node_yaml(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, + const cbm_gbuf_node_t *n) { + int len = 0; + char *src = n->file_path ? k8s_repo_read(ctx, n->file_path, &len) : NULL; + if (!src) { + return NULL; + } + size_t b = 0; + size_t e = (size_t)len; + int line = SKIP_ONE; + for (size_t i = 0; i < (size_t)len && n->start_line > 0; i++) { + if (src[i] != '\n') { + continue; + } + line++; + if (line == n->start_line) { + b = i + SKIP_ONE; + } + if (n->end_line > 0 && line == n->end_line + SKIP_ONE) { + e = i; + break; + } + } + yyjson_mut_val *v = b <= e ? cbm_helm_values_parse(h->doc, src + b, e - b) : NULL; + cbm_free(CBM_MEM_CLASS_OTHER, src); + return v; +} + +/* String property `key` of edge properties `props` into out; false: none. */ +static bool k8s_json_str(const char *props, const char *key, char *out, size_t sz) { + out[0] = '\0'; + yyjson_doc *d = props ? yyjson_read(props, strlen(props), 0) : NULL; + yyjson_val *v = d ? yyjson_obj_get(yyjson_doc_get_root(d), key) : NULL; + bool ok = yyjson_is_str(v); + if (ok) { + snprintf(out, sz, "%s", yyjson_get_str(v)); + } + if (d) { + yyjson_doc_free(d); + } + return ok; +} + +/* Does field `key` of `obj` give a value (a non-null scalar, a non-empty list + * or map)? */ +static bool k8s_helm_gives(yyjson_mut_val *obj, const char *key) { + yyjson_mut_val *v = yyjson_mut_is_obj(obj) ? yyjson_mut_obj_get(obj, key) : NULL; + if (!v || yyjson_mut_is_null(v)) { + return false; + } + if (yyjson_mut_is_arr(v)) { + return yyjson_mut_arr_size(v) > 0; + } + if (yyjson_mut_is_obj(v)) { + return yyjson_mut_obj_size(v) > 0; + } + return !yyjson_mut_is_str(v) || yyjson_mut_get_len(v) > 0; +} + +/* A HelmRelease's values: spec.values; NULL when there are none -- or when a + * source not read also gives some (valuesFrom, the chart's valuesFiles, a + * postRenderer that may patch the images): its images are then unknown. */ +static yyjson_mut_val *k8s_helm_flux_values(yyjson_mut_val *docv) { + yyjson_mut_val *spec = yyjson_mut_is_obj(docv) ? yyjson_mut_obj_get(docv, "spec") : NULL; + yyjson_mut_val *chart = yyjson_mut_is_obj(spec) ? yyjson_mut_obj_get(spec, "chart") : NULL; + yyjson_mut_val *cspec = yyjson_mut_is_obj(chart) ? yyjson_mut_obj_get(chart, "spec") : NULL; + if (k8s_helm_gives(spec, "valuesFrom") || k8s_helm_gives(spec, "postRenderers") || + k8s_helm_gives(cspec, "valuesFiles") || k8s_helm_gives(cspec, "valuesFile")) { + return NULL; + } + return yyjson_mut_is_obj(spec) ? yyjson_mut_obj_get(spec, "values") : NULL; +} + +/* A kustomize helmCharts entry's values: valuesFile, then valuesInline over + * it; NULL when neither -- or when a part is not read (additionalValuesFiles, + * a valuesMerge other than the default override, a valuesFile that cannot be + * read): its images are then unknown. */ +static yyjson_mut_val *k8s_helm_kustomize_values(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, + const cbm_gbuf_node_t *kust, const char *props) { + char chart[CBM_SZ_512] = ""; + char release[CBM_SZ_512] = ""; + char vfile[CBM_SZ_1K] = ""; + (void)k8s_json_str(props, "chart", chart, sizeof(chart)); + (void)k8s_json_str(props, "release", release, sizeof(release)); + (void)k8s_json_str(props, "values", vfile, sizeof(vfile)); + int len = 0; + char *src = kust->file_path ? k8s_repo_read(ctx, kust->file_path, &len) : NULL; + yyjson_mut_val *doc = src ? cbm_helm_values_parse(h->doc, src, (size_t)len) : NULL; + cbm_free(CBM_MEM_CLASS_OTHER, src); + /* kustomize runs the kustomization's own transformers over the inflated + * chart: images / patches / replacements may change what it deploys */ + static const char *const TRANSFORMS[] = { + "images", "patches", "patchesJson6902", "patchesStrategicMerge", + "replacements", "transformers", "components", NULL}; + for (int i = 0; TRANSFORMS[i]; i++) { + if (k8s_helm_gives(doc, TRANSFORMS[i])) { + return NULL; + } + } + yyjson_mut_val *charts = yyjson_mut_is_obj(doc) ? yyjson_mut_obj_get(doc, "helmCharts") : NULL; + yyjson_mut_val *inline_vals = NULL; + size_t idx, max; + yyjson_mut_val *c; + yyjson_mut_arr_foreach(charts, idx, max, c) { + const char *name = k8s_helm_meta_str(c, "name"); + const char *rel = k8s_helm_meta_str(c, "releaseName"); + if (name && strcmp(name, chart) == 0 && + (!release[0] || (rel && strcmp(rel, release) == 0))) { + const char *merge = k8s_helm_meta_str(c, "valuesMerge"); + if (k8s_helm_gives(c, "additionalValuesFiles") || + (merge && strcmp(merge, "override") != 0)) { + return NULL; + } + inline_vals = yyjson_mut_obj_get(c, "valuesInline"); + break; + } + } + yyjson_mut_val *file_vals = NULL; + if (vfile[0]) { + int fl = 0; + char *fs = k8s_repo_read(ctx, vfile, &fl); + file_vals = fs ? cbm_helm_values_parse(h->doc, fs, (size_t)fl) : NULL; + cbm_free(CBM_MEM_CLASS_OTHER, fs); + if (!file_vals) { + return NULL; /* a valuesFile not read: the release's values would be partial */ + } + } + if (!inline_vals && !file_vals) { + return NULL; + } + return cbm_helm_values_coalesce(h->doc, file_vals, inline_vals); +} + +/* Every release's images: HelmReleases and kustomize helmCharts from their + * DEPLOYS edges to a Chart, Terraform helm_releases from the context. */ +static void k8s_helm_link_releases(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8s_helm_t *h) { + k8s_dep_link_t l = {.ctx = ctx, .charts = k8s_index_by_file(ctx, "Chart")}; + const cbm_gbuf_edge_t **es = NULL; + int n = 0; + if (ctx->repo_path && cbm_gbuf_find_edges_by_type(ctx->gbuf, "DEPLOYS", &es, &n) == 0) { + /* collected first: release edges are DEPLOYS too */ + typedef struct { + int64_t src; + int64_t tgt; + const char *props; + } k8s_rel_t; + k8s_rel_t *rels = n ? cbm_arena_alloc(&h->a, (size_t)n * sizeof(*rels)) : NULL; + int nr = 0; + for (int i = 0; rels && i < n; i++) { + const char *p = es[i]->properties_json; + if (p && + (strstr(p, "\"via\":\"helmrelease\"") || strstr(p, "\"via\":\"helmcharts\"")) && + !strstr(p, "\"ref\":")) { + rels[nr++] = + (k8s_rel_t){es[i]->source_id, es[i]->target_id, cbm_arena_strdup(&h->a, p)}; + } + } + for (int i = 0; i < nr; i++) { + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, rels[i].src); + if (!src) { + continue; + } + bool hr = strstr(rels[i].props, "\"via\":\"helmrelease\"") != NULL; + yyjson_mut_val *rv = NULL; + if (hr) { + rv = k8s_helm_flux_values(k8s_helm_node_yaml(ctx, h, src)); + } else { + rv = k8s_helm_kustomize_values(ctx, h, src, rels[i].props); + } + k8s_helm_release(ctx, b, h, &l, rels[i].src, hr ? "helmrelease" : "helmcharts", + rels[i].tgt, rv); + } + } + for (int i = 0; i < ctx->tf_release_n; i++) { + cbm_tf_release_t *r = &ctx->tf_releases[i]; + if (r->values) { + yyjson_doc *jd = yyjson_read(r->values, strlen(r->values), 0); + yyjson_mut_val *rv = jd ? yyjson_val_mut_copy(h->doc, yyjson_doc_get_root(jd)) : NULL; + if (jd) { + yyjson_doc_free(jd); + } + k8s_helm_release(ctx, b, h, &l, r->src, "terraform", r->chart, rv); + cbm_free(CBM_MEM_CLASS_OTHER, r->values); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, ctx->tf_releases); + ctx->tf_releases = NULL; + ctx->tf_release_n = 0; + ctx->tf_release_cap = 0; + k8s_free_index(l.charts); +} + +static void k8s_helm_init(k8s_helm_t *h) { + *h = (k8s_helm_t){0}; + cbm_arena_init_lazy(&h->a, CBM_SZ_4K); + h->doc = yyjson_mut_doc_new(cbm_json_writer_allocator()); + h->charts = cbm_ht_create(0); + h->defines = cbm_ht_create(0); +} + +static void k8s_helm_free(k8s_helm_t *h) { + if (h->defines) { + cbm_ht_foreach(h->defines, k8s_helm_free_defines, NULL); + cbm_ht_free(h->defines); + } + if (h->charts) { + cbm_ht_free(h->charts); + } + if (h->doc) { + yyjson_mut_doc_free(h->doc); + } + cbm_free(CBM_MEM_CLASS_OTHER, h->t); + k8s_himgs_free(&h->dflt); + cbm_arena_destroy(&h->a); + if (h->edges > 0) { + cbm_log_info("pass.k8s.helm_images", "edges", itoa_k8s(h->edges)); + } + *h = (k8s_helm_t){0}; +} + +/* Each Image a chart's objects use (their USAGE {ref: image}, after * pass_docker.c moved references onto component Images): Chart -DEPLOYS - * {via: chart, ref: image}-> it. A templated image names no Image, and no - * values default is read. Edges are collected first: inserting while walking - * a chart's own DEPLOYS edges would grow the list being read. */ + * {via: chart, ref: image}-> it -- {default_of: the values keys} when every + * object of the chart uses it by a rendered default (the keys unioned), none + * when one names it literally. Edges are collected first: inserting while + * walking a chart's own DEPLOYS edges would grow the list being read. */ +typedef struct { + int64_t chart; + int64_t img; + bool literal; + bool lost; /* a mark that could not be carried whole: no edge, never an unmarked one */ + char *keys; /* JSON array (cbm_alloc); NULL: none yet */ +} k8s_cimg_t; + +/* JSON array `a` (owned, may be NULL) with the strings of JSON array value + * `bv` appended once; the result is owned. *lost when one could not be. */ +static char *k8s_keys_union(char *a, yyjson_val *bv, bool *lost) { + size_t idx, max; + yyjson_val *k; + yyjson_arr_foreach(bv, idx, max, k) { + const char *s = yyjson_get_str(k); + if (!s) { + continue; + } + /* JSON escaping writes at most 6 bytes (\u00XX) a byte */ + size_t qn = strlen(s) * 6 + PAIR_LEN + SKIP_ONE; + char *quoted = cbm_alloc(CBM_MEM_CLASS_OTHER, qn); + if (!quoted) { + *lost = true; + break; + } + quoted[0] = '"'; + cbm_json_escape(quoted + SKIP_ONE, (int)(qn - PAIR_LEN), s); + size_t ql = strlen(quoted); + quoted[ql++] = '"'; + quoted[ql] = '\0'; + if (a && strstr(a, quoted)) { + cbm_free(CBM_MEM_CLASS_OTHER, quoted); + continue; + } + size_t al = a ? strlen(a) : 0; + char *g = cbm_alloc(CBM_MEM_CLASS_OTHER, al + ql + 3); + if (!g) { + cbm_free(CBM_MEM_CLASS_OTHER, quoted); + *lost = true; + break; + } + if (a) { + snprintf(g, al + ql + 3, "%.*s,%s]", (int)(al - SKIP_ONE), a, quoted); + } else { + snprintf(g, ql + 3, "[%s]", quoted); + } + cbm_free(CBM_MEM_CLASS_OTHER, quoted); + cbm_free(CBM_MEM_CLASS_OTHER, a); + a = g; + } + return a; +} + static void k8s_link_chart_images(cbm_pipeline_ctx_t *ctx) { const cbm_gbuf_node_t **charts = NULL; int nc = 0; if (cbm_gbuf_find_by_label(ctx->gbuf, "Chart", &charts, &nc) != 0 || nc == 0) { return; } - int64_t *pairs = NULL; - int np = 0; + k8s_cimg_t *v = NULL; + int n = 0; int cap = 0; for (int c = 0; c < nc; c++) { const cbm_gbuf_edge_t **dep = NULL; @@ -3748,26 +5064,52 @@ static void k8s_link_chart_images(cbm_pipeline_ctx_t *ctx) { !strstr(up, "\"ref\":\"image\"")) { continue; } - if (np + PAIR_LEN > cap) { - int ncap = cap ? cap * PAIR_LEN : CBM_SZ_64; - int64_t *grown = - cbm_realloc(CBM_MEM_CLASS_OTHER, pairs, (size_t)ncap * sizeof(*pairs)); - if (!grown) { + int at = -SKIP_ONE; + for (int i = n - SKIP_ONE; i >= 0 && at < 0; i--) { + at = v[i].chart == charts[c]->id && v[i].img == img->id ? i : at; + } + if (at < 0) { + if (!k8s_grow((void **)&v, &cap, n, sizeof(*v))) { break; } - pairs = grown; - cap = ncap; + v[n] = (k8s_cimg_t){charts[c]->id, img->id, false, false, NULL}; + at = n++; + } + /* the USAGE's default_of, read as JSON (a key path may hold `]`) */ + yyjson_doc *ud = yyjson_read(up, strlen(up), 0); + yyjson_val *dv = ud ? yyjson_obj_get(yyjson_doc_get_root(ud), "default_of") : NULL; + if (!ud) { + v[at].lost = true; + } else if (!dv) { + v[at].literal = true; + } else if (yyjson_is_arr(dv)) { + v[at].keys = k8s_keys_union(v[at].keys, dv, &v[at].lost); + } else { + v[at].lost = true; + } + if (ud) { + yyjson_doc_free(ud); } - pairs[np++] = charts[c]->id; - pairs[np++] = img->id; } } } - for (int i = 0; i + SKIP_ONE < np; i += PAIR_LEN) { - cbm_gbuf_insert_edge(ctx->gbuf, pairs[i], pairs[i + SKIP_ONE], "DEPLOYS", - "{\"via\":\"chart\",\"ref\":\"image\"}"); + for (int i = 0; i < n; i++) { + bool marked = !v[i].literal && v[i].keys; + if (!v[i].literal && v[i].lost) { + cbm_free(CBM_MEM_CLASS_OTHER, v[i].keys); + continue; /* the mark is not whole: no edge rather than an unmarked one */ + } + size_t pn = (marked ? strlen(v[i].keys) : 0) + CBM_SZ_64; + char *props = cbm_alloc(CBM_MEM_CLASS_OTHER, pn); + if (props) { + snprintf(props, pn, "{\"via\":\"chart\",\"ref\":\"image\"%s%s}", + marked ? ",\"default_of\":" : "", marked ? v[i].keys : ""); + cbm_gbuf_insert_edge(ctx->gbuf, v[i].chart, v[i].img, "DEPLOYS", props); + cbm_free(CBM_MEM_CLASS_OTHER, props); + } + cbm_free(CBM_MEM_CLASS_OTHER, v[i].keys); } - cbm_free(CBM_MEM_CLASS_OTHER, pairs); + cbm_free(CBM_MEM_CLASS_OTHER, v); } /* A HelmRelease whose chartRef names OCIRepository `oci` (its USAGE edge @@ -3830,7 +5172,7 @@ static void k8s_prop_add(char *out, size_t out_sz, int *w, const char *key, cons * namespace, values, line} from the kustomization (its Kustomization * Resource, else its File -- the unit a Flux / Argo deploy reaches) to the * external Chart of that repository and name. */ -static void k8s_link_kcharts(k8s_dep_link_t *l, const k8s_facts_t *f) { +static void k8s_link_kcharts(k8s_dep_link_t *l, k8s_bundles_t *b, const k8s_facts_t *f) { for (int i = 0; i < f->kchart_n; i++) { const k8s_kchart_t *c = &f->kcharts[i]; if (!c->chart || strchr(c->chart, '/') || strchr(c->chart, '$')) { @@ -3838,8 +5180,18 @@ static void k8s_link_kcharts(k8s_dep_link_t *l, const k8s_facts_t *f) { } int64_t src = k8s_first_id(l->kusts, c->kust); src = src > 0 ? src : k8s_first_id(l->files, c->kust); + /* No repo: the chart in the kustomization's chartHome (charts/). */ + int64_t local = 0; + if (!c->url || !c->url[0]) { + char kdir[CBM_SZ_1K]; + char cdir[CBM_SZ_1K]; + k8s_dirname(c->kust, kdir, sizeof(kdir)); + snprintf(cdir, sizeof(cdir), "%s%scharts/%s", kdir, kdir[0] ? "/" : "", c->chart); + local = b->charts && cbm_ht_get(b->charts, cdir) ? k8s_chart_node(l, cdir) : 0; + } char props[CBM_SZ_4K]; - int w = snprintf(props, sizeof(props), "{\"via\":\"helmcharts\",\"external\":true"); + int w = snprintf(props, sizeof(props), "{\"via\":\"helmcharts\"%s", + local > 0 ? "" : ",\"external\":true"); k8s_prop_add(props, sizeof(props), &w, "chart", c->chart); k8s_prop_add(props, sizeof(props), &w, "url", c->url); k8s_prop_add(props, sizeof(props), &w, "version", c->version); @@ -3850,7 +5202,8 @@ static void k8s_link_kcharts(k8s_dep_link_t *l, const k8s_facts_t *f) { continue; } snprintf(props + w, sizeof(props) - (size_t)w, ",\"line\":%d}", c->line); - k8s_deploys_edge(l, src, k8s_external_chart(l, c->chart, c->url, c->kust), props); + k8s_deploys_edge( + l, src, local > 0 ? local : k8s_external_chart(l, c->chart, c->url, c->kust), props); } } @@ -4527,7 +5880,7 @@ static void k8s_link_deploys(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundl k8s_link_chart_refs(&l, &f->srcs[s]); } if (l.files && l.kusts) { - k8s_link_kcharts(&l, f); + k8s_link_kcharts(&l, b, f); } cbm_free(CBM_MEM_CLASS_OTHER, (void *)members.items); cbm_free(CBM_MEM_CLASS_OTHER, (void *)units.items); @@ -4603,6 +5956,14 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_chart_dir_add(&bundles, rel); helm_count++; break; + case K8S_WERF: { + char chart_dir[CBM_SZ_1K]; + if (handle_werf_chart(ctx, rel, p->source, chart_dir, sizeof(chart_dir))) { + k8s_chart_dir_register(&bundles, chart_dir); + helm_count++; + } + break; + } case K8S_KUSTOMIZE: k8s_kusts_push(&kusts, rel, p->source); k8s_kustomize_keep(&facts, rel, p->res); @@ -4647,6 +6008,9 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_link_crds(ctx, &bundles, &facts); k8s_link_deploys(ctx, &kusts, &bundles, &facts); k8s_link_images(ctx, &facts); + k8s_helm_t helm; + k8s_helm_init(&helm); + k8s_helm_collect(ctx, &facts, &helm); k8s_link_envs(ctx, &facts); k8s_link_patches(ctx, &kusts, &facts); k8s_link_renames(ctx, &kusts); @@ -4657,11 +6021,17 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_link_local_deps(ctx, &local_deps); k8s_link_includes(ctx, &bundles, files, file_count); k8s_link_chart_contents(ctx, &bundles); - k8s_bundles_free(&bundles); + /* The images chart templates render with their values (helm_template.c). */ + k8s_helm_link_defaults(ctx, &bundles, &helm); k8s_kusts_free(&kusts); /* Shell and Make deploy commands, against the nodes linked above. */ cbm_pipeline_pass_shell(ctx, files, file_count); cbm_pipeline_pass_terraform(ctx, files, file_count); + /* The images releases render (their values over the charts' defaults), + * before the image hubs move references. */ + k8s_helm_link_releases(ctx, &bundles, &helm); + k8s_helm_free(&helm); + k8s_bundles_free(&bundles); cbm_pipeline_pass_ansible(ctx, files, file_count); cbm_pipeline_pass_docker(ctx, files, file_count); /* The Images a chart's objects use, once the image hubs moved them. */ diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 3aadeb4dc..988da3060 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -3433,7 +3433,10 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB rc->project_name, rel)) || cbm_suppress_external_import_shadow(call->callee_name, res.strategy, &result->imports, imp_keys, imp_count, cbm_pipeline_get_pkgmap(), - cbm_pipeline_get_nsmap()); + cbm_pipeline_get_nsmap()) || + /* MUST match pass_calls.c exactly. */ + (lang == CBM_LANG_PHP && cbm_suppress_weak_member_match(true, true, res.strategy) && + cbm_pipeline_php_receiver_type_unindexed(&result->resolved_calls, call, rc->registry)); /* Service-pattern HTTP/ASYNC client call (`requests.get(url)`): the * service signal lives in the callee_name. The registry can mis-resolve diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c index 18f2b3134..ef3ba0f10 100644 --- a/src/pipeline/pass_pkgmap.c +++ b/src/pipeline/pass_pkgmap.c @@ -2115,6 +2115,37 @@ static const cbm_gbuf_node_t *resolve_sibling_file(const cbm_pipeline_ctx_t *ctx return found; } +/* The File an explicit relative import (`./x`, `../x`) names exactly, for + * files whose File QN differs from the module QN Strategy 1 tries + * (`../styles/global.css`, `../layouts/Layout.astro`, #2558). Resolved in + * stack buffers by cbm_repo_rel_join, which refuses a path above the + * repository root. */ +static const cbm_gbuf_node_t *resolve_relative_file(const cbm_pipeline_ctx_t *ctx, + const char *source_rel, const char *module_path, + bool php_only) { + if (!source_rel || !module_path || module_path[0] != '.' || + !(module_path[1] == '/' || (module_path[1] == '.' && module_path[2] == '/'))) { + return NULL; + } + char dir[PKGMAP_PATH_BUF]; + int dir_len = snprintf(dir, sizeof(dir), "%s", source_rel); + if (dir_len < 0 || (size_t)dir_len >= sizeof(dir)) { + return NULL; + } + char *slash = strrchr(dir, '/'); + if (slash) { + *slash = '\0'; + } else { + dir[0] = '\0'; + } + char path[PKGMAP_PATH_BUF]; + if (!cbm_repo_rel_join(dir, module_path, path, sizeof(path))) { + return NULL; + } + const cbm_gbuf_node_t *file = cbm_pipeline_file_node(ctx->gbuf, ctx->project_name, path); + return file && (!php_only || node_in_php_file(file)) ? file : NULL; +} + /* ── PHP PSR-4 class imports (#1186) ──────────────────────────────── */ typedef enum { @@ -2389,6 +2420,15 @@ static const cbm_gbuf_node_t *resolve_import_strategies( } } + /* Strategy 1c: relative non-module files can have a File QN that does not + * match the module QN above. Resolve the exact file after module and + * sibling lookup. */ + const cbm_gbuf_node_t *rel_file = + resolve_relative_file(ctx, source_rel, imp->module_path, php_only); + if (rel_file) { + return rel_file; + } + /* Strategy 2: namespace map. `using App.Utils`, `import com.example.Foo`, * `use App\Utils\Helper` name a NAMESPACE (or a member of it) that the * path-based QN cannot express. Try the full module path and progressively diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c index 8e6f82ce9..334322d15 100644 --- a/src/pipeline/pass_shell.c +++ b/src/pipeline/pass_shell.c @@ -376,10 +376,12 @@ typedef struct { const char *ci; /* static: the CI running the command; NULL: none */ const char *job; /* pass-arena: its job; NULL: none */ bool chart; - bool spawn; /* SPAWNS the script `tgt` / the Process `ext`, not DEPLOYS */ - bool make; /* SPAWNS the rule of target `ext` that makefile `path` reads */ - bool build; /* the image `ext` is built from the Dockerfile `path` */ - bool http; /* HTTP_CALLS the URL `ext` (scheme://host/path) with method `path` */ + bool spawn; /* SPAWNS the script `tgt` / the Process `ext`, not DEPLOYS */ + bool make; /* SPAWNS the rule of target `ext` that makefile `path` reads */ + bool build; /* the image `ext` is built from the Dockerfile `path` */ + bool http; /* HTTP_CALLS the URL `ext` (scheme://host/path) with method `path` */ + const char *build_args; /* pass-arena, build: --build-arg as a JSON object; NULL: none */ + const char *default_of; /* pass-arena, build: JSON array of the ${X:-d} defaults naming `ext` */ } sh_hit_t; typedef struct { @@ -413,9 +415,10 @@ typedef struct { const char *s; /* scalar value; NULL: unknown */ const char **items; /* array items (an item NULL: unknown); NULL: no array */ int n; + const char *dflt_of; /* JSON array: the variables whose ${X:-d} default made it */ } sh_val_t; -static const sh_val_t SH_UNKNOWN_VAL = {NULL, NULL, 0}; +static const sh_val_t SH_UNKNOWN_VAL = {NULL, NULL, 0, NULL}; typedef enum { SH_FREE, SH_AT, SH_LOST } sh_cwd_mode_t; @@ -464,8 +467,38 @@ typedef struct { sh_cwd_t *dirs; /* pushd stack */ int ndirs; int cap_dirs; + /* While set, an evaluation adds to dflt_acc (a JSON array) the variables + * whose ${X:-d} default supplied its value. */ + bool dflt_track; + const char *dflt_acc; } sh_t; +/* JSON array `list` (NULL: empty) with identifier `name` appended once. */ +static const char *sh_names_add(CBMArena *a, const char *list, const char *name, size_t nl) { + if (!list) { + return cbm_arena_sprintf(a, "[\"%.*s\"]", (int)nl, name); + } + char quoted[CBM_SZ_256]; + snprintf(quoted, sizeof(quoted), "\"%.*s\"", (int)nl, name); + if (strstr(list, quoted)) { + return list; + } + return cbm_arena_sprintf(a, "%.*s,%s]", (int)(strlen(list) - SKIP_ONE), list, quoted); +} + +/* JSON array `a` with the names of JSON array `b` appended once. */ +static const char *sh_names_union(CBMArena *ar, const char *a, const char *b) { + for (const char *p = b ? strchr(b, '"') : NULL; p;) { + const char *e = strchr(p + SKIP_ONE, '"'); + if (!e) { + break; + } + a = sh_names_add(ar, a, p + SKIP_ONE, (size_t)(e - p - SKIP_ONE)); + p = strchr(e + SKIP_ONE, '"'); + } + return a; +} + static void sh_set(sh_t *s, const char *name, const sh_val_t *v, bool local) { if (!sh_grow((void **)&s->undo, &s->cap_undo, s->nundo, sizeof(*s->undo))) { return; @@ -622,14 +655,14 @@ static const sh_val_t *sh_lookup(sh_t *s, const char *name, bool *found) { } sh_val_t *v = cbm_arena_alloc(s->a, sizeof(*v)); if (v) { - *v = (sh_val_t){s->script, NULL, 0}; + *v = (sh_val_t){s->script, NULL, 0, NULL}; } return v ? v : &SH_UNKNOWN_VAL; } if (strcmp(name, "PWD") == 0) { sh_val_t *v = cbm_arena_alloc(s->a, sizeof(*v)); if (v) { - *v = (sh_val_t){sh_cwd_value(s), NULL, 0}; + *v = (sh_val_t){sh_cwd_value(s), NULL, 0, NULL}; } return v ? v : &SH_UNKNOWN_VAL; } @@ -745,20 +778,29 @@ static const char *sh_eval_expansion(sh_t *s, TSNode n) { val = v->n > 0 ? v->items[0] : ""; } if (!op) { + if (found && s->dflt_track) { + s->dflt_acc = sh_names_union(s->a, s->dflt_acc, v->dflt_of); + } return found ? val : NULL; } bool colon = op[0] == ':'; bool use_default = !found || (colon && val && !val[0]); if (!use_default) { + if (s->dflt_track) { + s->dflt_acc = sh_names_union(s->a, s->dflt_acc, v->dflt_of); + } return val; } sh_str_t b = {0}; sh_concat(s, &b, n, op_end, close, true); const char *d = sh_str_done(&b); + if (s->dflt_track) { + s->dflt_acc = sh_names_add(s->a, s->dflt_acc, nm, strlen(nm)); + } if (d && strchr(op, '=')) { sh_val_t *nv = cbm_arena_alloc(s->a, sizeof(*nv)); if (nv) { - *nv = (sh_val_t){d, NULL, 0}; + *nv = (sh_val_t){d, NULL, 0, sh_names_add(s->a, NULL, nm, strlen(nm))}; sh_set(s, nm, nv, false); } } @@ -943,6 +985,9 @@ static const char *sh_eval(sh_t *s, TSNode n) { } bool found = false; const sh_val_t *val = sh_lookup(s, nm, &found); + if (found && s->dflt_track) { + s->dflt_acc = sh_names_union(s->a, s->dflt_acc, val->dflt_of); + } return found ? val->s : NULL; } if (strcmp(t, "expansion") == 0) { @@ -1319,6 +1364,8 @@ static void sh_hit_push(sh_t *s, TSNode cmd, sh_hit_t x) { x.line = (int)ts_node_start_point(cmd).row + SKIP_ONE + s->line_base; x.ext = x.ext ? cbm_arena_strdup(&h->arena, x.ext) : NULL; x.path = x.path ? cbm_arena_strdup(&h->arena, x.path) : NULL; + x.build_args = x.build_args ? cbm_arena_strdup(&h->arena, x.build_args) : NULL; + x.default_of = x.default_of ? cbm_arena_strdup(&h->arena, x.default_of) : NULL; x.ci = s->ci; x.job = s->job ? cbm_arena_strdup(&h->arena, s->job) : NULL; h->items[h->count++] = x; @@ -2268,7 +2315,11 @@ static void sh_assign(sh_t *s, TSNode n, bool local) { if (!nm || !v) { return; } - *v = (sh_val_t){"", NULL, 0}; + *v = (sh_val_t){"", NULL, 0, NULL}; + bool track = s->dflt_track; + const char *acc = s->dflt_acc; + s->dflt_track = true; + s->dflt_acc = NULL; if (sh_type(value, "array")) { uint32_t cnt = ts_node_named_child_count(value); const char **items = cbm_arena_alloc(s->a, (cnt ? cnt : SKIP_ONE) * sizeof(*items)); @@ -2282,10 +2333,13 @@ static void sh_assign(sh_t *s, TSNode n, bool local) { items[k++] = sh_eval_word(s, c); } } - *v = (sh_val_t){k > 0 ? items[0] : "", items, k}; + *v = (sh_val_t){k > 0 ? items[0] : "", items, k, NULL}; } else if (!ts_node_is_null(value)) { v->s = sh_eval(s, value); } + v->dflt_of = s->dflt_acc; + s->dflt_track = track; + s->dflt_acc = acc; bool append = false; for (uint32_t i = 0; i < ts_node_child_count(n); i++) { TSNode c = ts_node_child(n, i); @@ -2434,6 +2488,64 @@ static const char *sh_image_repo(sh_t *s, TSNode cmd, const sh_argv_t *a, int i) return sh_str_done(&out); } +/* JSON object `json` (NULL: none) with build argument `kv` ("K=V") added; a + * `K` alone (the environment's value), an unknown word or a secret (by name + * or value -- never kept in the graph, never an image) adds nothing. */ +static const char *sh_build_arg(sh_t *s, const char *json, const char *kv) { + const char *eq = kv ? strchr(kv, '=') : NULL; + if (!eq || eq == kv) { + return json; + } + for (const char *p = kv; p < eq; p++) { + if (!isalnum((unsigned char)*p) && *p != '_') { + return json; + } + } + const char *key = cbm_arena_strndup(s->a, kv, (size_t)(eq - kv)); + if (!key || cbm_is_secret_binding(key, eq + SKIP_ONE)) { + return json; + } + /* JSON escaping writes at most 6 bytes a byte: nothing is cut */ + int vn = (int)(strlen(eq + SKIP_ONE) * 6 + SKIP_ONE); + char *ev = cbm_arena_alloc(s->a, (size_t)vn); + if (!ev) { + return json; + } + cbm_json_escape(ev, vn, eq + SKIP_ONE); + const char *pair = cbm_arena_sprintf(s->a, "\"%.*s\":\"%s\"", (int)(eq - kv), kv, ev); + if (!pair) { + return json; + } + return json ? cbm_arena_sprintf(s->a, "%.*s,%s}", (int)(strlen(json) - SKIP_ONE), json, pair) + : cbm_arena_sprintf(s->a, "{%s}", pair); +} + +/* The variables whose ${X:-d} default supplied argv word `i` (a JSON array, + * NULL: none): the word evaluated once more with the defaults recorded. */ +static const char *sh_word_defaults(sh_t *s, TSNode cmd, const sh_argv_t *a, int i) { + uint32_t b = a->span[(size_t)i * PAIR_LEN]; + uint32_t e = a->span[(size_t)i * PAIR_LEN + SKIP_ONE]; + if (!memchr(s->src + b, '$', e - b)) { + return NULL; + } + TSNode w = ts_node_descendant_for_byte_range(cmd, b, e); + while (!ts_node_is_null(w) && (ts_node_start_byte(w) != b || ts_node_end_byte(w) != e)) { + w = ts_node_parent(w); + } + if (ts_node_is_null(w)) { + return NULL; + } + bool track = s->dflt_track; + const char *acc = s->dflt_acc; + s->dflt_track = true; + s->dflt_acc = NULL; + (void)sh_eval_word(s, w); + const char *out = s->dflt_acc; + s->dflt_track = track; + s->dflt_acc = acc; + return out; +} + /* docker / podman / nerdctl [buildx | image | builder] build, buildah bud: * each tagged image is built from the Dockerfile -f names (against the cwd), * else /Dockerfile -- one file of the repository. */ @@ -2467,17 +2579,20 @@ static void sh_build(sh_t *s, TSNode cmd, const sh_argv_t *a) { cbm_ht_set(s->hits->declarers, s->file, (void *)s->file); } int tags[SH_BUILD_TAGS]; + int tag_words[SH_BUILD_TAGS]; const char *tag_vals[SH_BUILD_TAGS]; int nt = 0; const char *dfile = NULL; bool dfile_known = true; const char *ctx = NULL; bool ctx_seen = false; + const char *args = NULL; for (; i < a->n; i++) { const char *w = a->v[i]; if (w && (sh_eq(w, "-t") || sh_eq(w, "--tag")) && i + SKIP_ONE < a->n) { if (nt < SH_BUILD_TAGS) { tags[nt] = i + SKIP_ONE; + tag_words[nt] = i + SKIP_ONE; tag_vals[nt++] = a->v[i + SKIP_ONE]; } i++; @@ -2485,9 +2600,15 @@ static void sh_build(sh_t *s, TSNode cmd, const sh_argv_t *a) { dfile = a->v[i + SKIP_ONE]; dfile_known = dfile != NULL; i++; + } else if (w && sh_eq(w, "--build-arg") && i + SKIP_ONE < a->n) { + args = sh_build_arg(s, args, a->v[i + SKIP_ONE]); + i++; + } else if (w && strncmp(w, "--build-arg=", strlen("--build-arg=")) == 0) { + args = sh_build_arg(s, args, w + strlen("--build-arg=")); } else if (w && strncmp(w, "--tag=", strlen("--tag=")) == 0) { if (nt < SH_BUILD_TAGS && w[strlen("--tag=")]) { tags[nt] = -SKIP_ONE; + tag_words[nt] = i; tag_vals[nt++] = w + strlen("--tag="); } } else if (w && strncmp(w, "--file=", strlen("--file=")) == 0) { @@ -2513,7 +2634,12 @@ static void sh_build(sh_t *s, TSNode cmd, const sh_argv_t *a) { } if (img && img[0]) { sh_hit_push(s, cmd, - (sh_hit_t){.via = "docker-build", .ext = img, .path = hit, .build = true}); + (sh_hit_t){.via = "docker-build", + .ext = img, + .path = hit, + .build = true, + .build_args = args, + .default_of = sh_word_defaults(s, cmd, a, tag_words[k])}); } } } @@ -3305,8 +3431,8 @@ static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t l sh_val_t *root = cbm_arena_alloc(&a, sizeof(*root)); sh_val_t *cur = cbm_arena_alloc(&a, sizeof(*cur)); if (root && cur) { - *root = (sh_val_t){SH_ROOT, NULL, 0}; - *cur = (sh_val_t){sh_anchor(&a, s.sdir), NULL, 0}; + *root = (sh_val_t){SH_ROOT, NULL, 0, NULL}; + *cur = (sh_val_t){sh_anchor(&a, s.sdir), NULL, 0, NULL}; cbm_ht_set(s.env, "__cbm_root", root); if (m.makefile) { cbm_ht_set(s.env, "__cbm_curdir", cur); @@ -3647,15 +3773,25 @@ static int sh_link_build(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, CBMHashTabl char esc_file[CBM_SZ_1K]; char esc_img[CBM_SZ_1K]; char ci[CBM_SZ_1K]; - char props[CBM_SZ_4K]; cbm_json_escape(esc_file, (int)sizeof(esc_file), x->file); cbm_json_escape(esc_img, (int)sizeof(esc_img), x->ext); sh_ci_props(x, ci, sizeof(ci)); - snprintf(props, sizeof(props), + /* sized to what it holds: default_of and build_args are never cut */ + size_t pn = strlen(esc_file) + strlen(esc_img) + strlen(ci) + + (x->default_of ? strlen(x->default_of) : 0) + + (x->build_args ? strlen(x->build_args) : 0) + CBM_SZ_256; + char *props = cbm_alloc(CBM_MEM_CLASS_OTHER, pn); + if (!props) { + return 0; + } + snprintf(props, pn, "{\"via\":\"build\",\"tool\":\"docker-build\",\"file\":\"%s\",\"image\":\"%s\"%s," - "\"line\":%d}", - esc_file, esc_img, ci, x->line); + "\"line\":%d%s%s%s%s}", + esc_file, esc_img, ci, x->line, x->default_of ? ",\"default_of\":" : "", + x->default_of ? x->default_of : "", x->build_args ? ",\"build_args\":" : "", + x->build_args ? x->build_args : ""); cbm_gbuf_insert_edge(ctx->gbuf, img, mod, "DEPENDS_ON", props); + cbm_free(CBM_MEM_CLASS_OTHER, props); return SKIP_ONE; } @@ -3852,7 +3988,7 @@ typedef struct { static void sh_ci_var(sh_t *s, const char *name, const char *value) { sh_val_t *v = value ? cbm_arena_alloc(s->a, sizeof(*v)) : NULL; if (v) { - *v = (sh_val_t){value, NULL, 0}; + *v = (sh_val_t){value, NULL, 0, NULL}; } sh_set(s, name, v ? v : &SH_UNKNOWN_VAL, false); } diff --git a/src/pipeline/pass_terraform.c b/src/pipeline/pass_terraform.c index 6328311c1..e9abd9c38 100644 --- a/src/pipeline/pass_terraform.c +++ b/src/pipeline/pass_terraform.c @@ -22,13 +22,16 @@ #include "foundation/log.h" #include "foundation/mem_core.h" #include "foundation/str_util.h" +#include "foundation/json_writer.h" /* cbm_json_writer_allocator */ #include "graph_buffer/graph_buffer.h" +#include "pipeline/helm_template.h" #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" #include "cbm.h" #include "helpers.h" #include "lang_specs.h" +#include #include #include #include @@ -249,6 +252,522 @@ static bool tf_plain_name(const char *s) { } /* One `resource "helm_release" "" { ... }` block. */ +/* ── helm_release values (for pass_k8s.c's release images) ────────── */ + +/* The expression inside wrapper nodes. */ +static TSNode tf_unwrap(TSNode n) { + while (tf_type(n, "expression") || tf_type(n, "literal_value") || + tf_type(n, "collection_value") || tf_type(n, "template_expr")) { + if (ts_node_named_child_count(n) != SKIP_ONE) { + break; + } + n = ts_node_named_child(n, 0); + } + return n; +} + +/* A heredoc's text (a `<<-` one dedented); NULL when it interpolates. */ +static char *tf_heredoc(const tf_t *t, TSNode n) { + TSNode open = {0}; + TSNode close = {0}; + bool dedent = false; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (tf_type(c, "heredoc_start")) { + uint32_t b = ts_node_start_byte(c); + dedent = ts_node_end_byte(c) - b >= 3 && t->src[b + PAIR_LEN] == '-'; + } else if (tf_type(c, "heredoc_identifier")) { + if (ts_node_is_null(open)) { + open = c; + } else { + close = c; + } + } else if (!tf_type(c, "template_literal")) { + return NULL; /* an interpolation or a directive: no literal */ + } + } + if (ts_node_is_null(open) || ts_node_is_null(close)) { + return NULL; + } + const char *b = t->src + ts_node_end_byte(open); + const char *e = t->src + ts_node_start_byte(close); + const char *nl = memchr(b, '\n', (size_t)(e - b)); + b = nl ? nl + SKIP_ONE : e; + while (e > b && e[-1] != '\n') { + e--; /* the closing line's indentation */ + } + size_t indent = (size_t)-1; + for (const char *p = b; dedent && p < e;) { + const char *le = memchr(p, '\n', (size_t)(e - p)); + le = le ? le : e; + size_t sp = 0; + while (p + sp < le && (p[sp] == ' ' || p[sp] == '\t')) { + sp++; + } + if (p + sp < le && sp < indent) { + indent = sp; + } + p = le + (le < e ? SKIP_ONE : 0); + } + if (!dedent || indent == (size_t)-1) { + indent = 0; + } + char *out = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)(e - b) + SKIP_ONE); + size_t k = 0; + for (const char *p = b; out && p < e;) { + const char *le = memchr(p, '\n', (size_t)(e - p)); + le = le ? le + SKIP_ONE : e; + size_t skip = 0; + while (skip < indent && p + skip < le && (p[skip] == ' ' || p[skip] == '\t')) { + skip++; + } + memcpy(out + k, p + skip, (size_t)(le - p) - skip); + k += (size_t)(le - p) - skip; + p = le; + } + if (out) { + out[k] = '\0'; + } + return out; +} + +/* HCL number literal s (digits, a fraction, an exponent) as Terraform turns a + * number into a string: exact decimal, no exponent, no trailing zeros (cty's + * big.Float Text('f', -1); 1.10 -> "1.1", 2e3 -> "2000"). False when it is + * no such literal or does not fit out. */ +static bool tf_number_text(const char *s, size_t n, char *out, size_t sz) { + enum { TF_MAX_EXP = 4096 }; + char dig[CBM_SZ_256]; + size_t nd = 0; + long exp = 0; + bool frac = false; + size_t i = 0; + for (; i < n && (isdigit((unsigned char)s[i]) || s[i] == '.'); i++) { + if (s[i] == '.') { + if (frac) { + return false; + } + frac = true; + continue; + } + if (nd + SKIP_ONE >= sizeof(dig)) { + return false; + } + dig[nd++] = s[i]; + exp -= frac ? SKIP_ONE : 0; + } + if (i < n) { + if (s[i] != 'e' && s[i] != 'E') { + return false; + } + i++; + bool neg = i < n && s[i] == '-'; + i += i < n && (s[i] == '-' || s[i] == '+'); + long e = 0; + if (i == n) { + return false; + } + for (; i < n; i++) { + if (!isdigit((unsigned char)s[i]) || e > TF_MAX_EXP) { + return false; + } + e = e * 10 + (s[i] - '0'); + } + exp += neg ? -e : e; + } + size_t b = 0; + while (b < nd && dig[b] == '0') { + b++; + } + if (b == nd) { + return snprintf(out, sz, "0") == SKIP_ONE; /* every digit 0 (or none) */ + } + size_t e = nd; + while (dig[e - SKIP_ONE] == '0') { /* stops at dig[b], which is no 0 */ + e--; + exp++; + } + size_t m = e - b; /* significant digits: dig[b, e) x 10^exp */ + size_t o = 0; + long point = (long)m + exp; /* digits before the decimal point */ + size_t need = exp >= 0 ? m + (size_t)exp + : point > 0 ? m + SKIP_ONE + : m + PAIR_LEN + (size_t)-point; + if (need + SKIP_ONE > sz) { + return false; + } + if (point <= 0) { + out[o++] = '0'; + out[o++] = '.'; + for (long z = 0; z < -point; z++) { + out[o++] = '0'; + } + } + for (size_t k = 0; k < m; k++) { + if (point > 0 && (long)k == point) { + out[o++] = '.'; + } + out[o++] = dig[b + k]; + } + for (long z = 0; z < exp; z++) { + out[o++] = '0'; + } + out[o] = '\0'; + return true; +} + +/* An HCL literal as JSON: a string without interpolation, a number, a bool, + * null, a tuple or object of those; NULL: anything else. `num_text`: a number + * as the string Terraform makes of it (a helm_release set value is a string), + * else a float64. */ +static yyjson_mut_val *tf_json_v(const tf_t *t, yyjson_mut_doc *doc, TSNode n, bool num_text) { + n = tf_unwrap(n); + if (tf_type(n, "string_lit") || tf_type(n, "quoted_template")) { + char buf[CBM_SZ_4K]; + size_t len = 0; + buf[0] = '\0'; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (tf_type(c, "quoted_template_start") || tf_type(c, "quoted_template_end")) { + continue; + } + if (!tf_type(c, "template_literal")) { + return NULL; + } + uint32_t b = ts_node_start_byte(c); + if (!tf_add(buf, sizeof(buf), &len, t->src + b, ts_node_end_byte(c) - b)) { + return NULL; + } + } + size_t k = 0; /* \" \\ \n \t */ + for (size_t i = 0; i < len; i++) { + if (buf[i] == '\\' && i + SKIP_ONE < len) { + char c = buf[++i]; + buf[k++] = c == 'n' ? '\n' : c == 't' ? '\t' : c; + } else { + buf[k++] = buf[i]; + } + } + return yyjson_mut_strncpy(doc, buf, k); + } + if (tf_type(n, "heredoc_template")) { + char *s = tf_heredoc(t, n); + yyjson_mut_val *v = s ? yyjson_mut_strcpy(doc, s) : NULL; + cbm_free(CBM_MEM_CLASS_OTHER, s); + return v; + } + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + if (tf_type(n, "numeric_lit")) { + char buf[CBM_SZ_256]; + if (num_text) { + return tf_number_text(t->src + b, e - b, buf, sizeof(buf)) ? yyjson_mut_strcpy(doc, buf) + : NULL; + } + snprintf(buf, sizeof(buf), "%.*s", (int)(e - b), t->src + b); + return yyjson_mut_real(doc, strtod(buf, NULL)); + } + if (tf_type(n, "bool_lit")) { + return yyjson_mut_bool(doc, e - b == strlen("true") && strncmp(t->src + b, "true", 4) == 0); + } + if (tf_type(n, "null_lit")) { + return yyjson_mut_null(doc); + } + if (tf_type(n, "tuple")) { + yyjson_mut_val *arr = yyjson_mut_arr(doc); + for (uint32_t i = 0; arr && i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (tf_type(c, "tuple_start") || tf_type(c, "tuple_end")) { + continue; + } + yyjson_mut_val *v = tf_json_v(t, doc, c, num_text); + if (!v) { + return NULL; + } + yyjson_mut_arr_append(arr, v); + } + return arr; + } + if (tf_type(n, "object")) { + yyjson_mut_val *obj = yyjson_mut_obj(doc); + for (uint32_t i = 0; obj && i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (!tf_type(c, "object_elem")) { + continue; + } + TSNode kn = tf_unwrap(ts_node_child_by_field_name(c, "key", (uint32_t)strlen("key"))); + TSNode vn = ts_node_child_by_field_name(c, "val", (uint32_t)strlen("val")); + yyjson_mut_val *key = NULL; + if (tf_type(kn, "variable_expr") && ts_node_named_child_count(kn) == SKIP_ONE) { + TSNode id = ts_node_named_child(kn, 0); + uint32_t kb = ts_node_start_byte(id); + key = yyjson_mut_strncpy(doc, t->src + kb, ts_node_end_byte(id) - kb); + } else { + key = tf_json_v(t, doc, kn, false); + } + yyjson_mut_val *val = tf_json_v(t, doc, vn, num_text); + if (!key || !yyjson_mut_is_str(key) || !val) { + return NULL; + } + (void)yyjson_mut_obj_put(obj, key, val); + } + return obj; + } + return NULL; +} + +static yyjson_mut_val *tf_json(const tf_t *t, yyjson_mut_doc *doc, TSNode n) { + return tf_json_v(t, doc, n, false); +} + +static char *tf_read(const char *path, size_t *len); + +/* A repository file's text (cbm_alloc; capped and padded like every file + * read; an empty file is empty text), NULL when unreadable -- or when it is + * no file discovery took in: a symlink, or a path through one, is never + * followed out of the repository. */ +static char *tf_read_repo(const tf_t *t, const char *path, size_t *len) { + if (!cbm_pipeline_file_node(t->ctx->gbuf, t->ctx->project_name, path)) { + return NULL; + } + char full[CBM_SZ_4K]; + int n = snprintf(full, sizeof(full), "%s/%s", t->ctx->repo_path, path); + if (n <= 0 || (size_t)n >= sizeof(full)) { + return NULL; + } + char *buf = tf_read(full, len); + cbm_path_info_t info; + if (!buf && cbm_path_info_utf8(full, &info) == CBM_PATH_INFO_OK && info.is_regular && + !info.is_symlink && info.size == 0) { + enum { TF_EMPTY_PAD = 16 }; + buf = cbm_calloc(CBM_MEM_CLASS_OTHER, TF_EMPTY_PAD); + *len = 0; + } + return buf; +} + +/* A `values = [...]` item as values: a YAML string / heredoc, file() + * of a repository file, yamlencode / jsonencode of a literal; NULL: none of + * those. */ +static yyjson_mut_val *tf_values_item(const tf_t *t, yyjson_mut_doc *doc, TSNode item, CBMArena *a, + const char ***files, int *nfiles) { + TSNode n = tf_unwrap(item); + if (tf_type(n, "function_call")) { + TSNode fn = ts_node_named_child(n, 0); + TSNode args = {0}; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + if (tf_type(ts_node_named_child(n, i), "function_arguments")) { + args = ts_node_named_child(n, i); + } + } + if (ts_node_is_null(fn) || ts_node_is_null(args) || + ts_node_named_child_count(args) != SKIP_ONE) { + return NULL; + } + uint32_t fb = ts_node_start_byte(fn); + size_t fl = ts_node_end_byte(fn) - fb; + TSNode arg = ts_node_named_child(args, 0); + if ((fl == strlen("yamlencode") && strncmp(t->src + fb, "yamlencode", fl) == 0) || + (fl == strlen("jsonencode") && strncmp(t->src + fb, "jsonencode", fl) == 0)) { + return tf_json(t, doc, arg); + } + if (fl == strlen("file") && strncmp(t->src + fb, "file", fl) == 0) { + char v[CBM_SZ_1K]; + char path[CBM_SZ_1K]; + if (!t->ctx->repo_path || !tf_string(t->src, t->mdir, arg, v, sizeof(v)) || + !tf_repo_path(t->mdir, v, path, sizeof(path))) { + return NULL; + } + const char **grown = cbm_arena_alloc(a, (size_t)(*nfiles + SKIP_ONE) * sizeof(*grown)); + if (grown) { + if (*nfiles) { + memcpy(grown, *files, (size_t)*nfiles * sizeof(*grown)); + } + grown[(*nfiles)++] = cbm_arena_strdup(a, path); + *files = grown; + } + size_t len = 0; + char *buf = tf_read_repo(t, path, &len); + yyjson_mut_val *out = buf ? cbm_helm_values_parse(doc, buf, len) : NULL; + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return out; + } + return NULL; + } + yyjson_mut_val *s = tf_json(t, doc, n); + return yyjson_mut_is_str(s) + ? cbm_helm_values_parse(doc, yyjson_mut_get_str(s), yyjson_mut_get_len(s)) + : NULL; +} + +/* A `set` entry (name / value / type; read with numbers as text) applied to + * root; false when it is no literal. */ +static bool tf_set_entry(yyjson_mut_doc *doc, yyjson_mut_val *root, yyjson_mut_val *entry) { + yyjson_mut_val *nm = yyjson_mut_obj_get(entry, "name"); + yyjson_mut_val *val = yyjson_mut_obj_get(entry, "value"); + yyjson_mut_val *ty = yyjson_mut_obj_get(entry, "type"); + if (!yyjson_mut_is_str(nm) || !val) { + return false; + } + const char *v = NULL; + if (yyjson_mut_is_str(val)) { + v = yyjson_mut_get_str(val); + } else if (yyjson_mut_is_bool(val)) { + v = yyjson_mut_get_bool(val) ? "true" : "false"; + } + bool as_string = yyjson_mut_is_str(ty) && strcmp(yyjson_mut_get_str(ty), "string") == 0; + return v && cbm_helm_values_set(doc, root, yyjson_mut_get_str(nm), v, as_string); +} + +/* A `set` block's attributes as an object; NULL when one is no literal. */ +static yyjson_mut_val *tf_set_block(const tf_t *t, yyjson_mut_doc *doc, TSNode block) { + yyjson_mut_val *entry = yyjson_mut_obj(doc); + for (uint32_t j = 0; entry && j < ts_node_named_child_count(block); j++) { + TSNode sb = ts_node_named_child(block, j); + if (!tf_type(sb, "body")) { + continue; + } + for (uint32_t k = 0; k < ts_node_named_child_count(sb); k++) { + TSNode at = ts_node_named_child(sb, k); + if (!tf_type(at, "attribute") || ts_node_named_child_count(at) < PAIR_LEN) { + continue; + } + TSNode key = ts_node_named_child(at, 0); + uint32_t b = ts_node_start_byte(key); + yyjson_mut_val *v = tf_json_v(t, doc, ts_node_named_child(at, SKIP_ONE), true); + if (!v) { + return NULL; + } + yyjson_mut_obj_add(entry, + yyjson_mut_strncpy(doc, t->src + b, ts_node_end_byte(key) - b), v); + } + } + return entry; +} + +/* Is block `a` a `dynamic "set"` / `"set_list"` / `"set_sensitive"` / + * `"postrender"` -- made by a for_each, never literal? */ +static bool tf_dynamic_set(const tf_t *t, TSNode a) { + TSNode kw = ts_node_named_child(a, 0); + TSNode label = ts_node_named_child(a, SKIP_ONE); + if (!tf_type(a, "block") || ts_node_is_null(kw) || ts_node_is_null(label)) { + return false; + } + uint32_t kb = ts_node_start_byte(kw); + if (ts_node_end_byte(kw) - kb != strlen("dynamic") || + strncmp(t->src + kb, "dynamic", strlen("dynamic")) != 0) { + return false; + } + uint32_t lb = ts_node_start_byte(label); + size_t ll = ts_node_end_byte(label) - lb; + return (ll == strlen("\"set\"") && strncmp(t->src + lb, "\"set\"", ll) == 0) || + (ll == strlen("\"set_list\"") && strncmp(t->src + lb, "\"set_list\"", ll) == 0) || + (ll == strlen("\"set_sensitive\"") && + strncmp(t->src + lb, "\"set_sensitive\"", ll) == 0) || + (ll == strlen("\"postrender\"") && strncmp(t->src + lb, "\"postrender\"", ll) == 0); +} + +/* Is node n an identifier spelling `word`? */ +static bool tf_word(const tf_t *t, TSNode n, const char *word) { + if (ts_node_is_null(n)) { + return false; + } + uint32_t b = ts_node_start_byte(n); + size_t l = ts_node_end_byte(n) - b; + return l == strlen(word) && strncmp(t->src + b, word, l) == 0; +} + +/* The values a helm_release body gives its chart -- `values` items + * coalesced in order, then each `set` -- as JSON (cbm_alloc); NULL when a + * part is no literal, is made by a `dynamic` block, is sensitive (its value + * is never read), or a postrender -- a block, an attribute or a dynamic one + * -- may rewrite what it deploys. */ +static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, int *nfiles) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(cbm_json_writer_allocator()); + yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; + bool ok = root != NULL; + for (int pass = 0; ok && pass < PAIR_LEN; pass++) { + for (uint32_t i = 0; ok && i < ts_node_named_child_count(body); i++) { + TSNode a = ts_node_named_child(body, i); + TSNode key = ts_node_named_child(a, 0); + uint32_t kb = ts_node_is_null(key) ? 0 : ts_node_start_byte(key); + size_t kl = ts_node_is_null(key) ? 0 : ts_node_end_byte(key) - kb; + bool is_set = kl == strlen("set") && strncmp(t->src + kb, "set", kl) == 0; + bool is_values = kl == strlen("values") && strncmp(t->src + kb, "values", kl) == 0; + /* set_list, set_sensitive and dynamic set values are not read: + * the release's values would be partial; a postrender may + * rewrite its images */ + if (tf_word(t, key, "set_list") || tf_word(t, key, "set_sensitive") || + tf_word(t, key, "postrender") || tf_dynamic_set(t, a)) { + ok = false; + break; + } + if (pass == 0 && is_values && tf_type(a, "attribute")) { + TSNode list = tf_unwrap(ts_node_named_child(a, SKIP_ONE)); + ok = tf_type(list, "tuple"); + for (uint32_t j = 0; ok && j < ts_node_named_child_count(list); j++) { + TSNode it = ts_node_named_child(list, j); + if (tf_type(it, "tuple_start") || tf_type(it, "tuple_end")) { + continue; + } + yyjson_mut_val *v = tf_values_item(t, doc, it, t->arena, files, nfiles); + ok = v && yyjson_mut_is_obj(v); + root = ok ? cbm_helm_values_coalesce(doc, root, v) : root; + } + } else if (pass == SKIP_ONE && is_set && tf_type(a, "block")) { + yyjson_mut_val *entry = tf_set_block(t, doc, a); + ok = entry && tf_set_entry(doc, root, entry); + } else if (pass == SKIP_ONE && is_set && tf_type(a, "attribute")) { + /* provider v3: set = [{ name, value, type }, ...] */ + yyjson_mut_val *list = tf_json_v(t, doc, ts_node_named_child(a, SKIP_ONE), true); + ok = yyjson_mut_is_arr(list); + size_t idx, max; + yyjson_mut_val *entry; + yyjson_mut_arr_foreach(list, idx, max, entry) { + ok = ok && tf_set_entry(doc, root, entry); + } + } + } + } + char *out = NULL; + if (ok) { + size_t len = 0; + out = yyjson_mut_val_write_opts(root, 0, cbm_json_writer_allocator(), &len, NULL); + } + if (doc) { + yyjson_mut_doc_free(doc); + } + return out; +} + +static void tf_release_push(tf_t *t, int64_t src, int64_t chart, TSNode body) { + cbm_pipeline_ctx_t *ctx = t->ctx; + if (ctx->tf_release_n == ctx->tf_release_cap) { + int cap = ctx->tf_release_cap ? ctx->tf_release_cap * PAIR_LEN : CBM_SZ_16; + cbm_tf_release_t *g = + cbm_realloc(CBM_MEM_CLASS_OTHER, ctx->tf_releases, (size_t)cap * sizeof(*g)); + if (!g) { + return; + } + ctx->tf_releases = g; + ctx->tf_release_cap = cap; + } + const char **files = NULL; + int nfiles = 0; + char *values = tf_release_values(t, body, &files, &nfiles); + ctx->tf_releases[ctx->tf_release_n++] = (cbm_tf_release_t){src, chart, values}; + /* the values files it reads, like a kustomization's valuesFile: an edit of + * one alone renders the release again (pipeline_incremental.c) */ + for (int i = 0; i < nfiles; i++) { + const char *mqn = cbm_fqn_module(t->arena, ctx->project_name, files[i]); + const cbm_gbuf_node_t *mod = mqn ? cbm_gbuf_find_by_qn(ctx->gbuf, mqn) : NULL; + if (mod && mod->id != src) { + cbm_gbuf_insert_edge(ctx->gbuf, src, mod->id, "IMPORTS", "{\"local_name\":\"values\"}"); + t->edges++; + } + } +} + static void tf_helm_release(tf_t *t, TSNode block, const char *name) { TSNode body = {0}; for (uint32_t i = 0; i < ts_node_named_child_count(block); i++) { @@ -320,6 +839,7 @@ static void tf_helm_release(tf_t *t, TSNode block, const char *name) { external ? ",\"external\":true" : "", (int)ts_node_start_point(block).row + SKIP_ONE); cbm_gbuf_insert_edge(t->ctx->gbuf, src, tgt, "DEPLOYS", props); t->edges++; + tf_release_push(t, src, tgt, body); } /* The text of a string_lit label: "helm_release" -> helm_release. */ diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 0c41d8c3b..306ff6226 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -525,6 +525,8 @@ const char *cbm_registry_value_target(const cbm_registry_t *r, const char *qn); int cbm_registry_find_by_name(const cbm_registry_t *r, const char *name, const char ***out, int *count); +bool cbm_registry_has_type_named(const cbm_registry_t *r, const char *name); + /* Return total number of entries. */ int cbm_registry_size(const cbm_registry_t *r); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 4182ca711..ce1c89f6f 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -2468,8 +2468,9 @@ static bool closure_dir_has_compose(const char *repo_path, const char *rel, cons return found; } -/* The nearest directory at or above `rel`'s that holds a Chart.yaml, into out - * ("" for the repository root); false when none does. */ +/* The nearest directory at or above `rel`'s that holds a Chart.yaml, or is a + * werf `.helm` chart, into out ("" for the repository root); false when none + * is. */ static bool closure_chart_dir_of(const char *repo_path, const char *rel, char *out, size_t sz) { snprintf(out, sz, "%s", rel); char *cut = strrchr(out, '/'); @@ -2487,6 +2488,9 @@ static bool closure_chart_dir_of(const char *repo_path, const char *rel, char *o return true; } } + if (out[0] && cbm_werf_helm_dir(repo_path, out)) { + return true; + } if (!out[0]) { return false; } @@ -2494,6 +2498,33 @@ static bool closure_chart_dir_of(const char *repo_path, const char *rel, char *o } } +/* Is `rel` a file a release reads as values (an IMPORTS {local_name: + * valuesFile | values} edge reaches its node)? */ +static bool closure_release_values(cbm_store_t *store, const char *project, const char *rel) { + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_file(store, project, rel, &nodes, &count) != CBM_STORE_OK) { + return false; + } + bool hit = false; + for (int i = 0; i < count && !hit; i++) { + cbm_edge_t *edges = NULL; + int ne = 0; + if (cbm_store_find_edges_by_target_type(store, nodes[i].id, "IMPORTS", &edges, &ne) != + CBM_STORE_OK) { + continue; + } + for (int j = 0; j < ne && !hit; j++) { + const char *p = edges[j].properties_json; + hit = p && (strstr(p, "\"local_name\":\"valuesFile\"") || + strstr(p, "\"local_name\":\"values\"")); + } + cbm_store_free_edges(edges, ne); + } + cbm_store_free_nodes(nodes, count); + return hit; +} + /* Kubernetes inputs reach past their own file (pass_k8s.c): a kustomization * decides which manifests deploy together and in which namespace, a * Chart.yaml links other charts, and a manifest's Services select workloads @@ -2510,7 +2541,9 @@ static const char *closure_k8s_decline(cbm_store_t *store, const char *project, if (cbm_is_kustomize_file(base) || strcmp(base, "Kustomization") == 0) { return "k8s_bundle_input_changed"; } - if (strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0) { + /* A werf.yaml names the chart of the `.helm` dir beside it. */ + if (strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0 || + strcmp(base, "werf.yaml") == 0 || strcmp(base, "werf.yml") == 0) { return "helm_chart_changed"; } /* A chart's values file CONFIGURES it and its templates/ are what it @@ -2530,6 +2563,12 @@ static const char *closure_k8s_decline(cbm_store_t *store, const char *project, if (strcmp(base, ".env") == 0 && closure_dir_has_compose(repo_path, rel, base)) { return "compose_env_changed"; } + /* A release's values file -- a kustomization's valuesFile, a Terraform + * helm_release's file(...) -- feeds the images the release renders, + * which a repair of the values file alone does not redraw. */ + if (closure_release_values(store, project, rel)) { + return "helm_release_values_changed"; + } cbm_node_t *nodes = NULL; int count = 0; if (cbm_store_find_nodes_by_file(store, project, rel, &nodes, &count) != CBM_STORE_OK) { diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index a1ab9c6e5..27c37098d 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -407,6 +407,16 @@ typedef struct { int reader_count; } cbm_envjoin_base_t; +/* A Terraform helm_release (pass_terraform.c): the chart it DEPLOYS and its + * values -- `values = [...]` coalesced in order, then `set` -- as JSON; NULL + * when a part is not a literal (an interpolation, a variable). pass_k8s.c + * renders the chart's images with them. */ +typedef struct cbm_tf_release { + int64_t src; /* the resource.helm_release node */ + int64_t chart; /* the Chart node */ + char *values; /* owned (cbm_free); NULL: unknown */ +} cbm_tf_release_t; + /* Shared context passed to each pass function. * Derived from cbm_pipeline_t fields during run. */ typedef struct { @@ -513,6 +523,12 @@ typedef struct { const struct cbm_doclink_scope *doc_link_base; int doc_link_base_count; bool doc_links_failed; + + /* The run's Terraform helm_releases (pass_terraform.c fills them, the K8s + * pass reads and frees them right after). */ + cbm_tf_release_t *tf_releases; + int tf_release_n; + int tf_release_cap; } cbm_pipeline_ctx_t; /* Origin-aware LSP returns only success/failure. Never retain partial output @@ -1097,6 +1113,9 @@ bool cbm_is_k8s_manifest(const char *name, const char *content); /* A Helm values file name: values.yaml / .yml, values-.yaml / .yml * (pass_k8s.c; beside Chart.yaml it CONFIGURES the chart). */ bool cbm_helm_values_name(const char *base); +/* Is `dir` (repository relative) a werf chart: a `.helm` directory without a + * Chart.yaml whose parent holds werf.yaml (pass_k8s.c)? */ +bool cbm_werf_helm_dir(const char *repo_path, const char *dir); /* Secret detection */ bool cbm_is_secret_binding(const char *key, const char *value); @@ -1296,6 +1315,7 @@ typedef struct { const char *via; /* "skaffold" / "compose" */ const char *image; /* the reference, variables expanded */ const char *dockerfile; /* repository path */ + const char *build_args; /* compose build.args as a JSON object; NULL: none */ int line; } cbm_build_site_t; typedef void (*cbm_build_site_fn)(const cbm_build_site_t *site, void *ud); @@ -1306,10 +1326,17 @@ typedef void (*cbm_build_site_fn)(const cbm_build_site_t *site, void *ud); * depends_on and links list, and the names of the environment variables its * environment sets (map keys / list items' names -- never a value), with * their 1-based lines and the service name each value addresses (a bare - * name, `name:port`, a URL's host; NULL: none -- cbm_env_address). */ + * name, `name:port`, a URL's host; NULL: none -- cbm_env_address). + * image_default_of: the variables whose inline default (${X:-d}) named the + * image; image_override_of: those whose default the project's .env replaced + * (JSON arrays, NULL: none), and default_image the image those defaults + * name. */ typedef struct { const char *name; const char *image; + const char *image_default_of; + const char *image_override_of; + const char *default_image; const char *dockerfile; const char **deps; int ndeps; @@ -1377,6 +1404,12 @@ bool cbm_repo_rel_join(const char *dir, const char *v, char *out, size_t sz); * for an official image; registry "unknown" when the reference names no * host -- none is invented), made when new; 0 for scratch or no image. */ int64_t cbm_pipeline_image_node(cbm_pipeline_ctx_t *ctx, const char *ref); +/* Does `ref` follow the image reference grammar (distribution/reference): + * [host[:port]/]path[:tag][@algo:hex], path components lowercase alphanumerics + * joined by `.`, `_`, `__` or dashes, a tag of word characters, `.` and `-` + * (128 at most)? A value rendered from a template is checked before it names + * an Image (pass_docker.c). */ +bool cbm_image_ref_valid(const char *ref); /* A chart repository URL as an identity (pass_k8s.c): `url` trimmed, its * scheme and host lower-cased, trailing `/` and `.git` dropped. False for no @@ -1767,7 +1800,8 @@ int cbm_delta_drop_edges_from_label(cbm_store_t *store, const char *project, con const char *label); /* The edges that keep a file-less shared node alive. */ #define SH_PROCESS_EDGES ((const char *const[]){"SPAWNS", NULL}) -#define DK_IMAGE_EDGES ((const char *const[]){"DEPENDS_ON", "USAGE", NULL}) +/* An Image a release DEPLOYS (pass_k8s.c release images) may have no other edge. */ +#define DK_IMAGE_EDGES ((const char *const[]){"DEPENDS_ON", "USAGE", "DEPLOYS", NULL}) /* Pre-seed proxy nodes with their REAL database ids and move the gbuf id * watermark above MAX(id); returns that max id, or -1 on failure. */ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf); diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index a4ca4cece..3dd11d5da 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -1732,6 +1732,21 @@ const char *cbm_registry_unique_field_qn(const cbm_registry_t *r, const char *me return only; } +bool cbm_registry_has_type_named(const cbm_registry_t *r, const char *name) { + if (!r || !name || !name[0]) { + return false; + } + const char **qns = NULL; + int count = 0; + cbm_registry_find_by_name(r, name, &qns, &count); + for (int i = 0; i < count; i++) { + if (cbm_label_is_type_like(cbm_registry_label_of(r, qns[i]))) { + return true; + } + } + return false; +} + int cbm_registry_size(const cbm_registry_t *r) { return r ? (int)cbm_ht_count(r->exact) : 0; } diff --git a/src/watcher/watcher.c b/src/watcher/watcher.c index f8d04c4c6..ac7c47b7f 100644 --- a/src/watcher/watcher.c +++ b/src/watcher/watcher.c @@ -522,14 +522,28 @@ static watcher_git_status_t git_repo_status(cbm_watcher_t *w, project_state_t *s * a `.git` file (the gitlink form used by linked worktrees and initialized * submodules). Deliberately a filesystem check, not a git invocation — the whole * point is to learn something `rev-parse` cannot tell us, because it walks up. */ +/* Preserve stat's link-following behavior while inspecting UTF-8 paths. */ +static int watcher_path_info(const char *path, cbm_path_info_t *info) { + int status = cbm_path_info_utf8(path, info); + if (status == CBM_PATH_INFO_OK && info->is_symlink) { + char canonical[CBM_SZ_4K]; + if (!cbm_canonical_path(path, canonical, sizeof(canonical))) { + return CBM_PATH_INFO_UNAVAILABLE; + } + status = cbm_path_info_utf8(canonical, info); + } + return status; +} + static bool git_has_own_dot_git(const char *root_path) { char path[CBM_SZ_4K]; int written = snprintf(path, sizeof(path), "%s/.git", root_path); if (written <= 0 || (size_t)written >= sizeof(path)) { return false; } - struct stat st; - return stat(path, &st) == 0 && (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)); + cbm_path_info_t info; + return watcher_path_info(path, &info) == CBM_PATH_INFO_OK && + (info.is_directory || info.is_regular); } /* Does the ancestor repository actually track anything inside this directory? @@ -990,13 +1004,17 @@ static root_status_t root_status(const char *root_path, int *out_errno) { if (!root_path) { return ROOT_UNCERTAIN; } - struct stat st; - if (stat(root_path, &st) == 0) { - /* Exists but is no longer a directory → the root directory is gone. */ - return S_ISDIR(st.st_mode) ? ROOT_PRESENT : ROOT_MISSING; + cbm_path_info_t info; + int status = watcher_path_info(root_path, &info); + if (status == CBM_PATH_INFO_OK) { + return info.is_directory ? ROOT_PRESENT : ROOT_MISSING; } +#ifdef _WIN32 + *out_errno = status == CBM_PATH_INFO_ABSENT ? ENOENT : EIO; +#else *out_errno = errno; - return cbm_watcher_root_missing_errno(errno) ? ROOT_MISSING : ROOT_UNCERTAIN; +#endif + return status == CBM_PATH_INFO_ABSENT ? ROOT_MISSING : ROOT_UNCERTAIN; } /* Sustained-absence window (seconds) before a missing root may be pruned. @@ -1338,8 +1356,8 @@ static void publish_status(project_state_t *s, bool scanned) { /* Init baseline for a project: check if git, get HEAD, count files */ static bool init_baseline(cbm_watcher_t *w, project_state_t *s) { - struct stat st; - if (stat(s->root_path, &st) != 0) { + cbm_path_info_t info; + if (watcher_path_info(s->root_path, &info) != CBM_PATH_INFO_OK || !info.is_directory) { cbm_log_warn("watcher.root_gone", "project", s->project_name, "path", s->root_path); s->baseline_done = true; s->is_git = false; diff --git a/tests/fixtures/complexity/Kubernetes/chart/Chart.yaml b/tests/fixtures/complexity/Kubernetes/chart/Chart.yaml new file mode 100644 index 000000000..7da879473 --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/chart/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: cx +version: 0.1.0 +appVersion: "1.0" diff --git a/tests/fixtures/complexity/Kubernetes/chart/templates/_helpers.tpl b/tests/fixtures/complexity/Kubernetes/chart/templates/_helpers.tpl new file mode 100644 index 000000000..f4aff46fe --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/chart/templates/_helpers.tpl @@ -0,0 +1,4 @@ +{{- define "cx.image" -}} +{{- $reg := .registry -}} +{{- printf "%s/%s:%s" $reg .repository .tag -}} +{{- end -}} diff --git a/tests/fixtures/complexity/Kubernetes/chart/templates/deploy.yaml b/tests/fixtures/complexity/Kubernetes/chart/templates/deploy.yaml new file mode 100644 index 000000000..52c14349c --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/chart/templates/deploy.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cx +spec: + template: + spec: + containers: + - name: main + image: "{{ include "cx.image" .Values.image }}" + {{- range .Values.sidecars }} + - name: {{ .name }} + image: "{{ .image.repository }}:{{ .image.tag }}" + {{- end }} diff --git a/tests/fixtures/complexity/Kubernetes/chart/values.yaml b/tests/fixtures/complexity/Kubernetes/chart/values.yaml new file mode 100644 index 000000000..c279c38a7 --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/chart/values.yaml @@ -0,0 +1,12 @@ +common: &common + pullPolicy: IfNotPresent +image: + <<: *common + registry: ghcr.io + repository: acme/cx + tag: "1.0" +sidecars: + - name: a + image: {repository: ghcr.io/acme/cx-side, tag: "2.0"} + - name: b + image: {repository: ghcr.io/acme/cx-side, tag: "2.1"} diff --git a/tests/fixtures/select-lanes/cases.tsv b/tests/fixtures/select-lanes/cases.tsv index 54a8404c2..29d07ca33 100644 --- a/tests/fixtures/select-lanes/cases.tsv +++ b/tests/fixtures/select-lanes/cases.tsv @@ -10,7 +10,7 @@ npm-readme-runs-the-test-legs pkg/npm/README.md T1-tests false lint lint-mem sec issue-template-is-docs .github/ISSUE_TEMPLATE/bug.yml T0a-docs false contracts security-static local-infra test-infrastructure/run.sh T0b-nonproduct false contracts lint lint-mem security-static lint-config scripts/memory-core-baseline.txt T0b-nonproduct false contracts lint lint-mem security-static -graph-ui-adds-guards graph-ui/src/App.tsx T0b-nonproduct false contracts lint lint-mem security-static smoke-ubuntu windows-guards +graph-ui-adds-guards graph-ui/src/App.tsx T0b-nonproduct false contracts lint lint-mem security-static smoke-ubuntu windows-guards codeql-js memwaste-tools-add-memwaste scripts/memwaste.sh T0b-nonproduct false contracts lint lint-mem memwaste security-static non-gating-script-runs-contracts-leg scripts/benchmark-index.sh T0b-nonproduct false contracts lint lint-mem security-static shard-completeness unix-x86 test-impact-tooling scripts/test-impact/replay-selector.sh T0b-nonproduct false contracts lint lint-mem security-static shard-completeness unix-x86 @@ -19,19 +19,19 @@ test-fixture tests/fixtures/cpp_include/a.h T1-tests false lint lint-mem securit contract-test tests/test_version_metadata_contract.sh T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows fuzz-harness tests/fuzz/fuzz_cypher.c T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows docs-plus-tests README.md tests/test_store.c T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows -lsp-language internal/cbm/lsp/go_lsp.c T2-extract false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows -lsp-core internal/cbm/lsp/type_registry.c T2-extract false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows -grammar-is-extract-not-full internal/cbm/vendored/grammars/go/parser.c T2-extract false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows -extraction internal/cbm/extract_calls.c T2-extract false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows -pipeline src/pipeline/pass_calls.c T3-mid false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows -mcp-plus-tests src/mcp/mcp.c tests/test_mcp.c T3-mid false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows -mid-plus-memwaste-tools src/store/store.c scripts/memwaste-report.py T3-mid false codeql-gate lint lint-mem memwaste security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows -daemon src/daemon/application.c T4-platform false FULL -msan -diag -memwaste -main-c-is-cli src/main.c T4-platform false FULL -msan -diag -memwaste -foundation-non-memory src/foundation/log.c T4-platform false FULL -msan -diag -memwaste -windows-guard-tests tests/windows/guard.ps1 T4-platform false FULL -msan -diag -memwaste -smoke-harness scripts/smoke-local.sh T4-platform false FULL -msan -diag -memwaste -package-wrappers pkg/npm/package.json T4-platform false FULL -msan -diag -memwaste +lsp-language internal/cbm/lsp/go_lsp.c T2-extract false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp +lsp-core internal/cbm/lsp/type_registry.c T2-extract false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp +grammar-is-extract-not-full internal/cbm/vendored/grammars/go/parser.c T2-extract false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp +extraction internal/cbm/extract_calls.c T2-extract false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp +pipeline src/pipeline/pass_calls.c T3-mid false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp +mcp-plus-tests src/mcp/mcp.c tests/test_mcp.c T3-mid false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp +mid-plus-memwaste-tools src/store/store.c scripts/memwaste-report.py T3-mid false lint lint-mem memwaste security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp codeql-py +daemon src/daemon/application.c T4-platform false FULL -msan -diag -memwaste -codeql-js -codeql-py -codeql-actions +main-c-is-cli src/main.c T4-platform false FULL -msan -diag -memwaste -codeql-js -codeql-py -codeql-actions +foundation-non-memory src/foundation/log.c T4-platform false FULL -msan -diag -memwaste -codeql-js -codeql-py -codeql-actions +windows-guard-tests tests/windows/guard.ps1 T4-platform false FULL -msan -diag -memwaste -codeql-cpp -codeql-js -codeql-py -codeql-actions +smoke-harness scripts/smoke-local.sh T4-platform false FULL -msan -diag -memwaste -codeql-cpp -codeql-js -codeql-py -codeql-actions +package-wrappers pkg/npm/package.json T4-platform false FULL -msan -diag -memwaste -codeql-cpp -codeql-js -codeql-py -codeql-actions memory-core src/foundation/mem_core.c T5-core true FULL gate-script scripts/ci/require-all-green.sh T5-core true FULL the-selector-itself scripts/ci/select-lanes.sh T5-core true FULL @@ -50,8 +50,8 @@ lint-config-file .cppcheck T0b-nonproduct false contracts lint lint-mem security script-c-tests-read scripts/security-vendored.sh T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows engine-config-c-tests-read .codebase-memory.json T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows test-failure-report scripts/suite-failure-report.sh T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows -smoke-security-script scripts/security-strings.sh T4-platform false FULL -msan -diag -memwaste -shipped-notices-are-pkg scripts/gen-third-party-notices.sh T4-platform false FULL -msan -diag -memwaste +smoke-security-script scripts/security-strings.sh T4-platform false FULL -msan -diag -memwaste -codeql-cpp -codeql-js -codeql-py -codeql-actions +shipped-notices-are-pkg scripts/gen-third-party-notices.sh T4-platform false FULL -msan -diag -memwaste -codeql-cpp -codeql-js -codeql-py -codeql-actions non-gate-workflow-forces-full .github/workflows/stale.yml T0b-nonproduct true FULL harness-main-forces-full tests/test_main.c T1-tests true FULL harness-framework-forces-full tests/test_framework.h T1-tests true FULL @@ -59,3 +59,7 @@ harness-driver-forces-full scripts/test.sh T5-core true FULL harness-scheduler-forces-full scripts/run-test-wave.py T5-core true FULL lower-tier-plus-trigger-is-full docs/x.md src/pipeline/a.c tests/test_main.c T3-mid true FULL empty-list-fails-safe - T0a-docs true FULL +graph-ui-python-agent graph-ui/agents/agent.py T0b-nonproduct false codeql-py contracts lint lint-mem security-static smoke-ubuntu windows-guards +vendored-js-is-not-scanned graph-ui/vendor/lib.js T0b-nonproduct false contracts lint lint-mem security-static smoke-ubuntu windows-guards +vendored-only-c-scans-c vendored/sqlite3/sqlite3.c T5-core true FULL +codeql-config-is-full .github/codeql/codeql-config.yml T5-core true FULL diff --git a/tests/test_agent_clients.c b/tests/test_agent_clients.c index df8515e73..f1bc3699e 100644 --- a/tests/test_agent_clients.c +++ b/tests/test_agent_clients.c @@ -1338,6 +1338,30 @@ TEST(client_adapter_opencode_exports_the_v2_default_definition_issue2077) { PASS(); } +/* augment() is awaited inline inside 'tool.execute.after', so a hook-augment + * that never exits would stall every grep/glob/read completion in the host + * session until the agent is restarted. The generated plugin must bound the + * wait: SIGKILL the child after AUGMENT_TIMEOUT_MS and resolve with '' (fail + * open). The promise keeps the first resolution, so the kill-triggered + * 'close' event arriving late cannot smuggle in partial output. Stream + * errors fire on child.stdin, not the child — the emitted listener and the + * guarded end() keep a failed write (EPIPE, ENOENT) from crashing the host + * as an unhandled 'error' event (reproduced under Bun 1.4 with a >64KB + * payload and a non-reading child). */ +TEST(client_adapter_opencode_bounds_a_hung_hook_augment_child) { + char *js = cbm_client_adapter_opencode("/usr/local/bin/codebase-memory-mcp"); + ASSERT_NOT_NULL(js); + ASSERT_NOT_NULL(strstr(js, "const AUGMENT_TIMEOUT_MS = 10000;")); + ASSERT_NOT_NULL(strstr(js, "const timer = setTimeout(")); + ASSERT_NOT_NULL(strstr(js, "child.kill('SIGKILL');")); + /* error and close must both stop the timer once they own the outcome. */ + ASSERT_NOT_NULL(strstr(js, "clearTimeout(timer);")); + /* stream-level failures must not escape as unhandled 'error' events. */ + ASSERT_NOT_NULL(strstr(js, "child.stdin.on('error', () => {});")); + free(js); + PASS(); +} + /* Empty/NULL inputs must not produce a module at all. */ TEST(client_adapter_rejects_missing_binary_path) { ASSERT_NULL(cbm_client_adapter_pi(NULL)); @@ -1385,5 +1409,6 @@ SUITE(agent_clients) { RUN_TEST(client_adapter_opencode_sends_the_required_hook_event); RUN_TEST(client_adapter_opencode_covers_lifecycle_read_and_compaction); RUN_TEST(client_adapter_opencode_exports_the_v2_default_definition_issue2077); + RUN_TEST(client_adapter_opencode_bounds_a_hung_hook_augment_child); RUN_TEST(client_adapter_rejects_missing_binary_path); } diff --git a/tests/test_codeql_retrace.sh b/tests/test_codeql_retrace.sh new file mode 100644 index 000000000..350271feb --- /dev/null +++ b/tests/test_codeql_retrace.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Exercise the real production Make rules with tiny C fixtures. CodeQL starts +# tracing after cbm-third-party, so every affected object must compile again, +# while unrelated objects must remain reused. No dry-run/timestamp assertions. +set -euo pipefail +ROOT="${1:-$(cd "$(dirname "$0")/.." && pwd)}" +python3 - "$ROOT" "$BASH" <<'PY' +import os +import pathlib +import re +import subprocess +import sys +import tempfile + +root, bash = pathlib.Path(sys.argv[1]), sys.argv[2] +failures = [] +with tempfile.TemporaryDirectory(prefix="cbm-codeql-retrace-") as directory: + work = pathlib.Path(directory) + + def write(path, text): + dest = work / path + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(text, encoding="utf-8") + + for path in ("Makefile.cbm", "scripts/ci/codeql-retrace-vendored.sh", + "scripts/ci/select-lanes.sh", ".github/codeql/codeql-config.yml"): + write(path, (root / path).read_text(encoding="utf-8")) + + # Nested and aliased includes deliberately escape the Makefile's shallow + # wildcard dependencies. The compiler really compiles these translation units. + write("vendored/mimalloc/src/static.c", + '#include "mimalloc/internal.h"\n#include "prim/osx/prim.c"\n') + write("vendored/mimalloc/include/mimalloc/internal.h", "#define MI_VALUE 7\n") + write("vendored/mimalloc/src/prim/osx/prim.c", '#include "../unix/prim.c"\n') + write("vendored/mimalloc/src/prim/unix/prim.c", "int mi_value(void) { return MI_VALUE; }\n") + write("internal/cbm/vendored/common/scanner.h", "#define SHARED_VALUE 3\n") + for language in ("go", "json"): + write(f"internal/cbm/grammar_{language}.c", + f'#include "vendored/grammars/{language}/parser.c"\n') + write(f"internal/cbm/vendored/grammars/{language}/parser.c", + '#include "nested/value.h"\n#include "../../common/scanner.h"\n' + f'int grammar_{language}(void) {{ return VALUE + SHARED_VALUE; }}\n') + write(f"internal/cbm/vendored/grammars/{language}/nested/value.h", "#define VALUE 2\n") + write("internal/cbm/vendored/grammars/shared/shared.h", "/* future shared grammar helper */\n") + write("internal/cbm/ts_runtime.c", '#include "vendored/ts_runtime/src/lib.c"\n') + write("internal/cbm/vendored/ts_runtime/src/lib.c", "int runtime_value(void) { return 1; }\n") + for path, symbol in (("vendored/sqlite3/sqlite3.c", "sqlite_value"), + ("vendored/tre/tre_all.c", "tre_value"), + ("internal/cbm/vendored/lz4/lz4.c", "lz4_value"), + ("internal/cbm/vendored/lz4/lz4hc.c", "lz4hc_value"), + ("internal/cbm/vendored/zstd/zstd.c", "zstd_value")): + write(path, f"int {symbol}(void) {{ return 1; }}\n") + write("internal/cbm/vendored/lz4/lz4.h", "/* shared LZ4 header */\n") + write("vendored/sqlite3/sqlite3.h", "/* sqlite header */\n") + write("internal/cbm/vendored/zstd/zstd.h", "/* zstd header */\n") + write("vendored/nomic/code_vectors_blob.S", ".text\n") + write("vendored/nomic/code_vectors.bin", "") + write("src/main.c", "int main(void) { return 0; }\n") + + def run(args): + proc = subprocess.run(args, cwd=work, capture_output=True, text=True) + if proc.returncode: + raise RuntimeError(f"{args}: exit {proc.returncode}\n{proc.stdout}\n{proc.stderr}") + return proc.stdout + + make = ["make", "--no-print-directory", "-f", "Makefile.cbm", "cbm-third-party", + "BUILD_DIR=build/c", "CC=" + os.environ.get("CC", "cc")] + + def compiled(output): + return set(re.findall(r" -c -o build/c/([^\s]+\.o)(?:\s|$)", output)) + + prebuilt = compiled(run(make)) + expected = {"prod_mimalloc.o", "prod_sqlite3.o", "prod_grammar_go.o", + "prod_grammar_json.o", "prod_ts_runtime.o", "prod_lz4.o", + "prod_lz4hc.o", "prod_zstd.o", "unixcoder_blob.o"} + # TRE is a Windows-only production object. All other objects are mandatory. + if prebuilt not in (expected, expected | {"prod_tre.o"}): + raise RuntimeError(f"incomplete prebuild: {sorted(prebuilt)}") + if compiled(run(make)): + raise RuntimeError("unchanged prebuild is not reused") + + cases = [ + ("nested allocator header", ["vendored/mimalloc/include/mimalloc/internal.h"], {"prod_mimalloc.o"}), + ("aliased allocator source", ["vendored/mimalloc/src/prim/unix/prim.c"], {"prod_mimalloc.o"}), + ("nested grammar header", ["internal/cbm/vendored/grammars/go/nested/value.h"], {"prod_grammar_go.o"}), + ("grammar wrapper", ["internal/cbm/grammar_go.c"], {"prod_grammar_go.o"}), + ("runtime wrapper", ["internal/cbm/ts_runtime.c"], {"prod_ts_runtime.o"}), + ("runtime source", ["internal/cbm/vendored/ts_runtime/src/lib.c"], {"prod_ts_runtime.o"}), + ("shared grammar header", ["internal/cbm/vendored/common/scanner.h"], {"prod_grammar_go.o", "prod_grammar_json.o"}), + ("unknown grammar family", ["internal/cbm/vendored/grammars/shared/shared.h"], {"prod_grammar_go.o", "prod_grammar_json.o"}), + ("LZ4 header", ["internal/cbm/vendored/lz4/lz4.h"], {"prod_lz4.o", "prod_lz4hc.o"}), + ("SQLite header", ["vendored/sqlite3/sqlite3.h"], {"prod_sqlite3.o"}), + ("zstd header", ["internal/cbm/vendored/zstd/zstd.h"], {"prod_zstd.o"}), + ("first-party only", ["src/main.c"], set()), + ("duplicate changes", ["internal/cbm/grammar_go.c", "internal/cbm/grammar_go.c"], {"prod_grammar_go.o"}), + ] + for label, paths, expected_objects in cases: + write("changed.txt", "\n".join(paths) + "\n") + run([bash, "scripts/ci/codeql-retrace-vendored.sh", "--changed", "changed.txt", "retrace.txt"]) + retrace = (work / "retrace.txt").read_text(encoding="utf-8").splitlines() + actual = compiled(run(make + [arg for path in retrace for arg in ("-W", path)])) + if actual != expected_objects: + failures.append(f"{label}: expected {sorted(expected_objects)}, compiled {sorted(actual)}") + else: + print(f"PASS {label}: {len(actual)}/{len(prebuilt)} third-party objects retraced") +if failures: + print("CODEQL RETRACE CONTRACT VIOLATED:\n " + "\n ".join(failures)) + sys.exit(1) +print("CodeQL retrace contract OK (real compiler + production Make rules)") +PY diff --git a/tests/test_cypher.c b/tests/test_cypher.c index e9d65b31f..ec3afbfc2 100644 --- a/tests/test_cypher.c +++ b/tests/test_cypher.c @@ -1917,6 +1917,32 @@ TEST(cypher_exists_has_outgoing_calls) { ASSERT_EQ(rc, 0); ASSERT_EQ(r.row_count, 2); cbm_cypher_result_free(&r); + + rc = cbm_cypher_execute(s, + "MATCH (a)-[:CALLS]->(b) WHERE b.name <> \"SubmitOrder\" " + "RETURN a.name, b.name", + "test", 0, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 2); + cbm_cypher_result_free(&r); + + rc = cbm_cypher_execute(s, + "MATCH (a)-[:CALLS]->(b) WHERE b.name = \"SubmitOrder\" OR " + "NOT b.name = \"SubmitOrder\" RETURN count(*)", + "test", 0, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 1); + ASSERT_STR_EQ(r.rows[0][0], "3"); + cbm_cypher_result_free(&r); + + rc = cbm_cypher_execute(s, + "MATCH (a)-[:CALLS]->(b) WITH a, b WHERE NOT b.name = " + "\"SubmitOrder\" RETURN a.name, b.name", + "test", 0, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 2); + cbm_cypher_result_free(&r); + cbm_store_close(s); PASS(); } @@ -3378,6 +3404,86 @@ TEST(cypher_exec_where_mixed_alias_xor) { PASS(); } +TEST(cypher_exec_where_not_target_node) { + cbm_store_t *s = setup_cypher_store(); + cbm_cypher_result_t r = {0}; + int rc = cbm_cypher_execute(s, + "MATCH (a)-[:CALLS]->(b) WHERE NOT b.name = \"SubmitOrder\" " + "RETURN a.name, b.name", + "test", 0, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 2); + cbm_cypher_result_free(&r); + cbm_store_close(s); + PASS(); +} + +TEST(cypher_exec_where_not_target_relationship) { + cbm_store_t *s = setup_cypher_multi_edge_store(); + cbm_cypher_result_t r = {0}; + int rc = cbm_cypher_execute(s, + "MATCH (a)-[r:HTTP_CALLS]->(b) WHERE NOT r.url_path = " + "\"/health\" RETURN r.url_path", + "testproj", 0, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 1); + ASSERT_STR_EQ(r.rows[0][0], "/api/orders"); + cbm_cypher_result_free(&r); + cbm_store_close(s); + PASS(); +} + +TEST(cypher_exec_where_not_source_node) { + cbm_store_t *s = setup_cypher_store(); + cbm_cypher_result_t r = {0}; + int rc = cbm_cypher_execute(s, + "MATCH (a)-[:CALLS]->(b) WHERE NOT a.name = " + "\"HandleOrder\" RETURN a.name, b.name", + "test", 0, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 1); + ASSERT_STR_EQ(r.rows[0][0], "ValidateOrder"); + cbm_cypher_result_free(&r); + cbm_store_close(s); + PASS(); +} + +TEST(cypher_exec_where_not_target_node_count) { + cbm_store_t *s = setup_cypher_store(); + cbm_cypher_result_t r = {0}; + int rc = cbm_cypher_execute(s, + "MATCH (a)-[:CALLS]->(b) WHERE NOT b.name = " + "\"SubmitOrder\" RETURN count(*)", + "test", 0, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 1); + ASSERT_STR_EQ(r.rows[0][0], "2"); + cbm_cypher_result_free(&r); + cbm_store_close(s); + PASS(); +} + +/* The early WHERE pass must wait for every alias a condition reads. A function + * condition is filed under its first argument's alias, so in + * coalesce(a.no_such_prop, b.name) the target b hid behind the bound a: the + * seed row read coalesce(NULL, ) as a definite mismatch and pruned + * every row before CALLS expansion, while the same predicate after WITH was + * right. */ +TEST(cypher_exec_where_func_arg_on_unbound_target) { + cbm_store_t *s = setup_cypher_store(); + cbm_cypher_result_t r = {0}; + int rc = cbm_cypher_execute(s, + "MATCH (a)-[:CALLS]->(b) WHERE coalesce(a.no_such_prop, b.name) = " + "\"SubmitOrder\" RETURN a.name", + "test", 0, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 1); + ASSERT_STR_EQ(r.rows[0][0], "ValidateOrder"); + cbm_cypher_result_free(&r); + cbm_store_close(s); + PASS(); +} + TEST(cypher_exec_where_in) { cbm_store_t *s = setup_cypher_store(); cbm_cypher_result_t r = {0}; @@ -6243,6 +6349,11 @@ SUITE(cypher) { RUN_TEST(cypher_exec_where_mixed_alias_and); RUN_TEST(cypher_exec_where_mixed_alias_or); RUN_TEST(cypher_exec_where_mixed_alias_xor); + RUN_TEST(cypher_exec_where_not_target_node); + RUN_TEST(cypher_exec_where_not_target_relationship); + RUN_TEST(cypher_exec_where_not_source_node); + RUN_TEST(cypher_exec_where_not_target_node_count); + RUN_TEST(cypher_exec_where_func_arg_on_unbound_target); RUN_TEST(cypher_exec_where_in); RUN_TEST(cypher_exec_where_not_in); RUN_TEST(cypher_exec_where_is_null); diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index 35ad5433c..ac03da3f3 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -2892,6 +2892,91 @@ TEST(daemon_ipc_local_participants_overlap_and_allow_modern_startup) { PASS(); } +TEST(daemon_ipc_windows_local_transition_release_reports_live_gate_contention) { +#ifndef _WIN32 + PASS(); +#else + static const char key[] = "6060717182829393"; + char parent[TEST_PATH_CAP] = {0}; + char canonical_parent[TEST_PATH_CAP] = {0}; + char runtime_dir[TEST_PATH_CAP] = {0}; + char legacy_pipe[CBM_DAEMON_IPC_WINDOWS_NAME_CAP] = {0}; + char legacy_startup[CBM_DAEMON_IPC_WINDOWS_NAME_CAP] = {0}; + cbm_daemon_ipc_endpoint_t *endpoint = NULL; + cbm_daemon_ipc_local_transition_t *transition = NULL; + cbm_daemon_ipc_startup_lock_t *startup = NULL; + + bool parent_ok = ipc_test_parent_new(parent, "local-release-busy") && + ipc_test_full_path(canonical_parent, parent); + bool names_ok = parent_ok && cbm_daemon_ipc_windows_legacy_names(canonical_parent, key, + legacy_pipe, legacy_startup); + if (names_ok) { + endpoint = cbm_daemon_ipc_endpoint_new(key, parent); + } + if (endpoint) { + ipc_test_copy_path(runtime_dir, cbm_daemon_ipc_endpoint_runtime_dir(endpoint)); + } + int acquired = + endpoint ? cbm_daemon_ipc_local_transition_try_acquire(endpoint, &transition) : -1; + int sealed = acquired == 1 ? cbm_daemon_ipc_local_transition_seal_legacy(transition) : -1; + int lifetime = + sealed == 1 ? cbm_daemon_ipc_local_transition_lifetime_probe(endpoint, transition) : -1; + bool begun = lifetime == 0 && cbm_daemon_ipc_local_transition_begin_work(transition); + int startup_acquired = begun ? cbm_daemon_ipc_startup_lock_try_acquire(endpoint, &startup) : -1; + int blocked = cbm_daemon_ipc_local_transition_try_release(&transition); + bool retained_while_busy = transition != NULL; + bool boolean_blocked = cbm_daemon_ipc_local_transition_release(&transition); + /* Keep the already-validated named mutex alive after the startup owner + * exits. Holding just this mutex exercises the second contention path. */ + wchar_t *legacy_name = names_ok ? cbm_utf8_to_wide(legacy_startup) : NULL; + HANDLE legacy_mutex = startup_acquired == 1 && legacy_name + ? OpenMutexW(SYNCHRONIZE | MUTEX_MODIFY_STATE, FALSE, legacy_name) + : NULL; + free(legacy_name); + bool startup_released = cbm_daemon_ipc_startup_lock_release(&startup); + DWORD mutex_wait = + startup_released && legacy_mutex ? WaitForSingleObject(legacy_mutex, 0) : WAIT_FAILED; + bool mutex_owned = mutex_wait == WAIT_OBJECT_0 || mutex_wait == WAIT_ABANDONED; + int legacy_blocked = + mutex_owned ? cbm_daemon_ipc_local_transition_try_release(&transition) : -1; + bool retained_while_legacy_busy = transition != NULL; + bool mutex_released = mutex_owned && ReleaseMutex(legacy_mutex); + if (legacy_mutex) { + (void)CloseHandle(legacy_mutex); + } + int legacy_retained = endpoint ? cbm_daemon_ipc_legacy_generation_probe(endpoint) : -1; + int released = cbm_daemon_ipc_local_transition_try_release(&transition); + bool consumed = transition == NULL; + int legacy_after = endpoint ? cbm_daemon_ipc_legacy_generation_probe(endpoint) : -1; + + (void)cbm_daemon_ipc_startup_lock_release(&startup); + (void)cbm_daemon_ipc_local_transition_release(&transition); + cbm_daemon_ipc_endpoint_free(endpoint); + ipc_test_remove_tree(runtime_dir, parent); + + ASSERT_TRUE(parent_ok); + ASSERT_TRUE(names_ok); + ASSERT_EQ(acquired, 1); + ASSERT_EQ(sealed, 1); + ASSERT_EQ(lifetime, 0); + ASSERT_TRUE(begun); + ASSERT_EQ(startup_acquired, 1); + ASSERT_EQ(blocked, 0); + ASSERT_TRUE(retained_while_busy); + ASSERT_FALSE(boolean_blocked); + ASSERT_TRUE(startup_released); + ASSERT_EQ(mutex_wait, WAIT_OBJECT_0); + ASSERT_EQ(legacy_blocked, 0); + ASSERT_TRUE(retained_while_legacy_busy); + ASSERT_TRUE(mutex_released); + ASSERT_EQ(legacy_retained, 1); + ASSERT_EQ(released, 1); + ASSERT_TRUE(consumed); + ASSERT_EQ(legacy_after, 0); + PASS(); +#endif +} + TEST(daemon_ipc_windows_local_transition_release_retries_retained_mutex) { #ifndef _WIN32 PASS(); @@ -2919,7 +3004,7 @@ TEST(daemon_ipc_windows_local_transition_release_retries_retained_mutex) { if (begun) { cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(1); } - bool first_release = cbm_daemon_ipc_local_transition_release(&transition); + int first_release = cbm_daemon_ipc_local_transition_try_release(&transition); bool retained_after_failure = transition != NULL; bool retry_release = cbm_daemon_ipc_local_transition_release(&transition); cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(0); @@ -2935,7 +3020,7 @@ TEST(daemon_ipc_windows_local_transition_release_retries_retained_mutex) { ASSERT_EQ(sealed, 1); ASSERT_EQ(lifetime, 0); ASSERT_TRUE(begun); - ASSERT_FALSE(first_release); + ASSERT_EQ(first_release, -1); ASSERT_TRUE(retained_after_failure); ASSERT_TRUE(retry_release); ASSERT_NULL(transition); @@ -5792,6 +5877,7 @@ SUITE(daemon_ipc) { RUN_TEST(daemon_ipc_activation_probe_ignores_matching_startup_claim_only); RUN_TEST(daemon_ipc_local_transition_coexists_with_active_daemon_lifetime); RUN_TEST(daemon_ipc_local_participants_overlap_and_allow_modern_startup); + RUN_TEST(daemon_ipc_windows_local_transition_release_reports_live_gate_contention); RUN_TEST(daemon_ipc_windows_local_transition_release_retries_retained_mutex); RUN_TEST(daemon_ipc_windows_startup_release_retains_retry_authority); RUN_TEST(daemon_ipc_frame_timeout_poisons_connection); diff --git a/tests/test_discover.c b/tests/test_discover.c index 3b1c5a213..c98ec26a8 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -2122,6 +2122,34 @@ static CBMLanguage discover_lang_of(const cbm_file_info_t *files, int count, con return CBM_LANG_COUNT; } +/* #1803: a symlink is skipped by policy on every walk. A bounded count must + * skip it too, not abort: aborting made auto-index admission refuse every + * repository containing any symlink ("unsafe_or_unavailable_path"), while + * an explicit index of the same tree succeeded. */ +TEST(discover_count_bounded_survives_a_symlink) { +#ifdef _WIN32 + SKIP_PLATFORM("Windows: symlinks need admin / symlink() unavailable"); +#else + char *base = th_mktempdir("cbm_disc_link"); + ASSERT(base != NULL); + + th_mkdir_p(TH_PATH(base, ".git")); + th_write_file(TH_PATH(base, "main.go"), "package main\n"); + th_write_file(TH_PATH(base, "sub/helper.go"), "package sub\n"); + ASSERT_EQ(symlink("sub", TH_PATH(base, "linkdir")), 0); + + cbm_discover_opts_t opts = {0}; + int count = -1; + /* deadline 0 = no deadline: only the symlink can decide this count. */ + cbm_discover_status_t status = cbm_discover_count_bounded(base, &opts, 50000, 0, &count); + ASSERT_EQ(status, CBM_DISCOVER_OK); + ASSERT_EQ(count, 2); + + th_cleanup(base); + PASS(); +#endif +} + /* Write raw bytes (may contain embedded NUL) to base/rel. Parent must exist. */ static int write_bytes_file(const char *path, const void *data, size_t n) { FILE *f = cbm_fopen(path, "wb"); @@ -2507,4 +2535,7 @@ SUITE(discover) { RUN_TEST(discover_nested_gitignore_below_ancestor_gitignore_issue1973); RUN_TEST(discover_nested_gitignore_negation_overrides_ancestor_pattern); RUN_TEST(discover_nested_gitignore_three_levels_deeper_file_wins); + + /* A symlink must not abort a bounded count (auto-index admission, #1803) */ + RUN_TEST(discover_count_bounded_survives_a_symlink); } diff --git a/tests/test_edge_imports.c b/tests/test_edge_imports.c index b5f1e7fbf..266cc255a 100644 --- a/tests/test_edge_imports.c +++ b/tests/test_edge_imports.c @@ -267,6 +267,102 @@ static int ei_edge_present(const EILangFile *files, int nfiles, const char *edge return got >= floor; } +typedef struct { + const char *importer; + const char *target; +} EIFileEdgeExpect; + +static int ei_file_edges_match(const EILangFile *files, int nfiles, const EIFileEdgeExpect *want, + int nwant, int expected_count) { + EILangProj lp; + cbm_store_t *store = ei_index_files(&lp, files, nfiles); + int ok = store && cbm_store_count_edges_by_type(store, lp.project, "IMPORTS") == expected_count; + if (store && !ok) { + int got = cbm_store_count_edges_by_type(store, lp.project, "IMPORTS"); + fprintf(stderr, " [IMPORTS] FAIL count=%d expected==%d\n", got, expected_count); + } + for (int i = 0; store && i < nwant; i++) { + int64_t source_id = ei_node_id_for_file_label(store, lp.project, want[i].importer, "File"); + cbm_edge_t *edges = NULL; + int edge_count = 0; + int found = 0; + if (source_id > 0 && + cbm_store_find_edges_by_source_type(store, source_id, "IMPORTS", &edges, &edge_count) == + CBM_STORE_OK) { + for (int j = 0; j < edge_count; j++) { + cbm_node_t target; + memset(&target, 0, sizeof(target)); + if (cbm_store_find_node_by_id(store, edges[j].target_id, &target) == CBM_STORE_OK) { + found = target.file_path && strcmp(target.file_path, want[i].target) == 0; + cbm_node_free_fields(&target); + } + if (found) { + break; + } + } + } + cbm_store_free_edges(edges, edge_count); + if (!found) { + fprintf(stderr, " [IMPORTS] FAIL %s -> %s\n", want[i].importer, want[i].target); + ok = 0; + } + } + ei_cleanup(&lp, store); + return ok; +} + +TEST(ei_relative_astro_css_imports_issue2558) { + static const EILangFile f[] = { + {"Badge.astro", "outside\n"}, + {"src/components/shared/Button.astro", + "---\nexport function buttonLabel(t: string): string { return t; }\n---\n