mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-10-27 06:53:11 +08:00
Martin Vogel
f7d6d34457
#581 (query-path memory growth) does not reproduce in our soak: a 10-min, 200-file query-only run shows RSS flat-to-shrinking and committed plateauing (mimalloc already purges on free, purge_delay=0). Rather than ship speculative mitigations for a leak we cannot measure, this adds the observability to find it where it actually occurs, plus a continuous guard. - diag: the diagnostics writer now appends a PERSISTENT NDJSON memory trajectory (one sample / 5s: rss, committed, peak fields, page_faults, fd, queries), kept on exit (rotates to a .1 generation past 8 MB) so users can send the time series post-mortem. The previous latest-snapshot file was overwritten every 5s and deleted on stop. - ci: the soak now also runs a read-only query-leak leg (CBM_SOAK_MODE query-leak, no reindex/mutate) on every platform including Windows, so a regression that introduces a query-path leak is caught. soak-test.sh RESULTS_DIR is now env-overridable so both legs keep separate artifacts. - docs: README Troubleshooting and Diagnostics section + a bug-report field explain the no-telemetry stance and how to capture and share the trajectory. Build clean; unit suite 5714/0; trajectory verified to persist post-exit. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
234 lines
8.6 KiB
YAML
234 lines
8.6 KiB
YAML
# Reusable: soak tests (quick + ASan, all platforms)
|
|
name: Soak
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
duration_minutes:
|
|
description: 'Soak duration in minutes'
|
|
type: number
|
|
default: 10
|
|
run_asan:
|
|
description: 'Run ASan soak in addition to quick soak'
|
|
type: boolean
|
|
default: false
|
|
version:
|
|
description: 'Version string for build'
|
|
type: string
|
|
default: ''
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
soak-quick:
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- os: ubuntu-latest
|
|
goos: linux
|
|
goarch: amd64
|
|
cc: gcc
|
|
cxx: g++
|
|
- os: ubuntu-24.04-arm
|
|
goos: linux
|
|
goarch: arm64
|
|
cc: gcc
|
|
cxx: g++
|
|
- os: macos-14
|
|
goos: darwin
|
|
goarch: arm64
|
|
cc: cc
|
|
cxx: c++
|
|
- os: macos-15-intel
|
|
goos: darwin
|
|
goarch: amd64
|
|
cc: cc
|
|
cxx: c++
|
|
runs-on: ${{ matrix.os }}
|
|
# BUG FIX: this was hard-coded to 30, but the caller (nightly-soak.yml)
|
|
# passes duration_minutes: 240. GitHub killed the job at 30 min, so the
|
|
# "4h nightly soak" was SILENTLY TRUNCATED to 30 min and never once ran
|
|
# multi-hour. Budget must always exceed the passed duration; 300 covers
|
|
# the 240-min nightly with headroom (build + analysis + idle phases).
|
|
timeout-minutes: 300
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- name: Install deps (Linux)
|
|
if: startsWith(matrix.os, 'ubuntu')
|
|
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 git
|
|
- name: Build
|
|
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}
|
|
- name: Soak (${{ inputs.duration_minutes }} min)
|
|
run: scripts/soak-test.sh build/c/codebase-memory-mcp ${{ inputs.duration_minutes }}
|
|
# #581 guard: read-only soak (never reindex/mutate) so any memory growth is
|
|
# a query-path leak, not WAL/indexing. Reuses the build above.
|
|
- name: Query-leak soak (#581, read-only)
|
|
env:
|
|
CBM_SOAK_MODE: query-leak
|
|
RESULTS_DIR: soak-results-query-leak
|
|
run: scripts/soak-test.sh build/c/codebase-memory-mcp ${{ inputs.duration_minutes }} --skip-crash-test
|
|
- name: Upload metrics
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: soak-quick-${{ matrix.goos }}-${{ matrix.goarch }}
|
|
path: soak-results/
|
|
retention-days: 14
|
|
- name: Upload query-leak metrics
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: soak-query-leak-${{ matrix.goos }}-${{ matrix.goarch }}
|
|
path: soak-results-query-leak/
|
|
retention-days: 14
|
|
|
|
soak-quick-windows:
|
|
runs-on: windows-latest
|
|
# BUG FIX (same 30→240 mismatch as soak-quick above): the caller passes
|
|
# duration_minutes: 240, so a 30-min cap truncated the nightly soak here
|
|
# too. 300 covers the 240-min nightly with headroom.
|
|
timeout-minutes: 300
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2
|
|
with:
|
|
msystem: CLANG64
|
|
path-type: inherit
|
|
install: >-
|
|
mingw-w64-clang-x86_64-clang
|
|
mingw-w64-clang-x86_64-zlib
|
|
mingw-w64-clang-x86_64-python3
|
|
make
|
|
git
|
|
coreutils
|
|
- name: Build
|
|
shell: msys2 {0}
|
|
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++
|
|
- name: Soak (${{ inputs.duration_minutes }} min)
|
|
shell: msys2 {0}
|
|
run: |
|
|
BIN=build/c/codebase-memory-mcp
|
|
[ -f "${BIN}.exe" ] && BIN="${BIN}.exe"
|
|
scripts/soak-test.sh "$BIN" ${{ inputs.duration_minutes }}
|
|
# #581 guard on Windows — the platform the bug is reported on.
|
|
- name: Query-leak soak (#581, read-only)
|
|
shell: msys2 {0}
|
|
env:
|
|
CBM_SOAK_MODE: query-leak
|
|
RESULTS_DIR: soak-results-query-leak
|
|
run: |
|
|
BIN=build/c/codebase-memory-mcp
|
|
[ -f "${BIN}.exe" ] && BIN="${BIN}.exe"
|
|
scripts/soak-test.sh "$BIN" ${{ inputs.duration_minutes }} --skip-crash-test
|
|
- name: Upload metrics
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: soak-quick-windows-amd64
|
|
path: soak-results/
|
|
retention-days: 14
|
|
- name: Upload query-leak metrics
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: soak-query-leak-windows-amd64
|
|
path: soak-results-query-leak/
|
|
retention-days: 14
|
|
|
|
soak-asan:
|
|
if: ${{ inputs.run_asan }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- os: ubuntu-latest
|
|
goos: linux
|
|
goarch: amd64
|
|
cc: gcc
|
|
cxx: g++
|
|
- os: ubuntu-24.04-arm
|
|
goos: linux
|
|
goarch: arm64
|
|
cc: gcc
|
|
cxx: g++
|
|
- os: macos-14
|
|
goos: darwin
|
|
goarch: arm64
|
|
cc: cc
|
|
cxx: c++
|
|
- os: macos-15-intel
|
|
goos: darwin
|
|
goarch: amd64
|
|
cc: cc
|
|
cxx: c++
|
|
runs-on: ${{ matrix.os }}
|
|
# ASan soak runs a FIXED 15-min soak (hard-coded below, NOT driven by
|
|
# inputs.duration_minutes), but the ASan-instrumented build is slow and
|
|
# leak reporting adds teardown time. 60 keeps the budget comfortably above
|
|
# the 15-min run so it is never truncated. (Same class of bug as the
|
|
# soak-quick 30→240 mismatch above — keep the timeout above the run length.)
|
|
timeout-minutes: 240
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- name: Install deps (Linux)
|
|
if: startsWith(matrix.os, 'ubuntu')
|
|
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 git
|
|
- name: Build (ASan)
|
|
run: |
|
|
SANITIZE="-fsanitize=address,undefined -fno-omit-frame-pointer"
|
|
scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} EXTRA_CFLAGS="$SANITIZE" EXTRA_LDFLAGS="$SANITIZE"
|
|
- name: ASan soak (15 min)
|
|
env:
|
|
ASAN_OPTIONS: "detect_leaks=1:halt_on_error=0:log_path=soak-results/asan"
|
|
run: scripts/soak-test.sh build/c/codebase-memory-mcp 15
|
|
- name: Upload metrics
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: soak-asan-${{ matrix.goos }}-${{ matrix.goarch }}
|
|
path: soak-results/
|
|
retention-days: 14
|
|
|
|
soak-asan-windows:
|
|
if: ${{ inputs.run_asan }}
|
|
runs-on: windows-latest
|
|
# FIXED 15-min soak (hard-coded below). MSYS2/Wine + ASan build is the
|
|
# slowest path; 60 keeps the budget well above the run length so it is
|
|
# never truncated.
|
|
timeout-minutes: 240
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2
|
|
with:
|
|
msystem: CLANG64
|
|
path-type: inherit
|
|
install: >-
|
|
mingw-w64-clang-x86_64-clang
|
|
mingw-w64-clang-x86_64-zlib
|
|
mingw-w64-clang-x86_64-python3
|
|
make
|
|
git
|
|
coreutils
|
|
- name: Build (ASan)
|
|
shell: msys2 {0}
|
|
run: |
|
|
SANITIZE="-fsanitize=address,undefined -fno-omit-frame-pointer"
|
|
scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++ EXTRA_CFLAGS="$SANITIZE" EXTRA_LDFLAGS="$SANITIZE"
|
|
- name: ASan soak (15 min, no leak detection)
|
|
shell: msys2 {0}
|
|
env:
|
|
ASAN_OPTIONS: "detect_leaks=0:halt_on_error=0:log_path=soak-results/asan"
|
|
run: |
|
|
BIN=build/c/codebase-memory-mcp
|
|
[ -f "${BIN}.exe" ] && BIN="${BIN}.exe"
|
|
scripts/soak-test.sh "$BIN" 15
|
|
- name: Upload metrics
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: soak-asan-windows-amd64
|
|
path: soak-results/
|
|
retention-days: 14
|