diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 761a101df..2ca74a2fb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -233,22 +233,18 @@ jobs: vt_api_key: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }} files: binaries/* - # VERSION + GH_TOKEN enable the pre-release ML false-positive tolerance: - # the gate may downgrade a single-engine Microsoft "!ml" verdict to - # TOLERATED, but only for -rc./-pre versions and only when hash-pinned - # Defender ENDPOINT evidence is attached to the draft release - # (defender-endpoint-verification.txt, from scripts/ci/av-endpoint-verify.sh). + # ZERO tolerance: any detection on any artifact blocks the release. + # False positives are resolved upstream (Microsoft false-positive + # submission for the exact hashes, then re-run this job — no rebuild), + # never by loosening this gate. - name: Wait for VirusTotal results env: VT_API_KEY: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }} VT_ANALYSIS: ${{ steps.virustotal.outputs.analysis }} - VERSION: ${{ inputs.version }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: scripts/ci/check-virustotal.sh - # Renders per-binary sha256 + VT links into the notes; when the gate - # tolerated an ML false positive it states that honestly instead of - # claiming "0 detections". + # Renders per-binary sha256 + VT links into the notes; only reachable + # after the zero-tolerance gate passed, so "0 detections" is verified. - name: Append VirusTotal scan links to release notes env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/ci/append-vt-notes.sh b/scripts/ci/append-vt-notes.sh index f3e7a41ff..56f5b42f4 100644 --- a/scripts/ci/append-vt-notes.sh +++ b/scripts/ci/append-vt-notes.sh @@ -1,28 +1,13 @@ #!/usr/bin/env bash # Append the Security Verification section to the release notes: per-binary -# sha256 + VirusTotal links, and — when the pre-release ML false-positive -# tolerance fired (scripts/ci/check-virustotal.sh wrote /tmp/vt_tolerated.tsv) -# — an honest annotation instead of a blanket "0 detections" claim. +# sha256 + VirusTotal links. This step only runs after check-virustotal.sh +# passed, and that gate is ZERO tolerance — any detection blocks the release — +# so "0 detections" here is a verified statement, never an assumption. # Expects: GH_TOKEN, VERSION; run from the verify job workspace (binaries/). set -euo pipefail -TOL=/tmp/vt_tolerated.tsv - TABLE="\n\n## Security Verification\n\n" -if [ -s "$TOL" ]; then - N=$(wc -l < "$TOL" | tr -d ' ') - TABLE+="All release binaries were scanned with 70+ antivirus engines. " - TABLE+="**${N} of them currently show a single machine-learning heuristic detection** " - TABLE+="(Microsoft \`Wacatac!ml\`) that we have verified to be a false positive: " - TABLE+="an up-to-date Microsoft Defender endpoint — signatures updated at scan time, " - TABLE+="real-time protection on — scans the identical bytes clean. The hash-pinned " - TABLE+="evidence is attached to this release as \`defender-endpoint-verification.txt\`, " - TABLE+="and a false-positive report for these exact hashes has been submitted to " - TABLE+="Microsoft. See **Antivirus false positives on release binaries** in the notes " - TABLE+="above for the full story and what we are doing about it.\n\n" -else - TABLE+="All release binaries scanned with 70+ antivirus engines — **0 detections**.\n\n" -fi +TABLE+="All release binaries scanned with 70+ antivirus engines — **0 detections**.\n\n" TABLE+="| Binary | SHA-256 | VirusTotal |\n" TABLE+="|--------|---------|------------|\n" @@ -34,13 +19,7 @@ for bin in binaries/codebase-memory-mcp-*; do label=$(echo "$name" | sed 's/^codebase-memory-mcp-//' | sed 's/\.exe$//') short="${sha256:0:20}..." vt_url="https://www.virustotal.com/gui/file/${sha256}/detection" - if [ -s "$TOL" ] && grep -q "^${name} " "$TOL"; then - completed=$(grep "^${name} " "$TOL" | cut -f6) - cell="[1/${completed} ⚠️ ML false positive, endpoint-verified clean](${vt_url})" - else - cell="[0 detections ✅](${vt_url})" - fi - TABLE+="| \`${label}\` | \`${short}\` | ${cell} |\n" + TABLE+="| \`${label}\` | \`${short}\` | [0 detections ✅](${vt_url}) |\n" done CURRENT=$(gh release view "$VERSION" \ diff --git a/scripts/ci/av-endpoint-verify.sh b/scripts/ci/av-endpoint-verify.sh deleted file mode 100644 index c996c6f9f..000000000 --- a/scripts/ci/av-endpoint-verify.sh +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env bash -# av-endpoint-verify.sh — verify release binaries on a REAL Microsoft Defender -# endpoint (the local Windows test VM) and produce the hash-pinned evidence -# file that scripts/ci/check-virustotal.sh accepts for pre-release ML false -# positives. -# -# VirusTotal's "Microsoft" engine and the shipping Defender product can -# disagree: VT runs a differently-configured engine instance, and for our -# large binaries it sits at an unstable ML decision boundary (Wacatac!ml came -# and went across four release cycles with no code-side cause). This script -# asks the authoritative source — Defender itself, signatures updated minutes -# before the scan, real-time protection on — about the exact bytes VT flagged, -# and records the answer: -# -# defender-endpoint-verification.txt -# # release/engine/signature header lines -# CLEAN|DETECTED -# -# The release gate downgrades a pre-release BLOCK to TOLERATED only when every -# flagged binary's sha256 appears here as CLEAN. -# -# Usage: scripts/ci/av-endpoint-verify.sh [--upload] -# version release tag holding the draft assets (e.g. v0.9.1-rc.1) -# asset-name archive basename without extension, as printed by the gate -# (e.g. codebase-memory-mcp-ui-linux-amd64) -# --upload attach/replace the evidence file on the (draft) release -# -# Runs on the dev machine: needs gh (authenticated) and the local Windows VM -# (test-infrastructure/vm/win.sh), plus ~300 MB VM disk per binary. -set -euo pipefail - -ROOT="$(cd "$(dirname "$0")/../.." && pwd)" -cd "$ROOT" -WIN=test-infrastructure/vm/win.sh -[ -x "$WIN" ] || { echo "av-endpoint-verify: $WIN not found — this runs on the dev machine" >&2; exit 2; } - -VERSION="${1:-}" -[ -n "$VERSION" ] || { echo "usage: scripts/ci/av-endpoint-verify.sh [--upload]" >&2; exit 2; } -shift -UPLOAD=false -NAMES=() -for arg in "$@"; do - case "$arg" in - --upload) UPLOAD=true ;; - *) NAMES+=("$arg") ;; - esac -done -[ "${#NAMES[@]}" -ge 1 ] || { echo "av-endpoint-verify: no asset names given" >&2; exit 2; } - -TMP="$(mktemp -d "${TMPDIR:-/tmp}/av-endpoint.XXXXXX")" -trap 'rm -rf "$TMP"' EXIT - -# ── Fetch + extract the exact release bytes ───────────────────────────────── -declare -a SHAS -for name in "${NAMES[@]}"; do - if gh release download "$VERSION" --pattern "$name.tar.gz" --dir "$TMP" 2>/dev/null; then - mkdir -p "$TMP/x-$name" - tar -xzf "$TMP/$name.tar.gz" -C "$TMP/x-$name" - BIN="$TMP/x-$name/codebase-memory-mcp" - elif gh release download "$VERSION" --pattern "$name.zip" --dir "$TMP" 2>/dev/null; then - mkdir -p "$TMP/x-$name" - unzip -oq "$TMP/$name.zip" -d "$TMP/x-$name" - BIN="$TMP/x-$name/codebase-memory-mcp.exe" - else - echo "av-endpoint-verify: no $name.tar.gz or $name.zip on release $VERSION" >&2 - exit 2 - fi - [ -f "$BIN" ] || { echo "av-endpoint-verify: archive $name held no binary" >&2; exit 2; } - mv "$BIN" "$TMP/probe-$name" - SHAS+=("$(shasum -a 256 "$TMP/probe-$name" | awk '{print $1}')") - echo "=== fetched $name (${SHAS[${#SHAS[@]}-1]}) ===" -done - -# ── Build the scan driver (pure ASCII: PowerShell 5.1 decodes BOM-less .ps1 -# as ANSI, and a stray non-ASCII byte corrupts the parse) ──────────────────── -PS1="$TMP/av-verify.ps1" -{ - echo '$mp = Join-Path $env:ProgramFiles "Windows Defender\MpCmdRun.exe"' - echo '& $mp -SignatureUpdate | Out-Null' - echo '$s = Get-MpComputerStatus' - echo 'Write-Output ("ENGINE=" + $s.AMEngineVersion)' - echo 'Write-Output ("SIGNATURES=" + $s.AntivirusSignatureVersion)' - echo 'Write-Output ("SIG_UPDATED=" + $s.AntivirusSignatureLastUpdated.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"))' - echo 'Write-Output ("RTP=" + $s.RealTimeProtectionEnabled)' - for name in "${NAMES[@]}"; do - echo '$f = "C:\Users\test\av-verify-'"$name"'"' - echo 'if (-not (Test-Path $f)) { Write-Output ("RESULT '"$name"' MISSING") } else {' - echo ' & $mp -Scan -ScanType 3 -File $f | Out-Null' - echo ' $code = $LASTEXITCODE' - # RTP quarantining the file between push and scan is itself a detection. - echo ' if ($code -eq 0 -and (Test-Path $f)) { Write-Output ("RESULT '"$name"' CLEAN") }' - echo ' else { Write-Output ("RESULT '"$name"' DETECTED code=" + $code) }' - echo ' Remove-Item $f -Force -ErrorAction SilentlyContinue' - echo '}' - done -} > "$PS1" -LC_ALL=C grep -q '[^ -~]' "$PS1" && { echo "av-endpoint-verify: driver is not pure ASCII" >&2; exit 2; } - -# ── Push and scan ─────────────────────────────────────────────────────────── -for name in "${NAMES[@]}"; do - echo "=== pushing $name to the VM ===" - "$WIN" push-file "$TMP/probe-$name" "C:/Users/test/av-verify-$name" -done -"$WIN" push-file "$PS1" "C:/Users/test/av-verify.ps1" -echo "=== updating signatures + scanning on the endpoint ===" -SCAN_OUT="$TMP/scan-out.txt" -"$WIN" sh "powershell -NoProfile -ExecutionPolicy Bypass -File C:/Users/test/av-verify.ps1" | tee "$SCAN_OUT" - -ENGINE=$(sed -n 's/^ENGINE=//p' "$SCAN_OUT" | tr -d '\r' | tail -1) -SIGS=$(sed -n 's/^SIGNATURES=//p' "$SCAN_OUT" | tr -d '\r' | tail -1) -SIG_UPDATED=$(sed -n 's/^SIG_UPDATED=//p' "$SCAN_OUT" | tr -d '\r' | tail -1) -RTP=$(sed -n 's/^RTP=//p' "$SCAN_OUT" | tr -d '\r' | tail -1) -[ -n "$ENGINE" ] && [ -n "$SIGS" ] || { echo "av-endpoint-verify: endpoint did not report engine/signature versions" >&2; exit 1; } -[ "$RTP" = "True" ] || { echo "av-endpoint-verify: real-time protection is OFF on the VM — evidence would be meaningless" >&2; exit 1; } - -# ── Write the evidence file ───────────────────────────────────────────────── -OUT="defender-endpoint-verification.txt" -{ - echo "# defender-endpoint-verification" - echo "# release: $VERSION" - echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "# venue: local Windows 11 ARM64 VM, Microsoft Defender endpoint, real-time protection on" - echo "# engine: $ENGINE" - echo "# signatures: $SIGS (updated $SIG_UPDATED)" -} > "$OUT" - -FAILED=false -for i in "${!NAMES[@]}"; do - name="${NAMES[$i]}" - verdict=$(sed -n "s/^RESULT $name //p" "$SCAN_OUT" | tr -d '\r' | tail -1) - case "$verdict" in - CLEAN) printf '%s %s CLEAN\n' "${SHAS[$i]}" "$name" >> "$OUT" ;; - *) - printf '%s %s DETECTED\n' "${SHAS[$i]}" "$name" >> "$OUT" - echo "av-endpoint-verify: $name verdict '$verdict' — NOT clean" >&2 - FAILED=true - ;; - esac -done - -echo "=== evidence written: $OUT ===" -cat "$OUT" - -if [ "$FAILED" = "true" ]; then - echo "av-endpoint-verify: Defender itself reports a detection — this is NOT" >&2 - echo "the false-positive pattern; do not release these binaries." >&2 - exit 1 -fi - -if [ "$UPLOAD" = "true" ]; then - gh release upload "$VERSION" "$OUT" --clobber - echo "=== uploaded $OUT to release $VERSION ===" -fi diff --git a/scripts/ci/check-virustotal.sh b/scripts/ci/check-virustotal.sh index 5e6c34b56..0c8ea5584 100755 --- a/scripts/ci/check-virustotal.sh +++ b/scripts/ci/check-virustotal.sh @@ -1,37 +1,10 @@ #!/usr/bin/env bash # Wait for VirusTotal scans to complete and check results. # Expects: VT_API_KEY, VT_ANALYSIS (comma-separated "file=URL" pairs) -# Optional: VERSION (release tag) + GH_TOKEN — these enable the PRE-RELEASE -# single-ML-detection tolerance described below. set -euo pipefail MIN_ENGINES=60 -rm -f /tmp/vt_gate_fail /tmp/vt_suspects.tsv /tmp/vt_tolerated.tsv - -# ── Pre-release ML false-positive tolerance ────────────────────────────────── -# A verdict may be downgraded from BLOCKED to TOLERATED only when ALL of: -# 1. the version is a pre-release (-rc. / -pre / -alpha / -beta), -# 2. every failing file was flagged by exactly ONE engine, -# 3. that engine is Microsoft and the verdict is a machine-learning -# heuristic (result string ends in "!ml") — never a signature name, -# 4. an up-to-date Microsoft Defender ENDPOINT has scanned the identical -# bytes clean, proven by a hash-pinned evidence file attached to the -# draft release (defender-endpoint-verification.txt, produced by -# scripts/ci/av-endpoint-verify.sh). -# -# Why this is sound: VT's "Microsoft" engine is a differently-configured -# instance of the same engine that ships in Windows, and for our large -# binaries it sits at an unstable ML decision boundary — across four release -# cycles the Wacatac!ml flag appeared and disappeared with no code-side cause -# (stripping, downloader removal and metadata changes each "fixed" it only -# until the next build). The shipping product's verdict on identical bytes, -# with signatures updated minutes before the scan, is the stronger statement -# of Microsoft's own estimate. Multi-engine hits, signature-named verdicts, -# non-Microsoft engines, timeouts and stable releases ALWAYS block. -TOLERANCE_ELIGIBLE=false -case "${VERSION:-}" in -*-rc.* | *-pre* | *-alpha* | *-beta*) TOLERANCE_ELIGIBLE=true ;; -esac +rm -f /tmp/vt_gate_fail echo "=== Waiting for VirusTotal scans to fully complete ===" @@ -95,22 +68,6 @@ print(f'{status},{malicious},{suspicious},{completed},{total}') echo "BLOCKED: $BASENAME flagged ($MALICIOUS malicious, $SUSPICIOUS suspicious / $COMPLETED engines)" echo " $URL" echo "FAIL" >> /tmp/vt_gate_fail - # Record WHO flagged it so the tolerance can classify the failure. - FLAG_INFO=$(echo "$RESULT" | python3 -c " -import json, sys -d = json.loads(sys.stdin.read()) -res = d.get('data', {}).get('attributes', {}).get('results', {}) -flagged = [(e, (r.get('result') or '?')) for e, r in res.items() - if r.get('category') in ('malicious', 'suspicious')] -engines = ';'.join(e for e, _ in flagged) -verdicts = ';'.join(v for _, v in flagged) -print(f'{len(flagged)}\t{engines}\t{verdicts}') -" 2>/dev/null || printf 'parse-error\tparse-error\tparse-error\n') - SHA256=$( (sha256sum "$FILE" 2>/dev/null || shasum -a 256 "$FILE" 2>/dev/null) \ - | awk 'NR==1{print $1}') - [ -n "$SHA256" ] || SHA256="unknown" - printf '%s\t%s\t%s\t%s\n' "$BASENAME" "$SHA256" "$FLAG_INFO" "$COMPLETED" \ - >> /tmp/vt_suspects.tsv else echo "OK: $BASENAME clean ($COMPLETED engines, 0 detections)" fi @@ -127,76 +84,8 @@ print(f'{len(flagged)}\t{engines}\t{verdicts}') fi done -if [ ! -f /tmp/vt_gate_fail ]; then - echo "=== All VirusTotal scans passed ===" - exit 0 -fi - -# ── Classify the failures against the tolerance conditions ────────────────── -FAILS=$(wc -l < /tmp/vt_gate_fail | tr -d ' ') -SUSPECTS=0 -[ -f /tmp/vt_suspects.tsv ] && SUSPECTS=$(wc -l < /tmp/vt_suspects.tsv | tr -d ' ') - -TOLERABLE="$TOLERANCE_ELIGIBLE" -# Timeouts and unparseable URLs produce FAIL lines with no suspect record — -# those are never tolerable, and the counts diverging catches them. -[ "$FAILS" -eq "$SUSPECTS" ] || TOLERABLE=false -if [ "$TOLERABLE" = "true" ]; then - while IFS=$'\t' read -r _name _sha nflagged engines verdicts _completed; do - [ "$nflagged" = "1" ] || TOLERABLE=false - [ "$engines" = "Microsoft" ] || TOLERABLE=false - case "$verdicts" in - *'!ml') ;; - *) TOLERABLE=false ;; - esac - done < /tmp/vt_suspects.tsv -fi - -if [ "$TOLERABLE" != "true" ]; then +if [ -f /tmp/vt_gate_fail ]; then echo "BLOCKED: One or more VirusTotal checks failed" exit 1 fi - -# ── Every failure matches the ML-FP pattern: require endpoint evidence ────── -EVIDENCE=/tmp/defender-endpoint-verification.txt -rm -f "$EVIDENCE" -if ! gh release download "${VERSION:?}" \ - --pattern defender-endpoint-verification.txt \ - --output "$EVIDENCE" --repo "${GITHUB_REPOSITORY:-DeusData/codebase-memory-mcp}" \ - 2>/dev/null; then - echo "BLOCKED: all detections match the pre-release ML false-positive pattern" - echo " (single engine, Microsoft, '!ml' verdict) but the release carries no" - echo " Defender endpoint evidence. To verify Microsoft's shipping product" - echo " against these exact bytes and attach the proof, run locally:" - echo "" - NAMES=$(cut -f1 /tmp/vt_suspects.tsv | sed 's/\.exe$//' | tr '\n' ' ') - echo " scripts/ci/av-endpoint-verify.sh $VERSION $NAMES--upload" - echo "" - echo " then RE-RUN THIS JOB (Re-run failed jobs — it does not rebuild, so" - echo " the bytes and hashes stay identical)." - exit 1 -fi - -MISSING=false -while IFS=$'\t' read -r name sha _rest; do - if grep -qE "^${sha}[[:space:]]+[^[:space:]]+[[:space:]]+CLEAN[[:space:]]*$" "$EVIDENCE"; then - echo "ENDPOINT-VERIFIED: $name ($sha) clean on a current Defender endpoint" - else - echo "BLOCKED: $name ($sha) has no CLEAN endpoint-evidence line" - MISSING=true - fi -done < /tmp/vt_suspects.tsv -if [ "$MISSING" = "true" ]; then - echo " Evidence file present but stale or incomplete — regenerate it with" - echo " scripts/ci/av-endpoint-verify.sh (hashes must match this build)." - exit 1 -fi - -cp /tmp/vt_suspects.tsv /tmp/vt_tolerated.tsv -echo "=== TOLERATED (pre-release ML false-positive policy) ===" -sed -n 's/^# //p' "$EVIDENCE" -echo "Every remaining engine reports clean; the detections above are Microsoft" -echo "ML heuristics that Microsoft's own shipping Defender does not reproduce." -echo "The release notes will state this transparently, with the evidence file" -echo "attached as a release asset." -exit 0 +echo "=== All VirusTotal scans passed ===" diff --git a/tests/test_vt_gate_tolerance_contract.sh b/tests/test_vt_gate_tolerance_contract.sh deleted file mode 100644 index 452b530cb..000000000 --- a/tests/test_vt_gate_tolerance_contract.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -# Contract: the VirusTotal gate's pre-release ML false-positive tolerance -# (scripts/ci/check-virustotal.sh) fails CLOSED in every direction. -# -# The tolerance may downgrade a BLOCK to TOLERATED only for the exact pattern -# pre-release version AND single engine AND Microsoft AND "!ml" verdict -# AND hash-pinned Defender endpoint evidence attached to the release. -# Anything else — stable version, multi-engine, signature-named verdict, -# non-Microsoft engine, missing or stale evidence — must still block. A bug in -# the tolerating direction ships a flagged binary, so every branch is pinned -# here with a stubbed VT API (curl) and release store (gh). -set -euo pipefail - -ROOT="$(cd "$(dirname "$0")/.." && pwd)" -GATE="$ROOT/scripts/ci/check-virustotal.sh" -FIX="$(mktemp -d "${TMPDIR:-/tmp}/vt-gate-contract.XXXXXX")" -trap 'rm -rf "$FIX"' EXIT - -fail() { echo "FAIL: $*" >&2; exit 1; } - -# ── Stub venue ────────────────────────────────────────────────────────────── -mkdir -p "$FIX/bin" "$FIX/work/binaries" "$FIX/responses" - -# curl: serve the canned VT analysis JSON selected by the analysis id in the URL. -cat > "$FIX/bin/curl" <<'EOF' -#!/usr/bin/env bash -url="${@: -1}" -id="${url##*/analyses/}" -resp="$STUB_RESPONSES/$id.json" -[ -f "$resp" ] || exit 22 -cat "$resp" -EOF - -# gh: "release download" delivers the evidence fixture if the scenario has one. -cat > "$FIX/bin/gh" <<'EOF' -#!/usr/bin/env bash -out="" -prev="" -for a in "$@"; do - [ "$prev" = "--output" ] && out="$a" - prev="$a" -done -[ -n "$STUB_EVIDENCE" ] && [ -f "$STUB_EVIDENCE" ] || exit 1 -cp "$STUB_EVIDENCE" "$out" -EOF -chmod +x "$FIX/bin/curl" "$FIX/bin/gh" - -mk_response() { # id malicious results-json - cat > "$FIX/responses/$1.json" < "$FIX/work/binaries/probe" -PROBE_SHA=$( (sha256sum "$FIX/work/binaries/probe" 2>/dev/null \ - || shasum -a 256 "$FIX/work/binaries/probe") | awk 'NR==1{print $1}') - -good_evidence() { - printf '# defender-endpoint-verification\n# engine: 1.1.1\n# signatures: 1.2.3\n%s probe CLEAN\n' "$PROBE_SHA" -} - -run_gate() { # version analysis-id evidence-file(optional) -> echoes exit code - rm -f /tmp/vt_gate_fail /tmp/vt_suspects.tsv /tmp/vt_tolerated.tsv - local rc=0 - (cd "$FIX/work" && - PATH="$FIX/bin:$PATH" \ - STUB_RESPONSES="$FIX/responses" \ - STUB_EVIDENCE="${3:-}" \ - GITHUB_REPOSITORY="stub/repo" \ - VT_API_KEY="stub" \ - VERSION="$1" \ - VT_ANALYSIS="binaries/probe=https://www.virustotal.com/gui/file-analysis/$2/detection" \ - bash "$GATE" >"$FIX/last.log" 2>&1) || rc=$? - echo "$rc" -} - -# ── 1. Clean scan passes, no tolerance involved ───────────────────────────── -[ "$(run_gate v0.9.1-rc.1 clean)" = "0" ] || fail "clean scan must pass" -[ ! -f /tmp/vt_tolerated.tsv ] || fail "clean scan must not write a tolerance record" - -# ── 2. Stable version: the exact ML-FP pattern with evidence still blocks ─── -good_evidence > "$FIX/evidence.txt" -[ "$(run_gate v0.9.1 msml "$FIX/evidence.txt")" != "0" ] || \ - fail "stable release must never tolerate" - -# ── 3. Pre-release + pattern + NO evidence blocks, with operator instructions ─ -[ "$(run_gate v0.9.1-rc.1 msml)" != "0" ] || fail "missing evidence must block" -grep -q "av-endpoint-verify.sh" "$FIX/last.log" || \ - fail "missing-evidence block must print the endpoint-verify instructions" - -# ── 4. Pre-release + pattern + matching evidence tolerates ────────────────── -[ "$(run_gate v0.9.1-rc.1 msml "$FIX/evidence.txt")" = "0" ] || \ - fail "pattern + evidence must tolerate (got a block)" -[ -s /tmp/vt_tolerated.tsv ] || fail "tolerance must record what it tolerated" -grep -q "TOLERATED" "$FIX/last.log" || fail "tolerance must announce itself" - -# ── 5. Stale evidence (wrong hash) blocks ─────────────────────────────────── -printf '0000000000000000000000000000000000000000000000000000000000000000 probe CLEAN\n' \ - > "$FIX/stale.txt" -[ "$(run_gate v0.9.1-rc.1 msml "$FIX/stale.txt")" != "0" ] || \ - fail "evidence for different bytes must block" - -# ── 6. Evidence that says DETECTED blocks ─────────────────────────────────── -printf '%s probe DETECTED\n' "$PROBE_SHA" > "$FIX/detected.txt" -[ "$(run_gate v0.9.1-rc.1 msml "$FIX/detected.txt")" != "0" ] || \ - fail "endpoint DETECTED must block" - -# ── 7. Signature-named Microsoft verdict blocks even with evidence ────────── -[ "$(run_gate v0.9.1-rc.1 mssig "$FIX/evidence.txt")" != "0" ] || \ - fail "non-!ml (signature) verdict must block" - -# ── 8. Non-Microsoft single-engine !ml blocks ─────────────────────────────── -[ "$(run_gate v0.9.1-rc.1 bkav "$FIX/evidence.txt")" != "0" ] || \ - fail "non-Microsoft engine must block" - -# ── 9. Multi-engine blocks ────────────────────────────────────────────────── -[ "$(run_gate v0.9.1-rc.1 two "$FIX/evidence.txt")" != "0" ] || \ - fail "multi-engine detection must block" - -rm -f /tmp/vt_gate_fail /tmp/vt_suspects.tsv /tmp/vt_tolerated.tsv -echo "PASS: VT gate tolerance fails closed in all nine pinned directions" diff --git a/tests/test_vt_gate_zero_tolerance_contract.sh b/tests/test_vt_gate_zero_tolerance_contract.sh new file mode 100644 index 000000000..15fd8ec7d --- /dev/null +++ b/tests/test_vt_gate_zero_tolerance_contract.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Contract: the VirusTotal release gate (scripts/ci/check-virustotal.sh) is +# ZERO tolerance — ANY detection on ANY artifact blocks, with no version +# carve-out, no engine carve-out, and no evidence side-channel. +# +# A tolerance path existed briefly (single-engine Microsoft "!ml" on +# pre-releases, endpoint-evidence-gated) and was deliberately reverted: the +# project does not ship binaries carrying a VirusTotal detection, full stop. +# False positives are resolved upstream (Microsoft FP submission for the exact +# hashes, then re-run the verify job — no rebuild), never by loosening the +# gate. This contract pins that decision against a stubbed VT API so a future +# "just tolerate this one heuristic" change has to consciously delete it. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +GATE="$ROOT/scripts/ci/check-virustotal.sh" +FIX="$(mktemp -d "${TMPDIR:-/tmp}/vt-gate-contract.XXXXXX")" +trap 'rm -rf "$FIX"' EXIT + +fail() { echo "FAIL: $*" >&2; exit 1; } + +# Tripwire for the specific reverted mechanism (behavioral checks below are +# the real contract; these names only catch that exact machinery returning). +for needle in defender-endpoint-verification av-endpoint-verify; do + if grep -q "$needle" "$GATE"; then + fail "gate references '$needle' — the reverted evidence side-channel is back" + fi +done + +mkdir -p "$FIX/bin" "$FIX/work/binaries" "$FIX/responses" + +# curl stub: serve the canned VT analysis JSON selected by the id in the URL. +cat > "$FIX/bin/curl" <<'EOF' +#!/usr/bin/env bash +url="${@: -1}" +id="${url##*/analyses/}" +resp="$STUB_RESPONSES/$id.json" +[ -f "$resp" ] || exit 22 +cat "$resp" +EOF +chmod +x "$FIX/bin/curl" + +mk_response() { # id malicious suspicious + cat > "$FIX/responses/$1.json" < "$FIX/work/binaries/probe" + +run_gate() { # version analysis-id -> echoes exit code + rm -f /tmp/vt_gate_fail + local rc=0 + (cd "$FIX/work" && + PATH="$FIX/bin:$PATH" \ + STUB_RESPONSES="$FIX/responses" \ + VT_API_KEY="stub" \ + VERSION="$1" \ + VT_ANALYSIS="binaries/probe=https://www.virustotal.com/gui/file-analysis/$2/detection" \ + bash "$GATE" >"$FIX/last.log" 2>&1) || rc=$? + echo "$rc" +} + +[ "$(run_gate v0.9.1 clean)" = "0" ] || fail "clean scan must pass" + +for version in v0.9.1 v0.9.1-rc.1 v0.9.1-pre1 v1.0.0-alpha.1; do + [ "$(run_gate "$version" one_malicious)" != "0" ] || \ + fail "1 malicious must block on $version (pre-release carve-outs are gone)" + [ "$(run_gate "$version" one_suspicious)" != "0" ] || \ + fail "1 suspicious must block on $version" +done + +rm -f /tmp/vt_gate_fail +echo "PASS: VT gate is zero tolerance — every detection blocks on every version"