Files
SkillSpector/scripts/compare_scan_accuracy.py
2026-08-21 10:08:55 -07:00

1876 lines
76 KiB
Python

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Compare two SkillSpector revisions on a human-adjudicated local corpus.
The corpus and generated report are intentionally external inputs. This keeps
private or disclosure-controlled fixtures out of the repository while making
the accuracy gate deterministic and reproducible.
"""
from __future__ import annotations
import argparse
import hashlib
import importlib
import io
import json
import os
import shutil
import signal
import subprocess
import sys
import tarfile
import tempfile
import threading
import time
from collections import Counter
from collections.abc import Iterator
from contextlib import contextmanager
from pathlib import Path
from typing import Any
SCHEMA_VERSION = 2
REQUIRED_CLASSIFICATIONS = frozenset({"maintained_benign", "approved_real_world"})
_SOURCE_SCAN_MODE = "--_source-bound-scan"
_GIT_TIMEOUT_SECONDS = 30.0
_IDENTITY_TIMEOUT_SECONDS = 120.0
_SCAN_TIMEOUT_SECONDS = 300.0
_GIT_STDOUT_LIMIT_BYTES = 64 * 1024 * 1024
_IDENTITY_STDOUT_LIMIT_BYTES = 16 * 1024 * 1024
_SCAN_STDOUT_LIMIT_BYTES = 32 * 1024 * 1024
_STDERR_LIMIT_BYTES = 1024 * 1024
_SOURCE_ARCHIVE_LIMIT_BYTES = 64 * 1024 * 1024
_SOURCE_ARCHIVE_MEMBER_LIMIT = 20_000
_ISOLATED_HOME_MARKER = "<fresh-owned-empty-home>"
_CONSOLE_ENTRYPOINT_BODY = b"""# -*- coding: utf-8 -*-
import sys
from skillspector.cli import app
if __name__ == "__main__":
if sys.argv[0].endswith("-script.pyw"):
sys.argv[0] = sys.argv[0][:-11]
elif sys.argv[0].endswith(".exe"):
sys.argv[0] = sys.argv[0][:-4]
sys.exit(app())
"""
_RUNTIME_IDENTITY_PROBE = r"""
import hashlib
import importlib.metadata
import json
import os
import platform
import sys
import urllib.parse
from pathlib import Path
MAX_DEPENDENCY_FILES = 200_000
MAX_DEPENDENCY_BYTES = 4 * 1024 * 1024 * 1024
EDITABLE_IGNORED_PARTS = {
".git", ".hg", ".svn", ".venv", "venv", "__pycache__",
".mypy_cache", ".pytest_cache", ".ruff_cache", ".tox", ".nox",
}
total_files = 0
total_bytes = 0
def hash_file(digest, label, path):
global total_files, total_bytes
if path.is_symlink() or not path.is_file():
raise RuntimeError(f"dependency identity path is not a regular file: {label}")
before = path.stat()
if total_files >= MAX_DEPENDENCY_FILES:
raise RuntimeError("installed dependency identity exceeds the file limit")
if before.st_size < 0 or total_bytes + before.st_size > MAX_DEPENDENCY_BYTES:
raise RuntimeError("installed dependency identity exceeds the byte limit")
encoded_label = label.encode("utf-8")
digest.update(len(encoded_label).to_bytes(8, "big"))
digest.update(encoded_label)
digest.update(before.st_size.to_bytes(8, "big"))
with path.open("rb") as stream:
opened_before = os.fstat(stream.fileno())
if (before.st_dev, before.st_ino, before.st_size) != (
opened_before.st_dev, opened_before.st_ino, opened_before.st_size
):
raise RuntimeError(f"dependency was swapped before hashing: {label}")
while True:
chunk = stream.read(1024 * 1024)
if not chunk:
break
digest.update(chunk)
opened_after = os.fstat(stream.fileno())
after = path.stat()
if (
before.st_dev,
before.st_ino,
before.st_size,
before.st_mtime_ns,
opened_before.st_dev,
opened_before.st_ino,
opened_before.st_size,
) != (
after.st_dev,
after.st_ino,
after.st_size,
after.st_mtime_ns,
opened_after.st_dev,
opened_after.st_ino,
opened_after.st_size,
):
raise RuntimeError(f"dependency changed while hashing: {label}")
total_files += 1
total_bytes += before.st_size
dependencies = []
seen_names = set()
for distribution in importlib.metadata.distributions():
name = distribution.metadata.get("Name")
if not isinstance(name, str) or not name.strip():
raise RuntimeError("installed distribution has no name")
normalized_name = name.strip().lower().replace("_", "-")
if normalized_name == "skillspector":
continue
if normalized_name in seen_names:
raise RuntimeError(f"duplicate installed distribution: {normalized_name}")
seen_names.add(normalized_name)
files = distribution.files
record = distribution.read_text("RECORD")
metadata = distribution.read_text("METADATA")
if files is None or record is None or metadata is None:
raise RuntimeError(f"installed distribution has incomplete identity metadata: {normalized_name}")
direct_url_text = distribution.read_text("direct_url.json") or ""
metadata_digest = hashlib.sha256()
for label, value in (("METADATA", metadata), ("RECORD", record), ("direct_url.json", direct_url_text)):
encoded_label = label.encode("utf-8")
encoded_value = value.encode("utf-8")
metadata_digest.update(len(encoded_label).to_bytes(8, "big"))
metadata_digest.update(encoded_label)
metadata_digest.update(len(encoded_value).to_bytes(8, "big"))
metadata_digest.update(encoded_value)
contents_digest = hashlib.sha256()
distribution_file_count = 0
distribution_bytes = 0
seen_paths = set()
for package_path in sorted(files, key=str):
located = Path(distribution.locate_file(package_path))
resolved = located.resolve(strict=True)
if resolved in seen_paths:
continue
seen_paths.add(resolved)
before_files = total_files
before_bytes = total_bytes
hash_file(contents_digest, f"installed:{package_path}", located)
distribution_file_count += total_files - before_files
distribution_bytes += total_bytes - before_bytes
editable = False
editable_file_count = 0
editable_bytes = 0
if direct_url_text:
direct_url = json.loads(direct_url_text)
if not isinstance(direct_url, dict):
raise RuntimeError(f"invalid direct_url.json for {normalized_name}")
directory_info = direct_url.get("dir_info")
editable = isinstance(directory_info, dict) and directory_info.get("editable") is True
if editable:
raw_url = direct_url.get("url")
if not isinstance(raw_url, str):
raise RuntimeError(f"editable dependency has no URL: {normalized_name}")
parsed = urllib.parse.urlsplit(raw_url)
if parsed.scheme != "file" or parsed.netloc not in {"", "localhost"}:
raise RuntimeError(f"editable dependency is not a local file target: {normalized_name}")
editable_root = Path(urllib.parse.unquote(parsed.path)).resolve(strict=True)
if not editable_root.is_dir():
raise RuntimeError(f"editable dependency target is not a directory: {normalized_name}")
for editable_path in sorted(editable_root.rglob("*")):
relative = editable_path.relative_to(editable_root)
if any(part in EDITABLE_IGNORED_PARTS for part in relative.parts):
continue
if editable_path.is_symlink():
raise RuntimeError(f"editable dependency contains a symlink: {normalized_name}")
if not editable_path.is_file():
continue
resolved = editable_path.resolve(strict=True)
if resolved in seen_paths:
continue
seen_paths.add(resolved)
before_files = total_files
before_bytes = total_bytes
hash_file(contents_digest, f"editable:{relative.as_posix()}", editable_path)
editable_file_count += total_files - before_files
editable_bytes += total_bytes - before_bytes
dependencies.append(
{
"name": normalized_name,
"version": distribution.version,
"recorded_file_count": len(files),
"distribution_metadata_sha256": f"sha256:{metadata_digest.hexdigest()}",
"installed_file_count": distribution_file_count,
"installed_bytes": distribution_bytes,
"installed_contents_sha256": f"sha256:{contents_digest.hexdigest()}",
"editable": editable,
"editable_file_count": editable_file_count,
"editable_bytes": editable_bytes,
}
)
payload = {
"python_version": platform.python_version(),
"python_implementation": platform.python_implementation(),
"python_cache_tag": sys.implementation.cache_tag,
"python_hexversion": sys.hexversion,
"platform": platform.platform(),
"machine": platform.machine(),
"byteorder": sys.byteorder,
"dependency_file_count": total_files,
"dependency_bytes": total_bytes,
"dependencies": sorted(dependencies, key=lambda item: item["name"]),
}
print(json.dumps(payload, sort_keys=True, separators=(",", ":")))
""".strip()
POLICY_FIELDS = (
"max_candidate_false_positives",
"max_candidate_false_negatives",
"max_false_positive_increase",
"max_false_negative_increase",
"max_per_rule_false_positive_increase",
"max_per_rule_false_negative_increase",
"max_per_cohort_false_positive_increase",
"max_per_cohort_false_negative_increase",
"max_per_case_false_positive_increase",
"max_per_case_false_negative_increase",
)
MANIFEST_FIELDS = frozenset({"schema_version", "material_regression_policy", "cases"})
CASE_FIELDS = frozenset({"id", "path", "classification", "expected_rules"})
APPROVAL_FIELDS = frozenset(
{
"schema_version",
"reviewer",
"rationale",
"corpus_identity",
"manifest_sha256",
"baseline_identity",
"candidate_identity",
"policy_sha256",
"violations",
"violations_sha256",
}
)
APPROVAL_IDENTITY_FIELDS = (
"revision",
"source_tree_git_oid",
"executable_sha256",
"python_executable_sha256",
"runtime_identity_sha256",
"dependency_identity_sha256",
"pyproject_sha256",
"lockfile_sha256",
"source_runner_sha256",
)
def _reject_duplicate_json_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
result: dict[str, Any] = {}
for key, value in pairs:
if key in result:
raise ValueError(f"Duplicate JSON field: {key}")
result[key] = value
return result
def _load_json_bytes(raw: bytes, path: Path) -> dict[str, Any]:
value = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_json_pairs)
if not isinstance(value, dict):
raise ValueError(f"Expected a JSON object in {path}")
return value
def _load_manifest(path: Path) -> tuple[dict[str, Any], bytes]:
raw = path.read_bytes()
return _load_json_bytes(raw, path), raw
def _json_sha256(value: object) -> str:
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
return f"sha256:{hashlib.sha256(encoded).hexdigest()}"
def _hash_record(digest: Any, label: bytes, value: bytes) -> None:
"""Add one unambiguous, length-delimited value to an evidence digest."""
digest.update(len(label).to_bytes(8, "big"))
digest.update(label)
digest.update(len(value).to_bytes(8, "big"))
digest.update(value)
def _resolve_case_path(corpus_root: Path, relative_path: str) -> Path:
root = corpus_root.resolve(strict=True)
target = (root / relative_path).resolve(strict=True)
if not target.is_relative_to(root) or not target.is_dir():
raise ValueError(f"Corpus case must be a directory below the corpus root: {relative_path}")
return target
def _corpus_identity(
corpus_root: Path,
cases: list[dict[str, Any]],
manifest_bytes: bytes,
) -> str:
"""Hash exact adjudication bytes plus every selected corpus path and byte."""
digest = hashlib.sha256()
_hash_record(digest, b"domain", b"skillspector-accuracy-corpus-v2")
_hash_record(digest, b"manifest", manifest_bytes)
seen: set[Path] = set()
root = corpus_root.resolve(strict=True)
for case in sorted(cases, key=lambda item: str(item["id"])):
target = _resolve_case_path(corpus_root, str(case["path"]))
for path in sorted(target.rglob("*")):
if path.is_symlink():
raise ValueError(f"Corpus snapshot does not follow symlinks: {path}")
if not path.is_file():
continue
resolved = path.resolve(strict=True)
if resolved in seen:
continue
seen.add(resolved)
relative = resolved.relative_to(root).as_posix().encode("utf-8")
_hash_record(digest, b"path", relative)
_hash_record(digest, b"contents", resolved.read_bytes())
return f"sha256:{digest.hexdigest()}"
def _file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return f"sha256:{digest.hexdigest()}"
def _terminate_process_tree(process: subprocess.Popen[bytes]) -> None:
"""Terminate a bounded subprocess and any children it placed in our session."""
try:
if os.name == "posix":
os.killpg(process.pid, signal.SIGKILL)
elif process.poll() is None: # pragma: no cover - exercised by Windows CI
process.kill()
except ProcessLookupError:
return
def _run_bounded(
command: list[str],
*,
cwd: Path,
env: dict[str, str] | None = None,
timeout_seconds: float,
stdout_limit_bytes: int,
stderr_limit_bytes: int,
) -> subprocess.CompletedProcess[bytes]:
"""Run a subprocess with wall-clock and captured-output limits.
Dedicated drain threads prevent a producer from filling a pipe. Each
thread retains at most its declared limit; the whole process group is
killed as soon as either stream crosses its bound.
"""
if timeout_seconds <= 0 or stdout_limit_bytes < 0 or stderr_limit_bytes < 0:
raise ValueError("Subprocess bounds must be positive")
process = subprocess.Popen(
command,
cwd=cwd,
env=env,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
start_new_session=(os.name == "posix"),
)
assert process.stdout is not None
assert process.stderr is not None
streams: dict[str, bytearray] = {"stdout": bytearray(), "stderr": bytearray()}
over_limit = threading.Event()
def drain(name: str, stream: Any, limit: int) -> None:
while True:
chunk = stream.read(64 * 1024)
if not chunk:
return
if len(streams[name]) + len(chunk) > limit:
remaining = max(0, limit - len(streams[name]))
streams[name].extend(chunk[:remaining])
over_limit.set()
return
streams[name].extend(chunk)
stdout_thread = threading.Thread(
target=drain,
args=("stdout", process.stdout, stdout_limit_bytes),
daemon=True,
)
stderr_thread = threading.Thread(
target=drain,
args=("stderr", process.stderr, stderr_limit_bytes),
daemon=True,
)
stdout_thread.start()
stderr_thread.start()
deadline = time.monotonic() + timeout_seconds
failure: str | None = None
try:
while process.poll() is None:
if over_limit.is_set():
failure = "output exceeded its byte limit"
_terminate_process_tree(process)
break
if time.monotonic() >= deadline:
failure = f"timed out after {timeout_seconds:g} seconds"
_terminate_process_tree(process)
break
time.sleep(0.01)
process.wait(timeout=5)
except subprocess.TimeoutExpired:
_terminate_process_tree(process)
process.wait()
failure = failure or "could not be terminated within its runtime limit"
finally:
stdout_thread.join(timeout=1)
stderr_thread.join(timeout=1)
if stdout_thread.is_alive() or stderr_thread.is_alive():
failure = failure or "left an output stream open after exit"
_terminate_process_tree(process)
process.stdout.close()
process.stderr.close()
stdout_thread.join(timeout=1)
stderr_thread.join(timeout=1)
if over_limit.is_set():
failure = "output exceeded its byte limit"
if failure:
raise RuntimeError(f"Bounded subprocess {failure}: {command[0]}")
return subprocess.CompletedProcess(
command,
process.returncode,
bytes(streams["stdout"]),
bytes(streams["stderr"]),
)
@contextmanager
def _fresh_owned_home() -> Iterator[Path]:
"""Yield a new private empty HOME and remove it without following symlinks."""
home = Path(tempfile.mkdtemp(prefix="skillspector-accuracy-home-"))
try:
home.chmod(0o700)
before = home.lstat()
if home.is_symlink() or not home.is_dir() or before.st_uid != os.geteuid():
raise RuntimeError("Could not create a private owned accuracy-gate HOME")
if any(home.iterdir()):
raise RuntimeError("Accuracy-gate HOME was not empty at creation")
yield home
finally:
if home.is_symlink():
home.unlink(missing_ok=True)
elif home.exists():
shutil.rmtree(home)
def _git_output(worktree: Path, *args: str) -> str:
try:
completed = _run_bounded(
["git", "-C", str(worktree), *args],
cwd=worktree,
timeout_seconds=_GIT_TIMEOUT_SECONDS,
stdout_limit_bytes=_GIT_STDOUT_LIMIT_BYTES,
stderr_limit_bytes=_STDERR_LIMIT_BYTES,
)
except RuntimeError as error:
raise ValueError(f"Cannot verify scanner worktree {worktree}: {error}") from error
if completed.returncode != 0:
detail = (
completed.stderr.decode("utf-8", errors="replace").strip()
or completed.stdout.decode("utf-8", errors="replace").strip()
or "git command failed"
)
raise ValueError(f"Cannot verify scanner worktree {worktree}: {detail}")
try:
return completed.stdout.decode("utf-8").strip()
except UnicodeDecodeError as error:
raise ValueError(f"Git returned non-UTF-8 identity output for {worktree}") from error
def _scan_environment(home: Path, *, disclose_home: bool = False) -> dict[str, str]:
"""Return the fixed, non-secret environment used by every accuracy scan."""
environment = {
"HOME": str(home) if disclose_home else _ISOLATED_HOME_MARKER,
"LANG": "C",
"LC_ALL": "C",
"PATH": os.defpath,
"PYTHONHASHSEED": "0",
"PYTHONNOUSERSITE": "1",
"SKILLSPECTOR_LOG_LEVEL": "WARNING",
"TZ": "UTC",
}
for name in ("SYSTEMROOT", "WINDIR"):
value = os.environ.get(name)
if value:
environment[name] = value
return environment
def _runtime_identity(interpreter: Path, worktree: Path) -> dict[str, Any]:
with _fresh_owned_home() as home:
execution_environment = _scan_environment(home, disclose_home=True)
completed = _run_bounded(
[str(interpreter), "-I", "-B", "-c", _RUNTIME_IDENTITY_PROBE],
cwd=worktree,
env=execution_environment,
timeout_seconds=_IDENTITY_TIMEOUT_SECONDS,
stdout_limit_bytes=_IDENTITY_STDOUT_LIMIT_BYTES,
stderr_limit_bytes=_STDERR_LIMIT_BYTES,
)
if completed.returncode != 0:
detail = (
completed.stderr.decode("utf-8", errors="replace").strip()
or completed.stdout.decode("utf-8", errors="replace").strip()
or "runtime probe failed"
)
raise ValueError(f"Cannot identify scanner dependency runtime: {detail}")
try:
identity = json.loads(completed.stdout.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as error:
raise ValueError("Scanner dependency runtime returned invalid identity JSON") from error
if not isinstance(identity, dict) or not isinstance(identity.get("dependencies"), list):
raise ValueError("Scanner dependency runtime returned an invalid identity object")
identity["probe_sha256"] = (
f"sha256:{hashlib.sha256(_RUNTIME_IDENTITY_PROBE.encode()).hexdigest()}"
)
evidence_environment = _scan_environment(Path(_ISOLATED_HOME_MARKER))
identity["environment"] = evidence_environment
identity["environment_sha256"] = _json_sha256(evidence_environment)
identity["dependency_identity_sha256"] = _json_sha256(identity["dependencies"])
identity["runtime_identity_sha256"] = _json_sha256(
{key: value for key, value in identity.items() if key != "runtime_identity_sha256"}
)
return identity
def _actual_source_files(source_root: Path) -> set[str]:
"""Inventory every regular source-tree file, including ignored files."""
actual: set[str] = set()
for directory, directory_names, file_names in os.walk(source_root, followlinks=False):
directory_path = Path(directory)
for name in directory_names:
child = directory_path / name
if child.is_symlink():
raise ValueError(f"Scanner source tree contains a symlink: {child}")
for name in file_names:
child = directory_path / name
if child.is_symlink() or not child.is_file():
raise ValueError(f"Scanner source tree contains a non-regular file: {child}")
actual.add(child.relative_to(source_root.parent).as_posix())
return actual
def _git_source_archive(worktree: Path, revision: str) -> bytes:
try:
completed = _run_bounded(
["git", "-C", str(worktree), "archive", "--format=tar", revision, "src"],
cwd=worktree,
timeout_seconds=_GIT_TIMEOUT_SECONDS,
stdout_limit_bytes=_SOURCE_ARCHIVE_LIMIT_BYTES,
stderr_limit_bytes=_STDERR_LIMIT_BYTES,
)
except RuntimeError as error:
raise ValueError(f"Cannot snapshot scanner revision {revision}: {error}") from error
if completed.returncode != 0:
detail = completed.stderr.decode("utf-8", errors="replace").strip()
raise ValueError(f"Cannot snapshot scanner revision {revision}: {detail}")
return completed.stdout
def _extract_source_archive(archive_bytes: bytes, destination: Path) -> Path:
"""Extract the Git-produced source archive without archive traversal semantics."""
destination.mkdir(mode=0o700, parents=True)
total_bytes = 0
with tarfile.open(fileobj=io.BytesIO(archive_bytes), mode="r:") as archive:
members = archive.getmembers()
if len(members) > _SOURCE_ARCHIVE_MEMBER_LIMIT:
raise ValueError("Scanner source snapshot exceeds the member limit")
for member in members:
pure_name = Path(member.name)
if pure_name.is_absolute() or ".." in pure_name.parts:
raise ValueError("Scanner source archive contains an unsafe path")
output = destination / pure_name
if not output.resolve(strict=False).is_relative_to(destination.resolve(strict=True)):
raise ValueError("Scanner source archive escapes its snapshot root")
if member.isdir():
output.mkdir(mode=0o700, parents=True, exist_ok=True)
continue
if not member.isfile() or member.issym() or member.islnk():
raise ValueError("Scanner source archive contains a non-regular member")
total_bytes += member.size
if total_bytes > _SOURCE_ARCHIVE_LIMIT_BYTES:
raise ValueError("Scanner source snapshot exceeds the byte limit")
stream = archive.extractfile(member)
if stream is None:
raise ValueError("Scanner source archive member could not be read")
output.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
with output.open("xb") as target:
shutil.copyfileobj(stream, target, length=1024 * 1024)
output.chmod(0o600)
source_root = (destination / "src").resolve(strict=True)
if not (source_root / "skillspector" / "__init__.py").is_file():
raise ValueError("Scanner source snapshot is missing the package entrypoint")
return source_root
def _copy_corpus_snapshot(
corpus_root: Path,
cases: list[dict[str, Any]],
destination: Path,
) -> None:
root = corpus_root.resolve(strict=True)
destination.mkdir(mode=0o700, parents=True)
copied: set[Path] = set()
for case in sorted(cases, key=lambda item: str(item["id"])):
source_case = _resolve_case_path(root, str(case["path"]))
relative_case = source_case.relative_to(root)
(destination / relative_case).mkdir(mode=0o700, parents=True, exist_ok=True)
for source in sorted(source_case.rglob("*")):
if source.is_symlink():
raise ValueError(f"Corpus snapshot does not follow symlinks: {source}")
relative = source.relative_to(root)
target = destination / relative
if source.is_dir():
target.mkdir(mode=0o700, parents=True, exist_ok=True)
continue
if not source.is_file():
raise ValueError(f"Corpus contains a non-regular path: {source}")
resolved = source.resolve(strict=True)
if resolved in copied:
continue
copied.add(resolved)
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
with source.open("rb") as source_stream, target.open("xb") as target_stream:
shutil.copyfileobj(source_stream, target_stream, length=1024 * 1024)
target.chmod(0o600)
@contextmanager
def _accuracy_snapshots(
*,
corpus_root: Path,
cases: list[dict[str, Any]],
manifest_bytes: bytes,
corpus_identity: str,
baseline_identity: dict[str, Any],
candidate_identity: dict[str, Any],
) -> Iterator[dict[str, Path]]:
"""Yield private immutable-by-construction inputs used by both scans."""
with tempfile.TemporaryDirectory(prefix="skillspector-accuracy-snapshot-") as temporary:
snapshot_root = Path(temporary)
snapshot_root.chmod(0o700)
if snapshot_root.lstat().st_uid != os.geteuid() or any(snapshot_root.iterdir()):
raise RuntimeError("Could not create a private empty accuracy snapshot root")
corpus_snapshot = snapshot_root / "corpus"
_copy_corpus_snapshot(corpus_root, cases, corpus_snapshot)
if _corpus_identity(corpus_snapshot, cases, manifest_bytes) != corpus_identity:
raise ValueError("Accuracy corpus changed while creating its private snapshot")
baseline_source = _extract_source_archive(
_git_source_archive(
Path(baseline_identity["worktree"]),
str(baseline_identity["revision"]),
),
snapshot_root / "baseline",
)
candidate_source = _extract_source_archive(
_git_source_archive(
Path(candidate_identity["worktree"]),
str(candidate_identity["revision"]),
),
snapshot_root / "candidate",
)
runner = snapshot_root / "compare_scan_accuracy.py"
runner.write_bytes(Path(__file__).resolve(strict=True).read_bytes())
runner.chmod(0o500)
expected_runner_sha256 = baseline_identity["source_runner_sha256"]
if (
expected_runner_sha256 != candidate_identity["source_runner_sha256"]
or _file_sha256(runner) != expected_runner_sha256
):
raise ValueError("Accuracy source runner changed while creating its private snapshot")
yield {
"corpus_root": corpus_snapshot,
"baseline_source": baseline_source,
"candidate_source": candidate_source,
"runner": runner,
}
def _console_python(executable: Path, worktree: Path) -> Path:
"""Validate the generated entrypoint and return its worktree-local Python."""
raw = executable.read_bytes()
shebang, separator, body = raw.partition(b"\n")
if not separator or body != _CONSOLE_ENTRYPOINT_BODY:
raise ValueError(
f"Scanner executable is not the expected immutable SkillSpector entrypoint: {executable}"
)
try:
shebang_text = shebang.decode("utf-8")
except UnicodeDecodeError as error:
raise ValueError(f"Scanner executable has an invalid shebang: {executable}") from error
if not shebang_text.startswith("#!"):
raise ValueError(f"Scanner executable has no Python shebang: {executable}")
interpreter_text = shebang_text[2:]
if not interpreter_text or any(character.isspace() for character in interpreter_text):
raise ValueError(f"Scanner executable has an ambiguous Python shebang: {executable}")
interpreter = Path(interpreter_text)
if not interpreter.is_absolute() or interpreter.parent != executable.parent:
raise ValueError(
f"Scanner executable must use a Python interpreter beside the entrypoint: {executable}"
)
# Check the lexical path before resolving a normal virtualenv interpreter
# symlink to its shared base runtime.
if not interpreter.is_relative_to(worktree):
raise ValueError(f"Scanner Python interpreter must be inside its worktree: {interpreter}")
resolved = interpreter.resolve(strict=True)
if not resolved.is_file() or not os.access(resolved, os.X_OK):
raise ValueError(f"Scanner Python interpreter is not executable: {resolved}")
return interpreter
def _source_bound_command(
*,
executable: Path,
target: Path | str,
worktree: Path,
source_root: Path | None = None,
runner: Path | None = None,
) -> list[str]:
"""Build the exact isolated command that imports the declared worktree source."""
interpreter = _console_python(executable, worktree)
return [
str(interpreter),
"-I",
"-B",
str((runner or Path(__file__)).resolve(strict=True)),
_SOURCE_SCAN_MODE,
str((source_root or (worktree / "src")).resolve(strict=True)),
str(target),
]
def _run_source_bound_scan(arguments: list[str]) -> int:
"""Internal subprocess mode: import only SkillSpector from the requested source tree."""
if len(arguments) != 2:
print("accuracy gate source runner received invalid arguments", file=sys.stderr)
return 2
source_root = Path(arguments[0]).resolve(strict=True)
expected_package = (source_root / "skillspector").resolve(strict=True)
target = Path(arguments[1]).resolve(strict=True)
# -I prevents cwd/PYTHONPATH/user-site shadowing. The explicit first path and
# fresh module namespace ensure an ignored installed package cannot win.
sys.path.insert(0, str(source_root))
for module_name in tuple(sys.modules):
if module_name == "skillspector" or module_name.startswith("skillspector."):
del sys.modules[module_name]
try:
skillspector = importlib.import_module("skillspector")
package_location = getattr(skillspector, "__file__", None)
if not isinstance(package_location, str):
raise RuntimeError("scanner package import has no source file")
package_file = Path(package_location).resolve(strict=True)
expected_package_file = (expected_package / "__init__.py").resolve(strict=True)
if package_file != expected_package_file:
raise RuntimeError(f"scanner package import is not revision source: {package_file}")
skillspector_cli = importlib.import_module("skillspector.cli")
cli_location = getattr(skillspector_cli, "__file__", None)
if not isinstance(cli_location, str):
raise RuntimeError("scanner CLI import has no source file")
cli_file = Path(cli_location).resolve(strict=True)
expected_cli_file = (expected_package / "cli.py").resolve(strict=True)
if cli_file != expected_cli_file:
raise RuntimeError(f"scanner CLI import is not revision source: {cli_file}")
app = skillspector_cli.app
except Exception as error:
print(f"accuracy gate source binding error: {error}", file=sys.stderr)
return 2
sys.argv = ["skillspector", "scan", str(target), "--format", "json", "--no-llm"]
result = app()
return result if isinstance(result, int) else 0
def _resolve_scanner_identity(
*,
executable: Path,
worktree: Path,
revision: str,
) -> dict[str, Any]:
"""Bind a revision label to one clean worktree and executable byte identity."""
if not revision or revision != revision.strip():
raise ValueError("Scanner revision must be a non-empty exact commit identity")
root = worktree.resolve(strict=True)
executable_path = executable.resolve(strict=True)
if not executable_path.is_file():
raise ValueError(f"Scanner executable is not a file: {executable_path}")
if not os.access(executable_path, os.X_OK):
raise ValueError(f"Scanner executable is not executable: {executable_path}")
if not executable_path.is_relative_to(root):
raise ValueError(f"Scanner executable must be inside its worktree: {executable_path}")
actual_root = Path(_git_output(root, "rev-parse", "--show-toplevel")).resolve(strict=True)
if actual_root != root:
raise ValueError(f"Scanner worktree must be the Git root: {root}")
actual_revision = _git_output(root, "rev-parse", "HEAD").lower()
if revision.strip().lower() != actual_revision:
raise ValueError(
f"Scanner revision mismatch for {root}: declared {revision}, actual {actual_revision}"
)
dirty = _git_output(root, "status", "--porcelain", "--untracked-files=all")
if dirty:
raise ValueError(f"Scanner worktree has changes and cannot be identified: {root}")
source_root = (root / "src").resolve(strict=True)
package_root = (source_root / "skillspector").resolve(strict=True)
if not package_root.is_dir():
raise ValueError(f"Scanner worktree has no SkillSpector source package: {package_root}")
source_tree_oid = _git_output(root, "rev-parse", f"{actual_revision}:src/skillspector")
tracked_source_files = set(
_git_output(
root,
"ls-tree",
"-r",
"--name-only",
actual_revision,
"--",
"src",
).splitlines()
)
required_source_files = {"src/skillspector/__init__.py", "src/skillspector/cli.py"}
if not required_source_files.issubset(tracked_source_files):
raise ValueError("Scanner revision is missing its tracked SkillSpector package entrypoints")
actual_source_files = _actual_source_files(source_root)
if actual_source_files != tracked_source_files:
unexpected = sorted(actual_source_files - tracked_source_files)
missing = sorted(tracked_source_files - actual_source_files)
detail_parts = []
if unexpected:
detail_parts.append("unexpected: " + ", ".join(unexpected[:5]))
if missing:
detail_parts.append("missing: " + ", ".join(missing[:5]))
raise ValueError(
"Scanner source inventory differs from the committed revision ("
+ "; ".join(detail_parts)
+ ")"
)
interpreter = _console_python(executable_path, root)
runner = Path(__file__).resolve(strict=True)
runtime_identity = _runtime_identity(interpreter, root)
pyproject = (root / "pyproject.toml").resolve(strict=True)
lockfile = (root / "uv.lock").resolve(strict=True)
return {
"declared_revision": revision,
"resolved_revision": actual_revision,
"revision": actual_revision,
"worktree": str(root),
"executable": str(executable_path),
"executable_relative_path": executable_path.relative_to(root).as_posix(),
"executable_sha256": _file_sha256(executable_path),
"python_executable": str(interpreter),
"python_executable_resolved": str(interpreter.resolve(strict=True)),
"python_executable_sha256": _file_sha256(interpreter),
"runtime_identity": runtime_identity,
"runtime_identity_sha256": runtime_identity["runtime_identity_sha256"],
"dependency_identity_sha256": runtime_identity["dependency_identity_sha256"],
"environment_sha256": runtime_identity["environment_sha256"],
"pyproject_sha256": _file_sha256(pyproject),
"lockfile": lockfile.name,
"lockfile_sha256": _file_sha256(lockfile),
"source_root": str(source_root),
"source_tree_git_oid": source_tree_oid,
"source_tree_revision": actual_revision,
"source_binding": "isolated-worktree-source-import",
"source_runner": str(runner),
"source_runner_sha256": _file_sha256(runner),
"worktree_clean": True,
# Compatibility field retained from the initial identity contract.
"tracked_worktree_clean": True,
}
def _run_scan(
executable: Path,
target: Path,
worktree: Path,
*,
source_root: Path | None = None,
runner: Path | None = None,
) -> dict[str, Any]:
command = _source_bound_command(
executable=executable,
target=target,
worktree=worktree,
source_root=source_root,
runner=runner,
)
with _fresh_owned_home() as home:
completed = _run_bounded(
command,
cwd=worktree,
env=_scan_environment(home, disclose_home=True),
timeout_seconds=_SCAN_TIMEOUT_SECONDS,
stdout_limit_bytes=_SCAN_STDOUT_LIMIT_BYTES,
stderr_limit_bytes=_STDERR_LIMIT_BYTES,
)
if completed.returncode not in {0, 1}:
raise RuntimeError(
f"Scanner exited {completed.returncode} for {target.name}: "
f"{completed.stderr.decode('utf-8', errors='replace').strip()}"
)
try:
report = json.loads(
completed.stdout.decode("utf-8"),
object_pairs_hook=_reject_duplicate_json_pairs,
)
except (UnicodeDecodeError, json.JSONDecodeError) as error:
raise RuntimeError(f"Scanner returned invalid JSON for {target.name}") from error
if not isinstance(report, dict):
raise RuntimeError(f"Scanner returned a non-object report for {target.name}")
return report
def _nonnegative_integer(value: object, field: str) -> int:
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
raise ValueError(f"Scanner JSON report has an invalid {field} field")
return value
def _validate_complete_report(report: dict[str, Any]) -> None:
"""Reject any failed, partial, or structurally ambiguous accuracy input."""
if report.get("execution_successful") is not True:
raise ValueError("Scanner JSON report is not execution-successful")
completeness = report.get("analysis_completeness")
if not isinstance(completeness, dict):
raise ValueError("Scanner JSON report has no analysis_completeness object")
if completeness.get("execution_successful") is not True:
raise ValueError("Scanner analysis completeness is not execution-successful")
if completeness.get("is_complete") is not True or completeness.get("status") != "complete":
raise ValueError("Scanner JSON report is not analysis-complete")
total_components = _nonnegative_integer(
completeness.get("total_components"),
"analysis_completeness.total_components",
)
scanned_components = _nonnegative_integer(
completeness.get("scanned_components"),
"analysis_completeness.scanned_components",
)
fully_inspected = _nonnegative_integer(
completeness.get("fully_inspected_files"),
"analysis_completeness.fully_inspected_files",
)
partially_inspected = _nonnegative_integer(
completeness.get("partially_inspected_files"),
"analysis_completeness.partially_inspected_files",
)
entirely_uninspected = _nonnegative_integer(
completeness.get("entirely_uninspected_files"),
"analysis_completeness.entirely_uninspected_files",
)
if scanned_components != total_components or partially_inspected or entirely_uninspected:
raise ValueError("Scanner JSON report completeness counters describe incomplete coverage")
if fully_inspected != total_components:
raise ValueError("Scanner JSON report has inconsistent fully-inspected coverage")
coverage = completeness.get("coverage_percent")
if not isinstance(coverage, (int, float)) or isinstance(coverage, bool) or coverage != 100:
raise ValueError("Scanner JSON report coverage_percent must be exactly 100")
for field in ("ledger_exceptions", "scope_exclusions", "limitations"):
value = completeness.get(field)
if not isinstance(value, list) or value:
raise ValueError(f"Scanner JSON report has non-empty or invalid {field}")
analyzer_statuses = completeness.get("analyzer_statuses")
if not isinstance(analyzer_statuses, list):
raise ValueError("Scanner JSON report has invalid analyzer_statuses")
for index, status in enumerate(analyzer_statuses):
if (
not isinstance(status, dict)
or not isinstance(status.get("analyzer_id"), str)
or not status["analyzer_id"].strip()
or status.get("status") not in {"completed", "not_applicable", "disabled"}
):
raise ValueError(f"Scanner JSON report has an invalid analyzer status at index {index}")
def _rule_counts(report: dict[str, Any], selected_rules: frozenset[str]) -> Counter[str]:
_validate_complete_report(report)
if "issues" not in report:
raise ValueError("Scanner JSON report is missing the 'issues' field")
issues = report["issues"]
if not isinstance(issues, list):
raise ValueError("Scanner JSON report has a non-list 'issues' field")
counts: Counter[str] = Counter()
for index, issue in enumerate(issues):
if (
not isinstance(issue, dict)
or not isinstance(issue.get("id"), str)
or not issue["id"].strip()
):
raise ValueError(f"Scanner JSON report has an invalid issue at index {index}")
rule_id = issue["id"]
raw_occurrences = issue.get("occurrences")
occurrence_count = 1
if raw_occurrences is not None:
if not isinstance(raw_occurrences, list):
raise ValueError(
f"Scanner JSON report has non-list occurrences at issue index {index}"
)
if any(not isinstance(occurrence, dict) for occurrence in raw_occurrences):
raise ValueError(
f"Scanner JSON report has an invalid occurrence at issue index {index}"
)
occurrence_count = max(1, len(raw_occurrences))
if not selected_rules or rule_id in selected_rules:
counts[rule_id] += occurrence_count
return counts
def _expected_range(value: object) -> tuple[int, int]:
if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
return value, value
if isinstance(value, dict):
unknown_fields = set(value) - {"min", "max"}
if unknown_fields:
raise ValueError(
"Expected rule count range has unknown field(s): "
+ ", ".join(sorted(str(field) for field in unknown_fields))
)
minimum = value.get("min", 0)
maximum = value.get("max", minimum)
if (
isinstance(minimum, int)
and not isinstance(minimum, bool)
and isinstance(maximum, int)
and not isinstance(maximum, bool)
and 0 <= minimum <= maximum
):
return minimum, maximum
raise ValueError("Expected rule counts must be a non-negative integer or {min, max} object")
def _adjudicate_counts(
case: dict[str, Any],
counts: Counter[str],
selected_rules: frozenset[str],
) -> dict[str, Any]:
raw_expected = case.get("expected_rules", {})
if not isinstance(raw_expected, dict):
raise ValueError(f"Case {case['id']} has a non-object expected_rules field")
expected: dict[str, tuple[int, int]] = {}
for rule_id, value in raw_expected.items():
if not isinstance(rule_id, str) or not rule_id.strip() or rule_id != rule_id.strip():
raise ValueError(f"Case {case['id']} has an empty expected rule id")
expected_range = _expected_range(value)
if not selected_rules or rule_id in selected_rules:
expected[rule_id] = expected_range
errors: list[str] = []
by_rule: dict[str, dict[str, int]] = {}
for rule_id in sorted(set(expected) | set(counts)):
minimum, maximum = expected.get(rule_id, (0, 0))
actual = counts.get(rule_id, 0)
false_positives = max(0, actual - maximum)
false_negatives = max(0, minimum - actual)
by_rule[rule_id] = {
"expected_min": minimum,
"expected_max": maximum,
"observed": actual,
"false_positives": false_positives,
"false_negatives": false_negatives,
}
if false_positives or false_negatives:
errors.append(f"{rule_id}: expected {minimum}..{maximum}, observed {actual}")
return {
"false_positives": sum(item["false_positives"] for item in by_rule.values()),
"false_negatives": sum(item["false_negatives"] for item in by_rule.values()),
"by_rule": by_rule,
"errors": errors,
}
def _validate_policy(manifest: dict[str, Any]) -> dict[str, int]:
raw_policy = manifest.get("material_regression_policy")
if not isinstance(raw_policy, dict):
raise ValueError("Accuracy manifest needs a material_regression_policy object")
unknown_fields = sorted(set(raw_policy) - set(POLICY_FIELDS))
if unknown_fields:
raise ValueError(
"Unknown material_regression_policy field(s): " + ", ".join(unknown_fields)
)
policy: dict[str, int] = {}
for field in POLICY_FIELDS:
value = raw_policy.get(field)
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
raise ValueError(f"material_regression_policy.{field} must be a non-negative integer")
policy[field] = value
return policy
def _aggregate_adjudication(
case_results: list[dict[str, Any]],
scanner: str,
classification: str | None = None,
) -> dict[str, Any]:
false_positives: Counter[str] = Counter()
false_negatives: Counter[str] = Counter()
for case in case_results:
if classification is not None and case["classification"] != classification:
continue
for rule_id, values in case["adjudication"][scanner]["by_rule"].items():
false_positives[rule_id] += values["false_positives"]
false_negatives[rule_id] += values["false_negatives"]
rules = sorted(set(false_positives) | set(false_negatives))
return {
"false_positives": sum(false_positives.values()),
"false_negatives": sum(false_negatives.values()),
"by_rule": {
rule_id: {
"false_positives": false_positives[rule_id],
"false_negatives": false_negatives[rule_id],
}
for rule_id in rules
},
}
def _adjudication_delta(
baseline: dict[str, Any],
candidate: dict[str, Any],
) -> dict[str, Any]:
rules = sorted(set(baseline["by_rule"]) | set(candidate["by_rule"]))
return {
"false_positives": candidate["false_positives"] - baseline["false_positives"],
"false_negatives": candidate["false_negatives"] - baseline["false_negatives"],
"by_rule": {
rule_id: {
"false_positives": candidate["by_rule"].get(rule_id, {}).get("false_positives", 0)
- baseline["by_rule"].get(rule_id, {}).get("false_positives", 0),
"false_negatives": candidate["by_rule"].get(rule_id, {}).get("false_negatives", 0)
- baseline["by_rule"].get(rule_id, {}).get("false_negatives", 0),
}
for rule_id in rules
},
}
def _material_regressions(
*,
policy: dict[str, int],
candidate: dict[str, Any],
delta: dict[str, Any],
by_classification: dict[str, dict[str, Any]],
case_results: list[dict[str, Any]],
) -> list[dict[str, Any]]:
regressions: list[dict[str, Any]] = []
def check(
metric: str,
observed: int,
limit_field: str,
*,
scope: str,
rule_id: str | None = None,
classification: str | None = None,
case_id: str | None = None,
) -> None:
limit = policy[limit_field]
if observed > limit:
record: dict[str, Any] = {
"metric": metric,
"observed": observed,
"limit": limit,
"scope": scope,
}
if rule_id is not None:
record["rule_id"] = rule_id
if classification is not None:
record["classification"] = classification
if case_id is not None:
record["case_id"] = case_id
regressions.append(record)
check(
"candidate_false_positives",
candidate["false_positives"],
"max_candidate_false_positives",
scope="global",
)
check(
"candidate_false_negatives",
candidate["false_negatives"],
"max_candidate_false_negatives",
scope="global",
)
check(
"false_positive_increase",
delta["false_positives"],
"max_false_positive_increase",
scope="global",
)
check(
"false_negative_increase",
delta["false_negatives"],
"max_false_negative_increase",
scope="global",
)
for rule_id, values in delta["by_rule"].items():
check(
"per_rule_false_positive_increase",
values["false_positives"],
"max_per_rule_false_positive_increase",
scope="global",
rule_id=rule_id,
)
check(
"per_rule_false_negative_increase",
values["false_negatives"],
"max_per_rule_false_negative_increase",
scope="global",
rule_id=rule_id,
)
for classification, adjudication in sorted(by_classification.items()):
cohort_delta = adjudication["delta"]
check(
"cohort_false_positive_increase",
cohort_delta["false_positives"],
"max_per_cohort_false_positive_increase",
scope="cohort",
classification=classification,
)
check(
"cohort_false_negative_increase",
cohort_delta["false_negatives"],
"max_per_cohort_false_negative_increase",
scope="cohort",
classification=classification,
)
for rule_id, values in cohort_delta["by_rule"].items():
check(
"per_rule_false_positive_increase",
values["false_positives"],
"max_per_rule_false_positive_increase",
scope="cohort",
classification=classification,
rule_id=rule_id,
)
check(
"per_rule_false_negative_increase",
values["false_negatives"],
"max_per_rule_false_negative_increase",
scope="cohort",
classification=classification,
rule_id=rule_id,
)
for case in sorted(case_results, key=lambda item: str(item["id"])):
case_delta = case["adjudication"]["delta"]
check(
"case_false_positive_increase",
case_delta["false_positives"],
"max_per_case_false_positive_increase",
scope="case",
classification=case["classification"],
case_id=case["id"],
)
check(
"case_false_negative_increase",
case_delta["false_negatives"],
"max_per_case_false_negative_increase",
scope="case",
classification=case["classification"],
case_id=case["id"],
)
for rule_id, values in case_delta["by_rule"].items():
check(
"per_rule_false_positive_increase",
values["false_positives"],
"max_per_rule_false_positive_increase",
scope="case",
classification=case["classification"],
case_id=case["id"],
rule_id=rule_id,
)
check(
"per_rule_false_negative_increase",
values["false_negatives"],
"max_per_rule_false_negative_increase",
scope="case",
classification=case["classification"],
case_id=case["id"],
rule_id=rule_id,
)
return regressions
def _approval_metadata(
artifact: Path | None,
reviewer: str | None,
*,
corpus_identity: str,
manifest_sha256: str,
baseline_identity: dict[str, Any],
candidate_identity: dict[str, Any],
policy: dict[str, int],
violations: list[dict[str, Any]],
) -> dict[str, Any] | None:
has_reviewer = isinstance(reviewer, str) and bool(reviewer.strip())
if (artifact is None) != (not has_reviewer):
raise ValueError("Material-regression approval requires both artifact and reviewer")
if artifact is None:
return None
assert reviewer is not None
artifact_path = artifact.resolve(strict=True)
if not artifact_path.is_file():
raise ValueError(f"Approval artifact is not a file: {artifact_path}")
raw = artifact_path.read_bytes()
document = _load_json_bytes(raw, artifact_path)
unknown_fields = sorted(set(document) - APPROVAL_FIELDS)
if unknown_fields:
raise ValueError("Unknown approval artifact field(s): " + ", ".join(unknown_fields))
if document.get("schema_version") != 1:
raise ValueError("Material-regression approval schema_version must be 1")
document_reviewer = document.get("reviewer")
if (
not isinstance(document_reviewer, str)
or not document_reviewer.strip()
or document_reviewer != document_reviewer.strip()
or document_reviewer != reviewer.strip()
):
raise ValueError("Material-regression approval reviewer does not match")
rationale = document.get("rationale")
if not isinstance(rationale, str) or not rationale.strip():
raise ValueError("Material-regression approval needs a non-empty rationale")
expected_baseline = {field: baseline_identity[field] for field in APPROVAL_IDENTITY_FIELDS}
expected_candidate = {field: candidate_identity[field] for field in APPROVAL_IDENTITY_FIELDS}
expected = {
"corpus_identity": corpus_identity,
"manifest_sha256": manifest_sha256,
"baseline_identity": expected_baseline,
"candidate_identity": expected_candidate,
"policy_sha256": _json_sha256(policy),
"violations": violations,
"violations_sha256": _json_sha256(violations),
}
for field, expected_value in expected.items():
if document.get(field) != expected_value:
raise ValueError(f"Material-regression approval is not bound to the exact {field}")
return {
"reviewer": document_reviewer,
"artifact": str(artifact_path),
"artifact_sha256": f"sha256:{hashlib.sha256(raw).hexdigest()}",
"binding_sha256": _json_sha256(expected),
"authorization": "evidence-only-untrusted-local-artifact",
}
def _snapshot_command_evidence(
command: list[str],
*,
scanner: str,
case_path: str,
) -> list[str]:
"""Replace random private snapshot paths with deterministic evidence markers."""
if len(command) != 7 or command[4] != _SOURCE_SCAN_MODE:
raise ValueError("Accuracy scanner command does not match the source-bound contract")
evidence = list(command)
evidence[3] = "<private-source-runner-snapshot>"
evidence[5] = f"<private-{scanner}-source-snapshot>"
evidence[6] = f"<private-corpus-snapshot>/{case_path}"
return evidence
def _scan_accuracy_cases(
*,
cases: list[dict[str, Any]],
snapshots: dict[str, Path],
baseline_identity: dict[str, Any],
candidate_identity: dict[str, Any],
baseline_scan_executable: Path,
candidate_scan_executable: Path,
selected_rules: frozenset[str],
) -> tuple[Counter[str], Counter[str], list[dict[str, Any]]]:
aggregate_baseline: Counter[str] = Counter()
aggregate_candidate: Counter[str] = Counter()
case_results: list[dict[str, Any]] = []
for case in sorted(cases, key=lambda item: str(item["id"])):
target = _resolve_case_path(snapshots["corpus_root"], str(case["path"]))
baseline_report = _run_scan(
baseline_scan_executable,
target,
Path(baseline_identity["worktree"]),
source_root=snapshots["baseline_source"],
runner=snapshots["runner"],
)
candidate_report = _run_scan(
candidate_scan_executable,
target,
Path(candidate_identity["worktree"]),
source_root=snapshots["candidate_source"],
runner=snapshots["runner"],
)
baseline_counts = _rule_counts(baseline_report, selected_rules)
candidate_counts = _rule_counts(candidate_report, selected_rules)
baseline_adjudication = _adjudicate_counts(case, baseline_counts, selected_rules)
candidate_adjudication = _adjudicate_counts(case, candidate_counts, selected_rules)
adjudication_delta = _adjudication_delta(
baseline_adjudication,
candidate_adjudication,
)
aggregate_baseline.update(baseline_counts)
aggregate_candidate.update(candidate_counts)
all_rules = sorted(set(baseline_counts) | set(candidate_counts))
case_results.append(
{
"id": case["id"],
"path": case["path"],
"classification": case["classification"],
"scan_execution": {
"private_input_snapshot": True,
"baseline": {
"command": _snapshot_command_evidence(
_source_bound_command(
executable=baseline_scan_executable,
target=target,
worktree=Path(baseline_identity["worktree"]),
source_root=snapshots["baseline_source"],
runner=snapshots["runner"],
),
scanner="baseline",
case_path=str(case["path"]),
),
"working_directory": baseline_identity["worktree"],
"source_root": "<private-baseline-source-snapshot>",
},
"candidate": {
"command": _snapshot_command_evidence(
_source_bound_command(
executable=candidate_scan_executable,
target=target,
worktree=Path(candidate_identity["worktree"]),
source_root=snapshots["candidate_source"],
runner=snapshots["runner"],
),
scanner="candidate",
case_path=str(case["path"]),
),
"working_directory": candidate_identity["worktree"],
"source_root": "<private-candidate-source-snapshot>",
},
},
"baseline": dict(sorted(baseline_counts.items())),
"candidate": dict(sorted(candidate_counts.items())),
"delta": {
rule_id: candidate_counts[rule_id] - baseline_counts[rule_id]
for rule_id in all_rules
},
"adjudication": {
"baseline": baseline_adjudication,
"candidate": candidate_adjudication,
"delta": adjudication_delta,
},
# Compatibility field retained as an explicit candidate-adjudication view.
"adjudication_errors": candidate_adjudication["errors"],
}
)
return aggregate_baseline, aggregate_candidate, case_results
def compare_scanners(
*,
manifest_path: Path,
corpus_root: Path,
baseline_executable: Path,
candidate_executable: Path,
baseline_worktree: Path,
candidate_worktree: Path,
baseline_revision: str,
candidate_revision: str,
invocation: list[str],
selected_rules: frozenset[str] = frozenset(),
approval_artifact: Path | None = None,
approval_reviewer: str | None = None,
) -> dict[str, Any]:
"""Run both scanners and return deterministic accuracy evidence."""
if not invocation or any(
not isinstance(argument, str) or not argument for argument in invocation
):
raise ValueError("Accuracy evidence requires the exact non-empty invocation")
if selected_rules:
raise ValueError("Partial rule selection is not allowed in the accuracy gate")
has_reviewer = isinstance(approval_reviewer, str) and bool(approval_reviewer.strip())
if (approval_artifact is None) != (not has_reviewer):
raise ValueError("Material-regression approval requires both artifact and reviewer")
manifest, manifest_bytes = _load_manifest(manifest_path)
unknown_manifest_fields = sorted(set(manifest) - MANIFEST_FIELDS)
if unknown_manifest_fields:
raise ValueError(
"Unknown accuracy manifest field(s): " + ", ".join(unknown_manifest_fields)
)
if manifest.get("schema_version") != SCHEMA_VERSION:
raise ValueError(f"Accuracy manifest schema_version must be {SCHEMA_VERSION}")
policy = _validate_policy(manifest)
raw_cases = manifest.get("cases")
if not isinstance(raw_cases, list) or not raw_cases:
raise ValueError("Accuracy manifest must contain a non-empty cases list")
cases: list[dict[str, Any]] = []
case_ids: set[str] = set()
case_paths: set[Path] = set()
classifications: set[str] = set()
for raw_case in raw_cases:
if not isinstance(raw_case, dict):
raise ValueError("Each accuracy case must be a JSON object")
unknown_case_fields = sorted(set(raw_case) - CASE_FIELDS)
if unknown_case_fields:
raise ValueError("Unknown accuracy case field(s): " + ", ".join(unknown_case_fields))
case_id = raw_case.get("id")
path = raw_case.get("path")
classification = raw_case.get("classification")
if (
not isinstance(case_id, str)
or not case_id.strip()
or case_id != case_id.strip()
or case_id in case_ids
):
raise ValueError("Each accuracy case needs a unique non-empty id")
if not isinstance(path, str) or not path.strip():
raise ValueError(f"Case {case_id} needs a non-empty path")
if (
not isinstance(classification, str)
or not classification.strip()
or classification != classification.strip()
or classification not in REQUIRED_CLASSIFICATIONS
):
raise ValueError(
f"Case {case_id} classification must be one of: "
+ ", ".join(sorted(REQUIRED_CLASSIFICATIONS))
)
if not isinstance(raw_case.get("expected_rules"), dict):
raise ValueError(f"Case {case_id} needs an explicit expected_rules object")
# Validate every adjudication, including rules outside an optional CLI filter,
# before either scanner executes.
_adjudicate_counts(raw_case, Counter(), frozenset())
resolved_case_path = _resolve_case_path(corpus_root, path)
if resolved_case_path in case_paths:
raise ValueError(f"Accuracy cases must reference unique corpus paths: {path}")
overlapping_path = next(
(
existing
for existing in case_paths
if resolved_case_path.is_relative_to(existing)
or existing.is_relative_to(resolved_case_path)
),
None,
)
if overlapping_path is not None:
raise ValueError(
"Accuracy case roots must not overlap as ancestors or descendants: "
f"{path} and {overlapping_path.relative_to(corpus_root.resolve(strict=True))}"
)
case_ids.add(case_id)
case_paths.add(resolved_case_path)
classifications.add(classification)
cases.append(raw_case)
missing_classifications = sorted(REQUIRED_CLASSIFICATIONS - classifications)
if missing_classifications:
raise ValueError(
"Accuracy manifest is missing required classifications: "
+ ", ".join(missing_classifications)
)
corpus_identity = _corpus_identity(corpus_root, cases, manifest_bytes)
manifest_sha256 = f"sha256:{hashlib.sha256(manifest_bytes).hexdigest()}"
baseline_identity = _resolve_scanner_identity(
executable=baseline_executable,
worktree=baseline_worktree,
revision=baseline_revision,
)
candidate_identity = _resolve_scanner_identity(
executable=candidate_executable,
worktree=candidate_worktree,
revision=candidate_revision,
)
baseline_scan_executable = Path(baseline_identity["executable"])
candidate_scan_executable = Path(candidate_identity["executable"])
with _accuracy_snapshots(
corpus_root=corpus_root,
cases=cases,
manifest_bytes=manifest_bytes,
corpus_identity=corpus_identity,
baseline_identity=baseline_identity,
candidate_identity=candidate_identity,
) as snapshots:
aggregate_baseline, aggregate_candidate, case_results = _scan_accuracy_cases(
cases=cases,
snapshots=snapshots,
baseline_identity=baseline_identity,
candidate_identity=candidate_identity,
baseline_scan_executable=baseline_scan_executable,
candidate_scan_executable=candidate_scan_executable,
selected_rules=selected_rules,
)
if manifest_path.read_bytes() != manifest_bytes:
raise ValueError("Accuracy manifest changed during comparison")
if _corpus_identity(corpus_root, cases, manifest_bytes) != corpus_identity:
raise ValueError("Accuracy corpus changed during comparison")
final_baseline_identity = _resolve_scanner_identity(
executable=baseline_scan_executable,
worktree=baseline_worktree,
revision=baseline_revision,
)
final_candidate_identity = _resolve_scanner_identity(
executable=candidate_scan_executable,
worktree=candidate_worktree,
revision=candidate_revision,
)
if final_baseline_identity != baseline_identity:
raise ValueError("Baseline scanner identity changed during comparison")
if final_candidate_identity != candidate_identity:
raise ValueError("Candidate scanner identity changed during comparison")
baseline_adjudication = _aggregate_adjudication(case_results, "baseline")
candidate_adjudication = _aggregate_adjudication(case_results, "candidate")
adjudication_delta = _adjudication_delta(baseline_adjudication, candidate_adjudication)
adjudication_by_classification: dict[str, dict[str, Any]] = {}
for classification in sorted(classifications):
classification_baseline = _aggregate_adjudication(
case_results,
"baseline",
classification,
)
classification_candidate = _aggregate_adjudication(
case_results,
"candidate",
classification,
)
adjudication_by_classification[classification] = {
"baseline": classification_baseline,
"candidate": classification_candidate,
"delta": _adjudication_delta(
classification_baseline,
classification_candidate,
),
}
regressions = _material_regressions(
policy=policy,
candidate=candidate_adjudication,
delta=adjudication_delta,
by_classification=adjudication_by_classification,
case_results=case_results,
)
approval = _approval_metadata(
approval_artifact,
approval_reviewer,
corpus_identity=corpus_identity,
manifest_sha256=manifest_sha256,
baseline_identity=baseline_identity,
candidate_identity=candidate_identity,
policy=policy,
violations=regressions,
)
# A local JSON document is useful review evidence, but it has no
# authentication root and therefore can never authorize a policy bypass.
approved = False
all_rules = sorted(
set(aggregate_baseline)
| set(aggregate_candidate)
| set(baseline_adjudication["by_rule"])
| set(candidate_adjudication["by_rule"])
)
result = {
"schema_version": SCHEMA_VERSION,
"corpus_identity": corpus_identity,
# Compatibility alias; unlike v1, this includes manifest/adjudication bytes.
"corpus_snapshot": corpus_identity,
"manifest": manifest_path.name,
"manifest_sha256": manifest_sha256,
"baseline": {
**baseline_identity,
"working_directory": baseline_identity["worktree"],
"command": _source_bound_command(
executable=baseline_scan_executable,
target="<case>",
worktree=Path(baseline_identity["worktree"]),
),
},
"candidate": {
**candidate_identity,
"working_directory": candidate_identity["worktree"],
"command": _source_bound_command(
executable=candidate_scan_executable,
target="<case>",
worktree=Path(candidate_identity["worktree"]),
),
},
"selected_rules": sorted(selected_rules),
"count_unit": "occurrence",
"required_classifications": sorted(REQUIRED_CLASSIFICATIONS),
"observed_classifications": sorted(classifications),
"execution": {
"invocation": list(invocation),
"configuration": {
"manifest": str(manifest_path.resolve(strict=True)),
"corpus_root": str(corpus_root.resolve(strict=True)),
"baseline_executable": baseline_identity["executable"],
"candidate_executable": candidate_identity["executable"],
"baseline_worktree": baseline_identity["worktree"],
"candidate_worktree": candidate_identity["worktree"],
"baseline_revision": baseline_identity["revision"],
"candidate_revision": candidate_identity["revision"],
"selected_rules": sorted(selected_rules),
"scan_arguments": ["scan", "<case>", "--format", "json", "--no-llm"],
"source_binding": "private-git-object-and-corpus-snapshot",
"source_runner": baseline_identity["source_runner"],
"source_runner_sha256": baseline_identity["source_runner_sha256"],
"baseline_environment": baseline_identity["runtime_identity"]["environment"],
"baseline_environment_sha256": baseline_identity["environment_sha256"],
"candidate_environment": candidate_identity["runtime_identity"]["environment"],
"candidate_environment_sha256": candidate_identity["environment_sha256"],
"approval_artifact": approval["artifact"] if approval else None,
"approval_reviewer": approval["reviewer"] if approval else None,
},
"inputs_verified_unchanged": True,
},
"per_rule": {
rule_id: {
"baseline": aggregate_baseline[rule_id],
"candidate": aggregate_candidate[rule_id],
"delta": aggregate_candidate[rule_id] - aggregate_baseline[rule_id],
"baseline_false_positives": baseline_adjudication["by_rule"]
.get(rule_id, {})
.get("false_positives", 0),
"candidate_false_positives": candidate_adjudication["by_rule"]
.get(rule_id, {})
.get("false_positives", 0),
"false_positive_delta": adjudication_delta["by_rule"]
.get(rule_id, {})
.get("false_positives", 0),
"baseline_false_negatives": baseline_adjudication["by_rule"]
.get(rule_id, {})
.get("false_negatives", 0),
"candidate_false_negatives": candidate_adjudication["by_rule"]
.get(rule_id, {})
.get("false_negatives", 0),
"false_negative_delta": adjudication_delta["by_rule"]
.get(rule_id, {})
.get("false_negatives", 0),
}
for rule_id in all_rules
},
"cases": case_results,
"adjudication": {
"baseline": baseline_adjudication,
"candidate": candidate_adjudication,
"delta": adjudication_delta,
"by_classification": adjudication_by_classification,
},
"material_regression": {
"policy": policy,
"violations": regressions,
"approval": approval,
"approved": approved,
},
}
result["passed"] = not regressions
if approval and _file_sha256(Path(approval["artifact"])) != approval["artifact_sha256"]:
raise ValueError("Material-regression approval artifact changed during comparison")
return result
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--corpus-root", type=Path, required=True)
parser.add_argument(
"--baseline-executable",
type=Path,
required=True,
help="executable file inside the baseline worktree",
)
parser.add_argument(
"--candidate-executable",
type=Path,
required=True,
help="executable file inside the candidate worktree",
)
parser.add_argument(
"--baseline-worktree",
type=Path,
required=True,
help="clean baseline Git worktree root used as the scanner working directory",
)
parser.add_argument(
"--candidate-worktree",
type=Path,
required=True,
help="clean candidate Git worktree root used as the scanner working directory",
)
parser.add_argument(
"--baseline-revision",
required=True,
help="full commit identity that must exactly match baseline-worktree HEAD",
)
parser.add_argument(
"--candidate-revision",
required=True,
help="full commit identity that must exactly match candidate-worktree HEAD",
)
parser.add_argument(
"--approval-artifact",
type=Path,
help=(
"schema-v1 JSON review record bound to this exact evidence set; "
"recorded for audit only and never waives regressions"
),
)
parser.add_argument(
"--approval-reviewer",
help="named reviewer required with --approval-artifact for audit attribution",
)
parser.add_argument("--output", type=Path)
return parser
def main(argv: list[str] | None = None) -> int:
effective_argv = list(sys.argv[1:] if argv is None else argv)
if effective_argv and effective_argv[0] == _SOURCE_SCAN_MODE:
return _run_source_bound_scan(effective_argv[1:])
args = _parser().parse_args(effective_argv)
invocation = [str(Path(sys.argv[0]).resolve()), *effective_argv]
try:
result = compare_scanners(
manifest_path=args.manifest,
corpus_root=args.corpus_root,
baseline_executable=args.baseline_executable,
candidate_executable=args.candidate_executable,
baseline_worktree=args.baseline_worktree,
candidate_worktree=args.candidate_worktree,
baseline_revision=args.baseline_revision,
candidate_revision=args.candidate_revision,
invocation=invocation,
selected_rules=frozenset(),
approval_artifact=args.approval_artifact,
approval_reviewer=args.approval_reviewer,
)
except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error:
print(f"accuracy gate error: {error}", file=sys.stderr)
return 2
rendered = json.dumps(result, indent=2, sort_keys=True) + "\n"
if args.output:
args.output.write_text(rendered, encoding="utf-8")
else:
print(rendered, end="")
return 0 if result["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())