fix(benchmarks): preserve candidate rank refresh defaults

scripts/benchmark-incremental-speed.py previously wrote rank_refresh=stale_on_exact for every matrix cell, including profiles labeled default. That changed candidate behavior and invalidated comparisons against binaries with different compiled policies.

Add RANK_REFRESH_CANDIDATE_DEFAULT and apply_rank_refresh_override() so default runs make no rank_refresh config write. Record the requested policy and whether an override was applied in synthetic, matrix, self-dogfood, and capability-quality reports; retain explicit eager, stale_on_exact, and stale_on_incremental experiments.

Document the comparison contract in docs/BENCHMARK_CAMPAIGN.md and cover CLI default, no-write, and explicit-write behavior in tests/test_benchmark_incremental_speed.py.

Verification: uv run python -m unittest tests.test_benchmark_incremental_speed tests.test_benchmark_campaign (90 tests); uv run ruff check scripts/benchmark-incremental-speed.py tests/test_benchmark_incremental_speed.py; bash scripts/check-source-safety.sh; git diff --check.
Signed-off-by: Andrew Hundt <ATHundt@gmail.com>

docs/BENCHMARK_CAMPAIGN.md

@@ -130,6 +130,12 @@ can therefore declare `"candidate_labels": ["latest"]` to restrict an ablation t
 candidates that accept it. Keep an unrestricted default profile for every candidate,
 and record fixed-default or unsupported capabilities in `capability_support`; do not
 pass an unknown flag to an old binary or pretend that its default is an ablation.
+The harness likewise leaves `rank_refresh` untouched by default, records
+`"rank_refresh": "candidate_default"` and
+`"rank_refresh_override_applied": false`, and therefore measures each candidate's
+real compiled/configured policy. Use `--rank-refresh eager`, `stale_on_exact`, or
+`stale_on_incremental` only for an explicit policy experiment on candidates known to
+support that value.
 
 Each semantic pair case also supplies a content-addressed replacement source. A real
 one-file mutation removes one judged positive and adds another, retaining pre/post

scripts/benchmark-incremental-speed.py

@@ -36,7 +36,8 @@ DEFAULT_FUNCTIONS_PER_FILE = 12
 DEFAULT_CHANGED_FILES = 2
 DEFAULT_MIN_SPEEDUP = 10.0
 DEFAULT_TIMEOUT_SECONDS = 240
-DEFAULT_RANK_REFRESH = "stale_on_exact"
+RANK_REFRESH_CANDIDATE_DEFAULT = "candidate_default"
+DEFAULT_RANK_REFRESH = RANK_REFRESH_CANDIDATE_DEFAULT
 DEFAULT_OVERHEAD_PROBES = 0
 DEFAULT_OVERHEAD_TOOL = "index_status"
 DEFAULT_FRONTIER_FILES = 16
@@ -1914,6 +1915,16 @@ def apply_config_overrides(
         run_config_set(binary, env, key, value, timeout)
 
 
+def apply_rank_refresh_override(
+    binary: Path, env: dict[str, str], policy: str, timeout: int
+) -> bool:
+    """Apply an explicit rank policy while preserving each candidate's default."""
+    if policy == RANK_REFRESH_CANDIDATE_DEFAULT:
+        return False
+    run_config_set(binary, env, "rank_refresh", policy, timeout)
+    return True
+
+
 def build_index_result(
     data: dict[str, Any],
     stderr: str,
@@ -4090,6 +4101,7 @@ def run_pair_quality_lifecycle(
     fresh_cache = work_root / "fresh-cache"
     fresh_cache.mkdir(parents=True, exist_ok=True)
     fresh_env = build_env(fresh_cache)
+    apply_rank_refresh_override(binary, fresh_env, args.rank_refresh, args.timeout)
     apply_config_overrides(binary, fresh_env, args.config_overrides, args.timeout)
     if args.transport == "mcp":
         with McpClient(binary, fresh_env, args.timeout) as client:
@@ -4184,6 +4196,10 @@ def run_capability_quality(args: argparse.Namespace, binary: Path) -> tuple[dict
         "mode": "capability_quality",
         "parameters": {
             "capability": capability,
+            "rank_refresh": args.rank_refresh,
+            "rank_refresh_override_applied": (
+                args.rank_refresh != RANK_REFRESH_CANDIDATE_DEFAULT
+            ),
             "index_mode": args.index_mode,
             "capability_applicability": index_mode_capability_applicability(args.index_mode),
             "config_profile": args.config_profile,
@@ -4226,6 +4242,7 @@ def run_capability_quality(args: argparse.Namespace, binary: Path) -> tuple[dict
             "semantic_edges": create_semantic_edges_quality_repo,
         }[capability]
         fixture = fixture_factory(repo_dir)
+        apply_rank_refresh_override(binary, case_env, args.rank_refresh, args.timeout)
         apply_config_overrides(binary, case_env, args.config_overrides, args.timeout)
         lifecycle = None
         if capability in {"similarity", "semantic_edges"}:
@@ -4323,7 +4340,7 @@ def run_matrix_case(
     case_env = dict(env)
     case_env["CBM_CACHE_DIR"] = str(cache_dir)
     run_config_set(binary, case_env, "incremental_reindex", "always", args.timeout)
-    run_config_set(binary, case_env, "rank_refresh", args.rank_refresh, args.timeout)
+    apply_rank_refresh_override(binary, case_env, args.rank_refresh, args.timeout)
     apply_config_overrides(binary, case_env, args.config_overrides, args.timeout)
 
     scenario_metadata = prepare_matrix_scenario(
@@ -4427,6 +4444,9 @@ def run_matrix(args: argparse.Namespace, binary: Path) -> tuple[dict[str, Any],
             "functions_per_file": args.functions_per_file,
             "frontier_files": args.frontier_files,
             "rank_refresh": args.rank_refresh,
+            "rank_refresh_override_applied": (
+                args.rank_refresh != RANK_REFRESH_CANDIDATE_DEFAULT
+            ),
             "index_mode": args.index_mode,
             "capability_applicability": index_mode_capability_applicability(args.index_mode),
             "config_profile": args.config_profile,
@@ -4483,7 +4503,7 @@ def run_self_dogfood_case(
     result: dict[str, Any] | None = None
     try:
         run_config_set(binary, case_env, "incremental_reindex", "always", args.timeout)
-        run_config_set(binary, case_env, "rank_refresh", args.rank_refresh, args.timeout)
+        apply_rank_refresh_override(binary, case_env, args.rank_refresh, args.timeout)
         apply_config_overrides(binary, case_env, args.config_overrides, args.timeout)
         if args.transport == "mcp":
             with McpClient(binary, case_env, args.timeout) as client:
@@ -4618,6 +4638,9 @@ def run_self_dogfood(args: argparse.Namespace, binary: Path) -> tuple[dict[str,
         "mode": "self_dogfood",
         "parameters": {
             "rank_refresh": args.rank_refresh,
+            "rank_refresh_override_applied": (
+                args.rank_refresh != RANK_REFRESH_CANDIDATE_DEFAULT
+            ),
             "index_mode": args.index_mode,
             "capability_applicability": index_mode_capability_applicability(args.index_mode),
             "config_profile": args.config_profile,
@@ -4686,8 +4709,17 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--rank-refresh",
-        choices=("eager", "stale_on_exact", "stale_on_incremental"),
+        choices=(
+            RANK_REFRESH_CANDIDATE_DEFAULT,
+            "eager",
+            "stale_on_exact",
+            "stale_on_incremental",
+        ),
         default=DEFAULT_RANK_REFRESH,
+        help=(
+            "Preserve the candidate's compiled/configured default unless an explicit policy "
+            "is selected. This is independent of --config-profile."
+        ),
     )
     parser.add_argument(
         "--config-profile",
@@ -4910,6 +4942,9 @@ def main() -> int:
             "changed_files": args.changed_files,
             "min_speedup": args.min_speedup,
             "rank_refresh": args.rank_refresh,
+            "rank_refresh_override_applied": (
+                args.rank_refresh != RANK_REFRESH_CANDIDATE_DEFAULT
+            ),
             "index_mode": args.index_mode,
             "capability_applicability": index_mode_capability_applicability(args.index_mode),
             "config_profile": args.config_profile,
@@ -4927,7 +4962,7 @@ def main() -> int:
         create_repo(repo_dir, args.files, args.functions_per_file)
         env = build_env(cache_dir)
         run_config_set(binary, env, "incremental_reindex", "always", args.timeout)
-        run_config_set(binary, env, "rank_refresh", args.rank_refresh, args.timeout)
+        apply_rank_refresh_override(binary, env, args.rank_refresh, args.timeout)
         apply_config_overrides(binary, env, args.config_overrides, args.timeout)
 
         if args.transport == "mcp":

tests/test_benchmark_incremental_speed.py

@@ -4,6 +4,7 @@ import json
 import os
 import sqlite3
 import subprocess
+import sys
 import tempfile
 import unittest
 from unittest import mock
@@ -18,6 +19,32 @@ SPEC.loader.exec_module(BENCHMARK)
 
 
 class BenchmarkIncrementalSpeedTest(unittest.TestCase):
+    def test_cli_default_preserves_candidate_rank_refresh_policy(self) -> None:
+        with mock.patch.object(sys, "argv", [str(SCRIPT)]):
+            args = BENCHMARK.parse_args()
+
+        self.assertEqual(args.rank_refresh, BENCHMARK.RANK_REFRESH_CANDIDATE_DEFAULT)
+
+    def test_candidate_default_rank_refresh_does_not_write_config_override(self) -> None:
+        with mock.patch.object(BENCHMARK, "run_config_set") as run:
+            applied = BENCHMARK.apply_rank_refresh_override(
+                Path("/tmp/cbm"), {}, BENCHMARK.RANK_REFRESH_CANDIDATE_DEFAULT, 30
+            )
+
+        self.assertFalse(applied)
+        run.assert_not_called()
+
+    def test_explicit_rank_refresh_writes_config_override(self) -> None:
+        with mock.patch.object(BENCHMARK, "run_config_set") as run:
+            applied = BENCHMARK.apply_rank_refresh_override(
+                Path("/tmp/cbm"), {}, "stale_on_exact", 30
+            )
+
+        self.assertTrue(applied)
+        run.assert_called_once_with(
+            Path("/tmp/cbm"), {}, "rank_refresh", "stale_on_exact", 30
+        )
+
     def test_stream_query_fingerprint_is_ordered_bounded_and_change_sensitive(self) -> None:
         with tempfile.TemporaryDirectory() as tmpdir:
             database = Path(tmpdir) / "graph.db"