From 575bc04dce077c5ea10b07ca2f1dd3a207302ce0 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 16 Jul 2026 18:35:29 -0400 Subject: [PATCH] fix(benchmarks): preserve candidate rank refresh defaults scripts/benchmark-incremental-speed.py previously wrote rank_refresh=stale_on_exact for every matrix cell, including profiles labeled default. That changed candidate behavior and invalidated comparisons against binaries with different compiled policies. Add RANK_REFRESH_CANDIDATE_DEFAULT and apply_rank_refresh_override() so default runs make no rank_refresh config write. Record the requested policy and whether an override was applied in synthetic, matrix, self-dogfood, and capability-quality reports; retain explicit eager, stale_on_exact, and stale_on_incremental experiments. Document the comparison contract in docs/BENCHMARK_CAMPAIGN.md and cover CLI default, no-write, and explicit-write behavior in tests/test_benchmark_incremental_speed.py. Verification: uv run python -m unittest tests.test_benchmark_incremental_speed tests.test_benchmark_campaign (90 tests); uv run ruff check scripts/benchmark-incremental-speed.py tests/test_benchmark_incremental_speed.py; bash scripts/check-source-safety.sh; git diff --check. Signed-off-by: Andrew Hundt --- docs/BENCHMARK_CAMPAIGN.md | 6 +++ scripts/benchmark-incremental-speed.py | 45 ++++++++++++++++++++--- tests/test_benchmark_incremental_speed.py | 27 ++++++++++++++ 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/docs/BENCHMARK_CAMPAIGN.md b/docs/BENCHMARK_CAMPAIGN.md index 8bcb5c976..831d24037 100644 --- a/docs/BENCHMARK_CAMPAIGN.md +++ b/docs/BENCHMARK_CAMPAIGN.md @@ -130,6 +130,12 @@ can therefore declare `"candidate_labels": ["latest"]` to restrict an ablation t candidates that accept it. Keep an unrestricted default profile for every candidate, and record fixed-default or unsupported capabilities in `capability_support`; do not pass an unknown flag to an old binary or pretend that its default is an ablation. +The harness likewise leaves `rank_refresh` untouched by default, records +`"rank_refresh": "candidate_default"` and +`"rank_refresh_override_applied": false`, and therefore measures each candidate's +real compiled/configured policy. Use `--rank-refresh eager`, `stale_on_exact`, or +`stale_on_incremental` only for an explicit policy experiment on candidates known to +support that value. Each semantic pair case also supplies a content-addressed replacement source. A real one-file mutation removes one judged positive and adds another, retaining pre/post diff --git a/scripts/benchmark-incremental-speed.py b/scripts/benchmark-incremental-speed.py index 45365ea4e..a92b5270b 100755 --- a/scripts/benchmark-incremental-speed.py +++ b/scripts/benchmark-incremental-speed.py @@ -36,7 +36,8 @@ DEFAULT_FUNCTIONS_PER_FILE = 12 DEFAULT_CHANGED_FILES = 2 DEFAULT_MIN_SPEEDUP = 10.0 DEFAULT_TIMEOUT_SECONDS = 240 -DEFAULT_RANK_REFRESH = "stale_on_exact" +RANK_REFRESH_CANDIDATE_DEFAULT = "candidate_default" +DEFAULT_RANK_REFRESH = RANK_REFRESH_CANDIDATE_DEFAULT DEFAULT_OVERHEAD_PROBES = 0 DEFAULT_OVERHEAD_TOOL = "index_status" DEFAULT_FRONTIER_FILES = 16 @@ -1914,6 +1915,16 @@ def apply_config_overrides( run_config_set(binary, env, key, value, timeout) +def apply_rank_refresh_override( + binary: Path, env: dict[str, str], policy: str, timeout: int +) -> bool: + """Apply an explicit rank policy while preserving each candidate's default.""" + if policy == RANK_REFRESH_CANDIDATE_DEFAULT: + return False + run_config_set(binary, env, "rank_refresh", policy, timeout) + return True + + def build_index_result( data: dict[str, Any], stderr: str, @@ -4090,6 +4101,7 @@ def run_pair_quality_lifecycle( fresh_cache = work_root / "fresh-cache" fresh_cache.mkdir(parents=True, exist_ok=True) fresh_env = build_env(fresh_cache) + apply_rank_refresh_override(binary, fresh_env, args.rank_refresh, args.timeout) apply_config_overrides(binary, fresh_env, args.config_overrides, args.timeout) if args.transport == "mcp": with McpClient(binary, fresh_env, args.timeout) as client: @@ -4184,6 +4196,10 @@ def run_capability_quality(args: argparse.Namespace, binary: Path) -> tuple[dict "mode": "capability_quality", "parameters": { "capability": capability, + "rank_refresh": args.rank_refresh, + "rank_refresh_override_applied": ( + args.rank_refresh != RANK_REFRESH_CANDIDATE_DEFAULT + ), "index_mode": args.index_mode, "capability_applicability": index_mode_capability_applicability(args.index_mode), "config_profile": args.config_profile, @@ -4226,6 +4242,7 @@ def run_capability_quality(args: argparse.Namespace, binary: Path) -> tuple[dict "semantic_edges": create_semantic_edges_quality_repo, }[capability] fixture = fixture_factory(repo_dir) + apply_rank_refresh_override(binary, case_env, args.rank_refresh, args.timeout) apply_config_overrides(binary, case_env, args.config_overrides, args.timeout) lifecycle = None if capability in {"similarity", "semantic_edges"}: @@ -4323,7 +4340,7 @@ def run_matrix_case( case_env = dict(env) case_env["CBM_CACHE_DIR"] = str(cache_dir) run_config_set(binary, case_env, "incremental_reindex", "always", args.timeout) - run_config_set(binary, case_env, "rank_refresh", args.rank_refresh, args.timeout) + apply_rank_refresh_override(binary, case_env, args.rank_refresh, args.timeout) apply_config_overrides(binary, case_env, args.config_overrides, args.timeout) scenario_metadata = prepare_matrix_scenario( @@ -4427,6 +4444,9 @@ def run_matrix(args: argparse.Namespace, binary: Path) -> tuple[dict[str, Any], "functions_per_file": args.functions_per_file, "frontier_files": args.frontier_files, "rank_refresh": args.rank_refresh, + "rank_refresh_override_applied": ( + args.rank_refresh != RANK_REFRESH_CANDIDATE_DEFAULT + ), "index_mode": args.index_mode, "capability_applicability": index_mode_capability_applicability(args.index_mode), "config_profile": args.config_profile, @@ -4483,7 +4503,7 @@ def run_self_dogfood_case( result: dict[str, Any] | None = None try: run_config_set(binary, case_env, "incremental_reindex", "always", args.timeout) - run_config_set(binary, case_env, "rank_refresh", args.rank_refresh, args.timeout) + apply_rank_refresh_override(binary, case_env, args.rank_refresh, args.timeout) apply_config_overrides(binary, case_env, args.config_overrides, args.timeout) if args.transport == "mcp": with McpClient(binary, case_env, args.timeout) as client: @@ -4618,6 +4638,9 @@ def run_self_dogfood(args: argparse.Namespace, binary: Path) -> tuple[dict[str, "mode": "self_dogfood", "parameters": { "rank_refresh": args.rank_refresh, + "rank_refresh_override_applied": ( + args.rank_refresh != RANK_REFRESH_CANDIDATE_DEFAULT + ), "index_mode": args.index_mode, "capability_applicability": index_mode_capability_applicability(args.index_mode), "config_profile": args.config_profile, @@ -4686,8 +4709,17 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--rank-refresh", - choices=("eager", "stale_on_exact", "stale_on_incremental"), + choices=( + RANK_REFRESH_CANDIDATE_DEFAULT, + "eager", + "stale_on_exact", + "stale_on_incremental", + ), default=DEFAULT_RANK_REFRESH, + help=( + "Preserve the candidate's compiled/configured default unless an explicit policy " + "is selected. This is independent of --config-profile." + ), ) parser.add_argument( "--config-profile", @@ -4910,6 +4942,9 @@ def main() -> int: "changed_files": args.changed_files, "min_speedup": args.min_speedup, "rank_refresh": args.rank_refresh, + "rank_refresh_override_applied": ( + args.rank_refresh != RANK_REFRESH_CANDIDATE_DEFAULT + ), "index_mode": args.index_mode, "capability_applicability": index_mode_capability_applicability(args.index_mode), "config_profile": args.config_profile, @@ -4927,7 +4962,7 @@ def main() -> int: create_repo(repo_dir, args.files, args.functions_per_file) env = build_env(cache_dir) run_config_set(binary, env, "incremental_reindex", "always", args.timeout) - run_config_set(binary, env, "rank_refresh", args.rank_refresh, args.timeout) + apply_rank_refresh_override(binary, env, args.rank_refresh, args.timeout) apply_config_overrides(binary, env, args.config_overrides, args.timeout) if args.transport == "mcp": diff --git a/tests/test_benchmark_incremental_speed.py b/tests/test_benchmark_incremental_speed.py index 4b43514c7..16ecdae11 100644 --- a/tests/test_benchmark_incremental_speed.py +++ b/tests/test_benchmark_incremental_speed.py @@ -4,6 +4,7 @@ import json import os import sqlite3 import subprocess +import sys import tempfile import unittest from unittest import mock @@ -18,6 +19,32 @@ SPEC.loader.exec_module(BENCHMARK) class BenchmarkIncrementalSpeedTest(unittest.TestCase): + def test_cli_default_preserves_candidate_rank_refresh_policy(self) -> None: + with mock.patch.object(sys, "argv", [str(SCRIPT)]): + args = BENCHMARK.parse_args() + + self.assertEqual(args.rank_refresh, BENCHMARK.RANK_REFRESH_CANDIDATE_DEFAULT) + + def test_candidate_default_rank_refresh_does_not_write_config_override(self) -> None: + with mock.patch.object(BENCHMARK, "run_config_set") as run: + applied = BENCHMARK.apply_rank_refresh_override( + Path("/tmp/cbm"), {}, BENCHMARK.RANK_REFRESH_CANDIDATE_DEFAULT, 30 + ) + + self.assertFalse(applied) + run.assert_not_called() + + def test_explicit_rank_refresh_writes_config_override(self) -> None: + with mock.patch.object(BENCHMARK, "run_config_set") as run: + applied = BENCHMARK.apply_rank_refresh_override( + Path("/tmp/cbm"), {}, "stale_on_exact", 30 + ) + + self.assertTrue(applied) + run.assert_called_once_with( + Path("/tmp/cbm"), {}, "rank_refresh", "stale_on_exact", 30 + ) + def test_stream_query_fingerprint_is_ordered_bounded_and_change_sensitive(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: database = Path(tmpdir) / "graph.db"