chore(evals): remove dead and test-only eval helpers

Remove private, unexported eval helpers that no product code, CI job, CLI command or doc calls: the unused TIER1/TIER1_CODE/EXTENDED/TIER3 task lists in comprehensive_benchmark (suite_runner keeps its own lists), memory.judge.create_batch_judge, and the test-only metrics.compute_bleu and datasets.generate_retrieval_probes, together with the asserts that only exercised them. None of these is in an __all__, a package re-export, the docs or a consumer repo.

Evidence: a word search for each name across the repo and the consumer repos finds only its definition, plus the removed test lines for the last two.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-10-09 17:23:49 -07:00
co-authored by Claude Opus 5.5
parent a9815bbb5f
commit a61487394f
6 changed files with 2 additions and 151 deletions
-23
View File
@@ -46,29 +46,6 @@ DEFAULT_TASKS = [
"gsm8k", # Math reasoning - generation-based, works with chat APIs
]
# Tier 1 standard benchmarks for the suite runner
TIER1_TASKS = [
"gsm8k", # Math reasoning
"truthfulqa_gen", # Factual accuracy
"mmlu", # General knowledge (57 subjects)
"arc_challenge", # Science reasoning
]
# Code tasks (need --confirm_run_unsafe_code)
TIER1_CODE_TASKS = [
"humaneval", # Code generation (pass@1)
]
EXTENDED_TASKS = [
"mmlu", # 57 subjects - comprehensive but slow
"gsm8k", # Math - requires multi-step reasoning
"arc_challenge", # Harder science reasoning
]
TIER3_TASKS = [
"hellaswag", # Commonsense reasoning
]
@dataclass
class BenchmarkResult:
-41
View File
@@ -1177,47 +1177,6 @@ def load_custom_dataset(path: Path | str) -> EvalSuite:
return EvalSuite.from_jsonl(path)
def generate_retrieval_probes(
context: str,
n_probes: int = 5,
) -> list[str]:
"""Generate retrieval probes from a context.
Extracts key facts/entities that should be retrievable
after compression.
Args:
context: The context to analyze
n_probes: Number of probes to generate
Returns:
List of fact strings to probe for
"""
import re
probes = []
# Look for specific patterns
patterns = [
r"\b[A-Z][a-z]+ [A-Z][a-z]+\b", # Names (e.g., "John Smith")
r"\b\d{4}-\d{2}-\d{2}\b", # Dates (e.g., "2024-01-15")
r"\b[A-Z]{2,}\b", # Acronyms (e.g., "API", "HTTP")
r"\b\d+\.\d+%?\b", # Numbers (e.g., "99.9%", "123.45")
r'"[^"]{5,50}"', # Quoted strings
r"\b[a-z_]+_[a-z_]+\b", # Snake case identifiers
]
for pattern in patterns:
matches = re.findall(pattern, context)
for match in matches[:2]: # Take up to 2 per pattern
if match not in probes:
probes.append(match.strip('"'))
if len(probes) >= n_probes:
return probes
return probes
# =============================================================================
# DATASET REGISTRY & UTILITIES
# =============================================================================
-26
View File
@@ -248,32 +248,6 @@ def _parse_judge_response(text: str) -> tuple[float, str]:
return score, reasoning
def create_batch_judge(
judge_fn: Callable[[str, str, str], tuple[float, str]],
max_concurrent: int = 5,
) -> Callable[[list[tuple[str, str, str]]], list[tuple[float, str]]]:
"""Create a batch judge function for parallel evaluation.
Args:
judge_fn: Single-item judge function.
max_concurrent: Maximum concurrent API calls.
Returns:
A function that takes a list of (question, ground_truth, prediction)
and returns a list of (score, reasoning).
"""
from concurrent.futures import ThreadPoolExecutor
def batch_judge(
items: list[tuple[str, str, str]],
) -> list[tuple[float, str]]:
with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
futures = [executor.submit(judge_fn, q, gt, pred) for q, gt, pred in items]
return [f.result() for f in futures]
return batch_judge
# Convenience function for simple scoring without LLM
def simple_judge(
question: str,
-44
View File
@@ -79,50 +79,6 @@ def compute_f1(response_a: str, response_b: str) -> float:
return 2 * precision * recall / (precision + recall)
def compute_bleu(response_a: str, response_b: str, max_n: int = 4) -> float:
"""Compute BLEU-like score between two responses.
Uses n-gram precision up to max_n.
"""
tokens_a = tokenize(response_a)
tokens_b = tokenize(response_b)
if not tokens_a or not tokens_b:
return 0.0
precisions = []
for n in range(1, max_n + 1):
# Get n-grams
ngrams_a = [tuple(tokens_a[i : i + n]) for i in range(len(tokens_a) - n + 1)]
ngrams_b = [tuple(tokens_b[i : i + n]) for i in range(len(tokens_b) - n + 1)]
if not ngrams_a:
break
counter_a = Counter(ngrams_a)
counter_b = Counter(ngrams_b)
# Clipped count
clipped = sum((counter_a & counter_b).values())
total = len(ngrams_a)
if total > 0:
precisions.append(clipped / total)
else:
precisions.append(0.0)
if not precisions or all(p == 0 for p in precisions):
return 0.0
# Geometric mean of precisions
import math
nonzero_precisions = [p for p in precisions if p > 0]
log_sum = sum(math.log(p) for p in nonzero_precisions) / len(nonzero_precisions)
return math.exp(log_sum)
def compute_rouge_l(response_a: str, response_b: str) -> float:
"""Compute ROUGE-L score (longest common subsequence).
+1 -11
View File
@@ -267,7 +267,7 @@ def test_load_bfcl_success_and_download_failure(monkeypatch: pytest.MonkeyPatch)
datasets.load_bfcl(category="parallel")
def test_tool_output_samples_custom_dataset_and_probe_generation(tmp_path) -> None:
def test_tool_output_samples_and_custom_dataset(tmp_path) -> None:
tool_outputs = datasets.load_tool_output_samples()
assert tool_outputs.name == "ToolOutputSamples"
assert len(tool_outputs.cases) >= 8
@@ -284,16 +284,6 @@ def test_tool_output_samples_custom_dataset_and_probe_generation(tmp_path) -> No
custom_suite = datasets.load_custom_dataset(custom_path)
assert custom_suite.cases[0].id == "case1"
probes = datasets.generate_retrieval_probes(
'Alice Smith deployed API on 2024-01-15 at 99.9% confidence for "Launch Ready" and build_id',
n_probes=5,
)
assert "Alice Smith" in probes
assert "2024-01-15" in probes
assert "API" in probes
assert "99.9" in probes
assert "Launch Ready" in probes
def test_dataset_registry_helpers(monkeypatch: pytest.MonkeyPatch) -> None:
categories = datasets.list_available_datasets()
+1 -6
View File
@@ -16,16 +16,11 @@ def test_normalize_tokenize_and_exact_match() -> None:
assert metrics.compute_exact_match("hello", "world") is False
def test_f1_bleu_and_rouge_cover_edge_cases() -> None:
def test_f1_and_rouge_cover_edge_cases() -> None:
assert metrics.compute_f1("", "value") == 0.0
assert metrics.compute_f1("alpha beta", "gamma delta") == 0.0
assert metrics.compute_f1("alpha beta gamma", "alpha gamma") == pytest.approx(0.8)
assert metrics.compute_bleu("", "value") == 0.0
assert metrics.compute_bleu("one", "one") == pytest.approx(1.0)
assert metrics.compute_bleu("alpha beta", "gamma delta") == 0.0
assert metrics.compute_bleu("alpha beta", "alpha beta gamma", max_n=4) == pytest.approx(1.0)
assert metrics.compute_rouge_l("", "value") == 0.0
assert metrics.compute_rouge_l("alpha beta", "gamma delta") == 0.0
assert metrics.compute_rouge_l("alpha beta gamma", "alpha gamma") == pytest.approx(0.8)