mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
chore(evals): remove dead and test-only eval helpers
Remove private, unexported eval helpers that no product code, CI job, CLI command or doc calls: the unused TIER1/TIER1_CODE/EXTENDED/TIER3 task lists in comprehensive_benchmark (suite_runner keeps its own lists), memory.judge.create_batch_judge, and the test-only metrics.compute_bleu and datasets.generate_retrieval_probes, together with the asserts that only exercised them. None of these is in an __all__, a package re-export, the docs or a consumer repo. Evidence: a word search for each name across the repo and the consumer repos finds only its definition, plus the removed test lines for the last two. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a9815bbb5f
commit
a61487394f
@@ -46,29 +46,6 @@ DEFAULT_TASKS = [
|
||||
"gsm8k", # Math reasoning - generation-based, works with chat APIs
|
||||
]
|
||||
|
||||
# Tier 1 standard benchmarks for the suite runner
|
||||
TIER1_TASKS = [
|
||||
"gsm8k", # Math reasoning
|
||||
"truthfulqa_gen", # Factual accuracy
|
||||
"mmlu", # General knowledge (57 subjects)
|
||||
"arc_challenge", # Science reasoning
|
||||
]
|
||||
|
||||
# Code tasks (need --confirm_run_unsafe_code)
|
||||
TIER1_CODE_TASKS = [
|
||||
"humaneval", # Code generation (pass@1)
|
||||
]
|
||||
|
||||
EXTENDED_TASKS = [
|
||||
"mmlu", # 57 subjects - comprehensive but slow
|
||||
"gsm8k", # Math - requires multi-step reasoning
|
||||
"arc_challenge", # Harder science reasoning
|
||||
]
|
||||
|
||||
TIER3_TASKS = [
|
||||
"hellaswag", # Commonsense reasoning
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class BenchmarkResult:
|
||||
|
||||
@@ -1177,47 +1177,6 @@ def load_custom_dataset(path: Path | str) -> EvalSuite:
|
||||
return EvalSuite.from_jsonl(path)
|
||||
|
||||
|
||||
def generate_retrieval_probes(
|
||||
context: str,
|
||||
n_probes: int = 5,
|
||||
) -> list[str]:
|
||||
"""Generate retrieval probes from a context.
|
||||
|
||||
Extracts key facts/entities that should be retrievable
|
||||
after compression.
|
||||
|
||||
Args:
|
||||
context: The context to analyze
|
||||
n_probes: Number of probes to generate
|
||||
|
||||
Returns:
|
||||
List of fact strings to probe for
|
||||
"""
|
||||
import re
|
||||
|
||||
probes = []
|
||||
|
||||
# Look for specific patterns
|
||||
patterns = [
|
||||
r"\b[A-Z][a-z]+ [A-Z][a-z]+\b", # Names (e.g., "John Smith")
|
||||
r"\b\d{4}-\d{2}-\d{2}\b", # Dates (e.g., "2024-01-15")
|
||||
r"\b[A-Z]{2,}\b", # Acronyms (e.g., "API", "HTTP")
|
||||
r"\b\d+\.\d+%?\b", # Numbers (e.g., "99.9%", "123.45")
|
||||
r'"[^"]{5,50}"', # Quoted strings
|
||||
r"\b[a-z_]+_[a-z_]+\b", # Snake case identifiers
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
matches = re.findall(pattern, context)
|
||||
for match in matches[:2]: # Take up to 2 per pattern
|
||||
if match not in probes:
|
||||
probes.append(match.strip('"'))
|
||||
if len(probes) >= n_probes:
|
||||
return probes
|
||||
|
||||
return probes
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DATASET REGISTRY & UTILITIES
|
||||
# =============================================================================
|
||||
|
||||
@@ -248,32 +248,6 @@ def _parse_judge_response(text: str) -> tuple[float, str]:
|
||||
return score, reasoning
|
||||
|
||||
|
||||
def create_batch_judge(
|
||||
judge_fn: Callable[[str, str, str], tuple[float, str]],
|
||||
max_concurrent: int = 5,
|
||||
) -> Callable[[list[tuple[str, str, str]]], list[tuple[float, str]]]:
|
||||
"""Create a batch judge function for parallel evaluation.
|
||||
|
||||
Args:
|
||||
judge_fn: Single-item judge function.
|
||||
max_concurrent: Maximum concurrent API calls.
|
||||
|
||||
Returns:
|
||||
A function that takes a list of (question, ground_truth, prediction)
|
||||
and returns a list of (score, reasoning).
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
def batch_judge(
|
||||
items: list[tuple[str, str, str]],
|
||||
) -> list[tuple[float, str]]:
|
||||
with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
|
||||
futures = [executor.submit(judge_fn, q, gt, pred) for q, gt, pred in items]
|
||||
return [f.result() for f in futures]
|
||||
|
||||
return batch_judge
|
||||
|
||||
|
||||
# Convenience function for simple scoring without LLM
|
||||
def simple_judge(
|
||||
question: str,
|
||||
|
||||
@@ -79,50 +79,6 @@ def compute_f1(response_a: str, response_b: str) -> float:
|
||||
return 2 * precision * recall / (precision + recall)
|
||||
|
||||
|
||||
def compute_bleu(response_a: str, response_b: str, max_n: int = 4) -> float:
|
||||
"""Compute BLEU-like score between two responses.
|
||||
|
||||
Uses n-gram precision up to max_n.
|
||||
"""
|
||||
tokens_a = tokenize(response_a)
|
||||
tokens_b = tokenize(response_b)
|
||||
|
||||
if not tokens_a or not tokens_b:
|
||||
return 0.0
|
||||
|
||||
precisions = []
|
||||
|
||||
for n in range(1, max_n + 1):
|
||||
# Get n-grams
|
||||
ngrams_a = [tuple(tokens_a[i : i + n]) for i in range(len(tokens_a) - n + 1)]
|
||||
ngrams_b = [tuple(tokens_b[i : i + n]) for i in range(len(tokens_b) - n + 1)]
|
||||
|
||||
if not ngrams_a:
|
||||
break
|
||||
|
||||
counter_a = Counter(ngrams_a)
|
||||
counter_b = Counter(ngrams_b)
|
||||
|
||||
# Clipped count
|
||||
clipped = sum((counter_a & counter_b).values())
|
||||
total = len(ngrams_a)
|
||||
|
||||
if total > 0:
|
||||
precisions.append(clipped / total)
|
||||
else:
|
||||
precisions.append(0.0)
|
||||
|
||||
if not precisions or all(p == 0 for p in precisions):
|
||||
return 0.0
|
||||
|
||||
# Geometric mean of precisions
|
||||
import math
|
||||
|
||||
nonzero_precisions = [p for p in precisions if p > 0]
|
||||
log_sum = sum(math.log(p) for p in nonzero_precisions) / len(nonzero_precisions)
|
||||
return math.exp(log_sum)
|
||||
|
||||
|
||||
def compute_rouge_l(response_a: str, response_b: str) -> float:
|
||||
"""Compute ROUGE-L score (longest common subsequence).
|
||||
|
||||
|
||||
@@ -267,7 +267,7 @@ def test_load_bfcl_success_and_download_failure(monkeypatch: pytest.MonkeyPatch)
|
||||
datasets.load_bfcl(category="parallel")
|
||||
|
||||
|
||||
def test_tool_output_samples_custom_dataset_and_probe_generation(tmp_path) -> None:
|
||||
def test_tool_output_samples_and_custom_dataset(tmp_path) -> None:
|
||||
tool_outputs = datasets.load_tool_output_samples()
|
||||
assert tool_outputs.name == "ToolOutputSamples"
|
||||
assert len(tool_outputs.cases) >= 8
|
||||
@@ -284,16 +284,6 @@ def test_tool_output_samples_custom_dataset_and_probe_generation(tmp_path) -> No
|
||||
custom_suite = datasets.load_custom_dataset(custom_path)
|
||||
assert custom_suite.cases[0].id == "case1"
|
||||
|
||||
probes = datasets.generate_retrieval_probes(
|
||||
'Alice Smith deployed API on 2024-01-15 at 99.9% confidence for "Launch Ready" and build_id',
|
||||
n_probes=5,
|
||||
)
|
||||
assert "Alice Smith" in probes
|
||||
assert "2024-01-15" in probes
|
||||
assert "API" in probes
|
||||
assert "99.9" in probes
|
||||
assert "Launch Ready" in probes
|
||||
|
||||
|
||||
def test_dataset_registry_helpers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
categories = datasets.list_available_datasets()
|
||||
|
||||
@@ -16,16 +16,11 @@ def test_normalize_tokenize_and_exact_match() -> None:
|
||||
assert metrics.compute_exact_match("hello", "world") is False
|
||||
|
||||
|
||||
def test_f1_bleu_and_rouge_cover_edge_cases() -> None:
|
||||
def test_f1_and_rouge_cover_edge_cases() -> None:
|
||||
assert metrics.compute_f1("", "value") == 0.0
|
||||
assert metrics.compute_f1("alpha beta", "gamma delta") == 0.0
|
||||
assert metrics.compute_f1("alpha beta gamma", "alpha gamma") == pytest.approx(0.8)
|
||||
|
||||
assert metrics.compute_bleu("", "value") == 0.0
|
||||
assert metrics.compute_bleu("one", "one") == pytest.approx(1.0)
|
||||
assert metrics.compute_bleu("alpha beta", "gamma delta") == 0.0
|
||||
assert metrics.compute_bleu("alpha beta", "alpha beta gamma", max_n=4) == pytest.approx(1.0)
|
||||
|
||||
assert metrics.compute_rouge_l("", "value") == 0.0
|
||||
assert metrics.compute_rouge_l("alpha beta", "gamma delta") == 0.0
|
||||
assert metrics.compute_rouge_l("alpha beta gamma", "alpha gamma") == pytest.approx(0.8)
|
||||
|
||||
Reference in New Issue
Block a user