mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
refactor(transforms): remove the unused summarize_dropped_items
compression_summary.summarize_dropped_items was SmartCrusher's dropped-item categoriser. SmartCrusher runs in Rust now and emits no such summary (smart_crusher/crusher.rs), and nothing in Python called the function. Only tests did, including two LLM eval files that evaluated the dead function. Removes the function, its four private helpers and three constants, and fixes the module docstring, which still said SmartCrusher used it. summarize_compressed_code (used by CodeCompressor) stays. Deletes test_compression_summary_hard_eval.py, test_compression_summary_tool_eval.py and test_compression_summary_eval.py, moving that file's one code-summary test into test_compression_summary.py, and drops the dropped-item cases from the other two files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a9815bbb5f
commit
bc0dc2f653
@@ -27,7 +27,7 @@
|
||||
//! - **Telemetry**: no-op.
|
||||
//! - **`_compress_text_within_items`**: pass-through (returns input
|
||||
//! unchanged) since text compression has its own port pipeline.
|
||||
//! - **`summarize_dropped_items`**: empty string.
|
||||
//! - **Dropped-item summary**: empty string.
|
||||
//!
|
||||
//! Parity fixtures will be recorded with all four disabled on the
|
||||
//! Python side, locking byte-equal output. The TOIN/CCR/feedback
|
||||
|
||||
@@ -1,80 +1,17 @@
|
||||
"""Compression summary generator — describes what was dropped.
|
||||
|
||||
When content is compressed, the LLM needs to know what it's missing.
|
||||
Instead of just "[480 items omitted]", we generate a categorical summary:
|
||||
"[480 items omitted: 150 log entries (3 with errors), 200 test results (12 failures)]"
|
||||
When code is compressed, the LLM needs to know what it's missing. Instead of
|
||||
just "[5 bodies compressed]", we list the removed function/class names (from
|
||||
the AST, language-agnostic): "5 bodies compressed: authenticate(), ...".
|
||||
|
||||
This helps the LLM decide whether to call headroom_retrieve and what to search for.
|
||||
|
||||
Used by:
|
||||
- SmartCrusher: categorizes dropped JSON items by field values
|
||||
- CodeCompressor: lists removed function/class names (from AST, language-agnostic)
|
||||
Used by CodeCompressor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections import Counter
|
||||
|
||||
|
||||
def summarize_dropped_items(
|
||||
all_items: list[dict],
|
||||
kept_items: list[dict],
|
||||
kept_indices: set[int] | None = None,
|
||||
max_categories: int = 5,
|
||||
max_notable: int = 3,
|
||||
) -> str:
|
||||
"""Generate a categorical summary of items that were dropped.
|
||||
|
||||
Args:
|
||||
all_items: The original full list of items.
|
||||
kept_items: The items that were kept after compression (used for count).
|
||||
kept_indices: Indices of kept items (preferred over identity comparison).
|
||||
max_categories: Maximum number of categories to show.
|
||||
max_notable: Maximum number of notable items to call out.
|
||||
|
||||
Returns:
|
||||
Summary string or empty string if no useful summary can be generated.
|
||||
"""
|
||||
if not all_items or len(kept_items) >= len(all_items):
|
||||
return ""
|
||||
|
||||
# Determine which items were dropped
|
||||
if kept_indices is not None:
|
||||
dropped = [item for i, item in enumerate(all_items) if i not in kept_indices]
|
||||
else:
|
||||
# Fallback: index-based comparison using JSON equality
|
||||
kept_json = {_item_key(item) for item in kept_items}
|
||||
dropped = [item for item in all_items if _item_key(item) not in kept_json]
|
||||
|
||||
if not dropped:
|
||||
return ""
|
||||
|
||||
# Strategy 1: Categorize by type/status/kind fields
|
||||
categories = _categorize_by_fields(dropped)
|
||||
|
||||
# Strategy 2: Find notable items (errors, failures, warnings)
|
||||
notable = _find_notable_items(dropped, max_notable)
|
||||
|
||||
# Build summary
|
||||
parts = []
|
||||
|
||||
if categories:
|
||||
cat_strs = []
|
||||
for field_val, count in categories.most_common(max_categories):
|
||||
cat_strs.append(f"{count} {field_val}")
|
||||
parts.append(", ".join(cat_strs))
|
||||
|
||||
if notable:
|
||||
parts.append(f"notable: {'; '.join(notable)}")
|
||||
|
||||
if not parts:
|
||||
# Fallback: just describe the data shape
|
||||
keys = _common_keys(dropped)
|
||||
if keys:
|
||||
parts.append(f"fields: {', '.join(keys[:5])}")
|
||||
|
||||
return "; ".join(parts)
|
||||
|
||||
|
||||
def summarize_compressed_code(
|
||||
@@ -117,98 +54,6 @@ def summarize_compressed_code(
|
||||
|
||||
# ---- Internal helpers ----
|
||||
|
||||
# Fields that commonly indicate item category/type
|
||||
_CATEGORY_FIELDS = (
|
||||
"type",
|
||||
"status",
|
||||
"kind",
|
||||
"category",
|
||||
"level",
|
||||
"severity",
|
||||
"state",
|
||||
"phase",
|
||||
"action",
|
||||
"event_type",
|
||||
"log_level",
|
||||
"result",
|
||||
"outcome",
|
||||
)
|
||||
|
||||
# Values that indicate something notable/important
|
||||
_NOTABLE_PATTERNS = re.compile(
|
||||
r"error|fail|critical|warning|exception|crash|timeout|denied|rejected|invalid",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Values that look like URLs or paths (not useful as categories)
|
||||
_URL_PATTERN = re.compile(r"^https?://|^/[a-z]", re.IGNORECASE)
|
||||
|
||||
|
||||
def _item_key(item: dict) -> str:
|
||||
"""Create a hashable key for an item (for dropped detection without id())."""
|
||||
# Use first few field values as a fingerprint
|
||||
parts = []
|
||||
for k, v in list(item.items())[:4]:
|
||||
parts.append(f"{k}={v}")
|
||||
return "|".join(parts)
|
||||
|
||||
|
||||
def _categorize_by_fields(items: list[dict]) -> Counter:
|
||||
"""Categorize items by their type/status/kind field values."""
|
||||
categories: Counter = Counter()
|
||||
|
||||
for item in items:
|
||||
categorized = False
|
||||
for field in _CATEGORY_FIELDS:
|
||||
val = item.get(field)
|
||||
if val and isinstance(val, str) and len(val) < 50:
|
||||
clean_val = val.replace("\n", " ").replace("\r", "").strip()
|
||||
if clean_val:
|
||||
categories[clean_val] += 1
|
||||
categorized = True
|
||||
break
|
||||
if not categorized:
|
||||
# Try to infer from the item's first short string field
|
||||
for key, val in item.items():
|
||||
if (
|
||||
isinstance(val, str)
|
||||
and 2 < len(val) < 30
|
||||
and key not in ("id", "name", "path", "url", "href", "email")
|
||||
and not _URL_PATTERN.match(val)
|
||||
):
|
||||
clean_val = val.replace("\n", " ").replace("\r", "").strip()
|
||||
categories[f"{key}={clean_val}"] += 1
|
||||
break
|
||||
|
||||
return categories
|
||||
|
||||
|
||||
def _find_notable_items(items: list[dict], max_notable: int) -> list[str]:
|
||||
"""Find items that contain error/failure/warning indicators."""
|
||||
notable = []
|
||||
for item in items:
|
||||
item_str = str(item)[:500]
|
||||
matches = _NOTABLE_PATTERNS.findall(item_str)
|
||||
if matches:
|
||||
name = item.get("name", item.get("id", item.get("path", "")))
|
||||
if name:
|
||||
clean_name = str(name).replace("\n", " ").strip()[:50]
|
||||
notable.append(f"{clean_name} ({matches[0]})")
|
||||
else:
|
||||
notable.append(matches[0])
|
||||
if len(notable) >= max_notable:
|
||||
break
|
||||
return notable
|
||||
|
||||
|
||||
def _common_keys(items: list[dict]) -> list[str]:
|
||||
"""Get the most common keys across items."""
|
||||
key_counts: Counter = Counter()
|
||||
for item in items[:50]:
|
||||
for key in item.keys():
|
||||
key_counts[key] += 1
|
||||
return [k for k, _ in key_counts.most_common(8)]
|
||||
|
||||
|
||||
def _extract_name_from_signature(sig: str) -> str:
|
||||
"""Extract the function/method name from a signature string.
|
||||
|
||||
@@ -3,87 +3,9 @@
|
||||
from headroom.transforms.compression_summary import (
|
||||
_extract_name_from_signature,
|
||||
summarize_compressed_code,
|
||||
summarize_dropped_items,
|
||||
)
|
||||
|
||||
|
||||
class TestSummarizeDroppedItems:
|
||||
def test_items_with_status_field(self):
|
||||
all_items = [{"id": i, "name": f"item-{i}", "status": "active"} for i in range(50)] + [
|
||||
{"id": i, "name": f"err-{i}", "status": "error"} for i in range(5)
|
||||
]
|
||||
kept = all_items[:3]
|
||||
summary = summarize_dropped_items(all_items, kept, kept_indices={0, 1, 2})
|
||||
assert "active" in summary
|
||||
assert summary
|
||||
|
||||
def test_items_with_type_field(self):
|
||||
all_items = [{"type": "log", "message": f"entry {i}"} for i in range(30)] + [
|
||||
{"type": "metric", "value": i} for i in range(20)
|
||||
]
|
||||
kept = all_items[:5]
|
||||
summary = summarize_dropped_items(all_items, kept, kept_indices={0, 1, 2, 3, 4})
|
||||
assert "log" in summary or "metric" in summary
|
||||
|
||||
def test_notable_items_with_errors(self):
|
||||
all_items = [{"name": f"test-{i}", "result": "pass"} for i in range(40)] + [
|
||||
{"name": "test-auth", "result": "fail", "error": "authentication failed"},
|
||||
]
|
||||
kept = all_items[:5]
|
||||
summary = summarize_dropped_items(all_items, kept, kept_indices=set(range(5)))
|
||||
assert summary
|
||||
|
||||
def test_no_dropped_items(self):
|
||||
items = [{"id": 1}, {"id": 2}]
|
||||
summary = summarize_dropped_items(items, items)
|
||||
assert summary == ""
|
||||
|
||||
def test_empty_input(self):
|
||||
summary = summarize_dropped_items([], [])
|
||||
assert summary == ""
|
||||
|
||||
def test_all_items_dropped(self):
|
||||
items = [{"status": "active", "name": f"item-{i}"} for i in range(20)]
|
||||
summary = summarize_dropped_items(items, [], kept_indices=set())
|
||||
assert "active" in summary
|
||||
|
||||
def test_mixed_category_fields(self):
|
||||
all_items = [{"level": "info", "msg": "something"} for _ in range(10)] + [
|
||||
{"level": "error", "msg": "bad thing"} for _ in range(3)
|
||||
]
|
||||
kept = all_items[:2]
|
||||
summary = summarize_dropped_items(all_items, kept, kept_indices={0, 1})
|
||||
assert summary
|
||||
|
||||
def test_items_without_category_fields(self):
|
||||
all_items = [{"code": 200, "count": i} for i in range(30)]
|
||||
kept = all_items[:3]
|
||||
summary = summarize_dropped_items(all_items, kept, kept_indices={0, 1, 2})
|
||||
assert summary # Should produce field-based fallback
|
||||
|
||||
def test_summary_not_too_long(self):
|
||||
all_items = [{"type": f"type_{i % 20}", "data": "x"} for i in range(100)]
|
||||
kept = all_items[:5]
|
||||
summary = summarize_dropped_items(all_items, kept, kept_indices=set(range(5)))
|
||||
assert len(summary) < 500
|
||||
|
||||
def test_url_values_excluded_from_categories(self):
|
||||
"""URL-like values should not be used as category labels."""
|
||||
all_items = [
|
||||
{"url": f"https://api.example.com/v1/items/{i}", "method": "GET"} for i in range(30)
|
||||
]
|
||||
kept = all_items[:3]
|
||||
summary = summarize_dropped_items(all_items, kept, kept_indices={0, 1, 2})
|
||||
assert "https://" not in summary
|
||||
|
||||
def test_fallback_without_kept_indices(self):
|
||||
"""Works correctly without kept_indices (uses _item_key fallback)."""
|
||||
all_items = [{"status": "active", "id": i} for i in range(20)]
|
||||
kept = [all_items[0], all_items[1]] # Copies from same list
|
||||
summary = summarize_dropped_items(all_items, kept)
|
||||
assert summary
|
||||
|
||||
|
||||
class TestSummarizeCompressedCode:
|
||||
def test_python_function_bodies(self):
|
||||
bodies = [
|
||||
@@ -134,6 +56,22 @@ class TestSummarizeCompressedCode:
|
||||
summary = summarize_compressed_code(bodies, 20)
|
||||
assert "+14 more" in summary # 20 - 6 shown
|
||||
|
||||
def test_real_python_module(self):
|
||||
bodies = [
|
||||
("def __init__(self, url: str, pool_size: int = 10):", "...", 8),
|
||||
("def connect(self) -> Any:", "...", 15),
|
||||
("def _create_new(self) -> Any:", "...", 22),
|
||||
("def release(self, conn: Any) -> None:", "...", 28),
|
||||
("def close_all(self) -> None:", "...", 33),
|
||||
("def create_engine(url: str) -> DatabaseConnection:", "...", 38),
|
||||
]
|
||||
summary = summarize_compressed_code(bodies, 6)
|
||||
assert "6 bodies compressed" in summary
|
||||
has_names = any(
|
||||
name in summary for name in ["connect()", "release()", "close_all()", "create_engine()"]
|
||||
)
|
||||
assert has_names, f"Summary missing function names: {summary}"
|
||||
|
||||
|
||||
class TestExtractNameFromSignature:
|
||||
def test_python_def(self):
|
||||
|
||||
@@ -1,217 +0,0 @@
|
||||
"""Eval: Compression summary quality — generic, unbiased.
|
||||
|
||||
Tests that compression summaries are:
|
||||
1. Accurate (categories match actual dropped items)
|
||||
2. Useful (contain information that would help retrieval)
|
||||
3. Not misleading (don't hallucinate categories)
|
||||
|
||||
These are NOT skewed to show summaries as amazing — they test
|
||||
real-world data patterns and verify correctness.
|
||||
"""
|
||||
|
||||
from headroom.transforms.compression_summary import (
|
||||
summarize_compressed_code,
|
||||
summarize_dropped_items,
|
||||
)
|
||||
|
||||
# ============================================================================
|
||||
# Realistic test data (modeled on actual tool outputs)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _make_github_issues(n: int) -> list[dict]:
|
||||
"""Realistic GitHub issues list."""
|
||||
statuses = ["open"] * (n // 2) + ["closed"] * (n // 4) + ["in_progress"] * (n // 4)
|
||||
issues = []
|
||||
for i in range(n):
|
||||
issue = {
|
||||
"id": i + 1,
|
||||
"title": f"Issue #{i + 1}: {'Fix auth bug' if i == 42 else 'General issue'}",
|
||||
"status": statuses[i % len(statuses)],
|
||||
"labels": ["bug"] if i % 10 == 0 else ["enhancement"],
|
||||
"assignee": f"user-{i % 5}",
|
||||
}
|
||||
if i in (42, 87):
|
||||
issue["status"] = "open"
|
||||
issue["labels"] = ["critical", "bug"]
|
||||
issue["title"] = f"CRITICAL: Auth failure in production (issue #{i + 1})"
|
||||
issues.append(issue)
|
||||
return issues
|
||||
|
||||
|
||||
def _make_test_results(n: int) -> list[dict]:
|
||||
"""Realistic test suite results."""
|
||||
results = []
|
||||
for i in range(n):
|
||||
result = {
|
||||
"name": f"test_{'auth' if i < 10 else 'general'}_{i}",
|
||||
"status": "pass",
|
||||
"duration_ms": 50 + i * 2,
|
||||
}
|
||||
if i in (3, 7, 45, 88):
|
||||
result["status"] = "fail"
|
||||
result["error"] = "AssertionError: expected True, got False"
|
||||
if i in (12, 67):
|
||||
result["status"] = "error"
|
||||
result["error"] = "TimeoutError: test exceeded 30s limit"
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def _make_log_entries(n: int) -> list[dict]:
|
||||
"""Realistic log entries."""
|
||||
entries = []
|
||||
for i in range(n):
|
||||
entry = {
|
||||
"timestamp": f"2024-01-15T10:{i:02d}:00Z",
|
||||
"level": "info",
|
||||
"message": f"Request processed in {10 + i}ms",
|
||||
"service": "api-gateway",
|
||||
}
|
||||
if i in (15, 45, 89):
|
||||
entry["level"] = "error"
|
||||
entry["message"] = "Connection refused: database pool exhausted"
|
||||
if i in (20, 50):
|
||||
entry["level"] = "warning"
|
||||
entry["message"] = "High memory usage: 85% threshold exceeded"
|
||||
entries.append(entry)
|
||||
return entries
|
||||
|
||||
|
||||
def _make_api_endpoints(n: int) -> list[dict]:
|
||||
"""Realistic API endpoint list."""
|
||||
return [
|
||||
{
|
||||
"path": f"/api/v1/{'users' if i < n // 3 else 'orders' if i < 2 * n // 3 else 'products'}/{i}",
|
||||
"method": "GET" if i % 3 else "POST",
|
||||
"status_code": 200 if i % 20 else 500,
|
||||
"latency_ms": 50 + i,
|
||||
}
|
||||
for i in range(n)
|
||||
]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Eval: Summary accuracy
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestSummaryAccuracy:
|
||||
"""Verify summaries accurately reflect what was dropped."""
|
||||
|
||||
def test_github_issues_categories_correct(self):
|
||||
"""Summary mentions actual status values from dropped items."""
|
||||
issues = _make_github_issues(100)
|
||||
kept = issues[:5]
|
||||
summary = summarize_dropped_items(issues, kept)
|
||||
|
||||
# Should mention the status values present in dropped items
|
||||
assert summary # Non-empty
|
||||
# At minimum, should contain some status category info
|
||||
has_category = any(s in summary.lower() for s in ["open", "closed", "in_progress"])
|
||||
assert has_category, f"Summary missing status categories: {summary}"
|
||||
|
||||
def test_test_results_mentions_failures(self):
|
||||
"""Summary mentions failures when test results are compressed."""
|
||||
results = _make_test_results(100)
|
||||
kept = results[:5]
|
||||
summary = summarize_dropped_items(results, kept)
|
||||
|
||||
assert summary
|
||||
# Should mention pass/fail somewhere
|
||||
has_result = any(s in summary.lower() for s in ["pass", "fail", "error"])
|
||||
assert has_result, f"Summary missing test result info: {summary}"
|
||||
|
||||
def test_log_entries_mentions_errors(self):
|
||||
"""Summary mentions error log entries."""
|
||||
logs = _make_log_entries(100)
|
||||
kept = logs[:3]
|
||||
summary = summarize_dropped_items(logs, kept)
|
||||
|
||||
assert summary
|
||||
# Should categorize by log level
|
||||
has_level = any(s in summary.lower() for s in ["info", "error", "warning"])
|
||||
assert has_level, f"Summary missing log level info: {summary}"
|
||||
|
||||
def test_no_hallucinated_categories(self):
|
||||
"""Summary should NOT mention categories that don't exist."""
|
||||
items = [{"status": "active", "id": i} for i in range(50)]
|
||||
kept = items[:3]
|
||||
summary = summarize_dropped_items(items, kept)
|
||||
|
||||
# Should NOT mention statuses that don't exist in the data
|
||||
assert "error" not in summary.lower() or "notable" in summary.lower()
|
||||
assert "fail" not in summary.lower()
|
||||
assert "critical" not in summary.lower()
|
||||
|
||||
def test_summary_proportional_to_data(self):
|
||||
"""Category counts in summary should roughly match actual data."""
|
||||
items = (
|
||||
[{"type": "log", "data": "x"}] * 100
|
||||
+ [{"type": "metric", "data": "y"}] * 50
|
||||
+ [{"type": "alert", "data": "z"}] * 10
|
||||
)
|
||||
kept = items[:3]
|
||||
summary = summarize_dropped_items(items, kept)
|
||||
|
||||
# "log" should appear with a higher count than "alert"
|
||||
# (We can't verify exact counts from the summary string,
|
||||
# but we verify the summary is non-empty and reasonable)
|
||||
assert summary
|
||||
assert len(summary) < 300
|
||||
|
||||
|
||||
class TestSummaryUsefulness:
|
||||
"""Verify summaries contain information useful for retrieval."""
|
||||
|
||||
def test_enough_info_to_search(self):
|
||||
"""Summary should contain terms the LLM could use as search queries."""
|
||||
results = _make_test_results(100)
|
||||
kept = results[:5]
|
||||
summary = summarize_dropped_items(results, kept)
|
||||
|
||||
# The LLM should be able to extract search terms from the summary
|
||||
# At minimum, it should know WHAT KIND of items are in the compressed data
|
||||
assert len(summary) > 10, "Summary too short to be useful"
|
||||
|
||||
def test_notable_items_actionable(self):
|
||||
"""Notable items should contain enough info to act on."""
|
||||
logs = _make_log_entries(100)
|
||||
kept = logs[:2]
|
||||
summary = summarize_dropped_items(logs, kept)
|
||||
|
||||
# If there are errors in the logs, the summary should help
|
||||
# the LLM decide to retrieve them
|
||||
assert summary
|
||||
# Just verify it's substantive enough
|
||||
assert len(summary.split()) > 3
|
||||
|
||||
def test_api_endpoints_described(self):
|
||||
"""API endpoint data should produce some useful description."""
|
||||
endpoints = _make_api_endpoints(60)
|
||||
kept = endpoints[:5]
|
||||
summary = summarize_dropped_items(endpoints, kept)
|
||||
|
||||
assert summary # Should produce SOMETHING, even without type/status fields
|
||||
|
||||
|
||||
class TestCodeSummaryAccuracy:
|
||||
"""Verify code summaries accurately describe removed sections."""
|
||||
|
||||
def test_real_python_module(self):
|
||||
"""Summary of a realistic Python module compression."""
|
||||
# Use AST-based summary (language-agnostic)
|
||||
bodies = [
|
||||
("def __init__(self, url: str, pool_size: int = 10):", "...", 8),
|
||||
("def connect(self) -> Any:", "...", 15),
|
||||
("def _create_new(self) -> Any:", "...", 22),
|
||||
("def release(self, conn: Any) -> None:", "...", 28),
|
||||
("def close_all(self) -> None:", "...", 33),
|
||||
("def create_engine(url: str) -> DatabaseConnection:", "...", 38),
|
||||
]
|
||||
summary = summarize_compressed_code(bodies, 6)
|
||||
assert "6 bodies compressed" in summary
|
||||
has_names = any(
|
||||
name in summary for name in ["connect()", "release()", "close_all()", "create_engine()"]
|
||||
)
|
||||
assert has_names, f"Summary missing function names: {summary}"
|
||||
@@ -1,251 +0,0 @@
|
||||
"""Hard eval: Cases where the LLM has NO reason to check compressed data.
|
||||
|
||||
The previous eval asked "are there failures?" — that's too easy, the LLM
|
||||
will proactively check regardless of summary.
|
||||
|
||||
This eval tests the SUBTLE case: the user asks a DIFFERENT question,
|
||||
but the answer is in the compressed data. The summary is the only hint.
|
||||
|
||||
Requires: ANTHROPIC_API_KEY in environment or .env file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests._dotenv import autouse_apply_env, load_env_overrides
|
||||
|
||||
_env_overrides = load_env_overrides()
|
||||
ANTHROPIC_KEY = os.environ.get("ANTHROPIC_API_KEY") or _env_overrides.get("ANTHROPIC_API_KEY", "")
|
||||
apply_dotenv = autouse_apply_env(_env_overrides)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not ANTHROPIC_KEY,
|
||||
reason="ANTHROPIC_API_KEY not set",
|
||||
)
|
||||
|
||||
HEADROOM_RETRIEVE_TOOL = {
|
||||
"name": "headroom_retrieve",
|
||||
"description": "Retrieve uncompressed content. Pass a query to search within it.",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"hash": {"type": "string"},
|
||||
"query": {"type": "string"},
|
||||
},
|
||||
"required": ["hash"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _call_claude(messages, tools, max_tokens=300):
|
||||
import httpx
|
||||
|
||||
resp = httpx.post(
|
||||
"https://api.anthropic.com/v1/messages",
|
||||
headers={
|
||||
"X-Api-Key": ANTHROPIC_KEY,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": "claude-sonnet-4-5-20250929",
|
||||
"max_tokens": max_tokens,
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _get_tool_calls(resp):
|
||||
return [
|
||||
{"name": b["name"], "input": b.get("input", {})}
|
||||
for b in resp.get("content", [])
|
||||
if b.get("type") == "tool_use"
|
||||
]
|
||||
|
||||
|
||||
def _get_text(resp):
|
||||
return " ".join(b.get("text", "") for b in resp.get("content", []) if b.get("type") == "text")
|
||||
|
||||
|
||||
class TestHardCases:
|
||||
"""Cases where the LLM wouldn't naturally check compressed data."""
|
||||
|
||||
def test_config_lookup_with_summary(self):
|
||||
"""User asks about a config value that's in compressed data.
|
||||
|
||||
The visible items are all about 'production' env.
|
||||
The compressed items include 'staging' configs.
|
||||
Summary mentions this. LLM should retrieve.
|
||||
"""
|
||||
visible = [
|
||||
{"env": "production", "key": "DATABASE_URL", "value": "postgres://prod-db:5432/app"},
|
||||
{"env": "production", "key": "REDIS_URL", "value": "redis://prod-cache:6379"},
|
||||
{"env": "production", "key": "API_RATE_LIMIT", "value": "1000"},
|
||||
]
|
||||
# Hidden in compressed: staging configs
|
||||
all_items = (
|
||||
visible
|
||||
+ [
|
||||
{
|
||||
"env": "staging",
|
||||
"key": "DATABASE_URL",
|
||||
"value": "postgres://staging-db:5432/app",
|
||||
},
|
||||
{"env": "staging", "key": "REDIS_URL", "value": "redis://staging-cache:6379"},
|
||||
{"env": "staging", "key": "DEBUG_MODE", "value": "true"},
|
||||
{"env": "staging", "key": "LOG_LEVEL", "value": "debug"},
|
||||
]
|
||||
* 10
|
||||
+ [
|
||||
{
|
||||
"env": "development",
|
||||
"key": "DATABASE_URL",
|
||||
"value": "postgres://localhost:5432/dev",
|
||||
},
|
||||
]
|
||||
* 5
|
||||
)
|
||||
|
||||
from headroom.transforms.compression_summary import summarize_dropped_items
|
||||
|
||||
summary = summarize_dropped_items(all_items, visible)
|
||||
|
||||
compressed_output = json.dumps(visible, indent=2)
|
||||
compressed_output += (
|
||||
f"\n[{len(all_items) - len(visible)} items compressed to {len(visible)}."
|
||||
f" Omitted: {summary}."
|
||||
f' Retrieve specific items: headroom_retrieve(hash="config_hash", query="search")]'
|
||||
)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"Here are the application configs:\n\n{compressed_output}\n\n"
|
||||
"What is the staging database URL?"
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
resp = _call_claude(messages, [HEADROOM_RETRIEVE_TOOL])
|
||||
tool_calls = _get_tool_calls(resp)
|
||||
text = _get_text(resp)
|
||||
|
||||
print(f"\n Summary: {summary}")
|
||||
print(f" Stop reason: {resp.get('stop_reason')}")
|
||||
print(f" Tool calls: {tool_calls}")
|
||||
if text:
|
||||
print(f" Text: {text[:200]}")
|
||||
|
||||
# WITH summary mentioning "staging" → should retrieve
|
||||
if resp.get("stop_reason") == "tool_use":
|
||||
assert tool_calls[0]["name"] == "headroom_retrieve"
|
||||
query = tool_calls[0]["input"].get("query", "").lower()
|
||||
assert "staging" in query or "database" in query
|
||||
print(" RESULT: Retrieved staging config ✓")
|
||||
else:
|
||||
# If LLM didn't retrieve, it should at least mention the data is compressed
|
||||
assert "compressed" in text.lower() or "staging" in text.lower()
|
||||
print(" RESULT: Mentioned compressed data but didn't retrieve")
|
||||
|
||||
def test_config_lookup_without_summary(self):
|
||||
"""Same question, but NO summary. LLM only sees production configs."""
|
||||
visible = [
|
||||
{"env": "production", "key": "DATABASE_URL", "value": "postgres://prod-db:5432/app"},
|
||||
{"env": "production", "key": "REDIS_URL", "value": "redis://prod-cache:6379"},
|
||||
{"env": "production", "key": "API_RATE_LIMIT", "value": "1000"},
|
||||
]
|
||||
|
||||
compressed_output = json.dumps(visible, indent=2)
|
||||
compressed_output += "\n[45 items compressed to 3. Retrieve more: hash=config_hash]"
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"Here are the application configs:\n\n{compressed_output}\n\n"
|
||||
"What is the staging database URL?"
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
resp = _call_claude(messages, [HEADROOM_RETRIEVE_TOOL])
|
||||
tool_calls = _get_tool_calls(resp)
|
||||
text = _get_text(resp)
|
||||
|
||||
print(f"\n Stop reason: {resp.get('stop_reason')}")
|
||||
print(f" Tool calls: {tool_calls}")
|
||||
if text:
|
||||
print(f" Text: {text[:200]}")
|
||||
|
||||
if resp.get("stop_reason") == "tool_use":
|
||||
print(" RESULT: LLM proactively retrieved (smart)")
|
||||
else:
|
||||
print(" RESULT: LLM did NOT retrieve staging config")
|
||||
|
||||
def test_specific_user_in_large_list_with_summary(self):
|
||||
"""Find a specific user in a compressed user list.
|
||||
|
||||
Summary mentions user roles. User asks about admins.
|
||||
"""
|
||||
visible = [
|
||||
{"id": i, "name": f"user_{i}", "role": "member", "email": f"user{i}@co.com"}
|
||||
for i in range(5)
|
||||
]
|
||||
all_items = (
|
||||
visible
|
||||
+ [
|
||||
{"id": i, "name": f"user_{i}", "role": "member", "email": f"user{i}@co.com"}
|
||||
for i in range(5, 95)
|
||||
]
|
||||
+ [
|
||||
{"id": 96, "name": "admin_sarah", "role": "admin", "email": "sarah@co.com"},
|
||||
{"id": 97, "name": "admin_mike", "role": "admin", "email": "mike@co.com"},
|
||||
{"id": 98, "name": "superadmin_jane", "role": "superadmin", "email": "jane@co.com"},
|
||||
]
|
||||
)
|
||||
|
||||
from headroom.transforms.compression_summary import summarize_dropped_items
|
||||
|
||||
summary = summarize_dropped_items(all_items, visible)
|
||||
|
||||
compressed_output = json.dumps(visible, indent=2)
|
||||
compressed_output += (
|
||||
f"\n[{len(all_items) - len(visible)} items compressed to {len(visible)}."
|
||||
f" Omitted: {summary}."
|
||||
f' Retrieve: headroom_retrieve(hash="users_hash", query="search")]'
|
||||
)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"Here's our user list:\n\n{compressed_output}\n\n"
|
||||
"Who are the admin users? I need to contact them."
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
resp = _call_claude(messages, [HEADROOM_RETRIEVE_TOOL])
|
||||
tool_calls = _get_tool_calls(resp)
|
||||
text = _get_text(resp)
|
||||
|
||||
print(f"\n Summary: {summary}")
|
||||
print(f" Stop reason: {resp.get('stop_reason')}")
|
||||
print(f" Tool calls: {tool_calls}")
|
||||
if text:
|
||||
print(f" Text: {text[:200]}")
|
||||
|
||||
if resp.get("stop_reason") == "tool_use":
|
||||
query = tool_calls[0]["input"].get("query", "").lower()
|
||||
assert "admin" in query
|
||||
print(f" RESULT: Retrieved admin users (query='{query}') ✓")
|
||||
else:
|
||||
print(" RESULT: Did not retrieve admin users")
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Integration eval: Compression summaries with real LLM calls.
|
||||
|
||||
Tests whether compression summaries actually help the LLM find information
|
||||
in compressed data. Compares behavior with and without summaries.
|
||||
Tests whether the code-compression summary actually helps the LLM tell which
|
||||
functions were removed from compressed code.
|
||||
|
||||
Requires: ANTHROPIC_API_KEY in environment or .env file.
|
||||
|
||||
@@ -10,7 +10,6 @@ Run: python -m pytest tests/test_compression_summary_integration.py -v -s
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
@@ -48,114 +47,8 @@ def _call_claude(messages: list[dict], max_tokens: int = 200) -> dict:
|
||||
return resp.json()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Test data: realistic tool output that gets compressed
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _make_test_suite_output(n: int = 100) -> list[dict]:
|
||||
"""Simulate a large test suite result (like from a CI/CD tool)."""
|
||||
results = []
|
||||
for i in range(n):
|
||||
result = {
|
||||
"test_name": f"test_module_{i // 10}.test_case_{i}",
|
||||
"status": "passed",
|
||||
"duration_ms": 50 + i * 3,
|
||||
"file": f"tests/test_module_{i // 10}.py",
|
||||
}
|
||||
# Inject specific failures that the LLM should find
|
||||
if i == 42:
|
||||
result["status"] = "failed"
|
||||
result["error"] = "AssertionError: expected status 200, got 401 in auth_middleware"
|
||||
result["test_name"] = "test_auth.test_login_with_expired_token"
|
||||
if i == 67:
|
||||
result["status"] = "failed"
|
||||
result["error"] = "TimeoutError: database connection pool exhausted after 30s"
|
||||
result["test_name"] = "test_database.test_concurrent_connections"
|
||||
if i == 88:
|
||||
result["status"] = "error"
|
||||
result["error"] = "ImportError: cannot import name 'NewFeature' from 'app.features'"
|
||||
result["test_name"] = "test_features.test_new_feature_integration"
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
class TestSummaryHelpfulness:
|
||||
"""Compare LLM accuracy with vs without compression summaries."""
|
||||
|
||||
def test_find_failures_with_summary(self):
|
||||
"""LLM can identify failure types from the summary alone."""
|
||||
test_results = _make_test_suite_output(100)
|
||||
|
||||
# Simulate compression: keep first 10, compress rest with summary
|
||||
kept = test_results[:10]
|
||||
from headroom.transforms.compression_summary import summarize_dropped_items
|
||||
|
||||
summary = summarize_dropped_items(test_results, kept)
|
||||
|
||||
compressed_output = json.dumps(kept, indent=2)
|
||||
compressed_output += f"\n[90 items compressed to 10. Omitted: {summary}. "
|
||||
compressed_output += (
|
||||
'Retrieve specific items: headroom_retrieve(hash="abc123", query="your search")]'
|
||||
)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Here are the test results from CI:\n\n"
|
||||
f"{compressed_output}\n\n"
|
||||
"Are there any test failures? What types of failures are there? "
|
||||
"Answer concisely."
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
resp = _call_claude(messages)
|
||||
text = resp.get("content", [{}])[0].get("text", "").lower()
|
||||
|
||||
# The LLM should mention failures (from the summary info)
|
||||
has_failure_info = any(
|
||||
word in text for word in ["fail", "error", "timeout", "assert", "import"]
|
||||
)
|
||||
print(f"\n Summary: {summary}")
|
||||
print(f" LLM response: {text[:200]}")
|
||||
print(f" Detected failure info: {has_failure_info}")
|
||||
|
||||
assert has_failure_info, f"LLM didn't detect failures from summary. Response: {text[:300]}"
|
||||
|
||||
def test_find_failures_without_summary(self):
|
||||
"""Baseline: LLM with NO summary — just '[90 items compressed]'."""
|
||||
test_results = _make_test_suite_output(100)
|
||||
|
||||
kept = test_results[:10]
|
||||
compressed_output = json.dumps(kept, indent=2)
|
||||
compressed_output += "\n[90 items compressed to 10. Retrieve more: hash=abc123]"
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Here are the test results from CI:\n\n"
|
||||
f"{compressed_output}\n\n"
|
||||
"Are there any test failures? What types of failures are there? "
|
||||
"Answer concisely."
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
resp = _call_claude(messages)
|
||||
text = resp.get("content", [{}])[0].get("text", "").lower()
|
||||
|
||||
# The LLM may or may not detect failures (it only sees 10 passing tests)
|
||||
has_failure_info = any(
|
||||
word in text for word in ["fail", "error", "timeout", "assert", "import"]
|
||||
)
|
||||
print(f"\n LLM response (no summary): {text[:200]}")
|
||||
print(f" Detected failure info: {has_failure_info}")
|
||||
|
||||
# We're NOT asserting here — this is the baseline.
|
||||
# We expect this to often MISS failures since the summary is generic.
|
||||
"""Check that the code summary carries information the LLM can use."""
|
||||
|
||||
def test_code_summary_helps_identify_functions(self):
|
||||
"""LLM can identify which functions were removed from compressed code."""
|
||||
|
||||
@@ -1,289 +0,0 @@
|
||||
"""Eval: Does the LLM invoke headroom_retrieve when summaries are present?
|
||||
|
||||
The REAL test — it's not enough for the LLM to know something is missing.
|
||||
It must actually call the tool to fetch it.
|
||||
|
||||
Compares:
|
||||
- WITH summary: LLM sees "2 failed, 1 error" → should call headroom_retrieve
|
||||
- WITHOUT summary: LLM sees "[90 items compressed]" → likely does NOT call tool
|
||||
|
||||
Requires: ANTHROPIC_API_KEY in environment or .env file.
|
||||
|
||||
Run: python -m pytest tests/test_compression_summary_tool_eval.py -v -s
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests._dotenv import autouse_apply_env, load_env_overrides
|
||||
|
||||
_env_overrides = load_env_overrides()
|
||||
ANTHROPIC_KEY = os.environ.get("ANTHROPIC_API_KEY") or _env_overrides.get("ANTHROPIC_API_KEY", "")
|
||||
apply_dotenv = autouse_apply_env(_env_overrides)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not ANTHROPIC_KEY,
|
||||
reason="ANTHROPIC_API_KEY not set — skipping integration tests",
|
||||
)
|
||||
|
||||
# The headroom_retrieve tool definition (same as what CCR injects)
|
||||
HEADROOM_RETRIEVE_TOOL = {
|
||||
"name": "headroom_retrieve",
|
||||
"description": (
|
||||
"Retrieve original uncompressed content from Headroom's compression cache. "
|
||||
"Use this when you need more details from compressed data. "
|
||||
"You can pass a query to search within the compressed content."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"hash": {
|
||||
"type": "string",
|
||||
"description": "The hash key from the compression marker",
|
||||
},
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Optional search query to find specific items within the compressed data",
|
||||
},
|
||||
},
|
||||
"required": ["hash"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _call_claude_with_tools(messages: list[dict], tools: list[dict], max_tokens: int = 300) -> dict:
|
||||
"""Make a real Anthropic API call with tool use."""
|
||||
import httpx
|
||||
|
||||
resp = httpx.post(
|
||||
"https://api.anthropic.com/v1/messages",
|
||||
headers={
|
||||
"X-Api-Key": ANTHROPIC_KEY,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": "claude-sonnet-4-5-20250929",
|
||||
"max_tokens": max_tokens,
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _make_test_results(n: int = 100) -> list[dict]:
|
||||
"""Test suite output with hidden failures in the compressed portion."""
|
||||
results = []
|
||||
for i in range(n):
|
||||
result = {
|
||||
"test_name": f"test_module_{i // 10}.test_case_{i}",
|
||||
"status": "passed",
|
||||
"duration_ms": 50 + i * 3,
|
||||
}
|
||||
if i == 42:
|
||||
result["status"] = "failed"
|
||||
result["error"] = "AssertionError: expected 200, got 401 in auth_middleware"
|
||||
result["test_name"] = "test_auth.test_login_expired_token"
|
||||
if i == 67:
|
||||
result["status"] = "failed"
|
||||
result["error"] = "TimeoutError: database pool exhausted after 30s"
|
||||
result["test_name"] = "test_database.test_concurrent_connections"
|
||||
if i == 88:
|
||||
result["status"] = "error"
|
||||
result["error"] = "ImportError: cannot import 'NewFeature'"
|
||||
result["test_name"] = "test_features.test_new_feature_integration"
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def _has_tool_use(response: dict) -> bool:
|
||||
"""Check if the response contains a tool_use block."""
|
||||
for block in response.get("content", []):
|
||||
if block.get("type") == "tool_use":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _get_tool_calls(response: dict) -> list[dict]:
|
||||
"""Extract all tool_use blocks from response."""
|
||||
calls = []
|
||||
for block in response.get("content", []):
|
||||
if block.get("type") == "tool_use":
|
||||
calls.append(
|
||||
{
|
||||
"name": block.get("name"),
|
||||
"input": block.get("input", {}),
|
||||
}
|
||||
)
|
||||
return calls
|
||||
|
||||
|
||||
class TestToolInvocationWithSummary:
|
||||
"""The real eval: does the LLM call headroom_retrieve?"""
|
||||
|
||||
def test_with_summary_triggers_tool_call(self):
|
||||
"""WITH compression summary → LLM should call headroom_retrieve."""
|
||||
test_results = _make_test_results(100)
|
||||
kept = test_results[:10] # All passing
|
||||
|
||||
from headroom.transforms.compression_summary import summarize_dropped_items
|
||||
|
||||
summary = summarize_dropped_items(test_results, kept)
|
||||
|
||||
compressed = json.dumps(kept, indent=2)
|
||||
compressed += (
|
||||
f"\n[90 items compressed to 10. Omitted: {summary}."
|
||||
f' Retrieve specific items: headroom_retrieve(hash="ccr_test_abc123", query="your search")]'
|
||||
)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Here are the test results from our CI pipeline:\n\n"
|
||||
f"{compressed}\n\n"
|
||||
"Tell me about any test failures. What went wrong?"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
resp = _call_claude_with_tools(messages, [HEADROOM_RETRIEVE_TOOL])
|
||||
|
||||
tool_calls = _get_tool_calls(resp)
|
||||
stop_reason = resp.get("stop_reason", "")
|
||||
|
||||
print(f"\n Summary: {summary}")
|
||||
print(f" Stop reason: {stop_reason}")
|
||||
print(f" Tool calls: {tool_calls}")
|
||||
|
||||
# With a summary showing failures, the LLM SHOULD call the tool
|
||||
if stop_reason == "tool_use":
|
||||
assert len(tool_calls) > 0
|
||||
call = tool_calls[0]
|
||||
assert call["name"] == "headroom_retrieve"
|
||||
assert call["input"].get("hash") == "ccr_test_abc123"
|
||||
# The query should be about failures/errors
|
||||
query = call["input"].get("query", "").lower()
|
||||
print(f" Query used: {query}")
|
||||
has_relevant_query = any(
|
||||
term in query for term in ["fail", "error", "issue", "problem", "broken", "test"]
|
||||
)
|
||||
assert has_relevant_query, f"Tool was called but query isn't relevant: {query}"
|
||||
print(" RESULT: LLM invoked headroom_retrieve with relevant query ✓")
|
||||
else:
|
||||
# LLM responded with text — check if it at least mentions the failures
|
||||
text = ""
|
||||
for block in resp.get("content", []):
|
||||
if block.get("type") == "text":
|
||||
text += block.get("text", "")
|
||||
print(f" LLM text response: {text[:200]}")
|
||||
# It's acceptable if the LLM mentions it WANTS to retrieve
|
||||
mentions_retrieval = any(
|
||||
term in text.lower()
|
||||
for term in ["retrieve", "headroom_retrieve", "fetch", "see more", "compressed"]
|
||||
)
|
||||
print(f" Mentions retrieval: {mentions_retrieval}")
|
||||
|
||||
def test_without_summary_baseline(self):
|
||||
"""WITHOUT compression summary → LLM likely does NOT call tool."""
|
||||
test_results = _make_test_results(100)
|
||||
kept = test_results[:10] # All passing
|
||||
|
||||
compressed = json.dumps(kept, indent=2)
|
||||
compressed += "\n[90 items compressed to 10. Retrieve more: hash=ccr_test_abc123]"
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Here are the test results from our CI pipeline:\n\n"
|
||||
f"{compressed}\n\n"
|
||||
"Tell me about any test failures. What went wrong?"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
resp = _call_claude_with_tools(messages, [HEADROOM_RETRIEVE_TOOL])
|
||||
|
||||
tool_calls = _get_tool_calls(resp)
|
||||
stop_reason = resp.get("stop_reason", "")
|
||||
|
||||
print(f"\n Stop reason: {stop_reason}")
|
||||
print(f" Tool calls: {tool_calls}")
|
||||
|
||||
if stop_reason == "tool_use":
|
||||
call = tool_calls[0]
|
||||
print(f" Query used: {call['input'].get('query', 'none')}")
|
||||
print(" RESULT: LLM DID invoke tool (may check proactively)")
|
||||
else:
|
||||
text = ""
|
||||
for block in resp.get("content", []):
|
||||
if block.get("type") == "text":
|
||||
text += block.get("text", "")
|
||||
print(f" LLM text response: {text[:200]}")
|
||||
print(" RESULT: LLM did NOT invoke tool — assumed all tests passed")
|
||||
|
||||
def test_code_summary_triggers_retrieval(self):
|
||||
"""Code compression summary → LLM should retrieve specific function."""
|
||||
compressed_code = '''class PaymentProcessor:
|
||||
"""Processes payments via Stripe."""
|
||||
|
||||
def __init__(self, api_key: str):
|
||||
# [2 lines omitted]
|
||||
pass
|
||||
|
||||
def charge(self, amount: float, currency: str, token: str) -> dict:
|
||||
# [8 lines omitted]
|
||||
pass
|
||||
|
||||
def refund(self, charge_id: str, amount: float = None) -> dict:
|
||||
# [3 lines omitted]
|
||||
pass
|
||||
|
||||
def get_balance(self) -> float:
|
||||
# [2 lines omitted]
|
||||
pass
|
||||
# [180 tokens compressed. removed: def charge (12 lines), def refund (6 lines). Retrieve full code: headroom_retrieve(hash="ccr_code_xyz", query="function name")]'''
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Here's the payment processor code:\n\n"
|
||||
f"```python\n{compressed_code}\n```\n\n"
|
||||
"There's a bug in the retry logic for failed charges. "
|
||||
"Can you find and fix it?"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
resp = _call_claude_with_tools(messages, [HEADROOM_RETRIEVE_TOOL])
|
||||
|
||||
tool_calls = _get_tool_calls(resp)
|
||||
stop_reason = resp.get("stop_reason", "")
|
||||
|
||||
print(f"\n Stop reason: {stop_reason}")
|
||||
print(f" Tool calls: {tool_calls}")
|
||||
|
||||
if stop_reason == "tool_use":
|
||||
call = tool_calls[0]
|
||||
assert call["name"] == "headroom_retrieve"
|
||||
query = call["input"].get("query", "").lower()
|
||||
print(f" Query: {query}")
|
||||
# Should be asking for the charge function specifically
|
||||
has_charge = any(term in query for term in ["charge", "retry", "payment", "stripe"])
|
||||
print(f" Targets charge/retry: {has_charge}")
|
||||
print(" RESULT: LLM invoked tool to get the charge() implementation ✓")
|
||||
else:
|
||||
text = ""
|
||||
for block in resp.get("content", []):
|
||||
if block.get("type") == "text":
|
||||
text += block.get("text", "")
|
||||
print(f" LLM text: {text[:200]}")
|
||||
print(" RESULT: LLM did NOT invoke tool")
|
||||
Reference in New Issue
Block a user