mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:10:00 +08:00
## Description Claude Code answers a ToolSearch call with `tool_reference` blocks plus a "Tool loaded." text block. `extract_user_query` takes the newest user text, so after an MCP tool is loaded the relevance query is "Tool loaded." and the question never reaches SmartCrusher. It's the same kind of bug as #4628 (`<system-reminder>`). Observed on Claude Code 2.1.319. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) ## Changes Made - In a turn whose tool_result carries `tool_reference` blocks, drop the exact "Tool loaded." text. Other text in that turn still counts, and "Tool loaded." outside such a turn is unchanged. - The `tool_reference` check is the stable signal; the exact string follows Claude Code's current wording. Happy to drop all text in such a turn instead if you prefer. ## Testing - [x] New tests: ToolSearch turn, ToolSearch turn with real user text, plain "Tool loaded." text. The first two fail without the fix. - [x] `ruff check`, `ruff format --check`; tests for `extract_user_query` callers pass. Fork CI: green. ### Test Output ``` $ pytest -q tests/test_utils.py 38 passed in 0.33s $ pytest -q tests/test_utils.py -k "tool_search or tool_loaded" 3 passed, 35 deselected in 0.18s $ ruff check headroom/utils.py tests/test_utils.py && ruff format --check headroom/utils.py tests/test_utils.py All checks passed! 2 files already formatted ``` ## Real Behavior Proof - Environment: macOS arm64, Claude Code (Sonnet) through `headroom proxy` (PyPI 0.43.1 vs source), an MCP tool returning 100 of this repo's issues. - Exact command / steps: 12 questions "Who opened issue #N, and is it open or closed?"; scripts and answers: https://github.com/cherylaurelia/headroom-demo/tree/main/bench - Observed result: the logged query was "Tool loaded." on 0.43.1, and the question with this fix. Accuracy 2/12 → 12/12 together with the SmartCrusher fix in `fix/keep-query-matches` (2/12 with that fix alone). - Not tested: agents other than Claude Code. ## Runtime Rollout Safety - Rollout-managed feature(s): n/a - Minimum rollout channel: n/a - Stable/default behavior changed: only for ToolSearch result turns - Kill switch / disable path: n/a - Unsafe override required: no - Qualification impact: none - Rollback path: revert ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] I did **not** edit `CHANGELOG.md` Written with Claude Code, which also ran the tests and measurements above. <!-- devin-review-badge-begin --> --- <a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4714" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4" alt="Devin Review"></picture></a> <!-- devin-review-badge-end --> Co-authored-by: Claude <noreply@anthropic.com>
322 lines
13 KiB
Python
322 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
from headroom import utils
|
|
|
|
|
|
class FakeProvider:
|
|
def __init__(self, result: str | None) -> None:
|
|
self.result = result
|
|
self.calls: list[tuple[int, int, str, int]] = []
|
|
|
|
def estimate_cost(
|
|
self, input_tokens: int, output_tokens: int, model: str, cached_tokens: int = 0
|
|
) -> str | None:
|
|
self.calls.append((input_tokens, output_tokens, model, cached_tokens))
|
|
return self.result
|
|
|
|
|
|
def test_hash_helpers_and_request_id() -> None:
|
|
request_id = utils.generate_request_id()
|
|
assert len(request_id) == 36
|
|
assert request_id.count("-") == 4
|
|
|
|
assert (
|
|
utils.compute_hash("hello")
|
|
== "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
|
|
)
|
|
assert utils.compute_hash("hi\ud800") == utils.compute_hash(
|
|
"hi\ud800".encode("utf-8", "surrogatepass")
|
|
)
|
|
assert utils.compute_short_hash("hello", length=8) == "2cf24dba"
|
|
|
|
|
|
def test_extract_user_query_and_message_hashes() -> None:
|
|
messages = [
|
|
{"role": "system", "content": "rules"},
|
|
{"role": "user", "content": " "},
|
|
{"role": "assistant", "content": "skip"},
|
|
{
|
|
"role": "user",
|
|
"content": [{"type": "image"}, {"type": "text", "text": " latest question "}],
|
|
},
|
|
]
|
|
assert utils.extract_user_query(messages) == "latest question"
|
|
assert utils.extract_user_query([{"role": "assistant", "content": "skip"}]) == ""
|
|
|
|
|
|
@pytest.mark.parametrize("latest_only", [False, True])
|
|
def test_extract_user_query_combines_text_blocks(latest_only: bool) -> None:
|
|
messages = [
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "text", "text": " Review incidents "},
|
|
{"type": "image", "source": "example"},
|
|
{"type": "text", "text": " "},
|
|
{"type": "text", "text": " Focus on Kafka replication "},
|
|
],
|
|
}
|
|
]
|
|
assert utils.extract_user_query(messages, latest_user_turn_only=latest_only) == (
|
|
"Review incidents\nFocus on Kafka replication"
|
|
)
|
|
|
|
|
|
def test_extract_user_query_skips_system_reminder_blocks() -> None:
|
|
messages = [
|
|
{"role": "user", "content": "older question"},
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "text",
|
|
"text": "<system-reminder>injected instructions</system-reminder>",
|
|
},
|
|
{"type": "text", "text": "real question"},
|
|
{"type": "text", "text": "<system-reminder>more instructions</system-reminder>"},
|
|
],
|
|
},
|
|
]
|
|
assert utils.extract_user_query(messages) == "real question"
|
|
assert utils.extract_user_query(messages, latest_user_turn_only=True) == "real question"
|
|
|
|
messages[-1]["content"] = [
|
|
{"type": "text", "text": "<system-reminder>injected</system-reminder>"}
|
|
]
|
|
assert utils.extract_user_query(messages) == "older question"
|
|
assert utils.extract_user_query(messages, latest_user_turn_only=True) == ""
|
|
|
|
|
|
@pytest.mark.parametrize("as_blocks", [False, True])
|
|
@pytest.mark.parametrize(
|
|
("content", "expected"),
|
|
[
|
|
("<system-reminder>context</system-reminder>\nFind Kafka lag", "Find Kafka lag"),
|
|
("Find <system-reminder>context</system-reminder> Kafka lag", "Find Kafka lag"),
|
|
(
|
|
"<system-reminder id='1'>a</system-reminder>Find <system-reminder>b</system-reminder>Kafka",
|
|
"Find Kafka",
|
|
),
|
|
("<System-Reminder>context</System-Reminder>Find Kafka", "Find Kafka"),
|
|
("İ<system-reminder>x</system-reminder>Kafka", "İ Kafka"),
|
|
("Find <system-reminder>unfinished", "Find <system-reminder>unfinished"),
|
|
(
|
|
"Find <system-reminder-extra>example</system-reminder> Kafka",
|
|
"Find <system-reminder-extra>example</system-reminder> Kafka",
|
|
),
|
|
],
|
|
)
|
|
def test_extract_user_query_keeps_text_around_reminders(
|
|
content: str, expected: str, as_blocks: bool
|
|
) -> None:
|
|
body = [{"type": "text", "text": content}] if as_blocks else content
|
|
messages = [{"role": "user", "content": body}]
|
|
assert utils.extract_user_query(messages, latest_user_turn_only=True) == expected
|
|
|
|
|
|
@pytest.mark.parametrize("as_blocks", [False, True])
|
|
def test_reminder_only_turn_keeps_user_query_fallback(as_blocks: bool) -> None:
|
|
reminder = "<system-reminder>context</system-reminder>"
|
|
content = [{"type": "text", "text": reminder}] if as_blocks else reminder
|
|
messages = [{"role": "user", "content": "older question"}, {"role": "user", "content": content}]
|
|
assert utils.extract_user_query(messages) == "older question"
|
|
assert utils.extract_user_query(messages, latest_user_turn_only=True) == ""
|
|
|
|
|
|
def test_extract_user_query_latest_user_turn_only() -> None:
|
|
"""The flag stops at the newest user turn instead of resurrecting an older one."""
|
|
# A tool_result continuation: the newest user turn carries no text at all.
|
|
messages = [
|
|
{"role": "user", "content": "the original question"},
|
|
{"role": "assistant", "content": "calling a tool"},
|
|
{"role": "user", "content": [{"type": "tool_result", "content": "output"}]},
|
|
]
|
|
|
|
# Default walks back and finds the older turn's question.
|
|
assert utils.extract_user_query(messages) == "the original question"
|
|
# Scoped to this turn, there is no question to score against.
|
|
assert utils.extract_user_query(messages, latest_user_turn_only=True) == ""
|
|
|
|
|
|
def test_extract_user_query_skips_tool_search_load_note() -> None:
|
|
"""Claude Code answers a ToolSearch call with ``tool_reference`` blocks plus a
|
|
"Tool loaded." text block; that note is not the user's question."""
|
|
messages = [
|
|
{"role": "user", "content": "Who opened issue #4475?"},
|
|
{"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "ToolSearch"}]},
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "tool_result",
|
|
"tool_use_id": "t1",
|
|
"content": [{"type": "tool_reference", "tool_name": "mcp__gh__list_issues"}],
|
|
},
|
|
{"type": "text", "text": "Tool loaded."},
|
|
],
|
|
},
|
|
{"role": "assistant", "content": [{"type": "tool_use", "id": "t2", "name": "list"}]},
|
|
{
|
|
"role": "user",
|
|
"content": [{"type": "tool_result", "tool_use_id": "t2", "content": "[]"}],
|
|
},
|
|
]
|
|
|
|
assert utils.extract_user_query(messages) == "Who opened issue #4475?"
|
|
assert utils.extract_user_query(messages, latest_user_turn_only=True) == ""
|
|
|
|
|
|
def test_extract_user_query_keeps_user_text_beside_tool_search_load() -> None:
|
|
"""Only the "Tool loaded." note is dropped; real text in that turn still counts."""
|
|
messages = [
|
|
{"role": "user", "content": "older question"},
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "tool_result",
|
|
"tool_use_id": "t1",
|
|
"content": [{"type": "tool_reference", "tool_name": "mcp__gh__list_issues"}],
|
|
},
|
|
{"type": "text", "text": "Tool loaded."},
|
|
{"type": "text", "text": "only open issues please"},
|
|
],
|
|
},
|
|
]
|
|
|
|
assert utils.extract_user_query(messages) == "only open issues please"
|
|
|
|
|
|
def test_extract_user_query_keeps_tool_loaded_text_without_tool_reference() -> None:
|
|
"""Outside a ToolSearch turn, "Tool loaded." is ordinary user text."""
|
|
messages = [{"role": "user", "content": [{"type": "text", "text": "Tool loaded."}]}]
|
|
|
|
assert utils.extract_user_query(messages) == "Tool loaded."
|
|
|
|
|
|
def test_extract_user_query_latest_turn_only_agrees_on_normal_turns() -> None:
|
|
"""With text present on the newest user turn, the flag changes nothing."""
|
|
messages = [
|
|
{"role": "user", "content": "older question"},
|
|
{"role": "assistant", "content": "answer"},
|
|
{"role": "user", "content": [{"type": "text", "text": " newest question "}]},
|
|
]
|
|
|
|
assert utils.extract_user_query(messages) == "newest question"
|
|
assert utils.extract_user_query(messages, latest_user_turn_only=True) == "newest question"
|
|
|
|
hash_one = utils.compute_messages_hash(messages)
|
|
hash_two = utils.compute_messages_hash(list(messages))
|
|
assert hash_one == hash_two
|
|
assert len(hash_one) == 16
|
|
|
|
prefix_default = utils.compute_prefix_hash(
|
|
[
|
|
{"role": "system", "content": "a"},
|
|
{"role": "system", "content": "b"},
|
|
{"role": "user", "content": "c"},
|
|
{"role": "assistant", "content": "d"},
|
|
]
|
|
)
|
|
prefix_explicit = utils.compute_prefix_hash(
|
|
[
|
|
{"role": "system", "content": "a"},
|
|
{"role": "system", "content": "b"},
|
|
{"role": "user", "content": "c"},
|
|
{"role": "assistant", "content": "d"},
|
|
],
|
|
prefix_count=3,
|
|
)
|
|
assert prefix_default == prefix_explicit
|
|
assert utils.compute_prefix_hash([]) == utils.compute_short_hash("")
|
|
|
|
|
|
def test_timestamp_marker_and_json_helpers() -> None:
|
|
ts = utils.format_timestamp(datetime(2026, 4, 23, 6, 0, 0))
|
|
assert ts == "2026-04-23T06:00:00Z"
|
|
assert utils.parse_timestamp(ts) == datetime(2026, 4, 23, 6, 0, 0)
|
|
assert utils.parse_timestamp("2026-04-23T06:00:00") == datetime(2026, 4, 23, 6, 0, 0)
|
|
|
|
|
|
def test_format_timestamp_normalizes_aware_datetimes() -> None:
|
|
# Shipped integrations pass datetime.now(timezone.utc) (aware). Appending a
|
|
# bare "Z" to an aware isoformat used to emit "...+00:00Z" — invalid ISO
|
|
# 8601 that datetime.fromisoformat itself rejects.
|
|
aware_utc = datetime(2026, 4, 23, 6, 0, 0, tzinfo=timezone.utc)
|
|
ts = utils.format_timestamp(aware_utc)
|
|
assert ts == "2026-04-23T06:00:00Z"
|
|
# The stored string must round-trip through a strict parser.
|
|
assert datetime.fromisoformat(ts.rstrip("Z")) == datetime(2026, 4, 23, 6, 0, 0)
|
|
|
|
# A non-UTC offset must be converted to UTC, not just stamped "Z" onto the
|
|
# local wall-clock time (which would mislabel the instant).
|
|
east = datetime(2026, 4, 23, 6, 0, 0, tzinfo=timezone(timedelta(hours=5)))
|
|
assert utils.format_timestamp(east) == "2026-04-23T01:00:00Z"
|
|
|
|
marker = utils.create_marker("tool_digest", sha256="abc", count="2")
|
|
assert marker == '<headroom:tool_digest sha256="abc" count="2">'
|
|
assert utils.create_tool_digest_marker("abc") == '<headroom:tool_digest sha256="abc">'
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("name", "args", "expected"),
|
|
[
|
|
("fast_hash", ("hello", 8), "5d41402a"),
|
|
("fast_hash", (b"hello",), "5d41402abc4b2a76"),
|
|
(
|
|
"create_dropped_context_marker",
|
|
("budget",),
|
|
'<headroom:dropped_context reason="budget">',
|
|
),
|
|
(
|
|
"create_dropped_context_marker",
|
|
("budget", 4),
|
|
'<headroom:dropped_context reason="budget" count="4">',
|
|
),
|
|
(
|
|
"create_truncated_marker",
|
|
(100, 25),
|
|
'<headroom:truncated original="100" truncated_to="25">',
|
|
),
|
|
(
|
|
"extract_markers",
|
|
(
|
|
'x <headroom:tool_digest sha256="abc"> y '
|
|
'<headroom:dropped_context reason="budget" count="2">',
|
|
),
|
|
[
|
|
{"type": "tool_digest", "attributes": {"sha256": "abc"}},
|
|
{"type": "dropped_context", "attributes": {"reason": "budget", "count": "2"}},
|
|
],
|
|
),
|
|
("safe_json_loads", ('{"ok": true}',), ({"ok": True}, True)),
|
|
("safe_json_loads", ("{bad",), (None, False)),
|
|
("safe_json_dumps", ({"emoji": "café"},), '{"emoji":"café"}'),
|
|
],
|
|
)
|
|
def test_deprecated_helpers_warn_and_still_work(name: str, args: tuple, expected: object) -> None:
|
|
with pytest.warns(DeprecationWarning, match=f"{name} is deprecated"):
|
|
assert getattr(utils, name)(*args) == expected
|
|
|
|
|
|
def test_cost_formatting_and_deep_copy() -> None:
|
|
provider = FakeProvider("1.25")
|
|
assert utils.estimate_cost(100, 50, "gpt-4o", cached_tokens=10, provider=provider) == 1.25
|
|
assert provider.calls == [(100, 50, "gpt-4o", 10)]
|
|
assert utils.estimate_cost(1, 1, "gpt-4o", provider=None) is None
|
|
|
|
none_provider = FakeProvider(None)
|
|
assert utils.estimate_cost(1, 1, "gpt-4o", provider=none_provider) is None
|
|
|
|
assert utils.format_cost(0.0099) == "$0.0099"
|
|
assert utils.format_cost(1.234) == "$1.23"
|
|
|
|
messages = [{"role": "user", "content": {"nested": ["a"]}}]
|
|
copied = utils.deep_copy_messages(messages)
|
|
copied[0]["content"]["nested"].append("b")
|
|
assert messages == [{"role": "user", "content": {"nested": ["a"]}}]
|