fix: use all user text blocks for relevance scoring (#4628)

## Description

`extract_user_query` stops at the first text block in a user message.
Relevance scoring can therefore miss the rest of the request. A leading
Claude Code `<system-reminder>` can also be mistaken for the request.

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)

## Changes Made

- Join the user text blocks in order and remove complete reminder spans
without discarding text around them.
- Keep incomplete reminder tags and retain the existing latest-turn
fallback behavior.
- Cover multi-block prompts, reminders within both string and block
content, and a relevance-split example.

## Testing

- [x] Focused unit and integration tests pass
- [x] Linting passes (`ruff check .` and `ruff format --check .`)
- [x] New tests added for the bug

### Test Output

```text
Windows / Python 3.12: 108 passed, 2 warnings
Ubuntu / Python 3.12: 108 passed, 2 warnings
Ubuntu: ruff check ., ruff format --check ., mypy headroom --ignore-missing-imports passed
```

The added tests fail on the preceding revision. The Ubuntu run built the
Rust extension from source and tested the final PR code. [Run
details](https://github.com/bingdongni/headroom/actions/runs/41793014635).

## Real Behavior Proof

- Environment: Ubuntu, Python 3.12, `HEADROOM_OFFLINE=1`; Rust extension
built from source with `maturin build --profile ci`.
- Exact command / steps: Run the [deterministic input
script](https://github.com/bingdongni/headroom/blob/5750c34/scripts/contributor-proof.py)
against the base and patched source trees, with the same compiled
extension. The [verification
run](https://github.com/bingdongni/headroom/actions/runs/41793014635)
records these commands and output:

```sh
git worktree add --detach /tmp/headroom-base 5c8772b721
cp headroom/_core*.so /tmp/headroom-base/headroom/
(cd /tmp/headroom-base && PYTHONPATH=. python "$GITHUB_WORKSPACE/scripts/contributor-proof.py")
PYTHONPATH=. python scripts/contributor-proof.py
```

- Observed result: On 200 generated records, the second text block asks
about Kafka replication. Before the fix, 1 of 20 matching records
remained visible; after, 9 remained visible. Both used
`smart_sample(200->15)`. This is one synthetic input, not a general
quality benchmark.
- Not tested: Live provider traffic, embedding-backed relevance, the
full test suite, or the Rust pre-push suite. This example explicitly
sets `with_compaction=False`, not the default lossless-first path.

## Runtime Rollout Safety

- Rollout-managed feature(s): None added.
- Minimum rollout channel: Not applicable; this changes the existing
query extraction path.
- Stable/default behavior changed: Yes, relevance-dependent compression
and expansion see the complete user query without reminder spans.
- Kill switch / disable path: No query-extraction-specific switch.
- Unsafe override required: No.
- Qualification impact: No rollout-managed feature is changed; broader
model-quality qualification was not run.
- Rollback path: Revert these commits.

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

## Checklist

- [x] My code follows the project's style guidelines
- [x] I have added tests that prove the fix
- [x] I did not edit `CHANGELOG.md`

---------

Co-authored-by: cuteNBD <206630735+bingdongni@users.noreply.github.com>
This commit is contained in:
cuteNBD
2026-11-04 09:19:03 -06:00
committed by GitHub
co-authored by cuteNBD
parent 92c4407e0e
commit 12dca26aa1
3 changed files with 138 additions and 7 deletions
+34 -7
View File
@@ -15,6 +15,9 @@ from typing import Any
MARKER_PREFIX = "<headroom:"
MARKER_SUFFIX = ">"
_REMINDER_OPEN = re.compile(r"<system-reminder(?=[\s>])[^<>]*>", re.IGNORECASE)
_REMINDER_CLOSE = re.compile(r"</system-reminder\s*>", re.IGNORECASE)
def generate_request_id() -> str:
"""Generate a unique request ID."""
@@ -61,6 +64,24 @@ def fast_hash(data: str | bytes, length: int = 16) -> str:
return hashlib.md5(data).hexdigest()[:length] # nosec B324
def _strip_system_reminders(text: str) -> str:
"""Remove complete harness reminders while keeping surrounding user text."""
parts: list[str] = []
cursor = 0
while opening := _REMINDER_OPEN.search(text, cursor):
closing = _REMINDER_CLOSE.search(text, opening.end())
if closing is None:
break
parts.append(text[cursor : opening.start()])
end = closing.end()
if opening.start() and end < len(text):
if not text[opening.start() - 1].isspace() and not text[end].isspace():
parts.append(" ")
cursor = end
parts.append(text[cursor:])
return "".join(parts).strip()
def extract_user_query(
messages: list[dict[str, Any]], *, latest_user_turn_only: bool = False
) -> str:
@@ -82,14 +103,20 @@ def extract_user_query(
for msg in reversed(messages):
if msg.get("role") == "user":
content = msg.get("content", "")
if isinstance(content, str) and content.strip():
return content.strip()
if isinstance(content, str):
user_text = _strip_system_reminders(content)
if user_text:
return user_text
if isinstance(content, list):
for block in content:
if isinstance(block, dict) and block.get("type") == "text":
text = str(block.get("text", "")).strip()
if text:
return text
user_text = "\n".join(
text
for block in content
if isinstance(block, dict) and block.get("type") == "text"
for text in (_strip_system_reminders(str(block.get("text", ""))),)
if text
)
if user_text:
return user_text
if latest_user_turn_only:
return ""
return ""
+25
View File
@@ -125,6 +125,31 @@ def test_split_keeps_relevant_drops_irrelevant():
assert "".join(t for _, t in runs) == content
def test_split_uses_all_user_text_blocks():
from headroom.relevance.bm25 import BM25Scorer
from headroom.utils import extract_user_query
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Review production incidents"},
{"type": "text", "text": "Focus on Kafka replication"},
],
}
]
content = (
"Production incident: payment gateway high latency\n\n"
"Kafka replication lag on broker 17 exceeded 30 minutes\n\n"
"Disk full on backup server\n\n"
"Production incident: expired TLS certificate\n"
)
runs = plan_relevance_split(content, extract_user_query(messages), BM25Scorer(), threshold=0.25)
kept = "".join(text for keep, text in runs if keep)
assert "Kafka replication lag" in kept
assert "Disk full on backup server" not in kept
def test_empty_query_yields_no_split():
assert plan_relevance_split("x\ny\n", "", KeywordScorer(), threshold=0.5) == [(True, "x\ny\n")]
+79
View File
@@ -48,6 +48,85 @@ def test_extract_user_query_and_message_hashes() -> None:
assert utils.extract_user_query([{"role": "assistant", "content": "skip"}]) == ""
@pytest.mark.parametrize("latest_only", [False, True])
def test_extract_user_query_combines_text_blocks(latest_only: bool) -> None:
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": " Review incidents "},
{"type": "image", "source": "example"},
{"type": "text", "text": " "},
{"type": "text", "text": " Focus on Kafka replication "},
],
}
]
assert utils.extract_user_query(messages, latest_user_turn_only=latest_only) == (
"Review incidents\nFocus on Kafka replication"
)
def test_extract_user_query_skips_system_reminder_blocks() -> None:
messages = [
{"role": "user", "content": "older question"},
{
"role": "user",
"content": [
{
"type": "text",
"text": "<system-reminder>injected instructions</system-reminder>",
},
{"type": "text", "text": "real question"},
{"type": "text", "text": "<system-reminder>more instructions</system-reminder>"},
],
},
]
assert utils.extract_user_query(messages) == "real question"
assert utils.extract_user_query(messages, latest_user_turn_only=True) == "real question"
messages[-1]["content"] = [
{"type": "text", "text": "<system-reminder>injected</system-reminder>"}
]
assert utils.extract_user_query(messages) == "older question"
assert utils.extract_user_query(messages, latest_user_turn_only=True) == ""
@pytest.mark.parametrize("as_blocks", [False, True])
@pytest.mark.parametrize(
("content", "expected"),
[
("<system-reminder>context</system-reminder>\nFind Kafka lag", "Find Kafka lag"),
("Find <system-reminder>context</system-reminder> Kafka lag", "Find Kafka lag"),
(
"<system-reminder id='1'>a</system-reminder>Find <system-reminder>b</system-reminder>Kafka",
"Find Kafka",
),
("<System-Reminder>context</System-Reminder>Find Kafka", "Find Kafka"),
("İ<system-reminder>x</system-reminder>Kafka", "İ Kafka"),
("Find <system-reminder>unfinished", "Find <system-reminder>unfinished"),
(
"Find <system-reminder-extra>example</system-reminder> Kafka",
"Find <system-reminder-extra>example</system-reminder> Kafka",
),
],
)
def test_extract_user_query_keeps_text_around_reminders(
content: str, expected: str, as_blocks: bool
) -> None:
body = [{"type": "text", "text": content}] if as_blocks else content
messages = [{"role": "user", "content": body}]
assert utils.extract_user_query(messages, latest_user_turn_only=True) == expected
@pytest.mark.parametrize("as_blocks", [False, True])
def test_reminder_only_turn_keeps_user_query_fallback(as_blocks: bool) -> None:
reminder = "<system-reminder>context</system-reminder>"
content = [{"type": "text", "text": reminder}] if as_blocks else reminder
messages = [{"role": "user", "content": "older question"}, {"role": "user", "content": content}]
assert utils.extract_user_query(messages) == "older question"
assert utils.extract_user_query(messages, latest_user_turn_only=True) == ""
def test_extract_user_query_latest_user_turn_only() -> None:
"""The flag stops at the newest user turn instead of resurrecting an older one."""
# A tool_result continuation: the newest user turn carries no text at all.