mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix: use all user text blocks for relevance scoring (#4628)
## Description
`extract_user_query` stops at the first text block in a user message.
Relevance scoring can therefore miss the rest of the request. A leading
Claude Code `<system-reminder>` can also be mistaken for the request.
## Type of Change
- [x] Bug fix (non-breaking change that fixes an issue)
## Changes Made
- Join the user text blocks in order and remove complete reminder spans
without discarding text around them.
- Keep incomplete reminder tags and retain the existing latest-turn
fallback behavior.
- Cover multi-block prompts, reminders within both string and block
content, and a relevance-split example.
## Testing
- [x] Focused unit and integration tests pass
- [x] Linting passes (`ruff check .` and `ruff format --check .`)
- [x] New tests added for the bug
### Test Output
```text
Windows / Python 3.12: 108 passed, 2 warnings
Ubuntu / Python 3.12: 108 passed, 2 warnings
Ubuntu: ruff check ., ruff format --check ., mypy headroom --ignore-missing-imports passed
```
The added tests fail on the preceding revision. The Ubuntu run built the
Rust extension from source and tested the final PR code. [Run
details](https://github.com/bingdongni/headroom/actions/runs/41793014635).
## Real Behavior Proof
- Environment: Ubuntu, Python 3.12, `HEADROOM_OFFLINE=1`; Rust extension
built from source with `maturin build --profile ci`.
- Exact command / steps: Run the [deterministic input
script](https://github.com/bingdongni/headroom/blob/5750c34/scripts/contributor-proof.py)
against the base and patched source trees, with the same compiled
extension. The [verification
run](https://github.com/bingdongni/headroom/actions/runs/41793014635)
records these commands and output:
```sh
git worktree add --detach /tmp/headroom-base 5c8772b721
cp headroom/_core*.so /tmp/headroom-base/headroom/
(cd /tmp/headroom-base && PYTHONPATH=. python "$GITHUB_WORKSPACE/scripts/contributor-proof.py")
PYTHONPATH=. python scripts/contributor-proof.py
```
- Observed result: On 200 generated records, the second text block asks
about Kafka replication. Before the fix, 1 of 20 matching records
remained visible; after, 9 remained visible. Both used
`smart_sample(200->15)`. This is one synthetic input, not a general
quality benchmark.
- Not tested: Live provider traffic, embedding-backed relevance, the
full test suite, or the Rust pre-push suite. This example explicitly
sets `with_compaction=False`, not the default lossless-first path.
## Runtime Rollout Safety
- Rollout-managed feature(s): None added.
- Minimum rollout channel: Not applicable; this changes the existing
query extraction path.
- Stable/default behavior changed: Yes, relevance-dependent compression
and expansion see the complete user query without reminder spans.
- Kill switch / disable path: No query-extraction-specific switch.
- Unsafe override required: No.
- Qualification impact: No rollout-managed feature is changed; broader
model-quality qualification was not run.
- Rollback path: Revert these commits.
## Review Readiness
- [x] I have performed a self-review
- [x] This PR is ready for human review
## Checklist
- [x] My code follows the project's style guidelines
- [x] I have added tests that prove the fix
- [x] I did not edit `CHANGELOG.md`
---------
Co-authored-by: cuteNBD <206630735+bingdongni@users.noreply.github.com>
This commit is contained in:
+34
-7
@@ -15,6 +15,9 @@ from typing import Any
|
||||
MARKER_PREFIX = "<headroom:"
|
||||
MARKER_SUFFIX = ">"
|
||||
|
||||
_REMINDER_OPEN = re.compile(r"<system-reminder(?=[\s>])[^<>]*>", re.IGNORECASE)
|
||||
_REMINDER_CLOSE = re.compile(r"</system-reminder\s*>", re.IGNORECASE)
|
||||
|
||||
|
||||
def generate_request_id() -> str:
|
||||
"""Generate a unique request ID."""
|
||||
@@ -61,6 +64,24 @@ def fast_hash(data: str | bytes, length: int = 16) -> str:
|
||||
return hashlib.md5(data).hexdigest()[:length] # nosec B324
|
||||
|
||||
|
||||
def _strip_system_reminders(text: str) -> str:
|
||||
"""Remove complete harness reminders while keeping surrounding user text."""
|
||||
parts: list[str] = []
|
||||
cursor = 0
|
||||
while opening := _REMINDER_OPEN.search(text, cursor):
|
||||
closing = _REMINDER_CLOSE.search(text, opening.end())
|
||||
if closing is None:
|
||||
break
|
||||
parts.append(text[cursor : opening.start()])
|
||||
end = closing.end()
|
||||
if opening.start() and end < len(text):
|
||||
if not text[opening.start() - 1].isspace() and not text[end].isspace():
|
||||
parts.append(" ")
|
||||
cursor = end
|
||||
parts.append(text[cursor:])
|
||||
return "".join(parts).strip()
|
||||
|
||||
|
||||
def extract_user_query(
|
||||
messages: list[dict[str, Any]], *, latest_user_turn_only: bool = False
|
||||
) -> str:
|
||||
@@ -82,14 +103,20 @@ def extract_user_query(
|
||||
for msg in reversed(messages):
|
||||
if msg.get("role") == "user":
|
||||
content = msg.get("content", "")
|
||||
if isinstance(content, str) and content.strip():
|
||||
return content.strip()
|
||||
if isinstance(content, str):
|
||||
user_text = _strip_system_reminders(content)
|
||||
if user_text:
|
||||
return user_text
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "text":
|
||||
text = str(block.get("text", "")).strip()
|
||||
if text:
|
||||
return text
|
||||
user_text = "\n".join(
|
||||
text
|
||||
for block in content
|
||||
if isinstance(block, dict) and block.get("type") == "text"
|
||||
for text in (_strip_system_reminders(str(block.get("text", ""))),)
|
||||
if text
|
||||
)
|
||||
if user_text:
|
||||
return user_text
|
||||
if latest_user_turn_only:
|
||||
return ""
|
||||
return ""
|
||||
|
||||
@@ -125,6 +125,31 @@ def test_split_keeps_relevant_drops_irrelevant():
|
||||
assert "".join(t for _, t in runs) == content
|
||||
|
||||
|
||||
def test_split_uses_all_user_text_blocks():
|
||||
from headroom.relevance.bm25 import BM25Scorer
|
||||
from headroom.utils import extract_user_query
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Review production incidents"},
|
||||
{"type": "text", "text": "Focus on Kafka replication"},
|
||||
],
|
||||
}
|
||||
]
|
||||
content = (
|
||||
"Production incident: payment gateway high latency\n\n"
|
||||
"Kafka replication lag on broker 17 exceeded 30 minutes\n\n"
|
||||
"Disk full on backup server\n\n"
|
||||
"Production incident: expired TLS certificate\n"
|
||||
)
|
||||
runs = plan_relevance_split(content, extract_user_query(messages), BM25Scorer(), threshold=0.25)
|
||||
kept = "".join(text for keep, text in runs if keep)
|
||||
assert "Kafka replication lag" in kept
|
||||
assert "Disk full on backup server" not in kept
|
||||
|
||||
|
||||
def test_empty_query_yields_no_split():
|
||||
assert plan_relevance_split("x\ny\n", "", KeywordScorer(), threshold=0.5) == [(True, "x\ny\n")]
|
||||
|
||||
|
||||
@@ -48,6 +48,85 @@ def test_extract_user_query_and_message_hashes() -> None:
|
||||
assert utils.extract_user_query([{"role": "assistant", "content": "skip"}]) == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("latest_only", [False, True])
|
||||
def test_extract_user_query_combines_text_blocks(latest_only: bool) -> None:
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": " Review incidents "},
|
||||
{"type": "image", "source": "example"},
|
||||
{"type": "text", "text": " "},
|
||||
{"type": "text", "text": " Focus on Kafka replication "},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert utils.extract_user_query(messages, latest_user_turn_only=latest_only) == (
|
||||
"Review incidents\nFocus on Kafka replication"
|
||||
)
|
||||
|
||||
|
||||
def test_extract_user_query_skips_system_reminder_blocks() -> None:
|
||||
messages = [
|
||||
{"role": "user", "content": "older question"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "<system-reminder>injected instructions</system-reminder>",
|
||||
},
|
||||
{"type": "text", "text": "real question"},
|
||||
{"type": "text", "text": "<system-reminder>more instructions</system-reminder>"},
|
||||
],
|
||||
},
|
||||
]
|
||||
assert utils.extract_user_query(messages) == "real question"
|
||||
assert utils.extract_user_query(messages, latest_user_turn_only=True) == "real question"
|
||||
|
||||
messages[-1]["content"] = [
|
||||
{"type": "text", "text": "<system-reminder>injected</system-reminder>"}
|
||||
]
|
||||
assert utils.extract_user_query(messages) == "older question"
|
||||
assert utils.extract_user_query(messages, latest_user_turn_only=True) == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("as_blocks", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("content", "expected"),
|
||||
[
|
||||
("<system-reminder>context</system-reminder>\nFind Kafka lag", "Find Kafka lag"),
|
||||
("Find <system-reminder>context</system-reminder> Kafka lag", "Find Kafka lag"),
|
||||
(
|
||||
"<system-reminder id='1'>a</system-reminder>Find <system-reminder>b</system-reminder>Kafka",
|
||||
"Find Kafka",
|
||||
),
|
||||
("<System-Reminder>context</System-Reminder>Find Kafka", "Find Kafka"),
|
||||
("İ<system-reminder>x</system-reminder>Kafka", "İ Kafka"),
|
||||
("Find <system-reminder>unfinished", "Find <system-reminder>unfinished"),
|
||||
(
|
||||
"Find <system-reminder-extra>example</system-reminder> Kafka",
|
||||
"Find <system-reminder-extra>example</system-reminder> Kafka",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_user_query_keeps_text_around_reminders(
|
||||
content: str, expected: str, as_blocks: bool
|
||||
) -> None:
|
||||
body = [{"type": "text", "text": content}] if as_blocks else content
|
||||
messages = [{"role": "user", "content": body}]
|
||||
assert utils.extract_user_query(messages, latest_user_turn_only=True) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("as_blocks", [False, True])
|
||||
def test_reminder_only_turn_keeps_user_query_fallback(as_blocks: bool) -> None:
|
||||
reminder = "<system-reminder>context</system-reminder>"
|
||||
content = [{"type": "text", "text": reminder}] if as_blocks else reminder
|
||||
messages = [{"role": "user", "content": "older question"}, {"role": "user", "content": content}]
|
||||
assert utils.extract_user_query(messages) == "older question"
|
||||
assert utils.extract_user_query(messages, latest_user_turn_only=True) == ""
|
||||
|
||||
|
||||
def test_extract_user_query_latest_user_turn_only() -> None:
|
||||
"""The flag stops at the newest user turn instead of resurrecting an older one."""
|
||||
# A tool_result continuation: the newest user turn carries no text at all.
|
||||
|
||||
Reference in New Issue
Block a user