mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(parser): stop counting HTML comments twice in waste signals (#3942)
## Root cause
In `detect_waste_signals`, the HTML noise signal joins matches from two
patterns:
```python
html_matches = HTML_TAG_PATTERN.findall(text) + HTML_COMMENT_PATTERN.findall(text)
```
`HTML_TAG_PATTERN` is `<[^>]+>`, which matches a full `<!-- ... -->`
comment (the span contains no bare `>`), and `HTML_COMMENT_PATTERN`
matches the same span again. Every comment is therefore counted twice,
and tags inside a comment are counted a third time as separate noise.
Measured on a text containing a single 608-character comment:
`html_noise_tokens` reports 304 instead of 152. The inflated signal
feeds `WasteSignals.total()`, which skews compression thresholds and
overreports noise in telemetry.
## Fix
Strip comments from the text before tag matching:
```python
comment_matches = HTML_COMMENT_PATTERN.findall(text)
html_matches = HTML_TAG_PATTERN.findall(HTML_COMMENT_PATTERN.sub("", text)) + comment_matches
```
Each comment is now counted exactly once (via the comment pattern), and
tags inside comments are no longer counted as separate noise. Plain tag
counting is unchanged: tags outside comments still match the same way on
the comment-free remainder.
## Test
Added four regression tests to `TestDetectWasteSignals`:
- a 608-character comment reports exactly its own token count (failed
before: reported double)
- tags inside a comment are not counted separately (failed before)
- mixed tags and comments each contribute exactly once (failed before)
- plain tags keep their existing count (guard, passes before and after)
Full `tests/test_parser.py` suite: 98 passed. `ruff check` and `ruff
format --check` clean on both touched files.
Same waste-signal accounting accuracy area as #1102 (whitespace signal
always zero), which this follows.
<!-- devin-review-badge-begin -->
---
<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/3942"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->
Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
JD Davis
parent
d503c575ea
commit
d0e9c4ca48
+4
-2
@@ -163,8 +163,10 @@ def detect_waste_signals(text: str, tokenizer: Tokenizer) -> WasteSignals:
|
||||
if not text:
|
||||
return signals
|
||||
|
||||
# HTML tags and comments
|
||||
html_matches = HTML_TAG_PATTERN.findall(text) + HTML_COMMENT_PATTERN.findall(text)
|
||||
# HTML tags and comments. Comments are removed before tag matching so a
|
||||
# comment is counted once (and tags inside comments are not counted).
|
||||
comment_matches = HTML_COMMENT_PATTERN.findall(text)
|
||||
html_matches = HTML_TAG_PATTERN.findall(HTML_COMMENT_PATTERN.sub("", text)) + comment_matches
|
||||
if html_matches:
|
||||
html_text = "".join(html_matches)
|
||||
signals.html_noise_tokens = tokenizer.count_text(html_text)
|
||||
|
||||
@@ -268,6 +268,37 @@ class TestDetectWasteSignals:
|
||||
assert signals.base64_tokens == 0
|
||||
assert signals.json_bloat_tokens == 0
|
||||
|
||||
def test_html_comment_counted_once(self, mock_tokenizer):
|
||||
"""A comment matches both HTML_TAG_PATTERN and HTML_COMMENT_PATTERN;
|
||||
it must contribute its token count once, not twice."""
|
||||
comment = "<!--" + "x" * 600 + "-->"
|
||||
text = "before " + comment + " after"
|
||||
signals = detect_waste_signals(text, mock_tokenizer)
|
||||
assert signals.html_noise_tokens == (len(comment) // 4 + 1)
|
||||
|
||||
def test_tags_inside_comments_not_counted(self, mock_tokenizer):
|
||||
"""Tags inside a comment are part of the comment, not extra noise."""
|
||||
comment = '<!-- <img src="diagram.png" alt="chart"> -->'
|
||||
text = "text " + comment + " end"
|
||||
signals = detect_waste_signals(text, mock_tokenizer)
|
||||
assert signals.html_noise_tokens == (len(comment) // 4 + 1)
|
||||
|
||||
def test_mixed_tags_and_comments_counted_once_each(self, mock_tokenizer):
|
||||
"""Plain tags and comments each contribute exactly once."""
|
||||
tag_open = '<div class="a">'
|
||||
tag_close = "</div>"
|
||||
comment = "<!-- note -->"
|
||||
text = tag_open + "content" + tag_close + comment + tag_open + "more" + tag_close
|
||||
signals = detect_waste_signals(text, mock_tokenizer)
|
||||
expected = 2 * len(tag_open) + 2 * len(tag_close) + len(comment)
|
||||
assert signals.html_noise_tokens == (expected // 4 + 1)
|
||||
|
||||
def test_plain_tags_unchanged(self, mock_tokenizer):
|
||||
"""Regular tags keep their existing count when no comments are present."""
|
||||
text = "<div>Hello</div>"
|
||||
signals = detect_waste_signals(text, mock_tokenizer)
|
||||
assert signals.html_noise_tokens == (len("<div>") + len("</div>")) // 4 + 1
|
||||
|
||||
|
||||
# --- TestIsRagContent ---
|
||||
|
||||
|
||||
Reference in New Issue
Block a user