mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(dedup): compare fold sizes in UTF-8 bytes, not characters
The pointer's arrow is 3 bytes in UTF-8, so a 44-character span folded into a 43-character pointer that is 45 bytes: shorter in characters, longer on the wire. Compare encoded lengths so a fold is always a net byte win, as the module comment promises. Adds the boundary test. Refs #4478 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
parent
7dc7cfc04e
commit
6bfa823f74
@@ -269,8 +269,11 @@ def dedup_blocks(
|
||||
span = lines[i : i + m[0]]
|
||||
span_text = "\n".join(span)
|
||||
ptr = _pointer(span, blocks[m[1]].turn, m[3])
|
||||
# Never fold when the pointer is not shorter than the span.
|
||||
if len(span_text) >= min_chars and len(ptr) < len(span_text):
|
||||
# Never fold when the pointer is not shorter than the span, in
|
||||
# UTF-8 bytes ("↑" is 3 bytes, so character counts mislead).
|
||||
if len(span_text) >= min_chars and len(ptr.encode("utf-8")) < len(
|
||||
span_text.encode("utf-8")
|
||||
):
|
||||
out.append(ptr)
|
||||
# Folded span is NOT verbatim in this block's output:
|
||||
# mark None so it can't seed a later contiguous match,
|
||||
|
||||
@@ -494,6 +494,18 @@ def test_fold_never_longer_than_span():
|
||||
out, stats = dedup_blocks([_blk(t, 1), _blk(t, 2)])
|
||||
assert stats["spans_folded"] == 0
|
||||
assert out[1].text == t
|
||||
# Byte boundary: the pointer's "↑" is 3 UTF-8 bytes. A 44-char span has a
|
||||
# 43-char pointer (shorter in characters) that is 45 bytes, so folding
|
||||
# would grow the output; it must stay. A 46-byte span folds.
|
||||
t44 = "a" * 25 + "\n" + "b" * 9 + "\n" + "c" * 8
|
||||
assert len(t44) == 44
|
||||
out, stats = dedup_blocks([_blk(t44, 1), _blk(t44, 2)])
|
||||
assert stats["spans_folded"] == 0
|
||||
assert out[1].text == t44
|
||||
t46 = "a" * 27 + "\n" + "b" * 9 + "\n" + "c" * 8
|
||||
out, stats = dedup_blocks([_blk(t46, 1), _blk(t46, 2)])
|
||||
assert stats["spans_folded"] == 1
|
||||
assert len(out[1].text.encode("utf-8")) < len(t46.encode("utf-8"))
|
||||
# A long span still folds.
|
||||
long_t = _code("", 5)
|
||||
out, stats = dedup_blocks([_blk(long_t, 1), _blk(long_t, 2)])
|
||||
|
||||
Reference in New Issue
Block a user