fix(dedup): compare fold sizes in UTF-8 bytes, not characters

The pointer's arrow is 3 bytes in UTF-8, so a 44-character span folded
into a 43-character pointer that is 45 bytes: shorter in characters,
longer on the wire. Compare encoded lengths so a fold is always a net
byte win, as the module comment promises. Adds the boundary test.

Refs #4478

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
HetCreep
2026-10-26 16:16:28 +07:00
co-authored by Claude Sonnet 5.5
parent 7dc7cfc04e
commit 6bfa823f74
2 changed files with 17 additions and 2 deletions
+5 -2
View File
@@ -269,8 +269,11 @@ def dedup_blocks(
span = lines[i : i + m[0]]
span_text = "\n".join(span)
ptr = _pointer(span, blocks[m[1]].turn, m[3])
# Never fold when the pointer is not shorter than the span.
if len(span_text) >= min_chars and len(ptr) < len(span_text):
# Never fold when the pointer is not shorter than the span, in
# UTF-8 bytes ("↑" is 3 bytes, so character counts mislead).
if len(span_text) >= min_chars and len(ptr.encode("utf-8")) < len(
span_text.encode("utf-8")
):
out.append(ptr)
# Folded span is NOT verbatim in this block's output:
# mark None so it can't seed a later contiguous match,
+12
View File
@@ -494,6 +494,18 @@ def test_fold_never_longer_than_span():
out, stats = dedup_blocks([_blk(t, 1), _blk(t, 2)])
assert stats["spans_folded"] == 0
assert out[1].text == t
# Byte boundary: the pointer's "↑" is 3 UTF-8 bytes. A 44-char span has a
# 43-char pointer (shorter in characters) that is 45 bytes, so folding
# would grow the output; it must stay. A 46-byte span folds.
t44 = "a" * 25 + "\n" + "b" * 9 + "\n" + "c" * 8
assert len(t44) == 44
out, stats = dedup_blocks([_blk(t44, 1), _blk(t44, 2)])
assert stats["spans_folded"] == 0
assert out[1].text == t44
t46 = "a" * 27 + "\n" + "b" * 9 + "\n" + "c" * 8
out, stats = dedup_blocks([_blk(t46, 1), _blk(t46, 2)])
assert stats["spans_folded"] == 1
assert len(out[1].text.encode("utf-8")) < len(t46.encode("utf-8"))
# A long span still folds.
long_t = _code("", 5)
out, stats = dedup_blocks([_blk(long_t, 1), _blk(long_t, 2)])