mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(compaction): reject negative dictionary indices and non-array legends in expand_compacted (#4470)
## Description
`expand_compacted()` promises `None` when a densified block cannot be
losslessly reversed, but value-factored (`__dict:K`) cells were decoded
with raw Python indexing:
- a wire index of `-1` silently decoded to the **last** legend value
(data corruption, no error);
- a non-array legend (`@tag=42`, `@tag="ab"`, `@tag={"a":1}`,
`@tag=null`) raised `TypeError`/`KeyError` or was indexed character by
character.
Fixes #4280
## Type of Change
- [x] Bug fix (non-breaking change that fixes an issue)
## Changes Made
- `headroom/transforms/compaction_codec.py`: reject a legend whose JSON
payload is not a list; reject negative dictionary indices before
indexing. Both paths return `None`, as the docstring already promises.
- `tests/test_transforms/test_compaction_codec.py`: regression tests for
the negative index, four non-array legend shapes, and a guard that valid
first/last indices, a quoted literal `"-1"`, `null`, and missing cells
in a dictionary column still decode.
## Testing
- [x] Unit tests pass locally
- [x] New tests added for the fix
```
$ python -m pytest tests/test_transforms/test_compaction_codec.py tests/test_densify_agent_mode.py -q
37 passed in 1.58s
# with the codec fix reverted, new tests:
5 failed, 1 passed, 17 deselected
$ ruff check / ruff format --check (touched files)
All checks passed! / 2 files already formatted
```
## Real Behavior Proof
- Environment: Windows 11, Python 3.12, local build of headroom with
native `headroom._core`, branch rebased on upstream/main
- Exact command / steps: `expand_compacted('[1]{tag:string}
__dict:1\n@tag=["first","last"]\n-1\n')` and
`expand_compacted('[1]{tag:string} __dict:1\n@tag=42\n0\n')`, plus the
codec and densify-agent-mode test files above
- Observed result: before: `[{"tag": "last"}]` and `TypeError`; after:
`None` for both; full codec + agent densification suites pass (37 tests)
- Not tested: declared row-count mismatch validation (separate issue
#4271)
## Runtime Rollout Safety
- Rollout-managed feature(s): none
- Minimum rollout channel: n/a, decoder correctness fix
- Stable/default behavior changed: only for malformed blocks, which now
return None instead of wrong data or an exception
- Kill switch / disable path: n/a
- Unsafe override required: no
- Qualification impact: none
- Rollback path: revert this commit
## Review Readiness
- [x] I have performed a self-review
- [x] This PR is ready for human review
🤖 Generated with [Claude Code](https://claude.com/claude-code)
<!-- devin-review-badge-begin -->
---
<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4470"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
JD Davis
parent
aebb729ef3
commit
3c1fcbdfde
@@ -383,9 +383,12 @@ def expand_compacted(text: str) -> list[dict[str, Any]] | None:
|
||||
if not eq or not name.startswith("@"):
|
||||
return None
|
||||
try:
|
||||
legends[name[1:]] = json.loads(payload)
|
||||
result = json.loads(payload)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
if not isinstance(result, list):
|
||||
return None
|
||||
legends[name[1:]] = result
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
for cells in _split_data_rows(rest, declared):
|
||||
@@ -404,7 +407,10 @@ def expand_compacted(text: str) -> list[dict[str, Any]] | None:
|
||||
_assign(record, col.path, None)
|
||||
continue
|
||||
try:
|
||||
_assign(record, col.path, legends[col.name][int(value)])
|
||||
idx_val = int(value)
|
||||
if idx_val < 0:
|
||||
return None
|
||||
_assign(record, col.path, legends[col.name][idx_val])
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
continue
|
||||
|
||||
@@ -15,6 +15,8 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from headroom.transforms.compaction_codec import (
|
||||
contains_removal_marker,
|
||||
expand_compacted,
|
||||
@@ -201,3 +203,24 @@ def test_single_column_null_survives_value_factoring() -> None:
|
||||
factored = factor_values(block)
|
||||
assert expand_compacted(factored) == expand_compacted(block)
|
||||
assert expand_compacted(factored) == [{"k": "x"}, {"k": "y"}, {"k": None}]
|
||||
|
||||
|
||||
def test_negative_dictionary_index_is_rejected() -> None:
|
||||
# Python's -1 would silently pick the last legend value.
|
||||
assert expand_compacted('[1]{tag:string} __dict:1\n@tag=["first","last"]\n-1\n') is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legend", ["42", '"ab"', '{"a":1}', "null"])
|
||||
def test_non_array_legend_is_rejected(legend: str) -> None:
|
||||
assert expand_compacted(f"[1]{{tag:string}} __dict:1\n@tag={legend}\n0\n") is None
|
||||
|
||||
|
||||
def test_dictionary_cells_still_decode_edge_values() -> None:
|
||||
block = '[5]{tag:string?} __dict:1\n@tag=["first","last"]\n0\n1\n"-1"\nnull\n\n'
|
||||
assert expand_compacted(block) == [
|
||||
{"tag": "first"},
|
||||
{"tag": "last"},
|
||||
{"tag": "-1"},
|
||||
{"tag": None},
|
||||
{},
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user