mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(tabular): keep empty records and separator-like Markdown body rows (#4582)
## Description Tabular ingest silently dropped real data rows: a CSV/TSV record whose fields are all empty (`,` or `"",""`) and any Markdown body row whose values look like a separator (`| --- | --- |`). Records disappeared with no marker. Closes #4279 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) ## Changes Made - `parse_csv`: keep every record; skip only truly blank lines (empty row or a single whitespace-only cell). - `parse_markdown_table`: drop only the header separator directly under the header row; later separator-like rows are data. - Regression tests for CSV, TSV, quoted empty CSV record and Markdown `---` body row. ## Testing - [x] Unit tests added/updated ``` $ python -m pytest tests/test_transforms_tabular.py -q 2 failed, 98 passed, 5 skipped # the 2 failures (test_verbose_markdown_compresses, test_compact_unique_csv_passes_through) fail identically on main in this env; on main the new tests also fail (6 failed) $ ruff check / ruff format --check (touched files) All checks passed! / 2 files already formatted ``` ## Real Behavior Proof - Environment: Windows 11, Python 3, local checkout rebased on upstream/main - Exact command / steps: `parse_csv("a,b\n1,2\n,\n3,4")` and `parse_markdown_table("| a | b |\n|---|---|\n| 1 | 2 |\n| --- | --- |")` via the new tests - Observed result: CSV returns 3 rows including `['', '']`; Markdown returns `[['1','2'],['---','---']]` - Not tested: end-to-end proxy compression on real tabular payloads ## Runtime Rollout Safety - Rollout-managed feature(s): none - Minimum rollout channel: stable - Stable/default behavior changed: yes, tabular parsers now keep rows they previously dropped - Kill switch / disable path: none needed; parser correctness fix - Unsafe override required: no - Qualification impact: none - Rollback path: revert this commit ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
JD Davis
parent
0d4c450d77
commit
5cebb90db1
@@ -74,7 +74,7 @@ def parse_csv(content: str, delimiter: str = ",") -> tuple[list[str], list[list[
|
||||
"""
|
||||
reader = csv.reader(io.StringIO(content), delimiter=delimiter)
|
||||
try:
|
||||
parsed = [row for row in reader if any(cell.strip() for cell in row)]
|
||||
parsed = [row for row in reader if row and not (len(row) == 1 and not row[0].strip())]
|
||||
except csv.Error:
|
||||
return [], []
|
||||
if not parsed:
|
||||
@@ -97,7 +97,10 @@ def parse_markdown_table(content: str) -> tuple[list[str], list[list[str]]]:
|
||||
if len(lines) < 2:
|
||||
return [], []
|
||||
headers = split_row(lines[0])
|
||||
rows = [split_row(ln) for ln in lines[1:] if not is_separator(ln)]
|
||||
body_lines = lines[1:]
|
||||
if body_lines and is_separator(body_lines[0]):
|
||||
body_lines = body_lines[1:]
|
||||
rows = [split_row(ln) for ln in body_lines]
|
||||
return headers, rows
|
||||
|
||||
|
||||
|
||||
@@ -410,6 +410,30 @@ def test_parse_markdown_table_drops_separator() -> None:
|
||||
assert all("---" not in cell for row in rows for cell in row)
|
||||
|
||||
|
||||
def test_parse_csv_keeps_empty_records() -> None:
|
||||
headers, rows = parse_csv("a,b\n1,2\n,\n3,4")
|
||||
assert headers == ["a", "b"]
|
||||
assert rows == [["1", "2"], ["", ""], ["3", "4"]]
|
||||
|
||||
|
||||
def test_parse_csv_keeps_empty_records_tsv() -> None:
|
||||
headers, rows = parse_csv("a\tb\n1\t2\n\t\n3\t4", delimiter="\t")
|
||||
assert headers == ["a", "b"]
|
||||
assert rows == [["1", "2"], ["", ""], ["3", "4"]]
|
||||
|
||||
|
||||
def test_parse_csv_keeps_quoted_empty_record() -> None:
|
||||
headers, rows = parse_csv('a,b\n"",""\n1,2')
|
||||
assert headers == ["a", "b"]
|
||||
assert rows[0] == ["", ""]
|
||||
|
||||
|
||||
def test_parse_markdown_table_keeps_separator_like_body_row() -> None:
|
||||
headers, rows = parse_markdown_table("| a | b |\n|---|---|\n| 1 | 2 |\n| --- | --- |")
|
||||
assert headers == ["a", "b"]
|
||||
assert rows == [["1", "2"], ["---", "---"]]
|
||||
|
||||
|
||||
def test_parse_tabular_rejects_ragged_fixed_width(monkeypatch) -> None:
|
||||
# Rows with differing cell counts can't be zipped under the headers
|
||||
# without misattributing columns (#1652) — must pass through.
|
||||
@@ -562,7 +586,7 @@ def test_parse_tabular_rejects_single_column_fixed_width(monkeypatch) -> None:
|
||||
def test_parse_tabular_none_when_no_data_rows_survive() -> None:
|
||||
# Detected as a markdown table, but it is header + separator rows only:
|
||||
# nothing survives as a data row, so parse_tabular bails to None.
|
||||
assert parse_tabular("| a | b |\n| --- | --- |\n| --- | --- |") is None
|
||||
assert parse_tabular("| a | b |\n| --- | --- |") is None
|
||||
|
||||
|
||||
def test_compression_ratio_zero_for_empty_original() -> None:
|
||||
|
||||
Reference in New Issue
Block a user