mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(tabular): keep interior blank CSV rows instead of dropping them
parse_csv dropped every all-blank row, so separator rows inside a table vanished and the row count changed before the lossless csv-schema step. It now trims all-blank rows only at the start and end (as spreadsheet_ingest._rows_to_csv trims the end) and keeps interior ones as empty cells; truly empty lines are still skipped. Refs #4546 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a2d4fe7238
commit
5beddc1e1d
@@ -74,9 +74,15 @@ def parse_csv(content: str, delimiter: str = ",") -> tuple[list[str], list[list[
|
||||
"""
|
||||
reader = csv.reader(io.StringIO(content), delimiter=delimiter)
|
||||
try:
|
||||
parsed = [row for row in reader if any(cell.strip() for cell in row)]
|
||||
parsed = [r for r in reader if len(r) > 1 or any(c.strip() for c in r)] # skip empty lines
|
||||
except csv.Error:
|
||||
return [], []
|
||||
# Trim all-blank rows at both ends, but keep interior ones (as empty
|
||||
# cells) so the row count survives, like spreadsheet_ingest._rows_to_csv.
|
||||
while parsed and not any(cell.strip() for cell in parsed[-1]):
|
||||
parsed.pop()
|
||||
while parsed and not any(cell.strip() for cell in parsed[0]):
|
||||
parsed.pop(0)
|
||||
if not parsed:
|
||||
return [], []
|
||||
headers = [h.strip() for h in parsed[0]]
|
||||
|
||||
@@ -403,6 +403,15 @@ def test_parse_csv_and_records() -> None:
|
||||
assert records[1] == {"name": "Bob", "age": "25", "city": "LA"}
|
||||
|
||||
|
||||
def test_parse_csv_keeps_interior_blank_rows_and_trims_the_ends() -> None:
|
||||
headers, rows = parse_csv("a,b\n1,2\n,\n3,4\n,\n5,6\n,\n")
|
||||
assert headers == ["a", "b"]
|
||||
assert rows == [["1", "2"], ["", ""], ["3", "4"], ["", ""], ["5", "6"]]
|
||||
# Empty and whitespace-only lines have no real cells and are still skipped.
|
||||
assert parse_csv("a,b\n1,2\n\n3,4\n")[1] == [["1", "2"], ["3", "4"]]
|
||||
assert parse_csv("a,b\n1,2\n \n3,4\n")[1] == [["1", "2"], ["3", "4"]]
|
||||
|
||||
|
||||
def test_parse_markdown_table_drops_separator() -> None:
|
||||
headers, rows = parse_markdown_table(MARKDOWN)
|
||||
assert headers == ["name", "age"]
|
||||
|
||||
Reference in New Issue
Block a user