mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(spreadsheet): stop .xls integer rendering fabricating digits above 2^53 (#3704)
Closes #3695. Follow-up to #3616, which fixed a real bug but introduced a narrower one at the top of the numeric range. ## Problem `_xls_cell` converted every whole-valued number to `int`. xlrd hands back an IEEE-754 double, and above 2^53 consecutive integers are no longer representable — so `int()` renders the double's *exact* value, which is not the number the user typed: ``` user typed: 123456789012345678 stored as: 1.2345678901234568e+17 int() shows: 123456789012345680 <- last two digits fabricated ``` This text goes to an agent. An ID column rendered as `123456789012345680` reads as an exact identifier; it isn't. That is the main concern — the `.xlsx`/`.xls` divergence #3616 set out to remove is the secondary one (both loaders showed the float repr before, and only diverge above 2^53 now). ## Fix One bounded condition: ```python if kind == xlrd.XL_CELL_NUMBER and float(value).is_integer() and abs(value) < 2**53: ``` Below 2^53 the double is exact, so `int()` is lossless and the `12.0 -> 12` win that motivated #3616 is preserved. At or above, fall through to the float repr — which signals approximation honestly *and* restores `_load_xlsx` parity. `< 2**53` rather than `<=`: 2^53 is representable but 2^53+1 is not, so the strict bound is the conventional "integers are exact below here" line. ## Verification New parametrized tests call `_xls_cell` directly with a stub cell (no `.xls` fixture needed). Verified fail-before / pass-after by reverting only the source file and keeping the tests: ``` 4 failed, 3 passed <- e.g. assert isinstance(123456789012345680, float) ``` After the fix: **52 passed** in `tests/test_transforms_tabular.py`. ## Also in this PR - Docstring note that sub-second precision is dropped (`xldate_as_tuple` returns whole seconds, so `12:34:56.5` renders `12:34:56` where openpyxl keeps microseconds). - Removed the stale `# pragma: no cover` on the `.xls` dispatch — it referenced a comment #3616 deleted, on a line the tests now exercise (confirmed via coverage: the line is no longer in the missing set). ## Note for reviewers These tests **silently skip** without `xlrd` installed. CI is fine (`[dev]` pins `xlrd>=2.0.1`), but a local venv without it shows `s` rather than failures. Still open in #3695 and deliberately out of scope here: replacing the `xlwt` dev dependency with a committed binary fixture, and covering `XL_CELL_ERROR`. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: JerrettDavis <2610199+JerrettDavis@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
copilot-swe-agent[bot]
JerrettDavis
parent
7f2766ca4b
commit
e8a510982a
@@ -77,6 +77,10 @@ def _xls_cell(cell: object, datemode: int) -> object:
|
||||
so a whole number arrives as ``12.0``. Left alone, the two loaders disagree
|
||||
about the same workbook -- ``45292.0`` here against ``2024-01-01 00:00:00``
|
||||
there -- and the date is not recoverable from the text.
|
||||
|
||||
Sub-second precision is dropped: ``xldate_as_tuple`` returns whole seconds, so
|
||||
``datetime(2024, 1, 1, 12, 34, 56, 500000)`` renders as ``2024-01-01 12:34:56``
|
||||
where openpyxl keeps the microseconds.
|
||||
"""
|
||||
import xlrd
|
||||
|
||||
@@ -153,5 +157,5 @@ def load_spreadsheet(path: str | Path) -> dict[str, str]:
|
||||
if suffix == ".xlsx":
|
||||
return _load_xlsx(p)
|
||||
if suffix == ".xls":
|
||||
return _load_xls(p) # pragma: no cover - legacy .xls path, see _load_xls
|
||||
return _load_xls(p)
|
||||
raise ValueError(f"Unsupported spreadsheet format '{suffix}'. Supported: .xlsx, .xls")
|
||||
|
||||
@@ -609,6 +609,59 @@ def test_load_xls_and_xlsx_agree_on_the_same_values(tmp_path) -> None:
|
||||
assert load_spreadsheet(xls_path) == load_spreadsheet(xlsx_path)
|
||||
|
||||
|
||||
class _StubXlsCell:
|
||||
"""The whole surface ``_xls_cell`` reads: xlrd's ``ctype`` and ``value``."""
|
||||
|
||||
def __init__(self, ctype: int, value: object) -> None:
|
||||
self.ctype = ctype
|
||||
self.value = value
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [12.0, 1e15, float(2**53 - 1), float(2**53), float(-(2**53))])
|
||||
def test_xls_cell_converts_exact_whole_numbers_to_int(value: float) -> None:
|
||||
"""At or below 2**53 every integer is representable, so ``int()`` loses nothing.
|
||||
|
||||
The bound is inclusive at both ends: +/-2**53 is exactly representable, and
|
||||
openpyxl reads the same value from an .xlsx as an ``int``, so excluding it
|
||||
would make the two loaders disagree at exactly the boundary.
|
||||
"""
|
||||
xlrd = pytest.importorskip("xlrd")
|
||||
|
||||
from headroom.transforms.spreadsheet_ingest import _xls_cell
|
||||
|
||||
rendered = _xls_cell(_StubXlsCell(xlrd.XL_CELL_NUMBER, value), 0)
|
||||
|
||||
assert isinstance(rendered, int)
|
||||
assert rendered == int(value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
[float(2**53 + 2), float(-(2**53) - 2), 1e16, 1e20, 123456789012345678.0],
|
||||
)
|
||||
def test_xls_cell_keeps_numbers_past_2_53_as_floats(value: float) -> None:
|
||||
"""Past 2**53 ``int()`` would fabricate digits the workbook never held.
|
||||
|
||||
``2**53 + 2`` is the first whole number above the boundary (``2**53 + 1``
|
||||
is not representable at all), and ``-(2**53) - 2`` its negative mirror.
|
||||
|
||||
xlrd hands back a double, and above 2**53 consecutive integers are no longer
|
||||
representable, so ``int()`` renders the double's exact value rather than the
|
||||
number that was typed: a cell holding 123456789012345678 prints as
|
||||
123456789012345680 -- an identifier that reads as exact and is wrong in its
|
||||
last two digits. The float repr says "approximate" out loud, and is also what
|
||||
the .xlsx loader shows for the same workbook.
|
||||
"""
|
||||
xlrd = pytest.importorskip("xlrd")
|
||||
|
||||
from headroom.transforms.spreadsheet_ingest import _xls_cell
|
||||
|
||||
rendered = _xls_cell(_StubXlsCell(xlrd.XL_CELL_NUMBER, value), 0)
|
||||
|
||||
assert isinstance(rendered, float)
|
||||
assert rendered == value
|
||||
|
||||
|
||||
def test_load_spreadsheet_rejects_unknown_extension(tmp_path) -> None:
|
||||
from headroom.transforms.spreadsheet_ingest import load_spreadsheet
|
||||
|
||||
|
||||
Reference in New Issue
Block a user