fix: preserve null and empty strings in lossless CSV compaction (#3665)

## Description

Fix strict-lossless CSV-schema compaction so nullable string columns
preserve the distinction between missing values, JSON `null`, and empty
strings.

Before this change, swapping `""` and `null` between two rows could
produce byte-identical `lossless:table` output.

Closes #

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)

## Changes Made

- Render JSON `null` explicitly as `null`.
- Quote empty strings as `""`.
- Quote the literal string `"null"` so it remains distinct from JSON
`null`.
- Add regression tests for the ambiguous cases and swapped-row
collision.

## Testing

- [x] New tests added for new functionality
- [x] Manual testing performed

### Test Output

```
cargo test -p headroom-core csv_formatter_distinguishes_missing_null_empty_and_literal_null
cargo test -p headroom-core csv_formatter_nullable_swap_is_not_identical
```

Both passed.

## Validation run:

https://github.com/upramod/invariant-compression-gate/actions/runs/35455082632

## Real Behavior Proof

- Environment: Headroom 0.36.0, 0.37.0, and current upstream `main`
- Exact command / steps: run strict-lossless compaction on two inputs
that swap `""` and `null` between rows
- Observed result: before the fix, outputs were byte-identical; after
the fix, they differ
- Not tested: full repository Python/lint/type-check suite

## Runtime Rollout Safety

- Rollout-managed feature(s): None
- Minimum rollout channel: N/A
- Stable/default behavior changed: Only CSV-schema rendering for
affected nullable string values
- Kill switch / disable path: Use another formatter or disable the
lossless CSV-schema path
- Unsafe override required: No
- Qualification impact: Localized to preserving scalar distinctions in
lossless CSV output
- Rollback path: Revert this commit

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

## Checklist

- [x] My code follows the project's style guidelines
- [x] I have performed a self-review of my code
- [x] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [ ] New and existing unit tests pass locally with my changes
- [x] I did **not** edit `CHANGELOG.md`
This commit is contained in:
Pramod Ubbala
2026-09-24 20:44:47 -05:00
committed by GitHub
parent 2e2758de95
commit 7ad9f809fa
@@ -337,11 +337,14 @@ fn humanize_bytes(n: usize) -> String {
fn json_scalar_to_csv(v: &Value) -> String {
match v {
Value::Null => String::new(),
// Keep null distinct from CellValue::Missing (empty field) and
// Value::String("") (quoted empty field). Literal string "null"
// is quoted below so these cases remain byte-distinguishable.
Value::Null => "null".to_string(),
Value::Bool(b) => if *b { "true" } else { "false" }.to_string(),
Value::Number(n) => n.to_string(),
Value::String(s) => {
if needs_csv_quote(s) {
if s.is_empty() || s == "null" || needs_csv_quote(s) {
csv_quote(s)
} else {
s.clone()
@@ -726,6 +729,43 @@ mod tests {
assert!(!without.contains("__dropped"));
}
#[test]
fn csv_formatter_distinguishes_missing_null_empty_and_literal_null() {
let c = Compaction::Table {
schema: Schema {
fields: vec![super::super::ir::FieldSpec {
name: "label".into(),
type_tag: "string".into(),
nullable: true,
}],
},
rows: vec![
Row::new(vec![CellValue::Missing]),
Row::new(vec![CellValue::Scalar(Value::Null)]),
Row::new(vec![CellValue::Scalar(Value::String(String::new()))]),
Row::new(vec![CellValue::Scalar(Value::String("null".into()))]),
],
original_count: 4,
};
let out = CsvSchemaFormatter::new().format(&c);
assert_eq!(out, "[4]{label:string?}\n\nnull\n\"\"\n\"null\"\n");
}
#[test]
fn csv_formatter_nullable_swap_is_not_identical() {
let a = vec![
json!({"id": 1, "label": ""}),
json!({"id": 2, "label": null}),
];
let b = vec![
json!({"id": 1, "label": null}),
json!({"id": 2, "label": ""}),
];
let fa = CsvSchemaFormatter::new().format(&compact(&a, &cfg()));
let fb = CsvSchemaFormatter::new().format(&compact(&b, &cfg()));
assert_ne!(fa, fb);
}
#[test]
fn estimate_matches_format_len() {
let items = vec![json!({"a": 1}), json!({"a": 2})];