mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix: preserve null and empty strings in lossless CSV compaction (#3665)
## Description Fix strict-lossless CSV-schema compaction so nullable string columns preserve the distinction between missing values, JSON `null`, and empty strings. Before this change, swapping `""` and `null` between two rows could produce byte-identical `lossless:table` output. Closes # ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) ## Changes Made - Render JSON `null` explicitly as `null`. - Quote empty strings as `""`. - Quote the literal string `"null"` so it remains distinct from JSON `null`. - Add regression tests for the ambiguous cases and swapped-row collision. ## Testing - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ``` cargo test -p headroom-core csv_formatter_distinguishes_missing_null_empty_and_literal_null cargo test -p headroom-core csv_formatter_nullable_swap_is_not_identical ``` Both passed. ## Validation run: https://github.com/upramod/invariant-compression-gate/actions/runs/35455082632 ## Real Behavior Proof - Environment: Headroom 0.36.0, 0.37.0, and current upstream `main` - Exact command / steps: run strict-lossless compaction on two inputs that swap `""` and `null` between rows - Observed result: before the fix, outputs were byte-identical; after the fix, they differ - Not tested: full repository Python/lint/type-check suite ## Runtime Rollout Safety - Rollout-managed feature(s): None - Minimum rollout channel: N/A - Stable/default behavior changed: Only CSV-schema rendering for affected nullable string values - Kill switch / disable path: Use another formatter or disable the lossless CSV-schema path - Unsafe override required: No - Qualification impact: Localized to preserving scalar distinctions in lossless CSV output - Rollback path: Revert this commit ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md`
This commit is contained in:
@@ -337,11 +337,14 @@ fn humanize_bytes(n: usize) -> String {
|
||||
|
||||
fn json_scalar_to_csv(v: &Value) -> String {
|
||||
match v {
|
||||
Value::Null => String::new(),
|
||||
// Keep null distinct from CellValue::Missing (empty field) and
|
||||
// Value::String("") (quoted empty field). Literal string "null"
|
||||
// is quoted below so these cases remain byte-distinguishable.
|
||||
Value::Null => "null".to_string(),
|
||||
Value::Bool(b) => if *b { "true" } else { "false" }.to_string(),
|
||||
Value::Number(n) => n.to_string(),
|
||||
Value::String(s) => {
|
||||
if needs_csv_quote(s) {
|
||||
if s.is_empty() || s == "null" || needs_csv_quote(s) {
|
||||
csv_quote(s)
|
||||
} else {
|
||||
s.clone()
|
||||
@@ -726,6 +729,43 @@ mod tests {
|
||||
assert!(!without.contains("__dropped"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_formatter_distinguishes_missing_null_empty_and_literal_null() {
|
||||
let c = Compaction::Table {
|
||||
schema: Schema {
|
||||
fields: vec![super::super::ir::FieldSpec {
|
||||
name: "label".into(),
|
||||
type_tag: "string".into(),
|
||||
nullable: true,
|
||||
}],
|
||||
},
|
||||
rows: vec![
|
||||
Row::new(vec![CellValue::Missing]),
|
||||
Row::new(vec![CellValue::Scalar(Value::Null)]),
|
||||
Row::new(vec![CellValue::Scalar(Value::String(String::new()))]),
|
||||
Row::new(vec![CellValue::Scalar(Value::String("null".into()))]),
|
||||
],
|
||||
original_count: 4,
|
||||
};
|
||||
let out = CsvSchemaFormatter::new().format(&c);
|
||||
assert_eq!(out, "[4]{label:string?}\n\nnull\n\"\"\n\"null\"\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_formatter_nullable_swap_is_not_identical() {
|
||||
let a = vec![
|
||||
json!({"id": 1, "label": ""}),
|
||||
json!({"id": 2, "label": null}),
|
||||
];
|
||||
let b = vec![
|
||||
json!({"id": 1, "label": null}),
|
||||
json!({"id": 2, "label": ""}),
|
||||
];
|
||||
let fa = CsvSchemaFormatter::new().format(&compact(&a, &cfg()));
|
||||
let fb = CsvSchemaFormatter::new().format(&compact(&b, &cfg()));
|
||||
assert_ne!(fa, fb);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_matches_format_len() {
|
||||
let items = vec![json!({"a": 1}), json!({"a": 2})];
|
||||
|
||||
Reference in New Issue
Block a user