fix(smart-crusher): recurse at adaptive array limit (#3770)

## Description

SmartCrusher currently stops after adaptive sizing when an object array
is small enough to keep every row. That skips the recursive string
handling used for smaller arrays, so arrays of 5 to 8 objects can pass
through long values without compression.

Closes #3760

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)
- [ ] New feature (non-breaking change that adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Documentation update
- [ ] Performance improvement
- [ ] Code refactoring (no functional changes)

## Changes Made

- Recurse into object-array rows when adaptive sizing keeps every row at
the analysis threshold.
- Add regression coverage for 5 to 8 object rows with long string values
and CCR markers.

## Testing

- [ ] Unit tests pass (`pytest`)
- [ ] Linting passes (`ruff check .`)
- [ ] Type checking passes (`mypy headroom`)
- [x] New tests added for new functionality
- [ ] Manual testing performed

### Test Output

```text
cargo fmt --all -- --check
exit 0

cargo test -p headroom-core --lib crush_recurses_into_object_arrays_at_adaptive_limit -- --nocapture
test transforms::smart_crusher::crusher::tests::crush_recurses_into_object_arrays_at_adaptive_limit ... ok
test result: ok. 1 passed; 0 failed

cargo test -p headroom-core --lib smart_crusher
test result: ok. 335 passed; 0 failed; 0 ignored

cargo clippy -p headroom-core --lib -- -D warnings
Finished `dev` profile; exit 0
```

## Real Behavior Proof

- Environment: Linux x86_64 checkout using the repository-pinned Rust
1.95.0 toolchain at the current main snapshot.
- Exact command / steps: `cargo test -p headroom-core --lib
crush_recurses_into_object_arrays_at_adaptive_limit -- --nocapture`; the
regression constructs object arrays with 5 through 8 rows and long
descriptions, then calls `SmartCrusher::crush`.
- Observed result: Before the fix, the merge-base test failed at n=5
with `none:adaptive_at_limit(5->5)` and no CCR marker. After the fix,
all n=5 through n=8 cases pass and emit `string_ccr` markers.
- Not tested: A published wheel, an external provider or proxy request,
and Windows or macOS runtime behavior.

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

## Checklist

- [x] My code follows the project's style guidelines
- [x] I have performed a self-review of my code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [x] I did **not** edit `CHANGELOG.md` - it is generated by
release-please from my Conventional Commit PR title (a CI guard enforces
this)

## Screenshots (if applicable)

N/A

## Runtime Rollout Safety

- Rollout-managed features: None.
- Runtime behavior: SmartCrusher recursion only; no configuration,
schema, migration, or deployment changes.
- Rollback: revert this PR.

---------

Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
This commit is contained in:
AxelRay
2026-09-28 13:27:49 -05:00
committed by GitHub
co-authored by JerrettDavis
parent de4cf7e7a1
commit 7790bdeebd
@@ -544,6 +544,36 @@ impl SmartCrusher {
// compressed / marker-substituted by the hook.
let result =
self.crush_array_with_source(&rows, arr, query_context, bias);
// Adaptive sizing can decide to keep every row even after the
// analysis threshold is crossed. Match the below-threshold path
// so nested values still receive their own safe transforms.
if result.strategy_info == "none:adaptive_at_limit" {
info_parts.push(format!(
"{}({}->{})",
result.strategy_info,
n,
result.items.len()
));
if prose_hook.is_some() {
return (Value::Array(rows), info_parts.join(","));
}
let mut processed: Vec<Value> = Vec::with_capacity(n);
for item in arr {
let (p_item, p_info) = self.process_value_with_hook(
item,
depth + 1,
query_context,
bias,
prose_hook,
);
processed.push(p_item);
if !p_info.is_empty() {
info_parts.push(p_info);
}
}
return (Value::Array(processed), info_parts.join(","));
}
// Lossless path won → substitute the array
// with the compacted string in place. This
// makes the lossless win visible to the
@@ -2038,6 +2068,40 @@ mod tests {
);
}
#[test]
fn crush_recurses_into_object_arrays_at_adaptive_limit() {
let c = crusher();
let description =
"Curated collection of business tables covering customer, product, ".repeat(12);
for n in 5..=8 {
let rows: Vec<Value> = (0..n)
.map(|i| {
json!({
"name": format!("domain_{i}"),
"description": description,
"tables": 12 + i,
"owner": "data-platform"
})
})
.collect();
let input = json!({"domains": rows}).to_string();
let result = c.crush(&input, "list the available domains", 1.0);
assert!(
result.strategy.contains("string_ccr:"),
"n={n} should recurse into rows at the adaptive limit: strategy={} output={}",
result.strategy,
result.compressed
);
assert!(
result.compressed.contains("<<ccr:"),
"n={n} should offload long row strings: {}",
result.compressed
);
}
}
/// Issue #3634 MCP `saga-mcp_task_get` payload (813 bytes, original
/// key order, tags encoded as a JSON string).
const MCP_TASK_GET_JSON: &str = r#"{"id":79,"epic_id":9,"title":"test","description":null,"status":"done","priority":"medium","sort_order":0,"assigned_to":null,"estimated_hours":null,"actual_hours":null,"due_date":null,"source_ref":null,"metadata":"{}","created_at":"2026-09-15 20:32:06","updated_at":"2026-09-16 11:59:27","description_locked":0,"is_deleted":0,"deleted_at":null,"deleted_by":null,"delete_reason":null,"epic_name":"latency-routing","tags":"[\"cherry-pick\",\"dedicated branch\"]","subtasks":[{"id":163,"task_id":79,"title":"test2","status":"todo","sort_order":1,"created_at":"2026-09-15 20:32:13","updated_at":"2026-09-16 14:02:26"},{"id":165,"task_id":79,"title":"test4","status":"todo","sort_order":2,"created_at":"2026-09-15 20:38:47","updated_at":"2026-09-17 20:29:04"}],"notes":[],"comments":[],"depends_on":[],"dependents":[]}"#;