mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:10:00 +08:00
@@ -446,10 +446,26 @@ impl SmartCrusher {
|
||||
let (crushed, info) =
|
||||
self.process_value_with_hook(&parsed, 0, query_context, bias, prose_hook);
|
||||
|
||||
// When an object and its descendants were left unchanged, keep
|
||||
// the original bytes. Re-serializing would rewrite Unicode
|
||||
// escapes and numeric lexical forms, then advertise a false
|
||||
// compression. Nested skip/compaction still serializes below
|
||||
// because those paths populate `info`. Spaced JSON without a
|
||||
// nested strategy is compact-serialized for historical crush()
|
||||
// / parity fixtures.
|
||||
if crushed == parsed
|
||||
&& info.is_empty()
|
||||
&& matches!(parsed, Value::Object(_))
|
||||
&& !has_json_insignificant_whitespace(content)
|
||||
{
|
||||
return (content.to_string(), false, String::new());
|
||||
}
|
||||
|
||||
// Re-serialize with Python `safe_json_dumps` formatting:
|
||||
// compact `(",", ":")` separators + `ensure_ascii=False`,
|
||||
// preserving object-key insertion order. Matches the Python
|
||||
// SmartCrusher output bytes the proxy writes.
|
||||
// SmartCrusher output bytes the proxy writes. When descendants
|
||||
// actually change, this path keeps strategy attribution.
|
||||
let result = crate::transforms::anchor_selector::python_safe_json_dumps(&crushed);
|
||||
let was_modified = result != content.trim();
|
||||
(result, was_modified, info)
|
||||
@@ -629,8 +645,11 @@ impl SmartCrusher {
|
||||
}
|
||||
}
|
||||
|
||||
// Second pass: if the object itself has many keys,
|
||||
// compress at the key level.
|
||||
// Second pass: crush_object preserves every property
|
||||
// (object fields are not interchangeable sampled
|
||||
// records). Nested arrays were already compacted above,
|
||||
// so large-array compaction and CCR row offload still
|
||||
// run without deleting their enclosing keys.
|
||||
if processed.len() >= self.config.min_items_to_analyze {
|
||||
let (crushed_dict, strategy) = crush_object(&processed, &self.config, bias);
|
||||
if strategy != "object:passthrough" {
|
||||
@@ -1213,6 +1232,36 @@ fn hash_canonical(canonical: &str) -> String {
|
||||
// here because `process_string`'s `string_ccr:<kind>` strategy-info
|
||||
// label is local to this module's debug-string convention.
|
||||
|
||||
/// True when `s` has JSON whitespace outside of string literals
|
||||
/// (spaces after `:` / `,`, pretty-print newlines, etc.). Used to
|
||||
/// keep compact unchanged objects on the original-bytes path while
|
||||
/// still compact-serializing spaced inputs for historical crush()
|
||||
/// output.
|
||||
fn has_json_insignificant_whitespace(s: &str) -> bool {
|
||||
let mut in_string = false;
|
||||
let mut escape = false;
|
||||
for ch in s.trim().chars() {
|
||||
if in_string {
|
||||
if escape {
|
||||
escape = false;
|
||||
} else if ch == '\\' {
|
||||
escape = true;
|
||||
} else if ch == '"' {
|
||||
in_string = false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ch == '"' {
|
||||
in_string = true;
|
||||
continue;
|
||||
}
|
||||
if ch.is_whitespace() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {
|
||||
use super::compaction::OpaqueKind;
|
||||
match kind {
|
||||
@@ -1533,18 +1582,70 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crush_serializes_with_python_safe_format() {
|
||||
fn crush_serializes_changed_objects_with_python_safe_format() {
|
||||
let c = crusher();
|
||||
// SmartCrusher uses Python's `safe_json_dumps`: compact
|
||||
// separators `(",", ":")` + `ensure_ascii=False`, preserving
|
||||
// object-key insertion order. A spaced input round-trips to
|
||||
// the compact form.
|
||||
let input = r#"{"a": 1, "b": 2, "c": 3}"#;
|
||||
let result = c.crush(input, "", 1.0);
|
||||
assert_eq!(
|
||||
result.compressed, r#"{"a":1,"b":2,"c":3}"#,
|
||||
"safe_json_dumps emits compact `,` / `:` separators"
|
||||
// When descendants change, SmartCrusher re-serializes with
|
||||
// Python's `safe_json_dumps`: compact separators `(",", ":")`
|
||||
// + `ensure_ascii=False`. Unchanged objects keep original
|
||||
// bytes instead of being rewritten.
|
||||
let mut inner = String::from("[");
|
||||
for i in 0..30 {
|
||||
if i > 0 {
|
||||
inner.push(',');
|
||||
}
|
||||
inner.push_str(r#"{"status": "ok"}"#);
|
||||
}
|
||||
inner.push(']');
|
||||
let input = format!(r#"{{"a": 1, "data": {inner}}}"#);
|
||||
let result = c.crush(&input, "", 1.0);
|
||||
assert!(
|
||||
result.was_modified,
|
||||
"nested compressible array must trigger serialization"
|
||||
);
|
||||
assert!(
|
||||
result.compressed.contains(r#""a":1"#),
|
||||
"safe_json_dumps emits compact `:`; got {}",
|
||||
result.compressed
|
||||
);
|
||||
assert!(
|
||||
!result.compressed.contains(": "),
|
||||
"changed output must not keep spaced separators: {}",
|
||||
result.compressed
|
||||
);
|
||||
assert_ne!(result.strategy, "passthrough");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crush_unchanged_object_keeps_original_bytes() {
|
||||
let c = crusher();
|
||||
// Compact input: escaped Unicode and numeric lexical forms must
|
||||
// survive the passthrough branch rather than being reserialized
|
||||
// (`\u00e9` → `é`, `1.0` → `1`).
|
||||
let compact = r#"{"cafe":"caf\u00e9","n":1.0,"ok":true}"#;
|
||||
let result = c.crush(compact, "", 1.0);
|
||||
assert_eq!(result.compressed, compact);
|
||||
assert!(!result.was_modified);
|
||||
assert_eq!(result.strategy, "passthrough");
|
||||
assert!(
|
||||
!result.compressed.contains("<<ccr:"),
|
||||
"unchanged object must not invent a CCR marker"
|
||||
);
|
||||
|
||||
// Spaced / pretty-printed unchanged objects still compact-serialize
|
||||
// (historical crush() output, required by parity fixtures) but
|
||||
// must not advertise a key-drop strategy or invent a CCR hash.
|
||||
let spaced = "{ \"cafe\": \"caf\\u00e9\", \"n\": 1.0, \"ok\": true }";
|
||||
let spaced_result = c.crush(spaced, "", 1.0);
|
||||
assert_eq!(spaced_result.strategy, "passthrough");
|
||||
assert!(
|
||||
!spaced_result.strategy.contains("adaptive"),
|
||||
"must not report a key-drop strategy: {}",
|
||||
spaced_result.strategy
|
||||
);
|
||||
assert!(!spaced_result.compressed.contains("<<ccr:"));
|
||||
let parsed: Value = serde_json::from_str(&spaced_result.compressed).unwrap();
|
||||
assert_eq!(parsed["cafe"], "café");
|
||||
assert_eq!(parsed["ok"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1552,6 +1653,7 @@ mod tests {
|
||||
let c = crusher();
|
||||
// Top-level dict with a nested array of 30 identical items.
|
||||
// The inner array should compress (low_uniqueness path).
|
||||
// Sibling fields stay even though the array is rewritten.
|
||||
let mut inner = String::from("[");
|
||||
for i in 0..30 {
|
||||
if i > 0 {
|
||||
@@ -1560,12 +1662,105 @@ mod tests {
|
||||
inner.push_str(r#"{"status":"ok"}"#);
|
||||
}
|
||||
inner.push(']');
|
||||
let input = format!(r#"{{"data": {}}}"#, inner);
|
||||
let input =
|
||||
format!(r#"{{"label":"keep","data":{inner},"empty":[],"flag":false,"missing":null}}"#);
|
||||
let result = c.crush(&input, "", 1.0);
|
||||
assert!(
|
||||
result.was_modified,
|
||||
"nested compressible array must be crushed even inside a wrapper object"
|
||||
);
|
||||
let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");
|
||||
assert_eq!(parsed["label"], "keep");
|
||||
assert_eq!(parsed["empty"], json!([]));
|
||||
assert_eq!(parsed["flag"], false);
|
||||
assert_eq!(parsed["missing"], Value::Null);
|
||||
assert!(
|
||||
parsed.get("data").is_some(),
|
||||
"enclosing array property must survive"
|
||||
);
|
||||
}
|
||||
|
||||
/// Issue #3634 MCP `saga-mcp_task_get` payload (813 bytes, original
|
||||
/// key order, tags encoded as a JSON string).
|
||||
const MCP_TASK_GET_JSON: &str = r#"{"id":79,"epic_id":9,"title":"test","description":null,"status":"done","priority":"medium","sort_order":0,"assigned_to":null,"estimated_hours":null,"actual_hours":null,"due_date":null,"source_ref":null,"metadata":"{}","created_at":"2026-09-15 20:32:06","updated_at":"2026-09-16 11:59:27","description_locked":0,"is_deleted":0,"deleted_at":null,"deleted_by":null,"delete_reason":null,"epic_name":"latency-routing","tags":"[\"cherry-pick\",\"dedicated branch\"]","subtasks":[{"id":163,"task_id":79,"title":"test2","status":"todo","sort_order":1,"created_at":"2026-09-15 20:32:13","updated_at":"2026-09-16 14:02:26"},{"id":165,"task_id":79,"title":"test4","status":"todo","sort_order":2,"created_at":"2026-09-15 20:38:47","updated_at":"2026-09-17 20:29:04"}],"notes":[],"comments":[],"depends_on":[],"dependents":[]}"#;
|
||||
|
||||
#[test]
|
||||
fn crush_preserves_mcp_task_object_fields() {
|
||||
let c = crusher();
|
||||
let result = c.crush(MCP_TASK_GET_JSON, "", 1.0);
|
||||
assert_eq!(
|
||||
result.compressed, MCP_TASK_GET_JSON,
|
||||
"compact unchanged object must stay byte-identical"
|
||||
);
|
||||
assert!(!result.was_modified);
|
||||
assert_eq!(result.strategy, "passthrough");
|
||||
assert!(
|
||||
!result.strategy.contains("adaptive"),
|
||||
"must not report a key-drop strategy: {}",
|
||||
result.strategy
|
||||
);
|
||||
assert!(
|
||||
!result.compressed.contains("<<ccr:"),
|
||||
"must not invent a CCR marker"
|
||||
);
|
||||
let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");
|
||||
assert_eq!(parsed["tags"], "[\"cherry-pick\",\"dedicated branch\"]");
|
||||
let subtasks = parsed["subtasks"].as_array().expect("subtasks array");
|
||||
assert_eq!(subtasks.len(), 2);
|
||||
assert_eq!(subtasks[0]["title"], "test2");
|
||||
assert_eq!(subtasks[1]["title"], "test4");
|
||||
assert_eq!(parsed["description"], Value::Null);
|
||||
assert_eq!(parsed["notes"], json!([]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_array_offload_keeps_enclosing_object_keys() {
|
||||
// Force the lossy row-drop path so CCR offload fires. The
|
||||
// enclosing object must keep every sibling field, and the
|
||||
// stored payload must round-trip as the original rows.
|
||||
let cfg = SmartCrusherConfig {
|
||||
lossless_min_savings_ratio: 0.99,
|
||||
..SmartCrusherConfig::default()
|
||||
};
|
||||
let c = SmartCrusher::new(cfg);
|
||||
let rows: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
|
||||
let doc = json!({
|
||||
"keep_me": "sibling",
|
||||
"rows": rows,
|
||||
"also_keep": null,
|
||||
"flag": true,
|
||||
"empty": []
|
||||
});
|
||||
let content = serde_json::to_string(&doc).unwrap();
|
||||
let result = c.crush(&content, "", 1.0);
|
||||
assert!(result.was_modified, "lossy nested array should compress");
|
||||
assert_ne!(result.strategy, "passthrough");
|
||||
let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");
|
||||
assert_eq!(parsed["keep_me"], "sibling");
|
||||
assert_eq!(parsed["also_keep"], Value::Null);
|
||||
assert_eq!(parsed["flag"], true);
|
||||
assert_eq!(parsed["empty"], json!([]));
|
||||
assert!(
|
||||
parsed.get("rows").is_some(),
|
||||
"enclosing array property must survive"
|
||||
);
|
||||
assert!(
|
||||
result.compressed.contains("<<ccr:"),
|
||||
"offload must emit a CCR marker: {}",
|
||||
result.compressed
|
||||
);
|
||||
let marker_at = result
|
||||
.compressed
|
||||
.find("<<ccr:")
|
||||
.expect("CCR marker present");
|
||||
let hash = &result.compressed[marker_at + 6..marker_at + 18];
|
||||
let stored = c
|
||||
.ccr_store()
|
||||
.expect("default crusher has a store")
|
||||
.get(hash)
|
||||
.expect("offloaded rows are retrievable");
|
||||
let recovered: Value = serde_json::from_str(&stored).expect("stored JSON");
|
||||
assert_eq!(recovered, Value::Array(rows));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2134,6 +2329,12 @@ mod tests {
|
||||
let lossy: Value = serde_json::from_str(&crusher().crush(&content, "", 1.0).compressed)
|
||||
.expect("default output is JSON");
|
||||
assert_ne!(lossy, doc, "default config should drop items");
|
||||
// Object keys are not sampled records: default mode may crush
|
||||
// nested arrays but must keep every property of `meta`.
|
||||
assert_eq!(
|
||||
lossy["meta"], doc["meta"],
|
||||
"default mode must keep object keys"
|
||||
);
|
||||
|
||||
let strict = SmartCrusher::new(SmartCrusherConfig {
|
||||
lossless_only: true,
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
//! - `crush_string_array` ← `_crush_string_array` (line 2727)
|
||||
//! - `crush_number_array` ← `_crush_number_array` (line 2810) — has BUG #1
|
||||
//! - `crush_object` ← `_crush_object` (line 3015)
|
||||
//! Object fields are independent properties, not sampled records.
|
||||
//! `crush_object` preserves every key; nested arrays are compacted
|
||||
//! by the recursive `process_value` pass before this function runs.
|
||||
//!
|
||||
//! Each takes a `&SmartCrusherConfig`, a `bias` multiplier, and returns
|
||||
//! `(crushed_items, strategy_string)`. Schema-preserving: the output
|
||||
@@ -379,139 +382,25 @@ pub fn crush_number_array(
|
||||
(kept_values, strategy)
|
||||
}
|
||||
|
||||
/// Crush a JSON object by selecting the most informative keys.
|
||||
/// Preserve every property of a JSON object.
|
||||
///
|
||||
/// Mirrors `_crush_object`. Treats key-value pairs as items and applies
|
||||
/// `compute_optimal_k` directly on `f"{k}: {json.dumps(v)}"` strings.
|
||||
/// Always-kept rules:
|
||||
/// - keys whose value contains an error keyword.
|
||||
/// - keys with small total token estimate (<=12 tokens via the rough
|
||||
/// `len(str)/4 + len(key)/4 + 2` heuristic).
|
||||
/// - first K_first and last K_last keys (insertion order — `IndexMap`
|
||||
/// preserves it via the `serde_json/preserve_order` feature).
|
||||
/// Object fields are independent named properties, not interchangeable
|
||||
/// sampled records. The previous key-selection algorithm dropped keys
|
||||
/// (no CCR marker, no recovery hash) which silently deleted structured
|
||||
/// fields such as `tags` and `subtasks` from MCP task objects. Nested
|
||||
/// arrays are still compacted by the recursive `process_value` pass
|
||||
/// *before* this function runs, so legitimate large-array compaction
|
||||
/// and CCR-backed row offload remain available without deleting their
|
||||
/// enclosing keys.
|
||||
///
|
||||
/// `config` and `bias` are retained so the production call site in
|
||||
/// `SmartCrusher::process_value_with_hook` stays unchanged.
|
||||
pub fn crush_object(
|
||||
obj: &Map<String, Value>,
|
||||
config: &SmartCrusherConfig,
|
||||
bias: f64,
|
||||
_config: &SmartCrusherConfig,
|
||||
_bias: f64,
|
||||
) -> (Map<String, Value>, String) {
|
||||
let n = obj.len();
|
||||
if n <= 8 {
|
||||
return (obj.clone(), "object:passthrough".to_string());
|
||||
}
|
||||
|
||||
// Estimate tokens per key-value pair. Python: `len(str)/4 + len(key)/4 + 2`.
|
||||
let mut kv_tokens: Vec<(String, usize)> = Vec::with_capacity(n);
|
||||
let mut total_tokens: usize = 0;
|
||||
for (key, val) in obj {
|
||||
let val_str = serde_json::to_string(val).unwrap_or_default();
|
||||
let tokens = val_str.len() / 4 + key.len() / 4 + 2;
|
||||
kv_tokens.push((key.clone(), tokens));
|
||||
total_tokens += tokens;
|
||||
}
|
||||
|
||||
if total_tokens < config.min_tokens_to_crush {
|
||||
return (obj.clone(), "object:passthrough".to_string());
|
||||
}
|
||||
|
||||
// Strict lossless mode: dropped keys get no CCR marker, so keep them all.
|
||||
if config.lossless_only {
|
||||
return (obj.clone(), "object:lossless_only".to_string());
|
||||
}
|
||||
|
||||
// Compute adaptive K on key-value string representations.
|
||||
let keys: Vec<&String> = obj.keys().collect();
|
||||
let kv_strings: Vec<String> = keys
|
||||
.iter()
|
||||
.map(|k| {
|
||||
format!(
|
||||
"{}: {}",
|
||||
k,
|
||||
serde_json::to_string(&obj[k.as_str()]).unwrap_or_default()
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let kv_refs: Vec<&str> = kv_strings.iter().map(|s| s.as_str()).collect();
|
||||
|
||||
let max_k = if config.max_items_after_crush > 0 {
|
||||
Some(config.max_items_after_crush)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let k_total = compute_optimal_k(&kv_refs, bias, 3, max_k);
|
||||
|
||||
if k_total >= n {
|
||||
return (obj.clone(), "object:passthrough".to_string());
|
||||
}
|
||||
|
||||
// Always keep: error-keyword values.
|
||||
let mut keep_keys: HashSet<String> = HashSet::new();
|
||||
for (key, val) in obj {
|
||||
let val_str = serde_json::to_string(val)
|
||||
.unwrap_or_default()
|
||||
.to_lowercase();
|
||||
if ERROR_KEYWORDS.iter().any(|kw| val_str.contains(kw)) {
|
||||
keep_keys.insert(key.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Always keep: small values (cheap to keep).
|
||||
// Python: `if tokens <= small_threshold // 4` where small_threshold=50,
|
||||
// so tokens <= 12.
|
||||
let small_threshold_tokens = 50_usize / 4;
|
||||
for (key, tokens) in &kv_tokens {
|
||||
if *tokens <= small_threshold_tokens {
|
||||
keep_keys.insert(key.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Boundary: first K_first and last K_last (over the key insertion order).
|
||||
let k_first = 1_usize.max(round_ties_even(k_total as f64 * config.first_fraction) as usize);
|
||||
let k_last = 1_usize.max(round_ties_even(k_total as f64 * config.last_fraction) as usize);
|
||||
for k in keys.iter().take(k_first) {
|
||||
keep_keys.insert((*k).clone());
|
||||
}
|
||||
for k in keys.iter().rev().take(k_last) {
|
||||
keep_keys.insert((*k).clone());
|
||||
}
|
||||
|
||||
// Stride fill. Python's cap recomputes the error-keyword count each
|
||||
// iteration (inefficient but deterministic). We can compute once
|
||||
// because once a key is in keep_keys, the count of error-flagged
|
||||
// entries grows monotonically — which means the cap effectively
|
||||
// grows. Mirror Python's behavior by recomputing.
|
||||
let remaining = k_total.saturating_sub(keep_keys.len());
|
||||
if remaining > 0 {
|
||||
let stride = ((n.saturating_sub(1)) / (remaining + 1)).max(1);
|
||||
let mut i: usize = 0;
|
||||
while i < n {
|
||||
// Python: `if len(keep_keys) >= k_total + len([k for k in keep_keys if any(kw in json.dumps(obj[k]).lower() for kw in keywords)])`
|
||||
let error_kept_count = keep_keys
|
||||
.iter()
|
||||
.filter(|k| {
|
||||
let s = serde_json::to_string(&obj[k.as_str()])
|
||||
.unwrap_or_default()
|
||||
.to_lowercase();
|
||||
ERROR_KEYWORDS.iter().any(|kw| s.contains(kw))
|
||||
})
|
||||
.count();
|
||||
if keep_keys.len() >= k_total + error_kept_count {
|
||||
break;
|
||||
}
|
||||
keep_keys.insert(keys[i].clone());
|
||||
i += stride;
|
||||
}
|
||||
}
|
||||
|
||||
// Build output preserving original key insertion order.
|
||||
let mut result: Map<String, Value> = Map::new();
|
||||
for k in &keys {
|
||||
if keep_keys.contains(k.as_str()) {
|
||||
result.insert((*k).clone(), obj[k.as_str()].clone());
|
||||
}
|
||||
}
|
||||
|
||||
let strategy = format!("object:adaptive({}->{} keys)", n, result.len());
|
||||
(result, strategy)
|
||||
(obj.clone(), "object:passthrough".to_string())
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
@@ -747,6 +636,61 @@ mod tests {
|
||||
|
||||
// ---------- crush_object ----------
|
||||
|
||||
/// Wide object above the historical n>8 / token-floor gate: arbitrary
|
||||
/// names, long strings, a nested dict, nulls, booleans, empty arrays.
|
||||
/// `expensive_index` places the nested dict at begin / middle / end.
|
||||
fn wide_object(expensive_index: usize) -> Map<String, Value> {
|
||||
let n = 15;
|
||||
let expensive_index = expensive_index.min(n - 1);
|
||||
let mut obj = Map::new();
|
||||
for i in 0..n {
|
||||
if i == expensive_index {
|
||||
obj.insert(
|
||||
format!("expensive_{i}"),
|
||||
json!({
|
||||
"nested": {
|
||||
"label": format!("nested-{i}"),
|
||||
"note": "x".repeat(120)
|
||||
},
|
||||
"items": [],
|
||||
"ok": true,
|
||||
"missing": null
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
obj.insert(
|
||||
format!("field_{i:02}"),
|
||||
json!(format!(
|
||||
"this is a relatively long value string for entry number {i} with content"
|
||||
)),
|
||||
);
|
||||
}
|
||||
}
|
||||
obj.insert("flag".to_string(), json!(false));
|
||||
obj.insert("empty".to_string(), json!([]));
|
||||
obj.insert("nada".to_string(), json!(null));
|
||||
obj
|
||||
}
|
||||
|
||||
fn assert_object_fully_preserved(obj: &Map<String, Value>, config: &SmartCrusherConfig) {
|
||||
let (out, strat) = crush_object(obj, config, 1.0);
|
||||
assert_eq!(out, *obj, "crush_object must keep every property");
|
||||
assert_eq!(out.len(), obj.len());
|
||||
for key in obj.keys() {
|
||||
assert!(out.contains_key(key), "dropped key {key}");
|
||||
assert_eq!(out.get(key), obj.get(key));
|
||||
}
|
||||
assert_eq!(strat, "object:passthrough");
|
||||
assert!(
|
||||
!strat.contains("adaptive"),
|
||||
"must not report a key-drop strategy: {strat}"
|
||||
);
|
||||
assert!(
|
||||
!strat.contains("ccr"),
|
||||
"must not invent a CCR hash: {strat}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_passthrough_when_few_keys() {
|
||||
let mut obj = Map::new();
|
||||
@@ -755,25 +699,29 @@ mod tests {
|
||||
}
|
||||
let (out, strat) = crush_object(&obj, &cfg(), 1.0);
|
||||
assert_eq!(out.len(), 5);
|
||||
assert_eq!(out, obj);
|
||||
assert_eq!(strat, "object:passthrough");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_passthrough_when_total_tokens_below_min() {
|
||||
// Many tiny keys/values: total_tokens stays below
|
||||
// min_tokens_to_crush=200.
|
||||
// Many tiny keys/values: historically total_tokens stayed below
|
||||
// min_tokens_to_crush=200 and skipped selection. Preservation is
|
||||
// now unconditional, so every key still survives.
|
||||
let mut obj = Map::new();
|
||||
for i in 0..30 {
|
||||
obj.insert(format!("k{}", i), json!(i));
|
||||
}
|
||||
let (_out, strat) = crush_object(&obj, &cfg(), 1.0);
|
||||
let (out, strat) = crush_object(&obj, &cfg(), 1.0);
|
||||
assert_eq!(out, obj);
|
||||
assert_eq!(strat, "object:passthrough");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_crushes_when_token_budget_exceeded() {
|
||||
// 30 keys, each with a long string value → total tokens > 200,
|
||||
// and unique k_total < n → actual crushing happens.
|
||||
fn object_preserves_every_key_when_token_budget_exceeded() {
|
||||
// 30 keys, each with a long string value → historically above
|
||||
// the token floor and subject to key-selection. Every property
|
||||
// must survive; no adaptive key-drop strategy.
|
||||
let mut obj = Map::new();
|
||||
for i in 0..30 {
|
||||
obj.insert(
|
||||
@@ -784,20 +732,27 @@ mod tests {
|
||||
)),
|
||||
);
|
||||
}
|
||||
let (out, strat) = crush_object(&obj, &cfg(), 1.0);
|
||||
// Either the optimizer kept all (if it deems them all distinct
|
||||
// enough — strategy = passthrough), or it crushed.
|
||||
if strat == "object:passthrough" {
|
||||
assert_eq!(out.len(), 30);
|
||||
} else {
|
||||
assert!(strat.starts_with("object:adaptive("));
|
||||
assert!(out.len() <= 30);
|
||||
assert_object_fully_preserved(&obj, &cfg());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_preserves_wide_object_with_mixed_value_types() {
|
||||
assert_object_fully_preserved(&wide_object(7), &cfg());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_preserves_expensive_fields_in_any_position() {
|
||||
// Reorder the expensive nested field into beginning, middle, and
|
||||
// end. Boundary/stride selection used to drop the middle one.
|
||||
for expensive_index in [0, 7, 14] {
|
||||
assert_object_fully_preserved(&wide_object(expensive_index), &cfg());
|
||||
assert_object_fully_preserved(&wide_object(expensive_index), &lossless_only_cfg());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_keeps_small_values() {
|
||||
// Mix of small + large values; small ones (<=12 tokens) always survive.
|
||||
// Mix of small + large values; every key survives, including tiny.
|
||||
let mut obj = Map::new();
|
||||
obj.insert("tiny".to_string(), json!(1));
|
||||
for i in 0..30 {
|
||||
@@ -810,6 +765,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
let (out, _) = crush_object(&obj, &cfg(), 1.0);
|
||||
assert_eq!(out, obj);
|
||||
assert!(
|
||||
out.contains_key("tiny"),
|
||||
"tiny key (small value) must survive"
|
||||
@@ -830,6 +786,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
let (out, _) = crush_object(&obj, &cfg(), 1.0);
|
||||
assert_eq!(out, obj);
|
||||
assert!(
|
||||
out.contains_key("msg1"),
|
||||
"key with error-keyword value must survive"
|
||||
@@ -887,12 +844,10 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
let (lossy, _) = crush_object(&obj, &cfg(), 1.0);
|
||||
assert!(lossy.len() < obj.len(), "default config should drop keys");
|
||||
|
||||
let (out, strat) = crush_object(&obj, &lossless_only_cfg(), 1.0);
|
||||
assert_eq!(out, obj, "lossless_only must keep every key");
|
||||
assert_eq!(strat, "object:lossless_only");
|
||||
// Default mode used to drop keys with no CCR marker. Preservation
|
||||
// is now unconditional, so both configs keep every property.
|
||||
assert_object_fully_preserved(&obj, &cfg());
|
||||
assert_object_fully_preserved(&obj, &lossless_only_cfg());
|
||||
}
|
||||
|
||||
// ---------- BUG #1 documentation test ----------
|
||||
|
||||
@@ -69,3 +69,254 @@ def test_compact_json_tool_result_compresses() -> None:
|
||||
)
|
||||
after = tok.count_messages(result.messages)
|
||||
assert after < before * 0.9, (before, after, result.transforms_applied[:5])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Issue #3634: SmartCrusher must not drop object fields (tags/subtasks)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Exact MCP `saga-mcp_task_get` payload from the issue (813 bytes, original
|
||||
# key order, tags encoded as a JSON string). Do not pretty-print or reorder.
|
||||
MCP_TASK_GET_JSON = (
|
||||
'{"id":79,"epic_id":9,"title":"test","description":null,'
|
||||
'"status":"done","priority":"medium","sort_order":0,'
|
||||
'"assigned_to":null,"estimated_hours":null,"actual_hours":null,'
|
||||
'"due_date":null,"source_ref":null,"metadata":"{}",'
|
||||
'"created_at":"2026-09-15 20:32:06","updated_at":"2026-09-16 11:59:27",'
|
||||
'"description_locked":0,"is_deleted":0,"deleted_at":null,'
|
||||
'"deleted_by":null,"delete_reason":null,"epic_name":"latency-routing",'
|
||||
'"tags":"[\\"cherry-pick\\",\\"dedicated branch\\"]",'
|
||||
'"subtasks":[{"id":163,"task_id":79,"title":"test2","status":"todo",'
|
||||
'"sort_order":1,"created_at":"2026-09-15 20:32:13",'
|
||||
'"updated_at":"2026-09-16 14:02:26"},'
|
||||
'{"id":165,"task_id":79,"title":"test4","status":"todo","sort_order":2,'
|
||||
'"created_at":"2026-09-15 20:38:47","updated_at":"2026-09-17 20:29:04"}],'
|
||||
'"notes":[],"comments":[],"depends_on":[],"dependents":[]}'
|
||||
)
|
||||
|
||||
_TOOL_CALL_ID = "call_task_get_79"
|
||||
_TOOL_NAME = "saga-mcp_task_get"
|
||||
|
||||
|
||||
def _offline_tokenizer():
|
||||
tokenizer = pytest.importorskip("headroom.tokenizers.estimator")
|
||||
from headroom.tokenizer import Tokenizer
|
||||
|
||||
return Tokenizer(tokenizer.EstimatingTokenCounter())
|
||||
|
||||
|
||||
def _object_preserving_router() -> ContentRouter:
|
||||
# Real ContentRouter + Rust SmartCrusher. Disable unrelated ML / log
|
||||
# fallbacks so a no-savings crush cannot be rewritten by Kompress.
|
||||
return ContentRouter(
|
||||
ContentRouterConfig(
|
||||
skip_user_messages=False,
|
||||
enable_kompress=False,
|
||||
enable_log_compressor=False,
|
||||
enable_search_compressor=False,
|
||||
relevance_split=False,
|
||||
protect_analysis_context=False,
|
||||
exclude_tools=set(),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _openai_task_messages(payload: str) -> list[dict]:
|
||||
return [
|
||||
{"role": "user", "content": "read task 79"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": _TOOL_CALL_ID,
|
||||
"type": "function",
|
||||
"function": {"name": _TOOL_NAME, "arguments": '{"id":79}'},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": _TOOL_CALL_ID,
|
||||
"content": payload,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _anthropic_task_messages(payload: str) -> list[dict]:
|
||||
return [
|
||||
{"role": "user", "content": "read task 79"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": _TOOL_CALL_ID,
|
||||
"name": _TOOL_NAME,
|
||||
"input": {"id": 79},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": _TOOL_CALL_ID,
|
||||
"content": payload,
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _apply(messages: list[dict]):
|
||||
router = _object_preserving_router()
|
||||
tok = _offline_tokenizer()
|
||||
return router.apply(
|
||||
[json.loads(json.dumps(m)) for m in messages],
|
||||
tok,
|
||||
context="read task 79",
|
||||
frozen_message_count=0,
|
||||
protect_recent=0,
|
||||
protect_analysis_context=False,
|
||||
)
|
||||
|
||||
|
||||
def _openai_tool_content(messages: list[dict]) -> str:
|
||||
for msg in messages:
|
||||
if msg.get("role") == "tool" and msg.get("tool_call_id") == _TOOL_CALL_ID:
|
||||
content = msg.get("content")
|
||||
assert isinstance(content, str), content
|
||||
return content
|
||||
raise AssertionError(f"OpenAI tool message {_TOOL_CALL_ID} missing")
|
||||
|
||||
|
||||
def _anthropic_tool_content(messages: list[dict]) -> str:
|
||||
for msg in messages:
|
||||
content = msg.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for block in content:
|
||||
if block.get("type") == "tool_result" and block.get("tool_use_id") == _TOOL_CALL_ID:
|
||||
inner = block.get("content")
|
||||
assert isinstance(inner, str), inner
|
||||
return inner
|
||||
raise AssertionError(f"Anthropic tool_result {_TOOL_CALL_ID} missing")
|
||||
|
||||
|
||||
def _assert_task_payload_preserved(original: str, compressed: str) -> dict:
|
||||
assert "<<ccr:" not in compressed, compressed[:200]
|
||||
parsed = json.loads(compressed)
|
||||
expected = json.loads(original)
|
||||
assert parsed == expected, (sorted(expected), sorted(parsed))
|
||||
assert "tags" in parsed
|
||||
assert "subtasks" in parsed
|
||||
assert parsed["tags"] == '["cherry-pick","dedicated branch"]'
|
||||
assert len(parsed["subtasks"]) == 2
|
||||
assert parsed["subtasks"][0]["title"] == "test2"
|
||||
assert parsed["subtasks"][1]["title"] == "test4"
|
||||
return parsed
|
||||
|
||||
|
||||
def _wide_renamed_task_json() -> str:
|
||||
"""Renamed, oversized variant so a below-threshold skip cannot hide key drop.
|
||||
|
||||
Field names are arbitrary (not tags/subtasks) and expensive values sit at
|
||||
the beginning, middle, and end so a boundary/stride selector cannot
|
||||
accidentally keep them all.
|
||||
"""
|
||||
original = json.loads(MCP_TASK_GET_JSON)
|
||||
expensive = {
|
||||
"nested": {"label": "child", "note": "n" * 180},
|
||||
"items": [],
|
||||
"ok": True,
|
||||
"missing": None,
|
||||
}
|
||||
wide: dict = {
|
||||
"leading_blob": expensive,
|
||||
"label_bundle": original.pop("tags"),
|
||||
"child_records": original.pop("subtasks"),
|
||||
}
|
||||
wide.update(original)
|
||||
for i in range(20):
|
||||
wide[f"extra_field_{i:02d}"] = (
|
||||
f"this is a relatively long value string for entry number {i} with content"
|
||||
)
|
||||
wide["trailing_blob"] = {
|
||||
"nested": {"label": "tail", "note": "t" * 180},
|
||||
"items": [],
|
||||
"ok": False,
|
||||
"missing": None,
|
||||
}
|
||||
payload = json.dumps(wide, separators=(",", ":"))
|
||||
assert len(payload) > 1500
|
||||
assert len(wide) > 8
|
||||
return payload
|
||||
|
||||
|
||||
def test_mcp_task_fixture_is_the_issue_payload() -> None:
|
||||
assert len(MCP_TASK_GET_JSON) == 813
|
||||
parsed = json.loads(MCP_TASK_GET_JSON)
|
||||
assert list(parsed)[21] == "tags"
|
||||
assert list(parsed)[22] == "subtasks"
|
||||
assert parsed["tags"] == '["cherry-pick","dedicated branch"]'
|
||||
|
||||
|
||||
def test_mcp_task_object_preserved_openai_tool_role() -> None:
|
||||
messages = _openai_task_messages(MCP_TASK_GET_JSON)
|
||||
result = _apply(messages)
|
||||
content = _openai_tool_content(result.messages)
|
||||
assert content == MCP_TASK_GET_JSON
|
||||
_assert_task_payload_preserved(MCP_TASK_GET_JSON, content)
|
||||
# Tool-call pairing stays intact.
|
||||
tool_msg = next(m for m in result.messages if m.get("role") == "tool")
|
||||
assistant = next(m for m in result.messages if m.get("role") == "assistant")
|
||||
assert tool_msg["tool_call_id"] == _TOOL_CALL_ID
|
||||
assert assistant["tool_calls"][0]["id"] == _TOOL_CALL_ID
|
||||
assert assistant["tool_calls"][0]["function"]["name"] == _TOOL_NAME
|
||||
assert not any("object:adaptive" in t for t in result.transforms_applied)
|
||||
|
||||
|
||||
def test_mcp_task_object_preserved_anthropic_tool_result() -> None:
|
||||
messages = _anthropic_task_messages(MCP_TASK_GET_JSON)
|
||||
result = _apply(messages)
|
||||
content = _anthropic_tool_content(result.messages)
|
||||
assert content == MCP_TASK_GET_JSON
|
||||
_assert_task_payload_preserved(MCP_TASK_GET_JSON, content)
|
||||
assistant = next(m for m in result.messages if m.get("role") == "assistant")
|
||||
assert assistant["content"][0]["id"] == _TOOL_CALL_ID
|
||||
user_blocks = next(
|
||||
m["content"]
|
||||
for m in result.messages
|
||||
if m.get("role") == "user" and isinstance(m.get("content"), list)
|
||||
)
|
||||
assert user_blocks[0]["tool_use_id"] == _TOOL_CALL_ID
|
||||
assert not any("object:adaptive" in t for t in result.transforms_applied)
|
||||
|
||||
|
||||
def test_wide_renamed_object_fields_preserved_openai_tool_role() -> None:
|
||||
payload = _wide_renamed_task_json()
|
||||
result = _apply(_openai_task_messages(payload))
|
||||
content = _openai_tool_content(result.messages)
|
||||
assert "<<ccr:" not in content
|
||||
parsed = json.loads(content)
|
||||
expected = json.loads(payload)
|
||||
assert parsed == expected
|
||||
assert "label_bundle" in parsed
|
||||
assert "child_records" in parsed
|
||||
assert "leading_blob" in parsed
|
||||
assert "trailing_blob" in parsed
|
||||
assert parsed["label_bundle"] == '["cherry-pick","dedicated branch"]'
|
||||
assert len(parsed["child_records"]) == 2
|
||||
assert parsed["leading_blob"]["nested"]["note"] == "n" * 180
|
||||
assert parsed["trailing_blob"]["nested"]["note"] == "t" * 180
|
||||
|
||||
|
||||
def test_malformed_json_tool_result_passes_through() -> None:
|
||||
payload = "{" + "not-json, still a tool result. " * 40
|
||||
assert len(payload) > 500
|
||||
result = _apply(_openai_task_messages(payload))
|
||||
content = _openai_tool_content(result.messages)
|
||||
assert content == payload
|
||||
assert "<<ccr:" not in content
|
||||
|
||||
Reference in New Issue
Block a user