fix: preserve JSON object fields in SmartCrusher

Fixes #3634
This commit is contained in:
Matt Van Horn
2026-09-18 01:55:51 -07:00
parent 964671d883
commit e214a5e953
3 changed files with 571 additions and 164 deletions
@@ -446,10 +446,26 @@ impl SmartCrusher {
let (crushed, info) =
self.process_value_with_hook(&parsed, 0, query_context, bias, prose_hook);
// When an object and its descendants were left unchanged, keep
// the original bytes. Re-serializing would rewrite Unicode
// escapes and numeric lexical forms, then advertise a false
// compression. Nested skip/compaction still serializes below
// because those paths populate `info`. Spaced JSON without a
// nested strategy is compact-serialized for historical crush()
// / parity fixtures.
if crushed == parsed
&& info.is_empty()
&& matches!(parsed, Value::Object(_))
&& !has_json_insignificant_whitespace(content)
{
return (content.to_string(), false, String::new());
}
// Re-serialize with Python `safe_json_dumps` formatting:
// compact `(",", ":")` separators + `ensure_ascii=False`,
// preserving object-key insertion order. Matches the Python
// SmartCrusher output bytes the proxy writes.
// SmartCrusher output bytes the proxy writes. When descendants
// actually change, this path keeps strategy attribution.
let result = crate::transforms::anchor_selector::python_safe_json_dumps(&crushed);
let was_modified = result != content.trim();
(result, was_modified, info)
@@ -629,8 +645,11 @@ impl SmartCrusher {
}
}
// Second pass: if the object itself has many keys,
// compress at the key level.
// Second pass: crush_object preserves every property
// (object fields are not interchangeable sampled
// records). Nested arrays were already compacted above,
// so large-array compaction and CCR row offload still
// run without deleting their enclosing keys.
if processed.len() >= self.config.min_items_to_analyze {
let (crushed_dict, strategy) = crush_object(&processed, &self.config, bias);
if strategy != "object:passthrough" {
@@ -1213,6 +1232,36 @@ fn hash_canonical(canonical: &str) -> String {
// here because `process_string`'s `string_ccr:<kind>` strategy-info
// label is local to this module's debug-string convention.
/// True when `s` has JSON whitespace outside of string literals
/// (spaces after `:` / `,`, pretty-print newlines, etc.). Used to
/// keep compact unchanged objects on the original-bytes path while
/// still compact-serializing spaced inputs for historical crush()
/// output.
fn has_json_insignificant_whitespace(s: &str) -> bool {
let mut in_string = false;
let mut escape = false;
for ch in s.trim().chars() {
if in_string {
if escape {
escape = false;
} else if ch == '\\' {
escape = true;
} else if ch == '"' {
in_string = false;
}
continue;
}
if ch == '"' {
in_string = true;
continue;
}
if ch.is_whitespace() {
return true;
}
}
false
}
fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {
use super::compaction::OpaqueKind;
match kind {
@@ -1533,18 +1582,70 @@ mod tests {
}
#[test]
fn crush_serializes_with_python_safe_format() {
fn crush_serializes_changed_objects_with_python_safe_format() {
let c = crusher();
// SmartCrusher uses Python's `safe_json_dumps`: compact
// separators `(",", ":")` + `ensure_ascii=False`, preserving
// object-key insertion order. A spaced input round-trips to
// the compact form.
let input = r#"{"a": 1, "b": 2, "c": 3}"#;
let result = c.crush(input, "", 1.0);
assert_eq!(
result.compressed, r#"{"a":1,"b":2,"c":3}"#,
"safe_json_dumps emits compact `,` / `:` separators"
// When descendants change, SmartCrusher re-serializes with
// Python's `safe_json_dumps`: compact separators `(",", ":")`
// + `ensure_ascii=False`. Unchanged objects keep original
// bytes instead of being rewritten.
let mut inner = String::from("[");
for i in 0..30 {
if i > 0 {
inner.push(',');
}
inner.push_str(r#"{"status": "ok"}"#);
}
inner.push(']');
let input = format!(r#"{{"a": 1, "data": {inner}}}"#);
let result = c.crush(&input, "", 1.0);
assert!(
result.was_modified,
"nested compressible array must trigger serialization"
);
assert!(
result.compressed.contains(r#""a":1"#),
"safe_json_dumps emits compact `:`; got {}",
result.compressed
);
assert!(
!result.compressed.contains(": "),
"changed output must not keep spaced separators: {}",
result.compressed
);
assert_ne!(result.strategy, "passthrough");
}
#[test]
fn crush_unchanged_object_keeps_original_bytes() {
let c = crusher();
// Compact input: escaped Unicode and numeric lexical forms must
// survive the passthrough branch rather than being reserialized
// (`\u00e9` → `é`, `1.0` → `1`).
let compact = r#"{"cafe":"caf\u00e9","n":1.0,"ok":true}"#;
let result = c.crush(compact, "", 1.0);
assert_eq!(result.compressed, compact);
assert!(!result.was_modified);
assert_eq!(result.strategy, "passthrough");
assert!(
!result.compressed.contains("<<ccr:"),
"unchanged object must not invent a CCR marker"
);
// Spaced / pretty-printed unchanged objects still compact-serialize
// (historical crush() output, required by parity fixtures) but
// must not advertise a key-drop strategy or invent a CCR hash.
let spaced = "{ \"cafe\": \"caf\\u00e9\", \"n\": 1.0, \"ok\": true }";
let spaced_result = c.crush(spaced, "", 1.0);
assert_eq!(spaced_result.strategy, "passthrough");
assert!(
!spaced_result.strategy.contains("adaptive"),
"must not report a key-drop strategy: {}",
spaced_result.strategy
);
assert!(!spaced_result.compressed.contains("<<ccr:"));
let parsed: Value = serde_json::from_str(&spaced_result.compressed).unwrap();
assert_eq!(parsed["cafe"], "café");
assert_eq!(parsed["ok"], true);
}
#[test]
@@ -1552,6 +1653,7 @@ mod tests {
let c = crusher();
// Top-level dict with a nested array of 30 identical items.
// The inner array should compress (low_uniqueness path).
// Sibling fields stay even though the array is rewritten.
let mut inner = String::from("[");
for i in 0..30 {
if i > 0 {
@@ -1560,12 +1662,105 @@ mod tests {
inner.push_str(r#"{"status":"ok"}"#);
}
inner.push(']');
let input = format!(r#"{{"data": {}}}"#, inner);
let input =
format!(r#"{{"label":"keep","data":{inner},"empty":[],"flag":false,"missing":null}}"#);
let result = c.crush(&input, "", 1.0);
assert!(
result.was_modified,
"nested compressible array must be crushed even inside a wrapper object"
);
let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");
assert_eq!(parsed["label"], "keep");
assert_eq!(parsed["empty"], json!([]));
assert_eq!(parsed["flag"], false);
assert_eq!(parsed["missing"], Value::Null);
assert!(
parsed.get("data").is_some(),
"enclosing array property must survive"
);
}
/// Issue #3634 MCP `saga-mcp_task_get` payload (813 bytes, original
/// key order, tags encoded as a JSON string).
const MCP_TASK_GET_JSON: &str = r#"{"id":79,"epic_id":9,"title":"test","description":null,"status":"done","priority":"medium","sort_order":0,"assigned_to":null,"estimated_hours":null,"actual_hours":null,"due_date":null,"source_ref":null,"metadata":"{}","created_at":"2026-09-15 20:32:06","updated_at":"2026-09-16 11:59:27","description_locked":0,"is_deleted":0,"deleted_at":null,"deleted_by":null,"delete_reason":null,"epic_name":"latency-routing","tags":"[\"cherry-pick\",\"dedicated branch\"]","subtasks":[{"id":163,"task_id":79,"title":"test2","status":"todo","sort_order":1,"created_at":"2026-09-15 20:32:13","updated_at":"2026-09-16 14:02:26"},{"id":165,"task_id":79,"title":"test4","status":"todo","sort_order":2,"created_at":"2026-09-15 20:38:47","updated_at":"2026-09-17 20:29:04"}],"notes":[],"comments":[],"depends_on":[],"dependents":[]}"#;
#[test]
fn crush_preserves_mcp_task_object_fields() {
let c = crusher();
let result = c.crush(MCP_TASK_GET_JSON, "", 1.0);
assert_eq!(
result.compressed, MCP_TASK_GET_JSON,
"compact unchanged object must stay byte-identical"
);
assert!(!result.was_modified);
assert_eq!(result.strategy, "passthrough");
assert!(
!result.strategy.contains("adaptive"),
"must not report a key-drop strategy: {}",
result.strategy
);
assert!(
!result.compressed.contains("<<ccr:"),
"must not invent a CCR marker"
);
let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");
assert_eq!(parsed["tags"], "[\"cherry-pick\",\"dedicated branch\"]");
let subtasks = parsed["subtasks"].as_array().expect("subtasks array");
assert_eq!(subtasks.len(), 2);
assert_eq!(subtasks[0]["title"], "test2");
assert_eq!(subtasks[1]["title"], "test4");
assert_eq!(parsed["description"], Value::Null);
assert_eq!(parsed["notes"], json!([]));
}
#[test]
fn nested_array_offload_keeps_enclosing_object_keys() {
// Force the lossy row-drop path so CCR offload fires. The
// enclosing object must keep every sibling field, and the
// stored payload must round-trip as the original rows.
let cfg = SmartCrusherConfig {
lossless_min_savings_ratio: 0.99,
..SmartCrusherConfig::default()
};
let c = SmartCrusher::new(cfg);
let rows: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
let doc = json!({
"keep_me": "sibling",
"rows": rows,
"also_keep": null,
"flag": true,
"empty": []
});
let content = serde_json::to_string(&doc).unwrap();
let result = c.crush(&content, "", 1.0);
assert!(result.was_modified, "lossy nested array should compress");
assert_ne!(result.strategy, "passthrough");
let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");
assert_eq!(parsed["keep_me"], "sibling");
assert_eq!(parsed["also_keep"], Value::Null);
assert_eq!(parsed["flag"], true);
assert_eq!(parsed["empty"], json!([]));
assert!(
parsed.get("rows").is_some(),
"enclosing array property must survive"
);
assert!(
result.compressed.contains("<<ccr:"),
"offload must emit a CCR marker: {}",
result.compressed
);
let marker_at = result
.compressed
.find("<<ccr:")
.expect("CCR marker present");
let hash = &result.compressed[marker_at + 6..marker_at + 18];
let stored = c
.ccr_store()
.expect("default crusher has a store")
.get(hash)
.expect("offloaded rows are retrievable");
let recovered: Value = serde_json::from_str(&stored).expect("stored JSON");
assert_eq!(recovered, Value::Array(rows));
}
#[test]
@@ -2134,6 +2329,12 @@ mod tests {
let lossy: Value = serde_json::from_str(&crusher().crush(&content, "", 1.0).compressed)
.expect("default output is JSON");
assert_ne!(lossy, doc, "default config should drop items");
// Object keys are not sampled records: default mode may crush
// nested arrays but must keep every property of `meta`.
assert_eq!(
lossy["meta"], doc["meta"],
"default mode must keep object keys"
);
let strict = SmartCrusher::new(SmartCrusherConfig {
lossless_only: true,
@@ -5,6 +5,9 @@
//! - `crush_string_array` ← `_crush_string_array` (line 2727)
//! - `crush_number_array` ← `_crush_number_array` (line 2810) — has BUG #1
//! - `crush_object` ← `_crush_object` (line 3015)
//! Object fields are independent properties, not sampled records.
//! `crush_object` preserves every key; nested arrays are compacted
//! by the recursive `process_value` pass before this function runs.
//!
//! Each takes a `&SmartCrusherConfig`, a `bias` multiplier, and returns
//! `(crushed_items, strategy_string)`. Schema-preserving: the output
@@ -379,139 +382,25 @@ pub fn crush_number_array(
(kept_values, strategy)
}
/// Crush a JSON object by selecting the most informative keys.
/// Preserve every property of a JSON object.
///
/// Mirrors `_crush_object`. Treats key-value pairs as items and applies
/// `compute_optimal_k` directly on `f"{k}: {json.dumps(v)}"` strings.
/// Always-kept rules:
/// - keys whose value contains an error keyword.
/// - keys with small total token estimate (<=12 tokens via the rough
/// `len(str)/4 + len(key)/4 + 2` heuristic).
/// - first K_first and last K_last keys (insertion order — `IndexMap`
/// preserves it via the `serde_json/preserve_order` feature).
/// Object fields are independent named properties, not interchangeable
/// sampled records. The previous key-selection algorithm dropped keys
/// (no CCR marker, no recovery hash) which silently deleted structured
/// fields such as `tags` and `subtasks` from MCP task objects. Nested
/// arrays are still compacted by the recursive `process_value` pass
/// *before* this function runs, so legitimate large-array compaction
/// and CCR-backed row offload remain available without deleting their
/// enclosing keys.
///
/// `config` and `bias` are retained so the production call site in
/// `SmartCrusher::process_value_with_hook` stays unchanged.
pub fn crush_object(
obj: &Map<String, Value>,
config: &SmartCrusherConfig,
bias: f64,
_config: &SmartCrusherConfig,
_bias: f64,
) -> (Map<String, Value>, String) {
let n = obj.len();
if n <= 8 {
return (obj.clone(), "object:passthrough".to_string());
}
// Estimate tokens per key-value pair. Python: `len(str)/4 + len(key)/4 + 2`.
let mut kv_tokens: Vec<(String, usize)> = Vec::with_capacity(n);
let mut total_tokens: usize = 0;
for (key, val) in obj {
let val_str = serde_json::to_string(val).unwrap_or_default();
let tokens = val_str.len() / 4 + key.len() / 4 + 2;
kv_tokens.push((key.clone(), tokens));
total_tokens += tokens;
}
if total_tokens < config.min_tokens_to_crush {
return (obj.clone(), "object:passthrough".to_string());
}
// Strict lossless mode: dropped keys get no CCR marker, so keep them all.
if config.lossless_only {
return (obj.clone(), "object:lossless_only".to_string());
}
// Compute adaptive K on key-value string representations.
let keys: Vec<&String> = obj.keys().collect();
let kv_strings: Vec<String> = keys
.iter()
.map(|k| {
format!(
"{}: {}",
k,
serde_json::to_string(&obj[k.as_str()]).unwrap_or_default()
)
})
.collect();
let kv_refs: Vec<&str> = kv_strings.iter().map(|s| s.as_str()).collect();
let max_k = if config.max_items_after_crush > 0 {
Some(config.max_items_after_crush)
} else {
None
};
let k_total = compute_optimal_k(&kv_refs, bias, 3, max_k);
if k_total >= n {
return (obj.clone(), "object:passthrough".to_string());
}
// Always keep: error-keyword values.
let mut keep_keys: HashSet<String> = HashSet::new();
for (key, val) in obj {
let val_str = serde_json::to_string(val)
.unwrap_or_default()
.to_lowercase();
if ERROR_KEYWORDS.iter().any(|kw| val_str.contains(kw)) {
keep_keys.insert(key.clone());
}
}
// Always keep: small values (cheap to keep).
// Python: `if tokens <= small_threshold // 4` where small_threshold=50,
// so tokens <= 12.
let small_threshold_tokens = 50_usize / 4;
for (key, tokens) in &kv_tokens {
if *tokens <= small_threshold_tokens {
keep_keys.insert(key.clone());
}
}
// Boundary: first K_first and last K_last (over the key insertion order).
let k_first = 1_usize.max(round_ties_even(k_total as f64 * config.first_fraction) as usize);
let k_last = 1_usize.max(round_ties_even(k_total as f64 * config.last_fraction) as usize);
for k in keys.iter().take(k_first) {
keep_keys.insert((*k).clone());
}
for k in keys.iter().rev().take(k_last) {
keep_keys.insert((*k).clone());
}
// Stride fill. Python's cap recomputes the error-keyword count each
// iteration (inefficient but deterministic). We can compute once
// because once a key is in keep_keys, the count of error-flagged
// entries grows monotonically — which means the cap effectively
// grows. Mirror Python's behavior by recomputing.
let remaining = k_total.saturating_sub(keep_keys.len());
if remaining > 0 {
let stride = ((n.saturating_sub(1)) / (remaining + 1)).max(1);
let mut i: usize = 0;
while i < n {
// Python: `if len(keep_keys) >= k_total + len([k for k in keep_keys if any(kw in json.dumps(obj[k]).lower() for kw in keywords)])`
let error_kept_count = keep_keys
.iter()
.filter(|k| {
let s = serde_json::to_string(&obj[k.as_str()])
.unwrap_or_default()
.to_lowercase();
ERROR_KEYWORDS.iter().any(|kw| s.contains(kw))
})
.count();
if keep_keys.len() >= k_total + error_kept_count {
break;
}
keep_keys.insert(keys[i].clone());
i += stride;
}
}
// Build output preserving original key insertion order.
let mut result: Map<String, Value> = Map::new();
for k in &keys {
if keep_keys.contains(k.as_str()) {
result.insert((*k).clone(), obj[k.as_str()].clone());
}
}
let strategy = format!("object:adaptive({}->{} keys)", n, result.len());
(result, strategy)
(obj.clone(), "object:passthrough".to_string())
}
// ---------- helpers ----------
@@ -747,6 +636,61 @@ mod tests {
// ---------- crush_object ----------
/// Wide object above the historical n>8 / token-floor gate: arbitrary
/// names, long strings, a nested dict, nulls, booleans, empty arrays.
/// `expensive_index` places the nested dict at begin / middle / end.
fn wide_object(expensive_index: usize) -> Map<String, Value> {
let n = 15;
let expensive_index = expensive_index.min(n - 1);
let mut obj = Map::new();
for i in 0..n {
if i == expensive_index {
obj.insert(
format!("expensive_{i}"),
json!({
"nested": {
"label": format!("nested-{i}"),
"note": "x".repeat(120)
},
"items": [],
"ok": true,
"missing": null
}),
);
} else {
obj.insert(
format!("field_{i:02}"),
json!(format!(
"this is a relatively long value string for entry number {i} with content"
)),
);
}
}
obj.insert("flag".to_string(), json!(false));
obj.insert("empty".to_string(), json!([]));
obj.insert("nada".to_string(), json!(null));
obj
}
fn assert_object_fully_preserved(obj: &Map<String, Value>, config: &SmartCrusherConfig) {
let (out, strat) = crush_object(obj, config, 1.0);
assert_eq!(out, *obj, "crush_object must keep every property");
assert_eq!(out.len(), obj.len());
for key in obj.keys() {
assert!(out.contains_key(key), "dropped key {key}");
assert_eq!(out.get(key), obj.get(key));
}
assert_eq!(strat, "object:passthrough");
assert!(
!strat.contains("adaptive"),
"must not report a key-drop strategy: {strat}"
);
assert!(
!strat.contains("ccr"),
"must not invent a CCR hash: {strat}"
);
}
#[test]
fn object_passthrough_when_few_keys() {
let mut obj = Map::new();
@@ -755,25 +699,29 @@ mod tests {
}
let (out, strat) = crush_object(&obj, &cfg(), 1.0);
assert_eq!(out.len(), 5);
assert_eq!(out, obj);
assert_eq!(strat, "object:passthrough");
}
#[test]
fn object_passthrough_when_total_tokens_below_min() {
// Many tiny keys/values: total_tokens stays below
// min_tokens_to_crush=200.
// Many tiny keys/values: historically total_tokens stayed below
// min_tokens_to_crush=200 and skipped selection. Preservation is
// now unconditional, so every key still survives.
let mut obj = Map::new();
for i in 0..30 {
obj.insert(format!("k{}", i), json!(i));
}
let (_out, strat) = crush_object(&obj, &cfg(), 1.0);
let (out, strat) = crush_object(&obj, &cfg(), 1.0);
assert_eq!(out, obj);
assert_eq!(strat, "object:passthrough");
}
#[test]
fn object_crushes_when_token_budget_exceeded() {
// 30 keys, each with a long string value → total tokens > 200,
// and unique k_total < n → actual crushing happens.
fn object_preserves_every_key_when_token_budget_exceeded() {
// 30 keys, each with a long string value → historically above
// the token floor and subject to key-selection. Every property
// must survive; no adaptive key-drop strategy.
let mut obj = Map::new();
for i in 0..30 {
obj.insert(
@@ -784,20 +732,27 @@ mod tests {
)),
);
}
let (out, strat) = crush_object(&obj, &cfg(), 1.0);
// Either the optimizer kept all (if it deems them all distinct
// enough — strategy = passthrough), or it crushed.
if strat == "object:passthrough" {
assert_eq!(out.len(), 30);
} else {
assert!(strat.starts_with("object:adaptive("));
assert!(out.len() <= 30);
assert_object_fully_preserved(&obj, &cfg());
}
#[test]
fn object_preserves_wide_object_with_mixed_value_types() {
assert_object_fully_preserved(&wide_object(7), &cfg());
}
#[test]
fn object_preserves_expensive_fields_in_any_position() {
// Reorder the expensive nested field into beginning, middle, and
// end. Boundary/stride selection used to drop the middle one.
for expensive_index in [0, 7, 14] {
assert_object_fully_preserved(&wide_object(expensive_index), &cfg());
assert_object_fully_preserved(&wide_object(expensive_index), &lossless_only_cfg());
}
}
#[test]
fn object_keeps_small_values() {
// Mix of small + large values; small ones (<=12 tokens) always survive.
// Mix of small + large values; every key survives, including tiny.
let mut obj = Map::new();
obj.insert("tiny".to_string(), json!(1));
for i in 0..30 {
@@ -810,6 +765,7 @@ mod tests {
);
}
let (out, _) = crush_object(&obj, &cfg(), 1.0);
assert_eq!(out, obj);
assert!(
out.contains_key("tiny"),
"tiny key (small value) must survive"
@@ -830,6 +786,7 @@ mod tests {
);
}
let (out, _) = crush_object(&obj, &cfg(), 1.0);
assert_eq!(out, obj);
assert!(
out.contains_key("msg1"),
"key with error-keyword value must survive"
@@ -887,12 +844,10 @@ mod tests {
);
}
let (lossy, _) = crush_object(&obj, &cfg(), 1.0);
assert!(lossy.len() < obj.len(), "default config should drop keys");
let (out, strat) = crush_object(&obj, &lossless_only_cfg(), 1.0);
assert_eq!(out, obj, "lossless_only must keep every key");
assert_eq!(strat, "object:lossless_only");
// Default mode used to drop keys with no CCR marker. Preservation
// is now unconditional, so both configs keep every property.
assert_object_fully_preserved(&obj, &cfg());
assert_object_fully_preserved(&obj, &lossless_only_cfg());
}
// ---------- BUG #1 documentation test ----------
+251
View File
@@ -69,3 +69,254 @@ def test_compact_json_tool_result_compresses() -> None:
)
after = tok.count_messages(result.messages)
assert after < before * 0.9, (before, after, result.transforms_applied[:5])
# ---------------------------------------------------------------------------
# Issue #3634: SmartCrusher must not drop object fields (tags/subtasks)
# ---------------------------------------------------------------------------
# Exact MCP `saga-mcp_task_get` payload from the issue (813 bytes, original
# key order, tags encoded as a JSON string). Do not pretty-print or reorder.
MCP_TASK_GET_JSON = (
'{"id":79,"epic_id":9,"title":"test","description":null,'
'"status":"done","priority":"medium","sort_order":0,'
'"assigned_to":null,"estimated_hours":null,"actual_hours":null,'
'"due_date":null,"source_ref":null,"metadata":"{}",'
'"created_at":"2026-09-15 20:32:06","updated_at":"2026-09-16 11:59:27",'
'"description_locked":0,"is_deleted":0,"deleted_at":null,'
'"deleted_by":null,"delete_reason":null,"epic_name":"latency-routing",'
'"tags":"[\\"cherry-pick\\",\\"dedicated branch\\"]",'
'"subtasks":[{"id":163,"task_id":79,"title":"test2","status":"todo",'
'"sort_order":1,"created_at":"2026-09-15 20:32:13",'
'"updated_at":"2026-09-16 14:02:26"},'
'{"id":165,"task_id":79,"title":"test4","status":"todo","sort_order":2,'
'"created_at":"2026-09-15 20:38:47","updated_at":"2026-09-17 20:29:04"}],'
'"notes":[],"comments":[],"depends_on":[],"dependents":[]}'
)
_TOOL_CALL_ID = "call_task_get_79"
_TOOL_NAME = "saga-mcp_task_get"
def _offline_tokenizer():
tokenizer = pytest.importorskip("headroom.tokenizers.estimator")
from headroom.tokenizer import Tokenizer
return Tokenizer(tokenizer.EstimatingTokenCounter())
def _object_preserving_router() -> ContentRouter:
# Real ContentRouter + Rust SmartCrusher. Disable unrelated ML / log
# fallbacks so a no-savings crush cannot be rewritten by Kompress.
return ContentRouter(
ContentRouterConfig(
skip_user_messages=False,
enable_kompress=False,
enable_log_compressor=False,
enable_search_compressor=False,
relevance_split=False,
protect_analysis_context=False,
exclude_tools=set(),
)
)
def _openai_task_messages(payload: str) -> list[dict]:
return [
{"role": "user", "content": "read task 79"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": _TOOL_CALL_ID,
"type": "function",
"function": {"name": _TOOL_NAME, "arguments": '{"id":79}'},
}
],
},
{
"role": "tool",
"tool_call_id": _TOOL_CALL_ID,
"content": payload,
},
]
def _anthropic_task_messages(payload: str) -> list[dict]:
return [
{"role": "user", "content": "read task 79"},
{
"role": "assistant",
"content": [
{
"type": "tool_use",
"id": _TOOL_CALL_ID,
"name": _TOOL_NAME,
"input": {"id": 79},
}
],
},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": _TOOL_CALL_ID,
"content": payload,
}
],
},
]
def _apply(messages: list[dict]):
router = _object_preserving_router()
tok = _offline_tokenizer()
return router.apply(
[json.loads(json.dumps(m)) for m in messages],
tok,
context="read task 79",
frozen_message_count=0,
protect_recent=0,
protect_analysis_context=False,
)
def _openai_tool_content(messages: list[dict]) -> str:
for msg in messages:
if msg.get("role") == "tool" and msg.get("tool_call_id") == _TOOL_CALL_ID:
content = msg.get("content")
assert isinstance(content, str), content
return content
raise AssertionError(f"OpenAI tool message {_TOOL_CALL_ID} missing")
def _anthropic_tool_content(messages: list[dict]) -> str:
for msg in messages:
content = msg.get("content")
if not isinstance(content, list):
continue
for block in content:
if block.get("type") == "tool_result" and block.get("tool_use_id") == _TOOL_CALL_ID:
inner = block.get("content")
assert isinstance(inner, str), inner
return inner
raise AssertionError(f"Anthropic tool_result {_TOOL_CALL_ID} missing")
def _assert_task_payload_preserved(original: str, compressed: str) -> dict:
assert "<<ccr:" not in compressed, compressed[:200]
parsed = json.loads(compressed)
expected = json.loads(original)
assert parsed == expected, (sorted(expected), sorted(parsed))
assert "tags" in parsed
assert "subtasks" in parsed
assert parsed["tags"] == '["cherry-pick","dedicated branch"]'
assert len(parsed["subtasks"]) == 2
assert parsed["subtasks"][0]["title"] == "test2"
assert parsed["subtasks"][1]["title"] == "test4"
return parsed
def _wide_renamed_task_json() -> str:
"""Renamed, oversized variant so a below-threshold skip cannot hide key drop.
Field names are arbitrary (not tags/subtasks) and expensive values sit at
the beginning, middle, and end so a boundary/stride selector cannot
accidentally keep them all.
"""
original = json.loads(MCP_TASK_GET_JSON)
expensive = {
"nested": {"label": "child", "note": "n" * 180},
"items": [],
"ok": True,
"missing": None,
}
wide: dict = {
"leading_blob": expensive,
"label_bundle": original.pop("tags"),
"child_records": original.pop("subtasks"),
}
wide.update(original)
for i in range(20):
wide[f"extra_field_{i:02d}"] = (
f"this is a relatively long value string for entry number {i} with content"
)
wide["trailing_blob"] = {
"nested": {"label": "tail", "note": "t" * 180},
"items": [],
"ok": False,
"missing": None,
}
payload = json.dumps(wide, separators=(",", ":"))
assert len(payload) > 1500
assert len(wide) > 8
return payload
def test_mcp_task_fixture_is_the_issue_payload() -> None:
assert len(MCP_TASK_GET_JSON) == 813
parsed = json.loads(MCP_TASK_GET_JSON)
assert list(parsed)[21] == "tags"
assert list(parsed)[22] == "subtasks"
assert parsed["tags"] == '["cherry-pick","dedicated branch"]'
def test_mcp_task_object_preserved_openai_tool_role() -> None:
messages = _openai_task_messages(MCP_TASK_GET_JSON)
result = _apply(messages)
content = _openai_tool_content(result.messages)
assert content == MCP_TASK_GET_JSON
_assert_task_payload_preserved(MCP_TASK_GET_JSON, content)
# Tool-call pairing stays intact.
tool_msg = next(m for m in result.messages if m.get("role") == "tool")
assistant = next(m for m in result.messages if m.get("role") == "assistant")
assert tool_msg["tool_call_id"] == _TOOL_CALL_ID
assert assistant["tool_calls"][0]["id"] == _TOOL_CALL_ID
assert assistant["tool_calls"][0]["function"]["name"] == _TOOL_NAME
assert not any("object:adaptive" in t for t in result.transforms_applied)
def test_mcp_task_object_preserved_anthropic_tool_result() -> None:
messages = _anthropic_task_messages(MCP_TASK_GET_JSON)
result = _apply(messages)
content = _anthropic_tool_content(result.messages)
assert content == MCP_TASK_GET_JSON
_assert_task_payload_preserved(MCP_TASK_GET_JSON, content)
assistant = next(m for m in result.messages if m.get("role") == "assistant")
assert assistant["content"][0]["id"] == _TOOL_CALL_ID
user_blocks = next(
m["content"]
for m in result.messages
if m.get("role") == "user" and isinstance(m.get("content"), list)
)
assert user_blocks[0]["tool_use_id"] == _TOOL_CALL_ID
assert not any("object:adaptive" in t for t in result.transforms_applied)
def test_wide_renamed_object_fields_preserved_openai_tool_role() -> None:
payload = _wide_renamed_task_json()
result = _apply(_openai_task_messages(payload))
content = _openai_tool_content(result.messages)
assert "<<ccr:" not in content
parsed = json.loads(content)
expected = json.loads(payload)
assert parsed == expected
assert "label_bundle" in parsed
assert "child_records" in parsed
assert "leading_blob" in parsed
assert "trailing_blob" in parsed
assert parsed["label_bundle"] == '["cherry-pick","dedicated branch"]'
assert len(parsed["child_records"]) == 2
assert parsed["leading_blob"]["nested"]["note"] == "n" * 180
assert parsed["trailing_blob"]["nested"]["note"] == "t" * 180
def test_malformed_json_tool_result_passes_through() -> None:
payload = "{" + "not-json, still a tool result. " * 40
assert len(payload) > 500
result = _apply(_openai_task_messages(payload))
content = _openai_tool_content(result.messages)
assert content == payload
assert "<<ccr:" not in content