diff --git a/docs/BENCHMARK_CAMPAIGN.md b/docs/BENCHMARK_CAMPAIGN.md index 111c7396c..eb013a3c8 100644 --- a/docs/BENCHMARK_CAMPAIGN.md +++ b/docs/BENCHMARK_CAMPAIGN.md @@ -107,12 +107,15 @@ Every invocation also writes: run directories; - `reports/summary.md`, regenerated from validated completion records only. -The report lists exact canonical response bytes and a clearly labeled deterministic -`ceil(UTF-8 bytes / 4)` token estimate. Pareto membership is restricted to candidates -that pass every applicable quality/correctness gate and have query latency, response -tokens, incremental latency, and peak RSS measurements. It maximizes quality while -minimizing those cost axes. Exact bytes remain visible so the token estimate is never -presented as tokenizer ground truth. +The report lists exact bytes from each tool's default response encoding and a clearly +labeled deterministic `ceil(UTF-8 bytes / 4)` token estimate. Each quality oracle makes +a second request with `format=json`; its latency and canonical JSON size are recorded +separately as `quality_probe_elapsed_ms` and `quality_response_bytes`, so parsing the +oracle cannot silently replace or inflate the default user-facing measurement. Pareto +membership is restricted to candidates that pass every applicable quality/correctness +gate and have query latency, response tokens, incremental latency, and peak RSS +measurements. It maximizes quality while minimizing those cost axes. Exact bytes remain +visible so the token estimate is never presented as tokenizer ground truth. Use `--audit-only` to scan and regenerate the report without running missing cells. Use `--minimum-free-gb` and `--stale-lock-hours` only when the recorded defaults are diff --git a/scripts/benchmark-incremental-speed.py b/scripts/benchmark-incremental-speed.py index c29f5404a..3796864a1 100755 --- a/scripts/benchmark-incremental-speed.py +++ b/scripts/benchmark-incremental-speed.py @@ -269,6 +269,20 @@ def unwrap_mcp_result(response: dict[str, Any]) -> dict[str, Any]: return result +def cli_result_text(stdout: str) -> str: + outer = json.loads(stdout) + if "content" in outer: + return str(outer["content"][0]["text"]) + return json.dumps(outer, separators=(",", ":"), sort_keys=True) + + +def mcp_result_text(response: dict[str, Any]) -> str: + result = response.get("result", {}) + if "content" in result: + return str(result["content"][0]["text"]) + return json.dumps(result, separators=(",", ":"), sort_keys=True) + + TOKEN_ESTIMATOR = "utf8_bytes_div_4_ceil" @@ -398,13 +412,19 @@ class McpClient: def call_tool( self, name: str, arguments: dict[str, Any] ) -> tuple[dict[str, Any], str, int, float]: + text, stderr, stdout_bytes, elapsed_ms = self.call_tool_text(name, arguments) + return json.loads(text), stderr, stdout_bytes, elapsed_ms + + def call_tool_text( + self, name: str, arguments: dict[str, Any] + ) -> tuple[str, str, int, float]: mark = self._stderr_mark() start = now_ms() response = self._request("tools/call", {"name": name, "arguments": arguments}) elapsed_ms = now_ms() - start stderr = self._stderr_since(mark) stdout_bytes = len(json.dumps(response, separators=(",", ":")).encode("utf-8")) - return unwrap_mcp_result(response), stderr, stdout_bytes, elapsed_ms + return mcp_result_text(response), stderr, stdout_bytes, elapsed_ms def log_tail(stderr: str) -> list[str]: @@ -725,8 +745,10 @@ def build_tool_call_result( stdout_bytes: int, elapsed_ms: float, include_logs: bool, + response_payload: bytes | None = None, ) -> dict[str, Any]: - payload = canonical_response_bytes(data) + quality_payload = canonical_response_bytes(data) + payload = response_payload if response_payload is not None else quality_payload result: dict[str, Any] = { "elapsed_ms": round(elapsed_ms, 3), # Preserve the historical field while separating transport framing from @@ -736,6 +758,8 @@ def build_tool_call_result( "response_bytes": len(payload), "response_token_estimate": estimate_response_tokens(payload), "token_estimator": TOKEN_ESTIMATOR, + "response_encoding": "tool_default" if response_payload is not None else "canonical_json", + "quality_response_bytes": len(quality_payload), "response": data, "freshness_state": response_freshness_state(data) or None, "freshness": response_freshness(data), @@ -754,14 +778,29 @@ def run_cli_tool_call( timeout: int, include_logs: bool, ) -> dict[str, Any]: - cmd = [str(binary), "cli", "--json", tool_name, json.dumps(arguments, separators=(",", ":"))] + encoded = json.dumps(arguments, separators=(",", ":")) + cmd = [str(binary), "cli", "--json", tool_name, encoded] proc, elapsed_ms = command_result(cmd, env, timeout) if proc.returncode != 0: raise command_failure(f"{tool_name}_call", cmd, env, proc, elapsed_ms) - data = unwrap_cli_json(proc.stdout) - return build_tool_call_result( - data, proc.stderr, len(proc.stdout.encode("utf-8")), elapsed_ms, include_logs + raw_payload = cli_result_text(proc.stdout).encode("utf-8") + quality_arguments = dict(arguments) + quality_arguments["format"] = "json" + quality_cmd = [ + str(binary), "cli", "--json", tool_name, + json.dumps(quality_arguments, separators=(",", ":")), + ] + quality_proc, quality_elapsed_ms = command_result(quality_cmd, env, timeout) + if quality_proc.returncode != 0: + raise command_failure( + f"{tool_name}_quality_call", quality_cmd, env, quality_proc, quality_elapsed_ms + ) + data = unwrap_cli_json(quality_proc.stdout) + result = build_tool_call_result( + data, proc.stderr, len(proc.stdout.encode("utf-8")), elapsed_ms, include_logs, raw_payload ) + result["quality_probe_elapsed_ms"] = round(quality_elapsed_ms, 3) + return result def run_mcp_tool_call( @@ -770,8 +809,15 @@ def run_mcp_tool_call( arguments: dict[str, Any], include_logs: bool, ) -> dict[str, Any]: - data, stderr, stdout_bytes, elapsed_ms = client.call_tool(tool_name, arguments) - return build_tool_call_result(data, stderr, stdout_bytes, elapsed_ms, include_logs) + raw_text, stderr, stdout_bytes, elapsed_ms = client.call_tool_text(tool_name, arguments) + quality_arguments = dict(arguments) + quality_arguments["format"] = "json" + data, _, _, quality_elapsed_ms = client.call_tool(tool_name, quality_arguments) + result = build_tool_call_result( + data, stderr, stdout_bytes, elapsed_ms, include_logs, raw_text.encode("utf-8") + ) + result["quality_probe_elapsed_ms"] = round(quality_elapsed_ms, 3) + return result def run_tool_call_for_transport( diff --git a/tests/test_benchmark_incremental_speed.py b/tests/test_benchmark_incremental_speed.py index 7620a5b17..9cb640e62 100644 --- a/tests/test_benchmark_incremental_speed.py +++ b/tests/test_benchmark_incremental_speed.py @@ -12,17 +12,56 @@ SPEC.loader.exec_module(BENCHMARK) class BenchmarkIncrementalSpeedTest(unittest.TestCase): - def test_tool_result_measures_canonical_payload_not_transport_envelope(self) -> None: + def test_tool_result_separates_default_payload_quality_json_and_transport(self) -> None: + default_payload = b"total: 1\nresults[1]{name}:\n alpha\n" result = BENCHMARK.build_tool_call_result( - {"name": "alpha", "items": [1, 2]}, "", 999, 12.5, False + {"name": "alpha", "items": [1, 2]}, "", 999, 12.5, False, + default_payload, ) canonical = b'{"items":[1,2],"name":"alpha"}' self.assertEqual(result["transport_response_bytes"], 999) - self.assertEqual(result["response_bytes"], len(canonical)) + self.assertEqual(result["response_bytes"], len(default_payload)) + self.assertEqual(result["quality_response_bytes"], len(canonical)) self.assertEqual( - result["response_token_estimate"], BENCHMARK.estimate_response_tokens(canonical) + result["response_token_estimate"], BENCHMARK.estimate_response_tokens(default_payload) ) self.assertEqual(result["token_estimator"], "utf8_bytes_div_4_ceil") + self.assertEqual(result["response_encoding"], "tool_default") + + def test_result_text_extractors_preserve_default_toon(self) -> None: + toon = "total: 1\nresults[1]{name}:\n alpha\n" + cli_stdout = '{"content":[{"type":"text","text":"total: 1\\nresults[1]{name}:\\n alpha\\n"}]}' + mcp_response = {"result": {"content": [{"type": "text", "text": toon}]}} + self.assertEqual(BENCHMARK.cli_result_text(cli_stdout), toon) + self.assertEqual(BENCHMARK.mcp_result_text(mcp_response), toon) + + def test_mcp_tool_call_measures_default_payload_and_uses_json_for_quality(self) -> None: + class FakeClient: + def call_tool_text(self, name, arguments): + self.default_call = (name, arguments) + return "total: 1\nresults[1]{name}:\n alpha\n", "default log", 321, 7.25 + + def call_tool(self, name, arguments): + self.quality_call = (name, arguments) + return {"results": [{"name": "alpha"}]}, "quality log", 654, 2.5 + + client = FakeClient() + result = BENCHMARK.run_mcp_tool_call( + client, "search_graph", {"name_pattern": "alpha"}, False + ) + + self.assertEqual( + client.default_call, ("search_graph", {"name_pattern": "alpha"}) + ) + self.assertEqual( + client.quality_call, + ("search_graph", {"name_pattern": "alpha", "format": "json"}), + ) + self.assertEqual(result["elapsed_ms"], 7.25) + self.assertEqual(result["quality_probe_elapsed_ms"], 2.5) + self.assertEqual(result["transport_response_bytes"], 321) + self.assertEqual(result["response_encoding"], "tool_default") + self.assertEqual(result["response"]["results"][0]["name"], "alpha") def test_quality_summary_requires_every_applicable_oracle(self) -> None: oracles = {