mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
refactor(integrations): share tool-wrapper metrics between langchain, crewai, autogen
ToolCompressionMetrics, ToolMetricsCollector and the body of _record_metrics were copied verbatim into the LangChain, CrewAI and AutoGen tool wrappers, and the UTC timestamp fix (#3117) had to land in each copy. OptimizationMetrics was likewise byte-identical in the LangChain, Agno and Strands model wrappers. They now live once in headroom/integrations/_metrics.py and every module re-exports them, so all existing import paths keep working. The shared helper is private (_record_tool_metrics), so no new public name lands in the agents modules. The per-framework classes are now one class: isinstance and == across frameworks now return True, and __module__ is headroom.integrations._metrics. Each module keeps its own _global_metrics, get_tool_metrics, reset_tool_metrics and compress_tool_result call (tests patch it per module), and passes its own logger so the savings log line keeps its logger name and text. The duplicated CrewAI and AutoGen collector tests go; tests/test_integrations/langchain/ test_agents.py covers the shared classes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assisted-by: claude-code:claude-opus-5-5 Headline-Session: 6e77a807-8dd4-4a77-a2fa-ddce6dc2515e Headline-AI-Share: 80% strict / 80% weighted
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a9815bbb5f
commit
8b12da71c9
@@ -0,0 +1,165 @@
|
||||
"""Metric types shared by the framework integrations.
|
||||
|
||||
The LangChain, CrewAI and AutoGen tool wrappers record the same per-tool
|
||||
compression metrics, and the LangChain, Agno and Strands model wrappers record
|
||||
the same per-call optimization metrics. Each integration re-exports these names
|
||||
from its own module, so ``headroom.integrations.<framework>`` keeps working.
|
||||
|
||||
Stdlib-only: importing this module must not pull in any agent framework.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class OptimizationMetrics:
|
||||
"""Metrics from a single optimization pass."""
|
||||
|
||||
request_id: str
|
||||
timestamp: datetime
|
||||
tokens_before: int
|
||||
tokens_after: int
|
||||
tokens_saved: int
|
||||
savings_percent: float
|
||||
transforms_applied: list[str]
|
||||
model: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCompressionMetrics:
|
||||
"""Metrics from a single tool compression.
|
||||
|
||||
Attributes:
|
||||
tool_name: Name of the tool that was invoked.
|
||||
timestamp: When the compression occurred.
|
||||
chars_before: Character count of the original output.
|
||||
chars_after: Character count after compression.
|
||||
chars_saved: Characters removed by compression.
|
||||
compression_ratio: Ratio of compressed to original size.
|
||||
was_compressed: Whether compression was actually applied.
|
||||
"""
|
||||
|
||||
tool_name: str
|
||||
timestamp: datetime
|
||||
chars_before: int
|
||||
chars_after: int
|
||||
chars_saved: int
|
||||
compression_ratio: float
|
||||
was_compressed: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolMetricsCollector:
|
||||
"""Collects compression metrics across all tool invocations.
|
||||
|
||||
Attributes:
|
||||
metrics: List of per-invocation metrics.
|
||||
"""
|
||||
|
||||
metrics: list[ToolCompressionMetrics] = field(default_factory=list)
|
||||
|
||||
def add(self, metric: ToolCompressionMetrics) -> None:
|
||||
"""Add a metric entry.
|
||||
|
||||
Args:
|
||||
metric: The compression metrics to record.
|
||||
"""
|
||||
self.metrics.append(metric)
|
||||
# Keep only last 1000
|
||||
if len(self.metrics) > 1000:
|
||||
self.metrics = self.metrics[-1000:]
|
||||
|
||||
def get_summary(self) -> dict[str, Any]:
|
||||
"""Get summary statistics.
|
||||
|
||||
Returns:
|
||||
Dict with total_invocations, total_compressions,
|
||||
total_chars_saved, average_compression_ratio, and
|
||||
per-tool breakdown.
|
||||
"""
|
||||
if not self.metrics:
|
||||
return {
|
||||
"total_invocations": 0,
|
||||
"total_compressions": 0,
|
||||
"total_chars_saved": 0,
|
||||
}
|
||||
|
||||
compressed = [m for m in self.metrics if m.was_compressed]
|
||||
return {
|
||||
"total_invocations": len(self.metrics),
|
||||
"total_compressions": len(compressed),
|
||||
"total_chars_saved": sum(m.chars_saved for m in self.metrics),
|
||||
"average_compression_ratio": (
|
||||
sum(m.compression_ratio for m in compressed) / len(compressed) if compressed else 0
|
||||
),
|
||||
"by_tool": self._get_by_tool_stats(),
|
||||
}
|
||||
|
||||
def _get_by_tool_stats(self) -> dict[str, dict[str, Any]]:
|
||||
"""Get per-tool statistics."""
|
||||
by_tool: dict[str, list[ToolCompressionMetrics]] = {}
|
||||
for m in self.metrics:
|
||||
if m.tool_name not in by_tool:
|
||||
by_tool[m.tool_name] = []
|
||||
by_tool[m.tool_name].append(m)
|
||||
|
||||
result = {}
|
||||
for name, tool_metrics in by_tool.items():
|
||||
compressed = [m for m in tool_metrics if m.was_compressed]
|
||||
result[name] = {
|
||||
"invocations": len(tool_metrics),
|
||||
"compressions": len(compressed),
|
||||
"chars_saved": sum(m.chars_saved for m in tool_metrics),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def _record_tool_metrics(
|
||||
collector: ToolMetricsCollector,
|
||||
tool_name: str,
|
||||
original: str,
|
||||
compressed: str,
|
||||
was_compressed: bool,
|
||||
log: logging.Logger,
|
||||
) -> None:
|
||||
"""Record one tool invocation in ``collector`` and log real savings.
|
||||
|
||||
Args:
|
||||
collector: The metrics collector.
|
||||
tool_name: Name of the tool.
|
||||
original: Original output.
|
||||
compressed: Compressed output.
|
||||
was_compressed: Whether compression was applied.
|
||||
log: The calling integration's logger, so the savings line keeps
|
||||
that integration's logger name.
|
||||
"""
|
||||
chars_before = len(original)
|
||||
chars_after = len(compressed)
|
||||
chars_saved = chars_before - chars_after
|
||||
|
||||
metric = ToolCompressionMetrics(
|
||||
tool_name=tool_name,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
chars_before=chars_before,
|
||||
chars_after=chars_after,
|
||||
chars_saved=max(0, chars_saved),
|
||||
compression_ratio=chars_after / chars_before if chars_before > 0 else 1.0,
|
||||
was_compressed=was_compressed and chars_saved > 0,
|
||||
)
|
||||
|
||||
collector.add(metric)
|
||||
|
||||
if was_compressed and chars_saved > 0:
|
||||
log.info(
|
||||
"HeadroomToolWrapper[%s]: %d -> %d chars (%d saved, %.1f%% of original)",
|
||||
tool_name,
|
||||
chars_before,
|
||||
chars_after,
|
||||
chars_saved,
|
||||
metric.compression_ratio * 100,
|
||||
)
|
||||
@@ -30,6 +30,7 @@ except ImportError:
|
||||
ModelResponse = dict # type: ignore[misc,assignment]
|
||||
|
||||
from headroom import HeadroomConfig, HeadroomMode
|
||||
from headroom.integrations._metrics import OptimizationMetrics
|
||||
from headroom.parser import _coerce_tool_call_to_dict
|
||||
from headroom.providers import OpenAIProvider
|
||||
from headroom.transforms import TransformPipeline
|
||||
@@ -50,20 +51,6 @@ def agno_available() -> bool:
|
||||
return AGNO_AVAILABLE
|
||||
|
||||
|
||||
@dataclass
|
||||
class OptimizationMetrics:
|
||||
"""Metrics from a single optimization pass."""
|
||||
|
||||
request_id: str
|
||||
timestamp: datetime
|
||||
tokens_before: int
|
||||
tokens_after: int
|
||||
tokens_saved: int
|
||||
savings_percent: float
|
||||
transforms_applied: list[str]
|
||||
model: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class HeadroomAgnoModel(Model): # type: ignore[misc]
|
||||
"""Agno model wrapper that applies Headroom optimizations.
|
||||
|
||||
@@ -35,8 +35,6 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import functools
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
@@ -47,6 +45,12 @@ except ImportError:
|
||||
AUTOGEN_AVAILABLE = False
|
||||
FunctionTool = object # type: ignore[misc,assignment]
|
||||
|
||||
# Re-exported: callers import ToolCompressionMetrics from this module.
|
||||
from headroom.integrations._metrics import ( # noqa: F401
|
||||
ToolCompressionMetrics,
|
||||
ToolMetricsCollector,
|
||||
_record_tool_metrics,
|
||||
)
|
||||
from headroom.integrations.mcp import compress_tool_result
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -60,94 +64,6 @@ def _check_autogen_available() -> None:
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCompressionMetrics:
|
||||
"""Metrics from a single tool compression.
|
||||
|
||||
Attributes:
|
||||
tool_name: Name of the tool that was invoked.
|
||||
timestamp: When the compression occurred.
|
||||
chars_before: Character count of the original output.
|
||||
chars_after: Character count after compression.
|
||||
chars_saved: Characters removed by compression.
|
||||
compression_ratio: Ratio of compressed to original size.
|
||||
was_compressed: Whether compression was actually applied.
|
||||
"""
|
||||
|
||||
tool_name: str
|
||||
timestamp: datetime
|
||||
chars_before: int
|
||||
chars_after: int
|
||||
chars_saved: int
|
||||
compression_ratio: float
|
||||
was_compressed: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolMetricsCollector:
|
||||
"""Collects compression metrics across all tool invocations.
|
||||
|
||||
Attributes:
|
||||
metrics: List of per-invocation metrics.
|
||||
"""
|
||||
|
||||
metrics: list[ToolCompressionMetrics] = field(default_factory=list)
|
||||
|
||||
def add(self, metric: ToolCompressionMetrics) -> None:
|
||||
"""Add a metric entry.
|
||||
|
||||
Args:
|
||||
metric: The compression metrics to record.
|
||||
"""
|
||||
self.metrics.append(metric)
|
||||
if len(self.metrics) > 1000:
|
||||
self.metrics = self.metrics[-1000:]
|
||||
|
||||
def get_summary(self) -> dict[str, Any]:
|
||||
"""Get summary statistics.
|
||||
|
||||
Returns:
|
||||
Dict with total_invocations, total_compressions,
|
||||
total_chars_saved, average_compression_ratio, and
|
||||
per-tool breakdown.
|
||||
"""
|
||||
if not self.metrics:
|
||||
return {
|
||||
"total_invocations": 0,
|
||||
"total_compressions": 0,
|
||||
"total_chars_saved": 0,
|
||||
}
|
||||
|
||||
compressed = [m for m in self.metrics if m.was_compressed]
|
||||
return {
|
||||
"total_invocations": len(self.metrics),
|
||||
"total_compressions": len(compressed),
|
||||
"total_chars_saved": sum(m.chars_saved for m in self.metrics),
|
||||
"average_compression_ratio": (
|
||||
sum(m.compression_ratio for m in compressed) / len(compressed) if compressed else 0
|
||||
),
|
||||
"by_tool": self._get_by_tool_stats(),
|
||||
}
|
||||
|
||||
def _get_by_tool_stats(self) -> dict[str, dict[str, Any]]:
|
||||
"""Get per-tool statistics."""
|
||||
by_tool: dict[str, list[ToolCompressionMetrics]] = {}
|
||||
for m in self.metrics:
|
||||
if m.tool_name not in by_tool:
|
||||
by_tool[m.tool_name] = []
|
||||
by_tool[m.tool_name].append(m)
|
||||
|
||||
result = {}
|
||||
for name, tool_metrics in by_tool.items():
|
||||
compressed = [m for m in tool_metrics if m.was_compressed]
|
||||
result[name] = {
|
||||
"invocations": len(tool_metrics),
|
||||
"compressions": len(compressed),
|
||||
"chars_saved": sum(m.chars_saved for m in tool_metrics),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
# Global metrics collector
|
||||
_global_metrics = ToolMetricsCollector()
|
||||
|
||||
@@ -220,31 +136,7 @@ def _record_metrics(
|
||||
compressed: Compressed output.
|
||||
was_compressed: Whether compression was applied.
|
||||
"""
|
||||
chars_before = len(original)
|
||||
chars_after = len(compressed)
|
||||
chars_saved = chars_before - chars_after
|
||||
|
||||
metric = ToolCompressionMetrics(
|
||||
tool_name=tool_name,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
chars_before=chars_before,
|
||||
chars_after=chars_after,
|
||||
chars_saved=max(0, chars_saved),
|
||||
compression_ratio=chars_after / chars_before if chars_before > 0 else 1.0,
|
||||
was_compressed=was_compressed and chars_saved > 0,
|
||||
)
|
||||
|
||||
collector.add(metric)
|
||||
|
||||
if was_compressed and chars_saved > 0:
|
||||
logger.info(
|
||||
"HeadroomToolWrapper[%s]: %d -> %d chars (%d saved, %.1f%% of original)",
|
||||
tool_name,
|
||||
chars_before,
|
||||
chars_after,
|
||||
chars_saved,
|
||||
metric.compression_ratio * 100,
|
||||
)
|
||||
_record_tool_metrics(collector, tool_name, original, compressed, was_compressed, logger)
|
||||
|
||||
|
||||
class HeadroomToolWrapper:
|
||||
|
||||
@@ -25,8 +25,6 @@ Example:
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
@@ -37,6 +35,12 @@ except ImportError:
|
||||
CREWAI_AVAILABLE = False
|
||||
BaseTool = object # type: ignore[misc,assignment]
|
||||
|
||||
# Re-exported: callers import ToolCompressionMetrics from this module.
|
||||
from headroom.integrations._metrics import ( # noqa: F401
|
||||
ToolCompressionMetrics,
|
||||
ToolMetricsCollector,
|
||||
_record_tool_metrics,
|
||||
)
|
||||
from headroom.integrations.mcp import compress_tool_result
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -50,94 +54,6 @@ def _check_crewai_available() -> None:
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCompressionMetrics:
|
||||
"""Metrics from a single tool compression.
|
||||
|
||||
Attributes:
|
||||
tool_name: Name of the tool that was invoked.
|
||||
timestamp: When the compression occurred.
|
||||
chars_before: Character count of the original output.
|
||||
chars_after: Character count after compression.
|
||||
chars_saved: Characters removed by compression.
|
||||
compression_ratio: Ratio of compressed to original size.
|
||||
was_compressed: Whether compression was actually applied.
|
||||
"""
|
||||
|
||||
tool_name: str
|
||||
timestamp: datetime
|
||||
chars_before: int
|
||||
chars_after: int
|
||||
chars_saved: int
|
||||
compression_ratio: float
|
||||
was_compressed: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolMetricsCollector:
|
||||
"""Collects compression metrics across all tool invocations.
|
||||
|
||||
Attributes:
|
||||
metrics: List of per-invocation metrics.
|
||||
"""
|
||||
|
||||
metrics: list[ToolCompressionMetrics] = field(default_factory=list)
|
||||
|
||||
def add(self, metric: ToolCompressionMetrics) -> None:
|
||||
"""Add a metric entry.
|
||||
|
||||
Args:
|
||||
metric: The compression metrics to record.
|
||||
"""
|
||||
self.metrics.append(metric)
|
||||
if len(self.metrics) > 1000:
|
||||
self.metrics = self.metrics[-1000:]
|
||||
|
||||
def get_summary(self) -> dict[str, Any]:
|
||||
"""Get summary statistics.
|
||||
|
||||
Returns:
|
||||
Dict with total_invocations, total_compressions,
|
||||
total_chars_saved, average_compression_ratio, and
|
||||
per-tool breakdown.
|
||||
"""
|
||||
if not self.metrics:
|
||||
return {
|
||||
"total_invocations": 0,
|
||||
"total_compressions": 0,
|
||||
"total_chars_saved": 0,
|
||||
}
|
||||
|
||||
compressed = [m for m in self.metrics if m.was_compressed]
|
||||
return {
|
||||
"total_invocations": len(self.metrics),
|
||||
"total_compressions": len(compressed),
|
||||
"total_chars_saved": sum(m.chars_saved for m in self.metrics),
|
||||
"average_compression_ratio": (
|
||||
sum(m.compression_ratio for m in compressed) / len(compressed) if compressed else 0
|
||||
),
|
||||
"by_tool": self._get_by_tool_stats(),
|
||||
}
|
||||
|
||||
def _get_by_tool_stats(self) -> dict[str, dict[str, Any]]:
|
||||
"""Get per-tool statistics."""
|
||||
by_tool: dict[str, list[ToolCompressionMetrics]] = {}
|
||||
for m in self.metrics:
|
||||
if m.tool_name not in by_tool:
|
||||
by_tool[m.tool_name] = []
|
||||
by_tool[m.tool_name].append(m)
|
||||
|
||||
result = {}
|
||||
for name, tool_metrics in by_tool.items():
|
||||
compressed = [m for m in tool_metrics if m.was_compressed]
|
||||
result[name] = {
|
||||
"invocations": len(tool_metrics),
|
||||
"compressions": len(compressed),
|
||||
"chars_saved": sum(m.chars_saved for m in tool_metrics),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
# Global metrics collector
|
||||
_global_metrics = ToolMetricsCollector()
|
||||
|
||||
@@ -288,31 +204,7 @@ class HeadroomToolWrapper(BaseTool): # type: ignore[misc]
|
||||
compressed: Compressed output.
|
||||
was_compressed: Whether compression was applied.
|
||||
"""
|
||||
chars_before = len(original)
|
||||
chars_after = len(compressed)
|
||||
chars_saved = chars_before - chars_after
|
||||
|
||||
metric = ToolCompressionMetrics(
|
||||
tool_name=self.name,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
chars_before=chars_before,
|
||||
chars_after=chars_after,
|
||||
chars_saved=max(0, chars_saved),
|
||||
compression_ratio=chars_after / chars_before if chars_before > 0 else 1.0,
|
||||
was_compressed=was_compressed and chars_saved > 0,
|
||||
)
|
||||
|
||||
self._metrics.add(metric)
|
||||
|
||||
if was_compressed and chars_saved > 0:
|
||||
logger.info(
|
||||
"HeadroomToolWrapper[%s]: %d -> %d chars (%d saved, %.1f%% of original)",
|
||||
self.name,
|
||||
chars_before,
|
||||
chars_after,
|
||||
chars_saved,
|
||||
metric.compression_ratio * 100,
|
||||
)
|
||||
_record_tool_metrics(self._metrics, self.name, original, compressed, was_compressed, logger)
|
||||
|
||||
|
||||
def wrap_tools_with_headroom(
|
||||
|
||||
@@ -24,8 +24,6 @@ Example:
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
# LangChain imports - these are optional dependencies
|
||||
@@ -39,6 +37,12 @@ except ImportError:
|
||||
StructuredTool = object # type: ignore[misc,assignment]
|
||||
Tool = object # type: ignore[misc,assignment]
|
||||
|
||||
# Re-exported: callers import ToolCompressionMetrics from this module.
|
||||
from headroom.integrations._metrics import ( # noqa: F401
|
||||
ToolCompressionMetrics,
|
||||
ToolMetricsCollector,
|
||||
_record_tool_metrics,
|
||||
)
|
||||
from headroom.integrations.mcp import compress_tool_result
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -54,71 +58,6 @@ def _check_langchain_available() -> None:
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCompressionMetrics:
|
||||
"""Metrics from a single tool compression."""
|
||||
|
||||
tool_name: str
|
||||
timestamp: datetime
|
||||
chars_before: int
|
||||
chars_after: int
|
||||
chars_saved: int
|
||||
compression_ratio: float
|
||||
was_compressed: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolMetricsCollector:
|
||||
"""Collects compression metrics across all tool invocations."""
|
||||
|
||||
metrics: list[ToolCompressionMetrics] = field(default_factory=list)
|
||||
|
||||
def add(self, metric: ToolCompressionMetrics) -> None:
|
||||
"""Add a metric entry."""
|
||||
self.metrics.append(metric)
|
||||
# Keep only last 1000
|
||||
if len(self.metrics) > 1000:
|
||||
self.metrics = self.metrics[-1000:]
|
||||
|
||||
def get_summary(self) -> dict[str, Any]:
|
||||
"""Get summary statistics."""
|
||||
if not self.metrics:
|
||||
return {
|
||||
"total_invocations": 0,
|
||||
"total_compressions": 0,
|
||||
"total_chars_saved": 0,
|
||||
}
|
||||
|
||||
compressed = [m for m in self.metrics if m.was_compressed]
|
||||
return {
|
||||
"total_invocations": len(self.metrics),
|
||||
"total_compressions": len(compressed),
|
||||
"total_chars_saved": sum(m.chars_saved for m in self.metrics),
|
||||
"average_compression_ratio": (
|
||||
sum(m.compression_ratio for m in compressed) / len(compressed) if compressed else 0
|
||||
),
|
||||
"by_tool": self._get_by_tool_stats(),
|
||||
}
|
||||
|
||||
def _get_by_tool_stats(self) -> dict[str, dict[str, Any]]:
|
||||
"""Get per-tool statistics."""
|
||||
by_tool: dict[str, list[ToolCompressionMetrics]] = {}
|
||||
for m in self.metrics:
|
||||
if m.tool_name not in by_tool:
|
||||
by_tool[m.tool_name] = []
|
||||
by_tool[m.tool_name].append(m)
|
||||
|
||||
result = {}
|
||||
for name, tool_metrics in by_tool.items():
|
||||
compressed = [m for m in tool_metrics if m.was_compressed]
|
||||
result[name] = {
|
||||
"invocations": len(tool_metrics),
|
||||
"compressions": len(compressed),
|
||||
"chars_saved": sum(m.chars_saved for m in tool_metrics),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
# Global metrics collector
|
||||
_global_metrics = ToolMetricsCollector()
|
||||
|
||||
@@ -283,27 +222,7 @@ class HeadroomToolWrapper:
|
||||
compressed: Compressed output.
|
||||
was_compressed: Whether compression was applied.
|
||||
"""
|
||||
chars_before = len(original)
|
||||
chars_after = len(compressed)
|
||||
chars_saved = chars_before - chars_after
|
||||
|
||||
metric = ToolCompressionMetrics(
|
||||
tool_name=self.name,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
chars_before=chars_before,
|
||||
chars_after=chars_after,
|
||||
chars_saved=max(0, chars_saved),
|
||||
compression_ratio=chars_after / chars_before if chars_before > 0 else 1.0,
|
||||
was_compressed=was_compressed and chars_saved > 0,
|
||||
)
|
||||
|
||||
self._metrics.add(metric)
|
||||
|
||||
if was_compressed and chars_saved > 0:
|
||||
logger.info(
|
||||
f"HeadroomToolWrapper[{self.name}]: {chars_before} -> {chars_after} chars "
|
||||
f"({chars_saved} saved, {metric.compression_ratio:.1%} of original)"
|
||||
)
|
||||
_record_tool_metrics(self._metrics, self.name, original, compressed, was_compressed, logger)
|
||||
|
||||
@staticmethod
|
||||
def _usable_args_schema(tool: Any) -> Any:
|
||||
|
||||
@@ -32,7 +32,6 @@ import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import AsyncIterator, Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
@@ -63,6 +62,7 @@ except ImportError:
|
||||
RunnableBinding = () # type: ignore[misc,assignment]
|
||||
|
||||
from headroom import HeadroomConfig, HeadroomMode
|
||||
from headroom.integrations._metrics import OptimizationMetrics
|
||||
from headroom.providers import OpenAIProvider
|
||||
from headroom.transforms import TransformPipeline
|
||||
|
||||
@@ -102,20 +102,6 @@ def langchain_available() -> bool:
|
||||
return LANGCHAIN_AVAILABLE
|
||||
|
||||
|
||||
@dataclass
|
||||
class OptimizationMetrics:
|
||||
"""Metrics from a single optimization pass."""
|
||||
|
||||
request_id: str
|
||||
timestamp: datetime
|
||||
tokens_before: int
|
||||
tokens_after: int
|
||||
tokens_saved: int
|
||||
savings_percent: float
|
||||
transforms_applied: list[str]
|
||||
model: str
|
||||
|
||||
|
||||
class HeadroomChatModel(BaseChatModel):
|
||||
"""LangChain chat model wrapper that applies Headroom optimizations.
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import AsyncGenerator, AsyncIterable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, TypeVar
|
||||
from uuid import uuid4
|
||||
@@ -36,6 +35,7 @@ except ImportError:
|
||||
T = TypeVar("T")
|
||||
|
||||
from headroom import HeadroomConfig # noqa: E402
|
||||
from headroom.integrations._metrics import OptimizationMetrics # noqa: E402
|
||||
from headroom.providers import OpenAIProvider # noqa: E402
|
||||
from headroom.transforms import TransformPipeline # noqa: E402
|
||||
|
||||
@@ -57,20 +57,6 @@ def strands_available() -> bool:
|
||||
return STRANDS_AVAILABLE
|
||||
|
||||
|
||||
@dataclass
|
||||
class OptimizationMetrics:
|
||||
"""Metrics from a single optimization pass."""
|
||||
|
||||
request_id: str
|
||||
timestamp: datetime
|
||||
tokens_before: int
|
||||
tokens_after: int
|
||||
tokens_saved: int
|
||||
savings_percent: float
|
||||
transforms_applied: list[str]
|
||||
model: str
|
||||
|
||||
|
||||
class HeadroomStrandsModel(Model): # type: ignore[misc]
|
||||
"""Strands model wrapper that applies Headroom optimizations.
|
||||
|
||||
|
||||
@@ -1,16 +1,13 @@
|
||||
"""Tests for AutoGen agent tool integration.
|
||||
|
||||
Tests cover:
|
||||
1. ToolCompressionMetrics - Dataclass for tool compression metrics
|
||||
2. ToolMetricsCollector - Collector for compression metrics
|
||||
3. HeadroomToolWrapper - Wrapper for AutoGen FunctionTool with compression
|
||||
4. wrap_tools_with_headroom - Convenience function for wrapping multiple tools
|
||||
5. get_tool_metrics / reset_tool_metrics - Global metrics access
|
||||
1. HeadroomToolWrapper - Wrapper for AutoGen FunctionTool with compression
|
||||
2. wrap_tools_with_headroom - Convenience function for wrapping multiple tools
|
||||
3. get_tool_metrics / reset_tool_metrics - Global metrics access
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -54,71 +51,6 @@ async def async_lookup(query: str) -> str:
|
||||
return _make_large_output()
|
||||
|
||||
|
||||
class TestToolCompressionMetrics:
|
||||
"""Tests for ToolCompressionMetrics dataclass."""
|
||||
|
||||
def test_create_metrics(self):
|
||||
from headroom.integrations.autogen.agents import ToolCompressionMetrics
|
||||
|
||||
metrics = ToolCompressionMetrics(
|
||||
tool_name="search",
|
||||
timestamp=datetime.now(),
|
||||
chars_before=5000,
|
||||
chars_after=2000,
|
||||
chars_saved=3000,
|
||||
compression_ratio=0.4,
|
||||
was_compressed=True,
|
||||
)
|
||||
|
||||
assert metrics.tool_name == "search"
|
||||
assert metrics.chars_before == 5000
|
||||
assert metrics.chars_saved == 3000
|
||||
assert metrics.was_compressed is True
|
||||
|
||||
def test_metrics_all_fields_required(self):
|
||||
from headroom.integrations.autogen.agents import ToolCompressionMetrics
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
ToolCompressionMetrics() # type: ignore[call-arg]
|
||||
|
||||
|
||||
class TestToolMetricsCollector:
|
||||
"""Tests for ToolMetricsCollector."""
|
||||
|
||||
def test_empty_summary(self):
|
||||
from headroom.integrations.autogen.agents import ToolMetricsCollector
|
||||
|
||||
collector = ToolMetricsCollector()
|
||||
summary = collector.get_summary()
|
||||
assert summary["total_invocations"] == 0
|
||||
assert summary["total_compressions"] == 0
|
||||
|
||||
def test_add_and_summary(self):
|
||||
from headroom.integrations.autogen.agents import (
|
||||
ToolCompressionMetrics,
|
||||
ToolMetricsCollector,
|
||||
)
|
||||
|
||||
collector = ToolMetricsCollector()
|
||||
collector.add(
|
||||
ToolCompressionMetrics(
|
||||
tool_name="search",
|
||||
timestamp=datetime.now(),
|
||||
chars_before=5000,
|
||||
chars_after=2000,
|
||||
chars_saved=3000,
|
||||
compression_ratio=0.4,
|
||||
was_compressed=True,
|
||||
)
|
||||
)
|
||||
|
||||
summary = collector.get_summary()
|
||||
assert summary["total_invocations"] == 1
|
||||
assert summary["total_compressions"] == 1
|
||||
assert summary["total_chars_saved"] == 3000
|
||||
assert "search" in summary["by_tool"]
|
||||
|
||||
|
||||
class TestGlobalMetrics:
|
||||
"""Tests for global metrics functions."""
|
||||
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
"""Tests for CrewAI agent tool integration.
|
||||
|
||||
Tests cover:
|
||||
1. ToolCompressionMetrics - Dataclass for tool compression metrics
|
||||
2. ToolMetricsCollector - Collector for compression metrics
|
||||
3. HeadroomToolWrapper - Wrapper for CrewAI tools with compression
|
||||
4. wrap_tools_with_headroom - Convenience function for wrapping multiple tools
|
||||
5. get_tool_metrics / reset_tool_metrics - Global metrics access
|
||||
1. HeadroomToolWrapper - Wrapper for CrewAI tools with compression
|
||||
2. wrap_tools_with_headroom - Convenience function for wrapping multiple tools
|
||||
3. get_tool_metrics / reset_tool_metrics - Global metrics access
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -30,92 +27,6 @@ def _make_large_output(n: int = 200) -> str:
|
||||
return json.dumps({"items": [{"id": i, "data": "x" * 50} for i in range(n)]})
|
||||
|
||||
|
||||
class TestToolCompressionMetrics:
|
||||
"""Tests for ToolCompressionMetrics dataclass."""
|
||||
|
||||
def test_create_metrics(self):
|
||||
from headroom.integrations.crewai.agents import ToolCompressionMetrics
|
||||
|
||||
metrics = ToolCompressionMetrics(
|
||||
tool_name="search",
|
||||
timestamp=datetime.now(),
|
||||
chars_before=5000,
|
||||
chars_after=2000,
|
||||
chars_saved=3000,
|
||||
compression_ratio=0.4,
|
||||
was_compressed=True,
|
||||
)
|
||||
|
||||
assert metrics.tool_name == "search"
|
||||
assert metrics.chars_before == 5000
|
||||
assert metrics.chars_saved == 3000
|
||||
assert metrics.was_compressed is True
|
||||
|
||||
def test_metrics_all_fields_required(self):
|
||||
from headroom.integrations.crewai.agents import ToolCompressionMetrics
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
ToolCompressionMetrics() # type: ignore[call-arg]
|
||||
|
||||
|
||||
class TestToolMetricsCollector:
|
||||
"""Tests for ToolMetricsCollector."""
|
||||
|
||||
def test_empty_summary(self):
|
||||
from headroom.integrations.crewai.agents import ToolMetricsCollector
|
||||
|
||||
collector = ToolMetricsCollector()
|
||||
summary = collector.get_summary()
|
||||
assert summary["total_invocations"] == 0
|
||||
assert summary["total_compressions"] == 0
|
||||
|
||||
def test_add_and_summary(self):
|
||||
from headroom.integrations.crewai.agents import (
|
||||
ToolCompressionMetrics,
|
||||
ToolMetricsCollector,
|
||||
)
|
||||
|
||||
collector = ToolMetricsCollector()
|
||||
collector.add(
|
||||
ToolCompressionMetrics(
|
||||
tool_name="search",
|
||||
timestamp=datetime.now(),
|
||||
chars_before=5000,
|
||||
chars_after=2000,
|
||||
chars_saved=3000,
|
||||
compression_ratio=0.4,
|
||||
was_compressed=True,
|
||||
)
|
||||
)
|
||||
|
||||
summary = collector.get_summary()
|
||||
assert summary["total_invocations"] == 1
|
||||
assert summary["total_compressions"] == 1
|
||||
assert summary["total_chars_saved"] == 3000
|
||||
assert "search" in summary["by_tool"]
|
||||
|
||||
def test_caps_at_1000(self):
|
||||
from headroom.integrations.crewai.agents import (
|
||||
ToolCompressionMetrics,
|
||||
ToolMetricsCollector,
|
||||
)
|
||||
|
||||
collector = ToolMetricsCollector()
|
||||
for _i in range(1050):
|
||||
collector.add(
|
||||
ToolCompressionMetrics(
|
||||
tool_name="t",
|
||||
timestamp=datetime.now(),
|
||||
chars_before=100,
|
||||
chars_after=100,
|
||||
chars_saved=0,
|
||||
compression_ratio=1.0,
|
||||
was_compressed=False,
|
||||
)
|
||||
)
|
||||
assert len(collector.metrics) == 1000
|
||||
|
||||
|
||||
class TestGlobalMetrics:
|
||||
"""Tests for global metrics functions."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user