mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
refactor(integrations): share the local/cloud compress backend of asgi and litellm_callback
CompressionMiddleware and HeadroomCallback carried identical copies of _local_compress, _cloud_compress, the cloud URL constant, the HEADROOM_API_KEY / HEADROOM_API_URL reads and cloud_mode. They now come from a CompressBackend mixin in headroom/integrations/_compress_backend.py, which imports neither starlette nor litellm. Method names stay (_local_compress and _cloud_compress are called by tests/test_offline_egress_chokepoint.py), _DEFAULT_CLOUD_URL and guard_egress stay importable from both modules, each class sets _log to its own module logger so the cloud-error warning keeps its logger name, and each class keeps its own aclose and min_tokens handling unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a9815bbb5f
commit
04d3810319
@@ -0,0 +1,118 @@
|
||||
"""Local or Headroom Cloud compression, shared by the ASGI and LiteLLM integrations.
|
||||
|
||||
``CompressionMiddleware`` (asgi.py) and ``HeadroomCallback`` (litellm_callback.py)
|
||||
compress the same way: in-process with ``headroom.compress()`` by default, or
|
||||
through the Headroom Cloud API when an API key is set.
|
||||
|
||||
This module must not import starlette or litellm, so either integration can be
|
||||
used without the other's dependency.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from headroom.offline import guard_egress
|
||||
|
||||
_DEFAULT_CLOUD_URL = "https://api.headroomlabs.ai"
|
||||
_DEFAULT_MODEL = "claude-sonnet-4-5-20250929"
|
||||
|
||||
|
||||
class CompressBackend:
|
||||
"""Mixin that picks local or cloud compression and holds the cloud client.
|
||||
|
||||
Subclasses call ``_init_compress_backend`` from ``__init__`` and set
|
||||
``_log`` to their module logger, so log lines keep that logger's name.
|
||||
"""
|
||||
|
||||
_log: logging.Logger
|
||||
|
||||
def _init_compress_backend(
|
||||
self,
|
||||
model_limit: int,
|
||||
hooks: Any,
|
||||
api_key: str | None,
|
||||
api_url: str | None,
|
||||
) -> None:
|
||||
self._model_limit = model_limit
|
||||
self._hooks = hooks
|
||||
|
||||
# Cloud mode: if api_key is set, compress via Headroom Cloud API
|
||||
# Falls back to HEADROOM_API_KEY env var
|
||||
self._api_key = api_key or os.environ.get("HEADROOM_API_KEY", "").strip() or None
|
||||
self._api_url = (
|
||||
api_url or os.environ.get("HEADROOM_API_URL", "").strip() or _DEFAULT_CLOUD_URL
|
||||
).rstrip("/")
|
||||
self._client: Any = None # Lazy-initialized httpx.AsyncClient
|
||||
|
||||
@property
|
||||
def cloud_mode(self) -> bool:
|
||||
"""Whether cloud compression is enabled."""
|
||||
return self._api_key is not None
|
||||
|
||||
def _local_compress(self, messages: list[dict], model: str) -> dict[str, Any] | None:
|
||||
"""Compress locally using headroom.compress()."""
|
||||
from headroom.compress import compress
|
||||
|
||||
result = compress(
|
||||
messages=messages,
|
||||
model=model or _DEFAULT_MODEL,
|
||||
model_limit=self._model_limit,
|
||||
hooks=self._hooks,
|
||||
)
|
||||
return {
|
||||
"messages": result.messages,
|
||||
"tokens_before": result.tokens_before,
|
||||
"tokens_after": result.tokens_after,
|
||||
"tokens_saved": result.tokens_saved,
|
||||
"compression_ratio": result.compression_ratio,
|
||||
}
|
||||
|
||||
async def _cloud_compress(self, messages: list[dict], model: str) -> dict[str, Any] | None:
|
||||
"""Compress via Headroom Cloud API (managed CCR, TOIN, analytics).
|
||||
|
||||
This is the one path in these integrations that puts the caller's
|
||||
prompt content on the wire to a Headroom-operated host, so it is
|
||||
exactly what HEADROOM_OFFLINE exists to stop. "Opt-in by configuration"
|
||||
was the old reason for leaving it open, and it is not good enough: an
|
||||
operator who sets an air-gap switch is overriding earlier configuration
|
||||
on purpose. The refusal is loud rather than a silent fall-through to
|
||||
local compression, because silently compressing locally would hide the
|
||||
fact that the deployment is no longer doing what it was configured to do.
|
||||
"""
|
||||
guard_egress("Headroom Cloud compression", self._api_url)
|
||||
if self._client is None:
|
||||
try:
|
||||
import httpx
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"httpx is required for Headroom Cloud mode: pip install httpx"
|
||||
) from e
|
||||
self._client = httpx.AsyncClient(timeout=30.0)
|
||||
|
||||
client = self._client
|
||||
assert client is not None
|
||||
resp = await client.post(
|
||||
f"{self._api_url}/v1/saas/compress",
|
||||
headers={
|
||||
"X-Headroom-Key": self._api_key,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
content=json.dumps(
|
||||
{
|
||||
"messages": messages,
|
||||
"model": model or _DEFAULT_MODEL,
|
||||
"model_limit": self._model_limit,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
if resp.status_code != 200:
|
||||
self._log.warning("Headroom Cloud API error: %d %s", resp.status_code, resp.text[:200])
|
||||
return None
|
||||
|
||||
result: dict[str, Any] = resp.json()
|
||||
return result
|
||||
@@ -28,18 +28,20 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import MutableMapping
|
||||
from typing import Any
|
||||
|
||||
from starlette.types import ASGIApp, Receive, Scope, Send
|
||||
|
||||
from headroom.offline import guard_egress
|
||||
# _DEFAULT_CLOUD_URL and guard_egress stay importable from this module.
|
||||
from headroom.integrations._compress_backend import ( # noqa: F401
|
||||
_DEFAULT_CLOUD_URL,
|
||||
CompressBackend,
|
||||
)
|
||||
from headroom.offline import guard_egress # noqa: F401
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_CLOUD_URL = "https://api.headroomlabs.ai"
|
||||
|
||||
# Paths that contain LLM messages to compress
|
||||
_LLM_PATHS = (
|
||||
"/v1/messages", # Anthropic
|
||||
@@ -49,7 +51,7 @@ _LLM_PATHS = (
|
||||
)
|
||||
|
||||
|
||||
class CompressionMiddleware:
|
||||
class CompressionMiddleware(CompressBackend):
|
||||
"""ASGI middleware that compresses LLM request messages.
|
||||
|
||||
Two modes:
|
||||
@@ -64,6 +66,8 @@ class CompressionMiddleware:
|
||||
- x-headroom-compressed: "true" if compression occurred
|
||||
"""
|
||||
|
||||
_log = logger
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app: ASGIApp,
|
||||
@@ -75,20 +79,7 @@ class CompressionMiddleware:
|
||||
) -> None:
|
||||
self.app = app
|
||||
self._min_tokens = min_tokens
|
||||
self._model_limit = model_limit
|
||||
self._hooks = hooks
|
||||
|
||||
# Cloud mode: if api_key is set, compress via Headroom Cloud API
|
||||
self._api_key = api_key or os.environ.get("HEADROOM_API_KEY", "").strip() or None
|
||||
self._api_url = (
|
||||
api_url or os.environ.get("HEADROOM_API_URL", "").strip() or _DEFAULT_CLOUD_URL
|
||||
).rstrip("/")
|
||||
self._client: Any = None # Lazy-initialized httpx.AsyncClient
|
||||
|
||||
@property
|
||||
def cloud_mode(self) -> bool:
|
||||
"""Whether cloud compression is enabled."""
|
||||
return self._api_key is not None
|
||||
self._init_compress_backend(model_limit, hooks, api_key, api_url)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Close the underlying httpx.AsyncClient, if one was created."""
|
||||
@@ -186,67 +177,3 @@ class CompressionMiddleware:
|
||||
await send(message)
|
||||
|
||||
await self.app(scope, modified_receive, metrics_send)
|
||||
|
||||
def _local_compress(self, messages: list[dict], model: str) -> dict[str, Any] | None:
|
||||
"""Compress locally using headroom.compress()."""
|
||||
from headroom.compress import compress
|
||||
|
||||
result = compress(
|
||||
messages=messages,
|
||||
model=model or "claude-sonnet-4-5-20250929",
|
||||
model_limit=self._model_limit,
|
||||
hooks=self._hooks,
|
||||
)
|
||||
return {
|
||||
"messages": result.messages,
|
||||
"tokens_before": result.tokens_before,
|
||||
"tokens_after": result.tokens_after,
|
||||
"tokens_saved": result.tokens_saved,
|
||||
"compression_ratio": result.compression_ratio,
|
||||
}
|
||||
|
||||
async def _cloud_compress(self, messages: list[dict], model: str) -> dict[str, Any] | None:
|
||||
"""Compress via Headroom Cloud API (managed CCR, TOIN, analytics).
|
||||
|
||||
This is the one path in this file that puts the caller's prompt
|
||||
content on the wire to a Headroom-operated host, so it is exactly what
|
||||
HEADROOM_OFFLINE exists to stop. "Opt-in by configuration" was the old
|
||||
reason for leaving it open, and it is not good enough: an operator who
|
||||
sets an air-gap switch is overriding earlier configuration on purpose.
|
||||
The refusal is loud rather than a silent fall-through to local
|
||||
compression, because silently compressing locally would hide the fact
|
||||
that the deployment is no longer doing what it was configured to do.
|
||||
"""
|
||||
guard_egress("Headroom Cloud compression", self._api_url)
|
||||
if self._client is None:
|
||||
try:
|
||||
import httpx
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"httpx is required for Headroom Cloud mode: pip install httpx"
|
||||
) from e
|
||||
self._client = httpx.AsyncClient(timeout=30.0)
|
||||
|
||||
client = self._client
|
||||
assert client is not None
|
||||
resp = await client.post(
|
||||
f"{self._api_url}/v1/saas/compress",
|
||||
headers={
|
||||
"X-Headroom-Key": self._api_key,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
content=json.dumps(
|
||||
{
|
||||
"messages": messages,
|
||||
"model": model or "claude-sonnet-4-5-20250929",
|
||||
"model_limit": self._model_limit,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
if resp.status_code != 200:
|
||||
logger.warning("Headroom Cloud API error: %d %s", resp.status_code, resp.text[:200])
|
||||
return None
|
||||
|
||||
result: dict[str, Any] = resp.json()
|
||||
return result
|
||||
|
||||
@@ -16,16 +16,18 @@ Cloud mode requires httpx: pip install httpx
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from headroom.offline import guard_egress
|
||||
# _DEFAULT_CLOUD_URL and guard_egress stay importable from this module.
|
||||
from headroom.integrations._compress_backend import ( # noqa: F401
|
||||
_DEFAULT_CLOUD_URL,
|
||||
CompressBackend,
|
||||
)
|
||||
from headroom.offline import guard_egress # noqa: F401
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_CLOUD_URL = "https://api.headroomlabs.ai"
|
||||
|
||||
|
||||
try:
|
||||
from litellm.integrations.custom_logger import CustomLogger as _CustomLogger
|
||||
@@ -33,7 +35,7 @@ except ImportError: # litellm not installed — fall back to plain object
|
||||
_CustomLogger = object # type: ignore[assignment,misc]
|
||||
|
||||
|
||||
class HeadroomCallback(_CustomLogger):
|
||||
class HeadroomCallback(CompressBackend, _CustomLogger):
|
||||
"""LiteLLM callback that compresses messages before each API call.
|
||||
|
||||
Implements the LiteLLM callback hooks looked up by name:
|
||||
@@ -63,6 +65,8 @@ class HeadroomCallback(_CustomLogger):
|
||||
HEADROOM_API_KEY: "hdr_xxx"
|
||||
"""
|
||||
|
||||
_log = logger
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
min_tokens: int = 500,
|
||||
@@ -73,30 +77,14 @@ class HeadroomCallback(_CustomLogger):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._min_tokens = min_tokens
|
||||
self._model_limit = model_limit
|
||||
self._hooks = hooks
|
||||
self._total_saved = 0
|
||||
|
||||
# Cloud mode: if api_key is set, compress via Headroom Cloud API
|
||||
# Falls back to HEADROOM_API_KEY env var
|
||||
import os
|
||||
|
||||
self._api_key = api_key or os.environ.get("HEADROOM_API_KEY", "").strip() or None
|
||||
self._api_url = (
|
||||
api_url or os.environ.get("HEADROOM_API_URL", "").strip() or _DEFAULT_CLOUD_URL
|
||||
).rstrip("/")
|
||||
self._client: Any = None # Lazy-initialized httpx.AsyncClient
|
||||
self._init_compress_backend(model_limit, hooks, api_key, api_url)
|
||||
|
||||
@property
|
||||
def total_tokens_saved(self) -> int:
|
||||
"""Total tokens saved across all calls."""
|
||||
return self._total_saved
|
||||
|
||||
@property
|
||||
def cloud_mode(self) -> bool:
|
||||
"""Whether cloud compression is enabled."""
|
||||
return self._api_key is not None
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Close the shared cloud HTTP client, if it was initialized.
|
||||
|
||||
@@ -167,70 +155,6 @@ class HeadroomCallback(_CustomLogger):
|
||||
|
||||
return data
|
||||
|
||||
def _local_compress(self, messages: list[dict], model: str) -> dict[str, Any] | None:
|
||||
"""Compress locally using headroom.compress()."""
|
||||
from headroom.compress import compress
|
||||
|
||||
result = compress(
|
||||
messages=messages,
|
||||
model=model or "claude-sonnet-4-5-20250929",
|
||||
model_limit=self._model_limit,
|
||||
hooks=self._hooks,
|
||||
)
|
||||
return {
|
||||
"messages": result.messages,
|
||||
"tokens_before": result.tokens_before,
|
||||
"tokens_after": result.tokens_after,
|
||||
"tokens_saved": result.tokens_saved,
|
||||
"compression_ratio": result.compression_ratio,
|
||||
}
|
||||
|
||||
async def _cloud_compress(self, messages: list[dict], model: str) -> dict[str, Any] | None:
|
||||
"""Compress via Headroom Cloud API (managed CCR, TOIN, analytics).
|
||||
|
||||
This is the one path in this file that puts the caller's prompt
|
||||
content on the wire to a Headroom-operated host, so it is exactly what
|
||||
HEADROOM_OFFLINE exists to stop. "Opt-in by configuration" was the old
|
||||
reason for leaving it open, and it is not good enough: an operator who
|
||||
sets an air-gap switch is overriding earlier configuration on purpose.
|
||||
The refusal is loud rather than a silent fall-through to local
|
||||
compression, because silently compressing locally would hide the fact
|
||||
that the deployment is no longer doing what it was configured to do.
|
||||
"""
|
||||
guard_egress("Headroom Cloud compression", self._api_url)
|
||||
if self._client is None:
|
||||
try:
|
||||
import httpx
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"httpx is required for Headroom Cloud mode: pip install httpx"
|
||||
) from e
|
||||
self._client = httpx.AsyncClient(timeout=30.0)
|
||||
|
||||
client = self._client
|
||||
assert client is not None
|
||||
resp = await client.post(
|
||||
f"{self._api_url}/v1/saas/compress",
|
||||
headers={
|
||||
"X-Headroom-Key": self._api_key,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
content=json.dumps(
|
||||
{
|
||||
"messages": messages,
|
||||
"model": model or "claude-sonnet-4-5-20250929",
|
||||
"model_limit": self._model_limit,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
if resp.status_code != 200:
|
||||
logger.warning("Headroom Cloud API error: %d %s", resp.status_code, resp.text[:200])
|
||||
return None
|
||||
|
||||
result: dict[str, Any] = resp.json()
|
||||
return result
|
||||
|
||||
async def async_post_call_success_hook(
|
||||
self,
|
||||
data: dict[str, Any],
|
||||
|
||||
Reference in New Issue
Block a user