mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(client): close upstream responses when wrapped streams are closed (#4482)
## Description Closing a streaming `HeadroomClient` response can leave its upstream HTTP response open. Repeatedly closing unread streams retains connections even after garbage collection and eventually exhausts the connection pool. `yield from stream` closes the generator returned by OpenAI's `Stream.__iter__()`, rather than the original stream. Closing the wrapper before its first iteration also skips its cleanup entirely. Replace it with an owning iterator that closes the upstream stream and saves metrics once on close, exhaustion, or error. ## Type of Change - [x] Bug fix ## Changes Made - Close the upstream source and its active iterator when distinct; handle their cleanup independently and save metrics once. - Add a runnable loopback reproducer for the connection and memory measurements. ## Testing - [x] 71 lifecycle, client, pipeline, and provider tests passed, including 15 lifecycle tests. The OpenAI response-close tests fail against the original implementation; two additional iterator-cleanup tests fail against the first PR revision. - [x] The Python pre-push suite passed: 177 passed, 4 skipped. - [x] Ruff lint/format and `git diff --check` passed. `mypy --platform linux headroom` passed for all 578 source files. ### Test Output ```text 71 passed in 10.81s 177 passed, 4 skipped, 1 warning in 27.62s Success: no issues found in 578 source files ``` ## Real Behavior Proof - Environment: Windows 11, Python 3.12.12, OpenAI SDK 2.41.0, HTTPX 0.28.1. - Exact command / steps: From this PR's checkout, run the commands below. The [reproducer](https://github.com/ernsahin/headroom/blob/codex/close-sdk-streams/scripts/repro_client_stream_close.py) starts a separate loopback SSE server and fresh client processes. It compares the original `_wrap_stream` from commit `3fef15d` with the patched method, using the same installed dependencies and normal GC. - Observed result: All three runs released every connection after the fix; memory results below. - Not tested: Live model endpoints. ```sh uv sync --extra dev uv run python scripts/repro_client_stream_close.py --cycles 500 --repeats 3 uv run python scripts/repro_client_stream_close.py --cycles 1001 --repeats 1 uv run python scripts/repro_client_stream_close.py --cycles 500 --repeats 3 --started --no-trace --timeout 20 ``` If the checkout is shallow, first fetch the baseline with `git fetch origin 3fef15d376d9899f77a28464e2e91f8b99514930`. Each run prints JSON with completed requests, connection counts, RSS, and Python heap samples. RSS is measured in the client process, excluding the server. Exact memory figures vary by environment. Across three fresh-process runs of 500 unread streams: | Result | Before | After | | --- | ---: | ---: | | Open connections after explicit GC | 500 | 0 | | Median client RSS increase | 7.03 MiB | 0.93 MiB | | Median Python heap retained after GC | 3.35 MiB | 0.10 MiB | With a 1,000-connection pool, request 1,001 failed with `PoolTimeout` before the fix; all 1,001 completed afterward. Closing after the first chunk also reproduced a GC cleanup deadlock inside the HTTPcore pool lock in 3/3 runs with memory tracing disabled. Each original-code run was stopped after 20 seconds; the fixed code completed 500/500 cycles in every run. GC timing depends on allocations. ## Runtime Rollout Safety - Rollout-managed feature(s): N/A; Python SDK resource cleanup. - Minimum rollout channel: N/A. - Stable/default behavior changed: Closing a wrapped stream releases its upstream response. - Kill switch / disable path: N/A. - Unsafe override required: No. - Qualification impact: No rollout-channel changes. - Rollback path: Revert this commit. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
+72
-9
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
@@ -38,6 +38,74 @@ from .utils import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _OwnedStream(Iterator[Any]):
|
||||
"""Iterator that owns and closes an upstream response stream."""
|
||||
|
||||
def __init__(self, stream: Iterable[Any], save_metrics: Callable[[], None]) -> None:
|
||||
self._source: Iterable[Any] | None = stream
|
||||
self._stream: Iterator[Any] | None = None
|
||||
self._save_metrics: Callable[[], None] | None = save_metrics
|
||||
self._finished = False
|
||||
|
||||
def __iter__(self) -> _OwnedStream:
|
||||
return self
|
||||
|
||||
def __next__(self) -> Any:
|
||||
if self._finished:
|
||||
raise StopIteration
|
||||
try:
|
||||
if self._stream is None:
|
||||
source = self._source
|
||||
if source is None:
|
||||
raise StopIteration
|
||||
self._stream = iter(source)
|
||||
return next(self._stream)
|
||||
except BaseException:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the upstream stream and save metrics exactly once."""
|
||||
if self._finished:
|
||||
return
|
||||
self._finished = True
|
||||
source = self._source
|
||||
stream = self._stream
|
||||
save_metrics = self._save_metrics
|
||||
try:
|
||||
candidates = (source,) if source is stream else (source, stream)
|
||||
for candidate in candidates:
|
||||
if candidate is None:
|
||||
continue
|
||||
close = getattr(candidate, "close", None)
|
||||
if close is not None:
|
||||
try:
|
||||
close()
|
||||
except Exception:
|
||||
logger.warning("Failed to close upstream response stream", exc_info=True)
|
||||
finally:
|
||||
self._source = None
|
||||
self._stream = None
|
||||
self._save_metrics = None
|
||||
if save_metrics is not None:
|
||||
save_metrics()
|
||||
|
||||
def __enter__(self) -> _OwnedStream:
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
||||
self.close()
|
||||
|
||||
def __del__(self) -> None:
|
||||
# The previous generator wrapper ran its finally block when abandoned
|
||||
# after iteration started. Preserve that cleanup for callers that do
|
||||
# not explicitly close a stream.
|
||||
try:
|
||||
self.close()
|
||||
except Exception:
|
||||
logger.debug("Failed to finalize abandoned response stream", exc_info=True)
|
||||
|
||||
|
||||
class ChatCompletions:
|
||||
"""Wrapper for chat.completions API (OpenAI-style)."""
|
||||
|
||||
@@ -644,16 +712,11 @@ class HeadroomClient:
|
||||
|
||||
def _wrap_stream(
|
||||
self,
|
||||
stream: Iterator[Any],
|
||||
stream: Iterable[Any],
|
||||
metrics: RequestMetrics,
|
||||
) -> Iterator[Any]:
|
||||
"""Wrap stream to pass through chunks and save metrics at end."""
|
||||
try:
|
||||
yield from stream
|
||||
finally:
|
||||
# Save metrics when stream completes
|
||||
# Note: output tokens unknown for streams
|
||||
self._storage.save(metrics)
|
||||
"""Wrap stream, closing the upstream and saving metrics exactly once."""
|
||||
return _OwnedStream(stream, lambda: self._storage.save(metrics))
|
||||
|
||||
def _extract_query(self, messages: list[dict[str, Any]]) -> str:
|
||||
"""Extract query from messages for semantic caching.
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
"""Reproduce client stream cleanup against a local SSE server.
|
||||
|
||||
Run the unread-stream proof with the checked-in defaults::
|
||||
|
||||
uv run python scripts/repro_client_stream_close.py
|
||||
uv run python scripts/repro_client_stream_close.py --cycles 1001
|
||||
|
||||
Pass ``--started`` to consume the first event before closing each stream.
|
||||
The baseline client method is loaded from ``--baseline-ref`` in a fresh child
|
||||
process; it is never taken from the current checkout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import ctypes
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
BASELINE_REF = "3fef15d376d9899f77a28464e2e91f8b99514930"
|
||||
|
||||
|
||||
def rss_bytes() -> int | None:
|
||||
"""Return this process's RSS using stdlib APIs where available."""
|
||||
if sys.platform == "win32":
|
||||
from ctypes import wintypes
|
||||
|
||||
class Counters(ctypes.Structure):
|
||||
_fields_ = [("cb", wintypes.DWORD), ("PageFaultCount", wintypes.DWORD)] + [
|
||||
(name, ctypes.c_size_t)
|
||||
for name in (
|
||||
"PeakWorkingSetSize",
|
||||
"WorkingSetSize",
|
||||
"QuotaPeakPagedPoolUsage",
|
||||
"QuotaPagedPoolUsage",
|
||||
"QuotaPeakNonPagedPoolUsage",
|
||||
"QuotaNonPagedPoolUsage",
|
||||
"PagefileUsage",
|
||||
"PeakPagefileUsage",
|
||||
"PrivateUsage",
|
||||
)
|
||||
]
|
||||
|
||||
counters = Counters()
|
||||
counters.cb = ctypes.sizeof(counters)
|
||||
kernel = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
kernel.GetCurrentProcess.restype = wintypes.HANDLE
|
||||
psapi = ctypes.WinDLL("psapi", use_last_error=True)
|
||||
psapi.GetProcessMemoryInfo.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
ctypes.POINTER(Counters),
|
||||
wintypes.DWORD,
|
||||
]
|
||||
if psapi.GetProcessMemoryInfo(
|
||||
kernel.GetCurrentProcess(), ctypes.byref(counters), counters.cb
|
||||
):
|
||||
return int(counters.WorkingSetSize)
|
||||
return None
|
||||
if sys.platform.startswith("linux"):
|
||||
try:
|
||||
resident_pages = int(Path("/proc/self/statm").read_text().split()[1])
|
||||
return resident_pages * os.sysconf("SC_PAGE_SIZE")
|
||||
except (OSError, IndexError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def pool_connections(client: object) -> int | None:
|
||||
try:
|
||||
return len(client._client._transport._pool.connections) # type: ignore[attr-defined]
|
||||
except (AttributeError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def load_baseline_method(module: object, ref: str) -> None:
|
||||
"""Install exactly ``HeadroomClient._wrap_stream`` from ``git show``."""
|
||||
source = subprocess.check_output(["git", "show", f"{ref}:headroom/client.py"], text=True)
|
||||
tree = ast.parse(source)
|
||||
client_class = next(
|
||||
node
|
||||
for node in tree.body
|
||||
if isinstance(node, ast.ClassDef) and node.name == "HeadroomClient"
|
||||
)
|
||||
method = next(
|
||||
node
|
||||
for node in client_class.body
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "_wrap_stream"
|
||||
)
|
||||
namespace = dict(vars(module))
|
||||
code = compile(
|
||||
ast.fix_missing_locations(ast.Module(body=[method], type_ignores=[])), "<baseline>", "exec"
|
||||
)
|
||||
exec(code, namespace)
|
||||
module.HeadroomClient._wrap_stream = namespace["_wrap_stream"] # type: ignore[attr-defined]
|
||||
|
||||
|
||||
class SSEHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def log_message(self, *_args: object) -> None:
|
||||
pass
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
self.rfile.read(int(self.headers["Content-Length"]))
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/event-stream")
|
||||
self.send_header("Content-Length", "1000000")
|
||||
self.end_headers()
|
||||
self.wfile.write(
|
||||
b'data: {"id":"local","object":"chat.completion.chunk",'
|
||||
b'"created":0,"model":"local","choices":[{"index":0,'
|
||||
b'"delta":{"content":"hi"},"finish_reason":null}]}'
|
||||
b"\n\n"
|
||||
)
|
||||
self.wfile.flush()
|
||||
self.connection.settimeout(120)
|
||||
try:
|
||||
self.connection.recv(1)
|
||||
except OSError:
|
||||
pass
|
||||
self.close_connection = True
|
||||
|
||||
|
||||
def write_progress(path: Path, completed: int, samples: list[dict[str, object]]) -> None:
|
||||
path.write_text(
|
||||
json.dumps({"completed": completed, "samples": samples}),
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
|
||||
|
||||
def child(args: argparse.Namespace) -> None:
|
||||
import httpx
|
||||
from openai import OpenAI
|
||||
|
||||
import headroom.client as client_module
|
||||
from headroom.providers.openai import OpenAIProvider
|
||||
|
||||
if args.variant == "baseline":
|
||||
load_baseline_method(client_module, args.baseline_ref)
|
||||
|
||||
progress_path = Path(args.progress)
|
||||
samples: list[dict[str, object]] = []
|
||||
with OpenAI(
|
||||
api_key="local-only",
|
||||
base_url=args.url,
|
||||
max_retries=0,
|
||||
http_client=httpx.Client(
|
||||
trust_env=False,
|
||||
limits=httpx.Limits(max_connections=1000),
|
||||
timeout=httpx.Timeout(10, pool=0.25),
|
||||
),
|
||||
) as original:
|
||||
client = client_module.HeadroomClient(
|
||||
original_client=original,
|
||||
provider=OpenAIProvider(),
|
||||
store_url=f"jsonl://{Path(args.store).resolve().as_posix()}",
|
||||
enable_cache_optimizer=False,
|
||||
)
|
||||
|
||||
def request() -> None:
|
||||
stream = client.chat.completions.create(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
)
|
||||
if args.started:
|
||||
next(stream)
|
||||
stream.close()
|
||||
del stream
|
||||
|
||||
for _ in range(5):
|
||||
warmup = original.chat.completions.create(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
)
|
||||
next(warmup)
|
||||
warmup.close()
|
||||
del warmup
|
||||
warmup = client.chat.completions.create(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
)
|
||||
next(warmup)
|
||||
warmup.close()
|
||||
del warmup
|
||||
gc.collect()
|
||||
import tracemalloc
|
||||
|
||||
if not args.no_trace:
|
||||
tracemalloc.start(1)
|
||||
|
||||
def sample(phase: str, cycles: int) -> None:
|
||||
heap, peak = tracemalloc.get_traced_memory()
|
||||
samples.append(
|
||||
{
|
||||
"phase": phase,
|
||||
"cycles": cycles,
|
||||
"rss": rss_bytes(),
|
||||
"heap": heap,
|
||||
"heap_peak": peak,
|
||||
"connections": pool_connections(original),
|
||||
}
|
||||
)
|
||||
write_progress(progress_path, completed, samples)
|
||||
|
||||
completed = 0
|
||||
sample("baseline", 0)
|
||||
started_at = time.perf_counter()
|
||||
error: dict[str, object] | None = None
|
||||
for request_number in range(1, args.cycles + 1):
|
||||
try:
|
||||
request()
|
||||
except Exception as exc: # includes httpx.PoolTimeout
|
||||
cause = exc.__cause__
|
||||
error = {
|
||||
"type": type(exc).__name__,
|
||||
"cause": type(cause).__name__ if cause is not None else None,
|
||||
"request": request_number,
|
||||
"completed": completed,
|
||||
"pool_timeout": type(exc).__name__ == "PoolTimeout"
|
||||
or type(cause).__name__ == "PoolTimeout",
|
||||
}
|
||||
break
|
||||
completed += 1
|
||||
if request_number in (10, 50, 100, 250, 500, 1000, args.cycles):
|
||||
sample("running", request_number)
|
||||
elapsed = time.perf_counter() - started_at
|
||||
sample("before_gc", completed)
|
||||
gc.collect()
|
||||
time.sleep(0.05)
|
||||
sample("after_gc", completed)
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"variant": args.variant,
|
||||
"started": args.started,
|
||||
"cycles": args.cycles,
|
||||
"completed": completed,
|
||||
"error": error,
|
||||
"elapsed": elapsed,
|
||||
"gc_enabled": gc.isenabled(),
|
||||
"samples": samples,
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def stop_process(process: subprocess.Popen[str]) -> None:
|
||||
if os.name == "nt":
|
||||
subprocess.run(
|
||||
["taskkill", "/PID", str(process.pid), "/T", "/F"],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
else:
|
||||
process.kill()
|
||||
try:
|
||||
process.communicate(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.communicate()
|
||||
|
||||
|
||||
def run_child(
|
||||
args: argparse.Namespace,
|
||||
server_port: int,
|
||||
variant: str,
|
||||
temporary: Path,
|
||||
) -> dict[str, object]:
|
||||
progress = temporary / "progress.json"
|
||||
store = temporary / "metrics.jsonl"
|
||||
command = [
|
||||
sys.executable,
|
||||
str(Path(__file__).resolve()),
|
||||
"--child",
|
||||
"--variant",
|
||||
variant,
|
||||
"--baseline-ref",
|
||||
args.baseline_ref,
|
||||
"--cycles",
|
||||
str(args.cycles),
|
||||
"--url",
|
||||
f"http://127.0.0.1:{server_port}/v1",
|
||||
"--progress",
|
||||
str(progress),
|
||||
"--store",
|
||||
str(store),
|
||||
]
|
||||
if args.started:
|
||||
command.append("--started")
|
||||
if args.no_trace:
|
||||
command.append("--no-trace")
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
env={**os.environ, "HEADROOM_OFFLINE": "1"},
|
||||
)
|
||||
try:
|
||||
stdout, stderr = process.communicate(timeout=args.timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
stop_process(process)
|
||||
record: dict[str, object] = {
|
||||
"variant": variant,
|
||||
"started": args.started,
|
||||
"cycles": args.cycles,
|
||||
"completed": None,
|
||||
"error": {"type": "TimeoutExpired"},
|
||||
}
|
||||
if progress.exists():
|
||||
record.update(json.loads(progress.read_text(encoding="utf-8")))
|
||||
return record
|
||||
if process.returncode:
|
||||
raise RuntimeError(
|
||||
f"{variant} child exited {process.returncode}: {stdout[-1000:]} {stderr[-1000:]}"
|
||||
)
|
||||
return json.loads(stdout)
|
||||
|
||||
|
||||
def parent(args: argparse.Namespace) -> None:
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), SSEHandler)
|
||||
server.daemon_threads = True
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
try:
|
||||
for repeat in range(1, args.repeats + 1):
|
||||
for variant in ("baseline", "fixed"):
|
||||
with tempfile.TemporaryDirectory(prefix="headroom-stream-close-") as directory:
|
||||
record = run_child(args, server.server_port, variant, Path(directory))
|
||||
record["repeat"] = repeat
|
||||
print(json.dumps(record), flush=True)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--child", action="store_true", help=argparse.SUPPRESS)
|
||||
parser.add_argument("--started", action="store_true")
|
||||
parser.add_argument("--cycles", type=int, default=500)
|
||||
parser.add_argument("--repeats", type=int, default=3)
|
||||
parser.add_argument("--baseline-ref", default=BASELINE_REF)
|
||||
parser.add_argument("--variant", choices=("baseline", "fixed"), help=argparse.SUPPRESS)
|
||||
parser.add_argument("--url", help=argparse.SUPPRESS)
|
||||
parser.add_argument("--progress", help=argparse.SUPPRESS)
|
||||
parser.add_argument("--store", help=argparse.SUPPRESS)
|
||||
parser.add_argument("--no-trace", action="store_true")
|
||||
parser.add_argument("--timeout", type=int, default=30)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
arguments = parse_args()
|
||||
if arguments.child:
|
||||
child(arguments)
|
||||
else:
|
||||
parent(arguments)
|
||||
@@ -0,0 +1,316 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import weakref
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from openai import OpenAI
|
||||
|
||||
from headroom.client import HeadroomClient
|
||||
|
||||
|
||||
class _ClosableIterator:
|
||||
def __init__(
|
||||
self,
|
||||
values: list[object] | None = None,
|
||||
error: Exception | None = None,
|
||||
close_error: Exception | None = None,
|
||||
):
|
||||
self._values = iter(values or [])
|
||||
self._error = error
|
||||
self._close_error = close_error
|
||||
self.closed = 0
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
if self._error is not None:
|
||||
error, self._error = self._error, None
|
||||
raise error
|
||||
return next(self._values)
|
||||
|
||||
def close(self):
|
||||
self.closed += 1
|
||||
if self._close_error is not None:
|
||||
raise self._close_error
|
||||
|
||||
|
||||
class _IterableOnly:
|
||||
def __init__(self, values: list[object]):
|
||||
self._values = values
|
||||
self.closed = 0
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._values)
|
||||
|
||||
def close(self):
|
||||
self.closed += 1
|
||||
|
||||
|
||||
class _RetainedIterable:
|
||||
def __init__(self):
|
||||
self.iterator = None
|
||||
self.closed = 0
|
||||
|
||||
def __iter__(self):
|
||||
def generate():
|
||||
try:
|
||||
yield "chunk"
|
||||
yield "later"
|
||||
finally:
|
||||
self.closed += 1
|
||||
|
||||
self.iterator = generate()
|
||||
return self.iterator
|
||||
|
||||
|
||||
class _DistinctClosableSource:
|
||||
def __init__(
|
||||
self, owner_error: Exception | None = None, iterator_error: Exception | None = None
|
||||
):
|
||||
self.owner_error = owner_error
|
||||
self.iterator_error = iterator_error
|
||||
self.closed = 0
|
||||
self.iterator = None
|
||||
|
||||
def __iter__(self):
|
||||
self.iterator = _ClosableIterator(["chunk"], close_error=self.iterator_error)
|
||||
return self.iterator
|
||||
|
||||
def close(self):
|
||||
self.closed += 1
|
||||
if self.owner_error is not None:
|
||||
raise self.owner_error
|
||||
|
||||
|
||||
class _Owner:
|
||||
def __init__(self):
|
||||
self.saved = []
|
||||
self._storage = SimpleNamespace(save=self.saved.append)
|
||||
|
||||
|
||||
class _ClosableSse(httpx.SyncByteStream):
|
||||
def __init__(self):
|
||||
self.closed = 0
|
||||
|
||||
def __iter__(self):
|
||||
yield (
|
||||
b'data: {"id":"x","object":"chat.completion.chunk","created":1,'
|
||||
b'"model":"gpt","choices":[{"index":0,"delta":{"content":"hi"},'
|
||||
b'"finish_reason":null}]}\n\n'
|
||||
)
|
||||
yield b"data: [DONE]\n\n"
|
||||
|
||||
def close(self):
|
||||
self.closed += 1
|
||||
|
||||
|
||||
def _openai_stream(body: _ClosableSse):
|
||||
def handler(request: httpx.Request):
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
stream=body,
|
||||
)
|
||||
|
||||
http_client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
client = OpenAI(api_key="test", http_client=http_client)
|
||||
stream = client.chat.completions.create(
|
||||
model="gpt-test",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
stream=True,
|
||||
)
|
||||
return stream, http_client
|
||||
|
||||
|
||||
def test_wrap_stream_closes_source_before_iteration_and_saves_once():
|
||||
owner = _Owner()
|
||||
source = _ClosableIterator(["chunk"])
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
wrapped.close()
|
||||
wrapped.close()
|
||||
|
||||
assert source.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_stream_closes_source_after_exhaustion_and_saves_once():
|
||||
owner = _Owner()
|
||||
source = _ClosableIterator(["chunk"])
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
assert list(wrapped) == ["chunk"]
|
||||
wrapped.close()
|
||||
|
||||
assert source.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_stream_does_not_iterate_after_close():
|
||||
owner = _Owner()
|
||||
source = _ClosableIterator(["chunk"])
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
wrapped.close()
|
||||
|
||||
with pytest.raises(StopIteration):
|
||||
next(wrapped)
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_stream_closes_source_on_error_and_saves_once():
|
||||
owner = _Owner()
|
||||
source = _ClosableIterator(error=RuntimeError("boom"))
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
with pytest.raises(RuntimeError, match="boom"):
|
||||
next(wrapped)
|
||||
|
||||
assert source.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_stream_supports_iterators_without_close():
|
||||
owner = _Owner()
|
||||
wrapped = HeadroomClient._wrap_stream(owner, iter(["chunk"]), "metrics")
|
||||
|
||||
assert list(wrapped) == ["chunk"]
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_stream_supports_iterable_only_sources():
|
||||
owner = _Owner()
|
||||
source = _IterableOnly(["chunk"])
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
assert list(wrapped) == ["chunk"]
|
||||
assert source.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_stream_preserves_iteration_error_when_close_fails():
|
||||
owner = _Owner()
|
||||
source = _ClosableIterator(
|
||||
error=RuntimeError("iteration"),
|
||||
close_error=RuntimeError("close"),
|
||||
)
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
with pytest.raises(RuntimeError, match="iteration"):
|
||||
next(wrapped)
|
||||
|
||||
assert source.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_stream_closes_retained_iterator_when_source_has_no_close():
|
||||
owner = _Owner()
|
||||
source = _RetainedIterable()
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
assert next(wrapped) == "chunk"
|
||||
wrapped.close()
|
||||
|
||||
assert source.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_stream_closes_distinct_source_and_iterator():
|
||||
owner = _Owner()
|
||||
source = _DistinctClosableSource()
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
assert next(wrapped) == "chunk"
|
||||
wrapped.close()
|
||||
|
||||
assert source.closed == 1
|
||||
assert source.iterator.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_stream_closes_both_owners_when_each_close_fails():
|
||||
owner = _Owner()
|
||||
source = _DistinctClosableSource(
|
||||
owner_error=RuntimeError("owner close"),
|
||||
iterator_error=RuntimeError("iterator close"),
|
||||
)
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
assert next(wrapped) == "chunk"
|
||||
wrapped.close()
|
||||
wrapped.close()
|
||||
|
||||
assert source.closed == 1
|
||||
assert source.iterator.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_stream_finalizes_started_abandoned_stream():
|
||||
owner = _Owner()
|
||||
source = _ClosableIterator(["chunk"])
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
next(wrapped)
|
||||
del wrapped
|
||||
gc.collect()
|
||||
|
||||
assert source.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_closed_stream_releases_source_and_owner():
|
||||
owner = _Owner()
|
||||
source = _ClosableIterator(["chunk"])
|
||||
owner_ref, source_ref = weakref.ref(owner), weakref.ref(source)
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
wrapped.close()
|
||||
del owner, source
|
||||
gc.collect()
|
||||
|
||||
assert owner_ref() is None
|
||||
assert source_ref() is None
|
||||
|
||||
|
||||
def test_wrap_stream_context_manager_closes_on_consumer_error():
|
||||
owner = _Owner()
|
||||
source = _ClosableIterator(["chunk"])
|
||||
|
||||
with pytest.raises(RuntimeError, match="consumer"):
|
||||
with HeadroomClient._wrap_stream(owner, source, "metrics") as wrapped:
|
||||
assert next(wrapped) == "chunk"
|
||||
raise RuntimeError("consumer")
|
||||
|
||||
assert source.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_real_openai_stream_closes_before_first_chunk():
|
||||
body = _ClosableSse()
|
||||
source, http_client = _openai_stream(body)
|
||||
owner = _Owner()
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
wrapped.close()
|
||||
http_client.close()
|
||||
|
||||
assert body.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
|
||||
|
||||
def test_wrap_real_openai_stream_closes_after_first_chunk():
|
||||
body = _ClosableSse()
|
||||
source, http_client = _openai_stream(body)
|
||||
owner = _Owner()
|
||||
wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")
|
||||
|
||||
next(wrapped)
|
||||
wrapped.close()
|
||||
http_client.close()
|
||||
|
||||
assert body.closed == 1
|
||||
assert owner.saved == ["metrics"]
|
||||
Reference in New Issue
Block a user