mirror of
https://github.com/TencentCloud/Octop.git
synced 2026-10-12 04:59:23 +08:00
Dictate with live partial results: the composer mic streams 16 kHz PCM over a WebSocket to Octop, which signs and proxies the audio to Tencent's realtime ASR V2 endpoint. Credentials never leave the server; the browser only learns a `stt_realtime` boolean from `/api/voice/active`. - settings: add a "realtime speech recognition" switch plus an AppID field to the Tencent Cloud voice drawer. Both live in `voice_providers.extra_json`, so no schema migration is needed. - interaction: with realtime on, the mic becomes press-and-hold (start on pointerdown, finish on release/leave/cancel) and stays available while a reply is streaming, so dictation can be queued. With the switch off the previous click-to-record one-shot upload path is unchanged. - protocol: realtime ASR V2 answers with `result` (`slice_type`, `index`, `voice_text_str`) while the docs describe `sentences`; both are normalized in `infra/voice/realtime.py`. - errors: new `VOICE_REALTIME_UNAVAILABLE` code plus localized copy for the Tencent realtime error codes (4002-4009, 5000-5002). Verified end-to-end against Tencent with real speech (edge-tts -> 16 kHz PCM over the live WebSocket) and by `make all`.
168 lines
6.2 KiB
Python
168 lines
6.2 KiB
Python
"""WebSocket / streaming helpers for integration tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import json
|
|
from collections.abc import AsyncIterator
|
|
from contextlib import asynccontextmanager
|
|
from typing import Any
|
|
from urllib.parse import unquote, urlsplit
|
|
|
|
import httpx
|
|
from starlette.websockets import WebSocketDisconnect
|
|
|
|
# A single-loop session should never stall; the cap keeps a regression from
|
|
# wedging the whole suite instead of failing this one test.
|
|
_WS_RECEIVE_TIMEOUT_S = 20.0
|
|
|
|
|
|
def ws_token(auth: dict[str, str]) -> str:
|
|
return auth["Authorization"].split(" ", 1)[1]
|
|
|
|
|
|
class ASGIWebSocketSession:
|
|
"""Drive an ASGI websocket endpoint on the caller's event loop.
|
|
|
|
starlette's sync ``TestClient`` runs the app on its own anyio portal loop.
|
|
That gives the process two loops over one ``OctopServer``, and loop-bound
|
|
primitives shared by both (``AgentManager._lock``) then deadlock or raise
|
|
"bound to a different event loop". Speaking ASGI directly keeps the
|
|
handler, the gateway workers and the test on the same loop, exactly like
|
|
uvicorn in production.
|
|
"""
|
|
|
|
def __init__(self, app: Any, path: str) -> None:
|
|
self._app = app
|
|
self._path = path
|
|
self._from_app: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
|
|
self._to_app: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
|
|
self._task: asyncio.Task[None] | None = None
|
|
|
|
def _scope(self) -> dict[str, Any]:
|
|
url = urlsplit(self._path)
|
|
return {
|
|
"type": "websocket",
|
|
"asgi": {"version": "3.0", "spec_version": "2.3"},
|
|
"http_version": "1.1",
|
|
"scheme": "ws",
|
|
"path": unquote(url.path),
|
|
"raw_path": url.path.encode("ascii"),
|
|
"query_string": url.query.encode("ascii"),
|
|
"root_path": "",
|
|
"headers": [(b"host", b"testserver")],
|
|
"client": ("testclient", 50000),
|
|
"server": ("testserver", 80),
|
|
"subprotocols": [],
|
|
"state": {},
|
|
"app": self._app,
|
|
}
|
|
|
|
async def _next_event(self, timeout: float) -> dict[str, Any]:
|
|
assert self._task is not None
|
|
getter: asyncio.Future[dict[str, Any]] = asyncio.ensure_future(self._from_app.get())
|
|
done, _pending = await asyncio.wait(
|
|
{getter, self._task},
|
|
timeout=timeout,
|
|
return_when=asyncio.FIRST_COMPLETED,
|
|
)
|
|
if getter in done:
|
|
return getter.result()
|
|
getter.cancel()
|
|
with contextlib.suppress(asyncio.CancelledError):
|
|
await getter
|
|
if self._task in done:
|
|
# Surface a handler crash instead of a bare timeout.
|
|
self._task.result()
|
|
raise AssertionError("websocket handler returned without sending an event")
|
|
raise TimeoutError(f"no websocket event after {timeout:.0f}s")
|
|
|
|
async def connect(self, timeout: float = _WS_RECEIVE_TIMEOUT_S) -> None:
|
|
self._task = asyncio.create_task(
|
|
self._app(self._scope(), self._to_app.get, self._from_app.put),
|
|
name=f"asgi-ws{self._path}",
|
|
)
|
|
await self._to_app.put({"type": "websocket.connect"})
|
|
event = await self._next_event(timeout)
|
|
if event["type"] == "websocket.close":
|
|
raise WebSocketDisconnect(event.get("code", 1000), event.get("reason"))
|
|
if event["type"] != "websocket.accept":
|
|
raise AssertionError(f"unexpected handshake event: {event['type']}")
|
|
|
|
async def send_json(self, payload: Any) -> None:
|
|
await self._to_app.put(
|
|
{"type": "websocket.receive", "text": json.dumps(payload, ensure_ascii=False)},
|
|
)
|
|
|
|
async def send_bytes(self, payload: bytes) -> None:
|
|
await self._to_app.put({"type": "websocket.receive", "bytes": payload})
|
|
|
|
async def receive_json(self, *, timeout: float = _WS_RECEIVE_TIMEOUT_S) -> dict[str, Any]:
|
|
event = await self._next_event(timeout)
|
|
if event["type"] == "websocket.close":
|
|
raise WebSocketDisconnect(event.get("code", 1000), event.get("reason"))
|
|
raw = event.get("text")
|
|
if raw is None:
|
|
raw = bytes(event.get("bytes") or b"").decode("utf-8")
|
|
parsed: dict[str, Any] = json.loads(raw)
|
|
return parsed
|
|
|
|
async def drain_turn(self, *, timeout: float = _WS_RECEIVE_TIMEOUT_S) -> list[dict[str, Any]]:
|
|
"""Collect frames until the turn reports ``done`` or ``error``."""
|
|
chunks: list[dict[str, Any]] = []
|
|
while True:
|
|
chunk = await self.receive_json(timeout=timeout)
|
|
chunks.append(chunk)
|
|
if chunk.get("type") in ("done", "error"):
|
|
return chunks
|
|
|
|
async def close(self) -> None:
|
|
if self._task is None:
|
|
return
|
|
task = self._task
|
|
self._task = None
|
|
await self._to_app.put({"type": "websocket.disconnect", "code": 1000})
|
|
try:
|
|
await asyncio.wait_for(task, timeout=_WS_RECEIVE_TIMEOUT_S)
|
|
except (asyncio.CancelledError, TimeoutError):
|
|
task.cancel()
|
|
with contextlib.suppress(asyncio.CancelledError):
|
|
await task
|
|
|
|
|
|
@asynccontextmanager
|
|
async def ws_connect(app: Any, path: str) -> AsyncIterator[ASGIWebSocketSession]:
|
|
"""Open an in-process websocket session against *app* on the current loop."""
|
|
session = ASGIWebSocketSession(app, path)
|
|
try:
|
|
await session.connect()
|
|
yield session
|
|
finally:
|
|
await session.close()
|
|
|
|
|
|
def chat_ws_path(agent_id: str, auth: dict[str, str]) -> str:
|
|
return f"/api/agents/{agent_id}/chat/ws?token={ws_token(auth)}"
|
|
|
|
|
|
async def ws_chat_turn(
|
|
client: httpx.AsyncClient,
|
|
agent_id: str,
|
|
auth: dict[str, str],
|
|
*,
|
|
mcp_servers: list[str] | None = None,
|
|
text: str = "hi",
|
|
) -> list[dict[str, Any]]:
|
|
body: dict[str, Any] = {
|
|
"type": "user_turn",
|
|
"text": text,
|
|
"messages": [{"role": "user", "content": text}],
|
|
}
|
|
if mcp_servers is not None:
|
|
body["mcp_servers"] = mcp_servers
|
|
app = client._octop_app # type: ignore[attr-defined]
|
|
async with ws_connect(app, chat_ws_path(agent_id, auth)) as ws:
|
|
await ws.send_json(body)
|
|
return await ws.drain_turn()
|