mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(server): require an API key on every route but /health * feat(shell): send the server's API key * feat(ctl): send the server's API key * feat(launch): hand the server's API key to the agent * feat(daemon): pass the serve's API key as apiKey The daemon reaches the serve's /v1/stats and prepare-stop itself, so the key travels in the start request, reaches the serve through its environment and is kept in the state file for re-adoption. The daemon's own FREETOKEN_API_KEY is no longer passed down to the serve. * fix(launch): keep the API key out of the variable dsh's web search reads dsh's web search reads DEEPSEEK_API_KEY and sends it to api.deepseek.com, so dsh now reads the key from FREETOKEN_API_KEY through the apiKeyEnv of its llm-deepseek section, and DEEPSEEK_API_KEY stays the dummy. * fix(server): check the API key on OPTIONS requests * fix(launch): write agent configs that hold the API key owner-only * chore: drop comments that restate the code
329 lines
13 KiB
Python
329 lines
13 KiB
Python
"""The FreeToken shell's view of a running server: an ordinary API client.
|
|
|
|
Chat goes through ``POST /v1/chat/completions`` (SSE) -- the same endpoint opencode and the
|
|
desktop app use -- so the shell shares one code path with every other client: prompt rendering,
|
|
sampling defaults, the reasoning split, tool-call parsing, request accounting. The status bar
|
|
reads the public control plane (``/health``, ``/v1/stats``, ``/v1/cache/status``, ``/v1/models``),
|
|
and ``/cache`` drives ``POST /v1/cache/rebuild``. There is no shell-private server interface, so
|
|
``ft shell`` can attach to any server, in this process or on another machine.
|
|
|
|
Transport follows what the repo already does: the ``openai`` SDK for generation (like
|
|
``freetoken.benchmark.client``), stdlib ``urllib`` for the control-plane JSON (like ``ft ctl``
|
|
and the daemon). Nothing here imports torch -- attaching to a remote server costs a few
|
|
milliseconds of imports, not a CUDA context.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from collections.abc import AsyncIterator, Callable
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
import openai
|
|
from openai import AsyncOpenAI
|
|
|
|
# Sent when no key is given: a server without --api-key accepts any string, and the SDK needs one.
|
|
LOCAL_API_KEY = "freetoken-local"
|
|
CONTROL_TIMEOUT = 10.0
|
|
# Per-read timeout on the chat stream. Generous: it has to cover the gap before the first
|
|
# token, which is a full prefill of the whole conversation on a cold cache.
|
|
CHAT_READ_TIMEOUT = 900.0
|
|
READY_POLL_INTERVAL = 0.5
|
|
|
|
|
|
class ShellClientError(Exception):
|
|
"""A request to the server failed, or the server refused it. Carries a message meant to be
|
|
printed to the user as-is."""
|
|
|
|
def __init__(self, message: str, *, status: int | None = None) -> None:
|
|
super().__init__(message)
|
|
self.status = status
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReasoningDelta:
|
|
text: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ContentDelta:
|
|
text: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TurnDone:
|
|
finish_reason: str | None = None
|
|
prompt_tokens: int = 0
|
|
completion_tokens: int = 0
|
|
|
|
|
|
ChatEvent = ReasoningDelta | ContentDelta | TurnDone
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Sampling:
|
|
max_tokens: int | None = None
|
|
temperature: float | None = None
|
|
top_p: float | None = None
|
|
top_k: int | None = None
|
|
|
|
|
|
def _error_text(payload: Any) -> str | None:
|
|
"""Pull a human-readable message out of the two error shapes the server emits: the OpenAI
|
|
envelope ``{"error": {"message": ...}}`` and the plain ``{"error": "..."}`` / ``{"detail":
|
|
...}`` used by the maintenance gates."""
|
|
if isinstance(payload, str):
|
|
return payload or None
|
|
if not isinstance(payload, dict):
|
|
return None
|
|
for key in ("error", "detail", "message"):
|
|
value = payload.get(key)
|
|
if isinstance(value, dict):
|
|
nested = _error_text(value)
|
|
if nested:
|
|
return nested
|
|
elif value:
|
|
return str(value)
|
|
return None
|
|
|
|
|
|
class ShellClient:
|
|
def __init__(
|
|
self,
|
|
origin: str,
|
|
*,
|
|
api_key: str | None = None,
|
|
timeout: float = CONTROL_TIMEOUT,
|
|
) -> None:
|
|
self.origin = origin.rstrip("/")
|
|
self.timeout = timeout
|
|
self._api_key = api_key or LOCAL_API_KEY
|
|
self._openai = AsyncOpenAI(
|
|
base_url=f"{self.origin}/v1",
|
|
api_key=self._api_key,
|
|
max_retries=0, # a retried chat request would generate the turn twice
|
|
timeout=openai.Timeout(CHAT_READ_TIMEOUT, connect=10.0),
|
|
)
|
|
|
|
async def aclose(self) -> None:
|
|
await self._openai.close()
|
|
|
|
# ---------------------------------------------------------------- control plane
|
|
|
|
def _request_json_blocking(
|
|
self, method: str, path: str, body: dict[str, Any] | None, timeout: float
|
|
) -> dict[str, Any]:
|
|
data = None
|
|
headers = {"Accept": "application/json", "Authorization": f"Bearer {self._api_key}"}
|
|
if body is not None:
|
|
data = json.dumps(body).encode("utf-8")
|
|
headers["Content-Type"] = "application/json"
|
|
request = urllib.request.Request(
|
|
f"{self.origin}{path}", data=data, headers=headers, method=method
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
raw = response.read()
|
|
except urllib.error.HTTPError as exc:
|
|
# A rebuild rejection (409/503) is a normal answer, not a transport failure: the
|
|
# body carries the status the caller wants to report, so hand it back as data.
|
|
raw = exc.read()
|
|
try:
|
|
doc = json.loads(raw.decode("utf-8"))
|
|
except (json.JSONDecodeError, UnicodeDecodeError):
|
|
doc = None
|
|
if isinstance(doc, dict) and doc.get("status"):
|
|
return doc
|
|
raise ShellClientError(
|
|
f"HTTP {exc.code}: {_error_text(doc) or exc.reason}", status=exc.code
|
|
) from exc
|
|
except urllib.error.URLError as exc:
|
|
raise ShellClientError(f"cannot reach {self.origin}: {exc.reason}") from exc
|
|
except (TimeoutError, OSError) as exc:
|
|
raise ShellClientError(f"cannot reach {self.origin}: {exc}") from exc
|
|
if not raw:
|
|
return {}
|
|
try:
|
|
doc = json.loads(raw.decode("utf-8"))
|
|
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
|
raise ShellClientError(f"{path} returned invalid JSON") from exc
|
|
if not isinstance(doc, dict):
|
|
raise ShellClientError(f"{path} returned a non-object JSON document")
|
|
return doc
|
|
|
|
async def _request_json(
|
|
self,
|
|
method: str,
|
|
path: str,
|
|
*,
|
|
body: dict[str, Any] | None = None,
|
|
timeout: float | None = None,
|
|
) -> dict[str, Any]:
|
|
return await asyncio.to_thread(
|
|
self._request_json_blocking, method, path, body, timeout or self.timeout
|
|
)
|
|
|
|
async def health(self) -> dict[str, Any]:
|
|
return await self._request_json("GET", "/health")
|
|
|
|
async def stats(self) -> dict[str, Any]:
|
|
return await self._request_json("GET", "/v1/stats")
|
|
|
|
async def cache_status(self) -> dict[str, Any]:
|
|
return await self._request_json("GET", "/v1/cache/status")
|
|
|
|
async def cache_rebuild(
|
|
self,
|
|
*,
|
|
moe_cache_size: int | None = None,
|
|
num_pages: int | None = None,
|
|
num_mamba_slots: int | None = None,
|
|
num_swa_pages: int | None = None,
|
|
wait: float = 300.0,
|
|
) -> dict[str, Any]:
|
|
"""Resize cache pools. Every count is in the endpoint's own unit -- slots for moe/mamba,
|
|
pages for kv and for the window pool (whose page is P on DSV4, 1 token on radix-SWA).
|
|
Untargeted pools are left out of the body entirely, which the server reads as 'keep'."""
|
|
body: dict[str, Any] = {"timeout": wait}
|
|
for key, value in (
|
|
("moe_cache_size", moe_cache_size),
|
|
("num_pages", num_pages),
|
|
("num_mamba_slots", num_mamba_slots),
|
|
("num_swa_pages", num_swa_pages),
|
|
):
|
|
if value is not None:
|
|
body[key] = value
|
|
# The server blocks for the whole rebuild, so the client must outwait it.
|
|
return await self._request_json(
|
|
"POST", "/v1/cache/rebuild", body=body, timeout=wait + self.timeout
|
|
)
|
|
|
|
async def model_id(self) -> str | None:
|
|
"""The first id from ``/v1/models``. None when the server reports none (it is a single-
|
|
model server, so this is the model the shell will be talking to)."""
|
|
doc = await self._request_json("GET", "/v1/models")
|
|
data = doc.get("data")
|
|
if not isinstance(data, list):
|
|
return None
|
|
for item in data:
|
|
if isinstance(item, dict) and isinstance(item.get("id"), str):
|
|
return item["id"]
|
|
return None
|
|
|
|
async def wait_until_ready(
|
|
self,
|
|
*,
|
|
on_progress: Callable[[dict[str, Any]], None] | None = None,
|
|
connect_grace: float = 0.0,
|
|
) -> dict[str, Any]:
|
|
"""Block until ``/health`` reports ok, returning that document.
|
|
|
|
``connect_grace`` is how long a *connection* failure is tolerated before giving up --
|
|
0 for attaching to a server that is supposed to be up already (so a typo'd address
|
|
fails immediately with the reason), a few seconds when we just started the server
|
|
ourselves and uvicorn may not have finished binding. A server that is up but still
|
|
loading weights is always waited on: that phase has no useful timeout, and ^C quits.
|
|
"""
|
|
deadline = time.monotonic() + connect_grace
|
|
while True:
|
|
try:
|
|
doc = await self.health()
|
|
except ShellClientError:
|
|
if time.monotonic() >= deadline:
|
|
raise
|
|
await asyncio.sleep(0.25)
|
|
continue
|
|
status = doc.get("status")
|
|
if status == "ok":
|
|
return doc
|
|
if status == "error":
|
|
raise ShellClientError(str(doc.get("message") or "the server reported an error"))
|
|
if on_progress is not None:
|
|
on_progress(doc)
|
|
await asyncio.sleep(READY_POLL_INTERVAL)
|
|
|
|
# ---------------------------------------------------------------- generation
|
|
|
|
async def chat(
|
|
self,
|
|
messages: list[dict[str, Any]],
|
|
*,
|
|
model: str,
|
|
sampling: Sampling,
|
|
chat_template_kwargs: dict[str, Any] | None = None,
|
|
) -> AsyncIterator[ChatEvent]:
|
|
"""Stream one turn. Yields reasoning/content deltas as the server splits them, then
|
|
exactly one :class:`TurnDone` carrying the authoritative token usage.
|
|
|
|
Cancelling the iteration (``break``, ^C) closes the HTTP response, which the server
|
|
reads as a client disconnect and turns into an abort -- the engine stops decoding
|
|
instead of finishing a turn nobody is reading.
|
|
"""
|
|
extra_body: dict[str, Any] = {}
|
|
if chat_template_kwargs:
|
|
extra_body["chat_template_kwargs"] = chat_template_kwargs
|
|
if sampling.top_k is not None:
|
|
extra_body["top_k"] = sampling.top_k # not an OpenAI field; FreeToken accepts it
|
|
|
|
request: dict[str, Any] = {
|
|
"model": model,
|
|
"messages": messages,
|
|
"stream": True,
|
|
"stream_options": {"include_usage": True},
|
|
}
|
|
if sampling.max_tokens is not None:
|
|
request["max_tokens"] = sampling.max_tokens
|
|
if sampling.temperature is not None:
|
|
request["temperature"] = sampling.temperature
|
|
if sampling.top_p is not None:
|
|
request["top_p"] = sampling.top_p
|
|
|
|
try:
|
|
stream = await self._openai.chat.completions.create(
|
|
**request, extra_body=extra_body or None
|
|
)
|
|
except openai.APIStatusError as exc:
|
|
raise ShellClientError(
|
|
_error_text(exc.body) or f"HTTP {exc.status_code}", status=exc.status_code
|
|
) from exc
|
|
except openai.APIConnectionError as exc:
|
|
raise ShellClientError(f"cannot reach {self.origin}: {exc}") from exc
|
|
|
|
finish_reason: str | None = None
|
|
prompt_tokens = 0
|
|
completion_tokens = 0
|
|
try:
|
|
async for chunk in stream:
|
|
usage = getattr(chunk, "usage", None)
|
|
if usage is not None:
|
|
prompt_tokens = int(getattr(usage, "prompt_tokens", 0) or 0)
|
|
completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
|
|
for choice in chunk.choices or []:
|
|
if choice.finish_reason:
|
|
finish_reason = choice.finish_reason
|
|
delta = choice.delta
|
|
if delta is None:
|
|
continue
|
|
# reasoning_content is FreeToken's (and vLLM/SGLang's) split channel; the
|
|
# SDK keeps unknown fields, so read it off the model as an extra.
|
|
reasoning = getattr(delta, "reasoning_content", None)
|
|
if reasoning:
|
|
yield ReasoningDelta(reasoning)
|
|
if delta.content:
|
|
yield ContentDelta(delta.content)
|
|
except openai.APIError as exc:
|
|
raise ShellClientError(f"stream failed: {exc}") from exc
|
|
finally:
|
|
await stream.close()
|
|
|
|
yield TurnDone(
|
|
finish_reason=finish_reason,
|
|
prompt_tokens=prompt_tokens,
|
|
completion_tokens=completion_tokens,
|
|
)
|