Files
Xiaoze Fan 9528190454 feat(server): add --api-key authentication (#639)
* feat(server): require an API key on every route but /health

* feat(shell): send the server's API key

* feat(ctl): send the server's API key

* feat(launch): hand the server's API key to the agent

* feat(daemon): pass the serve's API key as apiKey

The daemon reaches the serve's /v1/stats and prepare-stop itself, so the key
travels in the start request, reaches the serve through its environment and is
kept in the state file for re-adoption. The daemon's own FREETOKEN_API_KEY is no
longer passed down to the serve.

* fix(launch): keep the API key out of the variable dsh's web search reads

dsh's web search reads DEEPSEEK_API_KEY and sends it to api.deepseek.com, so
dsh now reads the key from FREETOKEN_API_KEY through the apiKeyEnv of its
llm-deepseek section, and DEEPSEEK_API_KEY stays the dummy.

* fix(server): check the API key on OPTIONS requests

* fix(launch): write agent configs that hold the API key owner-only

* chore: drop comments that restate the code
2026-10-10 00:36:04 -07:00

530 lines
20 KiB
Python

"""The daemon's HTTP control plane. camelCase JSON throughout. Loopback by default; an optional
``X-FT-Token`` shared secret gates everything except the daemon's own ``/health`` liveness probe.
Handlers are ``async`` and push every blocking call to an executor so the event loop never
blocks. Two executors: a small **lifecycle** pool for start/stop/switch, kept separate from the
**proxy/metrics** pool, so a storm of health/metrics polls against a loading serve can never
starve an operator's stop."""
from __future__ import annotations
import asyncio
import collections
import functools
import json
import os
import sys
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Callable
from fastapi import Depends, FastAPI, Header, HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse
from pydantic import BaseModel
from .accounting import AccountingOutboxError, AccountingPrepareError
from .serve_manager import Conflict
from .version import DAEMON_VERSION
class StartBody(BaseModel):
model: str
port: int | None = None
args: list[str] = []
apiKey: str | None = None
class StopBody(BaseModel):
force: bool = False
class SwitchBody(StartBody):
force: bool = False
class AccountingAckBody(BaseModel):
receiptId: str
class CheckpointBody(BaseModel):
id: str
args: list[str] = []
class CancelBody(BaseModel):
id: str
class BenchBody(BaseModel):
# Raw `ft bench bw` args (e.g. ["--dtype", "nvfp4", "--threshold", "2.5"]); empty = all dtypes.
args: list[str] = []
def _bench_profile_path(gpu_uuid: str | None) -> str | None:
# per-GPU profiles and no torch here: the serve's own card when its --gpu names one, else the newest file
from freetoken.moe.bench_profile import default_profile_path, latest_profile_path # torch-free
if gpu_uuid:
path = default_profile_path(gpu_uuid)
if os.path.isfile(path):
return path
return latest_profile_path()
def _serve_gpu_uuid(args: list[str]) -> str | None:
"""The full UUID a serve's `--gpu` pins, or None when there is none or it cannot be resolved."""
for i, a in enumerate(args):
val = a[len("--gpu="):] if a.startswith("--gpu=") else (args[i + 1] if a == "--gpu" and i + 1 < len(args) else None)
if not val:
continue
from freetoken.gpu_select import resolve_gpu_uuids
try:
resolved = resolve_gpu_uuids([val])
except ValueError:
return None
if resolved:
return resolved[0]
# no NVML: a UUID value still keys the profile file (canonical prefix), an index cannot
return "GPU-" + val[len("GPU-"):] if val.upper().startswith("GPU-") else None
return None
def _read_bench_profile(path: str | None) -> dict | None:
if path is None:
return None
try:
with open(path) as f:
return json.load(f)
except (OSError, ValueError):
return None
def _bench_sse(event: str, data) -> str:
return f"event: {event}\ndata: {json.dumps(data)}\n\n"
def _parse_ftbench(line: str) -> dict | None:
"""``FTBENCH <done> <total> <label>`` -> a progress dict (mirrors ft checkpoint's FTCONVERT)."""
parts = line.split(maxsplit=3)
if len(parts) < 4 or parts[0] != "FTBENCH":
return None
try:
return {"done": int(parts[1]), "total": int(parts[2]), "label": parts[3]}
except ValueError:
return None
def build_app(
*,
manager,
ring,
probe,
footprint_fn: Callable[[int | None], dict],
lifecycle_pool: ThreadPoolExecutor,
proxy_pool: ThreadPoolExecutor,
default_serve_port: int = 1919,
token: str | None = None,
checkpoints=None,
started_wall: float = 0.0,
wall_now: Callable[[], float] | None = None,
shutdown_hook: Callable[[], None] | None = None,
) -> FastAPI:
import time as _time
wall_now = wall_now or _time.time
app = FastAPI(title="FreeToken daemon", version=DAEMON_VERSION)
if shutdown_hook is not None:
@app.on_event("shutdown")
async def _on_shutdown() -> None:
# uvicorn fires this on SIGTERM/SIGINT. Run the (blocking) hook off-loop so the grace
# period in stop() can't wedge the event loop during shutdown.
loop = asyncio.get_running_loop()
try:
await loop.run_in_executor(None, shutdown_hook)
except Exception: # noqa: BLE001
pass
def require_token(x_ft_token: str | None = Header(default=None)) -> None:
if token is not None and x_ft_token != token:
raise HTTPException(status_code=401, detail="invalid or missing X-FT-Token")
auth = [Depends(require_token)]
async def run(pool: ThreadPoolExecutor, fn, *args):
loop = asyncio.get_running_loop()
return await loop.run_in_executor(pool, functools.partial(fn, *args))
def resolve_port(explicit: int | None) -> int:
if explicit is not None:
return explicit
st = manager.status()
return st.get("port") or default_serve_port
def reject_api_key_arg(args: list[str]) -> None:
# the daemon must know the key to reach the serve, so the key only travels as apiKey
# argparse also accepts any unambiguous prefix, and no other serve flag starts with --ap
flags = (a.split("=", 1)[0] for a in args)
if any(len(f) >= 4 and "--api-key".startswith(f) for f in flags):
raise HTTPException(
status_code=400, detail="pass the serve's API key as apiKey, not in args"
)
def accounting_error(exc: Exception) -> JSONResponse:
code = (
"accounting_outbox_failed"
if isinstance(exc, AccountingOutboxError)
else "accounting_prepare_failed"
)
return JSONResponse(
status_code=503,
content={
"error": str(exc),
"code": code,
"enginePreserved": True,
},
)
# ---- daemon self-health (never gated; always answers if the daemon is up) ----
@app.get("/health")
async def health():
st = manager.status()
return {
"status": "ok",
"version": DAEMON_VERSION,
"uptimeS": int(wall_now() - started_wall) if started_wall else 0,
"engineRunning": bool(st.get("running")),
}
# ---- engine lifecycle ----
@app.post("/engine/start", dependencies=auth)
async def engine_start(body: StartBody):
reject_api_key_arg(body.args)
port = resolve_port(body.port)
try:
return await run(
lifecycle_pool,
manager.start,
body.model,
port,
list(body.args),
body.apiKey or None,
)
except Conflict as exc:
st = manager.status()
return JSONResponse(
status_code=409,
content={
"error": str(exc),
"code": "serve_conflict",
"currentModel": st.get("model"),
"currentPort": st.get("port"),
},
)
except Exception as exc: # noqa: BLE001 — never propagate a 500-as-crash
raise HTTPException(status_code=500, detail=f"start failed: {exc}")
@app.post("/engine/stop", dependencies=auth)
async def engine_stop(body: StopBody | None = None):
try:
return await run(lifecycle_pool, manager.stop, None, bool(body and body.force))
except (AccountingPrepareError, AccountingOutboxError) as exc:
return accounting_error(exc)
@app.post("/shutdown", dependencies=auth)
async def shutdown_daemon(request: Request, body: StopBody | None = None):
# Tray "Stop daemon" stops everything: stop the engine FIRST so the default detach-on-exit can't
# leave the ~18GB serve orphaned, THEN bring the daemon down. We reply before uvicorn
# actually stops (it notices should_exit within ~0.1s) so the client still gets a clean 200.
try:
stopped = await run(lifecycle_pool, manager.shutdown, None, bool(body and body.force))
except (AccountingPrepareError, AccountingOutboxError) as exc:
return accounting_error(exc)
req = getattr(request.app.state, "request_shutdown", None)
if req is not None:
req()
return {
"stopping": True,
"already": stopped.get("already", False),
"accounting": stopped.get("accounting"),
}
@app.post("/engine/switch", dependencies=auth)
async def engine_switch(body: SwitchBody):
reject_api_key_arg(body.args)
port = resolve_port(body.port)
try:
return await run(
lifecycle_pool,
manager.switch,
body.model,
port,
list(body.args),
body.force,
body.apiKey or None,
)
except (AccountingPrepareError, AccountingOutboxError) as exc:
return accounting_error(exc)
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=500, detail=f"switch failed: {exc}")
# ---- durable accounting outbox ----
@app.get("/accounting/pending", dependencies=auth)
async def accounting_pending():
try:
receipts = await run(lifecycle_pool, manager.pending_accounting)
except AccountingOutboxError as exc:
return accounting_error(exc)
return {"receipts": receipts}
@app.post("/accounting/ack", dependencies=auth)
async def accounting_ack(body: AccountingAckBody):
try:
return await run(lifecycle_pool, manager.ack_accounting, body.receiptId)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc))
except AccountingOutboxError as exc:
return accounting_error(exc)
@app.get("/engine/status", dependencies=auth)
async def engine_status():
return manager.status()
@app.get("/engine/metrics", dependencies=auth)
async def engine_metrics():
pid = manager.current_pid()
return await run(proxy_pool, footprint_fn, pid)
@app.get("/engine/health", dependencies=auth)
async def engine_health():
st = manager.status()
if not st.get("running"):
return {"reachable": False, "running": False, "daemon": "up", **_engine_summary(st)}
port = st.get("port") or default_serve_port
doc = await run(proxy_pool, probe.health, port)
# The serve's own health fields (status/model/uptimeS/progress) are authoritative for
# "how is the model doing?"; the daemon only layers on what only it knows, never clobbering
# the serve's values.
doc["running"] = True
doc["daemon"] = "up"
doc.setdefault("port", st.get("port"))
doc.setdefault("pid", st.get("pid"))
doc.setdefault("lastExitCode", st.get("lastExitCode"))
return doc
@app.get("/engine/stats", dependencies=auth)
async def engine_stats():
st = manager.status()
if not st.get("running"):
return {"reachable": False, "running": False}
port = st.get("port") or default_serve_port
doc = await run(proxy_pool, probe.stats, port)
manager.observe_accounting(doc)
return doc
@app.get("/engine/logs", dependencies=auth)
async def engine_logs(request: Request, since: int = 0):
return _log_stream(request, ring, since)
# ---- checkpoint (phase 3; optional) ----
if checkpoints is not None:
@app.post("/checkpoint/start", dependencies=auth)
async def checkpoint_start(body: CheckpointBody):
# GPU exclusivity: a convert needs the GPU, so stop any serve first.
await run(lifecycle_pool, manager.stop)
try:
return await run(lifecycle_pool, checkpoints.start, body.id, list(body.args))
except Conflict as exc:
raise HTTPException(status_code=409, detail=str(exc))
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=500, detail=f"checkpoint start failed: {exc}")
@app.post("/checkpoint/cancel", dependencies=auth)
async def checkpoint_cancel(body: CancelBody):
return await run(lifecycle_pool, checkpoints.cancel, body.id)
@app.get("/checkpoint/status", dependencies=auth)
async def checkpoint_status():
return checkpoints.status()
# ---- hardware bandwidth bench (hardware-adaptive config) ----
@app.post("/bench/run", dependencies=auth)
async def bench_run(body: BenchBody):
# GPU exclusivity: the bench allocates transient device memory, so stop any serve first
# (mirrors /checkpoint/start). Runs `ft bench bw` on the engine HOST (so the profile lands
# where this daemon's serve reads it) and STREAMS progress back as SSE: `progress` events
# per measured format, then a terminal `result` (the profile) or `error` event. `body.args`
# is the raw arg list, so any `ft bench bw` flag (--dtype/--model/--threshold/...) passes
# through. torch stays out of the daemon (child process), which also frees VRAM on exit.
await run(lifecycle_pool, manager.stop)
async def gen():
env = {**os.environ, "FREETOKEN_BENCH_PROGRESS": "1"}
argv = [sys.executable, "-m", "freetoken.cli", "bench", "bw", *body.args]
try:
proc = await asyncio.create_subprocess_exec(
*argv, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, env=env
)
except Exception as exc: # noqa: BLE001
yield _bench_sse("error", {"message": f"failed to spawn bench: {exc}"})
return
tail: collections.deque = collections.deque(maxlen=8) # last non-progress lines (errors)
out_path: str | None = None
assert proc.stdout is not None
async for raw in proc.stdout:
line = raw.decode(errors="replace").rstrip()
prog = _parse_ftbench(line)
if prog is not None:
yield _bench_sse("progress", prog)
elif line.startswith("FTBENCH_OUT "):
out_path = line[len("FTBENCH_OUT "):]
elif line:
tail.append(line)
rc = await proc.wait()
if rc != 0:
yield _bench_sse("error", {"message": "\n".join(tail) or f"bench exited {rc}"})
return
# the file this run wrote (an older engine prints no FTBENCH_OUT: newest file, as before)
prof = _read_bench_profile(out_path or _bench_profile_path(None))
if prof is None:
yield _bench_sse("error", {"message": "bench finished but no profile was written"})
else:
yield _bench_sse("result", prof)
return StreamingResponse(gen(), media_type="text/event-stream")
@app.get("/bench/profile", dependencies=auth)
async def bench_profile():
def read() -> dict | None:
return _read_bench_profile(_bench_profile_path(serve_gpu_uuid()))
def serve_gpu_uuid() -> str | None:
# the running serve reports the full UUID of its card (/v1/stats gpus); a --gpu given as
# a UUID prefix would not match the profile file name
st = manager.status()
if st.get("running"):
try:
gpus = probe.stats(st.get("port") or default_serve_port).get("gpus") or []
if gpus and gpus[0].get("uuid"):
return gpus[0]["uuid"]
except Exception: # noqa: BLE001 -- the arg below is the fallback
pass
return _serve_gpu_uuid(manager.serve_args())
return await run(proxy_pool, read)
return app
def _engine_summary(st: dict) -> dict:
return {
"model": st.get("model"),
"port": st.get("port"),
"pid": st.get("pid"),
"uptimeS": st.get("uptimeS", 0),
"lastExitCode": st.get("lastExitCode"),
}
def _sse(rec: dict) -> str:
return f"id: {rec['seq']}\ndata: {json.dumps(rec)}\n\n"
def _sse_gap(dropped: int, from_seq: Any, to_seq: Any) -> str:
payload = {"kind": "gap", "dropped": dropped, "fromSeq": from_seq, "toSeq": to_seq}
return f"data: {json.dumps(payload)}\n\n"
def _log_stream(request: Request, ring, since: int) -> StreamingResponse:
"""SSE log stream with replay + live tail. Correctness points:
* subscribe BEFORE snapshotting the backlog, then dedupe live records by seq → no gap and
no duplicate across the replay→live boundary;
* per-subscriber bounded queue, drop-oldest on overflow via ``call_soon_threadsafe`` (the
mutation runs on the loop thread, so the reader never blocks) and a client-visible gap
sentinel so a slow client knows it lost lines;
* ``id:<seq>`` on every frame + ``Last-Event-ID`` honoured for native EventSource resume;
* a 15 s heartbeat + ``is_disconnected`` check so an idle client's disconnect is detected
and the subscriber is always removed in ``finally`` (no leak)."""
loop = asyncio.get_running_loop()
lei = request.headers.get("last-event-id")
if lei and lei.isdigit():
since = int(lei) + 1 # exclusive next-cursor
q: asyncio.Queue = asyncio.Queue(maxsize=1000)
drop = {"n": 0, "from": None, "to": None}
# Records with seq < boundary are already covered by the replayed backlog (they landed in the
# window between subscribe and the snapshot). Skipping them here keeps the gap counters honest
# — only genuinely-lost LIVE lines feed drop[]. Safe to set after subscribe: the
# scheduled _put callbacks only run once this handler yields control, by which point boundary
# is set.
boundary = {"v": 0}
def push(rec: dict) -> None:
def _put() -> None:
if rec["seq"] < boundary["v"]:
return # already delivered via backlog; don't enqueue or count it as dropped
if q.full():
try:
old = q.get_nowait()
drop["n"] += 1
if drop["from"] is None:
drop["from"] = old["seq"]
drop["to"] = old["seq"]
except asyncio.QueueEmpty: # pragma: no cover - race-only
pass
q.put_nowait(rec)
try:
loop.call_soon_threadsafe(_put)
except RuntimeError: # loop is closing during shutdown
pass
ring.subscribe(push)
backlog, cursor = ring.since(since)
boundary["v"] = cursor
async def gen():
try:
# If the ring evicted records at/after the client's cursor before it (re)connected,
# announce that lost prefix so the client knows its history is incomplete.
oldest = backlog[0]["seq"] if backlog else cursor
if oldest > since:
yield _sse_gap(oldest - since, since, oldest - 1)
for rec in backlog:
yield _sse(rec)
last_seq = cursor - 1
while True:
if await request.is_disconnected():
break
try:
rec = await asyncio.wait_for(q.get(), timeout=15.0)
except asyncio.TimeoutError:
yield ": ping\n\n"
continue
if rec["seq"] <= last_seq:
continue # already delivered in backlog
if drop["n"]:
# Snapshot + reset synchronously BEFORE yielding: during the yield the loop
# drains more _put callbacks that may mutate drop[], and those must not be
# wiped unreported.
n, frm, to = drop["n"], drop["from"], drop["to"]
drop["n"], drop["from"], drop["to"] = 0, None, None
yield _sse_gap(n, frm, to)
last_seq = rec["seq"]
yield _sse(rec)
finally:
ring.unsubscribe(push)
return StreamingResponse(
gen(),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)