mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
187 lines
6.9 KiB
Python
187 lines
6.9 KiB
Python
"""Backend startup/liveness supervision, decoupled from run_api_server so it is unit
|
|
testable without spawning real processes.
|
|
|
|
The scheduler/tokenizer worker processes are spawned by launch.start_subprocess, which
|
|
hands back a BackendHandle. ``drain_ready`` blocks until every worker has acked ready
|
|
(applying interleaved ("progress", desc, done, total) messages to a LoadProgress),
|
|
after which the caller flips the maintenance gate to "serving". Task 3 adds liveness
|
|
watching on top of this.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from queue import Empty
|
|
from typing import Any, Callable, List
|
|
|
|
|
|
def phase_slug(desc: str) -> str:
|
|
"""Normalize a progress-bar desc into the stable /health phase enum. Order matters:
|
|
the expert bars ("Loading experts …", "Loading expert banks …") both contain 'expert'
|
|
and must win over the generic 'weight' match ("Loading weights (FTW)")."""
|
|
d = (desc or "").lower()
|
|
if "expert" in d:
|
|
return "expert_banks"
|
|
if "weight" in d:
|
|
return "weights"
|
|
if "graph" in d or "warm" in d:
|
|
return "warmup"
|
|
return "other"
|
|
|
|
|
|
@dataclass
|
|
class LoadProgress:
|
|
desc: str = ""
|
|
done_bytes: int = 0
|
|
total_bytes: int = 0
|
|
|
|
def update(self, desc: str, done: int, total: int) -> None:
|
|
self.desc = desc
|
|
self.done_bytes = done
|
|
self.total_bytes = total
|
|
|
|
@property
|
|
def phase(self) -> str:
|
|
return phase_slug(self.desc)
|
|
|
|
|
|
@dataclass
|
|
class BackendHandle:
|
|
"""What start_subprocess hands back so run_api_server can supervise the workers it
|
|
no longer blocks on."""
|
|
|
|
ack_queue: Any
|
|
processes: List[Any] = field(default_factory=list)
|
|
expected_acks: int = 0
|
|
|
|
|
|
class WorkerDied(Exception):
|
|
def __init__(self, message: str) -> None:
|
|
super().__init__(message)
|
|
self.message = message
|
|
|
|
|
|
def _first_dead(processes: List[Any]) -> Any | None:
|
|
for p in processes:
|
|
try:
|
|
if not p.is_alive():
|
|
return p
|
|
except Exception: # noqa: BLE001 — treat an unqueryable handle as alive
|
|
continue
|
|
return None
|
|
|
|
|
|
def _as_error(msg: Any) -> str | None:
|
|
"""The reason a dying worker pushed as ("error", reason), or None for any other ack — lets
|
|
the supervisor report the real cause (e.g. a config ValueError) rather than the generic
|
|
"exited during load"."""
|
|
if isinstance(msg, tuple) and msg and msg[0] == "error":
|
|
return str(msg[1]) if len(msg) > 1 else "backend worker failed during load"
|
|
return None
|
|
|
|
|
|
def _drain_pending_error(get: Callable[[float], Any], attempts: int = 5, timeout: float = 0.02) -> str | None:
|
|
"""After a worker death, poll the ack queue briefly for a final ("error", reason) ack still
|
|
in flight (the queue feeder flushes on process exit), so the real reason wins over the
|
|
generic message. Bounded so a genuinely reason-less death still fails fast."""
|
|
for _ in range(attempts):
|
|
try:
|
|
msg = get(timeout)
|
|
except Empty:
|
|
continue
|
|
err = _as_error(msg)
|
|
if err is not None:
|
|
return err
|
|
return None
|
|
|
|
|
|
def drain_ready(
|
|
handle: BackendHandle,
|
|
progress: LoadProgress,
|
|
get: Callable[[float], Any] | None = None,
|
|
poll: float = 1.0,
|
|
on_meta: Callable[[dict], None] | None = None,
|
|
) -> None:
|
|
"""Block until ``expected_acks`` non-progress acks arrive, applying ("progress", …)
|
|
tuples to ``progress``. Between acks, poll worker liveness; raise WorkerDied if a worker
|
|
exits before readiness (e.g. CUDA OOM / corrupt weights during load).
|
|
|
|
("meta", payload) tuples carry optional backend metadata (per-unit cache byte costs);
|
|
they are forwarded to ``on_meta`` and do NOT count toward readiness. meta is optional --
|
|
an engine build that never emits it must not stall this drain, so nothing waits for it."""
|
|
if get is None:
|
|
def get(timeout: float) -> Any:
|
|
return handle.ack_queue.get(timeout=timeout)
|
|
|
|
ready = 0
|
|
while ready < handle.expected_acks:
|
|
try:
|
|
msg = get(poll)
|
|
except Empty:
|
|
dead = _first_dead(handle.processes)
|
|
if dead is not None:
|
|
# The worker may have pushed its real failure reason just before exiting; give
|
|
# the queue a brief window to surface it, else fall back to the generic message.
|
|
reason = _drain_pending_error(get)
|
|
raise WorkerDied(reason or f"backend worker {getattr(dead, 'name', '?')} exited during load")
|
|
continue
|
|
err = _as_error(msg)
|
|
if err is not None:
|
|
raise WorkerDied(err)
|
|
if isinstance(msg, tuple) and msg and msg[0] == "progress":
|
|
_tag, desc, done, total = msg
|
|
progress.update(desc, done, total)
|
|
elif isinstance(msg, tuple) and msg and msg[0] == "meta":
|
|
if on_meta is not None:
|
|
on_meta(msg[1] if len(msg) > 1 else {})
|
|
else:
|
|
ready += 1
|
|
|
|
|
|
def run_backend_supervisor(
|
|
handle: BackendHandle,
|
|
progress: LoadProgress,
|
|
on_ready: Callable[[], None],
|
|
on_failure: Callable[[str], None] | None = None,
|
|
poll: float = 1.0,
|
|
on_meta: Callable[[dict], None] | None = None,
|
|
is_shutting_down: Callable[[], bool] | None = None,
|
|
) -> None:
|
|
"""Drain to readiness (watching liveness), flip the gate via ``on_ready``, then watch
|
|
forever: a worker that dies AFTER ready (a scheduler crash while the frontend keeps
|
|
answering 200) also fires ``on_failure`` — closing the half-alive-engine gap.
|
|
|
|
``on_meta`` receives the optional ("meta", …) backend metadata during the drain.
|
|
|
|
``is_shutting_down`` distinguishes a crash from an orderly stop: on SIGTERM/SIGINT
|
|
(``ft serve`` ^C, or the desktop's stop) the api server sets a shutting-down flag before
|
|
the workers exit, so their expected deaths are NOT misreported. When the flag is set at
|
|
the moment a death is observed, the supervisor returns silently — no ``on_failure``, no
|
|
ERROR log, no "failed" latch. A death seen while the flag is clear is a real crash and
|
|
fires ``on_failure`` as before."""
|
|
|
|
def _shutting_down() -> bool:
|
|
return bool(is_shutting_down is not None and is_shutting_down())
|
|
|
|
try:
|
|
drain_ready(handle, progress, poll=poll, on_meta=on_meta)
|
|
except WorkerDied as exc:
|
|
# A worker dying mid-load during an orderly shutdown is expected, not a load failure.
|
|
if not _shutting_down() and on_failure is not None:
|
|
on_failure(exc.message)
|
|
return
|
|
|
|
on_ready()
|
|
|
|
while True:
|
|
dead = _first_dead(handle.processes)
|
|
if dead is not None:
|
|
if _shutting_down():
|
|
# Orderly stop in progress: the worker exit is expected — stay quiet.
|
|
return
|
|
if on_failure is not None:
|
|
on_failure(f"backend worker {getattr(dead, 'name', '?')} exited")
|
|
return
|
|
time.sleep(poll)
|