mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(mm): serve image input on the Qwen VL families - frontend: OpenAI image_url, Anthropic image blocks and Responses input_image (http(s), base64, gated file://) become MMItems in the tokenizer workers; image spans carry content pseudo ids so the radix cache keys them by content; mrope positions are precomputed per request - registry: EncoderSpec per family (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next register a vision encoder serving image) and ModelSpec.mm_processor; EngineConfig.active_encoders decides what a process builds, served_modalities what it accepts; --text-model-only, --mm-disable, --mm-encoder-weights, --mm-embed-cache-device, --mm-max-pixels - engine and scheduler: per-request encoder-cache row ownership, chunk planning on the batch (jobs, gather rows, scatter rows), items encoded right before the LM forward and gathered into batch.mm_embeds - models: the Qwen VL vision tower on the text tower's TP layers with DeepStack, block weights streamed from pinned host banks by default, merger outputs parked on the host between blocks; SupportsMultimodal (encode, place_encoder_weights) and a shared embed_input_ids - server: image inputs rejected with a client-facing reason when no vision encoder is served; GET /v1/stats reports model.input_modalities * feat(mm): replace the pixel cap with a token budget and processor kwargs - --image-min-tokens / --image-max-tokens replace --mm-max-pixels; MMProcessor.get_mm_processor_kwargs turns the MultimodalConfig into the processor call's arguments, the default passes mm.processor_kwargs through and Qwen VL converts the budget to pixel areas in size.shortest_edge / longest_edge - --mm-processor-kwargs hands a JSON object to the checkpoint's image processor call, applied after the token budget - MMProcessor is an ABC that owns the model path, the config and the lazily loaded image processor; get_mm_processor returns None when --mm-disable leaves no encoder to serve - the frontend tokenizer, the tokenizer worker and the offline LLM build one processor each and hand it to TokenizeManager, so the tokenize message carries no per-request pixel cap
229 lines
8.9 KiB
Python
229 lines
8.9 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
import multiprocessing as mp
|
|
import os
|
|
import sys
|
|
from dataclasses import replace
|
|
from typing import TYPE_CHECKING
|
|
|
|
from freetoken.distributed import DistributedInfo
|
|
from freetoken.utils import init_logger
|
|
|
|
if TYPE_CHECKING:
|
|
from .args import ServerArgs
|
|
from .supervisor import BackendHandle
|
|
|
|
|
|
def _report_startup_error(ack_queue: mp.Queue, exc: BaseException) -> None:
|
|
"""Tell the parent WHY this worker is dying — push an ("error", reason) ack before it exits,
|
|
so the supervisor reports the real cause (e.g. a config ValueError) instead of the generic
|
|
"backend worker … exited during load". Best-effort and flushed (close + join_thread), since
|
|
the process is about to terminate; a failure to report must never mask the original error."""
|
|
try:
|
|
ack_queue.put(("error", f"{type(exc).__name__}: {exc}"))
|
|
ack_queue.close()
|
|
ack_queue.join_thread()
|
|
except Exception: # noqa: BLE001 -- reporting is a nicety; never shadow the real exception
|
|
pass
|
|
|
|
|
|
def _detach_process_group() -> None:
|
|
"""Shell mode only: move this worker out of the terminal's foreground process group.
|
|
|
|
The shell binds ^C to "cancel this turn", but a terminal delivers SIGINT to the whole
|
|
foreground group — which, with the engine running in this same process, includes the
|
|
workers. They would take the same ^C and exit (``_run_scheduler`` below stops gracefully on
|
|
KeyboardInterrupt), leaving the shell chatting with a dead engine. Nothing depends on the
|
|
signal reaching them: the parent tears the workers down explicitly on every stop path
|
|
(uvicorn's lifespan, plus the SIGTERM/SIGHUP handler and the reap backstop in api_server).
|
|
|
|
``ft serve`` keeps the default — uvicorn owns ^C there, and the group-wide delivery is part
|
|
of how it stops."""
|
|
try:
|
|
os.setpgrp()
|
|
except OSError: # no job control (already a group leader / unusual environment)
|
|
pass
|
|
|
|
|
|
def _run_tokenize_worker(detach: bool, **kwargs) -> None:
|
|
"""Module-level so it survives the spawn pickle; exists only to detach the group first."""
|
|
if detach:
|
|
_detach_process_group()
|
|
from freetoken.tokenizer import tokenize_worker
|
|
|
|
tokenize_worker(**kwargs)
|
|
|
|
|
|
def _run_scheduler(args: ServerArgs, ack_queue: mp.Queue[str]) -> None:
|
|
if args.shell_mode:
|
|
_detach_process_group()
|
|
|
|
# published (not bound) here: the engine binds it after the allocator setup
|
|
from freetoken.gpu_select import set_assigned_gpu
|
|
|
|
# resolved UUIDs when we have them, the raw --gpu entries when NVML could not resolve them, else one CUDA ordinal per rank
|
|
targets = args.gpu_assigned or args.gpu or tuple(str(r) for r in range(args.tp_info.size))
|
|
set_assigned_gpu(targets[args.tp_info.rank])
|
|
|
|
import torch
|
|
from freetoken.scheduler import Scheduler
|
|
|
|
if args.tp_info.is_primary():
|
|
from freetoken.utils.progress import set_progress_sink
|
|
|
|
set_progress_sink(
|
|
lambda desc, done, total: ack_queue.put(("progress", desc, done, total))
|
|
)
|
|
|
|
with torch.inference_mode():
|
|
try:
|
|
scheduler = Scheduler(args)
|
|
scheduler.sync_all_ranks()
|
|
except Exception as exc: # noqa: BLE001 -- surface the reason, then let it propagate
|
|
# A startup failure (bad config, OOM, corrupt weights) would otherwise reach the
|
|
# parent only as a dead process -> a generic "exited during load". Push the real
|
|
# reason first so the supervisor (and the desktop failure modal) can surface it;
|
|
# the traceback still prints and the process still exits non-zero.
|
|
_report_startup_error(ack_queue, exc)
|
|
raise
|
|
|
|
if args.tp_info.is_primary():
|
|
# Report the real per-unit cache VRAM costs (KV/expert/mamba), the device-wide free
|
|
# VRAM, and the per-pool rebuild floors before the ready ack, so the supervisor has
|
|
# them (and the desktop's slider bounds) by the time the gate flips. Optional +
|
|
# best-effort: a failure here must never keep the model from serving, and older
|
|
# consumers ignore ("meta", …).
|
|
try:
|
|
from freetoken.kvcache.cache_status import compute_cache_status_meta
|
|
|
|
meta = compute_cache_status_meta(scheduler.engine)
|
|
# the parent must not touch CUDA to learn this
|
|
meta["gpus"] = scheduler.gpus
|
|
ack_queue.put(("meta", meta))
|
|
except Exception: # noqa: BLE001 -- metadata is a nicety; readiness is not
|
|
pass
|
|
ack_queue.put("Scheduler is ready")
|
|
# The supervisor stops draining ack_queue once ready, so uninstall the sink:
|
|
# runtime cache rebuilds re-run the graph capture (which emits progress) and
|
|
# would otherwise push onto a queue nobody reads for the server's lifetime.
|
|
set_progress_sink(None)
|
|
|
|
if args.silent_output:
|
|
logging.disable(logging.INFO)
|
|
|
|
try:
|
|
scheduler.run_forever()
|
|
except KeyboardInterrupt:
|
|
logger = init_logger(__name__)
|
|
if args.tp_info.is_primary():
|
|
print() # for a clean newline after ^C
|
|
logger.info("Scheduler exiting gracefully...")
|
|
scheduler.shutdown()
|
|
|
|
|
|
def launch_server(
|
|
run_shell: bool = False,
|
|
argv: list[str] | None = None,
|
|
prog: str | None = None,
|
|
) -> None:
|
|
from .api_server import run_api_server
|
|
from .args import parse_args
|
|
|
|
server_args, run_shell = parse_args(
|
|
sys.argv[1:] if argv is None else argv,
|
|
run_shell,
|
|
prog=prog,
|
|
)
|
|
logger = init_logger(__name__, "initializer")
|
|
|
|
if server_args.gpu:
|
|
# resolve here so a typo is one clear error before any worker spawns
|
|
from freetoken.gpu_select import resolve_gpu_uuids
|
|
|
|
try:
|
|
server_args = replace(server_args, gpu_assigned=resolve_gpu_uuids(server_args.gpu))
|
|
except ValueError as exc:
|
|
raise SystemExit(f"{prog or 'ft serve'}: error: {exc}") from exc
|
|
logger.info(
|
|
f"--gpu {','.join(server_args.gpu)} -> "
|
|
f"{', '.join(server_args.gpu_assigned) if server_args.gpu_assigned else 'resolved at CUDA init (no NVML)'}"
|
|
)
|
|
|
|
def start_subprocess() -> "BackendHandle":
|
|
import multiprocessing as mp
|
|
|
|
from .supervisor import BackendHandle
|
|
|
|
mp.set_start_method("spawn", force=True)
|
|
detach = server_args.shell_mode # see _detach_process_group
|
|
|
|
world_size = server_args.tp_info.size
|
|
ack_queue: mp.Queue = mp.Queue()
|
|
processes: list[mp.Process] = []
|
|
|
|
for i in range(world_size):
|
|
new_args = replace(server_args, tp_info=DistributedInfo(i, world_size))
|
|
p = mp.Process(
|
|
target=_run_scheduler,
|
|
args=(new_args, ack_queue),
|
|
daemon=False,
|
|
name=f"freetoken-TP{i}-scheduler",
|
|
)
|
|
p.start()
|
|
processes.append(p)
|
|
|
|
num_tokenizers = server_args.num_tokenizer
|
|
p = mp.Process(
|
|
target=_run_tokenize_worker,
|
|
kwargs={
|
|
"detach": detach,
|
|
"tokenizer_path": server_args.model_path,
|
|
"addr": server_args.zmq_detokenizer_addr,
|
|
"backend_addr": server_args.zmq_backend_addr,
|
|
"frontend_addr": server_args.zmq_frontend_addr,
|
|
"local_bs": 1,
|
|
"mm": server_args.mm,
|
|
"create": server_args.tokenizer_create_addr,
|
|
"tokenizer_id": num_tokenizers,
|
|
"ack_queue": ack_queue,
|
|
},
|
|
daemon=False,
|
|
name="freetoken-detokenizer-0",
|
|
)
|
|
p.start()
|
|
processes.append(p)
|
|
for i in range(num_tokenizers):
|
|
p = mp.Process(
|
|
target=_run_tokenize_worker,
|
|
kwargs={
|
|
"detach": detach,
|
|
"tokenizer_path": server_args.model_path,
|
|
"addr": server_args.zmq_tokenizer_addr,
|
|
"backend_addr": server_args.zmq_backend_addr,
|
|
"frontend_addr": server_args.zmq_frontend_addr,
|
|
"local_bs": 1,
|
|
"mm": server_args.mm,
|
|
"create": server_args.tokenizer_create_addr,
|
|
"tokenizer_id": i,
|
|
"ack_queue": ack_queue,
|
|
},
|
|
daemon=False,
|
|
name=f"freetoken-tokenizer-{i}",
|
|
)
|
|
p.start()
|
|
processes.append(p)
|
|
|
|
# Expected ready acks: 1 primary scheduler + num_tokenizers + 1 detokenizer.
|
|
return BackendHandle(
|
|
ack_queue=ack_queue,
|
|
processes=processes,
|
|
expected_acks=num_tokenizers + 2,
|
|
)
|
|
|
|
run_api_server(server_args, start_subprocess, run_shell=run_shell)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
launch_server()
|