mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal * feat(engine): Sampler.probs and sparse-q rejection * feat(spec): let a proposer declare slot states riding the GDN slots * feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families * feat(attention): record the source slot of each tracked request * feat(kernel): DFlash2 draft kernels * feat(models): DFlash2 draft family * feat(spec): DFlashWindow * feat(spec): DFlashProposer * docs(cli): add dflash to the speculative decoding section * refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs * fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply * refactor(spec): share the draft path check between the MTP and DFlash proposers * test(e2e): compare rebuilt outputs exactly only without hybrid MoE * refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config * perf(spec): compute the verify distribution once for sparse-q rejection * perf(models): let the feature sink add the residual on the rows it keeps * perf(models): share the DFlash draft K/V weights with the ring projection * fix(kernel): keep the dflash selector off zero-probability candidates * test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover * fix(spec): default and cap DFlash steps at the draft's block_size - 1 * fix(models): reject DFlash draft configs the draft forward cannot serve * test(e2e): read the MoE strategy from the rebuild test's extra args * docs(spec): say that slot-state owners write their own prefill track snapshot * test(models): move the DFlash draft parity tests under tests/models/dflash * refactor(spec): drop the no-op verify handler of the DFlash window * refactor(models): fold the DFlash candidate top-k into propose * refactor(spec): declare the draft step limits through Proposer.num_steps only * refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring * refactor(models): name the DFlash draft submodules like the other model families * refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py * refactor(spec): hand the verify the draft distribution over the whole vocabulary * refactor(spec): call the DFlashRing object dflash_ring and its length ring_len * refactor(models): split the DFlash draft's propose into compute_candidates and sample_path * refactor(spec): keep the draft distribution on its support in SpecProposal * fix(spec): drop the step's proposal once the verify has read it * refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink * refactor(models): read the declared Batch.features directly instead of through getattr * test(dflash): check the window rebuild inside the block forward test * test(spec): share the plain run of the verify contract and run the DFlash draft once per family * test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes * test(spec): check the verify probs of each row against a torch top-k and top-p reference * test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate * refactor(scheduler): let callers take the speculative counts from the status reporter * test(spec): keep the shared plain runs of the verify contract in their own temp dirs * test(models): give the ricursive test's stand-in batch a features field
1122 lines
42 KiB
Python
1122 lines
42 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
from dataclasses import dataclass, field
|
|
from typing import List, Tuple
|
|
|
|
import torch
|
|
from freetoken.mm.config import ENCODER_KINDS, MultimodalConfig
|
|
from freetoken.distributed import DistributedInfo
|
|
from freetoken.scheduler import SchedulerConfig
|
|
from freetoken.utils import init_logger
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class _DeprecatedAlias(argparse.Action):
|
|
"""An old flag: warns at parse time, converts the value if asked, stores it."""
|
|
|
|
def __init__(self, *args, new_flag: str, convert=None, **kwargs):
|
|
self.new_flag, self.convert = new_flag, convert
|
|
super().__init__(*args, **kwargs)
|
|
|
|
def __call__(self, parser, namespace, values, option_string=None):
|
|
logger.warning("%s is deprecated; use %s", option_string, self.new_flag)
|
|
setattr(namespace, self.dest, self.convert(values) if self.convert else values)
|
|
|
|
|
|
def _nvfp4_entry(value: str) -> str:
|
|
"""The --quant-backend entry an old --nvfp4-backend value stands for; auto stands for none."""
|
|
if value == "auto":
|
|
return ""
|
|
return "moe.nvfp4=" + {"flashinfer": "b12x"}.get(value, value)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ServerArgs(SchedulerConfig):
|
|
server_host: str = "127.0.0.1"
|
|
server_port: int = 1919
|
|
# kept out of repr: parse_args logs the whole ServerArgs
|
|
api_key: str | None = field(default=None, repr=False)
|
|
num_tokenizer: int = 0
|
|
silent_output: bool = False
|
|
# The terminal shell is attached to this server (ft shell --model / ft serve --shell-mode).
|
|
# The workers read it to leave the shell's foreground process group, so the ^C that cancels
|
|
# a turn cannot also kill the engine — see server/launch.py:_detach_process_group.
|
|
shell_mode: bool = False
|
|
served_model_name: str | None = None
|
|
tool_call_parser: str = "llama3"
|
|
# Reasoning parser that splits <think> reasoning from content for OpenAI
|
|
# responses. None disables it (default for models without a reasoning protocol).
|
|
reasoning_parser: str | None = None
|
|
# "model": fill unspecified request sampling params from generation_config.json
|
|
# (temperature/top_k/top_p), like sglang. "none": use framework defaults only.
|
|
sampling_defaults: str = "model"
|
|
# Default max output (decode) tokens for a request that omits one. None falls back to the
|
|
# adapter's built-in default (32k).
|
|
max_output_tokens: int | None = None
|
|
# Report the prefix-cache hit in each response's usage block (OpenAI
|
|
# prompt_tokens_details.cached_tokens, Anthropic cache_read_input_tokens, Responses
|
|
# input_tokens_details.cached_tokens). Mirrors sglang's --enable-cache-report.
|
|
enable_cache_report: bool = False
|
|
# Attach llama.cpp's per-request `timings` object to OpenAI chat/completions and
|
|
# Responses responses (plus queue_ms and, under speculative decoding, draft_n /
|
|
# draft_n_accepted). Opt-in like vLLM's --enable-per-request-metrics.
|
|
enable_timings_report: bool = False
|
|
anthropic_inline_system: str = "auto"
|
|
# Comma-separated hostname allowlist for client-supplied image URLs; empty admits any domain.
|
|
allowed_media_domains: str = ""
|
|
# Directory file:// image refs may be read from; empty rejects local files.
|
|
allowed_local_media_path: str = ""
|
|
# Comma-separated CORS allow-list for browser/webview clients (e.g. the desktop
|
|
# app). Empty string disables CORS headers entirely; "*" allows any origin.
|
|
cors_origins: str = "tauri://localhost,http://tauri.localhost,http://localhost:1420"
|
|
# --gpu entries in TP-rank order, empty = not given
|
|
gpu: tuple[str, ...] = ()
|
|
# full UUIDs resolved from --gpu, entry i = TP rank i; None = NVML unavailable, each worker then resolves its raw entry against CUDA's own enumeration
|
|
gpu_assigned: "tuple[str, ...] | None" = None
|
|
|
|
@property
|
|
def share_tokenizer(self) -> bool:
|
|
return self.num_tokenizer == 0
|
|
|
|
@property
|
|
def zmq_frontend_addr(self) -> str:
|
|
return "ipc:///tmp/freetoken_3" + self._unique_suffix
|
|
|
|
@property
|
|
def zmq_tokenizer_addr(self) -> str:
|
|
if self.share_tokenizer:
|
|
return self.zmq_detokenizer_addr
|
|
result = "ipc:///tmp/freetoken_4" + self._unique_suffix
|
|
assert result != self.zmq_detokenizer_addr
|
|
return result
|
|
|
|
@property
|
|
def tokenizer_create_addr(self) -> bool:
|
|
return self.share_tokenizer
|
|
|
|
@property
|
|
def backend_create_detokenizer_link(self) -> bool:
|
|
return not self.share_tokenizer
|
|
|
|
@property
|
|
def frontend_create_tokenizer_link(self) -> bool:
|
|
return not self.share_tokenizer
|
|
|
|
@property
|
|
def distributed_addr(self) -> str:
|
|
return f"tcp://127.0.0.1:{self.server_port + 1}"
|
|
|
|
|
|
def _json_object(text: str) -> dict:
|
|
try:
|
|
value = json.loads(text)
|
|
except json.JSONDecodeError as exc:
|
|
raise argparse.ArgumentTypeError(f"not valid JSON: {exc}") from None
|
|
if not isinstance(value, dict):
|
|
raise argparse.ArgumentTypeError("expected a JSON object")
|
|
return value
|
|
|
|
|
|
def parse_args(
|
|
args: List[str],
|
|
run_shell: bool = False,
|
|
prog: str | None = None,
|
|
) -> Tuple[ServerArgs, bool]:
|
|
"""
|
|
Parse command line arguments and return an EngineConfig.
|
|
|
|
Args:
|
|
args: Command line arguments (e.g., sys.argv[1:])
|
|
|
|
Returns:
|
|
EngineConfig instance with parsed arguments
|
|
"""
|
|
from freetoken.attention import validate_attn_backend
|
|
from freetoken.kvcache import SUPPORTED_CACHE_MANAGER
|
|
from freetoken.moe import MOE_STRATEGIES
|
|
from freetoken.moe.pool_policy import SCOPES
|
|
|
|
def _parse_quant_backend(value: str) -> str:
|
|
from freetoken.layers.quantization import QuantBackend
|
|
|
|
try:
|
|
QuantBackend.parse(value)
|
|
except ValueError as exc:
|
|
raise argparse.ArgumentTypeError(str(exc)) from None
|
|
return value
|
|
|
|
def _parse_moe_cache_rate(value: str) -> float:
|
|
try:
|
|
rate = float(value)
|
|
except ValueError as exc:
|
|
raise argparse.ArgumentTypeError("must be a number in [0, 1]") from exc
|
|
if not 0 <= rate <= 1:
|
|
raise argparse.ArgumentTypeError("must be in [0, 1]")
|
|
return rate
|
|
|
|
def _parse_mem_size(value: str) -> int:
|
|
units = {
|
|
"b": 1, "k": 1 << 10, "kb": 1 << 10, "kib": 1 << 10, "m": 1 << 20, "mb": 1 << 20, "mib": 1 << 20,
|
|
"g": 1 << 30, "gb": 1 << 30, "gib": 1 << 30,
|
|
}
|
|
text = value.strip().lower()
|
|
number = text.rstrip("abgikm")
|
|
unit = text[len(number):]
|
|
try:
|
|
size = int(float(number) * units[unit])
|
|
except (KeyError, ValueError, OverflowError) as exc:
|
|
raise argparse.ArgumentTypeError("must be a size with a unit, e.g. 12GiB or 800MiB") from exc
|
|
if size <= 0:
|
|
raise argparse.ArgumentTypeError("must be positive")
|
|
return size
|
|
|
|
def _parse_cache_split(value: str) -> str:
|
|
if value == "equal" or (value.startswith("profile:") and len(value) > len("profile:")):
|
|
return value
|
|
raise argparse.ArgumentTypeError("must be equal or profile:<path>")
|
|
|
|
def _positive_int(value: str) -> int:
|
|
try:
|
|
n = int(value)
|
|
except ValueError as exc:
|
|
raise argparse.ArgumentTypeError("must be a positive integer") from exc
|
|
if n < 1:
|
|
raise argparse.ArgumentTypeError("must be >= 1")
|
|
return n
|
|
|
|
def _lazy_gpu_arg(value: str) -> tuple[str, ...]:
|
|
from freetoken.gpu_select import gpu_arg
|
|
|
|
return gpu_arg(value)
|
|
|
|
def _infer_tool_call_parser(model_path: str) -> str:
|
|
try:
|
|
from freetoken.utils import cached_load_hf_config
|
|
|
|
cfg = cached_load_hf_config(model_path).to_dict()
|
|
except Exception:
|
|
cfg = {}
|
|
|
|
text_cfg = cfg.get("text_config") or {}
|
|
candidates = [
|
|
model_path,
|
|
str(cfg.get("model_type", "")),
|
|
str(text_cfg.get("model_type", "")),
|
|
" ".join(str(v) for v in cfg.get("architectures", []) or []),
|
|
" ".join(str(v) for v in text_cfg.get("architectures", []) or []),
|
|
]
|
|
marker = " ".join(candidates).lower()
|
|
# off the checkpoint only: the HF org name ricursiveAI would match other models' paths
|
|
if "ricursive" in " ".join(candidates[1:]).lower():
|
|
return "ricursive"
|
|
if "flint" in marker:
|
|
return "glm47"
|
|
if "gpt_oss" in marker or "gpt-oss" in marker or "gptoss" in marker:
|
|
return "gpt_oss"
|
|
# M3 first: its marker also contains the bare "minimax" substring, but the
|
|
# namespaced tool grammar is a different protocol from M2's.
|
|
if "minimax_m3" in marker or "minimax-m3" in marker or "minimaxm3" in marker:
|
|
return "minimax_m3"
|
|
if "minimax" in marker:
|
|
return "minimax"
|
|
if "muse_glimmer" in marker or "muse-glimmer" in marker or "museglimmer" in marker:
|
|
return "muse_glimmer"
|
|
if "gemma4" in marker:
|
|
return "gemma4"
|
|
if "qwen4_exp" in marker or "qwen4exp" in marker or "qwen3.8-flash" in marker:
|
|
return "qwen3_coder"
|
|
if (
|
|
"qwen3_5" in marker
|
|
or "qwen3.5" in marker
|
|
or ("qwen3" in marker and "coder" in marker)
|
|
):
|
|
return "qwen3_coder"
|
|
if "qwen" in marker:
|
|
return "qwen25"
|
|
if "deepseek_v41" in marker or "deepseekv41" in marker:
|
|
return "deepseekv41"
|
|
if "deepseek" in marker and ("v4" in marker or "deepseek_v4" in marker):
|
|
return "deepseekv32"
|
|
if "deepseek" in marker and ("v3.2" in marker or "v32" in marker):
|
|
return "deepseekv32"
|
|
if "glm" in marker:
|
|
return "glm47"
|
|
if "mistral" in marker:
|
|
return "mistral"
|
|
return "llama3"
|
|
|
|
def _infer_reasoning_parser(model_path: str) -> str | None:
|
|
try:
|
|
from freetoken.utils import cached_load_hf_config
|
|
|
|
cfg = cached_load_hf_config(model_path).to_dict()
|
|
except Exception:
|
|
cfg = {}
|
|
|
|
text_cfg = cfg.get("text_config") or {}
|
|
candidates = [
|
|
model_path,
|
|
str(cfg.get("model_type", "")),
|
|
str(text_cfg.get("model_type", "")),
|
|
" ".join(str(v) for v in cfg.get("architectures", []) or []),
|
|
" ".join(str(v) for v in text_cfg.get("architectures", []) or []),
|
|
]
|
|
marker = " ".join(candidates).lower()
|
|
if "ricursive" in " ".join(candidates[1:]).lower():
|
|
return "ricursive"
|
|
if "flint" in marker:
|
|
return "flint"
|
|
if "gpt_oss" in marker or "gpt-oss" in marker or "gptoss" in marker:
|
|
return "gpt_oss"
|
|
if "deepseek" in marker and any(
|
|
tag in marker for tag in ("v4", "deepseek_v4", "v3.2", "v32")
|
|
):
|
|
return "deepseekv32"
|
|
if "qwen4_exp" in marker or "qwen4exp" in marker or "qwen3.8-flash" in marker:
|
|
return "qwen3"
|
|
if "qwen3" in marker or "qwen3.5" in marker or "qwen3_5" in marker:
|
|
return "qwen3"
|
|
if "glm" in marker:
|
|
return "glm"
|
|
# M3 first ("minimax" is a substring): <mm:think> tags + 3 thinking gears,
|
|
# not M2's always-on implicit <think>.
|
|
if "minimax_m3" in marker or "minimax-m3" in marker or "minimaxm3" in marker:
|
|
return "minimax_m3"
|
|
if "minimax" in marker:
|
|
return "minimax"
|
|
if "muse_glimmer" in marker or "muse-glimmer" in marker or "museglimmer" in marker:
|
|
return "muse_glimmer"
|
|
if "gemma4" in marker:
|
|
return "gemma4"
|
|
return None
|
|
|
|
parser = argparse.ArgumentParser(prog=prog, description="FreeToken Server Arguments")
|
|
|
|
parser.add_argument(
|
|
"--model-path",
|
|
"--model",
|
|
type=str,
|
|
required=True,
|
|
help="The path of the model weights. This can be a local folder or a Hugging Face repo ID.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--dtype",
|
|
type=str,
|
|
default="auto",
|
|
choices=["auto", "float16", "bfloat16", "float32"],
|
|
help="Data type for model weights and activations. 'auto' will use FP16 for FP32/FP16 models and BF16 for BF16 models.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--hf-overrides",
|
|
type=_json_object,
|
|
default=None,
|
|
metavar="JSON",
|
|
help="JSON object applied to the checkpoint config the model is built from, as vLLM's "
|
|
"--hf-overrides: a nested config section is updated key by key, any other value is "
|
|
"replaced whole. A YaRN rope_parameters override extends the servable context to "
|
|
"original_max_position_embeddings * factor.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--tensor-parallel-size",
|
|
"--tp-size",
|
|
type=int,
|
|
default=1,
|
|
help="The tensor parallelism size.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--gpu",
|
|
type=_lazy_gpu_arg,
|
|
default=ServerArgs.gpu,
|
|
help=(
|
|
"GPU(s) to run on, comma-separated; entry i is TP rank i. Each entry is a GPU "
|
|
"UUID (GPU-xxxx..., as nvidia-smi -L prints) or an nvidia-smi index"
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-running-requests",
|
|
type=int,
|
|
dest="max_running_req",
|
|
default=ServerArgs.max_running_req,
|
|
help="The maximum number of running requests.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-seq-len-override",
|
|
type=int,
|
|
default=ServerArgs.max_seq_len_override,
|
|
help="The maximum sequence length override.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-output-tokens",
|
|
type=_positive_int,
|
|
default=ServerArgs.max_output_tokens,
|
|
help="Default max output tokens for requests that omit one (default 32k).",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--memory-ratio",
|
|
type=float,
|
|
default=ServerArgs.memory_ratio,
|
|
help=(
|
|
"Fraction of total GPU free memory the engine may use for weights + MoE "
|
|
"cache + KV cache combined; the remainder is reserved runtime headroom."
|
|
),
|
|
)
|
|
|
|
assert ServerArgs.use_dummy_weight == False
|
|
parser.add_argument(
|
|
"--dummy-weight",
|
|
action="store_true",
|
|
dest="use_dummy_weight",
|
|
help="Use dummy weights for testing.",
|
|
)
|
|
|
|
assert ServerArgs.use_pynccl == True
|
|
parser.add_argument(
|
|
"--disable-pynccl",
|
|
action="store_false",
|
|
dest="use_pynccl",
|
|
help="Disable PyNCCL for tensor parallelism.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--host",
|
|
type=str,
|
|
dest="server_host",
|
|
default=ServerArgs.server_host,
|
|
help="The host address for the server.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--port",
|
|
type=int,
|
|
dest="server_port",
|
|
default=ServerArgs.server_port,
|
|
help="The port number for the server to listen on.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--api-key",
|
|
type=str,
|
|
default=ServerArgs.api_key,
|
|
help="Require this key on every route except /health, sent as "
|
|
"'Authorization: Bearer <key>' or 'x-api-key: <key>'. Defaults to $FREETOKEN_API_KEY; "
|
|
"unset serves without authentication.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--cuda-graph-max-bs",
|
|
"--graph",
|
|
type=int,
|
|
default=ServerArgs.cuda_graph_max_bs,
|
|
help="The maximum batch size for CUDA graph capture. None means auto-tuning based on the GPU memory.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--num-tokenizer",
|
|
"--tokenizer-count",
|
|
type=int,
|
|
default=ServerArgs.num_tokenizer,
|
|
help="The number of tokenizer processes to launch. 0 means the tokenizer is shared with the detokenizer.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-prefill-length",
|
|
"--max-extend-length",
|
|
type=int,
|
|
dest="max_extend_tokens",
|
|
default=ServerArgs.max_extend_tokens,
|
|
help="Chunk Prefill maximum chunk size in tokens.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--decode-log-interval",
|
|
type=_positive_int,
|
|
default=ServerArgs.decode_log_interval,
|
|
help="Print one decode scheduler status line every N decode forwards.",
|
|
)
|
|
|
|
kv_capacity_group = parser.add_mutually_exclusive_group()
|
|
kv_capacity_group.add_argument(
|
|
"--num-pages",
|
|
dest="num_page_override",
|
|
type=int,
|
|
default=ServerArgs.num_page_override,
|
|
help="Set the maximum number of pages for KVCache.",
|
|
)
|
|
|
|
kv_capacity_group.add_argument(
|
|
"--num-tokens",
|
|
dest="num_token_override",
|
|
type=int,
|
|
default=ServerArgs.num_token_override,
|
|
help=(
|
|
"Total KV-cache capacity in tokens; must be a multiple of the resolved page "
|
|
"size (DSV4: 128 window page, TRTLLM backend: 64). Mutually exclusive with "
|
|
"--num-pages."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--page-size",
|
|
type=int,
|
|
default=ServerArgs.page_size,
|
|
help="Set the page size for system management.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--attention-backend",
|
|
"--attn",
|
|
type=validate_attn_backend,
|
|
default=ServerArgs.attention_backend,
|
|
help="The attention backend to use. If two backends are specified,"
|
|
" the first one is used for prefill and the second one for decode.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--model-source",
|
|
type=str,
|
|
default="huggingface",
|
|
choices=["huggingface", "modelscope"],
|
|
help="The source to download model from. Either 'huggingface' or 'modelscope'.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--cache-type",
|
|
type=str,
|
|
default=ServerArgs.cache_type,
|
|
choices=SUPPORTED_CACHE_MANAGER.supported_names(),
|
|
help="KV cache strategy (naive | radix). For hybrid GDN models 'radix' is materialized "
|
|
"as a GDN-aware radix (cross-request GDN-state prefix reuse); pass 'naive' to opt out.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--text-model-only",
|
|
action="store_true",
|
|
default=False,
|
|
help="Serve a multimodal checkpoint text-only: no encoder tower is built (its VRAM goes to "
|
|
"the KV/expert pools) and every multimodal input is rejected. Same as --mm-disable with "
|
|
"every encoder kind.",
|
|
)
|
|
parser.add_argument(
|
|
"--mm-disable",
|
|
nargs="+",
|
|
choices=list(ENCODER_KINDS),
|
|
default=[],
|
|
metavar="{vision,audio}",
|
|
help="Encoder towers to leave unbuilt; every input they would serve is rejected.",
|
|
)
|
|
parser.add_argument(
|
|
"--mmproj",
|
|
type=str,
|
|
default=None,
|
|
metavar="FILE",
|
|
help="llama.cpp vision tower (mmproj GGUF) for a GGUF model. Default: a *mmproj*.gguf that fits the model "
|
|
"in the model file's folder (BF16 > F32 > F16 > quantized when several are there); none found, or files of "
|
|
"several models, serves text only.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--image-min-tokens",
|
|
type=_positive_int,
|
|
default=MultimodalConfig.image_min_tokens,
|
|
help="Fewest tokens an image may take: the image processor scales smaller images up to it, "
|
|
"in the family's own units. Default: the processor's own limit.",
|
|
)
|
|
parser.add_argument(
|
|
"--image-max-tokens",
|
|
type=_positive_int,
|
|
default=MultimodalConfig.image_max_tokens,
|
|
help="Most tokens an image may take: the image processor scales larger images down to it, "
|
|
"in the family's own units (Qwen VL: one token per 32x32 pixels). Default: the processor's own limit.",
|
|
)
|
|
parser.add_argument(
|
|
"--mm-processor-kwargs",
|
|
type=_json_object,
|
|
default=None,
|
|
metavar="JSON",
|
|
help="JSON object of extra keyword arguments for the checkpoint's image processor call, "
|
|
"for family-specific knobs; applied after the token budget.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--mm-embed-cache-device",
|
|
choices=["cpu", "cuda"],
|
|
default=MultimodalConfig.embed_cache_device,
|
|
help="Storage for encoded image embeddings between prefill chunks.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--mm-encoder-weights",
|
|
choices=["gpu", "host"],
|
|
default=MultimodalConfig.encoder_weights,
|
|
help="Encoder tower block weights: pinned host banks streamed two blocks at a time behind the "
|
|
"compute (default, about 60 MiB of VRAM instead of the whole tower), or resident on the GPU.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--allowed-media-domains",
|
|
type=str,
|
|
default=ServerArgs.allowed_media_domains,
|
|
help="Comma-separated hostname allowlist for client-supplied image URLs. "
|
|
"Empty (default) allows any domain.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--allowed-local-media-path",
|
|
type=str,
|
|
default=ServerArgs.allowed_local_media_path,
|
|
help="Directory that file:// image refs may be read from. "
|
|
"Unset (default) rejects local files.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--enable-cache-report",
|
|
action="store_true",
|
|
default=ServerArgs.enable_cache_report,
|
|
help=(
|
|
"Return the number of prefix-cached prompt tokens in each response's usage block "
|
|
"(OpenAI usage.prompt_tokens_details.cached_tokens, Anthropic "
|
|
"usage.cache_read_input_tokens, Responses usage.input_tokens_details.cached_tokens). "
|
|
"On /v1/messages this also makes input_tokens EXCLUDE the cached prefix, matching "
|
|
"Anthropic billing semantics."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--enable-timings-report",
|
|
action="store_true",
|
|
default=ServerArgs.enable_timings_report,
|
|
help=(
|
|
"Attach a llama.cpp-shaped per-request timings object to every chat completion, "
|
|
"completion and Responses response (and to the final stream chunk), so frontends "
|
|
"like llama-swap and Open Web UI can show prefill/decode rates and cache hits. "
|
|
"Spans are measured in the scheduler; queue_ms is a FreeToken extension."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--anthropic-inline-system",
|
|
choices=("auto", "preserve", "fold"),
|
|
default=ServerArgs.anthropic_inline_system,
|
|
help="Preserve inline system instructions when supported by the renderer, "
|
|
"or fold them into nearby user/tool content without hoisting the prompt prefix.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--sampling-defaults",
|
|
type=str,
|
|
default=ServerArgs.sampling_defaults,
|
|
choices=["model", "none"],
|
|
help=(
|
|
"Source for unspecified request sampling params. 'model' fills "
|
|
"temperature/top_k/top_p from the checkpoint's generation_config.json "
|
|
"(recommended for reasoning models to avoid greedy repetition loops); "
|
|
"'none' uses framework defaults only."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--served-model-name",
|
|
type=str,
|
|
default=ServerArgs.served_model_name,
|
|
help="Model id returned by /v1/models. Defaults to the basename of --model.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--tool-call-parser",
|
|
type=str,
|
|
default="auto",
|
|
choices=[
|
|
"auto",
|
|
"llama3",
|
|
"qwen",
|
|
"qwen25",
|
|
"qwen3_coder",
|
|
"ricursive",
|
|
"mistral",
|
|
"deepseekv32",
|
|
"deepseekv41",
|
|
"gemma4",
|
|
"glm47",
|
|
"minimax",
|
|
"minimax_m3",
|
|
"muse_glimmer",
|
|
"gpt_oss",
|
|
"gpt-oss",
|
|
],
|
|
help="Tool-call parser format for OpenAI-compatible tool responses.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--reasoning-parser",
|
|
type=str,
|
|
default="auto",
|
|
choices=[
|
|
"auto", "off", "deepseekv32", "gpt_oss", "qwen3", "glm",
|
|
"minimax", "minimax_m3", "muse_glimmer", "gemma4", "flint", "ricursive",
|
|
],
|
|
help=(
|
|
"Reasoning parser that splits chain-of-thought into reasoning_content "
|
|
"for OpenAI responses. 'auto' selects per model family (gpt-oss Harmony, "
|
|
"<think> for qwen3/glm/minimax, <mm:think> for minimax-m3, ATEM to=self "
|
|
"channels for muse-glimmer, gemma thought, dsv4); 'off' disables it."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--moe-strategy",
|
|
default=ServerArgs.moe_strategy,
|
|
choices=["auto", *MOE_STRATEGIES],
|
|
help=(
|
|
"How the routed experts are served. 'auto' resolves a MoE model to 'offload', and to "
|
|
"resident 'fused' experts on unified-memory GPUs (GB10 / DGX Spark)."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--moe-backend",
|
|
dest="moe_strategy",
|
|
action=_DeprecatedAlias,
|
|
new_flag="--moe-strategy",
|
|
default=argparse.SUPPRESS,
|
|
choices=["auto", *MOE_STRATEGIES],
|
|
help="[Deprecated] Use --moe-strategy.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--quant-backend",
|
|
default=None,
|
|
type=_parse_quant_backend,
|
|
help=(
|
|
"Kernel per quantized layer type: comma-separated layer[.kind]=name entries, e.g. "
|
|
"'linear=marlin,moe=b12x' or 'moe.nvfp4=triton'. A layer-level entry applies to every "
|
|
"kind whose kernel table lists the name; unlisted tables stay automatic."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--ple-backend",
|
|
default=ServerArgs.ple_backend,
|
|
choices=["pinned", "disk"],
|
|
help=(
|
|
"Where a PLE n-gram table lives. 'disk' (default) reads rows straight from the "
|
|
"checkpoint files; 'pinned' preloads the whole table into page-locked host RAM."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--embed-weights",
|
|
default=ServerArgs.embed_weights,
|
|
choices=["gpu", "host"],
|
|
help=(
|
|
"Where the input token embedding table lives. 'gpu' (default) keeps it in VRAM; 'host' keeps it in "
|
|
"pinned host RAM and each lookup reads its rows over PCIe, so its VRAM goes to the expert cache and KV. "
|
|
"Not for a table tied to the LM head."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--swa-decoder-replay",
|
|
default=ServerArgs.swa_decoder_replay,
|
|
choices=["bounded", "exact"],
|
|
help=(
|
|
"DeepSeek-V4.1 Decoder SWA Bounded Replay. 'bounded' (default) runs the 20 decoder layers "
|
|
"on each prompt's last 128 tokens with their sliding window truncated there, as in the "
|
|
"tech report; 'exact' runs them on every prompt token (the reference numerics)."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--nvfp4-backend",
|
|
action=_DeprecatedAlias,
|
|
new_flag="--quant-backend moe.nvfp4=<marlin|b12x|triton>",
|
|
convert=_nvfp4_entry,
|
|
default=argparse.SUPPRESS,
|
|
choices=["auto", "marlin", "flashinfer", "triton"],
|
|
help="[Deprecated] Use --quant-backend moe.nvfp4=<marlin|b12x|triton> ('flashinfer' is b12x).",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--expert-load",
|
|
default=ServerArgs.expert_load,
|
|
choices=["auto", "serial", "parallel"],
|
|
help=(
|
|
"How MoE expert banks are read into host RAM. 'auto' (default) reads scattered "
|
|
"experts in parallel (fast) but falls back to serial when free RAM can't cover "
|
|
"the banks + the parallel reader's extra whole-shard buffer; 'serial' forces the "
|
|
"low-memory reclaimable read (slower); 'parallel' forces the fast read."
|
|
),
|
|
)
|
|
|
|
moe_cache_group = parser.add_mutually_exclusive_group()
|
|
moe_cache_group.add_argument(
|
|
"--moe-cache-size",
|
|
type=int,
|
|
default=ServerArgs.moe_cache_size,
|
|
help="The number of unified MoE expert slots on GPU.",
|
|
)
|
|
moe_cache_group.add_argument(
|
|
"--moe-cache-rate",
|
|
type=_parse_moe_cache_rate,
|
|
default=ServerArgs.moe_cache_rate,
|
|
help="The fraction of all MoE experts to keep in GPU cache.",
|
|
)
|
|
moe_cache_group.add_argument(
|
|
"--moe-cache-mem",
|
|
type=_parse_mem_size,
|
|
default=ServerArgs.moe_cache_mem,
|
|
help="GPU memory for the MoE expert slot cache, e.g. 12GiB or 800MiB (G, GB and GiB all mean GiB); the slot count is this over the bytes per slot.",
|
|
)
|
|
moe_cache_group.add_argument(
|
|
"--moe-cache-auto",
|
|
action="store_true",
|
|
default=ServerArgs.moe_cache_auto,
|
|
help=(
|
|
"Auto-pick --moe-cache-size from free VRAM and expert size, MoE-priority "
|
|
"(KV gets --kv-reserve-tokens as a floor). Not supported for owned-KV models."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--kv-reserve-tokens",
|
|
type=int,
|
|
default=ServerArgs.kv_reserve_tokens,
|
|
help="KV-cache token floor reserved before --moe-cache-auto fills experts.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--moe-cache-scope",
|
|
choices=SCOPES,
|
|
default=ServerArgs.moe_cache_scope,
|
|
help=(
|
|
"How MoE layers share the GPU expert slot cache: unified = one LRU pool per expert quant "
|
|
"(all layers when they share one quant format); per-layer = each layer has its own pool. "
|
|
"Pools are sized by --moe-cache-split."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--moe-cache-split",
|
|
type=_parse_cache_split,
|
|
default=ServerArgs.moe_cache_split,
|
|
help=(
|
|
"How pools share the slots: equal (by layer count), or profile:<path> = by expert use from a "
|
|
"profile file. With --moe-strategy offload, a pool smaller than the expert count limits the "
|
|
"decode batch to its slots / top_k."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--moe-cache-policy",
|
|
default=ServerArgs.moe_cache_policy,
|
|
choices=["lru"],
|
|
help="The unified MoE cache eviction policy.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--moe-cpu-threads",
|
|
type=int,
|
|
default=ServerArgs.moe_cpu_threads,
|
|
help=(
|
|
"Number of CPU worker threads for --moe-strategy cpu decode experts. "
|
|
"0 = auto (physical cores)."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--moe-cpu-layers",
|
|
type=str,
|
|
default=ServerArgs.moe_cpu_layers,
|
|
help=(
|
|
"With --moe-strategy offload/hybrid: which MoE layers compute on the "
|
|
"CPU executor instead of the GPU offload/PCIe path (where CUDA pinning "
|
|
"is quota-capped, e.g. WSL, their banks are OS-locked instead of pinned). Explicit id list ('3,7,11'), a count ('8' = 8 "
|
|
"layers evenly strided), a fraction ('0.5'), or 'auto'. 'auto' is for hosts that cap CUDA "
|
|
"pinned memory (Windows/WSL): it pins the banks layer by layer from the middle out until CUDA "
|
|
"refuses one and OS-locks the rest. FREETOKEN_PIN_BUDGET_GB, FTW checkpoints and models the CPU executor "
|
|
"cannot serve lock head+tail layers from a size estimate instead. Under hybrid, locked layers still decode from GPU slots "
|
|
"(--moe-locked-fetch). Any value, 'auto' included, commits to CPU "
|
|
"decode before the model is built, so the expert format must have a CPU executor "
|
|
"path (bf16, fp8_block, nvfp4, mxfp4); do not pass it on Linux. Unset = every layer on the "
|
|
"GPU; a boot whose banks exceed a known pin budget stops and asks for this flag."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--moe-hybrid-max-fetch",
|
|
type=int,
|
|
default=ServerArgs.moe_hybrid_max_fetch,
|
|
help=(
|
|
"For --moe-strategy hybrid: max experts fetched over PCIe per (layer, decode "
|
|
"step); the rest of that step's misses are computed on the CPU, overlapped. "
|
|
"-1 (default) = auto: fetch the benched pcie/cpu bandwidth fraction of each "
|
|
"step's misses (perfect overlap; needs the `ft bench bw` row of the layer's kind, "
|
|
"from `ft bench bw --model <checkpoint>`, else 1). "
|
|
"0 = never fetch (all misses on CPU); large = behaves like plain offload."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--moe-locked-fetch",
|
|
type=float,
|
|
default=ServerArgs.moe_locked_fetch,
|
|
help=(
|
|
"For --moe-strategy hybrid with locked layers (banks over the pin quota, OS-locked instead of pinned): "
|
|
"the share of a locked layer's misses fetched per decode step into GPU slots of its own pools, through "
|
|
"pinned staging that CPU threads fill; the rest are computed on the CPU. -1 (default) = auto: per layer from "
|
|
"the `ft bench bw` row of its kind, p / (2p + c) of the overlapped PCIe and CPU MoE rates (at most 1 expert "
|
|
"per step without a row). 0 = locked layers decode on the CPU only."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--moe-locked-max-fetch",
|
|
type=int,
|
|
default=ServerArgs.moe_locked_max_fetch,
|
|
help=(
|
|
"For --moe-locked-fetch: the most experts a locked layer fetches per decode step. Sizes the pinned "
|
|
"staging: this many rows of the widest expert, per bank."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--disable-moe-prefill-overlap",
|
|
action="store_false",
|
|
dest="moe_prefill_overlap",
|
|
default=ServerArgs.moe_prefill_overlap,
|
|
help=(
|
|
"Disable two-buffer overlap for prefill MoE expert copies. "
|
|
"By default, prefill overlap is enabled and requires "
|
|
"--moe-cache-size >= 2 * num_experts."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--enable-special-token-ckpt",
|
|
action="store_true",
|
|
dest="special_token_ckpt",
|
|
default=ServerArgs.special_token_ckpt,
|
|
help=(
|
|
"Checkpoint decode state at special tokens (currently the tool-call opener). "
|
|
"When a GDN-hybrid or SWA model samples its tool-call opener token, the "
|
|
"scheduler preserves a reuse point just after it (GDN: a state snapshot "
|
|
"donated to the prefix cache; SWA: the trailing window is kept resumable), so "
|
|
"a client that rewrites the echoed tool call only invalidates the call body, "
|
|
"not the turn."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--moe-collect-stats",
|
|
action="store_true",
|
|
default=ServerArgs.moe_collect_stats,
|
|
help=(
|
|
"With the offload-family strategies: after each decode log line, log the decode MoE stats since the "
|
|
"previous one (hit rate, and per layer-step the misses, experts fetched to the GPU and experts computed "
|
|
"on the CPU) for pinned layers, locked layers with GPU slots and CPU-only layers. Adds counter updates "
|
|
"to every decode step."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--moe-prefill-hit-d2d",
|
|
action="store_true",
|
|
dest="moe_prefill_hit_d2d",
|
|
default=ServerArgs.moe_prefill_hit_d2d,
|
|
help=(
|
|
"During prefill prefetch, copy cache-resident experts device-side into "
|
|
"the double buffer and stream only the misses over PCIe "
|
|
"(CUDA >= 13.0 or HIP with batch-copy support). Effective with "
|
|
"--moe-cache-size > 2 * num_experts."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--speculative-algorithm",
|
|
"--spec-algo",
|
|
type=str,
|
|
default=ServerArgs.speculative_algorithm,
|
|
help=(
|
|
"Speculative decoding with this draft method (mtp: the checkpoint's own MTP layers; dflash: a DFlash 2 draft "
|
|
"checkpoint from --speculative-draft-model-path). Off by default."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--speculative-num-steps",
|
|
"--spec-num-steps",
|
|
type=int,
|
|
default=ServerArgs.speculative_num_steps,
|
|
help="Draft tokens per decode step (k); 0 takes the method's default.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--speculative-draft-model-path",
|
|
"--spec-draft-model",
|
|
type=str,
|
|
default=ServerArgs.speculative_draft_model_path,
|
|
help="Draft weights; by default they come from the target checkpoint. Required for dflash, which reads a separate draft checkpoint.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--shell-mode",
|
|
action="store_true",
|
|
help="Run the server in shell mode.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--cors-origins",
|
|
type=str,
|
|
default=ServerArgs.cors_origins,
|
|
help=(
|
|
"Comma-separated CORS allow-list for browser/webview clients "
|
|
"(default: local Tauri/Vite dev origins). '' disables, '*' allows any."
|
|
),
|
|
)
|
|
|
|
# Parse arguments
|
|
kwargs = parser.parse_args(args).__dict__.copy()
|
|
|
|
# reject a too-long list here with a clear reason, not as a dead rank later
|
|
if len(kwargs["gpu"]) not in (0, kwargs["tensor_parallel_size"]):
|
|
if kwargs["tensor_parallel_size"] == 1 and len(kwargs["gpu"]) > 1:
|
|
parser.error("tensor parallelism is not supported yet: --gpu takes one entry")
|
|
parser.error(
|
|
f"--gpu has {len(kwargs['gpu'])} entries but --tensor-parallel-size is "
|
|
f"{kwargs['tensor_parallel_size']}; give one entry per TP rank"
|
|
)
|
|
|
|
# resolve some arguments
|
|
run_shell |= kwargs.pop("shell_mode")
|
|
kwargs["shell_mode"] = run_shell
|
|
if run_shell:
|
|
kwargs["cuda_graph_max_bs"] = 1
|
|
kwargs["max_running_req"] = 1
|
|
kwargs["silent_output"] = True
|
|
|
|
# the old flag stands in for one --quant-backend entry; next to the real flag it is a usage error
|
|
entry = kwargs.pop("nvfp4_backend", None)
|
|
if entry is not None:
|
|
if kwargs["quant_backend"] is not None:
|
|
parser.error("--nvfp4-backend cannot be combined with --quant-backend; write --quant-backend moe.nvfp4=... instead")
|
|
if entry:
|
|
kwargs["quant_backend"] = entry
|
|
|
|
# an empty flag usually means an unset shell variable; refuse it rather than serve open
|
|
if kwargs["api_key"] == "":
|
|
parser.error("--api-key is empty; omit it to serve without authentication")
|
|
from freetoken.launch import resolve_api_key
|
|
|
|
kwargs["api_key"] = resolve_api_key(kwargs["api_key"])
|
|
|
|
if kwargs["model_path"].startswith("~"):
|
|
kwargs["model_path"] = os.path.expanduser(kwargs["model_path"])
|
|
|
|
# every process reads the GGUF config, tokenizer workers included, so the choice travels in the environment
|
|
from freetoken.models.gguf.mmproj import MMPROJ_ENV
|
|
|
|
os.environ.pop(MMPROJ_ENV, None)
|
|
if (mmproj := kwargs.pop("mmproj")) is not None:
|
|
from freetoken.models.gguf.reader import is_gguf_path
|
|
|
|
if not is_gguf_path(kwargs["model_path"]):
|
|
parser.error("--mmproj needs a .gguf model file")
|
|
os.environ[MMPROJ_ENV] = os.path.abspath(os.path.expanduser(mmproj))
|
|
|
|
# a bad media root is a deployment mistake; fail at startup, not per request
|
|
if kwargs["allowed_local_media_path"]:
|
|
media_root = os.path.realpath(os.path.expanduser(kwargs["allowed_local_media_path"]))
|
|
if not os.path.isdir(media_root):
|
|
parser.error(f"--allowed-local-media-path {media_root} is not a directory")
|
|
kwargs["allowed_local_media_path"] = media_root
|
|
|
|
if kwargs["served_model_name"] is None:
|
|
kwargs["served_model_name"] = (
|
|
os.path.basename(os.path.normpath(kwargs["model_path"])) or kwargs["model_path"]
|
|
)
|
|
|
|
if kwargs["tool_call_parser"] == "auto":
|
|
kwargs["tool_call_parser"] = _infer_tool_call_parser(kwargs["model_path"])
|
|
|
|
if kwargs["reasoning_parser"] == "auto":
|
|
kwargs["reasoning_parser"] = _infer_reasoning_parser(kwargs["model_path"])
|
|
elif kwargs["reasoning_parser"] == "off":
|
|
kwargs["reasoning_parser"] = None
|
|
|
|
# Offload-family backends (offload/cpu/hybrid) need a slot cache; if the user gave no
|
|
# sizing flag at all, default to --moe-cache-auto so a bare `ft serve <FTW MoE>` works
|
|
# out of the box (the scheduler resolves the size from free VRAM). Explicit
|
|
# size/rate/auto is preserved.
|
|
from freetoken.moe import is_offload_moe_strategy
|
|
|
|
_no_cache_flag = (
|
|
kwargs["moe_cache_size"] == 0
|
|
and not kwargs["moe_cache_auto"]
|
|
and kwargs["moe_cache_mem"] is None
|
|
and (kwargs["moe_cache_rate"] is None or kwargs["moe_cache_rate"] == 0)
|
|
)
|
|
if is_offload_moe_strategy(kwargs["moe_strategy"]) and _no_cache_flag:
|
|
kwargs["moe_cache_auto"] = True
|
|
|
|
if kwargs["model_source"] == "modelscope":
|
|
model_path = kwargs["model_path"]
|
|
if not os.path.isdir(model_path):
|
|
from modelscope import snapshot_download
|
|
|
|
ignore_patterns = []
|
|
if kwargs["use_dummy_weight"]:
|
|
ignore_patterns = ["*.bin", "*.safetensors", "*.pt", "*.ckpt"]
|
|
model_path = snapshot_download(model_path, ignore_patterns=ignore_patterns)
|
|
kwargs["model_path"] = model_path
|
|
del kwargs["model_source"]
|
|
|
|
# "auto" (or an unspecified dtype) resolves to the checkpoint's dtype. Multimodal /
|
|
# hybrid configs (e.g. Qwen3.5-MoE) keep it under ``text_config`` and use the newer
|
|
# ``dtype`` key rather than top-level ``torch_dtype``, so check both; default bf16.
|
|
if (dtype_str := kwargs["dtype"]) in ("auto", None):
|
|
from freetoken.utils import cached_load_hf_config
|
|
|
|
cfg = cached_load_hf_config(kwargs["model_path"]).to_dict()
|
|
text_cfg = cfg.get("text_config") or {}
|
|
dtype_str = (
|
|
cfg.get("torch_dtype") or cfg.get("dtype")
|
|
or text_cfg.get("torch_dtype") or text_cfg.get("dtype") or "bfloat16"
|
|
)
|
|
|
|
DTYPE_MAP = {
|
|
"float16": torch.float16,
|
|
"bfloat16": torch.bfloat16,
|
|
"float32": torch.float32,
|
|
}
|
|
kwargs["dtype"] = DTYPE_MAP[dtype_str] if isinstance(dtype_str, str) else dtype_str
|
|
kwargs["tp_info"] = DistributedInfo(0, kwargs["tensor_parallel_size"])
|
|
del kwargs["tensor_parallel_size"]
|
|
|
|
disabled = set(ENCODER_KINDS) if kwargs.pop("text_model_only") else set()
|
|
disabled.update(kwargs.pop("mm_disable"))
|
|
image_min_tokens, image_max_tokens = kwargs.pop("image_min_tokens"), kwargs.pop("image_max_tokens")
|
|
if image_min_tokens is not None and image_max_tokens is not None and image_min_tokens > image_max_tokens:
|
|
parser.error(f"--image-min-tokens {image_min_tokens} exceeds --image-max-tokens {image_max_tokens}")
|
|
kwargs["mm"] = MultimodalConfig(
|
|
disabled_encoders=frozenset(disabled),
|
|
embed_cache_device=kwargs.pop("mm_embed_cache_device"),
|
|
encoder_weights=kwargs.pop("mm_encoder_weights"),
|
|
image_min_tokens=image_min_tokens,
|
|
image_max_tokens=image_max_tokens,
|
|
processor_kwargs=kwargs.pop("mm_processor_kwargs") or {},
|
|
)
|
|
kwargs["hf_overrides"] = kwargs["hf_overrides"] or {}
|
|
result = ServerArgs(**kwargs)
|
|
logger.info(f"Parsed arguments:\n{result}")
|
|
return result, run_shell
|