from __future__ import annotations import argparse import json import os from dataclasses import dataclass, field from typing import List, Tuple import torch from freetoken.mm.config import ENCODER_KINDS, MultimodalConfig from freetoken.distributed import DistributedInfo from freetoken.scheduler import SchedulerConfig from freetoken.utils import init_logger logger = init_logger(__name__) class _DeprecatedAlias(argparse.Action): """An old flag: warns at parse time, converts the value if asked, stores it.""" def __init__(self, *args, new_flag: str, convert=None, **kwargs): self.new_flag, self.convert = new_flag, convert super().__init__(*args, **kwargs) def __call__(self, parser, namespace, values, option_string=None): logger.warning("%s is deprecated; use %s", option_string, self.new_flag) setattr(namespace, self.dest, self.convert(values) if self.convert else values) def _nvfp4_entry(value: str) -> str: """The --quant-backend entry an old --nvfp4-backend value stands for; auto stands for none.""" if value == "auto": return "" return "moe.nvfp4=" + {"flashinfer": "b12x"}.get(value, value) @dataclass(frozen=True) class ServerArgs(SchedulerConfig): server_host: str = "127.0.0.1" server_port: int = 1919 # kept out of repr: parse_args logs the whole ServerArgs api_key: str | None = field(default=None, repr=False) num_tokenizer: int = 0 silent_output: bool = False # The terminal shell is attached to this server (ft shell --model / ft serve --shell-mode). # The workers read it to leave the shell's foreground process group, so the ^C that cancels # a turn cannot also kill the engine — see server/launch.py:_detach_process_group. shell_mode: bool = False served_model_name: str | None = None tool_call_parser: str = "llama3" # Reasoning parser that splits reasoning from content for OpenAI # responses. None disables it (default for models without a reasoning protocol). reasoning_parser: str | None = None # "model": fill unspecified request sampling params from generation_config.json # (temperature/top_k/top_p), like sglang. "none": use framework defaults only. sampling_defaults: str = "model" # Default max output (decode) tokens for a request that omits one. None falls back to the # adapter's built-in default (32k). max_output_tokens: int | None = None # Report the prefix-cache hit in each response's usage block (OpenAI # prompt_tokens_details.cached_tokens, Anthropic cache_read_input_tokens, Responses # input_tokens_details.cached_tokens). Mirrors sglang's --enable-cache-report. enable_cache_report: bool = False # Attach llama.cpp's per-request `timings` object to OpenAI chat/completions and # Responses responses (plus queue_ms and, under speculative decoding, draft_n / # draft_n_accepted). Opt-in like vLLM's --enable-per-request-metrics. enable_timings_report: bool = False anthropic_inline_system: str = "auto" # Comma-separated hostname allowlist for client-supplied image URLs; empty admits any domain. allowed_media_domains: str = "" # Directory file:// image refs may be read from; empty rejects local files. allowed_local_media_path: str = "" # Comma-separated CORS allow-list for browser/webview clients (e.g. the desktop # app). Empty string disables CORS headers entirely; "*" allows any origin. cors_origins: str = "tauri://localhost,http://tauri.localhost,http://localhost:1420" # --gpu entries in TP-rank order, empty = not given gpu: tuple[str, ...] = () # full UUIDs resolved from --gpu, entry i = TP rank i; None = NVML unavailable, each worker then resolves its raw entry against CUDA's own enumeration gpu_assigned: "tuple[str, ...] | None" = None @property def share_tokenizer(self) -> bool: return self.num_tokenizer == 0 @property def zmq_frontend_addr(self) -> str: return "ipc:///tmp/freetoken_3" + self._unique_suffix @property def zmq_tokenizer_addr(self) -> str: if self.share_tokenizer: return self.zmq_detokenizer_addr result = "ipc:///tmp/freetoken_4" + self._unique_suffix assert result != self.zmq_detokenizer_addr return result @property def tokenizer_create_addr(self) -> bool: return self.share_tokenizer @property def backend_create_detokenizer_link(self) -> bool: return not self.share_tokenizer @property def frontend_create_tokenizer_link(self) -> bool: return not self.share_tokenizer @property def distributed_addr(self) -> str: return f"tcp://127.0.0.1:{self.server_port + 1}" def _json_object(text: str) -> dict: try: value = json.loads(text) except json.JSONDecodeError as exc: raise argparse.ArgumentTypeError(f"not valid JSON: {exc}") from None if not isinstance(value, dict): raise argparse.ArgumentTypeError("expected a JSON object") return value def parse_args( args: List[str], run_shell: bool = False, prog: str | None = None, ) -> Tuple[ServerArgs, bool]: """ Parse command line arguments and return an EngineConfig. Args: args: Command line arguments (e.g., sys.argv[1:]) Returns: EngineConfig instance with parsed arguments """ from freetoken.attention import validate_attn_backend from freetoken.kvcache import SUPPORTED_CACHE_MANAGER from freetoken.moe import MOE_STRATEGIES from freetoken.moe.pool_policy import SCOPES def _parse_quant_backend(value: str) -> str: from freetoken.layers.quantization import QuantBackend try: QuantBackend.parse(value) except ValueError as exc: raise argparse.ArgumentTypeError(str(exc)) from None return value def _parse_moe_cache_rate(value: str) -> float: try: rate = float(value) except ValueError as exc: raise argparse.ArgumentTypeError("must be a number in [0, 1]") from exc if not 0 <= rate <= 1: raise argparse.ArgumentTypeError("must be in [0, 1]") return rate def _parse_mem_size(value: str) -> int: units = { "b": 1, "k": 1 << 10, "kb": 1 << 10, "kib": 1 << 10, "m": 1 << 20, "mb": 1 << 20, "mib": 1 << 20, "g": 1 << 30, "gb": 1 << 30, "gib": 1 << 30, } text = value.strip().lower() number = text.rstrip("abgikm") unit = text[len(number):] try: size = int(float(number) * units[unit]) except (KeyError, ValueError, OverflowError) as exc: raise argparse.ArgumentTypeError("must be a size with a unit, e.g. 12GiB or 800MiB") from exc if size <= 0: raise argparse.ArgumentTypeError("must be positive") return size def _parse_cache_split(value: str) -> str: if value == "equal" or (value.startswith("profile:") and len(value) > len("profile:")): return value raise argparse.ArgumentTypeError("must be equal or profile:") def _positive_int(value: str) -> int: try: n = int(value) except ValueError as exc: raise argparse.ArgumentTypeError("must be a positive integer") from exc if n < 1: raise argparse.ArgumentTypeError("must be >= 1") return n def _lazy_gpu_arg(value: str) -> tuple[str, ...]: from freetoken.gpu_select import gpu_arg return gpu_arg(value) def _infer_tool_call_parser(model_path: str) -> str: try: from freetoken.utils import cached_load_hf_config cfg = cached_load_hf_config(model_path).to_dict() except Exception: cfg = {} text_cfg = cfg.get("text_config") or {} candidates = [ model_path, str(cfg.get("model_type", "")), str(text_cfg.get("model_type", "")), " ".join(str(v) for v in cfg.get("architectures", []) or []), " ".join(str(v) for v in text_cfg.get("architectures", []) or []), ] marker = " ".join(candidates).lower() # off the checkpoint only: the HF org name ricursiveAI would match other models' paths if "ricursive" in " ".join(candidates[1:]).lower(): return "ricursive" if "flint" in marker: return "glm47" if "gpt_oss" in marker or "gpt-oss" in marker or "gptoss" in marker: return "gpt_oss" # M3 first: its marker also contains the bare "minimax" substring, but the # namespaced tool grammar is a different protocol from M2's. if "minimax_m3" in marker or "minimax-m3" in marker or "minimaxm3" in marker: return "minimax_m3" if "minimax" in marker: return "minimax" if "muse_glimmer" in marker or "muse-glimmer" in marker or "museglimmer" in marker: return "muse_glimmer" if "gemma4" in marker: return "gemma4" if "qwen4_exp" in marker or "qwen4exp" in marker or "qwen3.8-flash" in marker: return "qwen3_coder" if ( "qwen3_5" in marker or "qwen3.5" in marker or ("qwen3" in marker and "coder" in marker) ): return "qwen3_coder" if "qwen" in marker: return "qwen25" if "deepseek_v41" in marker or "deepseekv41" in marker: return "deepseekv41" if "deepseek" in marker and ("v4" in marker or "deepseek_v4" in marker): return "deepseekv32" if "deepseek" in marker and ("v3.2" in marker or "v32" in marker): return "deepseekv32" if "glm" in marker: return "glm47" if "mistral" in marker: return "mistral" return "llama3" def _infer_reasoning_parser(model_path: str) -> str | None: try: from freetoken.utils import cached_load_hf_config cfg = cached_load_hf_config(model_path).to_dict() except Exception: cfg = {} text_cfg = cfg.get("text_config") or {} candidates = [ model_path, str(cfg.get("model_type", "")), str(text_cfg.get("model_type", "")), " ".join(str(v) for v in cfg.get("architectures", []) or []), " ".join(str(v) for v in text_cfg.get("architectures", []) or []), ] marker = " ".join(candidates).lower() if "ricursive" in " ".join(candidates[1:]).lower(): return "ricursive" if "flint" in marker: return "flint" if "gpt_oss" in marker or "gpt-oss" in marker or "gptoss" in marker: return "gpt_oss" if "deepseek" in marker and any( tag in marker for tag in ("v4", "deepseek_v4", "v3.2", "v32") ): return "deepseekv32" if "qwen4_exp" in marker or "qwen4exp" in marker or "qwen3.8-flash" in marker: return "qwen3" if "qwen3" in marker or "qwen3.5" in marker or "qwen3_5" in marker: return "qwen3" if "glm" in marker: return "glm" # M3 first ("minimax" is a substring): tags + 3 thinking gears, # not M2's always-on implicit . if "minimax_m3" in marker or "minimax-m3" in marker or "minimaxm3" in marker: return "minimax_m3" if "minimax" in marker: return "minimax" if "muse_glimmer" in marker or "muse-glimmer" in marker or "museglimmer" in marker: return "muse_glimmer" if "gemma4" in marker: return "gemma4" return None parser = argparse.ArgumentParser(prog=prog, description="FreeToken Server Arguments") parser.add_argument( "--model-path", "--model", type=str, required=True, help="The path of the model weights. This can be a local folder or a Hugging Face repo ID.", ) parser.add_argument( "--dtype", type=str, default="auto", choices=["auto", "float16", "bfloat16", "float32"], help="Data type for model weights and activations. 'auto' will use FP16 for FP32/FP16 models and BF16 for BF16 models.", ) parser.add_argument( "--hf-overrides", type=_json_object, default=None, metavar="JSON", help="JSON object applied to the checkpoint config the model is built from, as vLLM's " "--hf-overrides: a nested config section is updated key by key, any other value is " "replaced whole. A YaRN rope_parameters override extends the servable context to " "original_max_position_embeddings * factor.", ) parser.add_argument( "--tensor-parallel-size", "--tp-size", type=int, default=1, help="The tensor parallelism size.", ) parser.add_argument( "--gpu", type=_lazy_gpu_arg, default=ServerArgs.gpu, help=( "GPU(s) to run on, comma-separated; entry i is TP rank i. Each entry is a GPU " "UUID (GPU-xxxx..., as nvidia-smi -L prints) or an nvidia-smi index" ), ) parser.add_argument( "--max-running-requests", type=int, dest="max_running_req", default=ServerArgs.max_running_req, help="The maximum number of running requests.", ) parser.add_argument( "--max-seq-len-override", type=int, default=ServerArgs.max_seq_len_override, help="The maximum sequence length override.", ) parser.add_argument( "--max-output-tokens", type=_positive_int, default=ServerArgs.max_output_tokens, help="Default max output tokens for requests that omit one (default 32k).", ) parser.add_argument( "--memory-ratio", type=float, default=ServerArgs.memory_ratio, help=( "Fraction of total GPU free memory the engine may use for weights + MoE " "cache + KV cache combined; the remainder is reserved runtime headroom." ), ) assert ServerArgs.use_dummy_weight == False parser.add_argument( "--dummy-weight", action="store_true", dest="use_dummy_weight", help="Use dummy weights for testing.", ) assert ServerArgs.use_pynccl == True parser.add_argument( "--disable-pynccl", action="store_false", dest="use_pynccl", help="Disable PyNCCL for tensor parallelism.", ) parser.add_argument( "--host", type=str, dest="server_host", default=ServerArgs.server_host, help="The host address for the server.", ) parser.add_argument( "--port", type=int, dest="server_port", default=ServerArgs.server_port, help="The port number for the server to listen on.", ) parser.add_argument( "--api-key", type=str, default=ServerArgs.api_key, help="Require this key on every route except /health, sent as " "'Authorization: Bearer ' or 'x-api-key: '. Defaults to $FREETOKEN_API_KEY; " "unset serves without authentication.", ) parser.add_argument( "--cuda-graph-max-bs", "--graph", type=int, default=ServerArgs.cuda_graph_max_bs, help="The maximum batch size for CUDA graph capture. None means auto-tuning based on the GPU memory.", ) parser.add_argument( "--num-tokenizer", "--tokenizer-count", type=int, default=ServerArgs.num_tokenizer, help="The number of tokenizer processes to launch. 0 means the tokenizer is shared with the detokenizer.", ) parser.add_argument( "--max-prefill-length", "--max-extend-length", type=int, dest="max_extend_tokens", default=ServerArgs.max_extend_tokens, help="Chunk Prefill maximum chunk size in tokens.", ) parser.add_argument( "--decode-log-interval", type=_positive_int, default=ServerArgs.decode_log_interval, help="Print one decode scheduler status line every N decode forwards.", ) kv_capacity_group = parser.add_mutually_exclusive_group() kv_capacity_group.add_argument( "--num-pages", dest="num_page_override", type=int, default=ServerArgs.num_page_override, help="Set the maximum number of pages for KVCache.", ) kv_capacity_group.add_argument( "--num-tokens", dest="num_token_override", type=int, default=ServerArgs.num_token_override, help=( "Total KV-cache capacity in tokens; must be a multiple of the resolved page " "size (DSV4: 128 window page, TRTLLM backend: 64). Mutually exclusive with " "--num-pages." ), ) parser.add_argument( "--page-size", type=int, default=ServerArgs.page_size, help="Set the page size for system management.", ) parser.add_argument( "--attention-backend", "--attn", type=validate_attn_backend, default=ServerArgs.attention_backend, help="The attention backend to use. If two backends are specified," " the first one is used for prefill and the second one for decode.", ) parser.add_argument( "--model-source", type=str, default="huggingface", choices=["huggingface", "modelscope"], help="The source to download model from. Either 'huggingface' or 'modelscope'.", ) parser.add_argument( "--cache-type", type=str, default=ServerArgs.cache_type, choices=SUPPORTED_CACHE_MANAGER.supported_names(), help="KV cache strategy (naive | radix). For hybrid GDN models 'radix' is materialized " "as a GDN-aware radix (cross-request GDN-state prefix reuse); pass 'naive' to opt out.", ) parser.add_argument( "--text-model-only", action="store_true", default=False, help="Serve a multimodal checkpoint text-only: no encoder tower is built (its VRAM goes to " "the KV/expert pools) and every multimodal input is rejected. Same as --mm-disable with " "every encoder kind.", ) parser.add_argument( "--mm-disable", nargs="+", choices=list(ENCODER_KINDS), default=[], metavar="{vision,audio}", help="Encoder towers to leave unbuilt; every input they would serve is rejected.", ) parser.add_argument( "--mmproj", type=str, default=None, metavar="FILE", help="llama.cpp vision tower (mmproj GGUF) for a GGUF model. Default: a *mmproj*.gguf that fits the model " "in the model file's folder (BF16 > F32 > F16 > quantized when several are there); none found, or files of " "several models, serves text only.", ) parser.add_argument( "--image-min-tokens", type=_positive_int, default=MultimodalConfig.image_min_tokens, help="Fewest tokens an image may take: the image processor scales smaller images up to it, " "in the family's own units. Default: the processor's own limit.", ) parser.add_argument( "--image-max-tokens", type=_positive_int, default=MultimodalConfig.image_max_tokens, help="Most tokens an image may take: the image processor scales larger images down to it, " "in the family's own units (Qwen VL: one token per 32x32 pixels). Default: the processor's own limit.", ) parser.add_argument( "--mm-processor-kwargs", type=_json_object, default=None, metavar="JSON", help="JSON object of extra keyword arguments for the checkpoint's image processor call, " "for family-specific knobs; applied after the token budget.", ) parser.add_argument( "--mm-embed-cache-device", choices=["cpu", "cuda"], default=MultimodalConfig.embed_cache_device, help="Storage for encoded image embeddings between prefill chunks.", ) parser.add_argument( "--mm-encoder-weights", choices=["gpu", "host"], default=MultimodalConfig.encoder_weights, help="Encoder tower block weights: pinned host banks streamed two blocks at a time behind the " "compute (default, about 60 MiB of VRAM instead of the whole tower), or resident on the GPU.", ) parser.add_argument( "--allowed-media-domains", type=str, default=ServerArgs.allowed_media_domains, help="Comma-separated hostname allowlist for client-supplied image URLs. " "Empty (default) allows any domain.", ) parser.add_argument( "--allowed-local-media-path", type=str, default=ServerArgs.allowed_local_media_path, help="Directory that file:// image refs may be read from. " "Unset (default) rejects local files.", ) parser.add_argument( "--enable-cache-report", action="store_true", default=ServerArgs.enable_cache_report, help=( "Return the number of prefix-cached prompt tokens in each response's usage block " "(OpenAI usage.prompt_tokens_details.cached_tokens, Anthropic " "usage.cache_read_input_tokens, Responses usage.input_tokens_details.cached_tokens). " "On /v1/messages this also makes input_tokens EXCLUDE the cached prefix, matching " "Anthropic billing semantics." ), ) parser.add_argument( "--enable-timings-report", action="store_true", default=ServerArgs.enable_timings_report, help=( "Attach a llama.cpp-shaped per-request timings object to every chat completion, " "completion and Responses response (and to the final stream chunk), so frontends " "like llama-swap and Open Web UI can show prefill/decode rates and cache hits. " "Spans are measured in the scheduler; queue_ms is a FreeToken extension." ), ) parser.add_argument( "--anthropic-inline-system", choices=("auto", "preserve", "fold"), default=ServerArgs.anthropic_inline_system, help="Preserve inline system instructions when supported by the renderer, " "or fold them into nearby user/tool content without hoisting the prompt prefix.", ) parser.add_argument( "--sampling-defaults", type=str, default=ServerArgs.sampling_defaults, choices=["model", "none"], help=( "Source for unspecified request sampling params. 'model' fills " "temperature/top_k/top_p from the checkpoint's generation_config.json " "(recommended for reasoning models to avoid greedy repetition loops); " "'none' uses framework defaults only." ), ) parser.add_argument( "--served-model-name", type=str, default=ServerArgs.served_model_name, help="Model id returned by /v1/models. Defaults to the basename of --model.", ) parser.add_argument( "--tool-call-parser", type=str, default="auto", choices=[ "auto", "llama3", "qwen", "qwen25", "qwen3_coder", "ricursive", "mistral", "deepseekv32", "deepseekv41", "gemma4", "glm47", "minimax", "minimax_m3", "muse_glimmer", "gpt_oss", "gpt-oss", ], help="Tool-call parser format for OpenAI-compatible tool responses.", ) parser.add_argument( "--reasoning-parser", type=str, default="auto", choices=[ "auto", "off", "deepseekv32", "gpt_oss", "qwen3", "glm", "minimax", "minimax_m3", "muse_glimmer", "gemma4", "flint", "ricursive", ], help=( "Reasoning parser that splits chain-of-thought into reasoning_content " "for OpenAI responses. 'auto' selects per model family (gpt-oss Harmony, " " for qwen3/glm/minimax, for minimax-m3, ATEM to=self " "channels for muse-glimmer, gemma thought, dsv4); 'off' disables it." ), ) parser.add_argument( "--moe-strategy", default=ServerArgs.moe_strategy, choices=["auto", *MOE_STRATEGIES], help=( "How the routed experts are served. 'auto' resolves a MoE model to 'offload', and to " "resident 'fused' experts on unified-memory GPUs (GB10 / DGX Spark)." ), ) parser.add_argument( "--moe-backend", dest="moe_strategy", action=_DeprecatedAlias, new_flag="--moe-strategy", default=argparse.SUPPRESS, choices=["auto", *MOE_STRATEGIES], help="[Deprecated] Use --moe-strategy.", ) parser.add_argument( "--quant-backend", default=None, type=_parse_quant_backend, help=( "Kernel per quantized layer type: comma-separated layer[.kind]=name entries, e.g. " "'linear=marlin,moe=b12x' or 'moe.nvfp4=triton'. A layer-level entry applies to every " "kind whose kernel table lists the name; unlisted tables stay automatic." ), ) parser.add_argument( "--ple-backend", default=ServerArgs.ple_backend, choices=["pinned", "disk"], help=( "Where a PLE n-gram table lives. 'disk' (default) reads rows straight from the " "checkpoint files; 'pinned' preloads the whole table into page-locked host RAM." ), ) parser.add_argument( "--embed-weights", default=ServerArgs.embed_weights, choices=["gpu", "host"], help=( "Where the input token embedding table lives. 'gpu' (default) keeps it in VRAM; 'host' keeps it in " "pinned host RAM and each lookup reads its rows over PCIe, so its VRAM goes to the expert cache and KV. " "Not for a table tied to the LM head." ), ) parser.add_argument( "--swa-decoder-replay", default=ServerArgs.swa_decoder_replay, choices=["bounded", "exact"], help=( "DeepSeek-V4.1 Decoder SWA Bounded Replay. 'bounded' (default) runs the 20 decoder layers " "on each prompt's last 128 tokens with their sliding window truncated there, as in the " "tech report; 'exact' runs them on every prompt token (the reference numerics)." ), ) parser.add_argument( "--nvfp4-backend", action=_DeprecatedAlias, new_flag="--quant-backend moe.nvfp4=", convert=_nvfp4_entry, default=argparse.SUPPRESS, choices=["auto", "marlin", "flashinfer", "triton"], help="[Deprecated] Use --quant-backend moe.nvfp4= ('flashinfer' is b12x).", ) parser.add_argument( "--expert-load", default=ServerArgs.expert_load, choices=["auto", "serial", "parallel"], help=( "How MoE expert banks are read into host RAM. 'auto' (default) reads scattered " "experts in parallel (fast) but falls back to serial when free RAM can't cover " "the banks + the parallel reader's extra whole-shard buffer; 'serial' forces the " "low-memory reclaimable read (slower); 'parallel' forces the fast read." ), ) moe_cache_group = parser.add_mutually_exclusive_group() moe_cache_group.add_argument( "--moe-cache-size", type=int, default=ServerArgs.moe_cache_size, help="The number of unified MoE expert slots on GPU.", ) moe_cache_group.add_argument( "--moe-cache-rate", type=_parse_moe_cache_rate, default=ServerArgs.moe_cache_rate, help="The fraction of all MoE experts to keep in GPU cache.", ) moe_cache_group.add_argument( "--moe-cache-mem", type=_parse_mem_size, default=ServerArgs.moe_cache_mem, help="GPU memory for the MoE expert slot cache, e.g. 12GiB or 800MiB (G, GB and GiB all mean GiB); the slot count is this over the bytes per slot.", ) moe_cache_group.add_argument( "--moe-cache-auto", action="store_true", default=ServerArgs.moe_cache_auto, help=( "Auto-pick --moe-cache-size from free VRAM and expert size, MoE-priority " "(KV gets --kv-reserve-tokens as a floor). Not supported for owned-KV models." ), ) parser.add_argument( "--kv-reserve-tokens", type=int, default=ServerArgs.kv_reserve_tokens, help="KV-cache token floor reserved before --moe-cache-auto fills experts.", ) parser.add_argument( "--moe-cache-scope", choices=SCOPES, default=ServerArgs.moe_cache_scope, help=( "How MoE layers share the GPU expert slot cache: unified = one LRU pool per expert quant " "(all layers when they share one quant format); per-layer = each layer has its own pool. " "Pools are sized by --moe-cache-split." ), ) parser.add_argument( "--moe-cache-split", type=_parse_cache_split, default=ServerArgs.moe_cache_split, help=( "How pools share the slots: equal (by layer count), or profile: = by expert use from a " "profile file. With --moe-strategy offload, a pool smaller than the expert count limits the " "decode batch to its slots / top_k." ), ) parser.add_argument( "--moe-cache-policy", default=ServerArgs.moe_cache_policy, choices=["lru"], help="The unified MoE cache eviction policy.", ) parser.add_argument( "--moe-cpu-threads", type=int, default=ServerArgs.moe_cpu_threads, help=( "Number of CPU worker threads for --moe-strategy cpu decode experts. " "0 = auto (physical cores)." ), ) parser.add_argument( "--moe-cpu-layers", type=str, default=ServerArgs.moe_cpu_layers, help=( "With --moe-strategy offload/hybrid: which MoE layers compute on the " "CPU executor instead of the GPU offload/PCIe path (where CUDA pinning " "is quota-capped, e.g. WSL, their banks are OS-locked instead of pinned). Explicit id list ('3,7,11'), a count ('8' = 8 " "layers evenly strided), a fraction ('0.5'), or 'auto'. 'auto' is for hosts that cap CUDA " "pinned memory (Windows/WSL): it pins the banks layer by layer from the middle out until CUDA " "refuses one and OS-locks the rest. FREETOKEN_PIN_BUDGET_GB, FTW checkpoints and models the CPU executor " "cannot serve lock head+tail layers from a size estimate instead. Under hybrid, locked layers still decode from GPU slots " "(--moe-locked-fetch). Any value, 'auto' included, commits to CPU " "decode before the model is built, so the expert format must have a CPU executor " "path (bf16, fp8_block, nvfp4, mxfp4); do not pass it on Linux. Unset = every layer on the " "GPU; a boot whose banks exceed a known pin budget stops and asks for this flag." ), ) parser.add_argument( "--moe-hybrid-max-fetch", type=int, default=ServerArgs.moe_hybrid_max_fetch, help=( "For --moe-strategy hybrid: max experts fetched over PCIe per (layer, decode " "step); the rest of that step's misses are computed on the CPU, overlapped. " "-1 (default) = auto: fetch the benched pcie/cpu bandwidth fraction of each " "step's misses (perfect overlap; needs the `ft bench bw` row of the layer's kind, " "from `ft bench bw --model `, else 1). " "0 = never fetch (all misses on CPU); large = behaves like plain offload." ), ) parser.add_argument( "--moe-locked-fetch", type=float, default=ServerArgs.moe_locked_fetch, help=( "For --moe-strategy hybrid with locked layers (banks over the pin quota, OS-locked instead of pinned): " "the share of a locked layer's misses fetched per decode step into GPU slots of its own pools, through " "pinned staging that CPU threads fill; the rest are computed on the CPU. -1 (default) = auto: per layer from " "the `ft bench bw` row of its kind, p / (2p + c) of the overlapped PCIe and CPU MoE rates (at most 1 expert " "per step without a row). 0 = locked layers decode on the CPU only." ), ) parser.add_argument( "--moe-locked-max-fetch", type=int, default=ServerArgs.moe_locked_max_fetch, help=( "For --moe-locked-fetch: the most experts a locked layer fetches per decode step. Sizes the pinned " "staging: this many rows of the widest expert, per bank." ), ) parser.add_argument( "--disable-moe-prefill-overlap", action="store_false", dest="moe_prefill_overlap", default=ServerArgs.moe_prefill_overlap, help=( "Disable two-buffer overlap for prefill MoE expert copies. " "By default, prefill overlap is enabled and requires " "--moe-cache-size >= 2 * num_experts." ), ) parser.add_argument( "--enable-special-token-ckpt", action="store_true", dest="special_token_ckpt", default=ServerArgs.special_token_ckpt, help=( "Checkpoint decode state at special tokens (currently the tool-call opener). " "When a GDN-hybrid or SWA model samples its tool-call opener token, the " "scheduler preserves a reuse point just after it (GDN: a state snapshot " "donated to the prefix cache; SWA: the trailing window is kept resumable), so " "a client that rewrites the echoed tool call only invalidates the call body, " "not the turn." ), ) parser.add_argument( "--moe-collect-stats", action="store_true", default=ServerArgs.moe_collect_stats, help=( "With the offload-family strategies: after each decode log line, log the decode MoE stats since the " "previous one (hit rate, and per layer-step the misses, experts fetched to the GPU and experts computed " "on the CPU) for pinned layers, locked layers with GPU slots and CPU-only layers. Adds counter updates " "to every decode step." ), ) parser.add_argument( "--moe-prefill-hit-d2d", action="store_true", dest="moe_prefill_hit_d2d", default=ServerArgs.moe_prefill_hit_d2d, help=( "During prefill prefetch, copy cache-resident experts device-side into " "the double buffer and stream only the misses over PCIe " "(CUDA >= 13.0 or HIP with batch-copy support). Effective with " "--moe-cache-size > 2 * num_experts." ), ) parser.add_argument( "--speculative-algorithm", "--spec-algo", type=str, default=ServerArgs.speculative_algorithm, help=( "Speculative decoding with this draft method (mtp: the checkpoint's own MTP layers; dflash: a DFlash 2 draft " "checkpoint from --speculative-draft-model-path). Off by default." ), ) parser.add_argument( "--speculative-num-steps", "--spec-num-steps", type=int, default=ServerArgs.speculative_num_steps, help="Draft tokens per decode step (k); 0 takes the method's default.", ) parser.add_argument( "--speculative-draft-model-path", "--spec-draft-model", type=str, default=ServerArgs.speculative_draft_model_path, help="Draft weights; by default they come from the target checkpoint. Required for dflash, which reads a separate draft checkpoint.", ) parser.add_argument( "--shell-mode", action="store_true", help="Run the server in shell mode.", ) parser.add_argument( "--cors-origins", type=str, default=ServerArgs.cors_origins, help=( "Comma-separated CORS allow-list for browser/webview clients " "(default: local Tauri/Vite dev origins). '' disables, '*' allows any." ), ) # Parse arguments kwargs = parser.parse_args(args).__dict__.copy() # reject a too-long list here with a clear reason, not as a dead rank later if len(kwargs["gpu"]) not in (0, kwargs["tensor_parallel_size"]): if kwargs["tensor_parallel_size"] == 1 and len(kwargs["gpu"]) > 1: parser.error("tensor parallelism is not supported yet: --gpu takes one entry") parser.error( f"--gpu has {len(kwargs['gpu'])} entries but --tensor-parallel-size is " f"{kwargs['tensor_parallel_size']}; give one entry per TP rank" ) # resolve some arguments run_shell |= kwargs.pop("shell_mode") kwargs["shell_mode"] = run_shell if run_shell: kwargs["cuda_graph_max_bs"] = 1 kwargs["max_running_req"] = 1 kwargs["silent_output"] = True # the old flag stands in for one --quant-backend entry; next to the real flag it is a usage error entry = kwargs.pop("nvfp4_backend", None) if entry is not None: if kwargs["quant_backend"] is not None: parser.error("--nvfp4-backend cannot be combined with --quant-backend; write --quant-backend moe.nvfp4=... instead") if entry: kwargs["quant_backend"] = entry # an empty flag usually means an unset shell variable; refuse it rather than serve open if kwargs["api_key"] == "": parser.error("--api-key is empty; omit it to serve without authentication") from freetoken.launch import resolve_api_key kwargs["api_key"] = resolve_api_key(kwargs["api_key"]) if kwargs["model_path"].startswith("~"): kwargs["model_path"] = os.path.expanduser(kwargs["model_path"]) # every process reads the GGUF config, tokenizer workers included, so the choice travels in the environment from freetoken.models.gguf.mmproj import MMPROJ_ENV os.environ.pop(MMPROJ_ENV, None) if (mmproj := kwargs.pop("mmproj")) is not None: from freetoken.models.gguf.reader import is_gguf_path if not is_gguf_path(kwargs["model_path"]): parser.error("--mmproj needs a .gguf model file") os.environ[MMPROJ_ENV] = os.path.abspath(os.path.expanduser(mmproj)) # a bad media root is a deployment mistake; fail at startup, not per request if kwargs["allowed_local_media_path"]: media_root = os.path.realpath(os.path.expanduser(kwargs["allowed_local_media_path"])) if not os.path.isdir(media_root): parser.error(f"--allowed-local-media-path {media_root} is not a directory") kwargs["allowed_local_media_path"] = media_root if kwargs["served_model_name"] is None: kwargs["served_model_name"] = ( os.path.basename(os.path.normpath(kwargs["model_path"])) or kwargs["model_path"] ) if kwargs["tool_call_parser"] == "auto": kwargs["tool_call_parser"] = _infer_tool_call_parser(kwargs["model_path"]) if kwargs["reasoning_parser"] == "auto": kwargs["reasoning_parser"] = _infer_reasoning_parser(kwargs["model_path"]) elif kwargs["reasoning_parser"] == "off": kwargs["reasoning_parser"] = None # Offload-family backends (offload/cpu/hybrid) need a slot cache; if the user gave no # sizing flag at all, default to --moe-cache-auto so a bare `ft serve ` works # out of the box (the scheduler resolves the size from free VRAM). Explicit # size/rate/auto is preserved. from freetoken.moe import is_offload_moe_strategy _no_cache_flag = ( kwargs["moe_cache_size"] == 0 and not kwargs["moe_cache_auto"] and kwargs["moe_cache_mem"] is None and (kwargs["moe_cache_rate"] is None or kwargs["moe_cache_rate"] == 0) ) if is_offload_moe_strategy(kwargs["moe_strategy"]) and _no_cache_flag: kwargs["moe_cache_auto"] = True if kwargs["model_source"] == "modelscope": model_path = kwargs["model_path"] if not os.path.isdir(model_path): from modelscope import snapshot_download ignore_patterns = [] if kwargs["use_dummy_weight"]: ignore_patterns = ["*.bin", "*.safetensors", "*.pt", "*.ckpt"] model_path = snapshot_download(model_path, ignore_patterns=ignore_patterns) kwargs["model_path"] = model_path del kwargs["model_source"] # "auto" (or an unspecified dtype) resolves to the checkpoint's dtype. Multimodal / # hybrid configs (e.g. Qwen3.5-MoE) keep it under ``text_config`` and use the newer # ``dtype`` key rather than top-level ``torch_dtype``, so check both; default bf16. if (dtype_str := kwargs["dtype"]) in ("auto", None): from freetoken.utils import cached_load_hf_config cfg = cached_load_hf_config(kwargs["model_path"]).to_dict() text_cfg = cfg.get("text_config") or {} dtype_str = ( cfg.get("torch_dtype") or cfg.get("dtype") or text_cfg.get("torch_dtype") or text_cfg.get("dtype") or "bfloat16" ) DTYPE_MAP = { "float16": torch.float16, "bfloat16": torch.bfloat16, "float32": torch.float32, } kwargs["dtype"] = DTYPE_MAP[dtype_str] if isinstance(dtype_str, str) else dtype_str kwargs["tp_info"] = DistributedInfo(0, kwargs["tensor_parallel_size"]) del kwargs["tensor_parallel_size"] disabled = set(ENCODER_KINDS) if kwargs.pop("text_model_only") else set() disabled.update(kwargs.pop("mm_disable")) image_min_tokens, image_max_tokens = kwargs.pop("image_min_tokens"), kwargs.pop("image_max_tokens") if image_min_tokens is not None and image_max_tokens is not None and image_min_tokens > image_max_tokens: parser.error(f"--image-min-tokens {image_min_tokens} exceeds --image-max-tokens {image_max_tokens}") kwargs["mm"] = MultimodalConfig( disabled_encoders=frozenset(disabled), embed_cache_device=kwargs.pop("mm_embed_cache_device"), encoder_weights=kwargs.pop("mm_encoder_weights"), image_min_tokens=image_min_tokens, image_max_tokens=image_max_tokens, processor_kwargs=kwargs.pop("mm_processor_kwargs") or {}, ) kwargs["hf_overrides"] = kwargs["hf_overrides"] or {} result = ServerArgs(**kwargs) logger.info(f"Parsed arguments:\n{result}") return result, run_shell