"""CLI: convert an HF safetensors checkpoint to a FreeToken Weight (FTW) checkpoint. ft checkpoint --model --out \ [--dtype bfloat16] [--moe-backend offload] [--quant-backend moe.nvfp4=b12x] [--shard-gib 8] [--gpu ] The output dir is self-contained: point the server's ``--model`` at it to load via the FTW fast path (auto-detected). """ from __future__ import annotations import argparse import time import torch from freetoken.gpu_select import assign_gpu, bind_assigned_gpu, single_gpu_arg from .convert import convert_checkpoint _DTYPES = {"bfloat16": torch.bfloat16, "float16": torch.float16, "float32": torch.float32} def _parse_quant_backend(value: str) -> str: from freetoken.layers.quantization import QuantBackend try: QuantBackend.parse(value) except ValueError as exc: raise argparse.ArgumentTypeError(str(exc)) from None return value def main(argv: list[str] | None = None, prog: str = "freetoken.checkpoint") -> int: p = argparse.ArgumentParser(prog=prog, description=__doc__) p.add_argument("--model", required=True, help="source HF safetensors checkpoint dir") p.add_argument("--out", required=True, help="output FTW checkpoint dir") p.add_argument("--dtype", choices=sorted(_DTYPES), default="bfloat16") p.add_argument("--moe-backend", default="offload", help="kept for compatibility: MoE experts always go to banks, which every strategy reads") p.add_argument("--quant-backend", type=_parse_quant_backend, default=None, help="kernel per quantized layer type, as for ft serve; the expert banks are packed for the " "chosen MoE kernel and the server picks that kernel back up from the FTW") p.add_argument("--shard-gib", type=float, default=8.0, help="max shard size in GiB") p.add_argument("--gpu", type=single_gpu_arg, default=None, help="GPU for the repack: a GPU UUID (GPU-xxxx..., as nvidia-smi -L prints) or " "an nvidia-smi index (default: the first visible GPU)") ns = p.parse_args(argv) # same as ft serve --gpu: resolve, then bind by UUID at CUDA init try: assign_gpu(ns.gpu) device = f"cuda:{bind_assigned_gpu().index}" except (ValueError, RuntimeError) as e: p.error(str(e)) shard_limit = int(ns.shard_gib * (1 << 30)) shard_limit -= shard_limit % 4096 # keep aligned t = time.perf_counter() index = convert_checkpoint( ns.model, ns.out, dtype=_DTYPES[ns.dtype], moe_backend=ns.moe_backend, quant_backend=ns.quant_backend, shard_limit=shard_limit, device=device, ) dt = time.perf_counter() - t c = index["counts"] gib = index["total_bytes"] / (1 << 30) print(f"\nwrote FTW checkpoint -> {ns.out}") print(f" tensors: {c['weight']} weight + {c['experts_bank']} experts_bank") print(f" FTW: {gib:.2f} GiB across {len(index['shards'])} shard(s) " f"(<= {ns.shard_gib} GiB each)") print(f" quant_format: {index['quant_format']} fingerprint={index['fingerprint']}") print(f" converted in {dt:.1f}s") return 0 if __name__ == "__main__": raise SystemExit(main())