mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(moe): serve NVFP4 and MXFP4 experts fused through the expert banks Resident experts skip the state dict and load through the offload expert-bank loader (HF pieces or FTW banks) into per-layer GPU banks, so fused serves bf16, fp8_block, NVFP4 and MXFP4 alike. Unified-memory GPUs auto-select fused for these formats and ignore offload-only flags with a warning. BREAKING CHANGE: ft checkpoint always writes MoE experts as banks; an FTW converted with --moe-backend fused or triton (dense experts) no longer loads and must be reconverted. * docs(moe): name only GB10 / DGX Spark for unified memory
78 lines
3.2 KiB
Python
78 lines
3.2 KiB
Python
"""CLI: convert an HF safetensors checkpoint to a FreeToken Weight (FTW) checkpoint.
|
|
|
|
ft checkpoint --model <hf_dir> --out <ftw_dir> \
|
|
[--dtype bfloat16] [--moe-backend offload] [--quant-backend moe.nvfp4=b12x] [--shard-gib 8] [--gpu <uuid-or-index>]
|
|
|
|
The output dir is self-contained: point the server's ``--model`` at it to load via the FTW
|
|
fast path (auto-detected).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import time
|
|
|
|
import torch
|
|
|
|
from freetoken.gpu_select import assign_gpu, bind_assigned_gpu, single_gpu_arg
|
|
|
|
from .convert import convert_checkpoint
|
|
|
|
_DTYPES = {"bfloat16": torch.bfloat16, "float16": torch.float16, "float32": torch.float32}
|
|
|
|
|
|
def _parse_quant_backend(value: str) -> str:
|
|
from freetoken.layers.quantization import QuantBackend
|
|
|
|
try:
|
|
QuantBackend.parse(value)
|
|
except ValueError as exc:
|
|
raise argparse.ArgumentTypeError(str(exc)) from None
|
|
return value
|
|
|
|
|
|
def main(argv: list[str] | None = None, prog: str = "freetoken.checkpoint") -> int:
|
|
p = argparse.ArgumentParser(prog=prog, description=__doc__)
|
|
p.add_argument("--model", required=True, help="source HF safetensors checkpoint dir")
|
|
p.add_argument("--out", required=True, help="output FTW checkpoint dir")
|
|
p.add_argument("--dtype", choices=sorted(_DTYPES), default="bfloat16")
|
|
p.add_argument("--moe-backend", default="offload",
|
|
help="kept for compatibility: MoE experts always go to banks, which every strategy reads")
|
|
p.add_argument("--quant-backend", type=_parse_quant_backend, default=None,
|
|
help="kernel per quantized layer type, as for ft serve; the expert banks are packed for the "
|
|
"chosen MoE kernel and the server picks that kernel back up from the FTW")
|
|
p.add_argument("--shard-gib", type=float, default=8.0, help="max shard size in GiB")
|
|
p.add_argument("--gpu", type=single_gpu_arg, default=None,
|
|
help="GPU for the repack: a GPU UUID (GPU-xxxx..., as nvidia-smi -L prints) or "
|
|
"an nvidia-smi index (default: the first visible GPU)")
|
|
ns = p.parse_args(argv)
|
|
|
|
# same as ft serve --gpu: resolve, then bind by UUID at CUDA init
|
|
try:
|
|
assign_gpu(ns.gpu)
|
|
device = f"cuda:{bind_assigned_gpu().index}"
|
|
except (ValueError, RuntimeError) as e:
|
|
p.error(str(e))
|
|
|
|
shard_limit = int(ns.shard_gib * (1 << 30))
|
|
shard_limit -= shard_limit % 4096 # keep aligned
|
|
t = time.perf_counter()
|
|
index = convert_checkpoint(
|
|
ns.model, ns.out, dtype=_DTYPES[ns.dtype],
|
|
moe_backend=ns.moe_backend, quant_backend=ns.quant_backend, shard_limit=shard_limit, device=device,
|
|
)
|
|
dt = time.perf_counter() - t
|
|
c = index["counts"]
|
|
gib = index["total_bytes"] / (1 << 30)
|
|
print(f"\nwrote FTW checkpoint -> {ns.out}")
|
|
print(f" tensors: {c['weight']} weight + {c['experts_bank']} experts_bank")
|
|
print(f" FTW: {gib:.2f} GiB across {len(index['shards'])} shard(s) "
|
|
f"(<= {ns.shard_gib} GiB each)")
|
|
print(f" quant_format: {index['quant_format']} fingerprint={index['fingerprint']}")
|
|
print(f" converted in {dt:.1f}s")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|