Model-Optimizer/tests/_test_utils/examples/run_command.py

# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import subprocess
import time
from pathlib import Path

from _test_utils.torch_dist.dist_utils import get_free_port

MODELOPT_ROOT = Path(__file__).parent.parent.parent.parent


def _extend_cmd_parts(cmd_parts: list[str], **kwargs):
    for key, value in kwargs.items():
        if value is not None:
            cmd_parts.extend([f"--{key}", str(value)])
    if kwargs.get("trust_remote_code", False):
        cmd_parts.append("--trust_remote_code")
    return cmd_parts


def run_example_command(cmd_parts: list[str], example_path: str):
    print(f"[{example_path}] Running command: {cmd_parts}")
    subprocess.run(cmd_parts, cwd=MODELOPT_ROOT / "examples" / example_path, check=True)


def run_command_in_background(cmd_parts, example_path, stdout=None, stderr=None, text=True):
    print(f"Running command in background: {' '.join(str(part) for part in cmd_parts)}")
    process = subprocess.Popen(
        cmd_parts,
        cwd=MODELOPT_ROOT / "examples" / example_path,
        stdout=stdout,
        stderr=stderr,
        text=text,
    )
    return process


def run_llm_autodeploy_command(
    model: str, quant: str, effective_bits: float, output_dir: str, **kwargs
):
    # Create temporary directory for saving the quantized checkpoint
    port = get_free_port()
    quantized_ckpt_dir = os.path.join(output_dir, "quantized_model")
    kwargs.update(
        {
            "hf_ckpt": model,
            "quant": quant,
            "effective_bits": effective_bits,
            "save_quantized_ckpt": quantized_ckpt_dir,
            "port": port,
        }
    )

    server_handler = None
    try:
        # Quantize and deploy the model to the background
        cmd_parts = _extend_cmd_parts(["scripts/run_auto_quant_and_deploy.sh"], **kwargs)
        # Pass None to stdout and stderr to see the output in the console
        server_handler = run_command_in_background(
            cmd_parts, "llm_autodeploy", stdout=None, stderr=None
        )

        # Wait for the server to start. We might need to buil
        time.sleep(100)

        # Test the deployment
        run_example_command(
            ["python", "api_client.py", "--prompt", "What is AI?", "--port", str(port)],
            "llm_autodeploy",
        )
    finally:
        if server_handler:
            server_handler.terminate()


def run_torch_onnx_command(*, quantize_mode: str, onnx_save_path: str, **kwargs):
    kwargs.update({"quantize_mode": quantize_mode, "onnx_save_path": onnx_save_path})
    cmd_parts = _extend_cmd_parts(["python", "torch_quant_to_onnx.py"], **kwargs)
    run_example_command(cmd_parts, "onnx_ptq")


def run_llm_ptq_command(*, model: str, quant: str, **kwargs):
    kwargs.update({"model": model, "quant": quant})
    kwargs.setdefault("tasks", "build")
    kwargs.setdefault("calib", 16)

    cmd_parts = _extend_cmd_parts(["scripts/huggingface_example.sh", "--no-verbose"], **kwargs)
    run_example_command(cmd_parts, "llm_ptq")


def run_vlm_ptq_command(*, model: str, type: str, quant: str, **kwargs):
    kwargs.update({"model": model, "type": type, "quant": quant})
    kwargs.setdefault("tasks", "build")
    kwargs.setdefault("calib", 16)

    cmd_parts = _extend_cmd_parts(["scripts/huggingface_example.sh"], **kwargs)
    run_example_command(cmd_parts, "vlm_ptq")


def run_diffusers_cmd(cmd_parts: list[str]):
    run_example_command(cmd_parts, "diffusers/quantization")


def run_llm_sparsity_command(
    *, model: str, output_dir: str, sparsity_fmt: str = "sparsegpt", **kwargs
):
    kwargs.update(
        {"model_name_or_path": model, "sparsity_fmt": sparsity_fmt, "output_dir": output_dir}
    )
    kwargs.setdefault("calib_size", 16)
    kwargs.setdefault("device", "cuda")
    kwargs.setdefault("dtype", "fp16")
    kwargs.setdefault("model_max_length", 1024)

    cmd_parts = _extend_cmd_parts(["python", "hf_pts.py"], **kwargs)
    run_example_command(cmd_parts, "llm_sparsity")


def run_llm_sparsity_ft_command(
    *, model: str, restore_path: str, output_dir: str, data_path: str, **kwargs
):
    kwargs.update(
        {
            "model": model,
            "restore_path": restore_path,
            "output_dir": output_dir,
            "data_path": data_path,
        }
    )
    kwargs.setdefault("num_epochs", 0.01)
    kwargs.setdefault("max_length", 128)
    kwargs.setdefault("train_bs", 1)
    kwargs.setdefault("eval_bs", 1)

    cmd_parts = _extend_cmd_parts(["bash", "launch_finetune.sh"], **kwargs)
    run_example_command(cmd_parts, "llm_sparsity")