# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re from pathlib import Path import torch import torch.nn.functional as F from datasets import load_dataset from diffusers.models.attention_processor import Attention, AttnProcessor from diffusers.models.lora import LoRACompatibleConv, LoRACompatibleLinear from diffusers.utils import load_image import modelopt.torch.quantization as mtq from modelopt.torch.quantization.plugins.diffusion.diffusers import AttentionModuleMixin USE_PEFT = True try: from peft.tuners.lora.layer import Conv2d as PEFTLoRAConv2d from peft.tuners.lora.layer import Linear as PEFTLoRALinear except ModuleNotFoundError: USE_PEFT = False # Model-specific filter functions for quantization def filter_func_default(name: str) -> bool: """Default filter function for general models.""" pattern = re.compile( r".*(time_emb_proj|time_embedding|conv_in|conv_out|conv_shortcut|add_embedding|pos_embed|time_text_embed|context_embedder|norm_out|x_embedder).*" ) return pattern.match(name) is not None def check_conv_and_mha(backbone, if_fp4, quantize_mha): for name, module in backbone.named_modules(): if isinstance(module, (torch.nn.Conv1d, torch.nn.Conv2d)) and if_fp4: module.weight_quantizer.disable() module.input_quantizer.disable() print(f"Disabled NVFP4 Conv layer quantization for layer {name}") elif isinstance(module, (Attention, AttentionModuleMixin)): head_size = int(module.inner_dim / module.heads) if not quantize_mha or head_size % 16 != 0: for attr in ( "q_bmm_quantizer", "k_bmm_quantizer", "v_bmm_quantizer", "softmax_quantizer", "bmm2_output_quantizer", ): if hasattr(module, attr): getattr(module, attr).disable() print(f"Disabled Attention layer quantization for layer {name}") def filter_func_ltx_video(name: str) -> bool: """Filter function specifically for LTX-Video models.""" pattern = re.compile( r".*(proj_in|time_embed|caption_projection|proj_out|patchify_proj|adaln_single|transformer_blocks\.(0|1|2|45|46|47)\.).*" ) return pattern.match(name) is not None def filter_func_flux_dev(name: str) -> bool: """Filter function specifically for Flux-dev models.""" pattern = re.compile( r"(proj_out.*|.*(time_text_embed|context_embedder|x_embedder|norm_out|time_guidance_embed|stream_modulation).*)" ) return pattern.match(name) is not None def filter_func_ltx2_vae(name: str) -> bool: """Filter for LTX-2 VAE: keeps only conv1/conv2 in up_blocks resnets.""" keep = re.compile(r".*up_blocks\.\d+\.resnets\.\d+\.conv[12](?:\.|$)") return not keep.match(name) def filter_func_wan_vae(name: str) -> bool: """Filter for Wan 2.2 VAE: keeps only conv1/conv2 in resnet blocks.""" keep = re.compile( r".*(down_blocks\.\d+\.(?:resnets\.\d+\.)?conv[12]" r"|mid_block\.resnets\.\d+\.conv[12]" r"|up_blocks\.\d+\.resnets\.\d+\.conv[12])(?:\.|$)" ) return not keep.match(name) def filter_func_wan_video(name: str) -> bool: """Filter function specifically for WAN-Video models.""" pattern = re.compile( r".*(patch_embedding|condition_embedder|proj_out|blocks\.(0|1|2|37|38|39)\.).*" ) return pattern.match(name) is not None # Qwen-Image's transformer has 60 ``transformer_blocks``. The recipe quantizes # only those blocks while keeping the first two and last two -- and everything # outside ``transformer_blocks`` -- in original precision. The model-agnostic, # config-driven form of this recipe (deriving the block count from the model) # lives in quantize.py; this name-only filter covers the plain FP8/NVFP4 path # for the full 60-block Qwen-Image transformer. QWEN_IMAGE_NUM_TRANSFORMER_BLOCKS = 60 _QWEN_IMAGE_BLOCK_RE = re.compile(r"(?:^|\.)transformer_blocks\.(\d+)(?:\.|$)") def filter_func_qwen_image(name: str) -> bool: """Filter function specifically for Qwen-Image models. Returns ``True`` for modules to keep in original precision (quantization disabled): everything outside ``transformer_blocks``, plus the first two and last two transformer blocks. """ match = _QWEN_IMAGE_BLOCK_RE.search(name) if match is None: return True block_idx = int(match.group(1)) return block_idx < 2 or block_idx >= QWEN_IMAGE_NUM_TRANSFORMER_BLOCKS - 2 def load_calib_prompts( batch_size, calib_data_path: str | Path = "Gustavosta/Stable-Diffusion-Prompts", split="train", column="Prompt", ) -> list[list[str]]: prompt_list: list[str] = [] if isinstance(calib_data_path, Path): with open(calib_data_path) as f: prompt_list = f.readlines() else: dataset = load_dataset(calib_data_path) prompt_list = list(dataset[split][column]) return [prompt_list[i : i + batch_size] for i in range(0, len(prompt_list), batch_size)] def load_calib_images(folder_path): images = [] for filename in os.listdir(folder_path): img_path = os.path.join(folder_path, filename) if os.path.isfile(img_path): image = load_image(img_path) if image is not None: images.append(image) return images def set_fmha(unet): for name, module in unet.named_modules(): if isinstance(module, Attention): module.set_processor(AttnProcessor()) def check_lora(unet): for name, module in unet.named_modules(): if isinstance(module, (LoRACompatibleConv, LoRACompatibleLinear)): assert module.lora_layer is None, ( f"To quantize {name}, LoRA layer should be fused/merged. Please" " fuse the LoRA layer before quantization." ) elif USE_PEFT and isinstance(module, (PEFTLoRAConv2d, PEFTLoRALinear)): assert module.merged, ( f"To quantize {name}, LoRA layer should be fused/merged. Please" " fuse the LoRA layer before quantization." ) def fp8_mha_disable(backbone, quantized_mha_output: bool = True): def mha_filter_func(name): pattern = re.compile( r".*(q_bmm_quantizer|k_bmm_quantizer|v_bmm_quantizer|softmax_quantizer).*" if quantized_mha_output else r".*(q_bmm_quantizer|k_bmm_quantizer|v_bmm_quantizer|softmax_quantizer|bmm2_output_quantizer).*" ) return pattern.match(name) is not None if hasattr(F, "scaled_dot_product_attention"): mtq.disable_quantizer(backbone, mha_filter_func)