Update 0.23.0 - OSS release

examples/diffusers/cache_diffusion/cache_diffusion/cachify.py

@@ -0,0 +1,138 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import fnmatch
+from contextlib import contextmanager
+
+from diffusers.models.attention import BasicTransformerBlock, JointTransformerBlock
+from diffusers.models.transformers.pixart_transformer_2d import PixArtTransformer2DModel
+from diffusers.models.transformers.transformer_sd3 import SD3Transformer2DModel
+from diffusers.models.unets.unet_2d_blocks import (
+    CrossAttnDownBlock2D,
+    CrossAttnUpBlock2D,
+    DownBlock2D,
+    UNetMidBlock2DCrossAttn,
+    UpBlock2D,
+)
+from diffusers.models.unets.unet_2d_condition import UNet2DConditionModel
+from diffusers.models.unets.unet_3d_blocks import (
+    CrossAttnDownBlockSpatioTemporal,
+    CrossAttnUpBlockSpatioTemporal,
+    DownBlockSpatioTemporal,
+    UNetMidBlockSpatioTemporal,
+    UpBlockSpatioTemporal,
+)
+from diffusers.models.unets.unet_spatio_temporal_condition import UNetSpatioTemporalConditionModel
+
+from .module import CachedModule
+from .utils import replace_module
+
+CACHED_PIPE = {
+    UNet2DConditionModel: (
+        DownBlock2D,
+        CrossAttnDownBlock2D,
+        UNetMidBlock2DCrossAttn,
+        CrossAttnUpBlock2D,
+        UpBlock2D,
+    ),
+    PixArtTransformer2DModel: (BasicTransformerBlock),
+    UNetSpatioTemporalConditionModel: (
+        CrossAttnDownBlockSpatioTemporal,
+        DownBlockSpatioTemporal,
+        UpBlockSpatioTemporal,
+        CrossAttnUpBlockSpatioTemporal,
+        UNetMidBlockSpatioTemporal,
+    ),
+    SD3Transformer2DModel: (JointTransformerBlock),
+}
+
+
+def _apply_to_modules(model, action, modules=None, config_list=None):
+    if hasattr(model, "use_trt_infer") and model.use_trt_infer:
+        for key, module in model.engines.items():
+            if isinstance(module, CachedModule):
+                action(module)
+            elif config_list:
+                for config in config_list:
+                    if _pass(key, config["wildcard_or_filter_func"]):
+                        model.engines[key] = CachedModule(module, config["select_cache_step_func"])
+    else:
+        for name, module in model.named_modules():
+            if isinstance(module, CachedModule):
+                action(module)
+            elif modules and config_list:
+                for config in config_list:
+                    if _pass(name, config["wildcard_or_filter_func"]) and isinstance(
+                        module, modules
+                    ):
+                        replace_module(
+                            model,
+                            name,
+                            CachedModule(module, config["select_cache_step_func"]),
+                        )
+
+
+def cachify(model, config_list, modules):
+    def cache_action(module):
+        pass  # No action needed, caching is handled in the loop itself
+
+    _apply_to_modules(model, cache_action, modules, config_list)
+
+
+def disable(pipe):
+    model = get_model(pipe)
+    _apply_to_modules(model, lambda module: module.disable_cache())
+
+
+def enable(pipe):
+    model = get_model(pipe)
+    _apply_to_modules(model, lambda module: module.enable_cache())
+
+
+def reset_status(pipe):
+    model = get_model(pipe)
+    _apply_to_modules(model, lambda module: setattr(module, "cur_step", 0))
+
+
+def _pass(name, wildcard_or_filter_func):
+    if isinstance(wildcard_or_filter_func, str):
+        return fnmatch.fnmatch(name, wildcard_or_filter_func)
+    elif callable(wildcard_or_filter_func):
+        return wildcard_or_filter_func(name)
+    else:
+        raise NotImplementedError(f"Unsupported type {type(wildcard_or_filter_func)}")
+
+
+def get_model(pipe):
+    if hasattr(pipe, "unet"):
+        return pipe.unet
+    elif hasattr(pipe, "transformer"):
+        return pipe.transformer
+    else:
+        raise KeyError
+
+
+@contextmanager
+def infer(pipe):
+    try:
+        yield pipe
+    finally:
+        reset_status(pipe)
+
+
+def prepare(pipe, config_list):
+    model = get_model(pipe)
+    assert model.__class__ in CACHED_PIPE.keys(), f"{model.__class__} is not supported!"
+    cachify(model, config_list, CACHED_PIPE[model.__class__])

examples/diffusers/cache_diffusion/cache_diffusion/module.py

@@ -0,0 +1,49 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from torch import nn
+
+
+class CachedModule(nn.Module):
+    def __init__(self, block, select_cache_step_func) -> None:
+        super().__init__()
+        self.block = block
+        self.select_cache_step_func = select_cache_step_func
+        self.cur_step = 0
+        self.cached_results = None
+        self.enabled = True
+
+    def __getattr__(self, name):
+        try:
+            return super().__getattr__(name)
+        except AttributeError:
+            return getattr(self.block, name)
+
+    def if_cache(self):
+        return self.select_cache_step_func(self.cur_step) and self.enabled
+
+    def enable_cache(self):
+        self.enabled = True
+
+    def disable_cache(self):
+        self.enabled = False
+        self.cur_step = 0
+
+    def forward(self, *args, **kwargs):
+        if not self.if_cache():
+            self.cached_results = self.block(*args, **kwargs)
+        if self.enabled:
+            self.cur_step += 1
+        return self.cached_results

examples/diffusers/cache_diffusion/cache_diffusion/utils.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+SDXL_DEFAULT_CONFIG = [
+    {
+        "wildcard_or_filter_func": lambda name: "up_blocks.2" not in name,
+        "select_cache_step_func": lambda step: (step % 2) != 0,
+    }
+]
+
+PIXART_DEFAULT_CONFIG = [
+    {
+        "wildcard_or_filter_func": lambda name: not re.search(
+            r"transformer_blocks\.(2[1-7])\.", name
+        ),
+        "select_cache_step_func": lambda step: (step % 3) != 0,
+    }
+]
+
+SVD_DEFAULT_CONFIG = [
+    {
+        "wildcard_or_filter_func": lambda name: "up_blocks.3" not in name,
+        "select_cache_step_func": lambda step: (step % 2) != 0,
+    }
+]
+
+SD3_DEFAULT_CONFIG = [
+    {
+        "wildcard_or_filter_func": lambda name: re.search(
+            r"^((?!transformer_blocks\.(1[6-9]|2[0-3])).)*$", name
+        ),
+        "select_cache_step_func": lambda step: (step % 2) != 0,
+    }
+]
+
+
+def replace_module(parent, name_path, new_module):
+    path_parts = name_path.split(".")
+    for part in path_parts[:-1]:
+        parent = getattr(parent, part)
+    setattr(parent, path_parts[-1], new_module)