fix: release VAE compute buffer after tiled encoding (#1465)

Co-authored-by: professor-moody <keys@nimbus.lan>

.dockerignore

@@ -1,4 +1,5 @@
 build*/
+docs/
 test/
 
 .cache/

.github/workflows/build.yml

@@ -21,11 +21,14 @@ on:
         "**/*.c",
         "**/*.cpp",
         "**/*.cu",
+        "examples/server/frontend",
+        "examples/server/frontend/**",
       ]
   pull_request:
     types: [opened, synchronize, reopened]
     paths:
       [
+        ".github/workflows/**",
         "**/CMakeLists.txt",
         "**/Makefile",
         "**/*.h",
@@ -33,11 +36,17 @@ on:
         "**/*.c",
         "**/*.cpp",
         "**/*.cu",
+        "examples/server/frontend",
+        "examples/server/frontend/**",
       ]
 
 env:
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   ubuntu-latest-cmake:
     runs-on: ubuntu-latest
@@ -49,6 +58,16 @@ jobs:
         with:
           submodules: recursive
 
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.15.1
+
       - name: Dependencies
         id: depends
         run: |
@@ -65,8 +84,8 @@ jobs:
 
       - name: Get commit hash
         id: commit
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/main' ) || github.event.inputs.create_release == 'true' }}
-        uses: pr-mpt/actions-commit-hash@v2
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        uses: prompt/actions-commit-hash@v2
 
       - name: Fetch system info
         id: system-info
@@ -92,6 +111,145 @@ jobs:
           path: |
             sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip
 
+  ubuntu-latest-cmake-vulkan:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.15.1
+
+      - name: Dependencies
+        id: depends
+        run: |
+          sudo apt-get update
+          sudo apt-get install build-essential libvulkan-dev glslc
+
+      - name: Build
+        id: cmake_build
+        run: |
+          mkdir build
+          cd build
+          cmake .. -DSD_BUILD_SHARED_LIBS=ON -DSD_VULKAN=ON
+          cmake --build . --config Release
+
+      - name: Get commit hash
+        id: commit
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        uses: prompt/actions-commit-hash@v2
+
+      - name: Fetch system info
+        id: system-info
+        run: |
+          echo "CPU_ARCH=`uname -m`" >> "$GITHUB_OUTPUT"
+          echo "OS_NAME=`lsb_release -s -i`" >> "$GITHUB_OUTPUT"
+          echo "OS_VERSION=`lsb_release -s -r`" >> "$GITHUB_OUTPUT"
+          echo "OS_TYPE=`uname -s`" >> "$GITHUB_OUTPUT"
+
+      - name: Pack artifacts
+        id: pack_artifacts
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        run: |
+          cp ggml/LICENSE ./build/bin/ggml.txt
+          cp LICENSE ./build/bin/stable-diffusion.cpp.txt
+          zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-vulkan.zip ./build/bin/*
+
+      - name: Upload artifacts
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-vulkan.zip
+          path: |
+            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-vulkan.zip
+
+  build-and-push-docker-images:
+    name: Build and push container images
+    if: ${{ github.event_name != 'pull_request' }}
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
+      artifact-metadata: write
+
+    strategy:
+      matrix:
+        variant: [musa, sycl, vulkan, cuda]
+
+    env:
+      REGISTRY: ghcr.io
+      IMAGE_NAME: ${{ github.repository }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.15.1
+
+      - name: Get commit hash
+        id: commit
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        uses: prompt/actions-commit-hash@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@v1.3.1
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+
+      - name: Build and push Docker image
+        id: build-push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64
+          push: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+          file: Dockerfile.${{ matrix.variant }}
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH_NAME }}-${{ matrix.variant }}
+          labels: ${{ steps.meta.outputs.labels }}
+          annotations: ${{ steps.meta.outputs.annotations }}
+
   macOS-latest-cmake:
     runs-on: macos-latest
 
@@ -102,6 +260,16 @@ jobs:
         with:
           submodules: recursive
 
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.15.1
+
       - name: Dependencies
         id: depends
         run: |
@@ -118,8 +286,8 @@ jobs:
 
       - name: Get commit hash
         id: commit
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/main' ) || github.event.inputs.create_release == 'true' }}
-        uses: pr-mpt/actions-commit-hash@v2
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        uses: prompt/actions-commit-hash@v2
 
       - name: Fetch system info
         id: system-info
@@ -146,7 +314,7 @@ jobs:
             sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip
 
   windows-latest-cmake:
-    runs-on: windows-2025
+    runs-on: windows-2022
 
     env:
       VULKAN_VERSION: 1.4.328.1
@@ -163,10 +331,8 @@ jobs:
           - build: "avx512"
             defines: "-DGGML_NATIVE=OFF -DGGML_AVX512=ON -DGGML_AVX=ON -DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON"
           - build: "cuda12"
-            defines: "-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_ARCHITECTURES=90;89;86;80;75"
-          # - build: "rocm5.5"
-          #   defines: '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS="gfx1100;gfx1102;gfx1030" -DSD_BUILD_SHARED_LIBS=ON'
-          - build: 'vulkan'
+            defines: "-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_ARCHITECTURES='61;70;75;80;86;89;90;100;120' -DCMAKE_CUDA_FLAGS='-Xcudafe \"--diag_suppress=177\" -Xcudafe \"--diag_suppress=550\"'"
+          - build: "vulkan"
             defines: "-DSD_VULKAN=ON -DSD_BUILD_SHARED_LIBS=ON"
     steps:
       - name: Clone
@@ -175,44 +341,45 @@ jobs:
         with:
           submodules: recursive
 
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.15.1
+
       - name: Install cuda-toolkit
         id: cuda-toolkit
         if: ${{ matrix.build == 'cuda12' }}
-        uses: Jimver/cuda-toolkit@v0.2.19
+        uses: Jimver/cuda-toolkit@v0.2.22
         with:
-          cuda: "12.6.2"
+          cuda: "12.8.1"
           method: "network"
           sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
 
-      - name: Install rocm-toolkit
-        id: rocm-toolkit
-        if: ${{ matrix.build == 'rocm5.5' }}
-        uses: Cyberhan123/rocm-toolkit@v0.1.0
-        with:
-          rocm: "5.5.0"
-
-      - name: Install Ninja
-        id: install-ninja
-        if: ${{ matrix.build == 'rocm5.5' }}
-        uses: urkle/action-get-ninja@v1
-        with:
-          version: 1.11.1
       - name: Install Vulkan SDK
         id: get_vulkan
-        if: ${{ matrix.build == 'vulkan' }} https://sdk.lunarg.com/sdk/download/1.4.328.1/windows/vulkansdk-windows-X64-1.4.328.1.exe
+        if: ${{ matrix.build == 'vulkan' }}
         run: |
           curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
           & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
           Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
           Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
 
+      - name: Activate MSVC environment
+        id: msvc_dev_cmd
+        uses: ilammy/msvc-dev-cmd@v1
+
       - name: Build
         id: cmake_build
         run: |
           mkdir build
           cd build
-          cmake .. ${{ matrix.defines }}
-          cmake --build . --config Release
+          cmake .. -DCMAKE_CXX_FLAGS='/bigobj' -G Ninja -DCMAKE_C_COMPILER=cl.exe -DCMAKE_CXX_COMPILER=cl.exe -DCMAKE_BUILD_TYPE=Release ${{ matrix.defines }}
+          cmake --build .
 
       - name: Check AVX512F support
         id: check_avx512f
@@ -230,7 +397,7 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: pr-mpt/actions-commit-hash@v2
+        uses: prompt/actions-commit-hash@v2
 
       - name: Pack artifacts
         id: pack_artifacts
@@ -277,6 +444,264 @@ jobs:
           path: |
             sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip
 
+  windows-latest-cmake-hip:
+    runs-on: windows-2022
+
+    env:
+      HIPSDK_INSTALLER_VERSION: "25.Q3"
+      GPU_TARGETS: "gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.15.1
+
+      - name: Cache ROCm Installation
+        id: cache-rocm
+        uses: actions/cache@v4
+        with:
+          path: C:\Program Files\AMD\ROCm
+          key: rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }}
+
+      - name: ccache
+        uses: ggml-org/ccache-action@v1.2.16
+        with:
+          key: windows-latest-cmake-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-x64
+          evict-old-files: 1d
+
+      - name: Install ROCm
+        if: steps.cache-rocm.outputs.cache-hit != 'true'
+        run: |
+          $ErrorActionPreference = "Stop"
+          write-host "Downloading AMD HIP SDK Installer"
+          Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-WinSvr2022-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe"
+          write-host "Installing AMD HIP SDK"
+          $proc = Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -PassThru
+          $completed = $proc.WaitForExit(600000)
+          if (-not $completed) {
+              Write-Error "ROCm installation timed out after 10 minutes. Killing the process"
+              $proc.Kill()
+              exit 1
+          }
+          if ($proc.ExitCode -ne 0) {
+              Write-Error "ROCm installation failed with exit code $($proc.ExitCode)"
+              exit 1
+          }
+          write-host "Completed AMD HIP SDK installation"
+
+      - name: Verify ROCm
+        run: |
+          # Find and test ROCm installation
+          $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 1
+          if (-not $clangPath) {
+            Write-Error "ROCm installation not found"
+            exit 1
+          }
+          & $clangPath.FullName --version
+          # Set HIP_PATH environment variable for later steps
+          echo "HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)" >> $env:GITHUB_ENV
+
+      - name: Build
+        run: |
+          mkdir build
+          cd build
+          $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}"
+          cmake .. `
+            -G "Unix Makefiles" `
+            -DSD_HIPBLAS=ON `
+            -DSD_BUILD_SHARED_LIBS=ON `
+            -DGGML_NATIVE=OFF `
+            -DCMAKE_C_COMPILER=clang `
+            -DCMAKE_CXX_COMPILER=clang++ `
+            -DCMAKE_BUILD_TYPE=Release `
+            -DGPU_TARGETS="${{ env.GPU_TARGETS }}"
+          cmake --build . --config Release --parallel ${env:NUMBER_OF_PROCESSORS}
+
+      - name: Get commit hash
+        id: commit
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        uses: prompt/actions-commit-hash@v2
+
+      - name: Pack artifacts
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        run: |
+          md "build\bin\rocblas\library\"
+          md "build\bin\hipblaslt\library"
+          cp "${env:HIP_PATH}\bin\hipblas.dll" "build\bin\"
+          cp "${env:HIP_PATH}\bin\hipblaslt.dll" "build\bin\"
+          cp "${env:HIP_PATH}\bin\rocblas.dll" "build\bin\"
+          cp "${env:HIP_PATH}\bin\rocblas\library\*" "build\bin\rocblas\library\"
+          cp "${env:HIP_PATH}\bin\hipblaslt\library\*" "build\bin\hipblaslt\library\"
+          7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-x64.zip .\build\bin\*
+
+      - name: Upload artifacts
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-x64.zip
+          path: |
+            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-x64.zip
+
+  ubuntu-latest-rocm:
+    runs-on: ubuntu-latest
+    container: rocm/dev-ubuntu-24.04:7.2
+
+    env:
+      ROCM_VERSION: "7.2"
+      UBUNTU_VERSION: "24.04"
+      GPU_TARGETS: "gfx1151;gfx1150;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201"
+
+    steps:
+      - run: apt-get update && apt-get install -y git
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.15.1
+
+      - name: Free disk space
+        run: |
+          # Remove preinstalled SDKs and caches not needed for this job
+          sudo rm -rf /usr/share/dotnet || true
+          sudo rm -rf /usr/local/lib/android || true
+          sudo rm -rf /opt/ghc || true
+          sudo rm -rf /usr/local/.ghcup || true
+          sudo rm -rf /opt/hostedtoolcache || true
+
+          # Remove old package lists and caches
+          sudo rm -rf /var/lib/apt/lists/* || true
+          sudo apt clean
+
+      - name: Dependencies
+        id: depends
+        run: |
+          sudo apt-get update
+          sudo apt install -y \
+            cmake \
+            hip-dev \
+            hipblas-dev \
+            ninja-build \
+            rocm-dev \
+            zip
+          # Clean apt caches to recover disk space
+          sudo apt clean
+          sudo rm -rf /var/lib/apt/lists/* || true
+
+      - name: Setup ROCm Environment
+        run: |
+          # Add ROCm to PATH for current session
+          echo "/opt/rocm/bin" >> $GITHUB_PATH
+
+          # Build regex pattern from ${{ env.GPU_TARGETS }} (match target as substring)
+          TARGET_REGEX="($(printf '%s' "${{ env.GPU_TARGETS }}" | sed 's/;/|/g'))"
+
+          # Remove library files for architectures we're not building for to save disk space
+          echo "Cleaning up unneeded architecture files..."
+          cd /opt/rocm/lib/rocblas/library
+          # Keep only our target architectures
+          for file in *; do
+            if printf '%s' "$file" | grep -q 'gfx'; then
+              if ! printf '%s' "$file" | grep -Eq "$TARGET_REGEX"; then
+                echo "Removing $file" &&
+                sudo rm -f "$file";
+              fi
+            fi
+          done
+
+          cd /opt/rocm/lib/hipblaslt/library
+          for file in *; do
+            if printf '%s' "$file" | grep -q 'gfx'; then
+              if ! printf '%s' "$file" | grep -Eq "$TARGET_REGEX"; then
+                echo "Removing $file" &&
+                sudo rm -f "$file";
+              fi
+            fi
+          done
+
+      - name: Build
+        id: cmake_build
+        run: |
+          mkdir build
+          cd build
+          cmake .. -G Ninja \
+            -DCMAKE_CXX_COMPILER=amdclang++ \
+            -DCMAKE_C_COMPILER=amdclang \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DSD_HIPBLAS=ON \
+            -DGPU_TARGETS="${{ env.GPU_TARGETS }}" \
+            -DAMDGPU_TARGETS="${{ env.GPU_TARGETS }}" \
+            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+            -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+            -DSD_BUILD_SHARED_LIBS=ON
+          cmake --build . --config Release
+
+      - name: Get commit hash
+        id: commit
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        uses: prompt/actions-commit-hash@v2
+
+      - name: Prepare artifacts
+        id: prepare_artifacts
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        run: |
+          # Copy licenses
+          cp ggml/LICENSE ./build/bin/ggml.txt
+          cp LICENSE ./build/bin/stable-diffusion.cpp.txt
+
+          # Move ROCm runtime libraries (to avoid double space consumption)
+          sudo mv /opt/rocm/lib/librocsparse.so* ./build/bin/
+          sudo mv /opt/rocm/lib/libhsa-runtime64.so* ./build/bin/
+          sudo mv /opt/rocm/lib/libamdhip64.so* ./build/bin/
+          sudo mv /opt/rocm/lib/libhipblas.so* ./build/bin/
+          sudo mv /opt/rocm/lib/libhipblaslt.so* ./build/bin/
+          sudo mv /opt/rocm/lib/librocblas.so* ./build/bin/
+          sudo mv /opt/rocm/lib/rocblas/ ./build/bin/
+          sudo mv /opt/rocm/lib/hipblaslt/ ./build/bin/
+
+      - name: Fetch system info
+        id: system-info
+        run: |
+          echo "CPU_ARCH=`uname -m`" >> "$GITHUB_OUTPUT"
+          echo "OS_NAME=`lsb_release -s -i`" >> "$GITHUB_OUTPUT"
+          echo "OS_VERSION=`lsb_release -s -r`" >> "$GITHUB_OUTPUT"
+          echo "OS_TYPE=`uname -s`" >> "$GITHUB_OUTPUT"
+
+      - name: Pack artifacts
+        id: pack_artifacts
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        run: |
+          cp ggml/LICENSE ./build/bin/ggml.txt
+          cp LICENSE ./build/bin/stable-diffusion.cpp.txt
+          zip -y -r sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm.zip ./build/bin
+
+      - name: Upload artifacts
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm.zip
+          path: |
+            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm.zip
+
   release:
     if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
 
@@ -284,8 +709,12 @@ jobs:
 
     needs:
       - ubuntu-latest-cmake
+      - ubuntu-latest-cmake-vulkan
+      - ubuntu-latest-rocm
+      - build-and-push-docker-images
       - macOS-latest-cmake
       - windows-latest-cmake
+      - windows-latest-cmake-hip
 
     steps:
       - name: Clone
@@ -308,7 +737,7 @@ jobs:
 
       - name: Get commit hash
         id: commit
-        uses: pr-mpt/actions-commit-hash@v2
+        uses: prompt/actions-commit-hash@v2
 
       - name: Create release
         id: create_release

.gitignore

@@ -12,3 +12,4 @@ test/
 output*.png
 models*
 *.log
+preview.png

.gitmodules

@@ -1,3 +1,12 @@
 [submodule "ggml"]
     path = ggml
 	url = https://github.com/ggml-org/ggml.git
+[submodule "examples/server/frontend"]
+	path = examples/server/frontend
+	url = https://github.com/leejet/sdcpp-webui.git
+[submodule "thirdparty/libwebp"]
+	path = thirdparty/libwebp
+	url = https://github.com/webmproject/libwebp.git
+[submodule "thirdparty/libwebm"]
+	path = thirdparty/libwebm
+	url = https://github.com/webmproject/libwebm.git

CMakeLists.txt

@@ -8,6 +8,15 @@ if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
     set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
 endif()
 
+if (MSVC)
+    add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
+    add_compile_definitions(_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING)
+    add_compile_options(
+        $<$<COMPILE_LANGUAGE:C>:/MP>
+        $<$<COMPILE_LANGUAGE:CXX>:/MP>
+    )
+endif()
+
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
@@ -17,6 +26,26 @@ else()
     set(SD_STANDALONE OFF)
 endif()
 
+set(SD_SUBMODULE_WEBP FALSE)
+if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/libwebp/CMakeLists.txt")
+    set(SD_SUBMODULE_WEBP TRUE)
+endif()
+if(SD_SUBMODULE_WEBP)
+    set(SD_WEBP_DEFAULT ON)
+else()
+    set(SD_WEBP_DEFAULT ${SD_USE_SYSTEM_WEBP})
+endif()
+
+set(SD_SUBMODULE_WEBM FALSE)
+if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/libwebm/CMakeLists.txt")
+    set(SD_SUBMODULE_WEBM TRUE)
+endif()
+if(SD_SUBMODULE_WEBM)
+    set(SD_WEBM_DEFAULT ON)
+else()
+    set(SD_WEBM_DEFAULT ${SD_USE_SYSTEM_WEBM})
+endif()
+
 #
 # Option list
 #
@@ -24,6 +53,10 @@ endif()
 # general
 #option(SD_BUILD_TESTS                "sd: build tests"    ${SD_STANDALONE})
 option(SD_BUILD_EXAMPLES             "sd: build examples" ${SD_STANDALONE})
+option(SD_WEBP                       "sd: enable WebP image I/O support" ${SD_WEBP_DEFAULT})
+option(SD_USE_SYSTEM_WEBP            "sd: link against system libwebp" OFF)
+option(SD_WEBM                       "sd: enable WebM video output support" ${SD_WEBM_DEFAULT})
+option(SD_USE_SYSTEM_WEBM            "sd: link against system libwebm" OFF)
 option(SD_CUDA                       "sd: cuda backend" OFF)
 option(SD_HIPBLAS                    "sd: rocm backend" OFF)
 option(SD_METAL                      "sd: metal backend" OFF)
@@ -31,7 +64,6 @@ option(SD_VULKAN                     "sd: vulkan backend" OFF)
 option(SD_OPENCL                     "sd: opencl backend" OFF)
 option(SD_SYCL                       "sd: sycl backend" OFF)
 option(SD_MUSA                       "sd: musa backend" OFF)
-option(SD_FAST_SOFTMAX               "sd: x1.5 faster softmax, indeterministic (sometimes, same seed don't generate same image), cuda only" OFF)
 option(SD_BUILD_SHARED_LIBS          "sd: build shared libs" OFF)
 option(SD_BUILD_SHARED_GGML_LIB      "sd: build ggml as a separate shared lib" OFF)
 option(SD_USE_SYSTEM_GGML            "sd: use system-installed GGML library" OFF)
@@ -65,26 +97,107 @@ if (SD_HIPBLAS)
     message("-- Use HIPBLAS as backend stable-diffusion")
     set(GGML_HIP ON)
     add_definitions(-DSD_USE_CUDA)
-    if(SD_FAST_SOFTMAX)
-        set(GGML_CUDA_FAST_SOFTMAX ON)
-    endif()
 endif ()
 
 if(SD_MUSA)
     message("-- Use MUSA as backend stable-diffusion")
     set(GGML_MUSA ON)
     add_definitions(-DSD_USE_CUDA)
-    if(SD_FAST_SOFTMAX)
-        set(GGML_CUDA_FAST_SOFTMAX ON)
+endif()
+
+if(SD_WEBP)
+    if(NOT SD_SUBMODULE_WEBP AND NOT SD_USE_SYSTEM_WEBP)
+        message(FATAL_ERROR "WebP support enabled but no source found.
+          Either initialize the submodule:\n  git submodule update --init thirdparty/libwebp\n\n"
+          "Or link against system library:\n  cmake (...) -DSD_USE_SYSTEM_WEBP=ON")
+    endif()
+    if(SD_USE_SYSTEM_WEBP)
+        find_package(WebP REQUIRED)
+        add_library(webp ALIAS WebP::webp)
+        # libwebp CMake target naming is not consistent across versions/distros.
+        # Some export WebP::libwebpmux, others export WebP::webpmux.
+        if(TARGET WebP::libwebpmux)
+            add_library(libwebpmux ALIAS WebP::libwebpmux)
+        elseif(TARGET WebP::webpmux)
+            add_library(libwebpmux ALIAS WebP::webpmux)
+        else()
+            message(FATAL_ERROR
+                "Could not find a compatible webpmux target in system WebP package. "
+                "Expected WebP::libwebpmux or WebP::webpmux."
+            )
+        endif()
+    endif()
+endif()
+
+if(SD_WEBM)
+    if(NOT SD_WEBP)
+        message(FATAL_ERROR "SD_WEBM requires SD_WEBP because WebM output reuses libwebp VP8 encoding.")
+    endif()
+    if(NOT SD_SUBMODULE_WEBM AND NOT SD_USE_SYSTEM_WEBM)
+        message(FATAL_ERROR "WebM support enabled but no source found.
+          Either initialize the submodule:\n  git submodule update --init thirdparty/libwebm\n\n"
+          "Or link against system library:\n  cmake (...) -DSD_USE_SYSTEM_WEBM=ON")
+    endif()
+    if(SD_USE_SYSTEM_WEBM)
+        find_path(WEBM_INCLUDE_DIR
+            NAMES mkvmuxer/mkvmuxer.h mkvparser/mkvparser.h common/webmids.h
+            PATH_SUFFIXES webm
+            REQUIRED)
+        find_library(WEBM_LIBRARY
+            NAMES webm libwebm
+            REQUIRED)
+
+        add_library(webm UNKNOWN IMPORTED)
+        set_target_properties(webm PROPERTIES
+            IMPORTED_LOCATION "${WEBM_LIBRARY}"
+            INTERFACE_INCLUDE_DIRECTORIES "${WEBM_INCLUDE_DIR}")
     endif()
 endif()
 
 set(SD_LIB stable-diffusion)
 
-file(GLOB SD_LIB_SOURCES
-    "*.h"
-    "*.cpp"
-    "*.hpp"
+file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
+    "src/*.h"
+    "src/*.cpp"
+    "src/*.hpp"
+    "src/model_io/*.h"
+    "src/model_io/*.cpp"
+    "src/tokenizers/*.h"
+    "src/tokenizers/*.cpp"
+    "src/tokenizers/vocab/*.h"
+    "src/tokenizers/vocab/*.cpp"
+)
+
+find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH)
+if(GIT_EXE)
+    execute_process(COMMAND ${GIT_EXE} describe --tags --abbrev=7 --dirty=+
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+        OUTPUT_VARIABLE SDCPP_BUILD_VERSION
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        ERROR_QUIET
+    )
+    execute_process(COMMAND ${GIT_EXE} rev-parse --short HEAD
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+        OUTPUT_VARIABLE SDCPP_BUILD_COMMIT
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        ERROR_QUIET
+    )
+endif()
+
+if(NOT SDCPP_BUILD_VERSION)
+    set(SDCPP_BUILD_VERSION unknown)
+endif()
+message(STATUS "stable-diffusion.cpp version ${SDCPP_BUILD_VERSION}")
+
+if(NOT SDCPP_BUILD_COMMIT)
+    set(SDCPP_BUILD_COMMIT unknown)
+endif()
+message(STATUS "stable-diffusion.cpp commit ${SDCPP_BUILD_COMMIT}")
+
+set_property(
+  SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src/version.cpp
+  APPEND PROPERTY COMPILE_DEFINITIONS
+  SDCPP_BUILD_COMMIT=${SDCPP_BUILD_COMMIT} SDCPP_BUILD_VERSION=${SDCPP_BUILD_VERSION}
 )
 
 if(SD_BUILD_SHARED_LIBS)
@@ -145,6 +258,7 @@ endif()
 add_subdirectory(thirdparty)
 
 target_link_libraries(${SD_LIB} PUBLIC ggml zip)
+target_include_directories(${SD_LIB} PUBLIC . src include)
 target_include_directories(${SD_LIB} PUBLIC . thirdparty)
 target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17)
 
@@ -153,7 +267,7 @@ if (SD_BUILD_EXAMPLES)
     add_subdirectory(examples)
 endif()
 
-set(SD_PUBLIC_HEADERS stable-diffusion.h)
+set(SD_PUBLIC_HEADERS include/stable-diffusion.h)
 set_target_properties(${SD_LIB} PROPERTIES PUBLIC_HEADER "${SD_PUBLIC_HEADERS}")
 
 install(TARGETS ${SD_LIB} LIBRARY PUBLIC_HEADER)

Dockerfile

@@ -1,4 +1,4 @@
-ARG UBUNTU_VERSION=22.04
+ARG UBUNTU_VERSION=24.04
 
 FROM ubuntu:$UBUNTU_VERSION AS build
 
@@ -17,6 +17,7 @@ RUN apt-get update && \
     apt-get install --yes --no-install-recommends libgomp1 && \
     apt-get clean
 
-COPY --from=build /sd.cpp/build/bin/sd /sd
+COPY --from=build /sd.cpp/build/bin/sd-cli /sd-cli
+COPY --from=build /sd.cpp/build/bin/sd-server /sd-server
 
-ENTRYPOINT [ "/sd" ]
+ENTRYPOINT [ "/sd-cli" ]

Dockerfile.cuda

@@ -0,0 +1,25 @@
+ARG CUDA_VERSION=12.6.3
+ARG UBUNTU_VERSION=24.04
+
+FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu${UBUNTU_VERSION} AS build
+
+RUN apt-get update && apt-get install -y --no-install-recommends build-essential git ccache cmake
+
+WORKDIR /sd.cpp
+
+COPY . .
+
+ARG CUDACXX=/usr/local/cuda/bin/nvcc
+RUN cmake . -B ./build -DSD_CUDA=ON
+RUN cmake --build ./build --config Release -j$(nproc)
+
+FROM nvidia/cuda:${CUDA_VERSION}-cudnn-runtime-ubuntu${UBUNTU_VERSION} AS runtime
+
+RUN apt-get update && \
+    apt-get install --yes --no-install-recommends libgomp1 && \
+    apt-get clean
+
+COPY --from=build /sd.cpp/build/bin/sd-cli /sd-cli
+COPY --from=build /sd.cpp/build/bin/sd-server /sd-server
+
+ENTRYPOINT [ "/sd-cli" ]

Dockerfile.musa

@@ -18,6 +18,7 @@ RUN mkdir build && cd build && \
 
 FROM mthreads/musa:${MUSA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}-amd64 as runtime
 
-COPY --from=build /sd.cpp/build/bin/sd /sd
+COPY --from=build /sd.cpp/build/bin/sd-cli /sd-cli
+COPY --from=build /sd.cpp/build/bin/sd-server /sd-server
 
-ENTRYPOINT [ "/sd" ]
+ENTRYPOINT [ "/sd-cli" ]

Dockerfile.sycl

@@ -14,6 +14,7 @@ RUN mkdir build && cd build && \
 
 FROM intel/oneapi-basekit:${SYCL_VERSION}-devel-ubuntu24.04 AS runtime
 
-COPY --from=build /sd.cpp/build/bin/sd /sd
+COPY --from=build /sd.cpp/build/bin/sd-cli /sd-cli
+COPY --from=build /sd.cpp/build/bin/sd-server /sd-server
 
-ENTRYPOINT [ "/sd" ]
+ENTRYPOINT [ "/sd-cli" ]

Dockerfile.vulkan

@@ -0,0 +1,23 @@
+ARG UBUNTU_VERSION=24.04
+
+FROM ubuntu:$UBUNTU_VERSION AS build
+
+RUN apt-get update && apt-get install -y --no-install-recommends build-essential git cmake libvulkan-dev glslc
+
+WORKDIR /sd.cpp
+
+COPY . .
+
+RUN cmake . -B ./build -DSD_VULKAN=ON
+RUN cmake --build ./build --config Release --parallel
+
+FROM ubuntu:$UBUNTU_VERSION AS runtime
+
+RUN apt-get update && \
+    apt-get install --yes --no-install-recommends libgomp1 libvulkan1 mesa-vulkan-drivers && \
+    apt-get clean
+
+COPY --from=build /sd.cpp/build/bin/sd-cli /sd-cli
+COPY --from=build /sd.cpp/build/bin/sd-server /sd-server
+
+ENTRYPOINT [ "/sd-cli" ]

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="./assets/cat_with_sd_cpp_42.png" width="360x">
+  <img src="./assets/logo.png" width="360x">
 </p>
 
 # stable-diffusion.cpp
@@ -15,6 +15,18 @@ API and command-line option may change frequently.***
 
 ## 🔥Important News
 
+* **2026/04/11** 🚀 stable-diffusion.cpp now uses a brand-new embedded web UI.  
+  👉 Details: [PR #1408](https://github.com/leejet/stable-diffusion.cpp/pull/1408)
+
+* **2026/01/18** 🚀 stable-diffusion.cpp now supports **FLUX.2-klein**  
+  👉 Details: [PR #1193](https://github.com/leejet/stable-diffusion.cpp/pull/1193)
+
+* **2025/12/01** 🚀 stable-diffusion.cpp now supports **Z-Image**  
+  👉 Details: [PR #1020](https://github.com/leejet/stable-diffusion.cpp/pull/1020)
+
+* **2025/11/30** 🚀 stable-diffusion.cpp now supports **FLUX.2-dev**  
+  👉 Details: [PR #1016](https://github.com/leejet/stable-diffusion.cpp/pull/1016)
+
 * **2025/10/13** 🚀 stable-diffusion.cpp now supports **Qwen-Image-Edit / Qwen-Image-Edit 2509**  
   👉 Details: [PR #877](https://github.com/leejet/stable-diffusion.cpp/pull/877)
 
@@ -29,19 +41,26 @@ API and command-line option may change frequently.***
 
 ## Features
 
-- Plain C/C++ implementation based on [ggml](https://github.com/ggerganov/ggml), working in the same way as [llama.cpp](https://github.com/ggerganov/llama.cpp)
+- Plain C/C++ implementation based on [ggml](https://github.com/ggml-org/ggml), working in the same way as [llama.cpp](https://github.com/ggml-org/llama.cpp)
 - Super lightweight and without external dependencies
 - Supported models
   - Image Models
     - SD1.x, SD2.x, [SD-Turbo](https://huggingface.co/stabilityai/sd-turbo)
     - SDXL, [SDXL-Turbo](https://huggingface.co/stabilityai/sdxl-turbo)
+    - [Some SD1.x and SDXL distilled models](./docs/distilled_sd.md)
     - [SD3/SD3.5](./docs/sd3.md)
-    - [Flux-dev/Flux-schnell](./docs/flux.md)
+    - [FLUX.1-dev/FLUX.1-schnell](./docs/flux.md)
+    - [FLUX.2-dev/FLUX.2-klein](./docs/flux2.md)
     - [Chroma](./docs/chroma.md)
+    - [Chroma1-Radiance](./docs/chroma_radiance.md)
     - [Qwen Image](./docs/qwen_image.md)
+    - [Z-Image](./docs/z_image.md)
+    - [Ovis-Image](./docs/ovis_image.md)
+    - [Anima](./docs/anima.md)
+    - [ERNIE-Image](./docs/ernie_image.md)
   - Image Edit Models
     - [FLUX.1-Kontext-dev](./docs/kontext.md)
-    - [Qwen Image Edit/Qwen Image Edit 2509](./docs/qwen_image_edit.md)
+    - [Qwen Image Edit series](./docs/qwen_image_edit.md)
   - Video Models
     - [Wan2.1/Wan2.2](./docs/wan.md)
   - [PhotoMaker](https://github.com/TencentARC/PhotoMaker) support.
@@ -58,9 +77,10 @@ API and command-line option may change frequently.***
   - OpenCL
   - SYCL
 - Supported weight formats
-  - Pytorch checkpoint (`.ckpt` or `.pth`)
-  - Safetensors (`./safetensors`)
+  - Pytorch checkpoint (`.ckpt` or `.pth` or `.pt`)
+  - Safetensors (`.safetensors`)
   - GGUF (`.gguf`)
+- Convert mode supports converting model weights to `.gguf` or `.safetensors`
 - Supported platforms
     - Linux
     - Mac OS
@@ -78,8 +98,11 @@ API and command-line option may change frequently.***
     - `DPM++ 2M`
     - [`DPM++ 2M v2`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/8457)
     - `DPM++ 2S a`
+    - `ER-SDE`
     - [`LCM`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/13952)
-- Cross-platform reproducibility (`--rng cuda`, consistent with the `stable-diffusion-webui GPU RNG`)
+- Cross-platform reproducibility
+    - `--rng cuda`, default, consistent with the `stable-diffusion-webui GPU RNG`
+    - `--rng cpu`, consistent with the `comfyui RNG`
 - Embedds generation parameters into png output as webui-compatible text string
 
 ## Quick Start
@@ -92,7 +115,7 @@ API and command-line option may change frequently.***
 ### Download model weights
 
 - download weights(.ckpt or .safetensors or .gguf). For example
-    - Stable Diffusion v1.5 from https://huggingface.co/runwayml/stable-diffusion-v1-5
+    - Stable Diffusion v1.5 from https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5 
 
     ```sh
     curl -L -O https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors
@@ -101,7 +124,7 @@ API and command-line option may change frequently.***
 ### Generate an image with just one command
 
 ```sh
-./bin/sd -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat"
+./bin/sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat"
 ```
 
 ***For detailed command-line arguments, check out [cli doc](./examples/cli/README.md).***
@@ -114,12 +137,17 @@ If you want to improve performance or reduce VRAM/RAM usage, please refer to [pe
 
 - [SD1.x/SD2.x/SDXL](./docs/sd.md)
 - [SD3/SD3.5](./docs/sd3.md)
-- [Flux-dev/Flux-schnell](./docs/flux.md)
+- [FLUX.1-dev/FLUX.1-schnell](./docs/flux.md)
+- [FLUX.2-dev/FLUX.2-klein](./docs/flux2.md)
 - [FLUX.1-Kontext-dev](./docs/kontext.md)
 - [Chroma](./docs/chroma.md)
 - [🔥Qwen Image](./docs/qwen_image.md)
-- [🔥Qwen Image Edit/Qwen Image Edit 2509](./docs/qwen_image_edit.md)
+- [🔥Qwen Image Edit series](./docs/qwen_image_edit.md)
 - [🔥Wan2.1/Wan2.2](./docs/wan.md)
+- [🔥Z-Image](./docs/z_image.md)
+- [Ovis-Image](./docs/ovis_image.md)
+- [Anima](./docs/anima.md)
+- [ERNIE-Image](./docs/ernie_image.md)
 - [LoRA](./docs/lora.md)
 - [LCM/LCM-LoRA](./docs/lcm.md)
 - [Using PhotoMaker to personalize image generation](./docs/photo_maker.md)
@@ -127,6 +155,7 @@ If you want to improve performance or reduce VRAM/RAM usage, please refer to [pe
 - [Using TAESD to faster decoding](./docs/taesd.md)
 - [Docker](./docs/docker.md)
 - [Quantization and GGUF](./docs/quantization_and_gguf.md)
+- [Inference acceleration via caching](./docs/caching.md)
 
 ## Bindings
 
@@ -150,6 +179,7 @@ These projects use `stable-diffusion.cpp` as a backend for their image generatio
 - [sd.cpp-webui](https://github.com/daniandtheweb/sd.cpp-webui)
 - [LocalAI](https://github.com/mudler/LocalAI)
 - [Neural-Pixel](https://github.com/Luiz-Alcantara/Neural-Pixel)
+- [KoboldCpp](https://github.com/LostRuins/koboldcpp)
 
 ## Contributors
 
@@ -163,7 +193,7 @@ Thank you to all the people who have already contributed to stable-diffusion.cpp
 
 ## References
 
-- [ggml](https://github.com/ggerganov/ggml)
+- [ggml](https://github.com/ggml-org/ggml)
 - [diffusers](https://github.com/huggingface/diffusers)
 - [stable-diffusion](https://github.com/CompVis/stable-diffusion)
 - [sd3-ref](https://github.com/Stability-AI/sd3-ref)

clip.hpp

@@ -1,969 +0,0 @@
-#ifndef __CLIP_HPP__
-#define __CLIP_HPP__
-
-#include "ggml_extend.hpp"
-#include "model.h"
-
-/*================================================== CLIPTokenizer ===================================================*/
-
-__STATIC_INLINE__ std::pair<std::unordered_map<std::string, float>, std::string> extract_and_remove_lora(std::string text) {
-    std::regex re("<lora:([^:]+):([^>]+)>");
-    std::smatch matches;
-    std::unordered_map<std::string, float> filename2multiplier;
-
-    while (std::regex_search(text, matches, re)) {
-        std::string filename = matches[1].str();
-        float multiplier     = std::stof(matches[2].str());
-
-        text = std::regex_replace(text, re, "", std::regex_constants::format_first_only);
-
-        if (multiplier == 0.f) {
-            continue;
-        }
-
-        if (filename2multiplier.find(filename) == filename2multiplier.end()) {
-            filename2multiplier[filename] = multiplier;
-        } else {
-            filename2multiplier[filename] += multiplier;
-        }
-    }
-
-    return std::make_pair(filename2multiplier, text);
-}
-
-__STATIC_INLINE__ std::vector<std::pair<int, std::u32string>> bytes_to_unicode() {
-    std::vector<std::pair<int, std::u32string>> byte_unicode_pairs;
-    std::set<int> byte_set;
-    for (int b = static_cast<int>('!'); b <= static_cast<int>('~'); ++b) {
-        byte_set.insert(b);
-        byte_unicode_pairs.push_back(std::pair<int, std::u32string>(b, unicode_value_to_utf32(b)));
-    }
-    for (int b = 161; b <= 172; ++b) {
-        byte_set.insert(b);
-        byte_unicode_pairs.push_back(std::pair<int, std::u32string>(b, unicode_value_to_utf32(b)));
-    }
-    for (int b = 174; b <= 255; ++b) {
-        byte_set.insert(b);
-        byte_unicode_pairs.push_back(std::pair<int, std::u32string>(b, unicode_value_to_utf32(b)));
-    }
-    int n = 0;
-    for (int b = 0; b < 256; ++b) {
-        if (byte_set.find(b) == byte_set.end()) {
-            byte_unicode_pairs.push_back(std::pair<int, std::u32string>(b, unicode_value_to_utf32(n + 256)));
-            ++n;
-        }
-    }
-    // LOG_DEBUG("byte_unicode_pairs %d", byte_unicode_pairs.size());
-    return byte_unicode_pairs;
-}
-
-// Ref: https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py
-
-typedef std::function<bool(std::string&, std::vector<int32_t>&)> on_new_token_cb_t;
-
-class CLIPTokenizer {
-private:
-    std::map<int, std::u32string> byte_encoder;
-    std::map<std::u32string, int> byte_decoder;
-    std::map<std::u32string, int> encoder;
-    std::map<int, std::u32string> decoder;
-    std::map<std::pair<std::u32string, std::u32string>, int> bpe_ranks;
-    std::regex pat;
-    int encoder_len;
-    int bpe_len;
-
-public:
-    const std::string UNK_TOKEN = "<|endoftext|>";
-    const std::string BOS_TOKEN = "<|startoftext|>";
-    const std::string EOS_TOKEN = "<|endoftext|>";
-    const std::string PAD_TOKEN = "<|endoftext|>";
-
-    const int UNK_TOKEN_ID = 49407;
-    const int BOS_TOKEN_ID = 49406;
-    const int EOS_TOKEN_ID = 49407;
-    const int PAD_TOKEN_ID = 49407;
-
-private:
-    static std::string strip(const std::string& str) {
-        std::string::size_type start = str.find_first_not_of(" \t\n\r\v\f");
-        std::string::size_type end   = str.find_last_not_of(" \t\n\r\v\f");
-
-        if (start == std::string::npos) {
-            // String contains only whitespace characters
-            return "";
-        }
-
-        return str.substr(start, end - start + 1);
-    }
-
-    static std::string whitespace_clean(std::string text) {
-        text = std::regex_replace(text, std::regex(R"(\s+)"), " ");
-        text = strip(text);
-        return text;
-    }
-
-    static std::set<std::pair<std::u32string, std::u32string>> get_pairs(const std::vector<std::u32string>& subwords) {
-        std::set<std::pair<std::u32string, std::u32string>> pairs;
-        if (subwords.size() == 0) {
-            return pairs;
-        }
-        std::u32string prev_subword = subwords[0];
-        for (int i = 1; i < subwords.size(); i++) {
-            std::u32string subword = subwords[i];
-            std::pair<std::u32string, std::u32string> pair(prev_subword, subword);
-            pairs.insert(pair);
-            prev_subword = subword;
-        }
-        return pairs;
-    }
-
-public:
-    CLIPTokenizer(int pad_token_id = 49407, const std::string& merges_utf8_str = "")
-        : PAD_TOKEN_ID(pad_token_id) {
-        if (merges_utf8_str.size() > 0) {
-            load_from_merges(merges_utf8_str);
-        } else {
-            load_from_merges(ModelLoader::load_merges());
-        }
-    }
-
-    void load_from_merges(const std::string& merges_utf8_str) {
-        auto byte_unicode_pairs = bytes_to_unicode();
-        // printf("byte_unicode_pairs have %lu pairs \n", byte_unicode_pairs.size());
-        byte_encoder = std::map<int, std::u32string>(byte_unicode_pairs.begin(), byte_unicode_pairs.end());
-        for (auto& pair : byte_unicode_pairs) {
-            byte_decoder[pair.second] = pair.first;
-        }
-        // for (auto & pair: byte_unicode_pairs) {
-        //     std::cout << pair.first << ": " << pair.second << std::endl;
-        // }
-        std::vector<std::u32string> merges;
-        size_t start = 0;
-        size_t pos;
-        std::u32string merges_utf32_str = utf8_to_utf32(merges_utf8_str);
-        while ((pos = merges_utf32_str.find('\n', start)) != std::string::npos) {
-            merges.push_back(merges_utf32_str.substr(start, pos - start));
-            start = pos + 1;
-        }
-        // LOG_DEBUG("merges size %llu", merges.size());
-        GGML_ASSERT(merges.size() == 48895);
-        merges = std::vector<std::u32string>(merges.begin() + 1, merges.end());
-        std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
-        for (const auto& merge : merges) {
-            size_t space_pos = merge.find(' ');
-            merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
-            // LOG_DEBUG("%s", utf32_to_utf8(merge.substr(space_pos + 1)).c_str());
-            // printf("%s :: %s | %s \n", utf32_to_utf8(merge).c_str(), utf32_to_utf8(merge.substr(0, space_pos)).c_str(),
-            //                     utf32_to_utf8(merge.substr(space_pos + 1)).c_str());
-        }
-        std::vector<std::u32string> vocab;
-        for (const auto& pair : byte_unicode_pairs) {
-            vocab.push_back(pair.second);
-        }
-        for (const auto& pair : byte_unicode_pairs) {
-            vocab.push_back(pair.second + utf8_to_utf32("</w>"));
-        }
-        for (const auto& merge : merge_pairs) {
-            vocab.push_back(merge.first + merge.second);
-        }
-        vocab.push_back(utf8_to_utf32("<|startoftext|>"));
-        vocab.push_back(utf8_to_utf32("<|endoftext|>"));
-        LOG_DEBUG("vocab size: %llu", vocab.size());
-        int i = 0;
-        for (const auto& token : vocab) {
-            encoder[token] = i;
-            decoder[i]     = token;
-            i++;
-        }
-        encoder_len = i;
-
-        auto it = encoder.find(utf8_to_utf32("img</w>"));
-        if (it != encoder.end()) {
-            LOG_DEBUG("trigger word img already in vocab");
-        } else {
-            LOG_DEBUG("trigger word img not in vocab yet");
-        }
-
-        int rank = 0;
-        for (const auto& merge : merge_pairs) {
-            bpe_ranks[merge] = rank++;
-        }
-        bpe_len = rank;
-    };
-
-    void add_token(const std::string& text) {
-        std::u32string token = utf8_to_utf32(text);
-        auto it              = encoder.find(token);
-        if (it != encoder.end()) {
-            encoder[token]       = encoder_len;
-            decoder[encoder_len] = token;
-            encoder_len++;
-        }
-    }
-
-    std::u32string bpe(const std::u32string& token) {
-        std::vector<std::u32string> word;
-
-        for (int i = 0; i < token.size() - 1; i++) {
-            word.emplace_back(1, token[i]);
-        }
-        word.push_back(token.substr(token.size() - 1) + utf8_to_utf32("</w>"));
-
-        std::set<std::pair<std::u32string, std::u32string>> pairs = get_pairs(word);
-
-        if (pairs.empty()) {
-            return token + utf8_to_utf32("</w>");
-        }
-
-        while (true) {
-            auto min_pair_iter = std::min_element(pairs.begin(),
-                                                  pairs.end(),
-                                                  [&](const std::pair<std::u32string, std::u32string>& a,
-                                                      const std::pair<std::u32string, std::u32string>& b) {
-                                                      if (bpe_ranks.find(a) == bpe_ranks.end()) {
-                                                          return false;
-                                                      } else if (bpe_ranks.find(b) == bpe_ranks.end()) {
-                                                          return true;
-                                                      }
-                                                      return bpe_ranks.at(a) < bpe_ranks.at(b);
-                                                  });
-
-            const std::pair<std::u32string, std::u32string>& bigram = *min_pair_iter;
-
-            if (bpe_ranks.find(bigram) == bpe_ranks.end()) {
-                break;
-            }
-
-            std::u32string first  = bigram.first;
-            std::u32string second = bigram.second;
-            std::vector<std::u32string> new_word;
-            int32_t i = 0;
-
-            while (i < word.size()) {
-                auto it = std::find(word.begin() + i, word.end(), first);
-                if (it == word.end()) {
-                    new_word.insert(new_word.end(), word.begin() + i, word.end());
-                    break;
-                }
-                new_word.insert(new_word.end(), word.begin() + i, it);
-                i = static_cast<int32_t>(std::distance(word.begin(), it));
-
-                if (word[i] == first && i < static_cast<int32_t>(word.size()) - 1 && word[i + 1] == second) {
-                    new_word.push_back(first + second);
-                    i += 2;
-                } else {
-                    new_word.push_back(word[i]);
-                    i += 1;
-                }
-            }
-
-            word = new_word;
-
-            if (word.size() == 1) {
-                break;
-            }
-            pairs = get_pairs(word);
-        }
-
-        std::u32string result;
-        for (int i = 0; i < word.size(); i++) {
-            result += word[i];
-            if (i != word.size() - 1) {
-                result += utf8_to_utf32(" ");
-            }
-        }
-
-        return result;
-    }
-
-    std::vector<int> tokenize(std::string text,
-                              on_new_token_cb_t on_new_token_cb,
-                              size_t max_length = 0,
-                              bool padding      = false) {
-        std::vector<int32_t> tokens = encode(text, on_new_token_cb);
-
-        tokens.insert(tokens.begin(), BOS_TOKEN_ID);
-        if (max_length > 0) {
-            if (tokens.size() > max_length - 1) {
-                tokens.resize(max_length - 1);
-                tokens.push_back(EOS_TOKEN_ID);
-            } else {
-                tokens.push_back(EOS_TOKEN_ID);
-                if (padding) {
-                    tokens.insert(tokens.end(), max_length - tokens.size(), PAD_TOKEN_ID);
-                }
-            }
-        }
-
-        return tokens;
-    }
-
-    void pad_tokens(std::vector<int>& tokens,
-                    std::vector<float>& weights,
-                    size_t max_length = 0,
-                    bool padding      = false) {
-        if (max_length > 0 && padding) {
-            size_t n = std::ceil(tokens.size() * 1.0 / (max_length - 2));
-            if (n == 0) {
-                n = 1;
-            }
-            size_t length = max_length * n;
-            LOG_DEBUG("token length: %llu", length);
-            std::vector<int> new_tokens;
-            std::vector<float> new_weights;
-            new_tokens.push_back(BOS_TOKEN_ID);
-            new_weights.push_back(1.0);
-            int token_idx = 0;
-            for (int i = 1; i < length; i++) {
-                if (token_idx >= tokens.size()) {
-                    break;
-                }
-                if (i % max_length == 0) {
-                    new_tokens.push_back(BOS_TOKEN_ID);
-                    new_weights.push_back(1.0);
-                } else if (i % max_length == max_length - 1) {
-                    new_tokens.push_back(EOS_TOKEN_ID);
-                    new_weights.push_back(1.0);
-                } else {
-                    new_tokens.push_back(tokens[token_idx]);
-                    new_weights.push_back(weights[token_idx]);
-                    token_idx++;
-                }
-            }
-
-            new_tokens.push_back(EOS_TOKEN_ID);
-            new_weights.push_back(1.0);
-            tokens  = new_tokens;
-            weights = new_weights;
-
-            if (padding) {
-                tokens.insert(tokens.end(), length - tokens.size(), PAD_TOKEN_ID);
-                weights.insert(weights.end(), length - weights.size(), 1.0);
-            }
-        }
-    }
-
-    std::string clean_up_tokenization(std::string& text) {
-        std::regex pattern(R"( ,)");
-        // Replace " ," with ","
-        std::string result = std::regex_replace(text, pattern, ",");
-        return result;
-    }
-
-    std::string decode(const std::vector<int>& tokens) {
-        std::string text = "";
-        for (int t : tokens) {
-            if (t == 49406 || t == 49407)
-                continue;
-            std::u32string ts = decoder[t];
-            // printf("%d, %s \n", t,  utf32_to_utf8(ts).c_str());
-            std::string s = utf32_to_utf8(ts);
-            if (s.length() >= 4) {
-                if (ends_with(s, "</w>")) {
-                    text += s.replace(s.length() - 4, s.length() - 1, "") + " ";
-                } else {
-                    text += s;
-                }
-            } else {
-                text += " " + s;
-            }
-        }
-        // std::vector<unsigned char> bytes;
-        // for (auto c : text){
-        //     bytes.push_back(byte_decoder[c]);
-        // }
-
-        // std::string s((char *)bytes.data());
-        // std::string s = "";
-        text = clean_up_tokenization(text);
-        return trim(text);
-    }
-
-    std::vector<int> encode(std::string text, on_new_token_cb_t on_new_token_cb) {
-        std::string original_text = text;
-        std::vector<int32_t> bpe_tokens;
-        text = whitespace_clean(text);
-        std::transform(text.begin(), text.end(), text.begin(), [](unsigned char c) { return std::tolower(c); });
-
-        std::regex pat(R"(<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[[:alpha:]]+|[[:digit:]]|[^[:space:][:alpha:][:digit:]]+)",
-                       std::regex::icase);
-
-        std::smatch matches;
-        std::string str = text;
-        std::vector<std::string> token_strs;
-        while (std::regex_search(str, matches, pat)) {
-            bool skip = on_new_token_cb(str, bpe_tokens);
-            if (skip) {
-                continue;
-            }
-            for (auto& token : matches) {
-                std::string token_str = token.str();
-                std::u32string utf32_token;
-                for (int i = 0; i < token_str.length(); i++) {
-                    unsigned char b = token_str[i];
-                    utf32_token += byte_encoder[b];
-                }
-                auto bpe_strs = bpe(utf32_token);
-                size_t start  = 0;
-                size_t pos;
-                while ((pos = bpe_strs.find(' ', start)) != std::u32string::npos) {
-                    auto bpe_str = bpe_strs.substr(start, pos - start);
-                    bpe_tokens.push_back(encoder[bpe_str]);
-                    token_strs.push_back(utf32_to_utf8(bpe_str));
-
-                    start = pos + 1;
-                }
-                auto bpe_str = bpe_strs.substr(start, bpe_strs.size() - start);
-                bpe_tokens.push_back(encoder[bpe_str]);
-                token_strs.push_back(utf32_to_utf8(bpe_str));
-            }
-            str = matches.suffix();
-        }
-        std::stringstream ss;
-        ss << "[";
-        for (auto token : token_strs) {
-            ss << "\"" << token << "\", ";
-        }
-        ss << "]";
-        // LOG_DEBUG("split prompt \"%s\" to tokens %s", original_text.c_str(), ss.str().c_str());
-        // printf("split prompt \"%s\" to tokens %s \n", original_text.c_str(), ss.str().c_str());
-        return bpe_tokens;
-    }
-};
-
-/*================================================ FrozenCLIPEmbedder ================================================*/
-
-// Ref: https://github.com/huggingface/transformers/blob/main/src/transformers/models/clip/modeling_clip.py
-
-struct CLIPMLP : public GGMLBlock {
-protected:
-    bool use_gelu;
-
-public:
-    CLIPMLP(int64_t d_model, int64_t intermediate_size) {
-        blocks["fc1"] = std::shared_ptr<GGMLBlock>(new Linear(d_model, intermediate_size));
-        blocks["fc2"] = std::shared_ptr<GGMLBlock>(new Linear(intermediate_size, d_model));
-
-        if (d_model == 1024 || d_model == 1280) {  // SD 2.x
-            use_gelu = true;
-        } else {  // SD 1.x
-            use_gelu = false;
-        }
-    }
-
-    struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
-        // x: [N, n_token, d_model]
-        auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
-        auto fc2 = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
-
-        x = fc1->forward(ctx, x);
-        if (use_gelu) {
-            x = ggml_gelu_inplace(ctx, x);
-        } else {
-            x = ggml_gelu_quick_inplace(ctx, x);
-        }
-        x = fc2->forward(ctx, x);
-        return x;
-    }
-};
-
-struct CLIPLayer : public GGMLBlock {
-protected:
-    int64_t d_model;  // hidden_size/embed_dim
-    int64_t n_head;
-    int64_t intermediate_size;
-
-public:
-    CLIPLayer(int64_t d_model,
-              int64_t n_head,
-              int64_t intermediate_size)
-        : d_model(d_model),
-          n_head(n_head),
-          intermediate_size(intermediate_size) {
-        blocks["self_attn"] = std::shared_ptr<GGMLBlock>(new MultiheadAttention(d_model, n_head, true, true));
-
-        blocks["layer_norm1"] = std::shared_ptr<GGMLBlock>(new LayerNorm(d_model));
-        blocks["layer_norm2"] = std::shared_ptr<GGMLBlock>(new LayerNorm(d_model));
-
-        blocks["mlp"] = std::shared_ptr<GGMLBlock>(new CLIPMLP(d_model, intermediate_size));
-    }
-
-    struct ggml_tensor* forward(struct ggml_context* ctx, ggml_backend_t backend, struct ggml_tensor* x, bool mask = true) {
-        // x: [N, n_token, d_model]
-        auto self_attn   = std::dynamic_pointer_cast<MultiheadAttention>(blocks["self_attn"]);
-        auto layer_norm1 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm1"]);
-        auto layer_norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm2"]);
-        auto mlp         = std::dynamic_pointer_cast<CLIPMLP>(blocks["mlp"]);
-
-        x = ggml_add(ctx, x, self_attn->forward(ctx, backend, layer_norm1->forward(ctx, x), mask));
-        x = ggml_add(ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x)));
-        return x;
-    }
-};
-
-struct CLIPEncoder : public GGMLBlock {
-protected:
-    int64_t n_layer;
-
-public:
-    CLIPEncoder(int64_t n_layer,
-                int64_t d_model,
-                int64_t n_head,
-                int64_t intermediate_size)
-        : n_layer(n_layer) {
-        for (int i = 0; i < n_layer; i++) {
-            std::string name = "layers." + std::to_string(i);
-            blocks[name]     = std::shared_ptr<GGMLBlock>(new CLIPLayer(d_model, n_head, intermediate_size));
-        }
-    }
-
-    struct ggml_tensor* forward(struct ggml_context* ctx,
-                                ggml_backend_t backend,
-                                struct ggml_tensor* x,
-                                int clip_skip = -1,
-                                bool mask     = true) {
-        // x: [N, n_token, d_model]
-        int layer_idx = n_layer - 1;
-        // LOG_DEBUG("clip_skip %d", clip_skip);
-        if (clip_skip > 0) {
-            layer_idx = n_layer - clip_skip;
-        }
-
-        for (int i = 0; i < n_layer; i++) {
-            // LOG_DEBUG("layer %d", i);
-            if (i == layer_idx + 1) {
-                break;
-            }
-            std::string name = "layers." + std::to_string(i);
-            auto layer       = std::dynamic_pointer_cast<CLIPLayer>(blocks[name]);
-            x                = layer->forward(ctx, backend, x, mask);  // [N, n_token, d_model]
-            // LOG_DEBUG("layer %d", i);
-        }
-        return x;
-    }
-};
-
-class CLIPEmbeddings : public GGMLBlock {
-protected:
-    int64_t embed_dim;
-    int64_t vocab_size;
-    int64_t num_positions;
-    bool force_clip_f32;
-
-    void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, const std::string prefix = "") override {
-        enum ggml_type token_wtype = GGML_TYPE_F32;
-        if (!force_clip_f32) {
-            token_wtype = get_type(prefix + "token_embedding.weight", tensor_types, GGML_TYPE_F32);
-            if (!support_get_rows(token_wtype)) {
-                token_wtype = GGML_TYPE_F32;
-            }
-        }
-        enum ggml_type position_wtype       = GGML_TYPE_F32;
-        params["token_embedding.weight"]    = ggml_new_tensor_2d(ctx, token_wtype, embed_dim, vocab_size);
-        params["position_embedding.weight"] = ggml_new_tensor_2d(ctx, position_wtype, embed_dim, num_positions);
-    }
-
-public:
-    CLIPEmbeddings(int64_t embed_dim,
-                   int64_t vocab_size    = 49408,
-                   int64_t num_positions = 77,
-                   bool force_clip_f32   = false)
-        : embed_dim(embed_dim),
-          vocab_size(vocab_size),
-          num_positions(num_positions),
-          force_clip_f32(force_clip_f32) {
-    }
-
-    struct ggml_tensor* get_token_embed_weight() {
-        return params["token_embedding.weight"];
-    }
-
-    struct ggml_tensor* forward(struct ggml_context* ctx,
-                                struct ggml_tensor* input_ids,
-                                struct ggml_tensor* custom_embed_weight) {
-        // input_ids: [N, n_token]
-        auto token_embed_weight    = params["token_embedding.weight"];
-        auto position_embed_weight = params["position_embedding.weight"];
-
-        GGML_ASSERT(input_ids->ne[0] == position_embed_weight->ne[1]);
-        input_ids            = ggml_reshape_3d(ctx, input_ids, input_ids->ne[0], 1, input_ids->ne[1]);
-        auto token_embedding = ggml_get_rows(ctx, custom_embed_weight != nullptr ? custom_embed_weight : token_embed_weight, input_ids);
-        token_embedding      = ggml_reshape_3d(ctx, token_embedding, token_embedding->ne[0], token_embedding->ne[1], token_embedding->ne[3]);
-
-        // token_embedding + position_embedding
-        auto x = ggml_add(ctx,
-                          token_embedding,
-                          position_embed_weight);  // [N, n_token, embed_dim]
-        return x;
-    }
-};
-
-class CLIPVisionEmbeddings : public GGMLBlock {
-protected:
-    int64_t embed_dim;
-    int64_t num_channels;
-    int64_t patch_size;
-    int64_t image_size;
-    int64_t num_patches;
-    int64_t num_positions;
-    void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, const std::string prefix = "") override {
-        enum ggml_type patch_wtype    = GGML_TYPE_F16;
-        enum ggml_type class_wtype    = GGML_TYPE_F32;
-        enum ggml_type position_wtype = GGML_TYPE_F32;
-
-        params["patch_embedding.weight"]    = ggml_new_tensor_4d(ctx, patch_wtype, patch_size, patch_size, num_channels, embed_dim);
-        params["class_embedding"]           = ggml_new_tensor_1d(ctx, class_wtype, embed_dim);
-        params["position_embedding.weight"] = ggml_new_tensor_2d(ctx, position_wtype, embed_dim, num_positions);
-    }
-
-public:
-    CLIPVisionEmbeddings(int64_t embed_dim,
-                         int64_t num_channels = 3,
-                         int64_t patch_size   = 14,
-                         int64_t image_size   = 224)
-        : embed_dim(embed_dim),
-          num_channels(num_channels),
-          patch_size(patch_size),
-          image_size(image_size) {
-        num_patches   = (image_size / patch_size) * (image_size / patch_size);
-        num_positions = num_patches + 1;
-    }
-
-    struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* pixel_values) {
-        // pixel_values: [N, num_channels, image_size, image_size]
-        // return: [N, num_positions, embed_dim]
-        GGML_ASSERT(pixel_values->ne[0] == image_size && pixel_values->ne[1] == image_size && pixel_values->ne[2] == num_channels);
-
-        auto patch_embed_weight    = params["patch_embedding.weight"];
-        auto class_embed_weight    = params["class_embedding"];
-        auto position_embed_weight = params["position_embedding.weight"];
-
-        // concat(patch_embedding, class_embedding) + position_embedding
-        struct ggml_tensor* patch_embedding;
-        int64_t N       = pixel_values->ne[3];
-        patch_embedding = ggml_nn_conv_2d(ctx, pixel_values, patch_embed_weight, nullptr, patch_size, patch_size);  // [N, embed_dim, image_size // pacht_size, image_size // pacht_size]
-        patch_embedding = ggml_reshape_3d(ctx, patch_embedding, num_patches, embed_dim, N);                         // [N, embed_dim, num_patches]
-        patch_embedding = ggml_cont(ctx, ggml_permute(ctx, patch_embedding, 1, 0, 2, 3));                           // [N, num_patches, embed_dim]
-        patch_embedding = ggml_reshape_4d(ctx, patch_embedding, 1, embed_dim, num_patches, N);                      // [N, num_patches, embed_dim, 1]
-
-        struct ggml_tensor* class_embedding = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, embed_dim, N);
-        class_embedding                     = ggml_repeat(ctx, class_embed_weight, class_embedding);      // [N, embed_dim]
-        class_embedding                     = ggml_reshape_4d(ctx, class_embedding, 1, embed_dim, 1, N);  // [N, 1, embed_dim, 1]
-
-        struct ggml_tensor* x = ggml_concat(ctx, class_embedding, patch_embedding, 2);  // [N, num_positions, embed_dim, 1]
-        x                     = ggml_reshape_3d(ctx, x, embed_dim, num_positions, N);   // [N, num_positions, embed_dim]
-        x                     = ggml_add(ctx, x, position_embed_weight);
-        return x;  // [N, num_positions, embed_dim]
-    }
-};
-
-// OPENAI_CLIP_VIT_L_14: https://huggingface.co/openai/clip-vit-large-patch14/blob/main/config.json
-// OPEN_CLIP_VIT_H_14: https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K/blob/main/config.json
-// OPEN_CLIP_VIT_BIGG_14: https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k/blob/main/config.json (CLIPTextModelWithProjection)
-
-enum CLIPVersion {
-    OPENAI_CLIP_VIT_L_14,   // SD 1.x and SDXL
-    OPEN_CLIP_VIT_H_14,     // SD 2.x
-    OPEN_CLIP_VIT_BIGG_14,  // SDXL
-};
-
-class CLIPTextModel : public GGMLBlock {
-protected:
-    void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, const std::string prefix = "") override {
-        if (version == OPEN_CLIP_VIT_BIGG_14) {
-            enum ggml_type wtype      = GGML_TYPE_F32;
-            params["text_projection"] = ggml_new_tensor_2d(ctx, wtype, projection_dim, hidden_size);
-        }
-    }
-
-public:
-    CLIPVersion version = OPENAI_CLIP_VIT_L_14;
-    // network hparams
-    int32_t vocab_size        = 49408;
-    int32_t n_token           = 77;  // max_position_embeddings
-    int32_t hidden_size       = 768;
-    int32_t intermediate_size = 3072;
-    int32_t n_head            = 12;
-    int32_t n_layer           = 12;    // num_hidden_layers
-    int32_t projection_dim    = 1280;  // only for OPEN_CLIP_VIT_BIGG_14
-    bool with_final_ln        = true;
-
-    CLIPTextModel(CLIPVersion version = OPENAI_CLIP_VIT_L_14,
-                  bool with_final_ln  = true,
-                  bool force_clip_f32 = false)
-        : version(version), with_final_ln(with_final_ln) {
-        if (version == OPEN_CLIP_VIT_H_14) {
-            hidden_size       = 1024;
-            intermediate_size = 4096;
-            n_head            = 16;
-            n_layer           = 24;
-        } else if (version == OPEN_CLIP_VIT_BIGG_14) {  // CLIPTextModelWithProjection
-            hidden_size       = 1280;
-            intermediate_size = 5120;
-            n_head            = 20;
-            n_layer           = 32;
-        }
-
-        blocks["embeddings"]       = std::shared_ptr<GGMLBlock>(new CLIPEmbeddings(hidden_size, vocab_size, n_token, force_clip_f32));
-        blocks["encoder"]          = std::shared_ptr<GGMLBlock>(new CLIPEncoder(n_layer, hidden_size, n_head, intermediate_size));
-        blocks["final_layer_norm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size));
-    }
-
-    struct ggml_tensor* get_token_embed_weight() {
-        auto embeddings = std::dynamic_pointer_cast<CLIPEmbeddings>(blocks["embeddings"]);
-        return embeddings->get_token_embed_weight();
-    }
-
-    struct ggml_tensor* forward(struct ggml_context* ctx,
-                                ggml_backend_t backend,
-                                struct ggml_tensor* input_ids,
-                                struct ggml_tensor* tkn_embeddings,
-                                size_t max_token_idx = 0,
-                                bool return_pooled   = false,
-                                int clip_skip        = -1) {
-        // input_ids: [N, n_token]
-        auto embeddings       = std::dynamic_pointer_cast<CLIPEmbeddings>(blocks["embeddings"]);
-        auto encoder          = std::dynamic_pointer_cast<CLIPEncoder>(blocks["encoder"]);
-        auto final_layer_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["final_layer_norm"]);
-
-        auto x = embeddings->forward(ctx, input_ids, tkn_embeddings);  // [N, n_token, hidden_size]
-        x      = encoder->forward(ctx, backend, x, return_pooled ? -1 : clip_skip, true);
-        if (return_pooled || with_final_ln) {
-            x = final_layer_norm->forward(ctx, x);
-        }
-
-        if (return_pooled) {
-            auto text_projection = params["text_projection"];
-            ggml_tensor* pooled  = ggml_view_1d(ctx, x, hidden_size, x->nb[1] * max_token_idx);
-            if (text_projection != nullptr) {
-                pooled = ggml_nn_linear(ctx, pooled, text_projection, nullptr);
-            } else {
-                LOG_DEBUG("identity projection");
-            }
-            return pooled;  // [hidden_size, 1, 1]
-        }
-
-        return x;  // [N, n_token, hidden_size]
-    }
-};
-
-class CLIPVisionModel : public GGMLBlock {
-public:
-    // network hparams
-    int32_t num_channels      = 3;
-    int32_t patch_size        = 14;
-    int32_t image_size        = 224;
-    int32_t num_positions     = 257;  // (image_size / patch_size)^2 + 1
-    int32_t hidden_size       = 1024;
-    int32_t intermediate_size = 4096;
-    int32_t n_head            = 16;
-    int32_t n_layer           = 24;
-
-public:
-    CLIPVisionModel(CLIPVersion version = OPENAI_CLIP_VIT_L_14) {
-        if (version == OPEN_CLIP_VIT_H_14) {
-            hidden_size       = 1280;
-            intermediate_size = 5120;
-            n_head            = 16;
-            n_layer           = 32;
-        } else if (version == OPEN_CLIP_VIT_BIGG_14) {
-            hidden_size       = 1664;
-            intermediate_size = 8192;
-            n_head            = 16;
-            n_layer           = 48;
-        }
-
-        blocks["embeddings"]     = std::shared_ptr<GGMLBlock>(new CLIPVisionEmbeddings(hidden_size, num_channels, patch_size, image_size));
-        blocks["pre_layernorm"]  = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size));
-        blocks["encoder"]        = std::shared_ptr<GGMLBlock>(new CLIPEncoder(n_layer, hidden_size, n_head, intermediate_size));
-        blocks["post_layernorm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size));
-    }
-
-    struct ggml_tensor* forward(struct ggml_context* ctx,
-                                ggml_backend_t backend,
-                                struct ggml_tensor* pixel_values,
-                                bool return_pooled = true,
-                                int clip_skip      = -1) {
-        // pixel_values: [N, num_channels, image_size, image_size]
-        auto embeddings     = std::dynamic_pointer_cast<CLIPVisionEmbeddings>(blocks["embeddings"]);
-        auto pre_layernorm  = std::dynamic_pointer_cast<LayerNorm>(blocks["pre_layernorm"]);
-        auto encoder        = std::dynamic_pointer_cast<CLIPEncoder>(blocks["encoder"]);
-        auto post_layernorm = std::dynamic_pointer_cast<LayerNorm>(blocks["post_layernorm"]);
-
-        auto x = embeddings->forward(ctx, pixel_values);  // [N, num_positions, embed_dim]
-        x      = pre_layernorm->forward(ctx, x);
-        x      = encoder->forward(ctx, backend, x, clip_skip, false);
-        // print_ggml_tensor(x, true, "ClipVisionModel x: ");
-        auto last_hidden_state = x;
-        x                      = post_layernorm->forward(ctx, x);  // [N, n_token, hidden_size]
-
-        GGML_ASSERT(x->ne[3] == 1);
-        if (return_pooled) {
-            ggml_tensor* pooled = ggml_cont(ctx, ggml_view_2d(ctx, x, x->ne[0], x->ne[2], x->nb[2], 0));
-            return pooled;  // [N, hidden_size]
-        } else {
-            // return x;  // [N, n_token, hidden_size]
-            return last_hidden_state;  // [N, n_token, hidden_size]
-        }
-    }
-};
-
-class CLIPProjection : public UnaryBlock {
-protected:
-    int64_t in_features;
-    int64_t out_features;
-    bool transpose_weight;
-
-    void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, const std::string prefix = "") override {
-        enum ggml_type wtype = get_type(prefix + "weight", tensor_types, GGML_TYPE_F32);
-        if (transpose_weight) {
-            params["weight"] = ggml_new_tensor_2d(ctx, wtype, out_features, in_features);
-        } else {
-            params["weight"] = ggml_new_tensor_2d(ctx, wtype, in_features, out_features);
-        }
-    }
-
-public:
-    CLIPProjection(int64_t in_features,
-                   int64_t out_features,
-                   bool transpose_weight = false)
-        : in_features(in_features),
-          out_features(out_features),
-          transpose_weight(transpose_weight) {}
-
-    struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
-        struct ggml_tensor* w = params["weight"];
-        if (transpose_weight) {
-            w = ggml_cont(ctx, ggml_transpose(ctx, w));
-        }
-        return ggml_nn_linear(ctx, x, w, nullptr);
-    }
-};
-
-class CLIPVisionModelProjection : public GGMLBlock {
-public:
-    int32_t hidden_size    = 1024;
-    int32_t projection_dim = 768;
-    int32_t image_size     = 224;
-
-public:
-    CLIPVisionModelProjection(CLIPVersion version   = OPENAI_CLIP_VIT_L_14,
-                              bool transpose_proj_w = false) {
-        if (version == OPEN_CLIP_VIT_H_14) {
-            hidden_size    = 1280;
-            projection_dim = 1024;
-        } else if (version == OPEN_CLIP_VIT_BIGG_14) {
-            hidden_size = 1664;
-        }
-
-        blocks["vision_model"]      = std::shared_ptr<GGMLBlock>(new CLIPVisionModel(version));
-        blocks["visual_projection"] = std::shared_ptr<GGMLBlock>(new CLIPProjection(hidden_size, projection_dim, transpose_proj_w));
-    }
-
-    struct ggml_tensor* forward(struct ggml_context* ctx,
-                                ggml_backend_t backend,
-                                struct ggml_tensor* pixel_values,
-                                bool return_pooled = true,
-                                int clip_skip      = -1) {
-        // pixel_values: [N, num_channels, image_size, image_size]
-        // return: [N, projection_dim] if return_pooled else [N, n_token, hidden_size]
-        auto vision_model      = std::dynamic_pointer_cast<CLIPVisionModel>(blocks["vision_model"]);
-        auto visual_projection = std::dynamic_pointer_cast<CLIPProjection>(blocks["visual_projection"]);
-
-        auto x = vision_model->forward(ctx, backend, pixel_values, return_pooled, clip_skip);  // [N, hidden_size] or [N, n_token, hidden_size]
-
-        if (return_pooled) {
-            x = visual_projection->forward(ctx, x);  // [N, projection_dim]
-        }
-
-        return x;
-    }
-};
-
-struct CLIPTextModelRunner : public GGMLRunner {
-    CLIPTextModel model;
-
-    CLIPTextModelRunner(ggml_backend_t backend,
-                        bool offload_params_to_cpu,
-                        const String2GGMLType& tensor_types,
-                        const std::string prefix,
-                        CLIPVersion version = OPENAI_CLIP_VIT_L_14,
-                        bool with_final_ln  = true,
-                        bool force_clip_f32 = false)
-        : GGMLRunner(backend, offload_params_to_cpu), model(version, with_final_ln, force_clip_f32) {
-        model.init(params_ctx, tensor_types, prefix);
-    }
-
-    std::string get_desc() override {
-        return "clip";
-    }
-
-    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
-        model.get_param_tensors(tensors, prefix);
-    }
-
-    struct ggml_tensor* forward(struct ggml_context* ctx,
-                                ggml_backend_t backend,
-                                struct ggml_tensor* input_ids,
-                                struct ggml_tensor* embeddings,
-                                size_t max_token_idx = 0,
-                                bool return_pooled   = false,
-                                int clip_skip        = -1) {
-        size_t N       = input_ids->ne[1];
-        size_t n_token = input_ids->ne[0];
-        if (input_ids->ne[0] > model.n_token) {
-            GGML_ASSERT(input_ids->ne[0] % model.n_token == 0);
-            input_ids = ggml_reshape_2d(ctx, input_ids, model.n_token, input_ids->ne[0] / model.n_token);
-        }
-
-        return model.forward(ctx, backend, input_ids, embeddings, max_token_idx, return_pooled, clip_skip);
-    }
-
-    struct ggml_cgraph* build_graph(struct ggml_tensor* input_ids,
-                                    int num_custom_embeddings    = 0,
-                                    void* custom_embeddings_data = nullptr,
-                                    size_t max_token_idx         = 0,
-                                    bool return_pooled           = false,
-                                    int clip_skip                = -1) {
-        struct ggml_cgraph* gf = ggml_new_graph(compute_ctx);
-
-        input_ids = to_backend(input_ids);
-
-        struct ggml_tensor* embeddings = nullptr;
-
-        if (num_custom_embeddings > 0 && custom_embeddings_data != nullptr) {
-            auto token_embed_weight = model.get_token_embed_weight();
-            auto custom_embeddings  = ggml_new_tensor_2d(compute_ctx,
-                                                         token_embed_weight->type,
-                                                         model.hidden_size,
-                                                         num_custom_embeddings);
-            set_backend_tensor_data(custom_embeddings, custom_embeddings_data);
-
-            // concatenate custom embeddings
-            embeddings = ggml_concat(compute_ctx, token_embed_weight, custom_embeddings, 1);
-        }
-
-        struct ggml_tensor* hidden_states = forward(compute_ctx, runtime_backend, input_ids, embeddings, max_token_idx, return_pooled, clip_skip);
-
-        ggml_build_forward_expand(gf, hidden_states);
-
-        return gf;
-    }
-
-    void compute(const int n_threads,
-                 struct ggml_tensor* input_ids,
-                 int num_custom_embeddings,
-                 void* custom_embeddings_data,
-                 size_t max_token_idx,
-                 bool return_pooled,
-                 int clip_skip,
-                 ggml_tensor** output,
-                 ggml_context* output_ctx = nullptr) {
-        auto get_graph = [&]() -> struct ggml_cgraph* {
-            return build_graph(input_ids, num_custom_embeddings, custom_embeddings_data, max_token_idx, return_pooled, clip_skip);
-        };
-        GGMLRunner::compute(get_graph, n_threads, true, output, output_ctx);
-    }
-};
-
-#endif  // __CLIP_HPP__

diffusion_model.hpp

@@ -1,323 +0,0 @@
-#ifndef __DIFFUSION_MODEL_H__
-#define __DIFFUSION_MODEL_H__
-
-#include "flux.hpp"
-#include "mmdit.hpp"
-#include "qwen_image.hpp"
-#include "unet.hpp"
-#include "wan.hpp"
-
-struct DiffusionParams {
-    struct ggml_tensor* x                     = nullptr;
-    struct ggml_tensor* timesteps             = nullptr;
-    struct ggml_tensor* context               = nullptr;
-    struct ggml_tensor* c_concat              = nullptr;
-    struct ggml_tensor* y                     = nullptr;
-    struct ggml_tensor* guidance              = nullptr;
-    std::vector<ggml_tensor*> ref_latents     = {};
-    bool increase_ref_index                   = false;
-    int num_video_frames                      = -1;
-    std::vector<struct ggml_tensor*> controls = {};
-    float control_strength                    = 0.f;
-    struct ggml_tensor* vace_context          = nullptr;
-    float vace_strength                       = 1.f;
-    std::vector<int> skip_layers              = {};
-};
-
-struct DiffusionModel {
-    virtual std::string get_desc()                                                      = 0;
-    virtual void compute(int n_threads,
-                         DiffusionParams diffusion_params,
-                         struct ggml_tensor** output     = nullptr,
-                         struct ggml_context* output_ctx = nullptr)                     = 0;
-    virtual void alloc_params_buffer()                                                  = 0;
-    virtual void free_params_buffer()                                                   = 0;
-    virtual void free_compute_buffer()                                                  = 0;
-    virtual void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) = 0;
-    virtual size_t get_params_buffer_size()                                             = 0;
-    virtual int64_t get_adm_in_channels()                                               = 0;
-};
-
-struct UNetModel : public DiffusionModel {
-    UNetModelRunner unet;
-
-    UNetModel(ggml_backend_t backend,
-              bool offload_params_to_cpu,
-              const String2GGMLType& tensor_types = {},
-              SDVersion version                   = VERSION_SD1,
-              bool flash_attn                     = false)
-        : unet(backend, offload_params_to_cpu, tensor_types, "model.diffusion_model", version, flash_attn) {
-    }
-
-    std::string get_desc() override {
-        return unet.get_desc();
-    }
-
-    void alloc_params_buffer() override {
-        unet.alloc_params_buffer();
-    }
-
-    void free_params_buffer() override {
-        unet.free_params_buffer();
-    }
-
-    void free_compute_buffer() override {
-        unet.free_compute_buffer();
-    }
-
-    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
-        unet.get_param_tensors(tensors, "model.diffusion_model");
-    }
-
-    size_t get_params_buffer_size() override {
-        return unet.get_params_buffer_size();
-    }
-
-    int64_t get_adm_in_channels() override {
-        return unet.unet.adm_in_channels;
-    }
-
-    void compute(int n_threads,
-                 DiffusionParams diffusion_params,
-                 struct ggml_tensor** output     = nullptr,
-                 struct ggml_context* output_ctx = nullptr) override {
-        return unet.compute(n_threads,
-                            diffusion_params.x,
-                            diffusion_params.timesteps,
-                            diffusion_params.context,
-                            diffusion_params.c_concat,
-                            diffusion_params.y,
-                            diffusion_params.num_video_frames,
-                            diffusion_params.controls,
-                            diffusion_params.control_strength, output, output_ctx);
-    }
-};
-
-struct MMDiTModel : public DiffusionModel {
-    MMDiTRunner mmdit;
-
-    MMDiTModel(ggml_backend_t backend,
-               bool offload_params_to_cpu,
-               bool flash_attn                     = false,
-               const String2GGMLType& tensor_types = {})
-        : mmdit(backend, offload_params_to_cpu, flash_attn, tensor_types, "model.diffusion_model") {
-    }
-
-    std::string get_desc() override {
-        return mmdit.get_desc();
-    }
-
-    void alloc_params_buffer() override {
-        mmdit.alloc_params_buffer();
-    }
-
-    void free_params_buffer() override {
-        mmdit.free_params_buffer();
-    }
-
-    void free_compute_buffer() override {
-        mmdit.free_compute_buffer();
-    }
-
-    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
-        mmdit.get_param_tensors(tensors, "model.diffusion_model");
-    }
-
-    size_t get_params_buffer_size() override {
-        return mmdit.get_params_buffer_size();
-    }
-
-    int64_t get_adm_in_channels() override {
-        return 768 + 1280;
-    }
-
-    void compute(int n_threads,
-                 DiffusionParams diffusion_params,
-                 struct ggml_tensor** output     = nullptr,
-                 struct ggml_context* output_ctx = nullptr) override {
-        return mmdit.compute(n_threads,
-                             diffusion_params.x,
-                             diffusion_params.timesteps,
-                             diffusion_params.context,
-                             diffusion_params.y,
-                             output,
-                             output_ctx,
-                             diffusion_params.skip_layers);
-    }
-};
-
-struct FluxModel : public DiffusionModel {
-    Flux::FluxRunner flux;
-
-    FluxModel(ggml_backend_t backend,
-              bool offload_params_to_cpu,
-              const String2GGMLType& tensor_types = {},
-              SDVersion version                   = VERSION_FLUX,
-              bool flash_attn                     = false,
-              bool use_mask                       = false)
-        : flux(backend, offload_params_to_cpu, tensor_types, "model.diffusion_model", version, flash_attn, use_mask) {
-    }
-
-    std::string get_desc() override {
-        return flux.get_desc();
-    }
-
-    void alloc_params_buffer() override {
-        flux.alloc_params_buffer();
-    }
-
-    void free_params_buffer() override {
-        flux.free_params_buffer();
-    }
-
-    void free_compute_buffer() override {
-        flux.free_compute_buffer();
-    }
-
-    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
-        flux.get_param_tensors(tensors, "model.diffusion_model");
-    }
-
-    size_t get_params_buffer_size() override {
-        return flux.get_params_buffer_size();
-    }
-
-    int64_t get_adm_in_channels() override {
-        return 768;
-    }
-
-    void compute(int n_threads,
-                 DiffusionParams diffusion_params,
-                 struct ggml_tensor** output     = nullptr,
-                 struct ggml_context* output_ctx = nullptr) override {
-        return flux.compute(n_threads,
-                            diffusion_params.x,
-                            diffusion_params.timesteps,
-                            diffusion_params.context,
-                            diffusion_params.c_concat,
-                            diffusion_params.y,
-                            diffusion_params.guidance,
-                            diffusion_params.ref_latents,
-                            diffusion_params.increase_ref_index,
-                            output,
-                            output_ctx,
-                            diffusion_params.skip_layers);
-    }
-};
-
-struct WanModel : public DiffusionModel {
-    std::string prefix;
-    WAN::WanRunner wan;
-
-    WanModel(ggml_backend_t backend,
-             bool offload_params_to_cpu,
-             const String2GGMLType& tensor_types = {},
-             const std::string prefix            = "model.diffusion_model",
-             SDVersion version                   = VERSION_WAN2,
-             bool flash_attn                     = false)
-        : prefix(prefix), wan(backend, offload_params_to_cpu, tensor_types, prefix, version, flash_attn) {
-    }
-
-    std::string get_desc() override {
-        return wan.get_desc();
-    }
-
-    void alloc_params_buffer() override {
-        wan.alloc_params_buffer();
-    }
-
-    void free_params_buffer() override {
-        wan.free_params_buffer();
-    }
-
-    void free_compute_buffer() override {
-        wan.free_compute_buffer();
-    }
-
-    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
-        wan.get_param_tensors(tensors, prefix);
-    }
-
-    size_t get_params_buffer_size() override {
-        return wan.get_params_buffer_size();
-    }
-
-    int64_t get_adm_in_channels() override {
-        return 768;
-    }
-
-    void compute(int n_threads,
-                 DiffusionParams diffusion_params,
-                 struct ggml_tensor** output     = nullptr,
-                 struct ggml_context* output_ctx = nullptr) override {
-        return wan.compute(n_threads,
-                           diffusion_params.x,
-                           diffusion_params.timesteps,
-                           diffusion_params.context,
-                           diffusion_params.y,
-                           diffusion_params.c_concat,
-                           nullptr,
-                           diffusion_params.vace_context,
-                           diffusion_params.vace_strength,
-                           output,
-                           output_ctx);
-    }
-};
-
-struct QwenImageModel : public DiffusionModel {
-    std::string prefix;
-    Qwen::QwenImageRunner qwen_image;
-
-    QwenImageModel(ggml_backend_t backend,
-                   bool offload_params_to_cpu,
-                   const String2GGMLType& tensor_types = {},
-                   const std::string prefix            = "model.diffusion_model",
-                   SDVersion version                   = VERSION_QWEN_IMAGE,
-                   bool flash_attn                     = false)
-        : prefix(prefix), qwen_image(backend, offload_params_to_cpu, tensor_types, prefix, version, flash_attn) {
-    }
-
-    std::string get_desc() override {
-        return qwen_image.get_desc();
-    }
-
-    void alloc_params_buffer() override {
-        qwen_image.alloc_params_buffer();
-    }
-
-    void free_params_buffer() override {
-        qwen_image.free_params_buffer();
-    }
-
-    void free_compute_buffer() override {
-        qwen_image.free_compute_buffer();
-    }
-
-    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
-        qwen_image.get_param_tensors(tensors, prefix);
-    }
-
-    size_t get_params_buffer_size() override {
-        return qwen_image.get_params_buffer_size();
-    }
-
-    int64_t get_adm_in_channels() override {
-        return 768;
-    }
-
-    void compute(int n_threads,
-                 DiffusionParams diffusion_params,
-                 struct ggml_tensor** output     = nullptr,
-                 struct ggml_context* output_ctx = nullptr) override {
-        return qwen_image.compute(n_threads,
-                                  diffusion_params.x,
-                                  diffusion_params.timesteps,
-                                  diffusion_params.context,
-                                  diffusion_params.ref_latents,
-                                  true,  // increase_ref_index
-                                  output,
-                                  output_ctx);
-    }
-};
-
-#endif

docs/anima.md

@@ -0,0 +1,21 @@
+# How to Use
+
+## Download weights
+
+- Download Anima
+    - safetensors: https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/diffusion_models
+    - gguf: https://huggingface.co/Bedovyy/Anima-GGUF/tree/main
+    - gguf Anima2: https://huggingface.co/JusteLeo/Anima2-GGUF/tree/main
+- Download vae
+    - safetensors: https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/vae
+- Download Qwen3-0.6B-Base
+    - safetensors: https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/text_encoders
+    - gguf: https://huggingface.co/mradermacher/Qwen3-0.6B-Base-GGUF/tree/main
+
+## Examples
+
+```sh
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\anima-preview.safetensors --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_06b_base.safetensors  -p "a lovely cat holding a sign says 'anima.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu --diffusion-fa
+```
+
+<img alt="anima image example" src="../assets/anima/example.png" />

docs/build.md

@@ -16,6 +16,26 @@ git submodule init
 git submodule update
 ```
 
+## WebP and WebM Support in Examples
+
+The example applications (`examples/cli` and `examples/server`) use `libwebp` to support WebP image I/O, and `examples/cli` can also use `libwebm` for `.webm` video output. Both are enabled by default. WebM output currently reuses `libwebp` to encode each frame as VP8 before muxing with `libwebm`.
+
+If you do not want WebP/WebM support, you can disable them at configure time:
+
+```shell
+mkdir build && cd build
+cmake .. -DSD_WEBP=OFF -DSD_WEBM=OFF
+cmake --build . --config Release
+```
+
+If the submodules are not available, you can also link against system packages instead:
+
+```shell
+mkdir build && cd build
+cmake .. -DSD_USE_SYSTEM_WEBP=ON -DSD_USE_SYSTEM_WEBM=ON
+cmake --build . --config Release
+```
+
 ## Build (CPU only)
 
 If you don't have a GPU or CUDA installed, you can build a CPU-only version.
@@ -157,7 +177,7 @@ ninja
 
 ## Build with SYCL
 
-Using SYCL makes the computation run on the Intel GPU. Please make sure you have installed the related driver and [Intel® oneAPI Base toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) before start. More details and steps can refer to [llama.cpp SYCL backend](https://github.com/ggerganov/llama.cpp/blob/master/docs/backend/SYCL.md#linux).
+Using SYCL makes the computation run on the Intel GPU. Please make sure you have installed the related driver and [Intel® oneAPI Base toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) before start. More details and steps can refer to [llama.cpp SYCL backend](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/SYCL.md#linux).
 
 ```shell
 # Export relevant ENV variables

docs/caching.md

@@ -0,0 +1,139 @@
+## Caching
+
+Caching methods accelerate diffusion inference by reusing intermediate computations when changes between steps are small.
+
+### Cache Modes
+
+| Mode | Target | Description |
+|------|--------|-------------|
+| `ucache` | UNET models | Condition-level caching with error tracking |
+| `easycache` | DiT models | Condition-level cache |
+| `dbcache` | DiT models | Block-level L1 residual threshold |
+| `taylorseer` | DiT models | Taylor series approximation |
+| `cache-dit` | DiT models | Combined DBCache + TaylorSeer |
+| `spectrum` | UNET and DiT models | Chebyshev + Taylor output forecasting |
+
+### UCache (UNET Models)
+
+UCache caches the residual difference (output - input) and reuses it when input changes are below threshold.
+
+```bash
+sd-cli -m model.safetensors -p "a cat" --cache-mode ucache --cache-option "threshold=1.5"
+```
+
+#### Parameters
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `threshold` | Error threshold for reuse decision | 1.0 |
+| `start` | Start caching at this percent of steps | 0.15 |
+| `end` | Stop caching at this percent of steps | 0.95 |
+| `decay` | Error decay rate (0-1) | 1.0 |
+| `relative` | Scale threshold by output norm (0/1) | 1 |
+| `reset` | Reset error after computing (0/1) | 1 |
+
+#### Reset Parameter
+
+The `reset` parameter controls error accumulation behavior:
+
+- `reset=1` (default): Resets accumulated error after each computed step. More aggressive caching, works well with most samplers.
+- `reset=0`: Keeps error accumulated. More conservative, recommended for `euler_a` sampler.
+
+### EasyCache (DiT Models)
+
+Condition-level caching for DiT models. Caches and reuses outputs when input changes are below threshold.
+
+```bash
+--cache-mode easycache --cache-option "threshold=0.3"
+```
+
+#### Parameters
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `threshold` | Input change threshold for reuse | 0.2 |
+| `start` | Start caching at this percent of steps | 0.15 |
+| `end` | Stop caching at this percent of steps | 0.95 |
+
+### Cache-DIT (DiT Models)
+
+For DiT models like FLUX and QWEN, use block-level caching modes.
+
+#### DBCache
+
+Caches blocks based on L1 residual difference threshold:
+
+```bash
+--cache-mode dbcache --cache-option "threshold=0.25,warmup=4"
+```
+
+#### TaylorSeer
+
+Uses Taylor series approximation to predict block outputs:
+
+```bash
+--cache-mode taylorseer
+```
+
+#### Cache-DIT (Combined)
+
+Combines DBCache and TaylorSeer:
+
+```bash
+--cache-mode cache-dit
+```
+
+#### Parameters
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `Fn` | Front blocks to always compute | 8 |
+| `Bn` | Back blocks to always compute | 0 |
+| `threshold` | L1 residual difference threshold | 0.08 |
+| `warmup` | Steps before caching starts | 8 |
+
+#### SCM Options
+
+Steps Computation Mask controls which steps can be cached:
+
+```bash
+--scm-mask "1,1,1,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,1,1"
+```
+
+Mask values: `1` = compute, `0` = can cache.
+
+| Policy | Description |
+|--------|-------------|
+| `dynamic` | Check threshold before caching |
+| `static` | Always cache on cacheable steps |
+
+```bash
+--scm-policy dynamic
+```
+
+### Spectrum (UNET and DiT Models)
+
+Spectrum uses Chebyshev polynomial fitting blended with Taylor extrapolation to predict denoised outputs, skipping entire forward passes. Based on the paper [Spectrum: Adaptive Spectral Feature Forecasting for Efficient Diffusion Sampling](https://github.com/tingyu215/Spectrum).
+
+```bash
+sd-cli -m model.safetensors -p "a cat" --cache-mode spectrum
+```
+
+#### Parameters
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `w` | Chebyshev vs Taylor blend weight (0=Taylor, 1=Chebyshev) | 0.40 |
+| `m` | Chebyshev polynomial degree | 3 |
+| `lam` | Ridge regression regularization | 1.0 |
+| `window` | Initial window size (compute every N steps) | 2 |
+| `flex` | Window growth per computed step after warmup | 0.50 |
+| `warmup` | Steps to always compute before caching starts | 4 |
+| `stop` | Stop caching at this fraction of total steps | 0.9 |
+
+### Performance Tips
+
+- Start with default thresholds and adjust based on output quality
+- Lower threshold = better quality, less speedup
+- Higher threshold = more speedup, potential quality loss
+- More steps generally means more caching opportunities

docs/chroma.md

@@ -15,7 +15,7 @@ You can run Chroma using stable-diffusion.cpp with a GPU that has 6GB or even 4G
 You can download the preconverted gguf weights from [silveroxides/Chroma-GGUF](https://huggingface.co/silveroxides/Chroma-GGUF), this way you don't have to do the conversion yourself.
 
 ```
-.\bin\Release\sd.exe -M convert -m ..\..\ComfyUI\models\unet\chroma-unlocked-v40.safetensors -o ..\models\chroma-unlocked-v40-q8_0.gguf -v --type q8_0
+.\bin\Release\sd-cli.exe -M convert -m ..\..\ComfyUI\models\unet\chroma-unlocked-v40.safetensors -o ..\models\chroma-unlocked-v40-q8_0.gguf -v --type q8_0
 ```
 
 ## Run
@@ -24,7 +24,7 @@ You can download the preconverted gguf weights from [silveroxides/Chroma-GGUF](h
 For example:
 
 ```
- .\bin\Release\sd.exe --diffusion-model  ..\models\chroma-unlocked-v40-q8_0.gguf --vae ..\models\ae.sft --t5xxl ..\models\t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'chroma.cpp'" --cfg-scale 4.0 --sampling-method euler -v --chroma-disable-dit-mask --clip-on-cpu
+ .\bin\Release\sd-cli.exe --diffusion-model  ..\models\chroma-unlocked-v40-q8_0.gguf --vae ..\models\ae.sft --t5xxl ..\models\t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'chroma.cpp'" --cfg-scale 4.0 --sampling-method euler -v --chroma-disable-dit-mask --clip-on-cpu
 ```
 
 ![](../assets/flux/chroma_v40.png)

docs/chroma_radiance.md

@@ -0,0 +1,21 @@
+# How to Use
+
+## Download weights
+
+- Download Chroma1-Radiance
+    - safetensors: https://huggingface.co/lodestones/Chroma1-Radiance/tree/main
+    - gguf: https://huggingface.co/silveroxides/Chroma1-Radiance-GGUF/tree/main
+
+- Download t5xxl
+    - safetensors: https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors
+
+## Examples
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Chroma1-Radiance-v0.4-Q8_0.gguf --t5xxl ..\..\ComfyUI\models\clip\t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'chroma  radiance cpp'" --cfg-scale 4.0 --sampling-method euler -v
+```
+
+<img alt="Chroma1-Radiance" src="../assets/flux/chroma1-radiance.png" />
+
+
+

docs/distilled_sd.md

@@ -0,0 +1,118 @@
+# Running distilled models: SSD1B, Vega and SDx.x with tiny U-Nets
+
+## Preface 
+
+These models feature a reduced U-Net architecture. Unlike standard SDXL models, the SSD-1B and Vega U-Net contains only one middle block and fewer attention layers in its up- and down-blocks, resulting in significantly smaller file sizes. Using these models can reduce inference time by more than 33%. For more details, refer to Segmind's paper: https://arxiv.org/abs/2401.02677v1.
+Similarly, SD1.x- and SD2.x-style models with a tiny U-Net consist of only 6 U-Net blocks, leading to very small files and time savings of up to 50%. For more information, see the paper: https://arxiv.org/pdf/2305.15798.pdf.
+
+## SSD1B
+
+Note that not all of these models follow the standard parameter naming conventions. However, several useful SSD-1B models are available online, such as:
+
+ * https://huggingface.co/segmind/SSD-1B/resolve/main/SSD-1B-A1111.safetensors
+ * https://huggingface.co/hassenhamdi/SSD-1B-fp8_e4m3fn/resolve/main/SSD-1B_fp8_e4m3fn.safetensors
+
+Useful LoRAs are also available:
+
+ * https://huggingface.co/seungminh/lora-swarovski-SSD-1B/resolve/main/pytorch_lora_weights.safetensors
+ * https://huggingface.co/kylielee505/mylcmlorassd/resolve/main/pytorch_lora_weights.safetensors
+
+## Vega
+
+Segmind's Vega model is available online here:
+
+ * https://huggingface.co/segmind/Segmind-Vega/resolve/main/segmind-vega.safetensors
+ 
+VegaRT is an example for an LCM-LoRA:
+
+ * https://huggingface.co/segmind/Segmind-VegaRT/resolve/main/pytorch_lora_weights.safetensors
+
+Both files can be used out-of-the-box, unlike the models described in next sections.
+
+
+## SD1.x, SD2.x with tiny U-Nets
+
+These models require conversion before use. You will need a Python script provided by the diffusers team, available on GitHub:
+
+ * https://raw.githubusercontent.com/huggingface/diffusers/refs/heads/main/scripts/convert_diffusers_to_original_stable_diffusion.py
+
+### SD2.x
+
+NotaAI provides the following model online:
+
+* https://huggingface.co/nota-ai/bk-sdm-v2-tiny
+
+Creating a .safetensors file involves two steps. First, run this short Python script to download the model from Hugging Face:
+
+```python
+from diffusers import StableDiffusionPipeline
+pipe = StableDiffusionPipeline.from_pretrained("nota-ai/bk-sdm-v2-tiny",cache_dir="./")
+```
+
+Second, create the .safetensors file by running:
+
+```bash
+python convert_diffusers_to_original_stable_diffusion.py \
+      --model_path  models--nota-ai--bk-sdm-v2-tiny/snapshots/68277af553777858cd47e133f92e4db47321bc74 \
+      --checkpoint_path bk-sdm-v2-tiny.safetensors --half --use_safetensors
+```
+
+This will generate the **file bk-sdm-v2-tiny.safetensors**, which is now ready for use with sd.cpp.
+
+### SD1.x
+
+Several Tiny SD 1.x models are available online, such as:
+
+ * https://huggingface.co/segmind/tiny-sd
+ * https://huggingface.co/segmind/portrait-finetuned
+ * https://huggingface.co/nota-ai/bk-sdm-tiny
+
+These models also require conversion, partly because some tensors are stored in a non-contiguous manner. To create a usable checkpoint file, follow these simple steps:
+Download and prepare the model using Python: 
+
+##### Download the model using Python on your computer, for example this way:
+
+```python
+import torch
+from diffusers import StableDiffusionPipeline
+pipe = StableDiffusionPipeline.from_pretrained("segmind/tiny-sd")
+unet=pipe.unet
+for param in unet.parameters():
+    param.data = param.data.contiguous()     # <- important here
+pipe.save_pretrained("segmindtiny-sd", safe_serialization=True)
+```
+
+##### Run the conversion script:
+
+```bash
+python convert_diffusers_to_original_stable_diffusion.py \
+      --model_path  ./segmindtiny-sd \
+      --checkpoint_path ./segmind_tiny-sd.safetensors  --half --use_safetensors
+```
+
+The file segmind_tiny-sd.safetensors will be generated and is now ready for use with sd.cpp. You can follow a similar process for the other models mentioned above.
+
+
+### SDXS-512-DreamShaper
+
+Another very tiny and **incredibly fast**  model is SDXS by IDKiro et al.  The authors refer to it as *"Real-Time One-Step Latent Diffusion Models with Image Conditions"*. For details read the paper: https://arxiv.org/pdf/2403.16627 . Once again the authors removed some more blocks of U-Net part and unlike other SD1 models they use an adjusted _AutoEncoderTiny_ instead of default _AutoEncoderKL_ for the VAE part.
+##### Some ready-to-run SDXS-512 model files are available online, such as:
+
+* https://huggingface.co/akleine/sdxs-512
+* https://huggingface.co/concedo/sdxs-512-tinySDdistilled-GGUF
+
+##### Run the model as follows:
+```bash
+~/stable-diffusion.cpp/build/bin/sd-cli -m sdxs.safetensors -p "portrait of a lovely cat" \
+  --cfg-scale 1 --steps 1
+```
+Both options: ``` --cfg-scale 1 ``` and  ``` --steps 1 ``` are mandatory here.
+
+### SDXS-512-0.9
+
+Even though the name "SDXS-512-0.9" is similar to "SDXS-512-DreamShaper", it is *completely different* but also **incredibly fast**. Sometimes it is preferred, so try it yourself.
+##### Download a ready-to-run file from here:
+
+* https://huggingface.co/akleine/sdxs-09
+
+For the use of this model, both options ``` --cfg-scale 1 ``` and ``` --steps 1 ``` are again absolutely necessary.

docs/docker.md

@@ -1,15 +1,39 @@
-## Docker
+# Docker
 
-### Building using Docker
+## Run CLI
+
+```shell
+docker run --rm -v /path/to/models:/models -v /path/to/output/:/output ghcr.io/leejet/stable-diffusion.cpp:master [args...]
+# For example
+# docker run --rm -v ./models:/models -v ./build:/output ghcr.io/leejet/stable-diffusion.cpp:master -m /models/sd-v1-4.ckpt -p "a lovely cat" -v -o /output/output.png
+```
+
+## Run server
+
+```shell
+docker run --rm --init -v /path/to/models:/models -v /path/to/output/:/output -p "1234:1234" --entrypoint "/sd-server" ghcr.io/leejet/stable-diffusion.cpp:master [args...]
+# For example
+# docker run --rm --init -v ./models:/models -v ./build:/output -p "1234:1234" --entrypoint "/sd-server" ghcr.io/leejet/stable-diffusion.cpp:master -m /models/sd-v1-4.ckpt -p "a lovely cat" -v -o /output/output.png
+```
+
+## Building using Docker
 
 ```shell
 docker build -t sd .
 ```
 
-### Run
+## Building variants using Docker
+
+Vulkan:
 
 ```shell
-docker run -v /path/to/models:/models -v /path/to/output/:/output sd [args...]
+docker build -f Dockerfile.vulkan -t sd .
+```
+
+## Run locally built image's CLI
+
+```shell
+docker run --rm -v /path/to/models:/models -v /path/to/output/:/output sd [args...]
 # For example
-# docker run -v ./models:/models -v ./build:/output sd -m /models/sd-v1-4.ckpt -p "a lovely cat" -v -o /output/output.png
-```
+# docker run --rm -v ./models:/models -v ./build:/output sd -m /models/sd-v1-4.ckpt -p "a lovely cat" -v -o /output/output.png
+```

docs/ernie_image.md

@@ -0,0 +1,35 @@
+# How to Use
+
+You can run ERNIE-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or even less.
+
+## Download weights
+
+- Download ERNIE-Image-Turbo
+    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/diffusion_models
+    - gguf: https://huggingface.co/unsloth/ERNIE-Image-Turbo-GGUF/tree/main
+- Download ERNIE-Image
+    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/diffusion_models
+    - gguf: https://huggingface.co/unsloth/ERNIE-Image-GGUF/tree/main
+- Download vae
+    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/vae
+- Download ministral 3b
+    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/text_encoders
+    - gguf: https://huggingface.co/unsloth/Ministral-3-3B-Instruct-2512-GGUF/tree/main
+
+## Examples
+
+### ERNIE-Image-Turbo
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\ernie-image-turbo.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 8 -v --offload-to-cpu --diffusion-fa
+```
+
+<img width="256" alt="ERNIE-Image Turbo example" src="../assets/ernie_image/turbo_example.png" />
+
+### ERNIE-Image
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\ernie-image-UD-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa
+```
+
+<img width="256" alt="ERNIE-Image example" src="../assets/ernie_image/example.png" />

docs/esrgan.md

@@ -1,9 +1,9 @@
 ## Using ESRGAN to upscale results
 
-You can use ESRGAN to upscale the generated images. At the moment, only the [RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth) model is supported. Support for more models of this architecture will be added soon.
+You can use ESRGAN—such as the model [RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth)—to upscale the generated images and improve their overall resolution and clarity.
 
 - Specify the model path using the `--upscale-model PATH` parameter. example:
 
 ```bash
-sd -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat" --upscale-model ../models/RealESRGAN_x4plus_anime_6B.pth
+sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat" --upscale-model ../models/RealESRGAN_x4plus_anime_6B.pth
 ```

docs/flux.md

@@ -15,9 +15,9 @@ You can run Flux using stable-diffusion.cpp with a GPU that has 6GB or even 4GB
 
 You can download the preconverted gguf weights from [FLUX.1-dev-gguf](https://huggingface.co/leejet/FLUX.1-dev-gguf) or [FLUX.1-schnell](https://huggingface.co/leejet/FLUX.1-schnell-gguf), this way you don't have to do the conversion yourself.
 
-Using fp16 will lead to overflow, but ggml's support for bf16 is not yet fully developed. Therefore, we need to convert flux to gguf format here, which also saves VRAM. For example:
+For example:
 ```
-.\bin\Release\sd.exe -M convert -m ..\..\ComfyUI\models\unet\flux1-dev.sft -o ..\models\flux1-dev-q8_0.gguf -v --type q8_0
+.\bin\Release\sd-cli.exe -M convert -m ..\..\ComfyUI\models\unet\flux1-dev.sft -o ..\models\flux1-dev-q8_0.gguf -v --type q8_0
 ```
 
 ## Run
@@ -28,7 +28,7 @@ Using fp16 will lead to overflow, but ggml's support for bf16 is not yet fully d
 For example:
 
 ```
- .\bin\Release\sd.exe --diffusion-model  ..\models\flux1-dev-q8_0.gguf --vae ..\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'flux.cpp'" --cfg-scale 1.0 --sampling-method euler -v --clip-on-cpu
+ .\bin\Release\sd-cli.exe --diffusion-model  ..\models\flux1-dev-q8_0.gguf --vae ..\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'flux.cpp'" --cfg-scale 1.0 --sampling-method euler -v --clip-on-cpu
 ```
 
 Using formats of different precisions will yield results of varying quality.
@@ -44,7 +44,7 @@ Using formats of different precisions will yield results of varying quality.
 
 
 ```
- .\bin\Release\sd.exe --diffusion-model  ..\models\flux1-schnell-q8_0.gguf --vae ..\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'flux.cpp'" --cfg-scale 1.0 --sampling-method euler -v --steps 4 --clip-on-cpu
+ .\bin\Release\sd-cli.exe --diffusion-model  ..\models\flux1-schnell-q8_0.gguf --vae ..\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'flux.cpp'" --cfg-scale 1.0 --sampling-method euler -v --steps 4 --clip-on-cpu
 ```
 
 | q8_0  |
@@ -60,7 +60,7 @@ Since many flux LoRA training libraries have used various LoRA naming formats, i
 - LoRA model from https://huggingface.co/XLabs-AI/flux-lora-collection/tree/main (using comfy converted version!!!)
 
 ```
-.\bin\Release\sd.exe --diffusion-model  ..\models\flux1-dev-q8_0.gguf --vae ...\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'flux.cpp'<lora:realism_lora_comfy_converted:1>" --cfg-scale 1.0 --sampling-method euler -v --lora-model-dir ../models --clip-on-cpu
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\flux1-dev-q8_0.gguf --vae ...\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'flux.cpp'<lora:realism_lora_comfy_converted:1>" --cfg-scale 1.0 --sampling-method euler -v --lora-model-dir ../models --clip-on-cpu
 ```
 
 ![output](../assets/flux/flux1-dev-q8_0%20with%20lora.png)

docs/flux2.md

@@ -0,0 +1,96 @@
+# How to Use
+
+## Flux.2-dev
+
+### Download weights
+
+- Download FLUX.2-dev
+    - gguf: https://huggingface.co/city96/FLUX.2-dev-gguf/tree/main
+- Download vae
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main
+- Download FLUX.2-small-decoder (full_encoder_small_decoder.safetensors) as an alternative VAE option
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-small-decoder/tree/main
+- Download Mistral-Small-3.2-24B-Instruct-2506-GGUF
+    - gguf: https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/tree/main
+
+### Examples
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux2-dev-Q4_K_S.gguf --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf -r .\kontext_input.png -p "change 'flux.cpp' to 'flux2-dev.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu
+```
+
+<img alt="flux2 example" src="../assets/flux2/example.png" />
+
+## Flux.2 klein 4B / Flux.2 klein base 4B
+
+### Download weights
+
+- Download FLUX.2-klein-4B
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-4B
+    - gguf: https://huggingface.co/leejet/FLUX.2-klein-4B-GGUF/tree/main
+- Download FLUX.2-klein-base-4B
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4B
+    - gguf: https://huggingface.co/leejet/FLUX.2-klein-base-4B-GGUF/tree/main
+- Download vae
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main
+- Download FLUX.2-small-decoder (full_encoder_small_decoder.safetensors) as an alternative VAE option
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-small-decoder/tree/main
+- Download Qwen3 4b
+    - safetensors: https://huggingface.co/Comfy-Org/flux2-klein-4B/tree/main/split_files/text_encoders
+    - gguf: https://huggingface.co/unsloth/Qwen3-4B-GGUF/tree/main
+
+### Examples
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu --diffusion-fa
+```
+
+<img alt="flux2-klein-4b" src="../assets/flux2/flux2-klein-4b.png" />
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_4b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu --steps 4
+```
+
+<img alt="flux2-klein-4b-edit" src="../assets/flux2/flux2-klein-4b-edit.png" />
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-base-4b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu --diffusion-fa
+```
+
+<img alt="flux2-klein-base-4b" src="../assets/flux2/flux2-klein-base-4b.png" />
+
+## Flux.2 klein 9B / Flux.2 klein base 9B
+
+### Download weights
+
+- Download FLUX.2-klein-9B
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-9B
+    - gguf: https://huggingface.co/leejet/FLUX.2-klein-9B-GGUF/tree/main
+- Download FLUX.2-klein-base-9B
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B
+    - gguf: https://huggingface.co/leejet/FLUX.2-klein-base-9B-GGUF/tree/main
+- Download vae
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main
+- Download Qwen3 8B
+    - safetensors: https://huggingface.co/Comfy-Org/flux2-klein-9B/tree/main/split_files/text_encoders
+    - gguf: https://huggingface.co/unsloth/Qwen3-8B-GGUF/tree/main
+
+### Examples
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu --diffusion-fa
+```
+
+<img alt="flux2-klein-9b" src="../assets/flux2/flux2-klein-9b.png" />
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_8b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu --steps 4
+```
+
+<img alt="flux2-klein-9b-edit" src="../assets/flux2/flux2-klein-9b-edit.png" />
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-base-9b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu --diffusion-fa
+```
+
+<img alt="flux2-klein-base-9b" src="../assets/flux2/flux2-klein-base-9b.png" />

docs/hipBLAS_on_Windows.md

@@ -82,4 +82,4 @@ cmake .. -G "Ninja" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_H
 cmake --build . --config Release
 ```
 
-If everything went OK, `build\bin\sd.exe` file should appear.
+If everything went OK, `build\bin\sd-cli.exe` file should appear.

docs/kontext.md

@@ -16,7 +16,7 @@ You can run Kontext using stable-diffusion.cpp with a GPU that has 6GB or even 4
 You can download the preconverted gguf weights from [FLUX.1-Kontext-dev-GGUF](https://huggingface.co/QuantStack/FLUX.1-Kontext-dev-GGUF), this way you don't have to do the conversion yourself.
 
 ```
-.\bin\Release\sd.exe -M convert -m ..\..\ComfyUI\models\unet\flux1-kontext-dev.safetensors -o ..\models\flux1-kontext-dev-q8_0.gguf -v --type q8_0
+.\bin\Release\sd-cli.exe -M convert -m ..\..\ComfyUI\models\unet\flux1-kontext-dev.safetensors -o ..\models\flux1-kontext-dev-q8_0.gguf -v --type q8_0
 ```
 
 ## Run
@@ -27,7 +27,7 @@ You can download the preconverted gguf weights from [FLUX.1-Kontext-dev-GGUF](ht
 For example:
 
 ```
- .\bin\Release\sd.exe -r .\flux1-dev-q8_0.png --diffusion-model  ..\models\flux1-kontext-dev-q8_0.gguf --vae ..\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors -p "change 'flux.cpp' to 'kontext.cpp'" --cfg-scale 1.0 --sampling-method euler -v --clip-on-cpu
+ .\bin\Release\sd-cli.exe -r .\flux1-dev-q8_0.png --diffusion-model  ..\models\flux1-kontext-dev-q8_0.gguf --vae ..\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors -p "change 'flux.cpp' to 'kontext.cpp'" --cfg-scale 1.0 --sampling-method euler -v --clip-on-cpu
 ```
 
 

docs/lcm.md

@@ -7,7 +7,7 @@
 Here's a simple example:
 
 ```
-./bin/sd -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat<lora:lcm-lora-sdv1-5:1>" --steps 4 --lora-model-dir ../models -v --cfg-scale 1
+./bin/sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat<lora:lcm-lora-sdv1-5:1>" --steps 4 --lora-model-dir ../models -v --cfg-scale 1
 ```
 
 | without LCM-LoRA (--cfg-scale 7)  | with LCM-LoRA (--cfg-scale 1)  |

docs/lora.md

@@ -7,43 +7,20 @@
 Here's a simple example:
 
 ```
-./bin/sd -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat<lora:marblesh:1>" --lora-model-dir ../models
+./bin/sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat<lora:marblesh:1>" --lora-model-dir ../models
 ```
 
 `../models/marblesh.safetensors` or `../models/marblesh.ckpt` will be applied to the model
 
-# Support matrix
+# Lora Apply Mode
 
-> ℹ️ CUDA `get_rows` support is defined here:  
-> [ggml-org/ggml/src/ggml-cuda/getrows.cu#L156](https://github.com/ggml-org/ggml/blob/7dee1d6a1e7611f238d09be96738388da97c88ed/src/ggml-cuda/getrows.cu#L156)  
-> Currently only the basic types + Q4/Q5/Q8 are implemented. K-quants are **not** supported.
+There are two ways to apply LoRA: **immediately** and **at_runtime**. You can specify it using the `--lora-apply-mode` parameter.
 
-NOTE: The other backends may have different support.
+By default, the mode is selected automatically:
+
+* If the model weights contain any quantized parameters, the **at_runtime** mode is used;
+* Otherwise, the **immediately** mode is used.
+
+The **immediately** mode may have precision and compatibility issues with quantized parameters, but it usually offers faster inference speed and, in some cases, lower memory usage.
+In contrast, the **at_runtime** mode provides better compatibility and higher precision, but inference may be slower and memory usage may be higher in some cases.
 
-| Quant / Type | CUDA | Vulkan |
-|--------------|------|--------|
-| F32          | ✔️   | ✔️   |
-| F16          | ✔️   | ✔️   |
-| BF16         | ✔️   | ✔️   |
-| I32          | ✔️   | ❌   |
-| Q4_0         | ✔️   | ✔️   |
-| Q4_1         | ✔️   | ✔️   |
-| Q5_0         | ✔️   | ✔️   |
-| Q5_1         | ✔️   | ✔️   |
-| Q8_0         | ✔️   | ✔️   |
-| Q2_K         | ❌   | ❌   |
-| Q3_K         | ❌   | ❌   |
-| Q4_K         | ❌   | ❌   |
-| Q5_K         | ❌   | ❌   |
-| Q6_K         | ❌   | ❌   |
-| Q8_K         | ❌   | ❌   |
-| IQ1_S        | ❌   | ✔️   |
-| IQ1_M        | ❌   | ✔️   |
-| IQ2_XXS      | ❌   | ✔️   |
-| IQ2_XS       | ❌   | ✔️   |
-| IQ2_S        | ❌   | ✔️   |
-| IQ3_XXS      | ❌   | ✔️   |
-| IQ3_S        | ❌   | ✔️   |
-| IQ4_XS       | ❌   | ✔️   |
-| IQ4_NL       | ❌   | ✔️   |
-| MXFP4        | ❌   | ✔️   |

docs/ovis_image.md

@@ -0,0 +1,19 @@
+# How to Use
+
+## Download weights
+
+- Download Ovis-Image-7B
+    - safetensors: https://huggingface.co/Comfy-Org/Ovis-Image/tree/main/split_files/diffusion_models
+    - gguf: https://huggingface.co/leejet/Ovis-Image-7B-GGUF
+- Download vae
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.1-schnell/tree/main
+- Download Ovis 2.5
+    - safetensors: https://huggingface.co/Comfy-Org/Ovis-Image/tree/main/split_files/text_encoders
+
+## Examples
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ovis_image-Q4_0.gguf --vae ..\..\ComfyUI\models\vae\ae.sft  --llm ..\..\ComfyUI\models\text_encoders\ovis_2.5.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa
+```
+
+<img alt="ovis image example" src="../assets/ovis_image/example.png" />

docs/photo_maker.md

@@ -27,7 +27,7 @@ If on low memory GPUs (<= 8GB), recommend running with ```--vae-on-cpu``` option
 Example:
 
 ```bash
-bin/sd -m ../models/sdxlUnstableDiffusers_v11.safetensors  --vae ../models/sdxl_vae.safetensors --photo-maker ../models/photomaker-v1.safetensors --pm-id-images-dir ../assets/photomaker_examples/scarletthead_woman -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" --cfg-scale 5.0  --sampling-method euler -H 1024 -W 1024 --pm-style-strength 10 --vae-on-cpu --steps 50
+bin/sd-cli -m ../models/sdxlUnstableDiffusers_v11.safetensors  --vae ../models/sdxl_vae.safetensors --photo-maker ../models/photomaker-v1.safetensors --pm-id-images-dir ../assets/photomaker_examples/scarletthead_woman -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" --cfg-scale 5.0  --sampling-method euler -H 1024 -W 1024 --pm-style-strength 10 --vae-on-cpu --steps 50
 ```
 
 ## PhotoMaker Version 2
@@ -40,7 +40,7 @@ Running PMV2 is now a two-step process:
 ```
 python face_detect.py input_image_dir
 ```
-An ```id_embeds.safetensors``` file will be generated in ```input_images_dir```
+An ```id_embeds.bin``` file will be generated in ```input_images_dir```
 
 **Note: this step is only needed to run once; the same ```id_embeds``` can be reused**
 
@@ -48,6 +48,6 @@ An ```id_embeds.safetensors``` file will be generated in ```input_images_dir```
 
   You can download ```photomaker-v2.safetensors``` from [here](https://huggingface.co/bssrdf/PhotoMakerV2)
 
-- All the command line parameters from Version 1 remain the same for Version 2
+- All the command line parameters from Version 1 remain the same for Version 2 plus one extra pointing to a valid ```id_embeds``` file:  --pm-id-embed-path [path_to__id_embeds.bin] 
 
 

docs/quantization_and_gguf.md

@@ -23,5 +23,5 @@ You can also convert weights in the formats `ckpt/safetensors/diffusers` to gguf
 For example:
 
 ```sh
-./bin/sd -M convert -m ../models/v1-5-pruned-emaonly.safetensors -o  ../models/v1-5-pruned-emaonly.q8_0.gguf -v --type q8_0
+./bin/sd-cli -M convert -m ../models/v1-5-pruned-emaonly.safetensors -o  ../models/v1-5-pruned-emaonly.q8_0.gguf -v --type q8_0
 ```

docs/qwen_image.md

@@ -14,7 +14,7 @@
 ## Examples
 
 ```
-.\bin\Release\sd.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\qwen-image-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --qwen2vl ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf  -p '一个穿着"QWEN"标志的T恤的中国美女正拿着黑色的马克笔面相镜头微笑。她身后的玻璃板上手写体写着 “一、Qwen-Image的技术路线： 探索视觉生成基础模型的极限，开创理解与生成一体化的未来。二、Qwen-Image的模型特色：1、复杂文字渲染。支持中英渲染、自动布局； 2、精准图像编辑。支持文字编辑、物体增减、风格变换。三、Qwen-Image的未来愿景：赋能专业内容创作、助力生成式AI发展。”' --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu -H 1024 -W 1024 --diffusion-fa --flow-shift 3
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\qwen-image-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf  -p '一个穿着"QWEN"标志的T恤的中国美女正拿着黑色的马克笔面相镜头微笑。她身后的玻璃板上手写体写着 “一、Qwen-Image的技术路线： 探索视觉生成基础模型的极限，开创理解与生成一体化的未来。二、Qwen-Image的模型特色：1、复杂文字渲染。支持中英渲染、自动布局； 2、精准图像编辑。支持文字编辑、物体增减、风格变换。三、Qwen-Image的未来愿景：赋能专业内容创作、助力生成式AI发展。”' --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu -H 1024 -W 1024 --diffusion-fa --flow-shift 3
 ```
 
 <img alt="qwen example" src="../assets/qwen/example.png" />

docs/qwen_image_edit.md

@@ -9,6 +9,9 @@
     - Qwen Image Edit 2509
         - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/tree/main/split_files/diffusion_models
         - gguf: https://huggingface.co/QuantStack/Qwen-Image-Edit-2509-GGUF/tree/main
+    - Qwen Image Edit 2511
+        - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/tree/main/split_files/diffusion_models
+        - gguf: https://huggingface.co/unsloth/Qwen-Image-Edit-2511-GGUF/tree/main
 - Download vae
     - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/vae
 - Download qwen_2.5_vl 7b
@@ -20,7 +23,7 @@
 ### Qwen Image Edit
 
 ```
-.\bin\Release\sd.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Qwen_Image_Edit-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --qwen2vl ..\..\ComfyUI\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --diffusion-fa --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'" --seed 1118877715456453
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Qwen_Image_Edit-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --diffusion-fa --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'" --seed 1118877715456453
 ```
 
 <img alt="qwen_image_edit" src="../assets/qwen/qwen_image_edit.png" />
@@ -29,7 +32,17 @@
 ### Qwen Image Edit 2509
 
 ```
-.\bin\Release\sd.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Qwen-Image-Edit-2509-Q4_K_S.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --qwen2vl ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf --qwen2vl_vision ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct.mmproj-Q8_0.gguf --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --diffusion-fa --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'Qwen Image Edit 2509'"
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Qwen-Image-Edit-2509-Q4_K_S.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf --llm_vision ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct.mmproj-Q8_0.gguf --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --diffusion-fa --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'Qwen Image Edit 2509'"
 ```
 
-<img alt="qwen_image_edit_2509" src="../assets/qwen/qwen_image_edit_2509.png" />
+<img alt="qwen_image_edit_2509" src="../assets/qwen/qwen_image_edit_2509.png" />
+
+### Qwen Image Edit 2511
+
+To use the new Qwen Image Edit 2511 mode, the  `--qwen-image-zero-cond-t` flag must be enabled; otherwise, image editing quality will degrade significantly.
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\qwen-image-edit-2511-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --diffusion-fa --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'"  --qwen-image-zero-cond-t
+```
+
+<img alt="qwen_image_edit_2509" src="../assets/qwen/qwen_image_edit_2511.png" />

docs/sd.md

@@ -9,12 +9,12 @@
 ### txt2img example
 
 ```sh
-./bin/sd -m ../models/sd-v1-4.ckpt -p "a lovely cat"
-# ./bin/sd -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat"
-# ./bin/sd -m ../models/sd_xl_base_1.0.safetensors --vae ../models/sdxl_vae-fp16-fix.safetensors -H 1024 -W 1024 -p "a lovely cat" -v
-# ./bin/sd -m ../models/sd3_medium_incl_clips_t5xxlfp16.safetensors -H 1024 -W 1024 -p 'a lovely cat holding a sign says \"Stable Diffusion CPP\"' --cfg-scale 4.5 --sampling-method euler -v --clip-on-cpu
-# ./bin/sd --diffusion-model  ../models/flux1-dev-q3_k.gguf --vae ../models/ae.sft --clip_l ../models/clip_l.safetensors --t5xxl ../models/t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'flux.cpp'" --cfg-scale 1.0 --sampling-method euler -v --clip-on-cpu
-# ./bin/sd -m  ..\models\sd3.5_large.safetensors --clip_l ..\models\clip_l.safetensors --clip_g ..\models\clip_g.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors  -H 1024 -W 1024 -p 'a lovely cat holding a sign says \"Stable diffusion 3.5 Large\"' --cfg-scale 4.5 --sampling-method euler -v --clip-on-cpu
+./bin/sd-cli -m ../models/sd-v1-4.ckpt -p "a lovely cat"
+# ./bin/sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat"
+# ./bin/sd-cli -m ../models/sd_xl_base_1.0.safetensors --vae ../models/sdxl_vae-fp16-fix.safetensors -H 1024 -W 1024 -p "a lovely cat" -v
+# ./bin/sd-cli -m ../models/sd3_medium_incl_clips_t5xxlfp16.safetensors -H 1024 -W 1024 -p 'a lovely cat holding a sign says \"Stable Diffusion CPP\"' --cfg-scale 4.5 --sampling-method euler -v --clip-on-cpu
+# ./bin/sd-cli --diffusion-model  ../models/flux1-dev-q3_k.gguf --vae ../models/ae.sft --clip_l ../models/clip_l.safetensors --t5xxl ../models/t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'flux.cpp'" --cfg-scale 1.0 --sampling-method euler -v --clip-on-cpu
+# ./bin/sd-cli -m  ..\models\sd3.5_large.safetensors --clip_l ..\models\clip_l.safetensors --clip_g ..\models\clip_g.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors  -H 1024 -W 1024 -p 'a lovely cat holding a sign says \"Stable diffusion 3.5 Large\"' --cfg-scale 4.5 --sampling-method euler -v --clip-on-cpu
 ```
 
 Using formats of different precisions will yield results of varying quality.
@@ -29,7 +29,7 @@ Using formats of different precisions will yield results of varying quality.
 
 
 ```
-./bin/sd -m ../models/sd-v1-4.ckpt -p "cat with blue eyes" -i ./output.png -o ./img2img_output.png --strength 0.4
+./bin/sd-cli -m ../models/sd-v1-4.ckpt -p "cat with blue eyes" -i ./output.png -o ./img2img_output.png --strength 0.4
 ```
 
 <p align="center">

docs/sd3.md

@@ -14,7 +14,7 @@
 For example:
 
 ```
-.\bin\Release\sd.exe -m  ..\models\sd3.5_large.safetensors --clip_l ..\models\clip_l.safetensors --clip_g ..\models\clip_g.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors  -H 1024 -W 1024 -p 'a lovely cat holding a sign says \"Stable diffusion 3.5 Large\"' --cfg-scale 4.5 --sampling-method euler -v --clip-on-cpu
+.\bin\Release\sd-cli.exe -m  ..\models\sd3.5_large.safetensors --clip_l ..\models\clip_l.safetensors --clip_g ..\models\clip_g.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors  -H 1024 -W 1024 -p 'a lovely cat holding a sign says \"Stable diffusion 3.5 Large\"' --cfg-scale 4.5 --sampling-method euler -v --clip-on-cpu
 ```
 
 ![](../assets/sd3.5_large.png)

docs/taesd.md

@@ -7,11 +7,33 @@ You can use TAESD to accelerate the decoding of latent images by following these
 Or curl
 
 ```bash
-curl -L -O https://huggingface.co/madebyollin/taesd/blob/main/diffusion_pytorch_model.safetensors
+curl -L -O https://huggingface.co/madebyollin/taesd/resolve/main/diffusion_pytorch_model.safetensors
 ```
 
 - Specify the model path using the `--taesd PATH` parameter. example:
 
 ```bash
-sd -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat" --taesd ../models/diffusion_pytorch_model.safetensors
-```
+sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat" --taesd ../models/diffusion_pytorch_model.safetensors
+```
+
+### Qwen-Image and wan (TAEHV)
+
+sd.cpp also supports [TAEHV](https://github.com/madebyollin/taehv) (#937), which can be used for Qwen-Image and wan.
+
+- For **Qwen-Image and wan2.1 and wan2.2-A14B**, download the wan2.1 tae [safetensors weights](https://github.com/madebyollin/taehv/blob/main/safetensors/taew2_1.safetensors)
+  
+  Or curl
+  
+  ```bash
+  curl -L -O https://github.com/madebyollin/taehv/raw/refs/heads/main/safetensors/taew2_1.safetensors
+  ```
+
+- For **wan2.2-TI2V-5B**, use the wan2.2 tae [safetensors weights](https://github.com/madebyollin/taehv/blob/main/safetensors/taew2_2.safetensors)
+  
+  Or curl
+  
+  ```bash
+  curl -L -O https://github.com/madebyollin/taehv/raw/refs/heads/main/safetensors/taew2_2.safetensors
+  ```
+
+Then simply replace the `--vae xxx.safetensors` with `--tae xxx.safetensors` in the commands. If it still out of VRAM, add `--vae-conv-direct` to your command though might be slower.

docs/wan.md

@@ -39,6 +39,9 @@
         - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/vae/wan_2.1_vae.safetensors
     - wan_2.2_vae (for Wan2.2 TI2V 5B only)
         - safetensors: https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/blob/main/split_files/vae/wan2.2_vae.safetensors
+
+    > Wan models vae requires really much VRAM! If you do not have enough VRAM, please try tae instead, though the results may be poorer. For tae usage, please refer to [taesd](taesd.md)
+
 - Download umt5_xxl
     - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/text_encoders/umt5_xxl_fp16.safetensors
     - gguf: https://huggingface.co/city96/umt5-xxl-encoder-gguf/tree/main
@@ -52,7 +55,7 @@
 ### Wan2.1 T2V 1.3B
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1_t2v_1.3B_fp16.safetensors --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --video-frames 33 --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1_t2v_1.3B_fp16.safetensors --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --video-frames 33 --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.1_1.3B_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -60,7 +63,7 @@
 ### Wan2.1 T2V 14B
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-t2v-14b-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa  --offload-to-cpu --video-frames 33 --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-t2v-14b-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa  --offload-to-cpu --video-frames 33 --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.1_14B_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -70,7 +73,7 @@
 ### Wan2.1 I2V 14B
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-i2v-14b-480p-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --clip_vision ..\..\ComfyUI\models\clip_vision\clip_vision_h.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --video-frames 33 --offload-to-cpu -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-i2v-14b-480p-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --clip_vision ..\..\ComfyUI\models\clip_vision\clip_vision_h.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --video-frames 33 --offload-to-cpu -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.1_14B_i2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -78,7 +81,7 @@
 ### Wan2.2 T2V A14B
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --video-frames 33 --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --video-frames 33 --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_14B_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -86,7 +89,7 @@
 ### Wan2.2 I2V A14B
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --video-frames 33 --offload-to-cpu -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --video-frames 33 --offload-to-cpu -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_14B_i2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -94,7 +97,7 @@
 ### Wan2.2 T2V A14B T2I
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --flow-shift 3.0
 ```
 
 <img width="832" height="480" alt="Wan2 2_14B_t2i" src="../assets/wan/Wan2.2_14B_t2i.png" />
@@ -102,7 +105,7 @@
 ### Wan2.2 T2V 14B with Lora
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat<lora:wan2.2_t2v_lightx2v_4steps_lora_v1.1_low_noise:1><lora:|high_noise|wan2.2_t2v_lightx2v_4steps_lora_v1.1_high_noise:1>" --cfg-scale 3.5 --sampling-method euler --steps 4 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 4 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --lora-model-dir ..\..\ComfyUI\models\loras --video-frames 33 --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat<lora:wan2.2_t2v_lightx2v_4steps_lora_v1.1_low_noise:1><lora:|high_noise|wan2.2_t2v_lightx2v_4steps_lora_v1.1_high_noise:1>" --cfg-scale 3.5 --sampling-method euler --steps 4 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 4 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --lora-model-dir ..\..\ComfyUI\models\loras --video-frames 33 --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_14B_t2v_lora.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -114,7 +117,7 @@
 #### T2V
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors --vae ..\..\ComfyUI\models\vae\wan2.2_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --offload-to-cpu --video-frames 33 --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors --vae ..\..\ComfyUI\models\vae\wan2.2_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --offload-to-cpu --video-frames 33 --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_5B_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -122,7 +125,7 @@
 #### I2V
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors --vae ..\..\ComfyUI\models\vae\wan2.2_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --offload-to-cpu --video-frames 33 -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors --vae ..\..\ComfyUI\models\vae\wan2.2_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --offload-to-cpu --video-frames 33 -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_5B_i2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -130,7 +133,7 @@
 ### Wan2.1 FLF2V 14B
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-flf2v-14b-720p-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --clip_vision ..\..\ComfyUI\models\clip_vision\clip_vision_h.safetensors -p "glass flower blossom" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --video-frames 33 --offload-to-cpu --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-flf2v-14b-720p-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --clip_vision ..\..\ComfyUI\models\clip_vision\clip_vision_h.safetensors -p "glass flower blossom" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --video-frames 33 --offload-to-cpu --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png --flow-shift 3.0
 ```
 
 
@@ -139,7 +142,7 @@
 ### Wan2.2 FLF2V 14B
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -p "glass flower blossom" -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --video-frames 33 --offload-to-cpu --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -p "glass flower blossom" -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --video-frames 33 --offload-to-cpu --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_14B_flf2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -149,7 +152,7 @@
 #### T2V
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --video-frames 1 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --video-frames 1 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_1.3B_vace_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -158,7 +161,7 @@
 #### R2V
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_1.3B_vace_r2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -169,7 +172,7 @@
 ```
 mkdir post+depth
 ffmpeg -i ..\..\ComfyUI\input\post+depth.mp4 -qscale:v 1 -vf fps=8 post+depth\frame_%04d.jpg
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa -i ..\..\ComfyUI\input\dance_girl.jpg --control-video ./post+depth --video-frames 33 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa -i ..\..\ComfyUI\input\dance_girl.jpg --control-video ./post+depth --video-frames 33 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_1.3B_vace_v2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -179,7 +182,7 @@ ffmpeg -i ..\..\ComfyUI\input\post+depth.mp4 -qscale:v 1 -vf fps=8 post+depth\fr
 #### T2V
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --video-frames 33 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --video-frames 33 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_14B_vace_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -188,7 +191,7 @@ ffmpeg -i ..\..\ComfyUI\input\post+depth.mp4 -qscale:v 1 -vf fps=8 post+depth\fr
 #### R2V
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_14B_vace_r2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -198,7 +201,7 @@ ffmpeg -i ..\..\ComfyUI\input\post+depth.mp4 -qscale:v 1 -vf fps=8 post+depth\fr
 #### V2V
 
 ```
-.\bin\Release\sd.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa -i ..\..\ComfyUI\input\dance_girl.jpg --control-video ./post+depth --video-frames 33 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors --t5xxl ..\..\ComfyUI\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa -i ..\..\ComfyUI\input\dance_girl.jpg --control-video ./post+depth --video-frames 33 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_14B_vace_v2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>

docs/z_image.md

@@ -0,0 +1,41 @@
+# How to Use
+
+You can run Z-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or even less.
+
+## Download weights
+
+- Download Z-Image-Turbo
+    - safetensors: https://huggingface.co/Comfy-Org/z_image_turbo/tree/main/split_files/diffusion_models
+    - gguf: https://huggingface.co/leejet/Z-Image-Turbo-GGUF/tree/main
+- Download Z-Image
+    - safetensors: https://huggingface.co/Comfy-Org/z_image/tree/main/split_files/diffusion_models
+    - gguf: https://huggingface.co/unsloth/Z-Image-GGUF/tree/main
+- Download vae
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.1-schnell/tree/main
+- Download Qwen3 4b
+    - safetensors: https://huggingface.co/Comfy-Org/z_image_turbo/tree/main/split_files/text_encoders
+    - gguf: https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/tree/main
+
+## Examples
+
+### Z-Image-Turbo
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  z_image_turbo-Q3_K.gguf --vae ..\..\ComfyUI\models\vae\ae.sft  --llm ..\..\ComfyUI\models\text_encoders\Qwen3-4B-Instruct-2507-Q4_K_M.gguf -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 1.0 -v --offload-to-cpu --diffusion-fa -H 1024 -W 512
+```
+
+<img width="256" alt="z-image example" src="../assets/z_image/q3_K.png" />
+
+### Z-Image-Base
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\z_image_bf16.safetensors --vae ..\..\ComfyUI\models\vae\ae.sft  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_4b.safetensors -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa -H 1024 -W 512
+```
+
+<img width="256" alt="z-image example" src="../assets/z_image/base_bf16.png" />
+
+## Comparison of Different Quantization Types
+
+| bf16 | q8_0 | q6_K | q5_0 | q4_K | q4_0 | q3_K | q2_K|
+|---|---|---|---|---|---|---|---|
+| <img width="256" alt="bf16" src="../assets/z_image/bf16.png" /> | <img width="256" alt="q8_0" src="../assets/z_image/q8_0.png" /> | <img width="256" alt="q6_K" src="../assets/z_image/q6_K.png" /> | <img width="256" alt="q5_0" src="../assets/z_image/q5_0.png" />  | <img width="256" alt="q4_K" src="../assets/z_image/q4_K.png" /> | <img width="256" alt="q4_0" src="../assets/z_image/q4_0.png" /> | <img width="256" alt="q3_K" src="../assets/z_image/q3_K.png" /> | <img width="256" alt="q2_K" src="../assets/z_image/q2_K.png" /> |

examples/CMakeLists.txt

@@ -1,3 +1,4 @@
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
-add_subdirectory(cli)
+add_subdirectory(cli)
+add_subdirectory(server)

examples/cli/CMakeLists.txt

@@ -1,6 +1,20 @@
-set(TARGET sd)
+set(TARGET sd-cli)
 
-add_executable(${TARGET} main.cpp)
+add_executable(${TARGET}
+    ../common/common.cpp
+    ../common/log.cpp
+    ../common/media_io.cpp
+    image_metadata.cpp
+    main.cpp
+)
 install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE stable-diffusion ${CMAKE_THREAD_LIBS_INIT})
-target_compile_features(${TARGET} PUBLIC c_std_11 cxx_std_17)
+target_link_libraries(${TARGET} PRIVATE stable-diffusion zip ${CMAKE_THREAD_LIBS_INIT})
+if(SD_WEBP)
+    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBP)
+    target_link_libraries(${TARGET} PRIVATE webp libwebpmux)
+endif()
+if(SD_WEBM)
+    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBM)
+    target_link_libraries(${TARGET} PRIVATE webm)
+endif()
+target_compile_features(${TARGET} PUBLIC c_std_11 cxx_std_17)

examples/cli/README.md

@@ -1,110 +1,197 @@
 # Run
 
 ```
-usage: ./bin/sd  [options]
+usage: ./bin/sd-cli  [options]
 
-Options:
+CLI Options:
+  -o, --output <string>         path to write result image to. you can use printf-style %d format specifiers for image
+                                sequences (default: ./output.png) (eg. output_%03d.png). Single-file video outputs
+                                support .avi, .webm, and animated .webp
+  --image <string>              path to the image to inspect (for metadata mode)
+  --metadata-format <string>    metadata output format, one of [text, json] (default: text)
+  --preview-path <string>       path to write preview image to (default: ./preview.png). Multi-frame previews support
+                                .avi, .webm, and animated .webp
+  --preview-interval <int>      interval in denoising steps between consecutive updates of the image preview file
+                                (default is 1, meaning updating at every step)
+  --output-begin-idx <int>      starting index for output image sequence, must be non-negative (default 0 if specified
+                                %d in output path, 1 otherwise)
+  --canny                       apply canny preprocessor (edge detection)
+  --convert-name                convert tensor name (for convert mode)
+  -v, --verbose                 print extra info
+  --color                       colors the logging tags according to level
+  --taesd-preview-only          prevents usage of taesd for decoding the final image. (for use with --preview tae)
+  --preview-noisy               enables previewing noisy inputs of the models rather than the denoised outputs
+  --metadata-raw                include raw hex previews for unparsed metadata payloads
+  --metadata-brief              truncate long metadata text values in text output
+  --metadata-all                include structural/container entries such as IHDR, IDAT, and non-metadata JPEG segments
+  -M, --mode                    run mode, one of [img_gen, vid_gen, upscale, convert, metadata], default: img_gen
+  --preview                     preview method. must be one of the following [none, proj, tae, vae] (default is none)
+  -h, --help                    show this help message and exit
+
+Context Options:
   -m, --model <string>                     path to full model
   --clip_l <string>                        path to the clip-l text encoder
   --clip_g <string>                        path to the clip-g text encoder
   --clip_vision <string>                   path to the clip-vision encoder
   --t5xxl <string>                         path to the t5xxl text encoder
-  --qwen2vl <string>                       path to the qwen2vl text encoder
-  --qwen2vl_vision <string>                path to the qwen2vl vit
+  --llm <string>                           path to the llm text encoder. For example: (qwenvl2.5 for qwen-image,
+                                           mistral-small3.2 for flux2, ...)
+  --llm_vision <string>                    path to the llm vit
+  --qwen2vl <string>                       alias of --llm. Deprecated.
+  --qwen2vl_vision <string>                alias of --llm_vision. Deprecated.
   --diffusion-model <string>               path to the standalone diffusion model
   --high-noise-diffusion-model <string>    path to the standalone high noise diffusion model
   --vae <string>                           path to standalone vae model
   --taesd <string>                         path to taesd. Using Tiny AutoEncoder for fast decoding (low quality)
+  --tae <string>                           alias of --taesd
   --control-net <string>                   path to control net model
   --embd-dir <string>                      embeddings directory
   --lora-model-dir <string>                lora model directory
-  -i, --init-img <string>                  path to the init image
-  --end-img <string>                       path to the end image, required by flf2v
+  --hires-upscalers-dir <string>           highres fix upscaler model directory
   --tensor-type-rules <string>             weight type per tensor pattern (example: "^vae\.=f16,model\.=q8_0")
   --photo-maker <string>                   path to PHOTOMAKER model
-  --pm-id-images-dir <string>              path to PHOTOMAKER input id images dir
-  --pm-id-embed-path <string>              path to PHOTOMAKER v2 id embed
-  --mask <string>                          path to the mask image
-  --control-image <string>                 path to control image, control net
-  --control-video <string>                 path to control video frames, It must be a directory path. The video frames inside should be stored as images in
-                                           lexicographical (character) order. For example, if the control video path is
-                                           `frames`, the directory contain images such as 00.png, 01.png, ... etc.
-  -o, --output <string>                    path to write result image to (default: ./output.png)
+  --upscale-model <string>                 path to esrgan model.
+  -t, --threads <int>                      number of threads to use during computation (default: -1). If threads <= 0,
+                                           then threads will be set to the number of CPU physical cores
+  --chroma-t5-mask-pad <int>               t5 mask pad size of chroma
+  --force-sdxl-vae-conv-scale              force use of conv scale on sdxl vae
+  --offload-to-cpu                         place the weights in RAM to save VRAM, and automatically load them into VRAM
+                                           when needed
+  --mmap                                   whether to memory-map model
+  --control-net-cpu                        keep controlnet in cpu (for low vram)
+  --clip-on-cpu                            keep clip in cpu (for low vram)
+  --vae-on-cpu                             keep vae in cpu (for low vram)
+  --fa                                     use flash attention
+  --diffusion-fa                           use flash attention in the diffusion model only
+  --diffusion-conv-direct                  use ggml_conv2d_direct in the diffusion model
+  --vae-conv-direct                        use ggml_conv2d_direct in the vae model
+  --circular                               enable circular padding for convolutions
+  --circularx                              enable circular RoPE wrapping on x-axis (width) only
+  --circulary                              enable circular RoPE wrapping on y-axis (height) only
+  --chroma-disable-dit-mask                disable dit mask for chroma
+  --qwen-image-zero-cond-t                 enable zero_cond_t for qwen image
+  --chroma-enable-t5-mask                  enable t5 mask for chroma
+  --type                                   weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K,
+                                           q4_K). If not specified, the default is the type of the weight file
+  --rng                                    RNG, one of [std_default, cuda, cpu], default: cuda(sd-webui), cpu(comfyui)
+  --sampler-rng                            sampler RNG, one of [std_default, cuda, cpu]. If not specified, use --rng
+  --prediction                             prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow,
+                                           flux2_flow]
+  --lora-apply-mode                        the way to apply LoRA, one of [auto, immediately, at_runtime], default is
+                                           auto. In auto mode, if the model weights contain any quantized parameters,
+                                           the at_runtime mode will be used; otherwise, immediately will be used.The
+                                           immediately mode may have precision and compatibility issues with quantized
+                                           parameters, but it usually offers faster inference speed and, in some cases,
+                                           lower memory usage. The at_runtime mode, on the other hand, is exactly the
+                                           opposite.
+
+Generation Options:
   -p, --prompt <string>                    the prompt to render
   -n, --negative-prompt <string>           the negative prompt (default: "")
-  --upscale-model <string>                 path to esrgan model.
-  -t, --threads <int>                      number of threads to use during computation (default: -1). If threads <= 0, then threads will be set to the number of
-                                           CPU physical cores
-  --upscale-repeats <int>                  Run the ESRGAN upscaler this many times (default: 1)
+  -i, --init-img <string>                  path to the init image
+  --end-img <string>                       path to the end image, required by flf2v
+  --mask <string>                          path to the mask image
+  --control-image <string>                 path to control image, control net
+  --control-video <string>                 path to control video frames, It must be a directory path. The video frames
+                                           inside should be stored as images in lexicographical (character) order. For
+                                           example, if the control video path is `frames`, the directory contain images
+                                           such as 00.png, 01.png, ... etc.
+  --pm-id-images-dir <string>              path to PHOTOMAKER input id images dir
+  --pm-id-embed-path <string>              path to PHOTOMAKER v2 id embed
+  --hires-upscaler <string>                highres fix upscaler, Lanczos, Nearest, Latent, Latent (nearest), Latent
+                                           (nearest-exact), Latent (antialiased), Latent (bicubic), Latent (bicubic
+                                           antialiased), or a model name under --hires-upscalers-dir (default: Latent)
   -H, --height <int>                       image height, in pixel space (default: 512)
   -W, --width <int>                        image width, in pixel space (default: 512)
   --steps <int>                            number of sample steps (default: 20)
   --high-noise-steps <int>                 (high noise) number of sample steps (default: -1 = auto)
-  --clip-skip <int>                        ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1). <= 0 represents unspecified,
-                                           will be 1 for SD1.x, 2 for SD2.x
+  --clip-skip <int>                        ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer
+                                           (default: -1). <= 0 represents unspecified, will be 1 for SD1.x, 2 for SD2.x
   -b, --batch-count <int>                  batch count
-  --chroma-t5-mask-pad <int>               t5 mask pad size of chroma
   --video-frames <int>                     video frames (default: 1)
   --fps <int>                              fps (default: 24)
-  --timestep-shift <int>                   shift timestep for NitroFusion models (default: 0). recommended N for NitroSD-Realism around 250 and 500 for
-                                           NitroSD-Vibrant
+  --timestep-shift <int>                   shift timestep for NitroFusion models (default: 0). recommended N for
+                                           NitroSD-Realism around 250 and 500 for NitroSD-Vibrant
+  --upscale-repeats <int>                  Run the ESRGAN upscaler this many times (default: 1)
+  --upscale-tile-size <int>                tile size for ESRGAN upscaling (default: 128)
+  --hires-width <int>                      highres fix target width, 0 to use --hires-scale (default: 0)
+  --hires-height <int>                     highres fix target height, 0 to use --hires-scale (default: 0)
+  --hires-steps <int>                      highres fix second pass sample steps, 0 to reuse --steps (default: 0)
+  --hires-upscale-tile-size <int>          highres fix upscaler tile size, reserved for model-backed upscalers (default:
+                                           128)
   --cfg-scale <float>                      unconditional guidance scale: (default: 7.0)
-  --img-cfg-scale <float>                  image guidance scale for inpaint or instruct-pix2pix models: (default: same as --cfg-scale)
+  --img-cfg-scale <float>                  image guidance scale for inpaint or instruct-pix2pix models: (default: same
+                                           as --cfg-scale)
   --guidance <float>                       distilled guidance scale for models with guidance input (default: 3.5)
-  --slg-scale <float>                      skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means disabled, a value of 2.5 is nice for sd3.5
-                                           medium
+  --slg-scale <float>                      skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means
+                                           disabled, a value of 2.5 is nice for sd3.5 medium
   --skip-layer-start <float>               SLG enabling point (default: 0.01)
   --skip-layer-end <float>                 SLG disabling point (default: 0.2)
-  --eta <float>                            eta in DDIM, only for DDIM and TCD (default: 0)
+  --eta <float>                            noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and
+                                           res_2s; 1 for euler_a, er_sde and dpm++2s_a)
+  --flow-shift <float>                     shift value for Flow models like SD3.x or WAN (default: auto)
   --high-noise-cfg-scale <float>           (high noise) unconditional guidance scale: (default: 7.0)
-  --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or instruct-pix2pix models (default: same as --cfg-scale)
-  --high-noise-guidance <float>            (high noise) distilled guidance scale for models with guidance input (default: 3.5)
-  --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default: 0)
+  --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or instruct-pix2pix models
+                                           (default: same as --cfg-scale)
+  --high-noise-guidance <float>            (high noise) distilled guidance scale for models with guidance input
+                                           (default: 3.5)
+  --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default:
+                                           0)
   --high-noise-skip-layer-start <float>    (high noise) SLG enabling point (default: 0.01)
   --high-noise-skip-layer-end <float>      (high noise) SLG disabling point (default: 0.2)
-  --high-noise-eta <float>                 (high noise) eta in DDIM, only for DDIM and TCD (default: 0)
+  --high-noise-eta <float>                 (high noise) noise multiplier (default: 0 for ddim_trailing, tcd,
+                                           res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)
   --strength <float>                       strength for noising/unnoising (default: 0.75)
-  --pm-style-strength <float>
-  --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image
-  --moe-boundary <float>                   timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if `--high-noise-steps` is set to -1
-  --flow-shift <float>                     shift value for Flow models like SD3.x or WAN (default: auto)
+  --pm-style-strength <float>              
+  --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full
+                                           destruction of information in init image
+  --moe-boundary <float>                   timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if
+                                           `--high-noise-steps` is set to -1
   --vace-strength <float>                  wan vace strength
   --vae-tile-overlap <float>               tile overlap for vae tiling, in fraction of tile size (default: 0.5)
-  --vae-tiling                             process vae in tiles to reduce memory usage
-  --force-sdxl-vae-conv-scale              force use of conv scale on sdxl vae
-  --offload-to-cpu                         place the weights in RAM to save VRAM, and automatically load them into VRAM when needed
-  --control-net-cpu                        keep controlnet in cpu (for low vram)
-  --clip-on-cpu                            keep clip in cpu (for low vram)
-  --vae-on-cpu                             keep vae in cpu (for low vram)
-  --diffusion-fa                           use flash attention in the diffusion model
-  --diffusion-conv-direct                  use ggml_conv2d_direct in the diffusion model
-  --vae-conv-direct                        use ggml_conv2d_direct in the vae model
-  --canny                                  apply canny preprocessor (edge detection)
-  -v, --verbose                            print extra info
-  --color                                  colors the logging tags according to level
-  --chroma-disable-dit-mask                disable dit mask for chroma
-  --chroma-enable-t5-mask                  enable t5 mask for chroma
-  --increase-ref-index                     automatically increase the indices of references images based on the order they are listed (starting with 1).
+  --hires-scale <float>                    highres fix scale when target size is not set (default: 2.0)
+  --hires-denoising-strength <float>       highres fix second pass denoising strength (default: 0.7)
+  --increase-ref-index                     automatically increase the indices of references images based on the order
+                                           they are listed (starting with 1).
   --disable-auto-resize-ref-image          disable auto resize of ref images
-  -M, --mode                               run mode, one of [img_gen, vid_gen, upscale, convert], default: img_gen
-  --type                                   weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K, q4_K). If not specified, the default is the
-                                           type of the weight file
-  --rng                                    RNG, one of [std_default, cuda], default: cuda
+  --disable-image-metadata                 do not embed generation metadata on image files
+  --vae-tiling                             process vae in tiles to reduce memory usage
+  --hires                                  enable highres fix
   -s, --seed                               RNG seed (default: 42, use random seed for < 0)
-  --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing,
-                                           tcd] (default: euler for Flux/SD3/Wan, euler_a otherwise)
-  --prediction                             prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow]
-  --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, smoothstep, sgm_uniform, simple], default:
+  --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m,
+                                           dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s,
+                                           er_sde] (default: euler for Flux/SD3/Wan, euler_a otherwise)
+  --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a,
+                                           dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep,
+                                           res_2s, er_sde] default: euler for Flux/SD3/Wan, euler_a otherwise
+  --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits,
+                                           smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent], default:
                                            discrete
+  --sigmas                                 custom sigma values for the sampler, comma-separated (e.g.,
+                                           "14.61,7.8,3.5,0.0").
   --skip-layers                            layers to skip for SLG steps (default: [7,8,9])
-  --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm,
-                                           ddim_trailing, tcd] default: euler for Flux/SD3/Wan, euler_a otherwise
-  --high-noise-scheduler                   (high noise) denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, smoothstep, sgm_uniform,
-                                           simple], default: discrete
   --high-noise-skip-layers                 (high noise) layers to skip for SLG steps (default: [7,8,9])
   -r, --ref-image                          reference image for Flux Kontext models (can be used multiple times)
-  -h, --help                               show this help message and exit
+  --cache-mode                             caching method: 'easycache' (DiT), 'ucache' (UNET),
+                                           'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT
+                                           Chebyshev+Taylor forecasting)
+  --cache-option                           named cache params (key=value format, comma-separated). easycache/ucache:
+                                           threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit:
+                                           Fn=,Bn=,threshold=,warmup=; spectrum: w=,m=,lam=,window=,flex=,warmup=,stop=.
+                                           Examples: "threshold=0.25" or "threshold=1.5,reset=0"
+  --scm-mask                               SCM steps mask for cache-dit: comma-separated 0/1 (e.g.,
+                                           "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache
+  --scm-policy                             SCM policy: 'dynamic' (default) or 'static'
   --vae-tile-size                          tile size for vae tiling, format [X]x[Y] (default: 32x32)
-  --vae-relative-tile-size                 relative tile size for vae tiling, format [X]x[Y], in fraction of image size if < 1, in number of tiles per dim if >=1
-                                           (overrides --vae-tile-size)
-```
+  --vae-relative-tile-size                 relative tile size for vae tiling, format [X]x[Y], in fraction of image size
+                                           if < 1, in number of tiles per dim if >=1 (overrides --vae-tile-size)
+```
+
+Metadata mode inspects PNG/JPEG container metadata without loading any model:
+
+```bash
+./bin/sd-cli -M metadata --image ./output.png
+./bin/sd-cli -M metadata --image ./output.jpg --metadata-format json
+./bin/sd-cli -M metadata --image ./output.png --metadata-raw
+./bin/sd-cli -M metadata --image ./output.png --metadata-all
+```

examples/cli/avi_writer.h

@@ -1,217 +0,0 @@
-#ifndef __AVI_WRITER_H__
-#define __AVI_WRITER_H__
-
-#include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-
-#include "stable-diffusion.h"
-
-#ifndef INCLUDE_STB_IMAGE_WRITE_H
-#include "stb_image_write.h"
-#endif
-
-typedef struct {
-    uint32_t offset;
-    uint32_t size;
-} avi_index_entry;
-
-// Write 32-bit little-endian integer
-void write_u32_le(FILE* f, uint32_t val) {
-    fwrite(&val, 4, 1, f);
-}
-
-// Write 16-bit little-endian integer
-void write_u16_le(FILE* f, uint16_t val) {
-    fwrite(&val, 2, 1, f);
-}
-
-/**
- * Create an MJPG AVI file from an array of sd_image_t images.
- * Images are encoded to JPEG using stb_image_write.
- *
- * @param filename Output AVI file name.
- * @param images Array of input images.
- * @param num_images Number of images in the array.
- * @param fps Frames per second for the video.
- * @param quality JPEG quality (0-100).
- * @return 0 on success, -1 on failure.
- */
-int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality = 90) {
-    if (num_images == 0) {
-        fprintf(stderr, "Error: Image array is empty.\n");
-        return -1;
-    }
-
-    FILE* f = fopen(filename, "wb");
-    if (!f) {
-        perror("Error opening file for writing");
-        return -1;
-    }
-
-    uint32_t width    = images[0].width;
-    uint32_t height   = images[0].height;
-    uint32_t channels = images[0].channel;
-    if (channels != 3 && channels != 4) {
-        fprintf(stderr, "Error: Unsupported channel count: %u\n", channels);
-        fclose(f);
-        return -1;
-    }
-
-    // --- RIFF AVI Header ---
-    fwrite("RIFF", 4, 1, f);
-    long riff_size_pos = ftell(f);
-    write_u32_le(f, 0);  // Placeholder for file size
-    fwrite("AVI ", 4, 1, f);
-
-    // 'hdrl' LIST (header list)
-    fwrite("LIST", 4, 1, f);
-    write_u32_le(f, 4 + 8 + 56 + 8 + 4 + 8 + 56 + 8 + 40);
-    fwrite("hdrl", 4, 1, f);
-
-    // 'avih' chunk (AVI main header)
-    fwrite("avih", 4, 1, f);
-    write_u32_le(f, 56);
-    write_u32_le(f, 1000000 / fps);       // Microseconds per frame
-    write_u32_le(f, 0);                   // Max bytes per second
-    write_u32_le(f, 0);                   // Padding granularity
-    write_u32_le(f, 0x110);               // Flags (HASINDEX | ISINTERLEAVED)
-    write_u32_le(f, num_images);          // Total frames
-    write_u32_le(f, 0);                   // Initial frames
-    write_u32_le(f, 1);                   // Number of streams
-    write_u32_le(f, width * height * 3);  // Suggested buffer size
-    write_u32_le(f, width);
-    write_u32_le(f, height);
-    write_u32_le(f, 0);  // Reserved
-    write_u32_le(f, 0);  // Reserved
-    write_u32_le(f, 0);  // Reserved
-    write_u32_le(f, 0);  // Reserved
-
-    // 'strl' LIST (stream list)
-    fwrite("LIST", 4, 1, f);
-    write_u32_le(f, 4 + 8 + 56 + 8 + 40);
-    fwrite("strl", 4, 1, f);
-
-    // 'strh' chunk (stream header)
-    fwrite("strh", 4, 1, f);
-    write_u32_le(f, 56);
-    fwrite("vids", 4, 1, f);              // Stream type: video
-    fwrite("MJPG", 4, 1, f);              // Codec: Motion JPEG
-    write_u32_le(f, 0);                   // Flags
-    write_u16_le(f, 0);                   // Priority
-    write_u16_le(f, 0);                   // Language
-    write_u32_le(f, 0);                   // Initial frames
-    write_u32_le(f, 1);                   // Scale
-    write_u32_le(f, fps);                 // Rate
-    write_u32_le(f, 0);                   // Start
-    write_u32_le(f, num_images);          // Length
-    write_u32_le(f, width * height * 3);  // Suggested buffer size
-    write_u32_le(f, (uint32_t)-1);        // Quality
-    write_u32_le(f, 0);                   // Sample size
-    write_u16_le(f, 0);                   // rcFrame.left
-    write_u16_le(f, 0);                   // rcFrame.top
-    write_u16_le(f, 0);                   // rcFrame.right
-    write_u16_le(f, 0);                   // rcFrame.bottom
-
-    // 'strf' chunk (stream format: BITMAPINFOHEADER)
-    fwrite("strf", 4, 1, f);
-    write_u32_le(f, 40);
-    write_u32_le(f, 40);  // biSize
-    write_u32_le(f, width);
-    write_u32_le(f, height);
-    write_u16_le(f, 1);                   // biPlanes
-    write_u16_le(f, 24);                  // biBitCount
-    fwrite("MJPG", 4, 1, f);              // biCompression (FOURCC)
-    write_u32_le(f, width * height * 3);  // biSizeImage
-    write_u32_le(f, 0);                   // XPelsPerMeter
-    write_u32_le(f, 0);                   // YPelsPerMeter
-    write_u32_le(f, 0);                   // Colors used
-    write_u32_le(f, 0);                   // Colors important
-
-    // 'movi' LIST (video frames)
-    // long movi_list_pos = ftell(f);
-    fwrite("LIST", 4, 1, f);
-    long movi_size_pos = ftell(f);
-    write_u32_le(f, 0);  // Placeholder for movi size
-    fwrite("movi", 4, 1, f);
-
-    avi_index_entry* index = (avi_index_entry*)malloc(sizeof(avi_index_entry) * num_images);
-    if (!index) {
-        fclose(f);
-        return -1;
-    }
-
-    // Encode and write each frame as JPEG
-    struct {
-        uint8_t* buf;
-        size_t size;
-    } jpeg_data;
-
-    for (int i = 0; i < num_images; i++) {
-        jpeg_data.buf  = nullptr;
-        jpeg_data.size = 0;
-
-        // Callback function to collect JPEG data into memory
-        auto write_to_buf = [](void* context, void* data, int size) {
-            auto jd = (decltype(jpeg_data)*)context;
-            jd->buf = (uint8_t*)realloc(jd->buf, jd->size + size);
-            memcpy(jd->buf + jd->size, data, size);
-            jd->size += size;
-        };
-
-        // Encode to JPEG in memory
-        stbi_write_jpg_to_func(
-            write_to_buf,
-            &jpeg_data,
-            images[i].width,
-            images[i].height,
-            channels,
-            images[i].data,
-            quality);
-
-        // Write '00dc' chunk (video frame)
-        fwrite("00dc", 4, 1, f);
-        write_u32_le(f, jpeg_data.size);
-        index[i].offset = ftell(f) - 8;
-        index[i].size   = jpeg_data.size;
-        fwrite(jpeg_data.buf, 1, jpeg_data.size, f);
-
-        // Align to even byte size
-        if (jpeg_data.size % 2)
-            fputc(0, f);
-
-        free(jpeg_data.buf);
-    }
-
-    // Finalize 'movi' size
-    long cur_pos   = ftell(f);
-    long movi_size = cur_pos - movi_size_pos - 4;
-    fseek(f, movi_size_pos, SEEK_SET);
-    write_u32_le(f, movi_size);
-    fseek(f, cur_pos, SEEK_SET);
-
-    // Write 'idx1' index
-    fwrite("idx1", 4, 1, f);
-    write_u32_le(f, num_images * 16);
-    for (int i = 0; i < num_images; i++) {
-        fwrite("00dc", 4, 1, f);
-        write_u32_le(f, 0x10);
-        write_u32_le(f, index[i].offset);
-        write_u32_le(f, index[i].size);
-    }
-
-    // Finalize RIFF size
-    cur_pos        = ftell(f);
-    long file_size = cur_pos - riff_size_pos - 4;
-    fseek(f, riff_size_pos, SEEK_SET);
-    write_u32_le(f, file_size);
-    fseek(f, cur_pos, SEEK_SET);
-
-    fclose(f);
-    free(index);
-
-    return 0;
-}
-
-#endif  // __AVI_WRITER_H__

examples/cli/image_metadata.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <iosfwd>
+#include <string>
+
+enum class MetadataOutputFormat {
+    TEXT,
+    JSON,
+};
+
+struct MetadataReadOptions {
+    MetadataOutputFormat output_format = MetadataOutputFormat::TEXT;
+    bool include_raw                   = false;
+    bool brief                         = false;
+    bool include_structural            = false;
+};
+
+bool print_image_metadata(const std::string& image_path,
+                          const MetadataReadOptions& options,
+                          std::ostream& out,
+                          std::string& error);

examples/common/common.h

@@ -0,0 +1,261 @@
+#ifndef __EXAMPLES_COMMON_COMMON_H__
+#define __EXAMPLES_COMMON_COMMON_H__
+
+#include <cmath>
+#include <cstdint>
+#include <functional>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "log.h"
+#include "resource_owners.hpp"
+#include "stable-diffusion.h"
+
+#define SAFE_STR(s) ((s) ? (s) : "")
+#define BOOL_STR(b) ((b) ? "true" : "false")
+
+extern const char* const modes_str[];
+#define SD_ALL_MODES_STR "img_gen, vid_gen, convert, upscale, metadata"
+
+enum SDMode {
+    IMG_GEN,
+    VID_GEN,
+    CONVERT,
+    UPSCALE,
+    METADATA,
+    MODE_COUNT
+};
+
+struct StringOption {
+    std::string short_name;
+    std::string long_name;
+    std::string desc;
+    std::string* target;
+};
+
+struct IntOption {
+    std::string short_name;
+    std::string long_name;
+    std::string desc;
+    int* target;
+};
+
+struct FloatOption {
+    std::string short_name;
+    std::string long_name;
+    std::string desc;
+    float* target;
+};
+
+struct BoolOption {
+    std::string short_name;
+    std::string long_name;
+    std::string desc;
+    bool keep_true;
+    bool* target;
+};
+
+struct ManualOption {
+    std::string short_name;
+    std::string long_name;
+    std::string desc;
+    std::function<int(int argc, const char** argv, int index)> cb;
+};
+
+struct ArgOptions {
+    std::vector<StringOption> string_options;
+    std::vector<IntOption> int_options;
+    std::vector<FloatOption> float_options;
+    std::vector<BoolOption> bool_options;
+    std::vector<ManualOption> manual_options;
+
+    static std::string wrap_text(const std::string& text, size_t width, size_t indent);
+    void print() const;
+};
+
+bool parse_options(int argc, const char** argv, const std::vector<ArgOptions>& options_list);
+bool decode_base64_image(const std::string& encoded_input,
+                         int target_channels,
+                         int expected_width,
+                         int expected_height,
+                         SDImageOwner& out_image);
+
+struct SDContextParams {
+    int n_threads = -1;
+    std::string model_path;
+    std::string clip_l_path;
+    std::string clip_g_path;
+    std::string clip_vision_path;
+    std::string t5xxl_path;
+    std::string llm_path;
+    std::string llm_vision_path;
+    std::string diffusion_model_path;
+    std::string high_noise_diffusion_model_path;
+    std::string vae_path;
+    std::string taesd_path;
+    std::string esrgan_path;
+    std::string control_net_path;
+    std::string embedding_dir;
+    std::string photo_maker_path;
+    sd_type_t wtype = SD_TYPE_COUNT;
+    std::string tensor_type_rules;
+    std::string lora_model_dir = ".";
+    std::string hires_upscalers_dir;
+
+    std::map<std::string, std::string> embedding_map;
+    std::vector<sd_embedding_t> embedding_vec;
+
+    rng_type_t rng_type         = CUDA_RNG;
+    rng_type_t sampler_rng_type = RNG_TYPE_COUNT;
+    bool offload_params_to_cpu  = false;
+    bool enable_mmap            = false;
+    bool control_net_cpu        = false;
+    bool clip_on_cpu            = false;
+    bool vae_on_cpu             = false;
+    bool flash_attn             = false;
+    bool diffusion_flash_attn   = false;
+    bool diffusion_conv_direct  = false;
+    bool vae_conv_direct        = false;
+
+    bool circular   = false;
+    bool circular_x = false;
+    bool circular_y = false;
+
+    bool chroma_use_dit_mask = true;
+    bool chroma_use_t5_mask  = false;
+    int chroma_t5_mask_pad   = 1;
+
+    bool qwen_image_zero_cond_t = false;
+
+    prediction_t prediction           = PREDICTION_COUNT;
+    lora_apply_mode_t lora_apply_mode = LORA_APPLY_AUTO;
+
+    bool force_sdxl_vae_conv_scale = false;
+
+    float flow_shift = INFINITY;
+    ArgOptions get_options();
+    void build_embedding_map();
+    bool resolve(SDMode mode);
+    bool validate(SDMode mode);
+    bool resolve_and_validate(SDMode mode);
+    std::string to_string() const;
+    sd_ctx_params_t to_sd_ctx_params_t(bool vae_decode_only, bool free_params_immediately, bool taesd_preview);
+};
+
+struct SDGenerationParams {
+    // User-facing input fields.
+    std::string prompt;
+    std::string negative_prompt;
+    int clip_skip              = -1;  // <= 0 represents unspecified
+    int width                  = -1;
+    int height                 = -1;
+    int batch_count            = 1;
+    int64_t seed               = 42;
+    float strength             = 0.75f;
+    float control_strength     = 0.9f;
+    bool auto_resize_ref_image = true;
+    bool increase_ref_index    = false;
+    bool embed_image_metadata  = true;
+
+    std::string init_image_path;
+    std::string end_image_path;
+    std::string mask_image_path;
+    std::string control_image_path;
+    std::vector<std::string> ref_image_paths;
+    std::string control_video_path;
+
+    sd_sample_params_t sample_params;
+    sd_sample_params_t high_noise_sample_params;
+    std::vector<int> skip_layers            = {7, 8, 9};
+    std::vector<int> high_noise_skip_layers = {7, 8, 9};
+
+    std::vector<float> custom_sigmas;
+
+    std::string cache_mode;
+    std::string cache_option;
+    std::string scm_mask;
+    bool scm_policy_dynamic = true;
+    sd_cache_params_t cache_params{};
+
+    float moe_boundary                   = 0.875f;
+    int video_frames                     = 1;
+    int fps                              = 16;
+    float vace_strength                  = 1.f;
+    sd_tiling_params_t vae_tiling_params = {false, 0, 0, 0.5f, 0.0f, 0.0f};
+
+    std::string pm_id_images_dir;
+    std::string pm_id_embed_path;
+    float pm_style_strength = 20.f;
+
+    int upscale_repeats   = 1;
+    int upscale_tile_size = 128;
+
+    bool hires_enabled         = false;
+    std::string hires_upscaler = "Latent";
+    std::string hires_upscaler_model_path;
+    float hires_scale              = 2.f;
+    int hires_width                = 0;
+    int hires_height               = 0;
+    int hires_steps                = 0;
+    float hires_denoising_strength = 0.7f;
+    int hires_upscale_tile_size    = 128;
+
+    std::map<std::string, float> lora_map;
+    std::map<std::string, float> high_noise_lora_map;
+
+    // Derived and normalized fields.
+    std::string prompt_with_lora;  // for metadata record only
+    std::vector<sd_lora_t> lora_vec;
+    sd_hires_upscaler_t resolved_hires_upscaler;
+
+    // Owned execution payload.
+    SDImageOwner init_image;
+    SDImageOwner end_image;
+    std::vector<SDImageOwner> ref_images;
+    SDImageOwner mask_image;
+    SDImageOwner control_image;
+    std::vector<SDImageOwner> pm_id_images;
+    std::vector<SDImageOwner> control_frames;
+
+    // Backing storage for sd_img_gen_params_t view fields.
+    std::vector<sd_image_t> ref_image_views;
+    std::vector<sd_image_t> pm_id_image_views;
+    std::vector<sd_image_t> control_frame_views;
+
+    SDGenerationParams();
+    SDGenerationParams(const SDGenerationParams& other)                = default;
+    SDGenerationParams& operator=(const SDGenerationParams& other)     = default;
+    SDGenerationParams(SDGenerationParams&& other) noexcept            = default;
+    SDGenerationParams& operator=(SDGenerationParams&& other) noexcept = default;
+    ArgOptions get_options();
+    bool from_json_str(const std::string& json_str,
+                       const std::function<std::string(const std::string&)>& lora_path_resolver = {});
+    bool initialize_cache_params();
+    void extract_and_remove_lora(const std::string& lora_model_dir);
+    bool width_and_height_are_set() const;
+    void set_width_and_height_if_unset(int w, int h);
+    int get_resolved_width() const;
+    int get_resolved_height() const;
+    bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false);
+    bool validate(SDMode mode);
+    bool resolve_and_validate(SDMode mode,
+                              const std::string& lora_model_dir,
+                              const std::string& hires_upscalers_dir,
+                              bool strict = false);
+    sd_img_gen_params_t to_sd_img_gen_params_t();
+    sd_vid_gen_params_t to_sd_vid_gen_params_t();
+    std::string to_string() const;
+};
+
+std::string version_string();
+std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
+                                            const SDGenerationParams& gen_params,
+                                            int64_t seed,
+                                            SDMode mode = IMG_GEN);
+std::string get_image_params(const SDContextParams& ctx_params,
+                             const SDGenerationParams& gen_params,
+                             int64_t seed,
+                             SDMode mode = IMG_GEN);
+
+#endif  // __EXAMPLES_COMMON_COMMON_H__

examples/common/log.cpp

@@ -0,0 +1,115 @@
+#include "log.h"
+
+#include <vector>
+
+bool log_verbose = false;
+bool log_color   = false;
+
+std::string sd_basename(const std::string& path) {
+    size_t pos = path.find_last_of('/');
+    if (pos != std::string::npos) {
+        return path.substr(pos + 1);
+    }
+    pos = path.find_last_of('\\');
+    if (pos != std::string::npos) {
+        return path.substr(pos + 1);
+    }
+    return path;
+}
+
+void print_utf8(FILE* stream, const char* utf8) {
+    if (!utf8) {
+        return;
+    }
+
+#ifdef _WIN32
+    HANDLE h = (stream == stderr)
+                   ? GetStdHandle(STD_ERROR_HANDLE)
+                   : GetStdHandle(STD_OUTPUT_HANDLE);
+
+    DWORD mode;
+    BOOL is_console = GetConsoleMode(h, &mode);
+
+    if (is_console) {
+        int wlen = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0);
+        if (wlen <= 0) {
+            return;
+        }
+
+        std::vector<wchar_t> wbuf(static_cast<size_t>(wlen));
+
+        MultiByteToWideChar(CP_UTF8, 0, utf8, -1, wbuf.data(), wlen);
+
+        DWORD written;
+        WriteConsoleW(h, wbuf.data(), wlen - 1, &written, NULL);
+    } else {
+        DWORD written;
+        WriteFile(h, utf8, (DWORD)strlen(utf8), &written, NULL);
+    }
+#else
+    fputs(utf8, stream);
+#endif
+}
+
+void log_print(enum sd_log_level_t level, const char* log, bool verbose, bool color) {
+    int tag_color;
+    const char* level_str;
+    FILE* out_stream = (level == SD_LOG_ERROR) ? stderr : stdout;
+
+    if (!log || (!verbose && level <= SD_LOG_DEBUG)) {
+        return;
+    }
+
+    switch (level) {
+        case SD_LOG_DEBUG:
+            tag_color = 37;
+            level_str = "DEBUG";
+            break;
+        case SD_LOG_INFO:
+            tag_color = 34;
+            level_str = "INFO";
+            break;
+        case SD_LOG_WARN:
+            tag_color = 35;
+            level_str = "WARN";
+            break;
+        case SD_LOG_ERROR:
+            tag_color = 31;
+            level_str = "ERROR";
+            break;
+        default:
+            tag_color = 33;
+            level_str = "?????";
+            break;
+    }
+
+    if (color) {
+        fprintf(out_stream, "\033[%d;1m[%-5s]\033[0m ", tag_color, level_str);
+    } else {
+        fprintf(out_stream, "[%-5s] ", level_str);
+    }
+    print_utf8(out_stream, log);
+    fflush(out_stream);
+}
+
+void example_log_printf(sd_log_level_t level, const char* file, int line, const char* format, ...) {
+    constexpr size_t LOG_BUFFER_SIZE = 4096;
+
+    va_list args;
+    va_start(args, format);
+
+    static char log_buffer[LOG_BUFFER_SIZE + 1];
+    int written = snprintf(log_buffer, LOG_BUFFER_SIZE, "%s:%-4d - ", sd_basename(file).c_str(), line);
+
+    if (written >= 0 && written < static_cast<int>(LOG_BUFFER_SIZE)) {
+        vsnprintf(log_buffer + written, LOG_BUFFER_SIZE - written, format, args);
+    }
+    size_t len = strlen(log_buffer);
+    if (len == 0 || log_buffer[len - 1] != '\n') {
+        strncat(log_buffer, "\n", LOG_BUFFER_SIZE - len);
+    }
+
+    log_print(level, log_buffer, log_verbose, log_color);
+
+    va_end(args);
+}

examples/common/log.h

@@ -0,0 +1,32 @@
+#ifndef __EXAMPLE_LOG_H__
+#define __EXAMPLE_LOG_H__
+
+#include <cstdarg>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+
+#if defined(_WIN32)
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif  // _WIN32
+
+#include "stable-diffusion.h"
+
+extern bool log_verbose;
+extern bool log_color;
+
+std::string sd_basename(const std::string& path);
+void print_utf8(FILE* stream, const char* utf8);
+void log_print(sd_log_level_t level, const char* log, bool verbose, bool color);
+void example_log_printf(sd_log_level_t level, const char* file, int line, const char* format, ...);
+
+#define LOG_DEBUG(format, ...) example_log_printf(SD_LOG_DEBUG, __FILE__, __LINE__, format, ##__VA_ARGS__)
+#define LOG_INFO(format, ...) example_log_printf(SD_LOG_INFO, __FILE__, __LINE__, format, ##__VA_ARGS__)
+#define LOG_WARN(format, ...) example_log_printf(SD_LOG_WARN, __FILE__, __LINE__, format, ##__VA_ARGS__)
+#define LOG_ERROR(format, ...) example_log_printf(SD_LOG_ERROR, __FILE__, __LINE__, format, ##__VA_ARGS__)
+
+#endif  // __EXAMPLE_LOG_H__

examples/common/media_io.h

@@ -0,0 +1,101 @@
+#ifndef __MEDIA_IO_H__
+#define __MEDIA_IO_H__
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "stable-diffusion.h"
+
+enum class EncodedImageFormat {
+    JPEG,
+    PNG,
+    WEBP,
+    UNKNOWN,
+};
+
+EncodedImageFormat encoded_image_format_from_path(const std::string& path);
+
+std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
+                                            const uint8_t* image,
+                                            int width,
+                                            int height,
+                                            int channels,
+                                            const std::string& parameters = "",
+                                            int quality                   = 90);
+
+bool write_image_to_file(const std::string& path,
+                         const uint8_t* image,
+                         int width,
+                         int height,
+                         int channels,
+                         const std::string& parameters = "",
+                         int quality                   = 90);
+
+uint8_t* load_image_from_file(const char* image_path,
+                              int& width,
+                              int& height,
+                              int expected_width   = 0,
+                              int expected_height  = 0,
+                              int expected_channel = 3);
+
+bool load_sd_image_from_file(sd_image_t* image,
+                             const char* image_path,
+                             int expected_width   = 0,
+                             int expected_height  = 0,
+                             int expected_channel = 3);
+
+uint8_t* load_image_from_memory(const char* image_bytes,
+                                int len,
+                                int& width,
+                                int& height,
+                                int expected_width   = 0,
+                                int expected_height  = 0,
+                                int expected_channel = 3);
+
+int create_mjpg_avi_from_sd_images(const char* filename,
+                                   sd_image_t* images,
+                                   int num_images,
+                                   int fps,
+                                   int quality = 90);
+std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images,
+                                                              int num_images,
+                                                              int fps,
+                                                              int quality = 90);
+
+#ifdef SD_USE_WEBP
+int create_animated_webp_from_sd_images(const char* filename,
+                                        sd_image_t* images,
+                                        int num_images,
+                                        int fps,
+                                        int quality = 90);
+std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(sd_image_t* images,
+                                                                   int num_images,
+                                                                   int fps,
+                                                                   int quality = 90);
+#endif
+
+#ifdef SD_USE_WEBM
+int create_webm_from_sd_images(const char* filename,
+                               sd_image_t* images,
+                               int num_images,
+                               int fps,
+                               int quality = 90);
+std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images,
+                                                          int num_images,
+                                                          int fps,
+                                                          int quality = 90);
+#endif
+
+int create_video_from_sd_images(const char* filename,
+                                sd_image_t* images,
+                                int num_images,
+                                int fps,
+                                int quality = 90);
+std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& output_format,
+                                                           sd_image_t* images,
+                                                           int num_images,
+                                                           int fps,
+                                                           int quality = 90);
+
+#endif  // __MEDIA_IO_H__

examples/common/resource_owners.hpp

@@ -0,0 +1,236 @@
+#ifndef __EXAMPLE_RESOURCE_OWNERS_H__
+#define __EXAMPLE_RESOURCE_OWNERS_H__
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "stable-diffusion.h"
+
+struct FreeDeleter {
+    void operator()(void* ptr) const {
+        free(ptr);
+    }
+};
+
+struct FileCloser {
+    void operator()(FILE* file) const {
+        if (file != nullptr) {
+            fclose(file);
+        }
+    }
+};
+
+struct SDCtxDeleter {
+    void operator()(sd_ctx_t* ctx) const {
+        if (ctx != nullptr) {
+            free_sd_ctx(ctx);
+        }
+    }
+};
+
+struct UpscalerCtxDeleter {
+    void operator()(upscaler_ctx_t* ctx) const {
+        if (ctx != nullptr) {
+            free_upscaler_ctx(ctx);
+        }
+    }
+};
+
+template <typename T>
+using FreeUniquePtr = std::unique_ptr<T, FreeDeleter>;
+
+using FilePtr        = std::unique_ptr<FILE, FileCloser>;
+using SDCtxPtr       = std::unique_ptr<sd_ctx_t, SDCtxDeleter>;
+using UpscalerCtxPtr = std::unique_ptr<upscaler_ctx_t, UpscalerCtxDeleter>;
+
+class SDImageOwner {
+private:
+    static sd_image_t copy_image(const sd_image_t& image) {
+        if (image.data == nullptr) {
+            return {image.width, image.height, image.channel, nullptr};
+        }
+
+        const size_t byte_count = static_cast<size_t>(image.width) * image.height * image.channel;
+        uint8_t* raw_copy       = static_cast<uint8_t*>(malloc(byte_count));
+        if (raw_copy == nullptr) {
+            return {0, 0, 0, nullptr};
+        }
+
+        std::memcpy(raw_copy, image.data, byte_count);
+        return {image.width, image.height, image.channel, raw_copy};
+    }
+
+    sd_image_t image_ = {0, 0, 0, nullptr};
+
+public:
+    SDImageOwner() = default;
+    explicit SDImageOwner(sd_image_t image)
+        : image_(image) {
+    }
+
+    SDImageOwner(const SDImageOwner& other)
+        : image_(copy_image(other.image_)) {
+    }
+
+    SDImageOwner& operator=(const SDImageOwner& other) {
+        if (this != &other) {
+            reset(copy_image(other.image_));
+        }
+        return *this;
+    }
+
+    SDImageOwner(SDImageOwner&& other) noexcept
+        : image_(other.release()) {
+    }
+
+    SDImageOwner& operator=(SDImageOwner&& other) noexcept {
+        if (this != &other) {
+            reset();
+            image_ = other.release();
+        }
+        return *this;
+    }
+
+    ~SDImageOwner() {
+        reset();
+    }
+
+    sd_image_t* put() {
+        if (image_.data != nullptr) {
+            free(image_.data);
+            image_.data = nullptr;
+        }
+        image_.width   = 0;
+        image_.height  = 0;
+        image_.channel = 0;
+        return &image_;
+    }
+
+    sd_image_t& get() {
+        return image_;
+    }
+
+    const sd_image_t& get() const {
+        return image_;
+    }
+
+    sd_image_t release() {
+        sd_image_t image = image_;
+        image_           = {0, 0, 0, nullptr};
+        return image;
+    }
+
+    void reset(sd_image_t image = {0, 0, 0, nullptr}) {
+        if (image_.data != nullptr) {
+            free(image_.data);
+        }
+        image_ = image;
+    }
+};
+
+class SDImageVec {
+private:
+    std::vector<sd_image_t> images_;
+
+public:
+    SDImageVec() = default;
+
+    SDImageVec(const SDImageVec&)            = delete;
+    SDImageVec& operator=(const SDImageVec&) = delete;
+
+    SDImageVec(SDImageVec&& other) noexcept
+        : images_(std::move(other.images_)) {
+    }
+
+    SDImageVec& operator=(SDImageVec&& other) noexcept {
+        if (this != &other) {
+            clear();
+            images_ = std::move(other.images_);
+        }
+        return *this;
+    }
+
+    ~SDImageVec() {
+        clear();
+    }
+
+    void push_back(sd_image_t image) {
+        images_.push_back(image);
+    }
+
+    void push_back(SDImageOwner&& image) {
+        images_.push_back(image.release());
+    }
+
+    void reserve(size_t count) {
+        images_.reserve(count);
+    }
+
+    void adopt(sd_image_t* images, int count) {
+        clear();
+        if (images == nullptr || count <= 0) {
+            free(images);
+            return;
+        }
+
+        images_.reserve(static_cast<size_t>(count));
+        for (int i = 0; i < count; ++i) {
+            images_.push_back(images[i]);
+        }
+        free(images);
+    }
+
+    size_t size() const {
+        return images_.size();
+    }
+
+    bool empty() const {
+        return images_.empty();
+    }
+
+    int count() const {
+        return static_cast<int>(images_.size());
+    }
+
+    explicit operator bool() const {
+        return !images_.empty();
+    }
+
+    sd_image_t* data() {
+        return images_.data();
+    }
+
+    const sd_image_t* data() const {
+        return images_.data();
+    }
+
+    sd_image_t& operator[](size_t index) {
+        return images_[index];
+    }
+
+    const sd_image_t& operator[](size_t index) const {
+        return images_[index];
+    }
+
+    std::vector<sd_image_t>& raw() {
+        return images_;
+    }
+
+    const std::vector<sd_image_t>& raw() const {
+        return images_;
+    }
+
+    void clear() {
+        for (sd_image_t& image : images_) {
+            free(image.data);
+            image.data = nullptr;
+        }
+        images_.clear();
+    }
+};
+
+#endif  // __EXAMPLE_RESOURCE_OWNERS_H__

examples/server/CMakeLists.txt

@@ -0,0 +1,104 @@
+set(TARGET sd-server)
+
+option(SD_SERVER_BUILD_FRONTEND "Build server frontend with pnpm" ON)
+
+set(FRONTEND_DIR "${CMAKE_CURRENT_SOURCE_DIR}/frontend")
+set(GENERATED_HTML_HEADER "${FRONTEND_DIR}/dist/gen_index_html.h")
+
+set(HAVE_FRONTEND_BUILD OFF)
+
+if(SD_SERVER_BUILD_FRONTEND AND EXISTS "${FRONTEND_DIR}")
+    if(WIN32)
+        find_program(PNPM_EXECUTABLE NAMES pnpm.cmd pnpm)
+    else()
+        find_program(PNPM_EXECUTABLE NAMES pnpm)
+    endif()
+
+    if(PNPM_EXECUTABLE)
+        message(STATUS "Frontend dir found: ${FRONTEND_DIR}")
+        message(STATUS "pnpm found: ${PNPM_EXECUTABLE}")
+
+        set(HAVE_FRONTEND_BUILD ON)
+
+        add_custom_target(${TARGET}_frontend_install
+            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" install
+            WORKING_DIRECTORY "${FRONTEND_DIR}"
+            COMMENT "Installing frontend dependencies"
+            VERBATIM
+        )
+
+        add_custom_target(${TARGET}_frontend_build
+            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" run build
+            WORKING_DIRECTORY "${FRONTEND_DIR}"
+            COMMENT "Building frontend"
+            VERBATIM
+        )
+
+        add_custom_target(${TARGET}_frontend_header
+            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" run build:header
+            WORKING_DIRECTORY "${FRONTEND_DIR}"
+            COMMENT "Generating gen_index_html.h"
+            VERBATIM
+        )
+
+        add_dependencies(${TARGET}_frontend_build ${TARGET}_frontend_install)
+        add_dependencies(${TARGET}_frontend_header ${TARGET}_frontend_build)
+
+        add_custom_target(${TARGET}_frontend
+            DEPENDS ${TARGET}_frontend_header
+        )
+
+        set_source_files_properties("${GENERATED_HTML_HEADER}" PROPERTIES GENERATED TRUE)
+    else()
+        if(EXISTS "${GENERATED_HTML_HEADER}")
+            message(STATUS "pnpm not found; using pre-built frontend header detected at ${GENERATED_HTML_HEADER}")
+            set(HAVE_FRONTEND_BUILD ON)
+            add_custom_target(${TARGET}_frontend)
+        else()
+            message(WARNING "pnpm not found; frontend build disabled.")
+        endif()
+    endif()
+else()
+    message(STATUS "Frontend disabled or directory not found: ${FRONTEND_DIR}")
+endif()
+
+add_executable(${TARGET}
+    ../common/common.cpp
+    ../common/log.cpp
+    ../common/media_io.cpp
+    main.cpp
+    runtime.cpp
+    async_jobs.cpp
+    routes_index.cpp
+    routes_openai.cpp
+    routes_sdapi.cpp
+    routes_sdcpp.cpp
+)
+
+if(HAVE_FRONTEND_BUILD)
+    add_dependencies(${TARGET} ${TARGET}_frontend)
+    target_sources(${TARGET} PRIVATE "${GENERATED_HTML_HEADER}")
+    target_include_directories(${TARGET} PRIVATE "${FRONTEND_DIR}/dist")
+    target_compile_definitions(${TARGET} PRIVATE HAVE_INDEX_HTML)
+    message(STATUS "HAVE_INDEX_HTML enabled")
+else()
+    message(STATUS "HAVE_INDEX_HTML disabled")
+endif()
+
+install(TARGETS ${TARGET} RUNTIME)
+target_link_libraries(${TARGET} PRIVATE stable-diffusion ${CMAKE_THREAD_LIBS_INIT})
+if(SD_WEBP)
+    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBP)
+    target_link_libraries(${TARGET} PRIVATE webp libwebpmux)
+endif()
+if(SD_WEBM)
+    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBM)
+    target_link_libraries(${TARGET} PRIVATE webm)
+endif()
+
+# due to httplib; it contains a pragma for MSVC, but other things need explicit flags
+if(WIN32 AND NOT MSVC)
+    target_link_libraries(${TARGET} PRIVATE ws2_32)
+endif()
+
+target_compile_features(${TARGET} PUBLIC c_std_11 cxx_std_17)

examples/server/README.md

@@ -0,0 +1,290 @@
+# Example
+
+The following example starts `sd-server` with a standalone diffusion model, VAE, and LLM text encoder:
+
+```
+.\bin\Release\sd-server.exe --diffusion-model  ..\models\diffusion_models\z_image_turbo_bf16.safetensors --vae ..\models\vae\ae.sft  --llm ..\models\text_encoders\qwen_3_4b.safetensors --diffusion-fa --offload-to-cpu -v --cfg-scale 1.0
+```
+
+What this example does:
+
+* `--diffusion-model` selects the standalone diffusion model
+* `--vae` selects the VAE decoder
+* `--llm` selects the text encoder / language model used by this pipeline
+* `--diffusion-fa` enables flash attention in the diffusion model
+* `--offload-to-cpu` reduces VRAM pressure by keeping weights in RAM when possible
+* `-v` enables verbose logging
+* `--cfg-scale 1.0` sets the default CFG scale for generation
+
+After the server starts successfully:
+
+* the web UI is available at `http://127.0.0.1:1234/`
+* the native async API is available under `/sdcpp/v1/...`
+* the compatibility APIs are available under `/v1/...` and `/sdapi/v1/...`
+
+If you want to use a different host or port, pass:
+
+```bash
+--listen-ip <ip> --listen-port <port>
+```
+
+# Frontend
+
+## Build with Frontend
+
+The server can optionally build the web frontend and embed it into the binary as `gen_index_html.h`.
+
+### Requirements
+
+Install the following tools:
+
+* **Node.js** ≥ 20
+  https://nodejs.org/
+
+* **pnpm** ≥ 10
+  Install via npm:
+
+```bash
+npm install -g pnpm
+```
+
+Verify installation:
+
+```bash
+node -v
+pnpm -v
+```
+
+### Install frontend dependencies
+
+Go to the frontend directory and install dependencies:
+
+```bash
+cd examples/server/frontend
+pnpm install
+```
+
+### Build the server with CMake
+
+Enable the frontend build option when configuring CMake:
+
+```bash
+cmake -B build -DSD_SERVER_BUILD_FRONTEND=ON
+cmake --build build --config Release
+```
+
+If `pnpm` is available, the build system will automatically run:
+
+```
+pnpm run build
+pnpm run build:header
+```
+
+and embed the generated frontend into the server binary.
+
+## Frontend Repository
+
+The web frontend is maintained in a **separate repository**, https://github.com/leejet/sdcpp-webui.
+
+If you want to modify the UI or frontend logic, please submit pull requests to the **frontend repository**.
+
+This repository (`stable-diffusion.cpp`) only vendors the frontend periodically. Changes from the frontend repo are synchronized:
+
+* approximately **every 1–2 weeks**, or
+* when there are **major frontend updates**
+
+Because of this, frontend changes will **not appear here immediately** after being merged upstream.
+
+## Using an external frontend
+
+By default, the server uses the **embedded frontend** generated during the build (`gen_index_html.h`).
+
+You can also serve a custom frontend file instead of the embedded one by using:
+
+```bash
+--serve-html-path <path-to-index.html>
+```
+
+For example:
+
+```bash
+sd-server --serve-html-path ./index.html
+```
+
+In this case, the server will load and serve the specified `index.html` file instead of the embedded frontend. This is useful when:
+
+* developing or testing frontend changes
+* using a custom UI
+* avoiding rebuilding the binary after frontend modifications
+
+# Run
+
+```
+usage: ./bin/sd-server  [options]
+
+Svr Options:
+  -l, --listen-ip <string>      server listen ip (default: 127.0.0.1)
+  --serve-html-path <string>    path to HTML file to serve at root (optional)
+  --listen-port <int>           server listen port (default: 1234)
+  -v, --verbose                 print extra info
+  --color                       colors the logging tags according to level
+  -h, --help                    show this help message and exit
+
+Context Options:
+  -m, --model <string>                     path to full model
+  --clip_l <string>                        path to the clip-l text encoder
+  --clip_g <string>                        path to the clip-g text encoder
+  --clip_vision <string>                   path to the clip-vision encoder
+  --t5xxl <string>                         path to the t5xxl text encoder
+  --llm <string>                           path to the llm text encoder. For example: (qwenvl2.5 for qwen-image,
+                                           mistral-small3.2 for flux2, ...)
+  --llm_vision <string>                    path to the llm vit
+  --qwen2vl <string>                       alias of --llm. Deprecated.
+  --qwen2vl_vision <string>                alias of --llm_vision. Deprecated.
+  --diffusion-model <string>               path to the standalone diffusion model
+  --high-noise-diffusion-model <string>    path to the standalone high noise diffusion model
+  --vae <string>                           path to standalone vae model
+  --taesd <string>                         path to taesd. Using Tiny AutoEncoder for fast decoding (low quality)
+  --tae <string>                           alias of --taesd
+  --control-net <string>                   path to control net model
+  --embd-dir <string>                      embeddings directory
+  --lora-model-dir <string>                lora model directory
+  --hires-upscalers-dir <string>           highres fix upscaler model directory
+  --tensor-type-rules <string>             weight type per tensor pattern (example: "^vae\.=f16,model\.=q8_0")
+  --photo-maker <string>                   path to PHOTOMAKER model
+  --upscale-model <string>                 path to esrgan model.
+  -t, --threads <int>                      number of threads to use during computation (default: -1). If threads <= 0,
+                                           then threads will be set to the number of CPU physical cores
+  --chroma-t5-mask-pad <int>               t5 mask pad size of chroma
+  --force-sdxl-vae-conv-scale              force use of conv scale on sdxl vae
+  --offload-to-cpu                         place the weights in RAM to save VRAM, and automatically load them into VRAM
+                                           when needed
+  --mmap                                   whether to memory-map model
+  --control-net-cpu                        keep controlnet in cpu (for low vram)
+  --clip-on-cpu                            keep clip in cpu (for low vram)
+  --vae-on-cpu                             keep vae in cpu (for low vram)
+  --fa                                     use flash attention
+  --diffusion-fa                           use flash attention in the diffusion model only
+  --diffusion-conv-direct                  use ggml_conv2d_direct in the diffusion model
+  --vae-conv-direct                        use ggml_conv2d_direct in the vae model
+  --circular                               enable circular padding for convolutions
+  --circularx                              enable circular RoPE wrapping on x-axis (width) only
+  --circulary                              enable circular RoPE wrapping on y-axis (height) only
+  --chroma-disable-dit-mask                disable dit mask for chroma
+  --qwen-image-zero-cond-t                 enable zero_cond_t for qwen image
+  --chroma-enable-t5-mask                  enable t5 mask for chroma
+  --type                                   weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K,
+                                           q4_K). If not specified, the default is the type of the weight file
+  --rng                                    RNG, one of [std_default, cuda, cpu], default: cuda(sd-webui), cpu(comfyui)
+  --sampler-rng                            sampler RNG, one of [std_default, cuda, cpu]. If not specified, use --rng
+  --prediction                             prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow,
+                                           flux2_flow]
+  --lora-apply-mode                        the way to apply LoRA, one of [auto, immediately, at_runtime], default is
+                                           auto. In auto mode, if the model weights contain any quantized parameters,
+                                           the at_runtime mode will be used; otherwise, immediately will be used.The
+                                           immediately mode may have precision and compatibility issues with quantized
+                                           parameters, but it usually offers faster inference speed and, in some cases,
+                                           lower memory usage. The at_runtime mode, on the other hand, is exactly the
+                                           opposite.
+
+Default Generation Options:
+  -p, --prompt <string>                    the prompt to render
+  -n, --negative-prompt <string>           the negative prompt (default: "")
+  -i, --init-img <string>                  path to the init image
+  --end-img <string>                       path to the end image, required by flf2v
+  --mask <string>                          path to the mask image
+  --control-image <string>                 path to control image, control net
+  --control-video <string>                 path to control video frames, It must be a directory path. The video frames
+                                           inside should be stored as images in lexicographical (character) order. For
+                                           example, if the control video path is `frames`, the directory contain images
+                                           such as 00.png, 01.png, ... etc.
+  --pm-id-images-dir <string>              path to PHOTOMAKER input id images dir
+  --pm-id-embed-path <string>              path to PHOTOMAKER v2 id embed
+  --hires-upscaler <string>                highres fix upscaler, Lanczos, Nearest, Latent, Latent (nearest), Latent
+                                           (nearest-exact), Latent (antialiased), Latent (bicubic), Latent (bicubic
+                                           antialiased), or a model name under --hires-upscalers-dir (default: Latent)
+  -H, --height <int>                       image height, in pixel space (default: 512)
+  -W, --width <int>                        image width, in pixel space (default: 512)
+  --steps <int>                            number of sample steps (default: 20)
+  --high-noise-steps <int>                 (high noise) number of sample steps (default: -1 = auto)
+  --clip-skip <int>                        ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer
+                                           (default: -1). <= 0 represents unspecified, will be 1 for SD1.x, 2 for SD2.x
+  -b, --batch-count <int>                  batch count
+  --video-frames <int>                     video frames (default: 1)
+  --fps <int>                              fps (default: 24)
+  --timestep-shift <int>                   shift timestep for NitroFusion models (default: 0). recommended N for
+                                           NitroSD-Realism around 250 and 500 for NitroSD-Vibrant
+  --upscale-repeats <int>                  Run the ESRGAN upscaler this many times (default: 1)
+  --upscale-tile-size <int>                tile size for ESRGAN upscaling (default: 128)
+  --hires-width <int>                      highres fix target width, 0 to use --hires-scale (default: 0)
+  --hires-height <int>                     highres fix target height, 0 to use --hires-scale (default: 0)
+  --hires-steps <int>                      highres fix second pass sample steps, 0 to reuse --steps (default: 0)
+  --hires-upscale-tile-size <int>          highres fix upscaler tile size, reserved for model-backed upscalers (default:
+                                           128)
+  --cfg-scale <float>                      unconditional guidance scale: (default: 7.0)
+  --img-cfg-scale <float>                  image guidance scale for inpaint or instruct-pix2pix models: (default: same
+                                           as --cfg-scale)
+  --guidance <float>                       distilled guidance scale for models with guidance input (default: 3.5)
+  --slg-scale <float>                      skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means
+                                           disabled, a value of 2.5 is nice for sd3.5 medium
+  --skip-layer-start <float>               SLG enabling point (default: 0.01)
+  --skip-layer-end <float>                 SLG disabling point (default: 0.2)
+  --eta <float>                            noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and
+                                           res_2s; 1 for euler_a, er_sde and dpm++2s_a)
+  --flow-shift <float>                     shift value for Flow models like SD3.x or WAN (default: auto)
+  --high-noise-cfg-scale <float>           (high noise) unconditional guidance scale: (default: 7.0)
+  --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or instruct-pix2pix models
+                                           (default: same as --cfg-scale)
+  --high-noise-guidance <float>            (high noise) distilled guidance scale for models with guidance input
+                                           (default: 3.5)
+  --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default:
+                                           0)
+  --high-noise-skip-layer-start <float>    (high noise) SLG enabling point (default: 0.01)
+  --high-noise-skip-layer-end <float>      (high noise) SLG disabling point (default: 0.2)
+  --high-noise-eta <float>                 (high noise) noise multiplier (default: 0 for ddim_trailing, tcd,
+                                           res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)
+  --strength <float>                       strength for noising/unnoising (default: 0.75)
+  --pm-style-strength <float>              
+  --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full
+                                           destruction of information in init image
+  --moe-boundary <float>                   timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if
+                                           `--high-noise-steps` is set to -1
+  --vace-strength <float>                  wan vace strength
+  --vae-tile-overlap <float>               tile overlap for vae tiling, in fraction of tile size (default: 0.5)
+  --hires-scale <float>                    highres fix scale when target size is not set (default: 2.0)
+  --hires-denoising-strength <float>       highres fix second pass denoising strength (default: 0.7)
+  --increase-ref-index                     automatically increase the indices of references images based on the order
+                                           they are listed (starting with 1).
+  --disable-auto-resize-ref-image          disable auto resize of ref images
+  --disable-image-metadata                 do not embed generation metadata on image files
+  --vae-tiling                             process vae in tiles to reduce memory usage
+  --hires                                  enable highres fix
+  -s, --seed                               RNG seed (default: 42, use random seed for < 0)
+  --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m,
+                                           dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s,
+                                           er_sde] (default: euler for Flux/SD3/Wan, euler_a otherwise)
+  --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a,
+                                           dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep,
+                                           res_2s, er_sde] default: euler for Flux/SD3/Wan, euler_a otherwise
+  --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits,
+                                           smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent], default:
+                                           discrete
+  --sigmas                                 custom sigma values for the sampler, comma-separated (e.g.,
+                                           "14.61,7.8,3.5,0.0").
+  --skip-layers                            layers to skip for SLG steps (default: [7,8,9])
+  --high-noise-skip-layers                 (high noise) layers to skip for SLG steps (default: [7,8,9])
+  -r, --ref-image                          reference image for Flux Kontext models (can be used multiple times)
+  --cache-mode                             caching method: 'easycache' (DiT), 'ucache' (UNET),
+                                           'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT
+                                           Chebyshev+Taylor forecasting)
+  --cache-option                           named cache params (key=value format, comma-separated). easycache/ucache:
+                                           threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit:
+                                           Fn=,Bn=,threshold=,warmup=; spectrum: w=,m=,lam=,window=,flex=,warmup=,stop=.
+                                           Examples: "threshold=0.25" or "threshold=1.5,reset=0"
+  --scm-mask                               SCM steps mask for cache-dit: comma-separated 0/1 (e.g.,
+                                           "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache
+  --scm-policy                             SCM policy: 'dynamic' (default) or 'static'
+  --vae-tile-size                          tile size for vae tiling, format [X]x[Y] (default: 32x32)
+  --vae-relative-tile-size                 relative tile size for vae tiling, format [X]x[Y], in fraction of image size
+                                           if < 1, in number of tiles per dim if >=1 (overrides --vae-tile-size)
+```

examples/server/async_jobs.cpp

@@ -0,0 +1,349 @@
+// Extracted from main.cpp during server refactor.
+
+#include "async_jobs.h"
+
+#include <iomanip>
+#include <sstream>
+
+#include "common/log.h"
+#include "common/media_io.h"
+#include "common/resource_owners.hpp"
+
+const char* async_job_kind_name(AsyncJobKind kind) {
+    switch (kind) {
+        case AsyncJobKind::ImgGen:
+            return "img_gen";
+        case AsyncJobKind::VidGen:
+            return "vid_gen";
+        default:
+            return "img_gen";
+    }
+}
+
+const char* async_job_status_name(AsyncJobStatus status) {
+    switch (status) {
+        case AsyncJobStatus::Queued:
+            return "queued";
+        case AsyncJobStatus::Generating:
+            return "generating";
+        case AsyncJobStatus::Completed:
+            return "completed";
+        case AsyncJobStatus::Failed:
+            return "failed";
+        case AsyncJobStatus::Cancelled:
+            return "cancelled";
+        default:
+            return "failed";
+    }
+}
+
+void purge_expired_jobs(AsyncJobManager& manager) {
+    const int64_t now = unix_timestamp_now();
+
+    for (auto it = manager.expired_jobs.begin(); it != manager.expired_jobs.end();) {
+        if (it->second <= now) {
+            it = manager.expired_jobs.erase(it);
+        } else {
+            ++it;
+        }
+    }
+
+    for (auto it = manager.jobs.begin(); it != manager.jobs.end();) {
+        const auto& job = it->second;
+        if (job->completed_at == 0) {
+            ++it;
+            continue;
+        }
+
+        int64_t ttl_seconds = job->status == AsyncJobStatus::Completed
+                                  ? manager.completed_ttl_seconds
+                                  : manager.failed_ttl_seconds;
+        if (now - job->completed_at >= ttl_seconds) {
+            manager.expired_jobs[job->id] = now + std::max<int64_t>(ttl_seconds, 60);
+            it                            = manager.jobs.erase(it);
+        } else {
+            ++it;
+        }
+    }
+}
+
+size_t count_pending_jobs(const AsyncJobManager& manager) {
+    size_t pending = 0;
+    for (const auto& entry : manager.jobs) {
+        if (entry.second->status == AsyncJobStatus::Queued ||
+            entry.second->status == AsyncJobStatus::Generating) {
+            ++pending;
+        }
+    }
+    return pending;
+}
+
+std::string make_async_job_id(AsyncJobManager& manager) {
+    std::ostringstream oss;
+    oss << "job_" << std::hex << unix_timestamp_now() << "_" << std::setw(8)
+        << std::setfill('0') << manager.next_id++;
+    return oss.str();
+}
+
+bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job) {
+    auto new_end = std::remove(manager.queue.begin(), manager.queue.end(), job.id);
+    if (new_end == manager.queue.end()) {
+        return false;
+    }
+
+    manager.queue.erase(new_end, manager.queue.end());
+    job.status       = AsyncJobStatus::Cancelled;
+    job.completed_at = unix_timestamp_now();
+    job.result_images_b64.clear();
+    job.result_media_b64.clear();
+    job.result_media_mime_type.clear();
+    job.result_frame_count = 0;
+    job.result_fps         = 0;
+    job.error_code         = "cancelled";
+    job.error_message      = "job cancelled by client";
+    return true;
+}
+
+json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJob& job) {
+    json result;
+    result["id"]             = job.id;
+    result["kind"]           = async_job_kind_name(job.kind);
+    result["status"]         = async_job_status_name(job.status);
+    result["created"]        = job.created_at;
+    result["started"]        = job.started_at == 0 ? json(nullptr) : json(job.started_at);
+    result["completed"]      = job.completed_at == 0 ? json(nullptr) : json(job.completed_at);
+    result["queue_position"] = 0;
+
+    if (job.status == AsyncJobStatus::Queued) {
+        size_t position = 1;
+        for (const auto& queued_id : manager.queue) {
+            if (queued_id == job.id) {
+                result["queue_position"] = position;
+                break;
+            }
+            ++position;
+        }
+    }
+
+    if (job.status == AsyncJobStatus::Completed) {
+        if (job.kind == AsyncJobKind::VidGen) {
+            result["result"] = {
+                {"output_format", job.vid_gen.output_format},
+                {"mime_type", job.result_media_mime_type},
+                {"fps", job.result_fps},
+                {"frame_count", job.result_frame_count},
+                {"b64_json", job.result_media_b64},
+            };
+        } else {
+            json images = json::array();
+            for (size_t i = 0; i < job.result_images_b64.size(); ++i) {
+                images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}});
+            }
+            result["result"] = {
+                {"output_format", job.img_gen.output_format},
+                {"images", images},
+            };
+        }
+        result["error"] = nullptr;
+    } else if (job.status == AsyncJobStatus::Failed ||
+               job.status == AsyncJobStatus::Cancelled) {
+        result["result"] = nullptr;
+        result["error"]  = {
+             {"code",
+             job.error_code.empty()
+                  ? (job.status == AsyncJobStatus::Cancelled ? "cancelled" : "generation_failed")
+                  : job.error_code},
+             {"message", job.error_message},
+        };
+    } else {
+        result["result"] = nullptr;
+        result["error"]  = nullptr;
+    }
+
+    return result;
+}
+
+bool execute_img_gen_job(ServerRuntime& runtime,
+                         AsyncGenerationJob& job,
+                         std::vector<std::string>& output_images,
+                         std::string& error_message) {
+    sd_img_gen_params_t params = job.img_gen.to_sd_img_gen_params_t();
+
+    SDImageVec results;
+
+    {
+        std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
+        sd_image_t* raw_results = generate_image(runtime.sd_ctx, &params);
+        results.adopt(raw_results, params.batch_count);
+    }
+
+    const int num_results = results.count();
+    if (num_results <= 0) {
+        error_message = "generate_image returned no results";
+        return false;
+    }
+
+    EncodedImageFormat encoded_format = EncodedImageFormat::PNG;
+    if (job.img_gen.output_format == "jpeg") {
+        encoded_format = EncodedImageFormat::JPEG;
+    } else if (job.img_gen.output_format == "webp") {
+        encoded_format = EncodedImageFormat::WEBP;
+    }
+
+    for (int i = 0; i < num_results; ++i) {
+        if (results[i].data == nullptr) {
+            continue;
+        }
+
+        const std::string metadata = job.img_gen.gen_params.embed_image_metadata
+                                         ? get_image_params(*runtime.ctx_params,
+                                                            job.img_gen.gen_params,
+                                                            job.img_gen.gen_params.seed + i)
+                                         : "";
+        auto image_bytes           = encode_image_to_vector(encoded_format,
+                                                            results[i].data,
+                                                            results[i].width,
+                                                            results[i].height,
+                                                            results[i].channel,
+                                                            metadata,
+                                                            job.img_gen.output_compression);
+        if (image_bytes.empty()) {
+            continue;
+        }
+        output_images.push_back(base64_encode(image_bytes));
+    }
+
+    if (output_images.empty()) {
+        error_message = "generate_image returned empty encoded outputs";
+        return false;
+    }
+
+    return true;
+}
+
+bool execute_vid_gen_job(ServerRuntime& runtime,
+                         AsyncGenerationJob& job,
+                         std::string& output_media_b64,
+                         std::string& output_media_mime_type,
+                         int& output_frame_count,
+                         int& output_fps,
+                         std::string& error_message) {
+    sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t();
+
+    SDImageVec results;
+    int num_results = 0;
+
+    {
+        std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
+        sd_image_t* raw_results = generate_video(runtime.sd_ctx, &params, &num_results);
+        results.adopt(raw_results, num_results);
+    }
+
+    num_results = results.count();
+    if (num_results <= 0) {
+        error_message = "generate_video returned no results";
+        return false;
+    }
+
+    std::vector<uint8_t> video_bytes = create_video_from_sd_images_to_vector(job.vid_gen.output_format,
+                                                                             results.data(),
+                                                                             num_results,
+                                                                             job.vid_gen.gen_params.fps,
+                                                                             job.vid_gen.output_compression);
+    if (video_bytes.empty()) {
+        error_message = "failed to encode generated video container";
+        return false;
+    }
+
+    output_media_b64       = base64_encode(video_bytes);
+    output_media_mime_type = video_mime_type(job.vid_gen.output_format);
+    output_frame_count     = num_results;
+    output_fps             = job.vid_gen.gen_params.fps;
+    return true;
+}
+
+void async_job_worker(ServerRuntime& runtime) {
+    AsyncJobManager& manager = *runtime.async_job_manager;
+
+    while (true) {
+        std::shared_ptr<AsyncGenerationJob> job;
+        {
+            std::unique_lock<std::mutex> lock(manager.mutex);
+            manager.cv.wait(lock, [&]() { return manager.stop || !manager.queue.empty(); });
+
+            if (manager.stop && manager.queue.empty()) {
+                break;
+            }
+
+            purge_expired_jobs(manager);
+            if (manager.queue.empty()) {
+                continue;
+            }
+
+            const std::string job_id = manager.queue.front();
+            manager.queue.pop_front();
+
+            auto it = manager.jobs.find(job_id);
+            if (it == manager.jobs.end()) {
+                continue;
+            }
+
+            job             = it->second;
+            job->status     = AsyncJobStatus::Generating;
+            job->started_at = unix_timestamp_now();
+        }
+
+        std::vector<std::string> output_images;
+        std::string output_media_b64;
+        std::string output_media_mime_type;
+        int output_frame_count = 0;
+        int output_fps         = 0;
+        std::string error_message;
+        bool ok = false;
+
+        if (job->kind == AsyncJobKind::ImgGen) {
+            ok = execute_img_gen_job(runtime, *job, output_images, error_message);
+        } else if (job->kind == AsyncJobKind::VidGen) {
+            ok = execute_vid_gen_job(runtime,
+                                     *job,
+                                     output_media_b64,
+                                     output_media_mime_type,
+                                     output_frame_count,
+                                     output_fps,
+                                     error_message);
+        } else {
+            error_message = "unsupported job kind";
+        }
+
+        {
+            std::lock_guard<std::mutex> lock(manager.mutex);
+            auto it = manager.jobs.find(job->id);
+            if (it == manager.jobs.end()) {
+                continue;
+            }
+
+            job->completed_at = unix_timestamp_now();
+            if (ok) {
+                job->status                 = AsyncJobStatus::Completed;
+                job->result_images_b64      = std::move(output_images);
+                job->result_media_b64       = std::move(output_media_b64);
+                job->result_media_mime_type = std::move(output_media_mime_type);
+                job->result_frame_count     = output_frame_count;
+                job->result_fps             = output_fps;
+                job->error_code.clear();
+                job->error_message.clear();
+            } else {
+                job->status        = AsyncJobStatus::Failed;
+                job->error_code    = "generation_failed";
+                job->error_message = error_message.empty() ? "unknown generation error" : error_message;
+                job->result_images_b64.clear();
+                job->result_media_b64.clear();
+                job->result_media_mime_type.clear();
+                job->result_frame_count = 0;
+                job->result_fps         = 0;
+            }
+
+            purge_expired_jobs(manager);
+        }
+    }
+}

examples/server/async_jobs.h

@@ -0,0 +1,78 @@
+#pragma once
+
+#include <condition_variable>
+#include <cstdint>
+#include <deque>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+
+#include "runtime.h"
+
+enum class AsyncJobKind {
+    ImgGen,
+    VidGen,
+};
+
+enum class AsyncJobStatus {
+    Queued,
+    Generating,
+    Completed,
+    Failed,
+    Cancelled,
+};
+
+const char* async_job_kind_name(AsyncJobKind kind);
+const char* async_job_status_name(AsyncJobStatus status);
+
+struct AsyncGenerationJob {
+    std::string id;
+    AsyncJobKind kind     = AsyncJobKind::ImgGen;
+    AsyncJobStatus status = AsyncJobStatus::Queued;
+    int64_t created_at    = unix_timestamp_now();
+    int64_t started_at    = 0;
+    int64_t completed_at  = 0;
+    ImgGenJobRequest img_gen;
+    VidGenJobRequest vid_gen;
+    std::vector<std::string> result_images_b64;
+    std::string result_media_b64;
+    std::string result_media_mime_type;
+    int result_frame_count = 0;
+    int result_fps         = 0;
+    std::string error_code;
+    std::string error_message;
+};
+
+struct AsyncJobManager {
+    std::mutex mutex;
+    std::condition_variable cv;
+    std::unordered_map<std::string, std::shared_ptr<AsyncGenerationJob>> jobs;
+    std::unordered_map<std::string, int64_t> expired_jobs;
+    std::deque<std::string> queue;
+    uint64_t next_id              = 0;
+    bool stop                     = false;
+    size_t max_pending_jobs       = 64;
+    int64_t completed_ttl_seconds = 600;
+    int64_t failed_ttl_seconds    = 600;
+};
+
+void purge_expired_jobs(AsyncJobManager& manager);
+size_t count_pending_jobs(const AsyncJobManager& manager);
+std::string make_async_job_id(AsyncJobManager& manager);
+bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job);
+json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJob& job);
+bool execute_img_gen_job(ServerRuntime& runtime,
+                         AsyncGenerationJob& job,
+                         std::vector<std::string>& output_images,
+                         std::string& error_message);
+bool execute_vid_gen_job(ServerRuntime& runtime,
+                         AsyncGenerationJob& job,
+                         std::string& output_media_b64,
+                         std::string& output_media_mime_type,
+                         int& output_frame_count,
+                         int& output_fps,
+                         std::string& error_message);
+void async_job_worker(ServerRuntime& runtime);

examples/server/main.cpp

@@ -0,0 +1,158 @@
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "httplib.h"
+
+#include "async_jobs.h"
+#include "common/common.h"
+#include "common/resource_owners.hpp"
+#include "routes.h"
+#include "runtime.h"
+
+#ifdef HAVE_INDEX_HTML
+#include "frontend/dist/gen_index_html.h"
+#endif
+
+static void print_usage(const char* argv0, const std::vector<ArgOptions>& options_list) {
+    std::cout << version_string() << "\n";
+    std::cout << "Usage: " << argv0 << " [options]\n\n";
+    std::cout << "Svr Options:\n";
+    options_list[0].print();
+    std::cout << "\nContext Options:\n";
+    options_list[1].print();
+    std::cout << "\nDefault Generation Options:\n";
+    options_list[2].print();
+}
+
+static void parse_args(int argc,
+                       const char** argv,
+                       SDSvrParams& svr_params,
+                       SDContextParams& ctx_params,
+                       SDGenerationParams& default_gen_params) {
+    std::vector<ArgOptions> options_vec = {
+        svr_params.get_options(),
+        ctx_params.get_options(),
+        default_gen_params.get_options(),
+    };
+
+    if (!parse_options(argc, argv, options_vec)) {
+        print_usage(argv[0], options_vec);
+        exit(svr_params.normal_exit ? 0 : 1);
+    }
+
+    const bool random_seed_requested = default_gen_params.seed < 0;
+
+    if (!svr_params.resolve_and_validate() ||
+        !ctx_params.resolve_and_validate(IMG_GEN) ||
+        !default_gen_params.resolve_and_validate(IMG_GEN,
+                                                 ctx_params.lora_model_dir,
+                                                 ctx_params.hires_upscalers_dir)) {
+        print_usage(argv[0], options_vec);
+        exit(1);
+    }
+
+    if (random_seed_requested) {
+        default_gen_params.seed = -1;
+    }
+}
+
+void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
+    SDSvrParams* svr_params = (SDSvrParams*)data;
+    log_print(level, log, svr_params->verbose, svr_params->color);
+}
+
+int main(int argc, const char** argv) {
+    if (argc > 1 && std::string(argv[1]) == "--version") {
+        std::cout << version_string() << "\n";
+        return EXIT_SUCCESS;
+    }
+    SDSvrParams svr_params;
+    SDContextParams ctx_params;
+    SDGenerationParams default_gen_params;
+    parse_args(argc, argv, svr_params, ctx_params, default_gen_params);
+
+    sd_set_log_callback(sd_log_cb, (void*)&svr_params);
+    log_verbose = svr_params.verbose;
+    log_color   = svr_params.color;
+
+    LOG_DEBUG("version: %s", version_string().c_str());
+    LOG_DEBUG("%s", sd_get_system_info());
+    LOG_DEBUG("%s", svr_params.to_string().c_str());
+    LOG_DEBUG("%s", ctx_params.to_string().c_str());
+    LOG_DEBUG("%s", default_gen_params.to_string().c_str());
+
+    sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(false, false, false);
+    SDCtxPtr sd_ctx(new_sd_ctx(&sd_ctx_params));
+
+    if (sd_ctx == nullptr) {
+        LOG_ERROR("new_sd_ctx_t failed");
+        return 1;
+    }
+
+    std::mutex sd_ctx_mutex;
+
+    std::vector<LoraEntry> lora_cache;
+    std::mutex lora_mutex;
+    std::vector<UpscalerEntry> upscaler_cache;
+    std::mutex upscaler_mutex;
+    AsyncJobManager async_job_manager;
+    ServerRuntime runtime = {
+        sd_ctx.get(),
+        &sd_ctx_mutex,
+        &svr_params,
+        &ctx_params,
+        &default_gen_params,
+        &lora_cache,
+        &lora_mutex,
+        &upscaler_cache,
+        &upscaler_mutex,
+        &async_job_manager,
+    };
+
+    std::thread async_worker(async_job_worker, std::ref(runtime));
+
+    httplib::Server svr;
+
+    svr.set_pre_routing_handler([](const httplib::Request& req, httplib::Response& res) {
+        std::string origin = req.get_header_value("Origin");
+        if (origin.empty()) {
+            origin = "*";
+        }
+        res.set_header("Access-Control-Allow-Origin", origin);
+        res.set_header("Access-Control-Allow-Credentials", "true");
+        res.set_header("Access-Control-Allow-Methods", "*");
+        res.set_header("Access-Control-Allow-Headers", "*");
+
+        if (req.method == "OPTIONS") {
+            res.status = 204;
+            return httplib::Server::HandlerResponse::Handled;
+        }
+        return httplib::Server::HandlerResponse::Unhandled;
+    });
+
+    std::string index_html;
+#ifdef HAVE_INDEX_HTML
+    index_html.assign(reinterpret_cast<const char*>(index_html_bytes), index_html_size);
+#else
+    index_html = "Stable Diffusion Server is running";
+#endif
+    register_index_endpoints(svr, svr_params, index_html);
+    register_openai_api_endpoints(svr, runtime);
+    register_sdapi_endpoints(svr, runtime);
+    register_sdcpp_api_endpoints(svr, runtime);
+
+    LOG_INFO("listening on: %s:%d\n", svr_params.listen_ip.c_str(), svr_params.listen_port);
+    svr.listen(svr_params.listen_ip, svr_params.listen_port);
+
+    {
+        std::lock_guard<std::mutex> lock(async_job_manager.mutex);
+        async_job_manager.stop = true;
+    }
+    async_job_manager.cv.notify_all();
+    async_worker.join();
+    return 0;
+}

examples/server/routes.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <string>
+
+#include "httplib.h"
+#include "runtime.h"
+
+void register_index_endpoints(httplib::Server& svr, const SDSvrParams& svr_params, const std::string& index_html);
+void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt);
+void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt);
+void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt);

examples/server/routes_index.cpp

@@ -0,0 +1,22 @@
+#include "routes.h"
+
+#include <fstream>
+#include <iterator>
+
+void register_index_endpoints(httplib::Server& svr, const SDSvrParams& svr_params, const std::string& index_html) {
+    const std::string serve_html_path = svr_params.serve_html_path;
+    svr.Get("/", [serve_html_path, index_html](const httplib::Request&, httplib::Response& res) {
+        if (!serve_html_path.empty()) {
+            std::ifstream file(serve_html_path);
+            if (file) {
+                std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
+                res.set_content(content, "text/html");
+            } else {
+                res.status = 500;
+                res.set_content("Error: Unable to read HTML file", "text/plain");
+            }
+        } else {
+            res.set_content(index_html, "text/html");
+        }
+    });
+}

examples/server/routes_openai.cpp

@@ -0,0 +1,388 @@
+#include "routes.h"
+
+#include <algorithm>
+#include <ctime>
+#include <regex>
+
+#include "common/common.h"
+#include "common/media_io.h"
+#include "common/resource_owners.hpp"
+
+static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) {
+    std::regex re("<sd_cpp_extra_args>(.*?)</sd_cpp_extra_args>");
+    std::smatch match;
+
+    std::string extracted;
+    if (std::regex_search(text, match, re)) {
+        extracted = match[1].str();
+        text      = std::regex_replace(text, re, "");
+    }
+    return extracted;
+}
+
+static bool build_openai_generation_request(const httplib::Request& req,
+                                            ServerRuntime& runtime,
+                                            ImgGenJobRequest& request,
+                                            std::string& error_message) {
+    if (req.body.empty()) {
+        error_message = "empty body";
+        return false;
+    }
+
+    json j                    = json::parse(req.body);
+    std::string prompt        = j.value("prompt", "");
+    int n                     = std::max(1, j.value("n", 1));
+    std::string size          = j.value("size", "");
+    std::string output_format = j.value("output_format", "png");
+    int output_compression    = j.value("output_compression", 100);
+    int width                 = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->width : 512;
+    int height                = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->height : 512;
+    if (!size.empty()) {
+        auto pos = size.find('x');
+        if (pos != std::string::npos) {
+            try {
+                width  = std::stoi(size.substr(0, pos));
+                height = std::stoi(size.substr(pos + 1));
+            } catch (...) {
+            }
+        }
+    }
+
+    if (prompt.empty()) {
+        error_message = "prompt required";
+        return false;
+    }
+
+    request.gen_params = *runtime.default_gen_params;
+    if (!assign_output_options(request, output_format, output_compression, true, error_message)) {
+        return false;
+    }
+
+    request.gen_params.prompt      = prompt;
+    request.gen_params.width       = width;
+    request.gen_params.height      = height;
+    request.gen_params.batch_count = n;
+
+    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
+    if (!sd_cpp_extra_args_str.empty() && !request.gen_params.from_json_str(sd_cpp_extra_args_str)) {
+        error_message = "invalid sd_cpp_extra_args";
+        return false;
+    }
+
+    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
+    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
+        error_message = "invalid params";
+        return false;
+    }
+    return true;
+}
+
+static bool build_openai_edit_request(const httplib::Request& req,
+                                      ServerRuntime& runtime,
+                                      ImgGenJobRequest& request,
+                                      std::string& error_message) {
+    if (!req.is_multipart_form_data()) {
+        error_message = "Content-Type must be multipart/form-data";
+        return false;
+    }
+
+    std::string prompt = req.form.get_field("prompt");
+    if (prompt.empty()) {
+        error_message = "prompt required";
+        return false;
+    }
+
+    size_t image_count    = req.form.get_file_count("image[]");
+    bool has_legacy_image = req.form.has_file("image");
+    if (image_count == 0 && !has_legacy_image) {
+        error_message = "at least one image[] required";
+        return false;
+    }
+
+    std::vector<std::vector<uint8_t>> images_bytes;
+    for (size_t i = 0; i < image_count; ++i) {
+        auto file = req.form.get_file("image[]", i);
+        images_bytes.emplace_back(file.content.begin(), file.content.end());
+    }
+    if (image_count == 0 && has_legacy_image) {
+        auto file = req.form.get_file("image");
+        images_bytes.emplace_back(file.content.begin(), file.content.end());
+    }
+
+    std::vector<uint8_t> mask_bytes;
+    if (req.form.has_file("mask")) {
+        auto file = req.form.get_file("mask");
+        mask_bytes.assign(file.content.begin(), file.content.end());
+    }
+
+    int n = 1;
+    if (req.form.has_field("n")) {
+        try {
+            n = std::stoi(req.form.get_field("n"));
+        } catch (...) {
+        }
+    }
+
+    std::string size = req.form.get_field("size");
+    int width        = -1;
+    int height       = -1;
+    if (!size.empty()) {
+        auto pos = size.find('x');
+        if (pos != std::string::npos) {
+            try {
+                width  = std::stoi(size.substr(0, pos));
+                height = std::stoi(size.substr(pos + 1));
+            } catch (...) {
+            }
+        }
+    }
+
+    std::string output_format = req.form.has_field("output_format")
+                                    ? req.form.get_field("output_format")
+                                    : "png";
+
+    int output_compression = 100;
+    try {
+        output_compression = std::stoi(req.form.get_field("output_compression"));
+    } catch (...) {
+    }
+
+    request.gen_params = *runtime.default_gen_params;
+    if (!assign_output_options(request, output_format, output_compression, false, error_message)) {
+        return false;
+    }
+
+    request.gen_params.prompt      = prompt;
+    request.gen_params.width       = width;
+    request.gen_params.height      = height;
+    request.gen_params.batch_count = n;
+
+    for (auto& bytes : images_bytes) {
+        int img_w           = 0;
+        int img_h           = 0;
+        uint8_t* raw_pixels = load_image_from_memory(
+            reinterpret_cast<const char*>(bytes.data()),
+            static_cast<int>(bytes.size()),
+            img_w, img_h,
+            width, height, 3);
+        if (raw_pixels == nullptr) {
+            continue;
+        }
+
+        SDImageOwner image_owner({(uint32_t)img_w, (uint32_t)img_h, 3, raw_pixels});
+        request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height);
+        request.gen_params.ref_images.push_back(std::move(image_owner));
+    }
+
+    if (!request.gen_params.ref_images.empty()) {
+        request.gen_params.init_image = request.gen_params.ref_images.front();
+    }
+
+    if (!mask_bytes.empty()) {
+        int expected_width  = 0;
+        int expected_height = 0;
+        if (request.gen_params.width_and_height_are_set()) {
+            expected_width  = request.gen_params.width;
+            expected_height = request.gen_params.height;
+        }
+        int mask_w = 0;
+        int mask_h = 0;
+
+        uint8_t* mask_raw = load_image_from_memory(
+            reinterpret_cast<const char*>(mask_bytes.data()),
+            static_cast<int>(mask_bytes.size()),
+            mask_w, mask_h,
+            expected_width, expected_height, 1);
+        request.gen_params.mask_image.reset({(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw});
+        const sd_image_t& mask_image = request.gen_params.mask_image.get();
+        request.gen_params.set_width_and_height_if_unset(mask_image.width, mask_image.height);
+    } else {
+        request.gen_params.mask_image.reset({
+            (uint32_t)request.gen_params.get_resolved_width(),
+            (uint32_t)request.gen_params.get_resolved_height(),
+            1,
+            nullptr,
+        });
+    }
+
+    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
+    if (!sd_cpp_extra_args_str.empty() && !request.gen_params.from_json_str(sd_cpp_extra_args_str)) {
+        error_message = "invalid sd_cpp_extra_args";
+        return false;
+    }
+
+    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
+    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
+        error_message = "invalid params";
+        return false;
+    }
+
+    return true;
+}
+
+static bool execute_sync_img_gen_request(ServerRuntime& runtime,
+                                         ImgGenJobRequest& request,
+                                         SDImageVec& results,
+                                         std::string& error_message) {
+    sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t();
+    int num_results                    = 0;
+
+    {
+        std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
+        sd_image_t* raw_results = generate_image(runtime.sd_ctx, &img_gen_params);
+        num_results             = request.gen_params.batch_count;
+        results.adopt(raw_results, num_results);
+    }
+
+    if (results.empty()) {
+        error_message = "generate_image returned no results";
+        return false;
+    }
+    return true;
+}
+
+void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
+    ServerRuntime* runtime = &rt;
+
+    svr.Get("/v1/models", [runtime](const httplib::Request&, httplib::Response& res) {
+        json r;
+        r["data"] = json::array();
+        r["data"].push_back({{"id", "sd-cpp-local"}, {"object", "model"}, {"owned_by", "local"}});
+        res.set_content(r.dump(), "application/json");
+    });
+
+    svr.Post("/v1/images/generations", [runtime](const httplib::Request& req, httplib::Response& res) {
+        try {
+            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
+                res.status = 400;
+                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
+                return;
+            }
+
+            ImgGenJobRequest request;
+            std::string error_message;
+            if (!build_openai_generation_request(req, *runtime, request, error_message)) {
+                res.status = 400;
+                res.set_content(json({{"error", error_message}}).dump(), "application/json");
+                return;
+            }
+
+            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
+
+            SDImageVec results;
+            if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) {
+                res.status = 500;
+                res.set_content(json({{"error", error_message}}).dump(), "application/json");
+                return;
+            }
+
+            json out;
+            out["created"]       = static_cast<long long>(std::time(nullptr));
+            out["data"]          = json::array();
+            out["output_format"] = request.output_format;
+
+            for (int i = 0; i < request.gen_params.batch_count; ++i) {
+                if (results[i].data == nullptr) {
+                    continue;
+                }
+                std::string params = request.gen_params.embed_image_metadata
+                                         ? get_image_params(*runtime->ctx_params,
+                                                            request.gen_params,
+                                                            request.gen_params.seed + i)
+                                         : "";
+                auto image_bytes   = encode_image_to_vector(request.output_format == "jpeg"
+                                                                ? EncodedImageFormat::JPEG
+                                                            : request.output_format == "webp"
+                                                                ? EncodedImageFormat::WEBP
+                                                                : EncodedImageFormat::PNG,
+                                                          results[i].data,
+                                                          results[i].width,
+                                                          results[i].height,
+                                                          results[i].channel,
+                                                          params,
+                                                          request.output_compression);
+                if (image_bytes.empty()) {
+                    LOG_ERROR("write image to mem failed");
+                    continue;
+                }
+
+                json item;
+                item["b64_json"] = base64_encode(image_bytes);
+                out["data"].push_back(item);
+            }
+
+            res.set_content(out.dump(), "application/json");
+            res.status = 200;
+
+        } catch (const std::exception& e) {
+            res.status = 500;
+            json err;
+            err["error"]   = "server_error";
+            err["message"] = e.what();
+            res.set_content(err.dump(), "application/json");
+        }
+    });
+
+    svr.Post("/v1/images/edits", [runtime](const httplib::Request& req, httplib::Response& res) {
+        try {
+            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
+                res.status = 400;
+                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
+                return;
+            }
+
+            ImgGenJobRequest request;
+            std::string error_message;
+            if (!build_openai_edit_request(req, *runtime, request, error_message)) {
+                res.status = 400;
+                res.set_content(json({{"error", error_message}}).dump(), "application/json");
+                return;
+            }
+
+            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
+
+            SDImageVec results;
+            if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) {
+                res.status = 500;
+                res.set_content(json({{"error", error_message}}).dump(), "application/json");
+                return;
+            }
+
+            json out;
+            out["created"]       = static_cast<long long>(std::time(nullptr));
+            out["data"]          = json::array();
+            out["output_format"] = request.output_format;
+
+            for (int i = 0; i < request.gen_params.batch_count; ++i) {
+                if (results[i].data == nullptr) {
+                    continue;
+                }
+                std::string params = request.gen_params.embed_image_metadata
+                                         ? get_image_params(*runtime->ctx_params,
+                                                            request.gen_params,
+                                                            request.gen_params.seed + i)
+                                         : "";
+                auto image_bytes   = encode_image_to_vector(request.output_format == "jpeg" ? EncodedImageFormat::JPEG : EncodedImageFormat::PNG,
+                                                          results[i].data,
+                                                          results[i].width,
+                                                          results[i].height,
+                                                          results[i].channel,
+                                                          params,
+                                                          request.output_compression);
+                json item;
+                item["b64_json"] = base64_encode(image_bytes);
+                out["data"].push_back(item);
+            }
+
+            res.set_content(out.dump(), "application/json");
+            res.status = 200;
+
+        } catch (const std::exception& e) {
+            res.status = 500;
+            json err;
+            err["error"]   = "server_error";
+            err["message"] = e.what();
+            res.set_content(err.dump(), "application/json");
+        }
+    });
+}

examples/server/routes_sdapi.cpp

@@ -0,0 +1,469 @@
+#include "routes.h"
+
+#include <algorithm>
+#include <cctype>
+#include <cstring>
+#include <regex>
+#include <string_view>
+#include <unordered_map>
+
+#include "common/common.h"
+#include "common/media_io.h"
+#include "common/resource_owners.hpp"
+
+namespace fs = std::filesystem;
+
+static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) {
+    std::regex re("<sd_cpp_extra_args>(.*?)</sd_cpp_extra_args>");
+    std::smatch match;
+
+    std::string extracted;
+    if (std::regex_search(text, match, re)) {
+        extracted = match[1].str();
+        text      = std::regex_replace(text, re, "");
+    }
+    return extracted;
+}
+
+static fs::path resolve_display_model_path(const ServerRuntime& runtime) {
+    const auto& ctx = *runtime.ctx_params;
+    if (!ctx.model_path.empty()) {
+        return fs::path(ctx.model_path);
+    }
+    if (!ctx.diffusion_model_path.empty()) {
+        return fs::path(ctx.diffusion_model_path);
+    }
+    return {};
+}
+
+static std::string lower_ascii(std::string value) {
+    std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
+        return static_cast<char>(std::tolower(c));
+    });
+    return value;
+}
+
+static enum sample_method_t get_sdapi_sample_method(std::string name) {
+    enum sample_method_t result = str_to_sample_method(name.c_str());
+    if (result != SAMPLE_METHOD_COUNT) {
+        return result;
+    }
+
+    name = lower_ascii(name);
+    static const std::unordered_map<std::string_view, sample_method_t> hardcoded{
+        {"euler a", EULER_A_SAMPLE_METHOD},
+        {"k_euler_a", EULER_A_SAMPLE_METHOD},
+        {"euler", EULER_SAMPLE_METHOD},
+        {"k_euler", EULER_SAMPLE_METHOD},
+        {"heun", HEUN_SAMPLE_METHOD},
+        {"k_heun", HEUN_SAMPLE_METHOD},
+        {"dpm2", DPM2_SAMPLE_METHOD},
+        {"k_dpm_2", DPM2_SAMPLE_METHOD},
+        {"lcm", LCM_SAMPLE_METHOD},
+        {"ddim", DDIM_TRAILING_SAMPLE_METHOD},
+        {"dpm++ 2m", DPMPP2M_SAMPLE_METHOD},
+        {"k_dpmpp_2m", DPMPP2M_SAMPLE_METHOD},
+        {"res multistep", RES_MULTISTEP_SAMPLE_METHOD},
+        {"k_res_multistep", RES_MULTISTEP_SAMPLE_METHOD},
+        {"res 2s", RES_2S_SAMPLE_METHOD},
+        {"k_res_2s", RES_2S_SAMPLE_METHOD},
+    };
+    auto it = hardcoded.find(name);
+    return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT;
+}
+
+static void assign_solid_mask(SDImageOwner& mask_owner, int width, int height) {
+    const size_t pixel_count = static_cast<size_t>(width) * static_cast<size_t>(height);
+    uint8_t* raw_mask        = static_cast<uint8_t*>(malloc(pixel_count));
+    if (raw_mask == nullptr) {
+        mask_owner.reset({0, 0, 1, nullptr});
+        return;
+    }
+    std::memset(raw_mask, 255, pixel_count);
+    mask_owner.reset({(uint32_t)width, (uint32_t)height, 1, raw_mask});
+}
+
+static bool build_sdapi_img_gen_request(const json& j,
+                                        ServerRuntime& runtime,
+                                        bool img2img,
+                                        ImgGenJobRequest& request,
+                                        std::string& error_message) {
+    std::string prompt          = j.value("prompt", "");
+    std::string negative_prompt = j.value("negative_prompt", "");
+    int width                   = j.value("width", 512);
+    int height                  = j.value("height", 512);
+    int steps                   = j.value("steps", runtime.default_gen_params->sample_params.sample_steps);
+    float cfg_scale             = j.value("cfg_scale", runtime.default_gen_params->sample_params.guidance.txt_cfg);
+    int64_t seed                = j.value("seed", -1);
+    int batch_size              = j.value("batch_size", 1);
+    int clip_skip               = j.value("clip_skip", -1);
+    std::string sampler_name    = j.value("sampler_name", "");
+    std::string scheduler_name  = j.value("scheduler", "");
+
+    if (width <= 0 || height <= 0) {
+        error_message = "width and height must be positive";
+        return false;
+    }
+
+    if (prompt.empty()) {
+        error_message = "prompt required";
+        return false;
+    }
+
+    request.gen_params = *runtime.default_gen_params;
+
+    request.gen_params.prompt                         = prompt;
+    request.gen_params.negative_prompt                = negative_prompt;
+    request.gen_params.seed                           = seed;
+    request.gen_params.sample_params.sample_steps     = steps;
+    request.gen_params.batch_count                    = batch_size;
+    request.gen_params.sample_params.guidance.txt_cfg = cfg_scale;
+    request.gen_params.width                          = j.value("width", -1);
+    request.gen_params.height                         = j.value("height", -1);
+
+    if (!img2img && j.value("enable_hr", false)) {
+        request.gen_params.hires_enabled = true;
+        request.gen_params.hires_scale   = j.value("hr_scale", request.gen_params.hires_scale);
+        request.gen_params.hires_width   = j.value("hr_resize_x", request.gen_params.hires_width);
+        request.gen_params.hires_height  = j.value("hr_resize_y", request.gen_params.hires_height);
+        request.gen_params.hires_steps   = j.value("hr_steps", request.gen_params.hires_steps);
+        request.gen_params.hires_denoising_strength =
+            j.value("denoising_strength", request.gen_params.hires_denoising_strength);
+
+        request.gen_params.hires_upscaler = j.value("hr_upscaler", request.gen_params.hires_upscaler);
+    }
+
+    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
+    if (!sd_cpp_extra_args_str.empty() && !request.gen_params.from_json_str(sd_cpp_extra_args_str)) {
+        error_message = "invalid sd_cpp_extra_args";
+        return false;
+    }
+
+    if (clip_skip > 0) {
+        request.gen_params.clip_skip = clip_skip;
+    }
+
+    enum sample_method_t sample_method = get_sdapi_sample_method(sampler_name);
+    if (sample_method != SAMPLE_METHOD_COUNT) {
+        request.gen_params.sample_params.sample_method = sample_method;
+    }
+
+    enum scheduler_t scheduler = str_to_scheduler(scheduler_name.c_str());
+    if (scheduler != SCHEDULER_COUNT) {
+        request.gen_params.sample_params.scheduler = scheduler;
+    }
+
+    if (j.contains("lora") && j["lora"].is_array()) {
+        request.gen_params.lora_map.clear();
+        request.gen_params.high_noise_lora_map.clear();
+
+        for (const auto& item : j["lora"]) {
+            if (!item.is_object()) {
+                continue;
+            }
+
+            std::string path   = item.value("path", "");
+            float multiplier   = item.value("multiplier", 1.0f);
+            bool is_high_noise = item.value("is_high_noise", false);
+
+            if (path.empty()) {
+                error_message = "lora.path required";
+                return false;
+            }
+
+            std::string fullpath = get_lora_full_path(runtime, path);
+            if (fullpath.empty()) {
+                error_message = "invalid lora path: " + path;
+                return false;
+            }
+
+            if (is_high_noise) {
+                request.gen_params.high_noise_lora_map[fullpath] += multiplier;
+            } else {
+                request.gen_params.lora_map[fullpath] += multiplier;
+            }
+        }
+    }
+
+    if (img2img) {
+        const int expected_width  = request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0;
+        const int expected_height = request.gen_params.width_and_height_are_set() ? request.gen_params.height : 0;
+
+        if (j.contains("init_images") && j["init_images"].is_array() && !j["init_images"].empty()) {
+            if (decode_base64_image(j["init_images"][0].get<std::string>(),
+                                    3,
+                                    expected_width,
+                                    expected_height,
+                                    request.gen_params.init_image)) {
+                const sd_image_t& image = request.gen_params.init_image.get();
+                request.gen_params.set_width_and_height_if_unset(image.width, image.height);
+            }
+        }
+
+        if (j.contains("mask") && j["mask"].is_string()) {
+            if (decode_base64_image(j["mask"].get<std::string>(),
+                                    1,
+                                    expected_width,
+                                    expected_height,
+                                    request.gen_params.mask_image)) {
+                const sd_image_t& image = request.gen_params.mask_image.get();
+                request.gen_params.set_width_and_height_if_unset(image.width, image.height);
+            }
+            sd_image_t& mask_image      = request.gen_params.mask_image.get();
+            bool inpainting_mask_invert = j.value("inpainting_mask_invert", 0) != 0;
+            if (inpainting_mask_invert && mask_image.data != nullptr) {
+                for (uint32_t i = 0; i < mask_image.width * mask_image.height; ++i) {
+                    mask_image.data[i] = 255 - mask_image.data[i];
+                }
+            }
+        } else {
+            const int resolved_width  = request.gen_params.get_resolved_width();
+            const int resolved_height = request.gen_params.get_resolved_height();
+            assign_solid_mask(request.gen_params.mask_image, resolved_width, resolved_height);
+        }
+
+        float denoising_strength = j.value("denoising_strength", -1.f);
+        if (denoising_strength >= 0.f) {
+            request.gen_params.strength = std::min(denoising_strength, 1.0f);
+        }
+    }
+
+    if (j.contains("extra_images") && j["extra_images"].is_array()) {
+        for (const auto& extra_image : j["extra_images"]) {
+            if (!extra_image.is_string()) {
+                continue;
+            }
+            SDImageOwner image_owner;
+            if (decode_base64_image(extra_image.get<std::string>(),
+                                    3,
+                                    request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0,
+                                    request.gen_params.width_and_height_are_set() ? request.gen_params.height : 0,
+                                    image_owner)) {
+                const sd_image_t& image = image_owner.get();
+                request.gen_params.set_width_and_height_if_unset(image.width, image.height);
+                request.gen_params.ref_images.push_back(std::move(image_owner));
+            }
+        }
+    }
+
+    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
+    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
+        error_message = "invalid params";
+        return false;
+    }
+
+    return true;
+}
+
+void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
+    ServerRuntime* runtime = &rt;
+
+    auto sdapi_any2img = [runtime](const httplib::Request& req, httplib::Response& res, bool img2img) {
+        try {
+            if (req.body.empty()) {
+                res.status = 400;
+                res.set_content(R"({"error":"empty body"})", "application/json");
+                return;
+            }
+            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
+                res.status = 400;
+                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
+                return;
+            }
+
+            json j = json::parse(req.body);
+            ImgGenJobRequest request;
+            std::string error_message;
+            if (!build_sdapi_img_gen_request(j, *runtime, img2img, request, error_message)) {
+                res.status = 400;
+                res.set_content(json({{"error", error_message}}).dump(), "application/json");
+                return;
+            }
+
+            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
+
+            sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t();
+            SDImageVec results;
+            int num_results = 0;
+
+            {
+                std::lock_guard<std::mutex> lock(*runtime->sd_ctx_mutex);
+                sd_image_t* raw_results = generate_image(runtime->sd_ctx, &img_gen_params);
+                num_results             = request.gen_params.batch_count;
+                results.adopt(raw_results, num_results);
+            }
+
+            if (results.empty()) {
+                res.status = 500;
+                res.set_content(R"({"error":"generate_image returned no results"})", "application/json");
+                return;
+            }
+
+            json out;
+            out["images"]     = json::array();
+            out["parameters"] = j;
+            out["info"]       = "";
+
+            for (int i = 0; i < num_results; ++i) {
+                if (results[i].data == nullptr) {
+                    continue;
+                }
+
+                std::string params = request.gen_params.embed_image_metadata
+                                         ? get_image_params(*runtime->ctx_params,
+                                                            request.gen_params,
+                                                            request.gen_params.seed + i)
+                                         : "";
+                auto image_bytes   = encode_image_to_vector(EncodedImageFormat::PNG,
+                                                            results[i].data,
+                                                            results[i].width,
+                                                            results[i].height,
+                                                            results[i].channel,
+                                                            params);
+
+                if (image_bytes.empty()) {
+                    LOG_ERROR("write image to mem failed");
+                    continue;
+                }
+
+                out["images"].push_back(base64_encode(image_bytes));
+            }
+
+            res.set_content(out.dump(), "application/json");
+            res.status = 200;
+
+        } catch (const std::exception& e) {
+            res.status = 500;
+            json err;
+            err["error"]   = "server_error";
+            err["message"] = e.what();
+            res.set_content(err.dump(), "application/json");
+        }
+    };
+
+    svr.Post("/sdapi/v1/txt2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) {
+        sdapi_any2img(req, res, false);
+    });
+
+    svr.Post("/sdapi/v1/img2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) {
+        sdapi_any2img(req, res, true);
+    });
+
+    svr.Get("/sdapi/v1/loras", [runtime](const httplib::Request&, httplib::Response& res) {
+        refresh_lora_cache(*runtime);
+
+        json result = json::array();
+        {
+            std::lock_guard<std::mutex> lock(*runtime->lora_mutex);
+            for (const auto& e : *runtime->lora_cache) {
+                json item;
+                item["name"] = e.name;
+                item["path"] = e.path;
+                result.push_back(item);
+            }
+        }
+
+        res.set_content(result.dump(), "application/json");
+    });
+
+    svr.Get("/sdapi/v1/upscalers", [runtime](const httplib::Request&, httplib::Response& res) {
+        refresh_upscaler_cache(*runtime);
+
+        auto make_builtin = [](const char* name) {
+            json item;
+            item["name"]       = name;
+            item["model_name"] = nullptr;
+            item["model_path"] = nullptr;
+            item["model_url"]  = nullptr;
+            item["scale"]      = 4;
+            return item;
+        };
+
+        json result = json::array();
+        result.push_back(make_builtin("None"));
+        result.push_back(make_builtin("Lanczos"));
+        result.push_back(make_builtin("Nearest"));
+
+        {
+            std::lock_guard<std::mutex> lock(*runtime->upscaler_mutex);
+            for (const auto& e : *runtime->upscaler_cache) {
+                json item;
+                item["name"]       = e.name;
+                item["model_name"] = e.model_name;
+                item["model_path"] = e.fullpath;
+                item["model_url"]  = nullptr;
+                item["scale"]      = e.scale;
+                result.push_back(item);
+            }
+        }
+
+        res.set_content(result.dump(), "application/json");
+    });
+
+    svr.Get("/sdapi/v1/latent-upscale-modes", [](const httplib::Request&, httplib::Response& res) {
+        json result = json::array({
+            {{"name", "Latent"}},
+            {{"name", "Latent (nearest)"}},
+            {{"name", "Latent (nearest-exact)"}},
+            {{"name", "Latent (antialiased)"}},
+            {{"name", "Latent (bicubic)"}},
+            {{"name", "Latent (bicubic antialiased)"}},
+        });
+        res.set_content(result.dump(), "application/json");
+    });
+
+    svr.Get("/sdapi/v1/samplers", [runtime](const httplib::Request&, httplib::Response& res) {
+        std::vector<std::string> sampler_names;
+        sampler_names.push_back("default");
+        for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) {
+            sampler_names.push_back(sd_sample_method_name((sample_method_t)i));
+        }
+        json r = json::array();
+        for (auto name : sampler_names) {
+            json entry;
+            entry["name"]    = name;
+            entry["aliases"] = json::array({name});
+            entry["options"] = json::object();
+            r.push_back(entry);
+        }
+        res.set_content(r.dump(), "application/json");
+    });
+
+    svr.Get("/sdapi/v1/schedulers", [runtime](const httplib::Request&, httplib::Response& res) {
+        std::vector<std::string> scheduler_names;
+        scheduler_names.push_back("default");
+        for (int i = 0; i < SCHEDULER_COUNT; i++) {
+            scheduler_names.push_back(sd_scheduler_name((scheduler_t)i));
+        }
+        json r = json::array();
+        for (auto name : scheduler_names) {
+            json entry;
+            entry["name"]  = name;
+            entry["label"] = name;
+            r.push_back(entry);
+        }
+        res.set_content(r.dump(), "application/json");
+    });
+
+    svr.Get("/sdapi/v1/sd-models", [runtime](const httplib::Request&, httplib::Response& res) {
+        fs::path model_path = resolve_display_model_path(*runtime);
+        json entry;
+        entry["title"]      = model_path.stem();
+        entry["model_name"] = model_path.stem();
+        entry["filename"]   = model_path.filename();
+        entry["hash"]       = "8888888888";
+        entry["sha256"]     = "8888888888888888888888888888888888888888888888888888888888888888";
+        entry["config"]     = nullptr;
+        json r              = json::array();
+        r.push_back(entry);
+        res.set_content(r.dump(), "application/json");
+    });
+
+    svr.Get("/sdapi/v1/options", [runtime](const httplib::Request&, httplib::Response& res) {
+        fs::path model_path = resolve_display_model_path(*runtime);
+        json r;
+        r["samples_format"]      = "png";
+        r["sd_model_checkpoint"] = model_path.stem();
+        res.set_content(r.dump(), "application/json");
+    });
+}

examples/server/routes_sdcpp.cpp

@@ -0,0 +1,588 @@
+#include "routes.h"
+
+#include <algorithm>
+#include <cmath>
+#include <filesystem>
+
+#include "async_jobs.h"
+#include "common/common.h"
+
+namespace fs = std::filesystem;
+
+static bool parse_cache_mode(const std::string& mode_str, sd_cache_mode_t& mode_out) {
+    if (mode_str == "disabled") {
+        mode_out = SD_CACHE_DISABLED;
+        return true;
+    }
+    if (mode_str == "easycache") {
+        mode_out = SD_CACHE_EASYCACHE;
+        return true;
+    }
+    if (mode_str == "ucache") {
+        mode_out = SD_CACHE_UCACHE;
+        return true;
+    }
+    if (mode_str == "dbcache") {
+        mode_out = SD_CACHE_DBCACHE;
+        return true;
+    }
+    if (mode_str == "taylorseer") {
+        mode_out = SD_CACHE_TAYLORSEER;
+        return true;
+    }
+    if (mode_str == "cache-dit") {
+        mode_out = SD_CACHE_CACHE_DIT;
+        return true;
+    }
+    if (mode_str == "spectrum") {
+        mode_out = SD_CACHE_SPECTRUM;
+        return true;
+    }
+    return false;
+}
+
+static json finite_number_or_null(float value) {
+    return std::isfinite(value) ? json(value) : json(nullptr);
+}
+
+static const char* capability_scheduler_name(enum scheduler_t scheduler) {
+    return scheduler < SCHEDULER_COUNT ? sd_scheduler_name(scheduler) : "default";
+}
+
+static const char* capability_sample_method_name(enum sample_method_t sample_method) {
+    return sample_method < SAMPLE_METHOD_COUNT ? sd_sample_method_name(sample_method) : "default";
+}
+
+static json make_vae_tiling_json(const sd_tiling_params_t& params) {
+    return {
+        {"enabled", params.enabled},
+        {"tile_size_x", params.tile_size_x},
+        {"tile_size_y", params.tile_size_y},
+        {"target_overlap", params.target_overlap},
+        {"rel_size_x", params.rel_size_x},
+        {"rel_size_y", params.rel_size_y},
+    };
+}
+
+static fs::path resolve_display_model_path(const ServerRuntime& runtime) {
+    const auto& ctx = *runtime.ctx_params;
+    if (!ctx.model_path.empty()) {
+        return fs::path(ctx.model_path);
+    }
+    if (!ctx.diffusion_model_path.empty()) {
+        return fs::path(ctx.diffusion_model_path);
+    }
+    return {};
+}
+
+static json make_sample_params_json(const sd_sample_params_t& sample_params, const std::vector<int>& skip_layers) {
+    const auto& guidance = sample_params.guidance;
+    return {
+        {"scheduler", capability_scheduler_name(sample_params.scheduler)},
+        {"sample_method", capability_sample_method_name(sample_params.sample_method)},
+        {"sample_steps", sample_params.sample_steps},
+        {"eta", finite_number_or_null(sample_params.eta)},
+        {"shifted_timestep", sample_params.shifted_timestep},
+        {"flow_shift", finite_number_or_null(sample_params.flow_shift)},
+        {"guidance",
+         {
+             {"txt_cfg", guidance.txt_cfg},
+             {"img_cfg", finite_number_or_null(guidance.img_cfg)},
+             {"distilled_guidance", guidance.distilled_guidance},
+             {"slg",
+              {
+                  {"layers", skip_layers},
+                  {"layer_start", guidance.slg.layer_start},
+                  {"layer_end", guidance.slg.layer_end},
+                  {"scale", guidance.slg.scale},
+              }},
+         }},
+    };
+}
+
+static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const std::string& output_format) {
+    return {
+        {"prompt", defaults.prompt},
+        {"negative_prompt", defaults.negative_prompt},
+        {"clip_skip", defaults.clip_skip},
+        {"width", defaults.width > 0 ? defaults.width : 512},
+        {"height", defaults.height > 0 ? defaults.height : 512},
+        {"strength", defaults.strength},
+        {"seed", defaults.seed},
+        {"batch_count", defaults.batch_count},
+        {"auto_resize_ref_image", defaults.auto_resize_ref_image},
+        {"increase_ref_index", defaults.increase_ref_index},
+        {"control_strength", defaults.control_strength},
+        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)},
+        {"hires",
+         {
+             {"enabled", defaults.hires_enabled},
+             {"upscaler", defaults.hires_upscaler},
+             {"scale", defaults.hires_scale},
+             {"target_width", defaults.hires_width},
+             {"target_height", defaults.hires_height},
+             {"steps", defaults.hires_steps},
+             {"denoising_strength", defaults.hires_denoising_strength},
+             {"upscale_tile_size", defaults.hires_upscale_tile_size},
+         }},
+        {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)},
+        {"cache_mode", defaults.cache_mode},
+        {"cache_option", defaults.cache_option},
+        {"scm_mask", defaults.scm_mask},
+        {"scm_policy_dynamic", defaults.scm_policy_dynamic},
+        {"output_format", output_format},
+        {"output_compression", 100},
+    };
+}
+
+static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const std::string& output_format) {
+    return {
+        {"prompt", defaults.prompt},
+        {"negative_prompt", defaults.negative_prompt},
+        {"clip_skip", defaults.clip_skip},
+        {"width", defaults.width > 0 ? defaults.width : 512},
+        {"height", defaults.height > 0 ? defaults.height : 512},
+        {"strength", defaults.strength},
+        {"seed", defaults.seed},
+        {"video_frames", defaults.video_frames},
+        {"fps", defaults.fps},
+        {"moe_boundary", defaults.moe_boundary},
+        {"vace_strength", defaults.vace_strength},
+        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)},
+        {"high_noise_sample_params", make_sample_params_json(defaults.high_noise_sample_params, defaults.high_noise_skip_layers)},
+        {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)},
+        {"cache_mode", defaults.cache_mode},
+        {"cache_option", defaults.cache_option},
+        {"scm_mask", defaults.scm_mask},
+        {"scm_policy_dynamic", defaults.scm_policy_dynamic},
+        {"output_format", output_format},
+        {"output_compression", 100},
+    };
+}
+
+static json make_img_gen_features_json() {
+    return {
+        {"init_image", true},
+        {"mask_image", true},
+        {"control_image", true},
+        {"ref_images", true},
+        {"lora", true},
+        {"vae_tiling", true},
+        {"hires", true},
+        {"cache", true},
+        {"cancel_queued", true},
+        {"cancel_generating", false},
+    };
+}
+
+static json make_vid_gen_features_json() {
+    return {
+        {"init_image", true},
+        {"end_image", true},
+        {"control_frames", true},
+        {"high_noise_sample_params", true},
+        {"lora", true},
+        {"vae_tiling", true},
+        {"cache", true},
+        {"cancel_queued", true},
+        {"cancel_generating", false},
+    };
+}
+
+static json make_capabilities_json(ServerRuntime& runtime) {
+    refresh_lora_cache(runtime);
+    refresh_upscaler_cache(runtime);
+
+    AsyncJobManager& manager  = *runtime.async_job_manager;
+    const auto& defaults      = *runtime.default_gen_params;
+    const fs::path model_path = resolve_display_model_path(runtime);
+    const bool supports_img   = runtime_supports_generation_mode(runtime, IMG_GEN);
+    const bool supports_vid   = runtime_supports_generation_mode(runtime, VID_GEN);
+    json samplers             = json::array();
+    json schedulers           = json::array();
+    json image_output_formats = supported_img_output_formats();
+    json video_output_formats = supported_vid_output_formats();
+    json available_loras      = json::array();
+    json available_upscalers  = json::array();
+    json supported_modes      = json::array();
+
+    for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) {
+        samplers.push_back(sd_sample_method_name((sample_method_t)i));
+    }
+
+    for (int i = 0; i < SCHEDULER_COUNT; ++i) {
+        schedulers.push_back(sd_scheduler_name((scheduler_t)i));
+    }
+
+    {
+        std::lock_guard<std::mutex> lock(*runtime.lora_mutex);
+        for (const auto& entry : *runtime.lora_cache) {
+            available_loras.push_back({
+                {"name", entry.name},
+                {"path", entry.path},
+            });
+        }
+    }
+
+    available_upscalers.push_back({
+        {"name", "None"},
+    });
+    available_upscalers.push_back({
+        {"name", "Lanczos"},
+    });
+    available_upscalers.push_back({
+        {"name", "Nearest"},
+    });
+    available_upscalers.push_back({
+        {"name", "Latent"},
+    });
+    available_upscalers.push_back({
+        {"name", "Latent (nearest)"},
+    });
+    available_upscalers.push_back({
+        {"name", "Latent (nearest-exact)"},
+    });
+    available_upscalers.push_back({
+        {"name", "Latent (antialiased)"},
+    });
+    available_upscalers.push_back({
+        {"name", "Latent (bicubic)"},
+    });
+    available_upscalers.push_back({
+        {"name", "Latent (bicubic antialiased)"},
+    });
+    {
+        std::lock_guard<std::mutex> lock(*runtime.upscaler_mutex);
+        for (const auto& entry : *runtime.upscaler_cache) {
+            available_upscalers.push_back({
+                {"name", entry.name},
+            });
+        }
+    }
+
+    if (supports_img) {
+        supported_modes.push_back("img_gen");
+    }
+    if (supports_vid) {
+        supported_modes.push_back("vid_gen");
+    }
+
+    std::string default_img_output_format = "png";
+    std::string default_vid_output_format = "avi";
+    if (!image_output_formats.empty()) {
+        default_img_output_format = image_output_formats[0].get<std::string>();
+    }
+    if (!video_output_formats.empty()) {
+        default_vid_output_format = video_output_formats[0].get<std::string>();
+    }
+
+    json defaults_by_mode       = json::object();
+    json output_formats_by_mode = json::object();
+    json features_by_mode       = json::object();
+    if (supports_img) {
+        defaults_by_mode["img_gen"]       = make_img_gen_defaults_json(defaults, default_img_output_format);
+        output_formats_by_mode["img_gen"] = image_output_formats;
+        features_by_mode["img_gen"]       = make_img_gen_features_json();
+    }
+    if (supports_vid) {
+        defaults_by_mode["vid_gen"]       = make_vid_gen_defaults_json(defaults, default_vid_output_format);
+        output_formats_by_mode["vid_gen"] = video_output_formats;
+        features_by_mode["vid_gen"]       = make_vid_gen_features_json();
+    }
+
+    json top_level_defaults       = json::object();
+    json top_level_output_formats = json::array();
+    json top_level_features       = {
+              {"cancel_queued", true},
+              {"cancel_generating", false},
+    };
+    std::string current_mode = "";
+    if (supports_img) {
+        current_mode             = "img_gen";
+        top_level_defaults       = defaults_by_mode["img_gen"];
+        top_level_output_formats = output_formats_by_mode["img_gen"];
+        top_level_features       = features_by_mode["img_gen"];
+    } else if (supports_vid) {
+        current_mode             = "vid_gen";
+        top_level_defaults       = defaults_by_mode["vid_gen"];
+        top_level_output_formats = output_formats_by_mode["vid_gen"];
+        top_level_features       = features_by_mode["vid_gen"];
+    }
+
+    json result;
+    result["model"] = {
+        {"name", model_path.filename().u8string()},
+        {"stem", model_path.stem().u8string()},
+        {"path", model_path.u8string()},
+    };
+    result["current_mode"]     = current_mode;
+    result["supported_modes"]  = supported_modes;
+    result["defaults"]         = top_level_defaults;
+    result["defaults_by_mode"] = defaults_by_mode;
+    result["limits"]           = {
+                  {"min_width", 64},
+                  {"max_width", 4096},
+                  {"min_height", 64},
+                  {"max_height", 4096},
+                  {"max_batch_count", 8},
+                  {"max_queue_size", manager.max_pending_jobs},
+    };
+    result["samplers"]               = samplers;
+    result["schedulers"]             = schedulers;
+    result["output_formats"]         = top_level_output_formats;
+    result["output_formats_by_mode"] = output_formats_by_mode;
+    result["features"]               = top_level_features;
+    result["features_by_mode"]       = features_by_mode;
+    result["loras"]                  = available_loras;
+    result["upscalers"]              = available_upscalers;
+    return result;
+}
+
+static bool parse_img_gen_request(const json& body,
+                                  ServerRuntime& runtime,
+                                  ImgGenJobRequest& request,
+                                  std::string& error_message) {
+    request.gen_params = *runtime.default_gen_params;
+
+    refresh_lora_cache(runtime);
+    if (!request.gen_params.from_json_str(body.dump(), [&](const std::string& path) {
+            return get_lora_full_path(runtime, path);
+        })) {
+        error_message = "invalid generation parameters";
+        return false;
+    }
+
+    std::string output_format = body.value("output_format", "png");
+    int output_compression    = body.value("output_compression", 100);
+    if (!assign_output_options(request, output_format, output_compression, true, error_message)) {
+        return false;
+    }
+    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
+    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
+        error_message = "invalid generation parameters";
+        return false;
+    }
+    return true;
+}
+
+static bool parse_vid_gen_request(const json& body,
+                                  ServerRuntime& runtime,
+                                  VidGenJobRequest& request,
+                                  std::string& error_message) {
+    request.gen_params = *runtime.default_gen_params;
+
+    refresh_lora_cache(runtime);
+    if (!request.gen_params.from_json_str(body.dump(), [&](const std::string& path) {
+            return get_lora_full_path(runtime, path);
+        })) {
+        error_message = "invalid generation parameters";
+        return false;
+    }
+
+    std::string output_format = body.value("output_format", "webm");
+    int output_compression    = body.value("output_compression", 100);
+    if (!assign_output_options(request, output_format, output_compression, error_message)) {
+        return false;
+    }
+    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
+    if (!request.gen_params.resolve_and_validate(VID_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
+        error_message = "invalid generation parameters";
+        return false;
+    }
+    return true;
+}
+
+void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
+    ServerRuntime* runtime = &rt;
+
+    svr.Get("/sdcpp/v1/capabilities", [runtime](const httplib::Request&, httplib::Response& res) {
+        res.status = 200;
+        res.set_content(make_capabilities_json(*runtime).dump(), "application/json");
+    });
+
+    svr.Post("/sdcpp/v1/img_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
+        try {
+            if (req.body.empty()) {
+                res.status = 400;
+                res.set_content(R"({"error":"empty body"})", "application/json");
+                return;
+            }
+            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
+                res.status = 400;
+                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
+                return;
+            }
+
+            json body = json::parse(req.body);
+            ImgGenJobRequest request;
+            std::string error_message;
+            if (!parse_img_gen_request(body, *runtime, request, error_message)) {
+                res.status = 400;
+                res.set_content(json({{"error", error_message}}).dump(), "application/json");
+                return;
+            }
+
+            AsyncJobManager& manager                = *runtime->async_job_manager;
+            std::shared_ptr<AsyncGenerationJob> job = std::make_shared<AsyncGenerationJob>();
+            job->kind                               = AsyncJobKind::ImgGen;
+            job->status                             = AsyncJobStatus::Queued;
+            job->created_at                         = unix_timestamp_now();
+            job->img_gen                            = std::move(request);
+
+            {
+                std::lock_guard<std::mutex> lock(manager.mutex);
+                purge_expired_jobs(manager);
+                if (count_pending_jobs(manager) >= manager.max_pending_jobs) {
+                    res.status = 429;
+                    res.set_content(R"({"error":"job queue is full"})", "application/json");
+                    return;
+                }
+                job->id               = make_async_job_id(manager);
+                manager.jobs[job->id] = job;
+                manager.queue.push_back(job->id);
+            }
+
+            manager.cv.notify_one();
+
+            json out;
+            out["id"]       = job->id;
+            out["kind"]     = async_job_kind_name(job->kind);
+            out["status"]   = async_job_status_name(job->status);
+            out["created"]  = job->created_at;
+            out["poll_url"] = "/sdcpp/v1/jobs/" + job->id;
+
+            res.status = 202;
+            res.set_content(out.dump(), "application/json");
+        } catch (const json::parse_error& e) {
+            res.status = 400;
+            res.set_content(json({{"error", "invalid json"}, {"message", e.what()}}).dump(), "application/json");
+        } catch (const std::exception& e) {
+            res.status = 500;
+            res.set_content(json({{"error", "server_error"}, {"message", e.what()}}).dump(), "application/json");
+        }
+    });
+
+    svr.Post("/sdcpp/v1/vid_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
+        try {
+            if (req.body.empty()) {
+                res.status = 400;
+                res.set_content(R"({"error":"empty body"})", "application/json");
+                return;
+            }
+            if (!runtime_supports_generation_mode(*runtime, VID_GEN)) {
+                res.status = 400;
+                res.set_content(json({{"error", unsupported_generation_mode_error(VID_GEN)}}).dump(), "application/json");
+                return;
+            }
+
+            json body = json::parse(req.body);
+            VidGenJobRequest request;
+            std::string error_message;
+            if (!parse_vid_gen_request(body, *runtime, request, error_message)) {
+                res.status = 400;
+                res.set_content(json({{"error", error_message}}).dump(), "application/json");
+                return;
+            }
+
+            AsyncJobManager& manager                = *runtime->async_job_manager;
+            std::shared_ptr<AsyncGenerationJob> job = std::make_shared<AsyncGenerationJob>();
+            job->kind                               = AsyncJobKind::VidGen;
+            job->status                             = AsyncJobStatus::Queued;
+            job->created_at                         = unix_timestamp_now();
+            job->vid_gen                            = std::move(request);
+
+            {
+                std::lock_guard<std::mutex> lock(manager.mutex);
+                purge_expired_jobs(manager);
+                if (count_pending_jobs(manager) >= manager.max_pending_jobs) {
+                    res.status = 429;
+                    res.set_content(R"({"error":"job queue is full"})", "application/json");
+                    return;
+                }
+                job->id               = make_async_job_id(manager);
+                manager.jobs[job->id] = job;
+                manager.queue.push_back(job->id);
+            }
+
+            manager.cv.notify_one();
+
+            json out;
+            out["id"]       = job->id;
+            out["kind"]     = async_job_kind_name(job->kind);
+            out["status"]   = async_job_status_name(job->status);
+            out["created"]  = job->created_at;
+            out["poll_url"] = "/sdcpp/v1/jobs/" + job->id;
+
+            res.status = 202;
+            res.set_content(out.dump(), "application/json");
+        } catch (const json::parse_error& e) {
+            res.status = 400;
+            res.set_content(json({{"error", "invalid json"}, {"message", e.what()}}).dump(), "application/json");
+        } catch (const std::exception& e) {
+            res.status = 500;
+            res.set_content(json({{"error", "server_error"}, {"message", e.what()}}).dump(), "application/json");
+        }
+    });
+
+    svr.Get(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+))", [runtime](const httplib::Request& req, httplib::Response& res) {
+        AsyncJobManager& manager = *runtime->async_job_manager;
+        std::lock_guard<std::mutex> lock(manager.mutex);
+        purge_expired_jobs(manager);
+
+        std::string job_id = req.matches[1];
+        auto it            = manager.jobs.find(job_id);
+        if (it == manager.jobs.end()) {
+            if (manager.expired_jobs.find(job_id) != manager.expired_jobs.end()) {
+                res.status = 410;
+                res.set_content(R"({"error":"job expired"})", "application/json");
+            } else {
+                res.status = 404;
+                res.set_content(R"({"error":"job not found"})", "application/json");
+            }
+            return;
+        }
+
+        res.status = 200;
+        res.set_content(make_async_job_json(manager, *it->second).dump(), "application/json");
+    });
+
+    svr.Post(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+)/cancel)", [runtime](const httplib::Request& req, httplib::Response& res) {
+        AsyncJobManager& manager = *runtime->async_job_manager;
+        std::lock_guard<std::mutex> lock(manager.mutex);
+        purge_expired_jobs(manager);
+
+        std::string job_id = req.matches[1];
+        auto it            = manager.jobs.find(job_id);
+        if (it == manager.jobs.end()) {
+            if (manager.expired_jobs.find(job_id) != manager.expired_jobs.end()) {
+                res.status = 410;
+                res.set_content(R"({"error":"job expired"})", "application/json");
+            } else {
+                res.status = 404;
+                res.set_content(R"({"error":"job not found"})", "application/json");
+            }
+            return;
+        }
+
+        auto& job = *it->second;
+        if (job.status == AsyncJobStatus::Queued) {
+            if (!cancel_queued_job(manager, job)) {
+                res.status = 409;
+                res.set_content(R"({"error":"job queue state changed before cancellation"})", "application/json");
+                return;
+            }
+            res.status = 200;
+            res.set_content(make_async_job_json(manager, job).dump(), "application/json");
+            return;
+        }
+
+        if (job.status == AsyncJobStatus::Generating) {
+            res.status = 409;
+            res.set_content(R"({"error":"job is currently generating and cannot be interrupted yet"})", "application/json");
+            return;
+        }
+
+        res.status = 200;
+        res.set_content(make_async_job_json(manager, job).dump(), "application/json");
+    });
+}

examples/server/runtime.cpp

@@ -0,0 +1,332 @@
+#include "runtime.h"
+
+#include <algorithm>
+#include <cctype>
+#include <chrono>
+#include <cstdlib>
+#include <filesystem>
+#include <mutex>
+#include <regex>
+#include <sstream>
+
+#include "common/common.h"
+#include "common/log.h"
+
+namespace fs = std::filesystem;
+
+static std::string lower_ascii(std::string value) {
+    std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
+        return static_cast<char>(std::tolower(c));
+    });
+    return value;
+}
+
+static bool is_supported_model_ext(const fs::path& p) {
+    auto ext = lower_ascii(p.extension().string());
+    return ext == ".gguf" || ext == ".pt" || ext == ".pth" || ext == ".safetensors";
+}
+
+static const std::string k_base64_chars =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    "abcdefghijklmnopqrstuvwxyz"
+    "0123456789+/";
+
+std::string base64_encode(const std::vector<uint8_t>& bytes) {
+    std::string ret;
+    int val  = 0;
+    int valb = -6;
+    for (uint8_t c : bytes) {
+        val = (val << 8) + c;
+        valb += 8;
+        while (valb >= 0) {
+            ret.push_back(k_base64_chars[(val >> valb) & 0x3F]);
+            valb -= 6;
+        }
+    }
+    if (valb > -6) {
+        ret.push_back(k_base64_chars[((val << 8) >> (valb + 8)) & 0x3F]);
+    }
+    while (ret.size() % 4) {
+        ret.push_back('=');
+    }
+    return ret;
+}
+
+std::string normalize_output_format(std::string output_format) {
+    std::transform(output_format.begin(), output_format.end(), output_format.begin(),
+                   [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+    return output_format;
+}
+
+std::vector<std::string> supported_img_output_formats(bool allow_webp) {
+    std::vector<std::string> formats = {"png", "jpeg"};
+#ifdef SD_USE_WEBP
+    if (allow_webp) {
+        formats.push_back("webp");
+    }
+#else
+    (void)allow_webp;
+#endif
+    return formats;
+}
+
+std::vector<std::string> supported_vid_output_formats() {
+    std::vector<std::string> formats;
+#ifdef SD_USE_WEBM
+    formats.push_back("webm");
+#endif
+#ifdef SD_USE_WEBP
+    formats.push_back("webp");
+#endif
+    formats.push_back("avi");
+    return formats;
+}
+
+static std::string valid_vid_output_formats_message() {
+    const std::vector<std::string> formats = supported_vid_output_formats();
+
+    std::string message = "invalid output_format, must be one of [";
+    for (size_t i = 0; i < formats.size(); ++i) {
+        if (i > 0) {
+            message += ", ";
+        }
+        message += formats[i];
+    }
+    message += "]";
+    return message;
+}
+
+bool assign_output_options(ImgGenJobRequest& request,
+                           std::string output_format,
+                           int output_compression,
+                           bool allow_webp,
+                           std::string& error_message) {
+    request.output_format      = normalize_output_format(std::move(output_format));
+    request.output_compression = std::clamp(output_compression, 0, 100);
+
+    const std::vector<std::string> valid_formats = supported_img_output_formats(allow_webp);
+    const bool valid_format                      = std::find(valid_formats.begin(),
+                                                             valid_formats.end(),
+                                                             request.output_format) != valid_formats.end();
+    if (!valid_format) {
+        error_message = "invalid output_format, must be one of [";
+        for (size_t i = 0; i < valid_formats.size(); ++i) {
+            if (i > 0) {
+                error_message += ", ";
+            }
+            error_message += valid_formats[i];
+        }
+        error_message += "]";
+        return false;
+    }
+
+    return true;
+}
+
+bool assign_output_options(VidGenJobRequest& request,
+                           std::string output_format,
+                           int output_compression,
+                           std::string& error_message) {
+    request.output_format      = normalize_output_format(std::move(output_format));
+    request.output_compression = std::clamp(output_compression, 0, 100);
+
+    if (request.output_format == "avi") {
+        return true;
+    }
+
+    if (request.output_format == "webm") {
+#ifdef SD_USE_WEBM
+        return true;
+#else
+        error_message = valid_vid_output_formats_message();
+        return false;
+#endif
+    }
+
+    if (request.output_format == "webp") {
+#ifdef SD_USE_WEBP
+        return true;
+#else
+        error_message = valid_vid_output_formats_message();
+        return false;
+#endif
+    }
+
+    error_message = valid_vid_output_formats_message();
+    return false;
+}
+
+std::string video_mime_type(const std::string& output_format) {
+    if (output_format == "webm") {
+        return "video/webm";
+    }
+    if (output_format == "webp") {
+        return "image/webp";
+    }
+    return "video/x-msvideo";
+}
+
+bool runtime_supports_generation_mode(const ServerRuntime& runtime, SDMode mode) {
+    if (mode == VID_GEN) {
+        return sd_ctx_supports_video_generation(runtime.sd_ctx);
+    }
+    if (mode == IMG_GEN) {
+        return sd_ctx_supports_image_generation(runtime.sd_ctx);
+    }
+    return true;
+}
+
+std::string unsupported_generation_mode_error(SDMode mode) {
+    if (mode == VID_GEN) {
+        return "loaded model does not support vid_gen";
+    }
+    if (mode == IMG_GEN) {
+        return "loaded model does not support img_gen";
+    }
+    return "loaded model does not support requested mode";
+}
+
+ArgOptions SDSvrParams::get_options() {
+    ArgOptions options;
+
+    options.string_options = {
+        {"-l", "--listen-ip", "server listen ip (default: 127.0.0.1)", &listen_ip},
+        {"", "--serve-html-path", "path to HTML file to serve at root (optional)", &serve_html_path},
+    };
+
+    options.int_options = {
+        {"", "--listen-port", "server listen port (default: 1234)", &listen_port},
+    };
+
+    options.bool_options = {
+        {"-v", "--verbose", "print extra info", true, &verbose},
+        {"", "--color", "colors the logging tags according to level", true, &color},
+    };
+
+    auto on_help_arg = [&](int, const char**, int) {
+        normal_exit = true;
+        return -1;
+    };
+
+    options.manual_options = {
+        {"-h", "--help", "show this help message and exit", on_help_arg},
+    };
+    return options;
+}
+
+bool SDSvrParams::validate() {
+    if (listen_ip.empty()) {
+        LOG_ERROR("error: the following arguments are required: listen_ip");
+        return false;
+    }
+
+    if (listen_port < 0 || listen_port > 65535) {
+        LOG_ERROR("error: listen_port should be in the range [0, 65535]");
+        return false;
+    }
+
+    if (!serve_html_path.empty() && !fs::exists(serve_html_path)) {
+        LOG_ERROR("error: serve_html_path file does not exist: %s", serve_html_path.c_str());
+        return false;
+    }
+    return true;
+}
+
+bool SDSvrParams::resolve_and_validate() {
+    if (!validate()) {
+        return false;
+    }
+    return true;
+}
+
+std::string SDSvrParams::to_string() const {
+    std::ostringstream oss;
+    oss << "SDSvrParams {\n"
+        << "  listen_ip: " << listen_ip << ",\n"
+        << "  listen_port: \"" << listen_port << "\",\n"
+        << "  serve_html_path: \"" << serve_html_path << "\",\n"
+        << "}";
+    return oss.str();
+}
+
+void refresh_lora_cache(ServerRuntime& rt) {
+    std::vector<LoraEntry> new_cache;
+
+    fs::path lora_dir = rt.ctx_params->lora_model_dir;
+    if (fs::exists(lora_dir) && fs::is_directory(lora_dir)) {
+        for (auto& entry : fs::recursive_directory_iterator(lora_dir)) {
+            if (!entry.is_regular_file()) {
+                continue;
+            }
+            const fs::path& p = entry.path();
+            if (!is_supported_model_ext(p)) {
+                continue;
+            }
+
+            LoraEntry lora_entry;
+            lora_entry.name     = p.stem().u8string();
+            lora_entry.fullpath = p.u8string();
+            std::string rel     = p.lexically_relative(lora_dir).u8string();
+            std::replace(rel.begin(), rel.end(), '\\', '/');
+            lora_entry.path = rel;
+
+            new_cache.push_back(std::move(lora_entry));
+        }
+    }
+
+    std::sort(new_cache.begin(), new_cache.end(), [](const LoraEntry& a, const LoraEntry& b) {
+        return a.path < b.path;
+    });
+
+    {
+        std::lock_guard<std::mutex> lock(*rt.lora_mutex);
+        *rt.lora_cache = std::move(new_cache);
+    }
+}
+
+std::string get_lora_full_path(ServerRuntime& rt, const std::string& path) {
+    std::lock_guard<std::mutex> lock(*rt.lora_mutex);
+    auto it = std::find_if(rt.lora_cache->begin(), rt.lora_cache->end(),
+                           [&](const LoraEntry& entry) { return entry.path == path; });
+    return it != rt.lora_cache->end() ? it->fullpath : "";
+}
+
+void refresh_upscaler_cache(ServerRuntime& rt) {
+    std::vector<UpscalerEntry> new_cache;
+
+    fs::path upscaler_dir = rt.ctx_params->hires_upscalers_dir;
+    if (fs::exists(upscaler_dir) && fs::is_directory(upscaler_dir)) {
+        for (auto& entry : fs::directory_iterator(upscaler_dir)) {
+            if (!entry.is_regular_file()) {
+                continue;
+            }
+            const fs::path& p = entry.path();
+            if (!is_supported_model_ext(p)) {
+                continue;
+            }
+
+            UpscalerEntry upscaler_entry;
+            upscaler_entry.name       = p.stem().u8string();
+            upscaler_entry.fullpath   = fs::absolute(p).lexically_normal().u8string();
+            upscaler_entry.model_name = "ESRGAN_4x";
+            upscaler_entry.path       = p.filename().u8string();
+
+            new_cache.push_back(std::move(upscaler_entry));
+        }
+    }
+
+    std::sort(new_cache.begin(), new_cache.end(), [](const UpscalerEntry& a, const UpscalerEntry& b) {
+        return a.name < b.name;
+    });
+
+    {
+        std::lock_guard<std::mutex> lock(*rt.upscaler_mutex);
+        *rt.upscaler_cache = std::move(new_cache);
+    }
+}
+
+int64_t unix_timestamp_now() {
+    return std::chrono::duration_cast<std::chrono::seconds>(
+               std::chrono::system_clock::now().time_since_epoch())
+        .count();
+}

examples/server/runtime.h

@@ -0,0 +1,100 @@
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#include <json.hpp>
+#include "common/common.h"
+#include "common/resource_owners.hpp"
+#include "stable-diffusion.h"
+
+using json = nlohmann::json;
+
+struct ArgOptions;
+struct SDContextParams;
+struct AsyncJobManager;
+
+struct SDSvrParams {
+    std::string listen_ip = "127.0.0.1";
+    int listen_port       = 1234;
+    std::string serve_html_path;
+    bool normal_exit = false;
+    bool verbose     = false;
+    bool color       = false;
+
+    ArgOptions get_options();
+    bool validate();
+    bool resolve_and_validate();
+    std::string to_string() const;
+};
+
+struct LoraEntry {
+    std::string name;
+    std::string path;
+    std::string fullpath;
+};
+
+struct UpscalerEntry {
+    std::string name;
+    std::string path;
+    std::string fullpath;
+    std::string model_name;
+    int scale = 4;
+};
+
+struct ServerRuntime {
+    sd_ctx_t* sd_ctx;
+    std::mutex* sd_ctx_mutex;
+    const SDSvrParams* svr_params;
+    const SDContextParams* ctx_params;
+    const SDGenerationParams* default_gen_params;
+    std::vector<LoraEntry>* lora_cache;
+    std::mutex* lora_mutex;
+    std::vector<UpscalerEntry>* upscaler_cache;
+    std::mutex* upscaler_mutex;
+    AsyncJobManager* async_job_manager;
+};
+
+struct ImgGenJobRequest {
+    SDGenerationParams gen_params;
+    std::string output_format = "png";
+    int output_compression    = 100;
+
+    sd_img_gen_params_t to_sd_img_gen_params_t() {
+        return gen_params.to_sd_img_gen_params_t();
+    }
+};
+
+struct VidGenJobRequest {
+    SDGenerationParams gen_params;
+    std::string output_format = "webm";
+    int output_compression    = 100;
+
+    sd_vid_gen_params_t to_sd_vid_gen_params_t() {
+        return gen_params.to_sd_vid_gen_params_t();
+    }
+};
+
+std::string base64_encode(const std::vector<uint8_t>& bytes);
+std::string normalize_output_format(std::string output_format);
+std::vector<std::string> supported_img_output_formats(bool allow_webp = true);
+std::vector<std::string> supported_vid_output_formats();
+bool assign_output_options(ImgGenJobRequest& request,
+                           std::string output_format,
+                           int output_compression,
+                           bool allow_webp,
+                           std::string& error_message);
+bool assign_output_options(VidGenJobRequest& request,
+                           std::string output_format,
+                           int output_compression,
+                           std::string& error_message);
+std::string video_mime_type(const std::string& output_format);
+bool runtime_supports_generation_mode(const ServerRuntime& runtime, SDMode mode);
+std::string unsupported_generation_mode_error(SDMode mode);
+void refresh_lora_cache(ServerRuntime& rt);
+std::string get_lora_full_path(ServerRuntime& rt, const std::string& path);
+void refresh_upscaler_cache(ServerRuntime& rt);
+int64_t unix_timestamp_now();

format-code.sh

@@ -1,4 +1,6 @@
-for f in *.cpp *.h *.hpp examples/cli/*.cpp examples/cli/*.h; do
+for f in src/*.cpp src/*.h src/*.hpp src/tokenizers/*.h src/tokenizers/*.cpp src/tokenizers/vocab/*.h src/tokenizers/vocab/*.cpp \
+         src/model_io/*.h src/model_io/*.cpp examples/cli/*.cpp examples/cli/*.h examples/server/*.cpp \
+         examples/common/*.hpp examples/common/*.h examples/common/*.cpp; do
   [[ "$f" == vocab* ]] && continue
   echo "formatting '$f'"
   # if [ "$f" != "stable-diffusion.h" ]; then

include/stable-diffusion.h

@@ -31,46 +31,51 @@ extern "C" {
 enum rng_type_t {
     STD_DEFAULT_RNG,
     CUDA_RNG,
+    CPU_RNG,
     RNG_TYPE_COUNT
 };
 
 enum sample_method_t {
-    SAMPLE_METHOD_DEFAULT,
-    EULER,
-    HEUN,
-    DPM2,
-    DPMPP2S_A,
-    DPMPP2M,
-    DPMPP2Mv2,
-    IPNDM,
-    IPNDM_V,
-    LCM,
-    DDIM_TRAILING,
-    TCD,
-    EULER_A,
+    EULER_SAMPLE_METHOD,
+    EULER_A_SAMPLE_METHOD,
+    HEUN_SAMPLE_METHOD,
+    DPM2_SAMPLE_METHOD,
+    DPMPP2S_A_SAMPLE_METHOD,
+    DPMPP2M_SAMPLE_METHOD,
+    DPMPP2Mv2_SAMPLE_METHOD,
+    IPNDM_SAMPLE_METHOD,
+    IPNDM_V_SAMPLE_METHOD,
+    LCM_SAMPLE_METHOD,
+    DDIM_TRAILING_SAMPLE_METHOD,
+    TCD_SAMPLE_METHOD,
+    RES_MULTISTEP_SAMPLE_METHOD,
+    RES_2S_SAMPLE_METHOD,
+    ER_SDE_SAMPLE_METHOD,
     SAMPLE_METHOD_COUNT
 };
 
 enum scheduler_t {
-    DEFAULT,
-    DISCRETE,
-    KARRAS,
-    EXPONENTIAL,
-    AYS,
-    GITS,
-    SGM_UNIFORM,
-    SIMPLE,
-    SMOOTHSTEP,
-    SCHEDULE_COUNT
+    DISCRETE_SCHEDULER,
+    KARRAS_SCHEDULER,
+    EXPONENTIAL_SCHEDULER,
+    AYS_SCHEDULER,
+    GITS_SCHEDULER,
+    SGM_UNIFORM_SCHEDULER,
+    SIMPLE_SCHEDULER,
+    SMOOTHSTEP_SCHEDULER,
+    KL_OPTIMAL_SCHEDULER,
+    LCM_SCHEDULER,
+    BONG_TANGENT_SCHEDULER,
+    SCHEDULER_COUNT
 };
 
 enum prediction_t {
-    DEFAULT_PRED,
     EPS_PRED,
     V_PRED,
     EDM_V_PRED,
-    SD3_FLOW_PRED,
+    FLOW_PRED,
     FLUX_FLOW_PRED,
+    FLUX2_FLOW_PRED,
     PREDICTION_COUNT
 };
 
@@ -116,7 +121,8 @@ enum sd_type_t {
     // SD_TYPE_IQ4_NL_4_8 = 37,
     // SD_TYPE_IQ4_NL_8_8 = 38,
     SD_TYPE_MXFP4 = 39,  // MXFP4 (1 block)
-    SD_TYPE_COUNT = 40,
+    SD_TYPE_NVFP4 = 40,  // NVFP4 (4 blocks, E4M3 scale)
+    SD_TYPE_COUNT = 41,
 };
 
 enum sd_log_level_t {
@@ -126,6 +132,21 @@ enum sd_log_level_t {
     SD_LOG_ERROR
 };
 
+enum preview_t {
+    PREVIEW_NONE,
+    PREVIEW_PROJ,
+    PREVIEW_TAE,
+    PREVIEW_VAE,
+    PREVIEW_COUNT
+};
+
+enum lora_apply_mode_t {
+    LORA_APPLY_AUTO,
+    LORA_APPLY_IMMEDIATELY,
+    LORA_APPLY_AT_RUNTIME,
+    LORA_APPLY_MODE_COUNT,
+};
+
 typedef struct {
     bool enabled;
     int tile_size_x;
@@ -135,40 +156,53 @@ typedef struct {
     float rel_size_y;
 } sd_tiling_params_t;
 
+typedef struct {
+    const char* name;
+    const char* path;
+} sd_embedding_t;
+
 typedef struct {
     const char* model_path;
     const char* clip_l_path;
     const char* clip_g_path;
     const char* clip_vision_path;
     const char* t5xxl_path;
-    const char* qwen2vl_path;
-    const char* qwen2vl_vision_path;
+    const char* llm_path;
+    const char* llm_vision_path;
     const char* diffusion_model_path;
     const char* high_noise_diffusion_model_path;
     const char* vae_path;
     const char* taesd_path;
     const char* control_net_path;
-    const char* lora_model_dir;
-    const char* embedding_dir;
+    const sd_embedding_t* embeddings;
+    uint32_t embedding_count;
     const char* photo_maker_path;
+    const char* tensor_type_rules;
     bool vae_decode_only;
     bool free_params_immediately;
     int n_threads;
     enum sd_type_t wtype;
     enum rng_type_t rng_type;
+    enum rng_type_t sampler_rng_type;
     enum prediction_t prediction;
+    enum lora_apply_mode_t lora_apply_mode;
     bool offload_params_to_cpu;
+    bool enable_mmap;
     bool keep_clip_on_cpu;
     bool keep_control_net_on_cpu;
     bool keep_vae_on_cpu;
+    bool flash_attn;
     bool diffusion_flash_attn;
+    bool tae_preview_only;
     bool diffusion_conv_direct;
     bool vae_conv_direct;
+    bool circular_x;
+    bool circular_y;
     bool force_sdxl_vae_conv_scale;
     bool chroma_use_dit_mask;
     bool chroma_use_t5_mask;
     int chroma_t5_mask_pad;
-    float flow_shift;
+    bool qwen_image_zero_cond_t;
 } sd_ctx_params_t;
 
 typedef struct {
@@ -200,6 +234,9 @@ typedef struct {
     int sample_steps;
     float eta;
     int shifted_timestep;
+    float* custom_sigmas;
+    int custom_sigmas_count;
+    float flow_shift;
 } sd_sample_params_t;
 
 typedef struct {
@@ -209,7 +246,78 @@ typedef struct {
     float style_strength;
 } sd_pm_params_t;  // photo maker
 
+enum sd_cache_mode_t {
+    SD_CACHE_DISABLED = 0,
+    SD_CACHE_EASYCACHE,
+    SD_CACHE_UCACHE,
+    SD_CACHE_DBCACHE,
+    SD_CACHE_TAYLORSEER,
+    SD_CACHE_CACHE_DIT,
+    SD_CACHE_SPECTRUM,
+};
+
 typedef struct {
+    enum sd_cache_mode_t mode;
+    float reuse_threshold;
+    float start_percent;
+    float end_percent;
+    float error_decay_rate;
+    bool use_relative_threshold;
+    bool reset_error_on_compute;
+    int Fn_compute_blocks;
+    int Bn_compute_blocks;
+    float residual_diff_threshold;
+    int max_warmup_steps;
+    int max_cached_steps;
+    int max_continuous_cached_steps;
+    int taylorseer_n_derivatives;
+    int taylorseer_skip_interval;
+    const char* scm_mask;
+    bool scm_policy_dynamic;
+    float spectrum_w;
+    int spectrum_m;
+    float spectrum_lam;
+    int spectrum_window_size;
+    float spectrum_flex_window;
+    int spectrum_warmup_steps;
+    float spectrum_stop_percent;
+} sd_cache_params_t;
+
+typedef struct {
+    bool is_high_noise;
+    float multiplier;
+    const char* path;
+} sd_lora_t;
+
+enum sd_hires_upscaler_t {
+    SD_HIRES_UPSCALER_NONE,
+    SD_HIRES_UPSCALER_LATENT,
+    SD_HIRES_UPSCALER_LATENT_NEAREST,
+    SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT,
+    SD_HIRES_UPSCALER_LATENT_ANTIALIASED,
+    SD_HIRES_UPSCALER_LATENT_BICUBIC,
+    SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED,
+    SD_HIRES_UPSCALER_LANCZOS,
+    SD_HIRES_UPSCALER_NEAREST,
+    SD_HIRES_UPSCALER_MODEL,
+    SD_HIRES_UPSCALER_COUNT,
+};
+
+typedef struct {
+    bool enabled;
+    enum sd_hires_upscaler_t upscaler;
+    const char* model_path;
+    float scale;
+    int target_width;
+    int target_height;
+    int steps;
+    float denoising_strength;
+    int upscale_tile_size;
+} sd_hires_params_t;
+
+typedef struct {
+    const sd_lora_t* loras;
+    uint32_t lora_count;
     const char* prompt;
     const char* negative_prompt;
     int clip_skip;
@@ -229,9 +337,13 @@ typedef struct {
     float control_strength;
     sd_pm_params_t pm_params;
     sd_tiling_params_t vae_tiling_params;
+    sd_cache_params_t cache;
+    sd_hires_params_t hires;
 } sd_img_gen_params_t;
 
 typedef struct {
+    const sd_lora_t* loras;
+    uint32_t lora_count;
     const char* prompt;
     const char* negative_prompt;
     int clip_skip;
@@ -248,17 +360,23 @@ typedef struct {
     int64_t seed;
     int video_frames;
     float vace_strength;
+    sd_tiling_params_t vae_tiling_params;
+    sd_cache_params_t cache;
 } sd_vid_gen_params_t;
 
 typedef struct sd_ctx_t sd_ctx_t;
 
 typedef void (*sd_log_cb_t)(enum sd_log_level_t level, const char* text, void* data);
 typedef void (*sd_progress_cb_t)(int step, int steps, float time, void* data);
+typedef void (*sd_preview_cb_t)(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data);
 
 SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data);
 SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
-SD_API int32_t get_num_physical_cores();
+SD_API void sd_set_preview_callback(sd_preview_cb_t cb, enum preview_t mode, int interval, bool denoised, bool noisy, void* data);
+SD_API int32_t sd_get_num_physical_cores();
 SD_API const char* sd_get_system_info();
+SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx);
+SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx);
 
 SD_API const char* sd_type_name(enum sd_type_t type);
 SD_API enum sd_type_t str_to_sd_type(const char* str);
@@ -266,21 +384,32 @@ SD_API const char* sd_rng_type_name(enum rng_type_t rng_type);
 SD_API enum rng_type_t str_to_rng_type(const char* str);
 SD_API const char* sd_sample_method_name(enum sample_method_t sample_method);
 SD_API enum sample_method_t str_to_sample_method(const char* str);
-SD_API const char* sd_schedule_name(enum scheduler_t scheduler);
-SD_API enum scheduler_t str_to_schedule(const char* str);
+SD_API const char* sd_scheduler_name(enum scheduler_t scheduler);
+SD_API enum scheduler_t str_to_scheduler(const char* str);
 SD_API const char* sd_prediction_name(enum prediction_t prediction);
 SD_API enum prediction_t str_to_prediction(const char* str);
+SD_API const char* sd_preview_name(enum preview_t preview);
+SD_API enum preview_t str_to_preview(const char* str);
+SD_API const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode);
+SD_API enum lora_apply_mode_t str_to_lora_apply_mode(const char* str);
+SD_API const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler);
+SD_API enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str);
+
+SD_API void sd_cache_params_init(sd_cache_params_t* cache_params);
+SD_API void sd_hires_params_init(sd_hires_params_t* hires_params);
 
 SD_API void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params);
 SD_API char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params);
 
 SD_API sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params);
 SD_API void free_sd_ctx(sd_ctx_t* sd_ctx);
-SD_API enum sample_method_t sd_get_default_sample_method(const sd_ctx_t* sd_ctx);
 
 SD_API void sd_sample_params_init(sd_sample_params_t* sample_params);
 SD_API char* sd_sample_params_to_str(const sd_sample_params_t* sample_params);
 
+SD_API enum sample_method_t sd_get_default_sample_method(const sd_ctx_t* sd_ctx);
+SD_API enum scheduler_t sd_get_default_scheduler(const sd_ctx_t* sd_ctx, enum sample_method_t sample_method);
+
 SD_API void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params);
 SD_API char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params);
 SD_API sd_image_t* generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* sd_img_gen_params);
@@ -293,7 +422,8 @@ typedef struct upscaler_ctx_t upscaler_ctx_t;
 SD_API upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path,
                                         bool offload_params_to_cpu,
                                         bool direct,
-                                        int n_threads);
+                                        int n_threads,
+                                        int tile_size);
 SD_API void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx);
 
 SD_API sd_image_t upscale(upscaler_ctx_t* upscaler_ctx,
@@ -306,7 +436,8 @@ SD_API bool convert(const char* input_path,
                     const char* vae_path,
                     const char* output_path,
                     enum sd_type_t output_type,
-                    const char* tensor_type_rules);
+                    const char* tensor_type_rules,
+                    bool convert_name);
 
 SD_API bool preprocess_canny(sd_image_t image,
                              float high_threshold,
@@ -315,6 +446,9 @@ SD_API bool preprocess_canny(sd_image_t image,
                              float strong,
                              bool inverse);
 
+SD_API const char* sd_commit(void);
+SD_API const char* sd_version(void);
+
 #ifdef __cplusplus
 }
 #endif

lora.hpp

@@ -1,897 +0,0 @@
-#ifndef __LORA_HPP__
-#define __LORA_HPP__
-
-#include <mutex>
-#include "ggml_extend.hpp"
-
-#define LORA_GRAPH_BASE_SIZE 10240
-
-struct LoraModel : public GGMLRunner {
-    enum lora_t {
-        REGULAR      = 0,
-        DIFFUSERS    = 1,
-        DIFFUSERS_2  = 2,
-        DIFFUSERS_3  = 3,
-        TRANSFORMERS = 4,
-        LORA_TYPE_COUNT
-    };
-
-    const std::string lora_ups[LORA_TYPE_COUNT] = {
-        ".lora_up",
-        "_lora.up",
-        ".lora_B",
-        ".lora.up",
-        ".lora_linear_layer.up",
-    };
-
-    const std::string lora_downs[LORA_TYPE_COUNT] = {
-        ".lora_down",
-        "_lora.down",
-        ".lora_A",
-        ".lora.down",
-        ".lora_linear_layer.down",
-    };
-
-    const std::string lora_pre[LORA_TYPE_COUNT] = {
-        "lora.",
-        "",
-        "",
-        "",
-        "",
-    };
-
-    const std::map<std::string, std::string> alt_names = {
-        // mmdit
-        {"final_layer.adaLN_modulation.1", "norm_out.linear"},
-        {"pos_embed", "pos_embed.proj"},
-        {"final_layer.linear", "proj_out"},
-        {"y_embedder.mlp.0", "time_text_embed.text_embedder.linear_1"},
-        {"y_embedder.mlp.2", "time_text_embed.text_embedder.linear_2"},
-        {"t_embedder.mlp.0", "time_text_embed.timestep_embedder.linear_1"},
-        {"t_embedder.mlp.2", "time_text_embed.timestep_embedder.linear_2"},
-        {"x_block.mlp.fc1", "ff.net.0.proj"},
-        {"x_block.mlp.fc2", "ff.net.2"},
-        {"context_block.mlp.fc1", "ff_context.net.0.proj"},
-        {"context_block.mlp.fc2", "ff_context.net.2"},
-        {"x_block.adaLN_modulation.1", "norm1.linear"},
-        {"context_block.adaLN_modulation.1", "norm1_context.linear"},
-        {"context_block.attn.proj", "attn.to_add_out"},
-        {"x_block.attn.proj", "attn.to_out.0"},
-        {"x_block.attn2.proj", "attn2.to_out.0"},
-        // flux
-        {"img_in", "x_embedder"},
-        // singlestream
-        {"linear2", "proj_out"},
-        {"modulation.lin", "norm.linear"},
-        // doublestream
-        {"txt_attn.proj", "attn.to_add_out"},
-        {"img_attn.proj", "attn.to_out.0"},
-        {"txt_mlp.0", "ff_context.net.0.proj"},
-        {"txt_mlp.2", "ff_context.net.2"},
-        {"img_mlp.0", "ff.net.0.proj"},
-        {"img_mlp.2", "ff.net.2"},
-        {"txt_mod.lin", "norm1_context.linear"},
-        {"img_mod.lin", "norm1.linear"},
-    };
-
-    const std::map<std::string, std::string> qkv_prefixes = {
-        // mmdit
-        {"context_block.attn.qkv", "attn.add_"},  // suffix "_proj"
-        {"x_block.attn.qkv", "attn.to_"},
-        {"x_block.attn2.qkv", "attn2.to_"},
-        // flux
-        // doublestream
-        {"txt_attn.qkv", "attn.add_"},  // suffix "_proj"
-        {"img_attn.qkv", "attn.to_"},
-    };
-    const std::map<std::string, std::string> qkvm_prefixes = {
-        // flux
-        // singlestream
-        {"linear1", ""},
-    };
-
-    const std::string* type_fingerprints = lora_ups;
-
-    float multiplier = 1.0f;
-    std::map<std::string, struct ggml_tensor*> lora_tensors;
-    std::map<ggml_tensor*, ggml_tensor*> original_tensor_to_final_tensor;
-    std::string file_path;
-    ModelLoader model_loader;
-    bool load_failed                = false;
-    bool applied                    = false;
-    std::vector<int> zero_index_vec = {0};
-    ggml_tensor* zero_index         = nullptr;
-    enum lora_t type                = REGULAR;
-
-    LoraModel(ggml_backend_t backend,
-              const std::string& file_path = "",
-              const std::string prefix     = "")
-        : file_path(file_path), GGMLRunner(backend, false) {
-        if (!model_loader.init_from_file(file_path, prefix)) {
-            load_failed = true;
-        }
-    }
-
-    std::string get_desc() override {
-        return "lora";
-    }
-
-    bool load_from_file(bool filter_tensor, int n_threads) {
-        LOG_INFO("loading LoRA from '%s'", file_path.c_str());
-
-        if (load_failed) {
-            LOG_ERROR("init lora model loader from file failed: '%s'", file_path.c_str());
-            return false;
-        }
-
-        std::unordered_map<std::string, TensorStorage> tensors_to_create;
-        std::mutex lora_mutex;
-        bool dry_run          = true;
-        auto on_new_tensor_cb = [&](const TensorStorage& tensor_storage, ggml_tensor** dst_tensor) -> bool {
-            if (dry_run) {
-                const std::string& name = tensor_storage.name;
-
-                if (filter_tensor && !contains(name, "lora")) {
-                    return true;
-                }
-
-                {
-                    std::lock_guard<std::mutex> lock(lora_mutex);
-                    for (int i = 0; i < LORA_TYPE_COUNT; i++) {
-                        if (name.find(type_fingerprints[i]) != std::string::npos) {
-                            type = (lora_t)i;
-                            break;
-                        }
-                    }
-                    tensors_to_create[name] = tensor_storage;
-                }
-            } else {
-                const std::string& name = tensor_storage.name;
-                auto iter               = lora_tensors.find(name);
-                if (iter != lora_tensors.end()) {
-                    *dst_tensor = iter->second;
-                }
-            }
-            return true;
-        };
-
-        model_loader.load_tensors(on_new_tensor_cb, n_threads);
-
-        for (const auto& pair : tensors_to_create) {
-            const auto& name         = pair.first;
-            const auto& ts           = pair.second;
-            struct ggml_tensor* real = ggml_new_tensor(params_ctx,
-                                                       ts.type,
-                                                       ts.n_dims,
-                                                       ts.ne);
-            lora_tensors[name]       = real;
-        }
-
-        alloc_params_buffer();
-
-        dry_run = false;
-        model_loader.load_tensors(on_new_tensor_cb, n_threads);
-
-        LOG_DEBUG("lora type: \"%s\"/\"%s\"", lora_downs[type].c_str(), lora_ups[type].c_str());
-
-        LOG_DEBUG("finished loaded lora");
-        return true;
-    }
-
-    ggml_tensor* to_f32(ggml_context* ctx, ggml_tensor* a) {
-        auto out = ggml_reshape_1d(ctx, a, ggml_nelements(a));
-        out      = ggml_get_rows(ctx, out, zero_index);
-        out      = ggml_reshape(ctx, out, a);
-        // auto out = ggml_cast(ctx, a, GGML_TYPE_F32);
-        return out;
-    }
-
-    std::vector<std::string> to_lora_keys(std::string blk_name, SDVersion version) {
-        std::vector<std::string> keys;
-        // if (!sd_version_is_sd3(version) || blk_name != "model.diffusion_model.pos_embed") {
-        size_t k_pos = blk_name.find(".weight");
-        if (k_pos == std::string::npos) {
-            return keys;
-        }
-        blk_name = blk_name.substr(0, k_pos);
-        // }
-        keys.push_back(blk_name);
-        keys.push_back("lora." + blk_name);
-        if (sd_version_is_dit(version)) {
-            if (blk_name.find("model.diffusion_model") != std::string::npos) {
-                blk_name.replace(blk_name.find("model.diffusion_model"), sizeof("model.diffusion_model") - 1, "transformer");
-            }
-
-            if (blk_name.find(".single_blocks") != std::string::npos) {
-                blk_name.replace(blk_name.find(".single_blocks"), sizeof(".single_blocks") - 1, ".single_transformer_blocks");
-            }
-            if (blk_name.find(".double_blocks") != std::string::npos) {
-                blk_name.replace(blk_name.find(".double_blocks"), sizeof(".double_blocks") - 1, ".transformer_blocks");
-            }
-
-            if (blk_name.find(".joint_blocks") != std::string::npos) {
-                blk_name.replace(blk_name.find(".joint_blocks"), sizeof(".joint_blocks") - 1, ".transformer_blocks");
-            }
-
-            if (blk_name.find("text_encoders.clip_l") != std::string::npos) {
-                blk_name.replace(blk_name.find("text_encoders.clip_l"), sizeof("text_encoders.clip_l") - 1, "cond_stage_model");
-            }
-
-            for (const auto& item : alt_names) {
-                size_t match = blk_name.find(item.first);
-                if (match != std::string::npos) {
-                    blk_name = blk_name.substr(0, match) + item.second;
-                }
-            }
-            for (const auto& prefix : qkv_prefixes) {
-                size_t match = blk_name.find(prefix.first);
-                if (match != std::string::npos) {
-                    std::string split_blk = "SPLIT|" + blk_name.substr(0, match) + prefix.second;
-                    keys.push_back(split_blk);
-                }
-            }
-            for (const auto& prefix : qkvm_prefixes) {
-                size_t match = blk_name.find(prefix.first);
-                if (match != std::string::npos) {
-                    std::string split_blk = "SPLIT_L|" + blk_name.substr(0, match) + prefix.second;
-                    keys.push_back(split_blk);
-                }
-            }
-            keys.push_back(blk_name);
-        }
-
-        std::vector<std::string> ret;
-        for (std::string& key : keys) {
-            ret.push_back(key);
-            replace_all_chars(key, '.', '_');
-            // fix for some sdxl lora, like lcm-lora-xl
-            if (key == "model_diffusion_model_output_blocks_2_2_conv") {
-                ret.push_back("model_diffusion_model_output_blocks_2_1_conv");
-            }
-            ret.push_back(key);
-        }
-        return ret;
-    }
-
-    struct ggml_cgraph* build_lora_graph(std::map<std::string, struct ggml_tensor*> model_tensors, SDVersion version) {
-        size_t lora_graph_size = LORA_GRAPH_BASE_SIZE + lora_tensors.size() * 10;
-        struct ggml_cgraph* gf = ggml_new_graph_custom(compute_ctx, lora_graph_size, false);
-
-        zero_index = ggml_new_tensor_1d(compute_ctx, GGML_TYPE_I32, 1);
-        set_backend_tensor_data(zero_index, zero_index_vec.data());
-        ggml_build_forward_expand(gf, zero_index);
-
-        original_tensor_to_final_tensor.clear();
-
-        std::set<std::string> applied_lora_tensors;
-        for (auto it : model_tensors) {
-            std::string model_tensor_name    = it.first;
-            struct ggml_tensor* model_tensor = model_tensors[it.first];
-
-            std::vector<std::string> keys = to_lora_keys(model_tensor_name, version);
-            bool is_bias                  = ends_with(model_tensor_name, ".bias");
-            if (keys.size() == 0) {
-                if (is_bias) {
-                    keys.push_back(model_tensor_name.substr(0, model_tensor_name.size() - 5));  // remove .bias
-                } else {
-                    continue;
-                }
-            }
-
-            for (auto& key : keys) {
-                bool is_qkv_split = starts_with(key, "SPLIT|");
-                if (is_qkv_split) {
-                    key = key.substr(sizeof("SPLIT|") - 1);
-                }
-                bool is_qkvm_split = starts_with(key, "SPLIT_L|");
-                if (is_qkvm_split) {
-                    key = key.substr(sizeof("SPLIT_L|") - 1);
-                }
-                struct ggml_tensor* updown = nullptr;
-                float scale_value          = 1.0f;
-                std::string full_key       = lora_pre[type] + key;
-                if (is_bias) {
-                    if (lora_tensors.find(full_key + ".diff_b") != lora_tensors.end()) {
-                        std::string diff_name = full_key + ".diff_b";
-                        ggml_tensor* diff     = lora_tensors[diff_name];
-                        updown                = to_f32(compute_ctx, diff);
-                        applied_lora_tensors.insert(diff_name);
-                    } else {
-                        continue;
-                    }
-                } else if (lora_tensors.find(full_key + ".diff") != lora_tensors.end()) {
-                    std::string diff_name = full_key + ".diff";
-                    ggml_tensor* diff     = lora_tensors[diff_name];
-                    updown                = to_f32(compute_ctx, diff);
-                    applied_lora_tensors.insert(diff_name);
-                } else if (lora_tensors.find(full_key + ".hada_w1_a") != lora_tensors.end()) {
-                    // LoHa mode
-
-                    // TODO: split qkv convention for LoHas (is it ever used?)
-                    if (is_qkv_split || is_qkvm_split) {
-                        LOG_ERROR("Split qkv isn't supported for LoHa models.");
-                        break;
-                    }
-                    std::string alpha_name = "";
-
-                    ggml_tensor* hada_1_mid  = nullptr;  // tau for tucker decomposition
-                    ggml_tensor* hada_1_up   = nullptr;
-                    ggml_tensor* hada_1_down = nullptr;
-
-                    ggml_tensor* hada_2_mid  = nullptr;  // tau for tucker decomposition
-                    ggml_tensor* hada_2_up   = nullptr;
-                    ggml_tensor* hada_2_down = nullptr;
-
-                    std::string hada_1_mid_name  = "";
-                    std::string hada_1_down_name = "";
-                    std::string hada_1_up_name   = "";
-
-                    std::string hada_2_mid_name  = "";
-                    std::string hada_2_down_name = "";
-                    std::string hada_2_up_name   = "";
-
-                    hada_1_down_name = full_key + ".hada_w1_b";
-                    hada_1_up_name   = full_key + ".hada_w1_a";
-                    hada_1_mid_name  = full_key + ".hada_t1";
-                    if (lora_tensors.find(hada_1_down_name) != lora_tensors.end()) {
-                        hada_1_down = to_f32(compute_ctx, lora_tensors[hada_1_down_name]);
-                    }
-                    if (lora_tensors.find(hada_1_up_name) != lora_tensors.end()) {
-                        hada_1_up = to_f32(compute_ctx, lora_tensors[hada_1_up_name]);
-                    }
-                    if (lora_tensors.find(hada_1_mid_name) != lora_tensors.end()) {
-                        hada_1_mid = to_f32(compute_ctx, lora_tensors[hada_1_mid_name]);
-                        applied_lora_tensors.insert(hada_1_mid_name);
-                        hada_1_up = ggml_cont(compute_ctx, ggml_transpose(compute_ctx, hada_1_up));
-                    }
-
-                    hada_2_down_name = full_key + ".hada_w2_b";
-                    hada_2_up_name   = full_key + ".hada_w2_a";
-                    hada_2_mid_name  = full_key + ".hada_t2";
-                    if (lora_tensors.find(hada_2_down_name) != lora_tensors.end()) {
-                        hada_2_down = to_f32(compute_ctx, lora_tensors[hada_2_down_name]);
-                    }
-                    if (lora_tensors.find(hada_2_up_name) != lora_tensors.end()) {
-                        hada_2_up = to_f32(compute_ctx, lora_tensors[hada_2_up_name]);
-                    }
-                    if (lora_tensors.find(hada_2_mid_name) != lora_tensors.end()) {
-                        hada_2_mid = to_f32(compute_ctx, lora_tensors[hada_2_mid_name]);
-                        applied_lora_tensors.insert(hada_2_mid_name);
-                        hada_2_up = ggml_cont(compute_ctx, ggml_transpose(compute_ctx, hada_2_up));
-                    }
-
-                    alpha_name = full_key + ".alpha";
-
-                    applied_lora_tensors.insert(hada_1_down_name);
-                    applied_lora_tensors.insert(hada_1_up_name);
-                    applied_lora_tensors.insert(hada_2_down_name);
-                    applied_lora_tensors.insert(hada_2_up_name);
-
-                    applied_lora_tensors.insert(alpha_name);
-                    if (hada_1_up == nullptr || hada_1_down == nullptr || hada_2_up == nullptr || hada_2_down == nullptr) {
-                        continue;
-                    }
-
-                    struct ggml_tensor* updown_1 = ggml_merge_lora(compute_ctx, hada_1_down, hada_1_up, hada_1_mid);
-                    struct ggml_tensor* updown_2 = ggml_merge_lora(compute_ctx, hada_2_down, hada_2_up, hada_2_mid);
-                    updown                       = ggml_mul_inplace(compute_ctx, updown_1, updown_2);
-
-                    // calc_scale
-                    // TODO: .dora_scale?
-                    int64_t rank = hada_1_down->ne[ggml_n_dims(hada_1_down) - 1];
-                    if (lora_tensors.find(alpha_name) != lora_tensors.end()) {
-                        float alpha = ggml_backend_tensor_get_f32(lora_tensors[alpha_name]);
-                        scale_value = alpha / rank;
-                    }
-                } else if (lora_tensors.find(full_key + ".lokr_w1") != lora_tensors.end() || lora_tensors.find(full_key + ".lokr_w1_a") != lora_tensors.end()) {
-                    // LoKr mode
-
-                    // TODO: split qkv convention for LoKrs (is it ever used?)
-                    if (is_qkv_split || is_qkvm_split) {
-                        LOG_ERROR("Split qkv isn't supported for LoKr models.");
-                        break;
-                    }
-
-                    std::string alpha_name = full_key + ".alpha";
-
-                    ggml_tensor* lokr_w1 = nullptr;
-                    ggml_tensor* lokr_w2 = nullptr;
-
-                    std::string lokr_w1_name = "";
-                    std::string lokr_w2_name = "";
-
-                    lokr_w1_name = full_key + ".lokr_w1";
-                    lokr_w2_name = full_key + ".lokr_w2";
-
-                    if (lora_tensors.find(lokr_w1_name) != lora_tensors.end()) {
-                        lokr_w1 = to_f32(compute_ctx, lora_tensors[lokr_w1_name]);
-                        applied_lora_tensors.insert(lokr_w1_name);
-                    } else {
-                        ggml_tensor* down     = nullptr;
-                        ggml_tensor* up       = nullptr;
-                        std::string down_name = lokr_w1_name + "_b";
-                        std::string up_name   = lokr_w1_name + "_a";
-                        if (lora_tensors.find(down_name) != lora_tensors.end()) {
-                            // w1 should not be low rank normally, sometimes w1 and w2 are swapped
-                            down = to_f32(compute_ctx, lora_tensors[down_name]);
-                            applied_lora_tensors.insert(down_name);
-
-                            int64_t rank = down->ne[ggml_n_dims(down) - 1];
-                            if (lora_tensors.find(alpha_name) != lora_tensors.end()) {
-                                float alpha = ggml_backend_tensor_get_f32(lora_tensors[alpha_name]);
-                                scale_value = alpha / rank;
-                            }
-                        }
-                        if (lora_tensors.find(up_name) != lora_tensors.end()) {
-                            up = to_f32(compute_ctx, lora_tensors[up_name]);
-                            applied_lora_tensors.insert(up_name);
-                        }
-                        lokr_w1 = ggml_merge_lora(compute_ctx, down, up);
-                    }
-                    if (lora_tensors.find(lokr_w2_name) != lora_tensors.end()) {
-                        lokr_w2 = to_f32(compute_ctx, lora_tensors[lokr_w2_name]);
-                        applied_lora_tensors.insert(lokr_w2_name);
-                    } else {
-                        ggml_tensor* down     = nullptr;
-                        ggml_tensor* up       = nullptr;
-                        std::string down_name = lokr_w2_name + "_b";
-                        std::string up_name   = lokr_w2_name + "_a";
-                        if (lora_tensors.find(down_name) != lora_tensors.end()) {
-                            down = to_f32(compute_ctx, lora_tensors[down_name]);
-                            applied_lora_tensors.insert(down_name);
-
-                            int64_t rank = down->ne[ggml_n_dims(down) - 1];
-                            if (lora_tensors.find(alpha_name) != lora_tensors.end()) {
-                                float alpha = ggml_backend_tensor_get_f32(lora_tensors[alpha_name]);
-                                scale_value = alpha / rank;
-                            }
-                        }
-                        if (lora_tensors.find(up_name) != lora_tensors.end()) {
-                            up = to_f32(compute_ctx, lora_tensors[up_name]);
-                            applied_lora_tensors.insert(up_name);
-                        }
-                        lokr_w2 = ggml_merge_lora(compute_ctx, down, up);
-                    }
-
-                    // Technically it might be unused, but I believe it's the expected behavior
-                    applied_lora_tensors.insert(alpha_name);
-
-                    updown = ggml_kronecker(compute_ctx, lokr_w1, lokr_w2);
-
-                } else {
-                    // LoRA mode
-                    ggml_tensor* lora_mid  = nullptr;  // tau for tucker decomposition
-                    ggml_tensor* lora_up   = nullptr;
-                    ggml_tensor* lora_down = nullptr;
-
-                    std::string alpha_name         = "";
-                    std::string scale_name         = "";
-                    std::string split_q_scale_name = "";
-                    std::string lora_mid_name      = "";
-                    std::string lora_down_name     = "";
-                    std::string lora_up_name       = "";
-
-                    if (is_qkv_split) {
-                        std::string suffix  = "";
-                        auto split_q_d_name = full_key + "q" + suffix + lora_downs[type] + ".weight";
-
-                        if (lora_tensors.find(split_q_d_name) == lora_tensors.end()) {
-                            suffix         = "_proj";
-                            split_q_d_name = full_key + "q" + suffix + lora_downs[type] + ".weight";
-                        }
-                        if (lora_tensors.find(split_q_d_name) != lora_tensors.end()) {
-                            // print_ggml_tensor(it.second, true);  //[3072, 21504, 1, 1]
-                            // find qkv and mlp up parts in LoRA model
-                            auto split_k_d_name = full_key + "k" + suffix + lora_downs[type] + ".weight";
-                            auto split_v_d_name = full_key + "v" + suffix + lora_downs[type] + ".weight";
-
-                            auto split_q_u_name = full_key + "q" + suffix + lora_ups[type] + ".weight";
-                            auto split_k_u_name = full_key + "k" + suffix + lora_ups[type] + ".weight";
-                            auto split_v_u_name = full_key + "v" + suffix + lora_ups[type] + ".weight";
-
-                            auto split_q_scale_name = full_key + "q" + suffix + ".scale";
-                            auto split_k_scale_name = full_key + "k" + suffix + ".scale";
-                            auto split_v_scale_name = full_key + "v" + suffix + ".scale";
-
-                            auto split_q_alpha_name = full_key + "q" + suffix + ".alpha";
-                            auto split_k_alpha_name = full_key + "k" + suffix + ".alpha";
-                            auto split_v_alpha_name = full_key + "v" + suffix + ".alpha";
-
-                            ggml_tensor* lora_q_down = nullptr;
-                            ggml_tensor* lora_q_up   = nullptr;
-                            ggml_tensor* lora_k_down = nullptr;
-                            ggml_tensor* lora_k_up   = nullptr;
-                            ggml_tensor* lora_v_down = nullptr;
-                            ggml_tensor* lora_v_up   = nullptr;
-
-                            lora_q_down = to_f32(compute_ctx, lora_tensors[split_q_d_name]);
-
-                            if (lora_tensors.find(split_q_u_name) != lora_tensors.end()) {
-                                lora_q_up = to_f32(compute_ctx, lora_tensors[split_q_u_name]);
-                            }
-
-                            if (lora_tensors.find(split_k_d_name) != lora_tensors.end()) {
-                                lora_k_down = to_f32(compute_ctx, lora_tensors[split_k_d_name]);
-                            }
-
-                            if (lora_tensors.find(split_k_u_name) != lora_tensors.end()) {
-                                lora_k_up = to_f32(compute_ctx, lora_tensors[split_k_u_name]);
-                            }
-
-                            if (lora_tensors.find(split_v_d_name) != lora_tensors.end()) {
-                                lora_v_down = to_f32(compute_ctx, lora_tensors[split_v_d_name]);
-                            }
-
-                            if (lora_tensors.find(split_v_u_name) != lora_tensors.end()) {
-                                lora_v_up = to_f32(compute_ctx, lora_tensors[split_v_u_name]);
-                            }
-
-                            float q_rank = lora_q_up->ne[0];
-                            float k_rank = lora_k_up->ne[0];
-                            float v_rank = lora_v_up->ne[0];
-
-                            float lora_q_scale = 1;
-                            float lora_k_scale = 1;
-                            float lora_v_scale = 1;
-
-                            if (lora_tensors.find(split_q_scale_name) != lora_tensors.end()) {
-                                lora_q_scale = ggml_backend_tensor_get_f32(lora_tensors[split_q_scale_name]);
-                                applied_lora_tensors.insert(split_q_scale_name);
-                            }
-                            if (lora_tensors.find(split_k_scale_name) != lora_tensors.end()) {
-                                lora_k_scale = ggml_backend_tensor_get_f32(lora_tensors[split_k_scale_name]);
-                                applied_lora_tensors.insert(split_k_scale_name);
-                            }
-                            if (lora_tensors.find(split_v_scale_name) != lora_tensors.end()) {
-                                lora_v_scale = ggml_backend_tensor_get_f32(lora_tensors[split_v_scale_name]);
-                                applied_lora_tensors.insert(split_v_scale_name);
-                            }
-
-                            if (lora_tensors.find(split_q_alpha_name) != lora_tensors.end()) {
-                                float lora_q_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_q_alpha_name]);
-                                applied_lora_tensors.insert(split_q_alpha_name);
-                                lora_q_scale = lora_q_alpha / q_rank;
-                            }
-                            if (lora_tensors.find(split_k_alpha_name) != lora_tensors.end()) {
-                                float lora_k_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_k_alpha_name]);
-                                applied_lora_tensors.insert(split_k_alpha_name);
-                                lora_k_scale = lora_k_alpha / k_rank;
-                            }
-                            if (lora_tensors.find(split_v_alpha_name) != lora_tensors.end()) {
-                                float lora_v_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_v_alpha_name]);
-                                applied_lora_tensors.insert(split_v_alpha_name);
-                                lora_v_scale = lora_v_alpha / v_rank;
-                            }
-
-                            ggml_scale_inplace(compute_ctx, lora_q_down, lora_q_scale);
-                            ggml_scale_inplace(compute_ctx, lora_k_down, lora_k_scale);
-                            ggml_scale_inplace(compute_ctx, lora_v_down, lora_v_scale);
-
-                            // print_ggml_tensor(lora_q_down, true);  //[3072, R, 1, 1]
-                            // print_ggml_tensor(lora_k_down, true);  //[3072, R, 1, 1]
-                            // print_ggml_tensor(lora_v_down, true);  //[3072, R, 1, 1]
-                            // print_ggml_tensor(lora_q_up, true);    //[R, 3072, 1, 1]
-                            // print_ggml_tensor(lora_k_up, true);    //[R, 3072, 1, 1]
-                            // print_ggml_tensor(lora_v_up, true);    //[R, 3072, 1, 1]
-
-                            // these need to be stitched together this way:
-                            //                          |q_up,0   ,0   |
-                            //                          |0   ,k_up,0   |
-                            //                          |0   ,0   ,v_up|
-                            // (q_down,k_down,v_down) . (q   ,k   ,v)
-
-                            // up_concat will be [9216, R*3, 1, 1]
-                            // down_concat will be [R*3, 3072, 1, 1]
-                            ggml_tensor* lora_down_concat = ggml_concat(compute_ctx, ggml_concat(compute_ctx, lora_q_down, lora_k_down, 1), lora_v_down, 1);
-
-                            ggml_tensor* z = ggml_dup_tensor(compute_ctx, lora_q_up);
-                            ggml_scale(compute_ctx, z, 0);
-                            ggml_tensor* zz = ggml_concat(compute_ctx, z, z, 1);
-
-                            ggml_tensor* q_up = ggml_concat(compute_ctx, lora_q_up, zz, 1);
-                            ggml_tensor* k_up = ggml_concat(compute_ctx, ggml_concat(compute_ctx, z, lora_k_up, 1), z, 1);
-                            ggml_tensor* v_up = ggml_concat(compute_ctx, zz, lora_v_up, 1);
-                            // print_ggml_tensor(q_up, true);  //[R, 9216, 1, 1]
-                            // print_ggml_tensor(k_up, true);  //[R, 9216, 1, 1]
-                            // print_ggml_tensor(v_up, true);  //[R, 9216, 1, 1]
-                            ggml_tensor* lora_up_concat = ggml_concat(compute_ctx, ggml_concat(compute_ctx, q_up, k_up, 0), v_up, 0);
-                            // print_ggml_tensor(lora_up_concat, true);  //[R*3, 9216, 1, 1]
-
-                            lora_down = ggml_cont(compute_ctx, lora_down_concat);
-                            lora_up   = ggml_cont(compute_ctx, lora_up_concat);
-
-                            applied_lora_tensors.insert(split_q_u_name);
-                            applied_lora_tensors.insert(split_k_u_name);
-                            applied_lora_tensors.insert(split_v_u_name);
-
-                            applied_lora_tensors.insert(split_q_d_name);
-                            applied_lora_tensors.insert(split_k_d_name);
-                            applied_lora_tensors.insert(split_v_d_name);
-                        }
-                    } else if (is_qkvm_split) {
-                        auto split_q_d_name = full_key + "attn.to_q" + lora_downs[type] + ".weight";
-                        if (lora_tensors.find(split_q_d_name) != lora_tensors.end()) {
-                            // print_ggml_tensor(it.second, true);  //[3072, 21504, 1, 1]
-                            // find qkv and mlp up parts in LoRA model
-                            auto split_k_d_name = full_key + "attn.to_k" + lora_downs[type] + ".weight";
-                            auto split_v_d_name = full_key + "attn.to_v" + lora_downs[type] + ".weight";
-
-                            auto split_q_u_name = full_key + "attn.to_q" + lora_ups[type] + ".weight";
-                            auto split_k_u_name = full_key + "attn.to_k" + lora_ups[type] + ".weight";
-                            auto split_v_u_name = full_key + "attn.to_v" + lora_ups[type] + ".weight";
-
-                            auto split_m_d_name = full_key + "proj_mlp" + lora_downs[type] + ".weight";
-                            auto split_m_u_name = full_key + "proj_mlp" + lora_ups[type] + ".weight";
-
-                            auto split_q_scale_name = full_key + "attn.to_q" + ".scale";
-                            auto split_k_scale_name = full_key + "attn.to_k" + ".scale";
-                            auto split_v_scale_name = full_key + "attn.to_v" + ".scale";
-                            auto split_m_scale_name = full_key + "proj_mlp" + ".scale";
-
-                            auto split_q_alpha_name = full_key + "attn.to_q" + ".alpha";
-                            auto split_k_alpha_name = full_key + "attn.to_k" + ".alpha";
-                            auto split_v_alpha_name = full_key + "attn.to_v" + ".alpha";
-                            auto split_m_alpha_name = full_key + "proj_mlp" + ".alpha";
-
-                            ggml_tensor* lora_q_down = nullptr;
-                            ggml_tensor* lora_q_up   = nullptr;
-                            ggml_tensor* lora_k_down = nullptr;
-                            ggml_tensor* lora_k_up   = nullptr;
-                            ggml_tensor* lora_v_down = nullptr;
-                            ggml_tensor* lora_v_up   = nullptr;
-
-                            ggml_tensor* lora_m_down = nullptr;
-                            ggml_tensor* lora_m_up   = nullptr;
-
-                            lora_q_up = to_f32(compute_ctx, lora_tensors[split_q_u_name]);
-
-                            if (lora_tensors.find(split_q_d_name) != lora_tensors.end()) {
-                                lora_q_down = to_f32(compute_ctx, lora_tensors[split_q_d_name]);
-                            }
-
-                            if (lora_tensors.find(split_q_u_name) != lora_tensors.end()) {
-                                lora_q_up = to_f32(compute_ctx, lora_tensors[split_q_u_name]);
-                            }
-
-                            if (lora_tensors.find(split_k_d_name) != lora_tensors.end()) {
-                                lora_k_down = to_f32(compute_ctx, lora_tensors[split_k_d_name]);
-                            }
-
-                            if (lora_tensors.find(split_k_u_name) != lora_tensors.end()) {
-                                lora_k_up = to_f32(compute_ctx, lora_tensors[split_k_u_name]);
-                            }
-
-                            if (lora_tensors.find(split_v_d_name) != lora_tensors.end()) {
-                                lora_v_down = to_f32(compute_ctx, lora_tensors[split_v_d_name]);
-                            }
-
-                            if (lora_tensors.find(split_v_u_name) != lora_tensors.end()) {
-                                lora_v_up = to_f32(compute_ctx, lora_tensors[split_v_u_name]);
-                            }
-
-                            if (lora_tensors.find(split_m_d_name) != lora_tensors.end()) {
-                                lora_m_down = to_f32(compute_ctx, lora_tensors[split_m_d_name]);
-                            }
-
-                            if (lora_tensors.find(split_m_u_name) != lora_tensors.end()) {
-                                lora_m_up = to_f32(compute_ctx, lora_tensors[split_m_u_name]);
-                            }
-
-                            float q_rank = lora_q_up->ne[0];
-                            float k_rank = lora_k_up->ne[0];
-                            float v_rank = lora_v_up->ne[0];
-                            float m_rank = lora_v_up->ne[0];
-
-                            float lora_q_scale = 1;
-                            float lora_k_scale = 1;
-                            float lora_v_scale = 1;
-                            float lora_m_scale = 1;
-
-                            if (lora_tensors.find(split_q_scale_name) != lora_tensors.end()) {
-                                lora_q_scale = ggml_backend_tensor_get_f32(lora_tensors[split_q_scale_name]);
-                                applied_lora_tensors.insert(split_q_scale_name);
-                            }
-                            if (lora_tensors.find(split_k_scale_name) != lora_tensors.end()) {
-                                lora_k_scale = ggml_backend_tensor_get_f32(lora_tensors[split_k_scale_name]);
-                                applied_lora_tensors.insert(split_k_scale_name);
-                            }
-                            if (lora_tensors.find(split_v_scale_name) != lora_tensors.end()) {
-                                lora_v_scale = ggml_backend_tensor_get_f32(lora_tensors[split_v_scale_name]);
-                                applied_lora_tensors.insert(split_v_scale_name);
-                            }
-                            if (lora_tensors.find(split_m_scale_name) != lora_tensors.end()) {
-                                lora_m_scale = ggml_backend_tensor_get_f32(lora_tensors[split_m_scale_name]);
-                                applied_lora_tensors.insert(split_m_scale_name);
-                            }
-
-                            if (lora_tensors.find(split_q_alpha_name) != lora_tensors.end()) {
-                                float lora_q_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_q_alpha_name]);
-                                applied_lora_tensors.insert(split_q_alpha_name);
-                                lora_q_scale = lora_q_alpha / q_rank;
-                            }
-                            if (lora_tensors.find(split_k_alpha_name) != lora_tensors.end()) {
-                                float lora_k_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_k_alpha_name]);
-                                applied_lora_tensors.insert(split_k_alpha_name);
-                                lora_k_scale = lora_k_alpha / k_rank;
-                            }
-                            if (lora_tensors.find(split_v_alpha_name) != lora_tensors.end()) {
-                                float lora_v_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_v_alpha_name]);
-                                applied_lora_tensors.insert(split_v_alpha_name);
-                                lora_v_scale = lora_v_alpha / v_rank;
-                            }
-                            if (lora_tensors.find(split_m_alpha_name) != lora_tensors.end()) {
-                                float lora_m_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_m_alpha_name]);
-                                applied_lora_tensors.insert(split_m_alpha_name);
-                                lora_m_scale = lora_m_alpha / m_rank;
-                            }
-
-                            ggml_scale_inplace(compute_ctx, lora_q_down, lora_q_scale);
-                            ggml_scale_inplace(compute_ctx, lora_k_down, lora_k_scale);
-                            ggml_scale_inplace(compute_ctx, lora_v_down, lora_v_scale);
-                            ggml_scale_inplace(compute_ctx, lora_m_down, lora_m_scale);
-
-                            // print_ggml_tensor(lora_q_down, true);  //[3072, R, 1, 1]
-                            // print_ggml_tensor(lora_k_down, true);  //[3072, R, 1, 1]
-                            // print_ggml_tensor(lora_v_down, true);  //[3072, R, 1, 1]
-                            // print_ggml_tensor(lora_m_down, true);  //[3072, R, 1, 1]
-                            // print_ggml_tensor(lora_q_up, true);  //[R, 3072, 1, 1]
-                            // print_ggml_tensor(lora_k_up, true);  //[R, 3072, 1, 1]
-                            // print_ggml_tensor(lora_v_up, true);  //[R, 3072, 1, 1]
-                            // print_ggml_tensor(lora_m_up, true);  //[R, 12288, 1, 1]
-
-                            // these need to be stitched together this way:
-                            //                                 |q_up,0   ,0   ,0   |
-                            //                                 |0   ,k_up,0   ,0   |
-                            //                                 |0   ,0   ,v_up,0   |
-                            //                                 |0   ,0   ,0   ,m_up|
-                            // (q_down,k_down,v_down,m_down) . (q   ,k   ,v   ,m)
-
-                            // up_concat will be [21504, R*4, 1, 1]
-                            // down_concat will be [R*4, 3072, 1, 1]
-
-                            ggml_tensor* lora_down_concat = ggml_concat(compute_ctx, ggml_concat(compute_ctx, lora_q_down, lora_k_down, 1), ggml_concat(compute_ctx, lora_v_down, lora_m_down, 1), 1);
-                            // print_ggml_tensor(lora_down_concat, true);  //[3072, R*4, 1, 1]
-
-                            // this also means that if rank is bigger than 672, it is less memory efficient to do it this way (should be fine)
-                            // print_ggml_tensor(lora_q_up, true);  //[3072, R, 1, 1]
-                            ggml_tensor* z     = ggml_dup_tensor(compute_ctx, lora_q_up);
-                            ggml_tensor* mlp_z = ggml_dup_tensor(compute_ctx, lora_m_up);
-                            ggml_scale(compute_ctx, z, 0);
-                            ggml_scale(compute_ctx, mlp_z, 0);
-                            ggml_tensor* zz = ggml_concat(compute_ctx, z, z, 1);
-
-                            ggml_tensor* q_up = ggml_concat(compute_ctx, ggml_concat(compute_ctx, lora_q_up, zz, 1), mlp_z, 1);
-                            ggml_tensor* k_up = ggml_concat(compute_ctx, ggml_concat(compute_ctx, z, lora_k_up, 1), ggml_concat(compute_ctx, z, mlp_z, 1), 1);
-                            ggml_tensor* v_up = ggml_concat(compute_ctx, ggml_concat(compute_ctx, zz, lora_v_up, 1), mlp_z, 1);
-                            ggml_tensor* m_up = ggml_concat(compute_ctx, ggml_concat(compute_ctx, zz, z, 1), lora_m_up, 1);
-                            // print_ggml_tensor(q_up, true);  //[R, 21504, 1, 1]
-                            // print_ggml_tensor(k_up, true);  //[R, 21504, 1, 1]
-                            // print_ggml_tensor(v_up, true);  //[R, 21504, 1, 1]
-                            // print_ggml_tensor(m_up, true);  //[R, 21504, 1, 1]
-
-                            ggml_tensor* lora_up_concat = ggml_concat(compute_ctx, ggml_concat(compute_ctx, q_up, k_up, 0), ggml_concat(compute_ctx, v_up, m_up, 0), 0);
-                            // print_ggml_tensor(lora_up_concat, true);  //[R*4, 21504, 1, 1]
-
-                            lora_down = ggml_cont(compute_ctx, lora_down_concat);
-                            lora_up   = ggml_cont(compute_ctx, lora_up_concat);
-
-                            applied_lora_tensors.insert(split_q_u_name);
-                            applied_lora_tensors.insert(split_k_u_name);
-                            applied_lora_tensors.insert(split_v_u_name);
-                            applied_lora_tensors.insert(split_m_u_name);
-
-                            applied_lora_tensors.insert(split_q_d_name);
-                            applied_lora_tensors.insert(split_k_d_name);
-                            applied_lora_tensors.insert(split_v_d_name);
-                            applied_lora_tensors.insert(split_m_d_name);
-                        }
-                    } else {
-                        lora_up_name   = full_key + lora_ups[type] + ".weight";
-                        lora_down_name = full_key + lora_downs[type] + ".weight";
-                        lora_mid_name  = full_key + ".lora_mid.weight";
-
-                        alpha_name = full_key + ".alpha";
-                        scale_name = full_key + ".scale";
-
-                        if (lora_tensors.find(lora_up_name) != lora_tensors.end()) {
-                            lora_up = to_f32(compute_ctx, lora_tensors[lora_up_name]);
-                            applied_lora_tensors.insert(lora_up_name);
-                        }
-
-                        if (lora_tensors.find(lora_down_name) != lora_tensors.end()) {
-                            lora_down = to_f32(compute_ctx, lora_tensors[lora_down_name]);
-                            applied_lora_tensors.insert(lora_down_name);
-                        }
-
-                        if (lora_tensors.find(lora_mid_name) != lora_tensors.end()) {
-                            lora_mid = to_f32(compute_ctx, lora_tensors[lora_mid_name]);
-                            applied_lora_tensors.insert(lora_mid_name);
-                        }
-                    }
-
-                    if (lora_up == nullptr || lora_down == nullptr) {
-                        continue;
-                    }
-                    // calc_scale
-                    // TODO: .dora_scale?
-                    int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1];
-                    if (lora_tensors.find(scale_name) != lora_tensors.end()) {
-                        scale_value = ggml_backend_tensor_get_f32(lora_tensors[scale_name]);
-                        applied_lora_tensors.insert(scale_name);
-                    } else if (lora_tensors.find(alpha_name) != lora_tensors.end()) {
-                        float alpha = ggml_backend_tensor_get_f32(lora_tensors[alpha_name]);
-                        scale_value = alpha / rank;
-                        // LOG_DEBUG("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value);
-                        applied_lora_tensors.insert(alpha_name);
-                    }
-
-                    updown = ggml_merge_lora(compute_ctx, lora_down, lora_up, lora_mid);
-                }
-                scale_value *= multiplier;
-                ggml_tensor* original_tensor = model_tensor;
-                if (!ggml_backend_is_cpu(runtime_backend) && ggml_backend_buffer_is_host(original_tensor->buffer)) {
-                    model_tensor = ggml_dup_tensor(compute_ctx, model_tensor);
-                    set_backend_tensor_data(model_tensor, original_tensor->data);
-                }
-                updown = ggml_reshape(compute_ctx, updown, model_tensor);
-                GGML_ASSERT(ggml_nelements(updown) == ggml_nelements(model_tensor));
-                updown = ggml_scale_inplace(compute_ctx, updown, scale_value);
-                ggml_tensor* final_tensor;
-                if (model_tensor->type != GGML_TYPE_F32 && model_tensor->type != GGML_TYPE_F16) {
-                    final_tensor = to_f32(compute_ctx, model_tensor);
-                    final_tensor = ggml_add_inplace(compute_ctx, final_tensor, updown);
-                    final_tensor = ggml_cpy(compute_ctx, final_tensor, model_tensor);
-                } else {
-                    final_tensor = ggml_add_inplace(compute_ctx, model_tensor, updown);
-                }
-                ggml_build_forward_expand(gf, final_tensor);
-                if (!ggml_backend_is_cpu(runtime_backend) && ggml_backend_buffer_is_host(original_tensor->buffer)) {
-                    original_tensor_to_final_tensor[original_tensor] = final_tensor;
-                }
-                break;
-            }
-        }
-        size_t total_lora_tensors_count   = 0;
-        size_t applied_lora_tensors_count = 0;
-
-        for (auto& kv : lora_tensors) {
-            total_lora_tensors_count++;
-            if (applied_lora_tensors.find(kv.first) == applied_lora_tensors.end()) {
-                LOG_WARN("unused lora tensor |%s|", kv.first.c_str());
-                print_ggml_tensor(kv.second, true);
-                // exit(0);
-            } else {
-                applied_lora_tensors_count++;
-            }
-        }
-        /* Don't worry if this message shows up twice in the logs per LoRA,
-         * this function is called once to calculate the required buffer size
-         * and then again to actually generate a graph to be used */
-        if (applied_lora_tensors_count != total_lora_tensors_count) {
-            LOG_WARN("Only (%lu / %lu) LoRA tensors will be applied",
-                     applied_lora_tensors_count, total_lora_tensors_count);
-        } else {
-            LOG_DEBUG("(%lu / %lu) LoRA tensors will be applied",
-                      applied_lora_tensors_count, total_lora_tensors_count);
-        }
-
-        return gf;
-    }
-
-    void apply(std::map<std::string, struct ggml_tensor*> model_tensors, SDVersion version, int n_threads) {
-        auto get_graph = [&]() -> struct ggml_cgraph* {
-            return build_lora_graph(model_tensors, version);
-        };
-        GGMLRunner::compute(get_graph, n_threads, false);
-        for (auto item : original_tensor_to_final_tensor) {
-            ggml_tensor* original_tensor = item.first;
-            ggml_tensor* final_tensor    = item.second;
-
-            ggml_backend_tensor_copy(final_tensor, original_tensor);
-        }
-        original_tensor_to_final_tensor.clear();
-        GGMLRunner::free_compute_buffer();
-    }
-};
-
-#endif  // __LORA_HPP__

model.h

@@ -1,292 +0,0 @@
-#ifndef __MODEL_H__
-#define __MODEL_H__
-
-#include <functional>
-#include <map>
-#include <memory>
-#include <set>
-#include <sstream>
-#include <string>
-#include <tuple>
-#include <utility>
-#include <vector>
-
-#include "ggml-backend.h"
-#include "ggml.h"
-#include "gguf.h"
-#include "json.hpp"
-#include "zip.h"
-
-#define SD_MAX_DIMS 5
-
-enum SDVersion {
-    VERSION_SD1,
-    VERSION_SD1_INPAINT,
-    VERSION_SD1_PIX2PIX,
-    VERSION_SD2,
-    VERSION_SD2_INPAINT,
-    VERSION_SDXL,
-    VERSION_SDXL_INPAINT,
-    VERSION_SDXL_PIX2PIX,
-    VERSION_SVD,
-    VERSION_SD3,
-    VERSION_FLUX,
-    VERSION_FLUX_FILL,
-    VERSION_FLUX_CONTROLS,
-    VERSION_FLEX_2,
-    VERSION_WAN2,
-    VERSION_WAN2_2_I2V,
-    VERSION_WAN2_2_TI2V,
-    VERSION_QWEN_IMAGE,
-    VERSION_COUNT,
-};
-
-static inline bool sd_version_is_sd1(SDVersion version) {
-    if (version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX) {
-        return true;
-    }
-    return false;
-}
-
-static inline bool sd_version_is_sd2(SDVersion version) {
-    if (version == VERSION_SD2 || version == VERSION_SD2_INPAINT) {
-        return true;
-    }
-    return false;
-}
-
-static inline bool sd_version_is_sdxl(SDVersion version) {
-    if (version == VERSION_SDXL || version == VERSION_SDXL_INPAINT || version == VERSION_SDXL_PIX2PIX) {
-        return true;
-    }
-    return false;
-}
-
-static inline bool sd_version_is_sd3(SDVersion version) {
-    if (version == VERSION_SD3) {
-        return true;
-    }
-    return false;
-}
-
-static inline bool sd_version_is_flux(SDVersion version) {
-    if (version == VERSION_FLUX || version == VERSION_FLUX_FILL || version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2) {
-        return true;
-    }
-    return false;
-}
-
-static inline bool sd_version_is_wan(SDVersion version) {
-    if (version == VERSION_WAN2 || version == VERSION_WAN2_2_I2V || version == VERSION_WAN2_2_TI2V) {
-        return true;
-    }
-    return false;
-}
-
-static inline bool sd_version_is_qwen_image(SDVersion version) {
-    if (version == VERSION_QWEN_IMAGE) {
-        return true;
-    }
-    return false;
-}
-
-static inline bool sd_version_is_inpaint(SDVersion version) {
-    if (version == VERSION_SD1_INPAINT || version == VERSION_SD2_INPAINT || version == VERSION_SDXL_INPAINT || version == VERSION_FLUX_FILL || version == VERSION_FLEX_2) {
-        return true;
-    }
-    return false;
-}
-
-static inline bool sd_version_is_dit(SDVersion version) {
-    if (sd_version_is_flux(version) ||
-        sd_version_is_sd3(version) ||
-        sd_version_is_wan(version) ||
-        sd_version_is_qwen_image(version)) {
-        return true;
-    }
-    return false;
-}
-
-static inline bool sd_version_is_unet_edit(SDVersion version) {
-    return version == VERSION_SD1_PIX2PIX || version == VERSION_SDXL_PIX2PIX;
-}
-
-static inline bool sd_version_is_control(SDVersion version) {
-    return version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2;
-}
-
-static bool sd_version_is_inpaint_or_unet_edit(SDVersion version) {
-    return sd_version_is_unet_edit(version) || sd_version_is_inpaint(version) || sd_version_is_control(version);
-}
-
-enum PMVersion {
-    PM_VERSION_1,
-    PM_VERSION_2,
-};
-
-struct TensorStorage {
-    std::string name;
-    ggml_type type          = GGML_TYPE_F32;
-    bool is_bf16            = false;
-    bool is_f8_e4m3         = false;
-    bool is_f8_e5m2         = false;
-    bool is_f64             = false;
-    bool is_i64             = false;
-    int64_t ne[SD_MAX_DIMS] = {1, 1, 1, 1, 1};
-    int n_dims              = 0;
-
-    size_t file_index = 0;
-    int index_in_zip  = -1;  // >= means stored in a zip file
-    uint64_t offset   = 0;   // offset in file
-
-    TensorStorage() = default;
-
-    TensorStorage(std::string name, ggml_type type, const int64_t* ne, int n_dims, size_t file_index, size_t offset = 0)
-        : name(std::move(name)), type(type), n_dims(n_dims), file_index(file_index), offset(offset) {
-        for (int i = 0; i < n_dims; i++) {
-            this->ne[i] = ne[i];
-        }
-    }
-
-    int64_t nelements() const {
-        int64_t n = 1;
-        for (int i = 0; i < SD_MAX_DIMS; i++) {
-            n *= ne[i];
-        }
-        return n;
-    }
-
-    int64_t nbytes() const {
-        return nelements() * ggml_type_size(type) / ggml_blck_size(type);
-    }
-
-    int64_t nbytes_to_read() const {
-        if (is_bf16 || is_f8_e4m3 || is_f8_e5m2) {
-            return nbytes() / 2;
-        } else if (is_f64 || is_i64) {
-            return nbytes() * 2;
-        } else {
-            return nbytes();
-        }
-    }
-
-    void unsqueeze() {
-        if (n_dims == 2) {
-            n_dims = 4;
-            ne[3]  = ne[1];
-            ne[2]  = ne[0];
-            ne[1]  = 1;
-            ne[0]  = 1;
-        }
-    }
-
-    std::vector<TensorStorage> chunk(size_t n) {
-        std::vector<TensorStorage> chunks;
-        uint64_t chunk_size = nbytes_to_read() / n;
-        // printf("%d/%d\n", chunk_size, nbytes_to_read());
-        reverse_ne();
-        for (size_t i = 0; i < n; i++) {
-            TensorStorage chunk_i = *this;
-            chunk_i.ne[0]         = ne[0] / n;
-            chunk_i.offset        = offset + i * chunk_size;
-            chunk_i.reverse_ne();
-            chunks.push_back(chunk_i);
-        }
-        reverse_ne();
-        return chunks;
-    }
-
-    void reverse_ne() {
-        int64_t new_ne[SD_MAX_DIMS] = {1, 1, 1, 1, 1};
-        for (int i = 0; i < n_dims; i++) {
-            new_ne[i] = ne[n_dims - 1 - i];
-        }
-        for (int i = 0; i < n_dims; i++) {
-            ne[i] = new_ne[i];
-        }
-    }
-
-    std::string to_string() const {
-        std::stringstream ss;
-        const char* type_name = ggml_type_name(type);
-        if (is_bf16) {
-            type_name = "bf16";
-        } else if (is_f8_e4m3) {
-            type_name = "f8_e4m3";
-        } else if (is_f8_e5m2) {
-            type_name = "f8_e5m2";
-        } else if (is_f64) {
-            type_name = "f64";
-        } else if (is_i64) {
-            type_name = "i64";
-        }
-        ss << name << " | " << type_name << " | ";
-        ss << n_dims << " [";
-        for (int i = 0; i < SD_MAX_DIMS; i++) {
-            ss << ne[i];
-            if (i != SD_MAX_DIMS - 1) {
-                ss << ", ";
-            }
-        }
-        ss << "]";
-        return ss.str();
-    }
-};
-
-typedef std::function<bool(const TensorStorage&, ggml_tensor**)> on_new_tensor_cb_t;
-
-typedef std::map<std::string, enum ggml_type> String2GGMLType;
-
-class ModelLoader {
-protected:
-    std::vector<std::string> file_paths_;
-    std::vector<TensorStorage> tensor_storages;
-
-    bool parse_data_pkl(uint8_t* buffer,
-                        size_t buffer_size,
-                        zip_t* zip,
-                        std::string dir,
-                        size_t file_index,
-                        const std::string prefix);
-
-    bool init_from_gguf_file(const std::string& file_path, const std::string& prefix = "");
-    bool init_from_safetensors_file(const std::string& file_path, const std::string& prefix = "");
-    bool init_from_ckpt_file(const std::string& file_path, const std::string& prefix = "");
-    bool init_from_diffusers_file(const std::string& file_path, const std::string& prefix = "");
-
-public:
-    String2GGMLType tensor_storages_types;
-
-    bool init_from_file(const std::string& file_path, const std::string& prefix = "");
-    bool model_is_unet();
-    SDVersion get_sd_version();
-    std::map<ggml_type, uint32_t> get_wtype_stat();
-    std::map<ggml_type, uint32_t> get_conditioner_wtype_stat();
-    std::map<ggml_type, uint32_t> get_diffusion_model_wtype_stat();
-    std::map<ggml_type, uint32_t> get_vae_wtype_stat();
-    void set_wtype_override(ggml_type wtype, std::string prefix = "");
-    bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads = 0);
-    bool load_tensors(std::map<std::string, struct ggml_tensor*>& tensors,
-                      std::set<std::string> ignore_tensors = {},
-                      int n_threads                        = 0);
-
-    std::vector<std::string> get_tensor_names() const {
-        std::vector<std::string> names;
-        for (const auto& ts : tensor_storages) {
-            names.push_back(ts.name);
-        }
-        return names;
-    }
-
-    bool save_to_gguf_file(const std::string& file_path, ggml_type type, const std::string& tensor_type_rules);
-    bool tensor_should_be_converted(const TensorStorage& tensor_storage, ggml_type type);
-    int64_t get_params_mem_size(ggml_backend_t backend, ggml_type type = GGML_TYPE_COUNT);
-    ~ModelLoader() = default;
-
-    static std::string load_merges();
-    static std::string load_qwen2_merges();
-    static std::string load_t5_tokenizer_json();
-    static std::string load_umt5_tokenizer_json();
-};
-
-#endif  // __MODEL_H__

preprocessing.hpp

@@ -1,226 +0,0 @@
-#ifndef __PREPROCESSING_HPP__
-#define __PREPROCESSING_HPP__
-
-#include "ggml_extend.hpp"
-#define M_PI_ 3.14159265358979323846
-
-void convolve(struct ggml_tensor* input, struct ggml_tensor* output, struct ggml_tensor* kernel, int padding) {
-    struct ggml_init_params params;
-    params.mem_size                 = 80 * input->ne[0] * input->ne[1];  // 20M for 512x512
-    params.mem_buffer               = nullptr;
-    params.no_alloc                 = false;
-    struct ggml_context* ctx0       = ggml_init(params);
-    struct ggml_tensor* kernel_fp16 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F16, kernel->ne[0], kernel->ne[1], 1, 1);
-    ggml_fp32_to_fp16_row((float*)kernel->data, (ggml_fp16_t*)kernel_fp16->data, ggml_nelements(kernel));
-    ggml_tensor* h  = ggml_conv_2d(ctx0, kernel_fp16, input, 1, 1, padding, padding, 1, 1);
-    ggml_cgraph* gf = ggml_new_graph(ctx0);
-    ggml_build_forward_expand(gf, ggml_cpy(ctx0, h, output));
-    ggml_graph_compute_with_ctx(ctx0, gf, 1);
-    ggml_free(ctx0);
-}
-
-void gaussian_kernel(struct ggml_tensor* kernel) {
-    int ks_mid   = kernel->ne[0] / 2;
-    float sigma  = 1.4f;
-    float normal = 1.f / (2.0f * M_PI_ * powf(sigma, 2.0f));
-    for (int y = 0; y < kernel->ne[0]; y++) {
-        float gx = -ks_mid + y;
-        for (int x = 0; x < kernel->ne[1]; x++) {
-            float gy = -ks_mid + x;
-            float k_ = expf(-((gx * gx + gy * gy) / (2.0f * powf(sigma, 2.0f)))) * normal;
-            ggml_tensor_set_f32(kernel, k_, x, y);
-        }
-    }
-}
-
-void grayscale(struct ggml_tensor* rgb_img, struct ggml_tensor* grayscale) {
-    for (int iy = 0; iy < rgb_img->ne[1]; iy++) {
-        for (int ix = 0; ix < rgb_img->ne[0]; ix++) {
-            float r    = ggml_tensor_get_f32(rgb_img, ix, iy);
-            float g    = ggml_tensor_get_f32(rgb_img, ix, iy, 1);
-            float b    = ggml_tensor_get_f32(rgb_img, ix, iy, 2);
-            float gray = 0.2989f * r + 0.5870f * g + 0.1140f * b;
-            ggml_tensor_set_f32(grayscale, gray, ix, iy);
-        }
-    }
-}
-
-void prop_hypot(struct ggml_tensor* x, struct ggml_tensor* y, struct ggml_tensor* h) {
-    int n_elements = ggml_nelements(h);
-    float* dx      = (float*)x->data;
-    float* dy      = (float*)y->data;
-    float* dh      = (float*)h->data;
-    for (int i = 0; i < n_elements; i++) {
-        dh[i] = sqrtf(dx[i] * dx[i] + dy[i] * dy[i]);
-    }
-}
-
-void prop_arctan2(struct ggml_tensor* x, struct ggml_tensor* y, struct ggml_tensor* h) {
-    int n_elements = ggml_nelements(h);
-    float* dx      = (float*)x->data;
-    float* dy      = (float*)y->data;
-    float* dh      = (float*)h->data;
-    for (int i = 0; i < n_elements; i++) {
-        dh[i] = atan2f(dy[i], dx[i]);
-    }
-}
-
-void normalize_tensor(struct ggml_tensor* g) {
-    int n_elements = ggml_nelements(g);
-    float* dg      = (float*)g->data;
-    float max      = -INFINITY;
-    for (int i = 0; i < n_elements; i++) {
-        max = dg[i] > max ? dg[i] : max;
-    }
-    max = 1.0f / max;
-    for (int i = 0; i < n_elements; i++) {
-        dg[i] *= max;
-    }
-}
-
-void non_max_supression(struct ggml_tensor* result, struct ggml_tensor* G, struct ggml_tensor* D) {
-    for (int iy = 1; iy < result->ne[1] - 1; iy++) {
-        for (int ix = 1; ix < result->ne[0] - 1; ix++) {
-            float angle = ggml_tensor_get_f32(D, ix, iy) * 180.0f / M_PI_;
-            angle       = angle < 0.0f ? angle += 180.0f : angle;
-            float q     = 1.0f;
-            float r     = 1.0f;
-
-            // angle 0
-            if ((0 >= angle && angle < 22.5f) || (157.5f >= angle && angle <= 180)) {
-                q = ggml_tensor_get_f32(G, ix, iy + 1);
-                r = ggml_tensor_get_f32(G, ix, iy - 1);
-            }
-            // angle 45
-            else if (22.5f >= angle && angle < 67.5f) {
-                q = ggml_tensor_get_f32(G, ix + 1, iy - 1);
-                r = ggml_tensor_get_f32(G, ix - 1, iy + 1);
-            }
-            // angle 90
-            else if (67.5f >= angle && angle < 112.5) {
-                q = ggml_tensor_get_f32(G, ix + 1, iy);
-                r = ggml_tensor_get_f32(G, ix - 1, iy);
-            }
-            // angle 135
-            else if (112.5 >= angle && angle < 157.5f) {
-                q = ggml_tensor_get_f32(G, ix - 1, iy - 1);
-                r = ggml_tensor_get_f32(G, ix + 1, iy + 1);
-            }
-
-            float cur = ggml_tensor_get_f32(G, ix, iy);
-            if ((cur >= q) && (cur >= r)) {
-                ggml_tensor_set_f32(result, cur, ix, iy);
-            } else {
-                ggml_tensor_set_f32(result, 0.0f, ix, iy);
-            }
-        }
-    }
-}
-
-void threshold_hystersis(struct ggml_tensor* img, float high_threshold, float low_threshold, float weak, float strong) {
-    int n_elements = ggml_nelements(img);
-    float* imd     = (float*)img->data;
-    float max      = -INFINITY;
-    for (int i = 0; i < n_elements; i++) {
-        max = imd[i] > max ? imd[i] : max;
-    }
-    float ht = max * high_threshold;
-    float lt = ht * low_threshold;
-    for (int i = 0; i < n_elements; i++) {
-        float img_v = imd[i];
-        if (img_v >= ht) {  // strong pixel
-            imd[i] = strong;
-        } else if (img_v <= ht && img_v >= lt) {  // strong pixel
-            imd[i] = weak;
-        }
-    }
-
-    for (int iy = 0; iy < img->ne[1]; iy++) {
-        for (int ix = 0; ix < img->ne[0]; ix++) {
-            if (ix >= 3 && ix <= img->ne[0] - 3 && iy >= 3 && iy <= img->ne[1] - 3) {
-                ggml_tensor_set_f32(img, ggml_tensor_get_f32(img, ix, iy), ix, iy);
-            } else {
-                ggml_tensor_set_f32(img, 0.0f, ix, iy);
-            }
-        }
-    }
-
-    // hysteresis
-    for (int iy = 1; iy < img->ne[1] - 1; iy++) {
-        for (int ix = 1; ix < img->ne[0] - 1; ix++) {
-            float imd_v = ggml_tensor_get_f32(img, ix, iy);
-            if (imd_v == weak) {
-                if (ggml_tensor_get_f32(img, ix + 1, iy - 1) == strong || ggml_tensor_get_f32(img, ix + 1, iy) == strong ||
-                    ggml_tensor_get_f32(img, ix, iy - 1) == strong || ggml_tensor_get_f32(img, ix, iy + 1) == strong ||
-                    ggml_tensor_get_f32(img, ix - 1, iy - 1) == strong || ggml_tensor_get_f32(img, ix - 1, iy) == strong) {
-                    ggml_tensor_set_f32(img, strong, ix, iy);
-                } else {
-                    ggml_tensor_set_f32(img, 0.0f, ix, iy);
-                }
-            }
-        }
-    }
-}
-
-bool preprocess_canny(sd_image_t img, float high_threshold, float low_threshold, float weak, float strong, bool inverse) {
-    struct ggml_init_params params;
-    params.mem_size               = static_cast<size_t>(40 * img.width * img.height);  // 10MB for 512x512
-    params.mem_buffer             = nullptr;
-    params.no_alloc               = false;
-    struct ggml_context* work_ctx = ggml_init(params);
-
-    if (!work_ctx) {
-        LOG_ERROR("ggml_init() failed");
-        return false;
-    }
-
-    float kX[9] = {
-        -1, 0, 1,
-        -2, 0, 2,
-        -1, 0, 1};
-
-    float kY[9] = {
-        1, 2, 1,
-        0, 0, 0,
-        -1, -2, -1};
-
-    // generate kernel
-    int kernel_size             = 5;
-    struct ggml_tensor* gkernel = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, kernel_size, kernel_size, 1, 1);
-    struct ggml_tensor* sf_kx   = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, 3, 3, 1, 1);
-    memcpy(sf_kx->data, kX, ggml_nbytes(sf_kx));
-    struct ggml_tensor* sf_ky = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, 3, 3, 1, 1);
-    memcpy(sf_ky->data, kY, ggml_nbytes(sf_ky));
-    gaussian_kernel(gkernel);
-    struct ggml_tensor* image      = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, img.width, img.height, 3, 1);
-    struct ggml_tensor* image_gray = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, img.width, img.height, 1, 1);
-    struct ggml_tensor* iX         = ggml_dup_tensor(work_ctx, image_gray);
-    struct ggml_tensor* iY         = ggml_dup_tensor(work_ctx, image_gray);
-    struct ggml_tensor* G          = ggml_dup_tensor(work_ctx, image_gray);
-    struct ggml_tensor* tetha      = ggml_dup_tensor(work_ctx, image_gray);
-    sd_image_to_tensor(img, image);
-    grayscale(image, image_gray);
-    convolve(image_gray, image_gray, gkernel, 2);
-    convolve(image_gray, iX, sf_kx, 1);
-    convolve(image_gray, iY, sf_ky, 1);
-    prop_hypot(iX, iY, G);
-    normalize_tensor(G);
-    prop_arctan2(iX, iY, tetha);
-    non_max_supression(image_gray, G, tetha);
-    threshold_hystersis(image_gray, high_threshold, low_threshold, weak, strong);
-    // to RGB channels
-    for (int iy = 0; iy < img.height; iy++) {
-        for (int ix = 0; ix < img.width; ix++) {
-            float gray = ggml_tensor_get_f32(image_gray, ix, iy);
-            gray       = inverse ? 1.0f - gray : gray;
-            ggml_tensor_set_f32(image, gray, ix, iy);
-            ggml_tensor_set_f32(image, gray, ix, iy, 1);
-            ggml_tensor_set_f32(image, gray, ix, iy, 2);
-        }
-    }
-    sd_tensor_to_image(image, img.data);
-    ggml_free(work_ctx);
-    return true;
-}
-
-#endif  // __PREPROCESSING_HPP__