enable flash attn by default

.github/pull_request_template.md

@@ -1,15 +0,0 @@
-## Summary
-
-<!-- Describe what changed and why. Keep the PR focused on one clear change. -->
-
-## Related Issue / Discussion
-
-<!-- Link related issues, discussions, or previous PRs if applicable. -->
-
-## Additional Information
-
-<!-- Add verification notes, screenshots, sample output, or other context when applicable. -->
-
-## Checklist
-
-- [ ] I have read and confirmed this PR follows the [contribution guidelines](https://github.com/leejet/stable-diffusion.cpp/blob/master/CONTRIBUTING.md).

.github/workflows/build.yml

@@ -21,14 +21,11 @@ on:
         "**/*.c",
         "**/*.cpp",
         "**/*.cu",
-        "examples/server/frontend",
-        "examples/server/frontend/**",
       ]
   pull_request:
     types: [opened, synchronize, reopened]
     paths:
       [
-        ".github/workflows/**",
         "**/CMakeLists.txt",
         "**/Makefile",
         "**/*.h",
@@ -36,8 +33,6 @@ on:
         "**/*.c",
         "**/*.cpp",
         "**/*.cu",
-        "examples/server/frontend",
-        "examples/server/frontend/**",
       ]
 
 env:
@@ -58,16 +53,6 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 10.15.1
-
       - name: Dependencies
         id: depends
         run: |
@@ -85,7 +70,7 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        uses: pr-mpt/actions-commit-hash@v2
 
       - name: Fetch system info
         id: system-info
@@ -121,16 +106,6 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 10.15.1
-
       - name: Dependencies
         id: depends
         run: |
@@ -148,7 +123,7 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        uses: pr-mpt/actions-commit-hash@v2
 
       - name: Fetch system info
         id: system-info
@@ -176,7 +151,6 @@ jobs:
 
   build-and-push-docker-images:
     name: Build and push container images
-    if: ${{ github.event_name != 'pull_request' }}
     runs-on: ubuntu-latest
 
     permissions:
@@ -188,7 +162,7 @@ jobs:
 
     strategy:
       matrix:
-        variant: [musa, sycl, vulkan, cuda]
+        variant: [musa, sycl, vulkan]
 
     env:
       REGISTRY: ghcr.io
@@ -200,20 +174,10 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 10.15.1
-
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        uses: pr-mpt/actions-commit-hash@v2
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -242,7 +206,6 @@ jobs:
         id: build-push
         uses: docker/build-push-action@v6
         with:
-          context: .
           platforms: linux/amd64
           push: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
           file: Dockerfile.${{ matrix.variant }}
@@ -260,16 +223,6 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 10.15.1
-
       - name: Dependencies
         id: depends
         run: |
@@ -287,7 +240,7 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        uses: pr-mpt/actions-commit-hash@v2
 
       - name: Fetch system info
         id: system-info
@@ -341,16 +294,6 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 10.15.1
-
       - name: Install cuda-toolkit
         id: cuda-toolkit
         if: ${{ matrix.build == 'cuda12' }}
@@ -397,7 +340,7 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        uses: pr-mpt/actions-commit-hash@v2
 
       - name: Pack artifacts
         id: pack_artifacts
@@ -444,114 +387,18 @@ jobs:
           path: |
             sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip
 
-  windows-latest-rocm:
-    runs-on: windows-2022
-
-    env:
-      ROCM_VERSION: "7.12.0"
-      GPU_TARGETS: "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1200;gfx1201"
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: recursive
-
-      - name: Cache ROCm Installation
-        id: cache-rocm
-        uses: actions/cache@v4
-        with:
-          path: C:\TheRock\build
-          key: rocm-${{ env.ROCM_VERSION }}-gfx1151-${{ runner.os }}
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.16
-        with:
-          key: windows-latest-rocm-${{ env.ROCM_VERSION }}-x64
-          evict-old-files: 1d
-
-      - name: Install ROCm
-        if: steps.cache-rocm.outputs.cache-hit != 'true'
-        run: |
-          $ErrorActionPreference = "Stop"
-          write-host "Downloading AMD ROCm ${{ env.ROCM_VERSION }} tarball"
-          Invoke-WebRequest -Uri "https://repo.amd.com/rocm/tarball/therock-dist-windows-gfx1151-${{ env.ROCM_VERSION }}.tar.gz" -OutFile "${env:RUNNER_TEMP}\rocm.tar.gz"
-          write-host "Extracting ROCm tarball"
-          mkdir C:\TheRock\build -Force
-          tar -xzf "${env:RUNNER_TEMP}\rocm.tar.gz" -C C:\TheRock\build --strip-components=1
-          write-host "Completed ROCm extraction"
-
-      - name: Setup ROCm Environment
-        run: |
-          $rocmPath = "C:\TheRock\build"
-          echo "HIP_PATH=$rocmPath" >> $env:GITHUB_ENV
-          echo "HIP_DEVICE_LIB_PATH=$rocmPath\lib\llvm\amdgcn\bitcode" >> $env:GITHUB_ENV
-          echo "HIP_PLATFORM=amd" >> $env:GITHUB_ENV
-          echo "LLVM_PATH=$rocmPath\lib\llvm" >> $env:GITHUB_ENV
-          echo "$rocmPath\bin" >> $env:GITHUB_PATH
-          echo "$rocmPath\lib\llvm\bin" >> $env:GITHUB_PATH
-
-      - name: Build
-        run: |
-          mkdir build
-          cd build
-          cmake .. `
-            -G "Unix Makefiles" `
-            -DCMAKE_PREFIX_PATH="${env:HIP_PATH}" `
-            -DSD_HIPBLAS=ON `
-            -DSD_BUILD_SHARED_LIBS=ON `
-            -DGGML_NATIVE=OFF `
-            -DCMAKE_C_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
-            -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang++.exe" `
-            -DCMAKE_HIP_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
-            -DHIP_PATH="${env:HIP_PATH}" `
-            -DCMAKE_BUILD_TYPE=Release `
-            -DGPU_TARGETS="${{ env.GPU_TARGETS }}"
-          cmake --build . --config Release --parallel ${env:NUMBER_OF_PROCESSORS}
-
-      - name: Get commit hash
-        id: commit
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: pr-mpt/actions-commit-hash@v2
-
-      - name: Pack artifacts
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        run: |
-          cp "${env:HIP_PATH}\bin\hipblas.dll" "build\bin\"
-          cp "${env:HIP_PATH}\bin\libhipblaslt.dll" "build\bin\"
-          cp "${env:HIP_PATH}\bin\rocblas.dll" "build\bin\"
-          7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip .\build\bin\*
-
-      - name: Upload artifacts
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip
-          path: |
-            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip
-
   windows-latest-cmake-hip:
     runs-on: windows-2022
 
     env:
-      HIPSDK_INSTALLER_VERSION: "26.Q1"
-      ROCM_VERSION: "7.1.1"
-      GPU_TARGETS: "gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
+      HIPSDK_INSTALLER_VERSION: "25.Q3"
+      GPU_TARGETS: "gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
 
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: recursive
 
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 10.15.1
-
       - name: Cache ROCm Installation
         id: cache-rocm
         uses: actions/cache@v4
@@ -570,7 +417,7 @@ jobs:
         run: |
           $ErrorActionPreference = "Stop"
           write-host "Downloading AMD HIP SDK Installer"
-          Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-Win11-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe"
+          Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-WinSvr2022-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe"
           write-host "Installing AMD HIP SDK"
           $proc = Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -PassThru
           $completed = $proc.WaitForExit(600000)
@@ -616,171 +463,27 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        uses: pr-mpt/actions-commit-hash@v2
 
       - name: Pack artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
         run: |
           md "build\bin\rocblas\library\"
           md "build\bin\hipblaslt\library"
-          cp "${env:HIP_PATH}\bin\libhipblas.dll" "build\bin\"
-          cp "${env:HIP_PATH}\bin\libhipblaslt.dll" "build\bin\"
+          cp "${env:HIP_PATH}\bin\hipblas.dll" "build\bin\"
+          cp "${env:HIP_PATH}\bin\hipblaslt.dll" "build\bin\"
           cp "${env:HIP_PATH}\bin\rocblas.dll" "build\bin\"
           cp "${env:HIP_PATH}\bin\rocblas\library\*" "build\bin\rocblas\library\"
           cp "${env:HIP_PATH}\bin\hipblaslt\library\*" "build\bin\hipblaslt\library\"
-          7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip .\build\bin\*
+          7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-x64.zip .\build\bin\*
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
         uses: actions/upload-artifact@v4
         with:
-          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip
+          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-x64.zip
           path: |
-            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip
-
-  ubuntu-latest-rocm:
-    runs-on: ubuntu-24.04
-
-    env:
-      UBUNTU_VERSION: "24.04"
-
-    strategy:
-      matrix:
-        include:
-          - ROCM_VERSION: "7.2.1"
-            gpu_targets: "gfx908;gfx90a;gfx942;gfx1030;gfx1031;gfx1032;gfx1100;gfx1101;gfx1102;gfx1151;gfx1150;gfx1200;gfx1201"
-            build: 'x64'
-          - ROCM_VERSION: "7.12.0"
-            gpu_targets: "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1200;gfx1201"
-            build: x64
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 10.15.1
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.16
-        with:
-          key: ubuntu-rocm-cmake-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
-          evict-old-files: 1d
-
-      - name: Dependencies
-        id: depends
-        run: |
-          sudo apt install -y build-essential cmake wget zip ninja-build
-
-      - name: Setup Legacy ROCm
-        if: matrix.ROCM_VERSION == '7.2.1'
-        id: legacy_env
-        run: |
-          sudo mkdir --parents --mode=0755 /etc/apt/keyrings
-          wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \
-            gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
-
-          sudo tee /etc/apt/sources.list.d/rocm.list << EOF
-          deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${{ matrix.ROCM_VERSION }} noble main
-          EOF
-
-          sudo tee /etc/apt/preferences.d/rocm-pin-600 << EOF
-          Package: *
-          Pin: release o=repo.radeon.com
-          Pin-Priority: 600
-          EOF
-
-          sudo apt update
-          sudo apt-get install -y libssl-dev rocm-hip-sdk
-
-      - name: Free disk space
-        run: |
-          # Remove preinstalled SDKs and caches not needed for this job
-          sudo rm -rf /usr/share/dotnet || true
-          sudo rm -rf /usr/local/lib/android || true
-          sudo rm -rf /opt/ghc || true
-          sudo rm -rf /usr/local/.ghcup || true
-          sudo rm -rf /opt/hostedtoolcache || true
-
-          # Remove old package lists and caches
-          sudo rm -rf /var/lib/apt/lists/* || true
-          sudo apt clean
-
-      - name: Setup TheRock
-        if: matrix.ROCM_VERSION != '7.2.1'
-        id: therock_env
-        run: |
-          wget https://repo.amd.com/rocm/tarball/therock-dist-linux-gfx1151-${{ matrix.ROCM_VERSION }}.tar.gz
-          mkdir install
-          tar -xf *.tar.gz -C install
-          export ROCM_PATH=$(pwd)/install
-          echo ROCM_PATH=$ROCM_PATH >> $GITHUB_ENV
-          echo PATH=$PATH:$ROCM_PATH/bin >> $GITHUB_ENV
-          echo LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/llvm/lib:$ROCM_PATH/lib/rocprofiler-systems >> $GITHUB_ENV
-
-      - name: Build
-        id: cmake_build
-        run: |
-          mkdir build
-          cd build
-          cmake .. -G Ninja \
-            -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
-            -DCMAKE_HIP_FLAGS="-mllvm --amdgpu-unroll-threshold-local=600" \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DSD_HIPBLAS=ON \
-            -DHIP_PLATFORM=amd \
-            -DGPU_TARGETS="${{ matrix.gpu_targets }}" \
-            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
-            -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-            -DSD_BUILD_SHARED_LIBS=ON
-          cmake --build . --config Release
-
-      - name: Get commit hash
-        id: commit
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
-
-      - name: Prepare artifacts
-        id: prepare_artifacts
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        run: |
-          # Copy licenses
-          cp ggml/LICENSE ./build/bin/ggml.txt
-          cp LICENSE ./build/bin/stable-diffusion.cpp.txt
-
-      - name: Fetch system info
-        id: system-info
-        run: |
-          echo "CPU_ARCH=`uname -m`" >> "$GITHUB_OUTPUT"
-          echo "OS_NAME=`lsb_release -s -i`" >> "$GITHUB_OUTPUT"
-          echo "OS_VERSION=`lsb_release -s -r`" >> "$GITHUB_OUTPUT"
-          echo "OS_TYPE=`uname -s`" >> "$GITHUB_OUTPUT"
-
-      - name: Pack artifacts
-        id: pack_artifacts
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        run: |
-          cp ggml/LICENSE ./build/bin/ggml.txt
-          cp LICENSE ./build/bin/stable-diffusion.cpp.txt
-          zip -y -r sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip ./build/bin
-
-      - name: Upload artifacts
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip
-          path: |
-            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip
+            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-x64.zip
 
   release:
     if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
@@ -790,12 +493,10 @@ jobs:
     needs:
       - ubuntu-latest-cmake
       - ubuntu-latest-cmake-vulkan
-      - ubuntu-latest-rocm
       - build-and-push-docker-images
       - macOS-latest-cmake
       - windows-latest-cmake
       - windows-latest-cmake-hip
-      - windows-latest-rocm
 
     steps:
       - name: Clone
@@ -818,7 +519,7 @@ jobs:
 
       - name: Get commit hash
         id: commit
-        uses: prompt/actions-commit-hash@v2
+        uses: pr-mpt/actions-commit-hash@v2
 
       - name: Create release
         id: create_release

.gitmodules

@@ -1,12 +1,3 @@
 [submodule "ggml"]
     path = ggml
 	url = https://github.com/ggml-org/ggml.git
-[submodule "examples/server/frontend"]
-	path = examples/server/frontend
-	url = https://github.com/leejet/sdcpp-webui.git
-[submodule "thirdparty/libwebp"]
-	path = thirdparty/libwebp
-	url = https://github.com/webmproject/libwebp.git
-[submodule "thirdparty/libwebm"]
-	path = thirdparty/libwebm
-	url = https://github.com/webmproject/libwebm.git

CMakeLists.txt

@@ -11,10 +11,6 @@ endif()
 if (MSVC)
     add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
     add_compile_definitions(_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING)
-    add_compile_options(
-        $<$<COMPILE_LANGUAGE:C>:/MP>
-        $<$<COMPILE_LANGUAGE:CXX>:/MP>
-    )
 endif()
 
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
@@ -26,26 +22,6 @@ else()
     set(SD_STANDALONE OFF)
 endif()
 
-set(SD_SUBMODULE_WEBP FALSE)
-if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/libwebp/CMakeLists.txt")
-    set(SD_SUBMODULE_WEBP TRUE)
-endif()
-if(SD_SUBMODULE_WEBP)
-    set(SD_WEBP_DEFAULT ON)
-else()
-    set(SD_WEBP_DEFAULT ${SD_USE_SYSTEM_WEBP})
-endif()
-
-set(SD_SUBMODULE_WEBM FALSE)
-if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/libwebm/CMakeLists.txt")
-    set(SD_SUBMODULE_WEBM TRUE)
-endif()
-if(SD_SUBMODULE_WEBM)
-    set(SD_WEBM_DEFAULT ON)
-else()
-    set(SD_WEBM_DEFAULT ${SD_USE_SYSTEM_WEBM})
-endif()
-
 #
 # Option list
 #
@@ -53,10 +29,6 @@ endif()
 # general
 #option(SD_BUILD_TESTS                "sd: build tests"    ${SD_STANDALONE})
 option(SD_BUILD_EXAMPLES             "sd: build examples" ${SD_STANDALONE})
-option(SD_WEBP                       "sd: enable WebP image I/O support" ${SD_WEBP_DEFAULT})
-option(SD_USE_SYSTEM_WEBP            "sd: link against system libwebp" OFF)
-option(SD_WEBM                       "sd: enable WebM video output support" ${SD_WEBM_DEFAULT})
-option(SD_USE_SYSTEM_WEBM            "sd: link against system libwebm" OFF)
 option(SD_CUDA                       "sd: cuda backend" OFF)
 option(SD_HIPBLAS                    "sd: rocm backend" OFF)
 option(SD_METAL                      "sd: metal backend" OFF)
@@ -64,125 +36,60 @@ option(SD_VULKAN                     "sd: vulkan backend" OFF)
 option(SD_OPENCL                     "sd: opencl backend" OFF)
 option(SD_SYCL                       "sd: sycl backend" OFF)
 option(SD_MUSA                       "sd: musa backend" OFF)
+option(SD_FAST_SOFTMAX               "sd: x1.5 faster softmax, indeterministic (sometimes, same seed don't generate same image), cuda only" OFF)
 option(SD_BUILD_SHARED_LIBS          "sd: build shared libs" OFF)
 option(SD_BUILD_SHARED_GGML_LIB      "sd: build ggml as a separate shared lib" OFF)
 option(SD_USE_SYSTEM_GGML            "sd: use system-installed GGML library" OFF)
 #option(SD_BUILD_SERVER               "sd: build server example"                           ON)
 
-set(CMAKE_C_STANDARD 11)
-set(CMAKE_C_STANDARD_REQUIRED true)
-
-set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_STANDARD_REQUIRED true)
-
 if(SD_CUDA)
     message("-- Use CUDA as backend stable-diffusion")
     set(GGML_CUDA ON)
+    add_definitions(-DSD_USE_CUDA)
 endif()
 
 if(SD_METAL)
     message("-- Use Metal as backend stable-diffusion")
     set(GGML_METAL ON)
+    add_definitions(-DSD_USE_METAL)
 endif()
 
 if (SD_VULKAN)
     message("-- Use Vulkan as backend stable-diffusion")
     set(GGML_VULKAN ON)
+    add_definitions(-DSD_USE_VULKAN)
 endif ()
 
 if (SD_OPENCL)
     message("-- Use OpenCL as backend stable-diffusion")
     set(GGML_OPENCL ON)
+    add_definitions(-DSD_USE_OPENCL)
 endif ()
 
 if (SD_HIPBLAS)
     message("-- Use HIPBLAS as backend stable-diffusion")
     set(GGML_HIP ON)
+    add_definitions(-DSD_USE_CUDA)
+    if(SD_FAST_SOFTMAX)
+        set(GGML_CUDA_FAST_SOFTMAX ON)
+    endif()
 endif ()
 
 if(SD_MUSA)
     message("-- Use MUSA as backend stable-diffusion")
     set(GGML_MUSA ON)
-endif()
-
-if(SD_WEBP)
-    if(NOT SD_SUBMODULE_WEBP AND NOT SD_USE_SYSTEM_WEBP)
-        message(FATAL_ERROR "WebP support enabled but no source found.
-          Either initialize the submodule:\n  git submodule update --init thirdparty/libwebp\n\n"
-          "Or link against system library:\n  cmake (...) -DSD_USE_SYSTEM_WEBP=ON")
-    endif()
-    if(SD_USE_SYSTEM_WEBP)
-        find_package(WebP)
-        if(WebP_FOUND)
-            add_library(webp ALIAS WebP::webp)
-            # libwebp CMake target naming is not consistent across versions/distros.
-            # Some export WebP::libwebpmux, others export WebP::webpmux.
-            if(TARGET WebP::libwebpmux)
-                add_library(libwebpmux ALIAS WebP::libwebpmux)
-            elseif(TARGET WebP::webpmux)
-                add_library(libwebpmux ALIAS WebP::webpmux)
-            else()
-                message(FATAL_ERROR
-                    "Could not find a compatible webpmux target in system WebP package. "
-                    "Expected WebP::libwebpmux or WebP::webpmux."
-                )
-            endif()
-        else()
-            find_package(PkgConfig REQUIRED)
-            pkg_check_modules(WebP REQUIRED IMPORTED_TARGET GLOBAL libwebp)
-            pkg_check_modules(WebPMux REQUIRED IMPORTED_TARGET GLOBAL libwebpmux)
-            link_libraries(PkgConfig::WebP)
-            link_libraries(PkgConfig::WebPMux)
-            add_library(libwebpmux ALIAS PkgConfig::WebPMux)
-        endif()
-    endif()
-endif()
-
-if(SD_WEBM)
-    if(NOT SD_WEBP)
-        message(FATAL_ERROR "SD_WEBM requires SD_WEBP because WebM output reuses libwebp VP8 encoding.")
-    endif()
-    if(NOT SD_SUBMODULE_WEBM AND NOT SD_USE_SYSTEM_WEBM)
-        message(FATAL_ERROR "WebM support enabled but no source found.
-          Either initialize the submodule:\n  git submodule update --init thirdparty/libwebm\n\n"
-          "Or link against system library:\n  cmake (...) -DSD_USE_SYSTEM_WEBM=ON")
-    endif()
-    if(SD_USE_SYSTEM_WEBM)
-        find_package(PkgConfig)
-        if(PkgConfig_FOUND)
-            pkg_check_modules(WebM REQUIRED IMPORTED_TARGET GLOBAL libwebm)
-        endif()
-        if(PkgConfig_FOUND AND WebM_FOUND)
-            link_libraries(PkgConfig::WebM)
-        else()
-            find_path(WEBM_INCLUDE_DIR
-                NAMES mkvmuxer/mkvmuxer.h mkvparser/mkvparser.h common/webmids.h
-                PATH_SUFFIXES webm
-                REQUIRED)
-            find_library(WEBM_LIBRARY
-                NAMES webm libwebm
-                REQUIRED)
-
-            add_library(webm UNKNOWN IMPORTED)
-            set_target_properties(webm PROPERTIES
-                IMPORTED_LOCATION "${WEBM_LIBRARY}"
-                INTERFACE_INCLUDE_DIRECTORIES "${WEBM_INCLUDE_DIR}")
-        endif()
+    add_definitions(-DSD_USE_CUDA)
+    if(SD_FAST_SOFTMAX)
+        set(GGML_CUDA_FAST_SOFTMAX ON)
     endif()
 endif()
 
 set(SD_LIB stable-diffusion)
 
-file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
-    "src/*.h"
-    "src/*.cpp"
-    "src/*.hpp"
-    "src/model_io/*.h"
-    "src/model_io/*.cpp"
-    "src/tokenizers/*.h"
-    "src/tokenizers/*.cpp"
-    "src/tokenizers/vocab/*.h"
-    "src/tokenizers/vocab/*.cpp"
+file(GLOB SD_LIB_SOURCES
+    "*.h"
+    "*.cpp"
+    "*.hpp"
 )
 
 find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH)
@@ -212,7 +119,7 @@ endif()
 message(STATUS "stable-diffusion.cpp commit ${SDCPP_BUILD_COMMIT}")
 
 set_property(
-  SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src/version.cpp
+  SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/version.cpp
   APPEND PROPERTY COMPILE_DEFINITIONS
   SDCPP_BUILD_COMMIT=${SDCPP_BUILD_COMMIT} SDCPP_BUILD_VERSION=${SDCPP_BUILD_VERSION}
 )
@@ -239,6 +146,7 @@ if(SD_SYCL)
     message("-- Use SYCL as backend stable-diffusion")
     set(GGML_SYCL ON)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing -fsycl")
+    add_definitions(-DSD_USE_SYCL)
     # disable fast-math on host, see:
     # https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/2021-10/fp-model-fp.html
     if (WIN32)
@@ -274,7 +182,6 @@ endif()
 add_subdirectory(thirdparty)
 
 target_link_libraries(${SD_LIB} PUBLIC ggml zip)
-target_include_directories(${SD_LIB} PUBLIC . src include)
 target_include_directories(${SD_LIB} PUBLIC . thirdparty)
 target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17)
 
@@ -283,7 +190,7 @@ if (SD_BUILD_EXAMPLES)
     add_subdirectory(examples)
 endif()
 
-set(SD_PUBLIC_HEADERS include/stable-diffusion.h)
+set(SD_PUBLIC_HEADERS stable-diffusion.h)
 set_target_properties(${SD_LIB} PROPERTIES PUBLIC_HEADER "${SD_PUBLIC_HEADERS}")
 
 install(TARGETS ${SD_LIB} LIBRARY PUBLIC_HEADER)

CONTRIBUTING.md

@@ -1,65 +0,0 @@
-# Contributing
-
-This document collects general contribution conventions for this repository.
-
-## Before You Start
-
-Before opening a PR, please search existing PRs to avoid duplicating ongoing work.
-
-For large-scale refactors or changes with broad impact, please open an issue first to discuss the approach before submitting a PR.
-
-If you want to update a third-party dependency, please open an issue first instead of submitting a direct PR. See [Dependency Updates](#dependency-updates) for details.
-
-## Pull Requests
-
-Keep each PR focused on one clear change. Large or overly complex PRs are harder to review and may not be merged.
-
-Follow Conventional Commit-style subjects seen in history: `feat:`, `fix:`, `refactor:`, `ci:`, `docs:`, `chore:`. Keep subjects imperative and scoped.
-
-PRs should include:
-
-- What changed and why (short problem/solution summary).
-- Verification evidence when applicable (commands and key outputs).
-- Linked issue/PR context when applicable.
-- Screenshots or sample outputs for UI/visual behavior changes.
-
-## Code Style
-
-Format code according to the repository style before submitting changes.
-
-Formatting follows `.clang-format` (Chromium base, 4-space indent, no tabs). Run `format-code.sh` before opening a PR. Keep C++ standard at C++17-compatible patterns used in this repo.
-
-Naming conventions:
-
-- Use `PascalCase` for class/struct/type names.
-- In `PascalCase` names, preserve common abbreviations in uppercase, for example `SD`, `API`, `HTTP`, `JSON`, `RGB`, `VAE`, `TAE`, `LoRA`, and `WebP`.
-- Use `snake_case` for functions, methods, variables, and file names unless an existing API requires a different style.
-- Use a trailing underscore for private data member names, for example `hidden_size_` or `tokenizer_`.
-- Use `.h` for C and C++ header files. Do not introduce new `.hpp` headers.
-- Use macro-based header include guards instead of `#pragma once`.
-- Format header include guards as `__SD_{PATH}__`, where `{PATH}` is the header path in uppercase snake case without the file extension. For example, `src/sample.h` should use `__SD_SAMPLE_H__`.
-- Do not introduce anonymous namespaces in new or modified code; prefer `static` file-local functions/variables or an explicit named namespace when scoping is needed.
-- In `class`/`struct` definitions, place data members before member functions unless an existing type already clearly follows a different pattern.
-- Keep `test_*.cpp` / `test_*.py` naming for tests.
-
-Some older code in the project may not fully follow the current conventions. Please do not submit PRs that only rewrite existing code to match style rules.
-
-## AI-Assisted Contributions
-
-AI tools may be used to assist development, but contributors are responsible for the quality and correctness of the submitted code.
-
-If any part of a contribution was generated with AI assistance, the contributor must perform a thorough human review before submitting the PR and understand every changed line.
-
-Do not list AI tools as co-authors. The human contributor is the sole responsible author of the submitted code.
-
-Please do not submit AI-generated code that you do not understand, and do not include meaningless experiments, temporary test code, or unrelated generated output in a PR.
-
-## Dependency Updates
-
-Do not submit PRs that update `ggml`. `ggml` updates are performed only after local validation by the maintainer.
-
-Other third-party dependencies are not updated unless necessary. If you want to update a dependency, please open an issue first instead of submitting a direct PR.
-
-## Security & Configuration
-
-Do not commit model weights, secrets, or local absolute paths. Keep large binaries out of git unless intentionally tracked release assets.

Dockerfile.cuda

@@ -1,25 +0,0 @@
-ARG CUDA_VERSION=12.6.3
-ARG UBUNTU_VERSION=24.04
-
-FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu${UBUNTU_VERSION} AS build
-
-RUN apt-get update && apt-get install -y --no-install-recommends build-essential git ccache cmake
-
-WORKDIR /sd.cpp
-
-COPY . .
-
-ARG CUDACXX=/usr/local/cuda/bin/nvcc
-RUN cmake . -B ./build -DSD_CUDA=ON
-RUN cmake --build ./build --config Release -j$(nproc)
-
-FROM nvidia/cuda:${CUDA_VERSION}-cudnn-runtime-ubuntu${UBUNTU_VERSION} AS runtime
-
-RUN apt-get update && \
-    apt-get install --yes --no-install-recommends libgomp1 && \
-    apt-get clean
-
-COPY --from=build /sd.cpp/build/bin/sd-cli /sd-cli
-COPY --from=build /sd.cpp/build/bin/sd-server /sd-server
-
-ENTRYPOINT [ "/sd-cli" ]

README.md

@@ -15,12 +15,6 @@ API and command-line option may change frequently.***
 
 ## 🔥Important News
 
-* **2026/04/11** 🚀 stable-diffusion.cpp now uses a brand-new embedded web UI.  
-  👉 Details: [PR #1408](https://github.com/leejet/stable-diffusion.cpp/pull/1408)
-
-* **2026/01/18** 🚀 stable-diffusion.cpp now supports **FLUX.2-klein**  
-  👉 Details: [PR #1193](https://github.com/leejet/stable-diffusion.cpp/pull/1193)
-
 * **2025/12/01** 🚀 stable-diffusion.cpp now supports **Z-Image**  
   👉 Details: [PR #1020](https://github.com/leejet/stable-diffusion.cpp/pull/1020)
 
@@ -49,16 +43,13 @@ API and command-line option may change frequently.***
     - SDXL, [SDXL-Turbo](https://huggingface.co/stabilityai/sdxl-turbo)
     - [Some SD1.x and SDXL distilled models](./docs/distilled_sd.md)
     - [SD3/SD3.5](./docs/sd3.md)
-    - [FLUX.1-dev/FLUX.1-schnell](./docs/flux.md)
-    - [FLUX.2-dev/FLUX.2-klein](./docs/flux2.md)
+    - [FlUX.1-dev/FlUX.1-schnell](./docs/flux.md)
+    - [FLUX.2-dev](./docs/flux2.md)
     - [Chroma](./docs/chroma.md)
     - [Chroma1-Radiance](./docs/chroma_radiance.md)
     - [Qwen Image](./docs/qwen_image.md)
     - [Z-Image](./docs/z_image.md)
     - [Ovis-Image](./docs/ovis_image.md)
-    - [Anima](./docs/anima.md)
-    - [ERNIE-Image](./docs/ernie_image.md)
-    - [HiDream-O1-Image](./docs/hidream_o1_image.md)
   - Image Edit Models
     - [FLUX.1-Kontext-dev](./docs/kontext.md)
     - [Qwen Image Edit series](./docs/qwen_image_edit.md)
@@ -78,10 +69,9 @@ API and command-line option may change frequently.***
   - OpenCL
   - SYCL
 - Supported weight formats
-  - Pytorch checkpoint (`.ckpt` or `.pth` or `.pt`)
+  - Pytorch checkpoint (`.ckpt` or `.pth`)
   - Safetensors (`.safetensors`)
   - GGUF (`.gguf`)
-- Convert mode supports converting model weights to `.gguf` or `.safetensors`
 - Supported platforms
     - Linux
     - Mac OS
@@ -99,7 +89,6 @@ API and command-line option may change frequently.***
     - `DPM++ 2M`
     - [`DPM++ 2M v2`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/8457)
     - `DPM++ 2S a`
-    - `ER-SDE`
     - [`LCM`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/13952)
 - Cross-platform reproducibility
     - `--rng cuda`, default, consistent with the `stable-diffusion-webui GPU RNG`
@@ -133,15 +122,13 @@ API and command-line option may change frequently.***
 ## Performance
 
 If you want to improve performance or reduce VRAM/RAM usage, please refer to [performance guide](./docs/performance.md).
-For runtime and parameter backend placement, see the [backend selection guide](./docs/backend.md).
 
 ## More Guides
 
-- [Backend selection](./docs/backend.md)
 - [SD1.x/SD2.x/SDXL](./docs/sd.md)
 - [SD3/SD3.5](./docs/sd3.md)
-- [FLUX.1-dev/FLUX.1-schnell](./docs/flux.md)
-- [FLUX.2-dev/FLUX.2-klein](./docs/flux2.md)
+- [FlUX.1-dev/FlUX.1-schnell](./docs/flux.md)
+- [FLUX.2-dev](./docs/flux2.md)
 - [FLUX.1-Kontext-dev](./docs/kontext.md)
 - [Chroma](./docs/chroma.md)
 - [🔥Qwen Image](./docs/qwen_image.md)
@@ -149,9 +136,6 @@ For runtime and parameter backend placement, see the [backend selection guide](.
 - [🔥Wan2.1/Wan2.2](./docs/wan.md)
 - [🔥Z-Image](./docs/z_image.md)
 - [Ovis-Image](./docs/ovis_image.md)
-- [Anima](./docs/anima.md)
-- [ERNIE-Image](./docs/ernie_image.md)
-- [HiDream-O1-Image](./docs/hidream_o1_image.md)
 - [LoRA](./docs/lora.md)
 - [LCM/LCM-LoRA](./docs/lcm.md)
 - [Using PhotoMaker to personalize image generation](./docs/photo_maker.md)
@@ -160,6 +144,7 @@ For runtime and parameter backend placement, see the [backend selection guide](.
 - [Docker](./docs/docker.md)
 - [Quantization and GGUF](./docs/quantization_and_gguf.md)
 - [Inference acceleration via caching](./docs/caching.md)
+- [Troubleshooting](./docs/troubleshooting.md)
 
 ## Bindings
 
@@ -167,7 +152,6 @@ These projects wrap `stable-diffusion.cpp` for easier use in other languages/fra
 
 * Golang (non-cgo): [seasonjs/stable-diffusion](https://github.com/seasonjs/stable-diffusion)
 * Golang (cgo): [Binozo/GoStableDiffusion](https://github.com/Binozo/GoStableDiffusion)
-* Golang (non-cgo): [l8bloom/gosd](https://github.com/l8bloom/gosd)
 * C#: [DarthAffe/StableDiffusion.NET](https://github.com/DarthAffe/StableDiffusion.NET)
 * Python: [william-murray1204/stable-diffusion-cpp-python](https://github.com/william-murray1204/stable-diffusion-cpp-python)
 * Rust: [newfla/diffusion-rs](https://github.com/newfla/diffusion-rs)

cache_dit.hpp

@@ -8,9 +8,7 @@
 #include <unordered_map>
 #include <vector>
 
-#include "condition_cache_utils.hpp"
 #include "ggml_extend.hpp"
-#include "tensor.hpp"
 
 struct DBCacheConfig {
     bool enabled                        = false;
@@ -605,6 +603,87 @@ inline std::vector<int> generate_scm_mask(
     return mask;
 }
 
+inline std::vector<int> get_scm_preset(const std::string& preset, int total_steps) {
+    struct Preset {
+        std::vector<int> compute_bins;
+        std::vector<int> cache_bins;
+    };
+
+    Preset slow   = {{8, 3, 3, 2, 1, 1}, {1, 2, 2, 2, 3}};
+    Preset medium = {{6, 2, 2, 2, 2, 1}, {1, 3, 3, 3, 3}};
+    Preset fast   = {{6, 1, 1, 1, 1, 1}, {1, 3, 4, 5, 4}};
+    Preset ultra  = {{4, 1, 1, 1, 1}, {2, 5, 6, 7}};
+
+    Preset* p = nullptr;
+    if (preset == "slow" || preset == "s" || preset == "S")
+        p = &slow;
+    else if (preset == "medium" || preset == "m" || preset == "M")
+        p = &medium;
+    else if (preset == "fast" || preset == "f" || preset == "F")
+        p = &fast;
+    else if (preset == "ultra" || preset == "u" || preset == "U")
+        p = &ultra;
+    else
+        return {};
+
+    if (total_steps != 28 && total_steps > 0) {
+        float scale = static_cast<float>(total_steps) / 28.0f;
+        std::vector<int> scaled_compute, scaled_cache;
+
+        for (int v : p->compute_bins) {
+            scaled_compute.push_back(std::max(1, static_cast<int>(v * scale + 0.5f)));
+        }
+        for (int v : p->cache_bins) {
+            scaled_cache.push_back(std::max(1, static_cast<int>(v * scale + 0.5f)));
+        }
+
+        return generate_scm_mask(scaled_compute, scaled_cache, total_steps);
+    }
+
+    return generate_scm_mask(p->compute_bins, p->cache_bins, total_steps);
+}
+
+inline float get_preset_threshold(const std::string& preset) {
+    if (preset == "slow" || preset == "s" || preset == "S")
+        return 0.20f;
+    if (preset == "medium" || preset == "m" || preset == "M")
+        return 0.25f;
+    if (preset == "fast" || preset == "f" || preset == "F")
+        return 0.30f;
+    if (preset == "ultra" || preset == "u" || preset == "U")
+        return 0.34f;
+    return 0.08f;
+}
+
+inline int get_preset_warmup(const std::string& preset) {
+    if (preset == "slow" || preset == "s" || preset == "S")
+        return 8;
+    if (preset == "medium" || preset == "m" || preset == "M")
+        return 6;
+    if (preset == "fast" || preset == "f" || preset == "F")
+        return 6;
+    if (preset == "ultra" || preset == "u" || preset == "U")
+        return 4;
+    return 8;
+}
+
+inline int get_preset_Fn(const std::string& preset) {
+    if (preset == "slow" || preset == "s" || preset == "S")
+        return 8;
+    if (preset == "medium" || preset == "m" || preset == "M")
+        return 8;
+    if (preset == "fast" || preset == "f" || preset == "F")
+        return 6;
+    if (preset == "ultra" || preset == "u" || preset == "U")
+        return 4;
+    return 8;
+}
+
+inline int get_preset_Bn(const std::string& preset) {
+    (void)preset;
+    return 0;
+}
+
 inline void parse_dbcache_options(const std::string& opts, DBCacheConfig& cfg) {
     if (opts.empty())
         return;
@@ -773,37 +852,35 @@ struct CacheDitConditionState {
         return it != cache_diffs.end() && !it->second.diff.empty();
     }
 
-    void update_cache(const void* cond, const sd::Tensor<float>& input, const sd::Tensor<float>& output) {
+    void update_cache(const void* cond, const float* input, const float* output, size_t size) {
         CacheEntry& entry = cache_diffs[cond];
-        if (!sd::store_condition_cache_diff(&entry.diff, input, output)) {
-            entry.prev_input.clear();
-            entry.prev_output.clear();
-            entry.has_prev = false;
-            return;
+        entry.diff.resize(size);
+        for (size_t i = 0; i < size; i++) {
+            entry.diff[i] = output[i] - input[i];
         }
 
-        size_t size              = static_cast<size_t>(output.numel());
-        const float* input_data  = input.data();
-        const float* output_data = output.data();
         entry.prev_input.resize(size);
         entry.prev_output.resize(size);
         for (size_t i = 0; i < size; i++) {
-            entry.prev_input[i]  = input_data[i];
-            entry.prev_output[i] = output_data[i];
+            entry.prev_input[i]  = input[i];
+            entry.prev_output[i] = output[i];
         }
         entry.has_prev = true;
     }
 
-    void apply_cache(const void* cond,
-                     const sd::Tensor<float>& input,
-                     sd::Tensor<float>* output) {
+    void apply_cache(const void* cond, const float* input, float* output, size_t size) {
         auto it = cache_diffs.find(cond);
         if (it == cache_diffs.end() || it->second.diff.empty())
             return;
-        sd::apply_condition_cache_diff(it->second.diff, input, output);
+        if (it->second.diff.size() != size)
+            return;
+
+        for (size_t i = 0; i < size; i++) {
+            output[i] = input[i] + it->second.diff[i];
+        }
     }
 
-    bool before_condition(const void* cond, const sd::Tensor<float>& input, sd::Tensor<float>* output, float sigma, int step_index) {
+    bool before_condition(const void* cond, struct ggml_tensor* input, struct ggml_tensor* output, float sigma, int step_index) {
         if (!enabled() || step_index < 0)
             return false;
 
@@ -823,7 +900,8 @@ struct CacheDitConditionState {
 
         if (skip_current_step) {
             if (has_cache(cond)) {
-                apply_cache(cond, input, output);
+                apply_cache(cond, (float*)input->data, (float*)output->data,
+                            static_cast<size_t>(ggml_nelements(output)));
                 return true;
             }
             return false;
@@ -836,13 +914,13 @@ struct CacheDitConditionState {
         if (it == cache_diffs.end() || !it->second.has_prev)
             return false;
 
-        size_t ne = static_cast<size_t>(input.numel());
+        size_t ne = static_cast<size_t>(ggml_nelements(input));
         if (it->second.prev_input.size() != ne)
             return false;
 
-        const float* input_data = input.data();
-        float diff              = CacheDitState::calculate_residual_diff(
-                         it->second.prev_input.data(), input_data, ne);
+        float* input_data = (float*)input->data;
+        float diff        = CacheDitState::calculate_residual_diff(
+                   it->second.prev_input.data(), input_data, ne);
 
         float effective_threshold = config.residual_diff_threshold;
         if (config.Fn_compute_blocks > 0) {
@@ -862,7 +940,7 @@ struct CacheDitConditionState {
             cached_steps.push_back(current_step_index);
             continuous_cached_steps++;
             accumulated_residual_diff += diff;
-            apply_cache(cond, input, output);
+            apply_cache(cond, input_data, (float*)output->data, ne);
             return true;
         }
 
@@ -870,14 +948,15 @@ struct CacheDitConditionState {
         return false;
     }
 
-    void after_condition(const void* cond, const sd::Tensor<float>& input, const sd::Tensor<float>& output) {
+    void after_condition(const void* cond, struct ggml_tensor* input, struct ggml_tensor* output) {
         if (!step_is_active())
             return;
 
-        update_cache(cond, input, output);
+        size_t ne = static_cast<size_t>(ggml_nelements(output));
+        update_cache(cond, (float*)input->data, (float*)output->data, ne);
 
         if (cond == anchor_condition && taylor_config.enabled) {
-            taylor_state.update_derivatives(output.data(), static_cast<size_t>(output.numel()), current_step_index);
+            taylor_state.update_derivatives((float*)output->data, ne, current_step_index);
         }
     }
 

common.hpp

@@ -1,9 +1,7 @@
-#ifndef __COMMON_BLOCK_HPP__
-#define __COMMON_BLOCK_HPP__
+#ifndef __COMMON_HPP__
+#define __COMMON_HPP__
 
-#include "ggml-backend.h"
 #include "ggml_extend.hpp"
-#include "util.h"
 
 class DownSampleBlock : public GGMLBlock {
 protected:
@@ -25,7 +23,7 @@ public:
         }
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         // x: [N, channels, h, w]
         if (vae_downsample) {
             auto conv = std::dynamic_pointer_cast<Conv2d>(blocks["conv"]);
@@ -54,7 +52,7 @@ public:
         blocks["conv"] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, out_channels, {3, 3}, {1, 1}, {1, 1}));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         // x: [N, channels, h, w]
         auto conv = std::dynamic_pointer_cast<Conv2d>(blocks["conv"]);
 
@@ -123,7 +121,7 @@ public:
         }
     }
 
-    virtual ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* emb = nullptr) {
+    virtual struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x, struct ggml_tensor* emb = nullptr) {
         // For dims==3, we reduce dimension from 5d to 4d by merging h and w, in order not to change ggml
         // [N, c, t, h, w] => [N, c, t, h * w]
         // x: [N, channels, h, w] if dims == 2 else [N, channels, t, h, w]
@@ -190,7 +188,7 @@ public:
         blocks["proj"] = std::shared_ptr<GGMLBlock>(new Linear(dim_in, dim_out * 2));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
         // x: [ne3, ne2, ne1, dim_in]
         // return: [ne3, ne2, ne1, dim_out]
         auto proj = std::dynamic_pointer_cast<Linear>(blocks["proj"]);
@@ -202,7 +200,7 @@ public:
 
         gate = ggml_cont(ctx->ggml_ctx, gate);
 
-        gate = ggml_ext_gelu(ctx->ggml_ctx, gate, true);
+        gate = ggml_gelu_inplace(ctx->ggml_ctx, gate);
 
         x = ggml_mul(ctx->ggml_ctx, x, gate);  // [ne3, ne2, ne1, dim_out]
 
@@ -216,13 +214,13 @@ public:
         blocks["proj"] = std::shared_ptr<GGMLBlock>(new Linear(dim_in, dim_out, bias));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
         // x: [ne3, ne2, ne1, dim_in]
         // return: [ne3, ne2, ne1, dim_out]
         auto proj = std::dynamic_pointer_cast<Linear>(blocks["proj"]);
 
         x = proj->forward(ctx, x);
-        x = ggml_ext_gelu(ctx->ggml_ctx, x, true);
+        x = ggml_gelu_inplace(ctx->ggml_ctx, x);
         return x;
     }
 };
@@ -250,6 +248,9 @@ public:
         float scale         = 1.f;
         if (precision_fix) {
             scale = 1.f / 128.f;
+#ifdef SD_USE_VULKAN
+            force_prec_f32 = true;
+#endif
         }
         // The purpose of the scale here is to prevent NaN issues in certain situations.
         // For example, when using Vulkan without enabling force_prec_f32,
@@ -257,15 +258,12 @@ public:
         blocks["net.2"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, dim_out, true, false, force_prec_f32, scale));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         // x: [ne3, ne2, ne1, dim]
         // return: [ne3, ne2, ne1, dim_out]
 
         auto net_0 = std::dynamic_pointer_cast<UnaryBlock>(blocks["net.0"]);
         auto net_2 = std::dynamic_pointer_cast<Linear>(blocks["net.2"]);
-        if (sd_backend_is(ctx->backend, "Vulkan")) {
-            net_2->set_force_prec_f32(true);
-        }
 
         x = net_0->forward(ctx, x);  // [ne3, ne2, ne1, inner_dim]
         x = net_2->forward(ctx, x);  // [ne3, ne2, ne1, dim_out]
@@ -279,7 +277,6 @@ protected:
     int64_t context_dim;
     int64_t n_head;
     int64_t d_head;
-    bool xtra_dim = false;
 
 public:
     CrossAttention(int64_t query_dim,
@@ -291,11 +288,7 @@ public:
           query_dim(query_dim),
           context_dim(context_dim) {
         int64_t inner_dim = d_head * n_head;
-        if (context_dim == 320 && d_head == 320) {
-            // LOG_DEBUG("CrossAttention: temp set dim to 1024 for sdxs_09");
-            xtra_dim    = true;
-            context_dim = 1024;
-        }
+
         blocks["to_q"] = std::shared_ptr<GGMLBlock>(new Linear(query_dim, inner_dim, false));
         blocks["to_k"] = std::shared_ptr<GGMLBlock>(new Linear(context_dim, inner_dim, false));
         blocks["to_v"] = std::shared_ptr<GGMLBlock>(new Linear(context_dim, inner_dim, false));
@@ -304,9 +297,9 @@ public:
         // to_out_1 is nn.Dropout(), skip for inference
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x,
-                         ggml_tensor* context) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* x,
+                                struct ggml_tensor* context) {
         // x: [N, n_token, query_dim]
         // context: [N, n_context, context_dim]
         // return: [N, n_token, query_dim]
@@ -320,17 +313,11 @@ public:
         int64_t n_context = context->ne[1];
         int64_t inner_dim = d_head * n_head;
 
-        auto q = to_q->forward(ctx, x);  // [N, n_token, inner_dim]
-        if (xtra_dim) {
-            // LOG_DEBUG("CrossAttention: temp set dim to 1024 for sdxs_09");
-            context->ne[0] = 1024;  // patch dim
-        }
+        auto q = to_q->forward(ctx, x);        // [N, n_token, inner_dim]
         auto k = to_k->forward(ctx, context);  // [N, n_context, inner_dim]
         auto v = to_v->forward(ctx, context);  // [N, n_context, inner_dim]
-        if (xtra_dim) {
-            context->ne[0] = 320;  // reset dim to orig
-        }
-        x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, nullptr, false, ctx->flash_attn_enabled);  // [N, n_token, inner_dim]
+
+        x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, nullptr, false, false, ctx->flash_attn_enabled);  // [N, n_token, inner_dim]
 
         x = to_out_0->forward(ctx, x);  // [N, n_token, query_dim]
         return x;
@@ -368,9 +355,9 @@ public:
         }
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x,
-                         ggml_tensor* context) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* x,
+                                struct ggml_tensor* context) {
         // x: [N, n_token, query_dim]
         // context: [N, n_context, context_dim]
         // return: [N, n_token, query_dim]
@@ -419,7 +406,7 @@ protected:
     int64_t context_dim = 768;  // hidden_size, 1024 for VERSION_SD2
     bool use_linear     = false;
 
-    void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") {
+    void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") {
         auto iter = tensor_storage_map.find(prefix + "proj_out.weight");
         if (iter != tensor_storage_map.end()) {
             int64_t inner_dim = n_head * d_head;
@@ -469,9 +456,9 @@ public:
         }
     }
 
-    virtual ggml_tensor* forward(GGMLRunnerContext* ctx,
-                                 ggml_tensor* x,
-                                 ggml_tensor* context) {
+    virtual struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                        struct ggml_tensor* x,
+                                        struct ggml_tensor* context) {
         // x: [N, in_channels, h, w]
         // context: [N, max_position(aka n_token), hidden_size(aka context_dim)]
         auto norm     = std::dynamic_pointer_cast<GroupNorm32>(blocks["norm"]);
@@ -523,7 +510,7 @@ public:
 
 class AlphaBlender : public GGMLBlock {
 protected:
-    void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") override {
+    void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") override {
         // Get the type of the "mix_factor" tensor from the input tensors map with the specified prefix
         enum ggml_type wtype = GGML_TYPE_F32;
         params["mix_factor"] = ggml_new_tensor_1d(ctx, wtype, 1);
@@ -543,14 +530,14 @@ public:
         // since mix_factor.shape is [1,], we don't need rearrange using  rearrange_pattern
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x_spatial,
-                         ggml_tensor* x_temporal) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* x_spatial,
+                                struct ggml_tensor* x_temporal) {
         // image_only_indicator is always tensor([0.])
         float alpha = get_alpha();
         auto x      = ggml_add(ctx->ggml_ctx,
-                               ggml_ext_scale(ctx->ggml_ctx, x_spatial, alpha),
-                               ggml_ext_scale(ctx->ggml_ctx, x_temporal, 1.0f - alpha));
+                               ggml_scale(ctx->ggml_ctx, x_spatial, alpha),
+                               ggml_scale(ctx->ggml_ctx, x_temporal, 1.0f - alpha));
         return x;
     }
 };
@@ -568,10 +555,10 @@ public:
         blocks["time_mixer"] = std::shared_ptr<GGMLBlock>(new AlphaBlender());
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x,
-                         ggml_tensor* emb,
-                         int num_video_frames) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* x,
+                                struct ggml_tensor* emb,
+                                int num_video_frames) {
         // x: [N, channels, h, w] aka [b*t, channels, h, w]
         // emb: [N, emb_channels] aka [b*t, emb_channels]
         // image_only_indicator is always tensor([0.])
@@ -603,4 +590,4 @@ public:
     }
 };
 
-#endif  // __COMMON_BLOCK_HPP__
+#endif  // __COMMON_HPP__

control.hpp

@@ -1,7 +1,8 @@
 #ifndef __CONTROL_HPP__
 #define __CONTROL_HPP__
 
-#include "common_block.hpp"
+#include "common.hpp"
+#include "ggml_extend.hpp"
 #include "model.h"
 
 #define CONTROL_NET_GRAPH_SIZE 1536
@@ -164,26 +165,26 @@ public:
         blocks["middle_block_out.0"] = std::shared_ptr<GGMLBlock>(make_zero_conv(ch));
     }
 
-    ggml_tensor* resblock_forward(std::string name,
-                                  GGMLRunnerContext* ctx,
-                                  ggml_tensor* x,
-                                  ggml_tensor* emb) {
+    struct ggml_tensor* resblock_forward(std::string name,
+                                         GGMLRunnerContext* ctx,
+                                         struct ggml_tensor* x,
+                                         struct ggml_tensor* emb) {
         auto block = std::dynamic_pointer_cast<ResBlock>(blocks[name]);
         return block->forward(ctx, x, emb);
     }
 
-    ggml_tensor* attention_layer_forward(std::string name,
-                                         GGMLRunnerContext* ctx,
-                                         ggml_tensor* x,
-                                         ggml_tensor* context) {
+    struct ggml_tensor* attention_layer_forward(std::string name,
+                                                GGMLRunnerContext* ctx,
+                                                struct ggml_tensor* x,
+                                                struct ggml_tensor* context) {
         auto block = std::dynamic_pointer_cast<SpatialTransformer>(blocks[name]);
         return block->forward(ctx, x, context);
     }
 
-    ggml_tensor* input_hint_block_forward(GGMLRunnerContext* ctx,
-                                          ggml_tensor* hint,
-                                          ggml_tensor* emb,
-                                          ggml_tensor* context) {
+    struct ggml_tensor* input_hint_block_forward(GGMLRunnerContext* ctx,
+                                                 struct ggml_tensor* hint,
+                                                 struct ggml_tensor* emb,
+                                                 struct ggml_tensor* context) {
         int num_input_blocks = 15;
         auto h               = hint;
         for (int i = 0; i < num_input_blocks; i++) {
@@ -198,13 +199,13 @@ public:
         return h;
     }
 
-    std::vector<ggml_tensor*> forward(GGMLRunnerContext* ctx,
-                                      ggml_tensor* x,
-                                      ggml_tensor* hint,
-                                      ggml_tensor* guided_hint,
-                                      ggml_tensor* timesteps,
-                                      ggml_tensor* context,
-                                      ggml_tensor* y = nullptr) {
+    std::vector<struct ggml_tensor*> forward(GGMLRunnerContext* ctx,
+                                             struct ggml_tensor* x,
+                                             struct ggml_tensor* hint,
+                                             struct ggml_tensor* guided_hint,
+                                             struct ggml_tensor* timesteps,
+                                             struct ggml_tensor* context,
+                                             struct ggml_tensor* y = nullptr) {
         // x: [N, in_channels, h, w] or [N, in_channels/2, h, w]
         // timesteps: [N,]
         // context: [N, max_position, hidden_size] or [1, max_position, hidden_size]. for example, [N, 77, 768]
@@ -246,7 +247,7 @@ public:
             emb = ggml_add(ctx->ggml_ctx, emb, label_emb);  // [N, time_embed_dim]
         }
 
-        std::vector<ggml_tensor*> outs;
+        std::vector<struct ggml_tensor*> outs;
 
         if (guided_hint == nullptr) {
             guided_hint = input_hint_block_forward(ctx, hint, emb, context);
@@ -310,19 +311,17 @@ struct ControlNet : public GGMLRunner {
     SDVersion version = VERSION_SD1;
     ControlNetBlock control_net;
 
-    ggml_backend_buffer_t control_buffer = nullptr;
+    ggml_backend_buffer_t control_buffer = nullptr;  // keep control output tensors in backend memory
     ggml_context* control_ctx            = nullptr;
-    std::vector<ggml_tensor*> control_outputs_ggml;
-    ggml_tensor* guided_hint_output_ggml = nullptr;
-    std::vector<sd::Tensor<float>> controls;
-    sd::Tensor<float> guided_hint;
-    bool guided_hint_cached = false;
+    std::vector<struct ggml_tensor*> controls;  // (12 input block outputs, 1 middle block output) SD 1.5
+    struct ggml_tensor* guided_hint = nullptr;  // guided_hint cache, for faster inference
+    bool guided_hint_cached         = false;
 
     ControlNet(ggml_backend_t backend,
-               ggml_backend_t params_backend,
+               bool offload_params_to_cpu,
                const String2TensorStorage& tensor_storage_map = {},
                SDVersion version                              = VERSION_SD1)
-        : GGMLRunner(backend, params_backend), control_net(version) {
+        : GGMLRunner(backend, offload_params_to_cpu), control_net(version) {
         control_net.init(params_ctx, tensor_storage_map, "");
     }
 
@@ -330,23 +329,23 @@ struct ControlNet : public GGMLRunner {
         free_control_ctx();
     }
 
-    void alloc_control_ctx(std::vector<ggml_tensor*> outs) {
-        ggml_init_params params;
+    void alloc_control_ctx(std::vector<struct ggml_tensor*> outs) {
+        struct ggml_init_params params;
         params.mem_size   = static_cast<size_t>(outs.size() * ggml_tensor_overhead()) + 1024 * 1024;
         params.mem_buffer = nullptr;
         params.no_alloc   = true;
         control_ctx       = ggml_init(params);
 
-        control_outputs_ggml.resize(outs.size() - 1);
+        controls.resize(outs.size() - 1);
 
         size_t control_buffer_size = 0;
 
-        guided_hint_output_ggml = ggml_dup_tensor(control_ctx, outs[0]);
-        control_buffer_size += ggml_nbytes(guided_hint_output_ggml);
+        guided_hint = ggml_dup_tensor(control_ctx, outs[0]);
+        control_buffer_size += ggml_nbytes(guided_hint);
 
         for (int i = 0; i < outs.size() - 1; i++) {
-            control_outputs_ggml[i] = ggml_dup_tensor(control_ctx, outs[i + 1]);
-            control_buffer_size += ggml_nbytes(control_outputs_ggml[i]);
+            controls[i] = ggml_dup_tensor(control_ctx, outs[i + 1]);
+            control_buffer_size += ggml_nbytes(controls[i]);
         }
 
         control_buffer = ggml_backend_alloc_ctx_tensors(control_ctx, runtime_backend);
@@ -363,10 +362,8 @@ struct ControlNet : public GGMLRunner {
             ggml_free(control_ctx);
             control_ctx = nullptr;
         }
-        guided_hint_output_ggml = nullptr;
-        guided_hint_cached      = false;
-        guided_hint             = {};
-        control_outputs_ggml.clear();
+        guided_hint        = nullptr;
+        guided_hint_cached = false;
         controls.clear();
     }
 
@@ -374,37 +371,33 @@ struct ControlNet : public GGMLRunner {
         return "control_net";
     }
 
-    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string prefix) {
+    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
         control_net.get_param_tensors(tensors, prefix);
     }
 
-    ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
-                             const sd::Tensor<float>& hint_tensor,
-                             const sd::Tensor<float>& timesteps_tensor,
-                             const sd::Tensor<float>& context_tensor = {},
-                             const sd::Tensor<float>& y_tensor       = {}) {
-        ggml_cgraph* gf = new_graph_custom(CONTROL_NET_GRAPH_SIZE);
+    struct ggml_cgraph* build_graph(struct ggml_tensor* x,
+                                    struct ggml_tensor* hint,
+                                    struct ggml_tensor* timesteps,
+                                    struct ggml_tensor* context,
+                                    struct ggml_tensor* y = nullptr) {
+        struct ggml_cgraph* gf = new_graph_custom(CONTROL_NET_GRAPH_SIZE);
 
-        ggml_tensor* x         = make_input(x_tensor);
-        ggml_tensor* hint      = nullptr;
-        ggml_tensor* timesteps = make_input(timesteps_tensor);
-        ggml_tensor* context   = make_optional_input(context_tensor);
-        ggml_tensor* y         = make_optional_input(y_tensor);
-
-        ggml_tensor* guided_hint_input = nullptr;
-        if (guided_hint_cached && !guided_hint.empty()) {
-            guided_hint_input = make_input(guided_hint);
-            hint              = nullptr;
+        x = to_backend(x);
+        if (guided_hint_cached) {
+            hint = nullptr;
         } else {
-            hint = make_input(hint_tensor);
+            hint = to_backend(hint);
         }
+        context   = to_backend(context);
+        y         = to_backend(y);
+        timesteps = to_backend(timesteps);
 
         auto runner_ctx = get_context();
 
         auto outs = control_net.forward(&runner_ctx,
                                         x,
                                         hint,
-                                        guided_hint_input,
+                                        guided_hint_cached ? guided_hint : nullptr,
                                         timesteps,
                                         context,
                                         y);
@@ -413,46 +406,36 @@ struct ControlNet : public GGMLRunner {
             alloc_control_ctx(outs);
         }
 
-        ggml_build_forward_expand(gf, ggml_cpy(compute_ctx, outs[0], guided_hint_output_ggml));
+        ggml_build_forward_expand(gf, ggml_cpy(compute_ctx, outs[0], guided_hint));
         for (int i = 0; i < outs.size() - 1; i++) {
-            ggml_build_forward_expand(gf, ggml_cpy(compute_ctx, outs[i + 1], control_outputs_ggml[i]));
+            ggml_build_forward_expand(gf, ggml_cpy(compute_ctx, outs[i + 1], controls[i]));
         }
 
         return gf;
     }
 
-    std::optional<std::vector<sd::Tensor<float>>> compute(int n_threads,
-                                                          const sd::Tensor<float>& x,
-                                                          const sd::Tensor<float>& hint,
-                                                          const sd::Tensor<float>& timesteps,
-                                                          const sd::Tensor<float>& context = {},
-                                                          const sd::Tensor<float>& y       = {}) {
+    bool compute(int n_threads,
+                 struct ggml_tensor* x,
+                 struct ggml_tensor* hint,
+                 struct ggml_tensor* timesteps,
+                 struct ggml_tensor* context,
+                 struct ggml_tensor* y,
+                 struct ggml_tensor** output     = nullptr,
+                 struct ggml_context* output_ctx = nullptr) {
         // x: [N, in_channels, h, w]
         // timesteps: [N, ]
         // context: [N, max_position, hidden_size]([N, 77, 768]) or [1, max_position, hidden_size]
         // y: [N, adm_in_channels] or [1, adm_in_channels]
-        auto get_graph = [&]() -> ggml_cgraph* {
+        auto get_graph = [&]() -> struct ggml_cgraph* {
             return build_graph(x, hint, timesteps, context, y);
         };
 
-        auto compute_result = GGMLRunner::compute<float>(get_graph, n_threads, false);
-        if (!compute_result.has_value()) {
-            return std::nullopt;
+        bool res = GGMLRunner::compute(get_graph, n_threads, false, output, output_ctx);
+        if (res) {
+            // cache guided_hint
+            guided_hint_cached = true;
         }
-
-        if (guided_hint_output_ggml != nullptr) {
-            guided_hint = restore_trailing_singleton_dims(sd::make_sd_tensor_from_ggml<float>(guided_hint_output_ggml),
-                                                          4);
-        }
-        controls.clear();
-        controls.reserve(control_outputs_ggml.size());
-        for (ggml_tensor* control : control_outputs_ggml) {
-            auto control_host = restore_trailing_singleton_dims(sd::make_sd_tensor_from_ggml<float>(control), 4);
-            GGML_ASSERT(!control_host.empty());
-            controls.push_back(std::move(control_host));
-        }
-        guided_hint_cached = true;
-        return controls;
+        return res;
     }
 
     bool load_from_file(const std::string& file_path, int n_threads) {
@@ -480,4 +463,4 @@ struct ControlNet : public GGMLRunner {
     }
 };
 
-#endif  // __CONTROL_HPP__
+#endif  // __CONTROL_HPP__

diffusion_model.hpp

@@ -0,0 +1,450 @@
+#ifndef __DIFFUSION_MODEL_H__
+#define __DIFFUSION_MODEL_H__
+
+#include "flux.hpp"
+#include "mmdit.hpp"
+#include "qwen_image.hpp"
+#include "unet.hpp"
+#include "wan.hpp"
+#include "z_image.hpp"
+
+struct DiffusionParams {
+    struct ggml_tensor* x                     = nullptr;
+    struct ggml_tensor* timesteps             = nullptr;
+    struct ggml_tensor* context               = nullptr;
+    struct ggml_tensor* c_concat              = nullptr;
+    struct ggml_tensor* y                     = nullptr;
+    struct ggml_tensor* guidance              = nullptr;
+    std::vector<ggml_tensor*> ref_latents     = {};
+    bool increase_ref_index                   = false;
+    int num_video_frames                      = -1;
+    std::vector<struct ggml_tensor*> controls = {};
+    float control_strength                    = 0.f;
+    struct ggml_tensor* vace_context          = nullptr;
+    float vace_strength                       = 1.f;
+    std::vector<int> skip_layers              = {};
+};
+
+struct DiffusionModel {
+    virtual std::string get_desc()                                                      = 0;
+    virtual bool compute(int n_threads,
+                         DiffusionParams diffusion_params,
+                         struct ggml_tensor** output     = nullptr,
+                         struct ggml_context* output_ctx = nullptr)                     = 0;
+    virtual void alloc_params_buffer()                                                  = 0;
+    virtual void free_params_buffer()                                                   = 0;
+    virtual void free_compute_buffer()                                                  = 0;
+    virtual void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) = 0;
+    virtual size_t get_params_buffer_size()                                             = 0;
+    virtual void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter){};
+    virtual int64_t get_adm_in_channels()                            = 0;
+    virtual void set_flash_attention_enabled(bool enabled)           = 0;
+    virtual void set_circular_axes(bool circular_x, bool circular_y) = 0;
+};
+
+struct UNetModel : public DiffusionModel {
+    UNetModelRunner unet;
+
+    UNetModel(ggml_backend_t backend,
+              bool offload_params_to_cpu,
+              const String2TensorStorage& tensor_storage_map = {},
+              SDVersion version                              = VERSION_SD1)
+        : unet(backend, offload_params_to_cpu, tensor_storage_map, "model.diffusion_model", version) {
+    }
+
+    std::string get_desc() override {
+        return unet.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        unet.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        unet.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        unet.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
+        unet.get_param_tensors(tensors, "model.diffusion_model");
+    }
+
+    size_t get_params_buffer_size() override {
+        return unet.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        unet.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return unet.unet.adm_in_channels;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        unet.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        unet.set_circular_axes(circular_x, circular_y);
+    }
+
+    bool compute(int n_threads,
+                 DiffusionParams diffusion_params,
+                 struct ggml_tensor** output     = nullptr,
+                 struct ggml_context* output_ctx = nullptr) override {
+        return unet.compute(n_threads,
+                            diffusion_params.x,
+                            diffusion_params.timesteps,
+                            diffusion_params.context,
+                            diffusion_params.c_concat,
+                            diffusion_params.y,
+                            diffusion_params.num_video_frames,
+                            diffusion_params.controls,
+                            diffusion_params.control_strength, output, output_ctx);
+    }
+};
+
+struct MMDiTModel : public DiffusionModel {
+    MMDiTRunner mmdit;
+
+    MMDiTModel(ggml_backend_t backend,
+               bool offload_params_to_cpu,
+               const String2TensorStorage& tensor_storage_map = {})
+        : mmdit(backend, offload_params_to_cpu, tensor_storage_map, "model.diffusion_model") {
+    }
+
+    std::string get_desc() override {
+        return mmdit.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        mmdit.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        mmdit.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        mmdit.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
+        mmdit.get_param_tensors(tensors, "model.diffusion_model");
+    }
+
+    size_t get_params_buffer_size() override {
+        return mmdit.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        mmdit.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return 768 + 1280;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        mmdit.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        mmdit.set_circular_axes(circular_x, circular_y);
+    }
+
+    bool compute(int n_threads,
+                 DiffusionParams diffusion_params,
+                 struct ggml_tensor** output     = nullptr,
+                 struct ggml_context* output_ctx = nullptr) override {
+        return mmdit.compute(n_threads,
+                             diffusion_params.x,
+                             diffusion_params.timesteps,
+                             diffusion_params.context,
+                             diffusion_params.y,
+                             output,
+                             output_ctx,
+                             diffusion_params.skip_layers);
+    }
+};
+
+struct FluxModel : public DiffusionModel {
+    Flux::FluxRunner flux;
+
+    FluxModel(ggml_backend_t backend,
+              bool offload_params_to_cpu,
+              const String2TensorStorage& tensor_storage_map = {},
+              SDVersion version                              = VERSION_FLUX,
+              bool use_mask                                  = false)
+        : flux(backend, offload_params_to_cpu, tensor_storage_map, "model.diffusion_model", version, use_mask) {
+    }
+
+    std::string get_desc() override {
+        return flux.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        flux.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        flux.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        flux.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
+        flux.get_param_tensors(tensors, "model.diffusion_model");
+    }
+
+    size_t get_params_buffer_size() override {
+        return flux.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        flux.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return 768;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        flux.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        flux.set_circular_axes(circular_x, circular_y);
+    }
+
+    bool compute(int n_threads,
+                 DiffusionParams diffusion_params,
+                 struct ggml_tensor** output     = nullptr,
+                 struct ggml_context* output_ctx = nullptr) override {
+        return flux.compute(n_threads,
+                            diffusion_params.x,
+                            diffusion_params.timesteps,
+                            diffusion_params.context,
+                            diffusion_params.c_concat,
+                            diffusion_params.y,
+                            diffusion_params.guidance,
+                            diffusion_params.ref_latents,
+                            diffusion_params.increase_ref_index,
+                            output,
+                            output_ctx,
+                            diffusion_params.skip_layers);
+    }
+};
+
+struct WanModel : public DiffusionModel {
+    std::string prefix;
+    WAN::WanRunner wan;
+
+    WanModel(ggml_backend_t backend,
+             bool offload_params_to_cpu,
+             const String2TensorStorage& tensor_storage_map = {},
+             const std::string prefix                       = "model.diffusion_model",
+             SDVersion version                              = VERSION_WAN2)
+        : prefix(prefix), wan(backend, offload_params_to_cpu, tensor_storage_map, prefix, version) {
+    }
+
+    std::string get_desc() override {
+        return wan.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        wan.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        wan.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        wan.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
+        wan.get_param_tensors(tensors, prefix);
+    }
+
+    size_t get_params_buffer_size() override {
+        return wan.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        wan.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return 768;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        wan.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        wan.set_circular_axes(circular_x, circular_y);
+    }
+
+    bool compute(int n_threads,
+                 DiffusionParams diffusion_params,
+                 struct ggml_tensor** output     = nullptr,
+                 struct ggml_context* output_ctx = nullptr) override {
+        return wan.compute(n_threads,
+                           diffusion_params.x,
+                           diffusion_params.timesteps,
+                           diffusion_params.context,
+                           diffusion_params.y,
+                           diffusion_params.c_concat,
+                           nullptr,
+                           diffusion_params.vace_context,
+                           diffusion_params.vace_strength,
+                           output,
+                           output_ctx);
+    }
+};
+
+struct QwenImageModel : public DiffusionModel {
+    std::string prefix;
+    Qwen::QwenImageRunner qwen_image;
+
+    QwenImageModel(ggml_backend_t backend,
+                   bool offload_params_to_cpu,
+                   const String2TensorStorage& tensor_storage_map = {},
+                   const std::string prefix                       = "model.diffusion_model",
+                   SDVersion version                              = VERSION_QWEN_IMAGE,
+                   bool zero_cond_t                               = false)
+        : prefix(prefix), qwen_image(backend, offload_params_to_cpu, tensor_storage_map, prefix, version, zero_cond_t) {
+    }
+
+    std::string get_desc() override {
+        return qwen_image.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        qwen_image.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        qwen_image.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        qwen_image.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
+        qwen_image.get_param_tensors(tensors, prefix);
+    }
+
+    size_t get_params_buffer_size() override {
+        return qwen_image.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        qwen_image.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return 768;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        qwen_image.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        qwen_image.set_circular_axes(circular_x, circular_y);
+    }
+
+    bool compute(int n_threads,
+                 DiffusionParams diffusion_params,
+                 struct ggml_tensor** output     = nullptr,
+                 struct ggml_context* output_ctx = nullptr) override {
+        return qwen_image.compute(n_threads,
+                                  diffusion_params.x,
+                                  diffusion_params.timesteps,
+                                  diffusion_params.context,
+                                  diffusion_params.ref_latents,
+                                  true,  // increase_ref_index
+                                  output,
+                                  output_ctx);
+    }
+};
+
+struct ZImageModel : public DiffusionModel {
+    std::string prefix;
+    ZImage::ZImageRunner z_image;
+
+    ZImageModel(ggml_backend_t backend,
+                bool offload_params_to_cpu,
+                const String2TensorStorage& tensor_storage_map = {},
+                const std::string prefix                       = "model.diffusion_model",
+                SDVersion version                              = VERSION_Z_IMAGE)
+        : prefix(prefix), z_image(backend, offload_params_to_cpu, tensor_storage_map, prefix, version) {
+    }
+
+    std::string get_desc() override {
+        return z_image.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        z_image.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        z_image.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        z_image.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
+        z_image.get_param_tensors(tensors, prefix);
+    }
+
+    size_t get_params_buffer_size() override {
+        return z_image.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        z_image.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return 768;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        z_image.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        z_image.set_circular_axes(circular_x, circular_y);
+    }
+
+    bool compute(int n_threads,
+                 DiffusionParams diffusion_params,
+                 struct ggml_tensor** output     = nullptr,
+                 struct ggml_context* output_ctx = nullptr) override {
+        return z_image.compute(n_threads,
+                               diffusion_params.x,
+                               diffusion_params.timesteps,
+                               diffusion_params.context,
+                               diffusion_params.ref_latents,
+                               true,  // increase_ref_index
+                               output,
+                               output_ctx);
+    }
+};
+
+#endif

docs/anima.md

@@ -1,21 +0,0 @@
-# How to Use
-
-## Download weights
-
-- Download Anima
-    - safetensors: https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/diffusion_models
-    - gguf: https://huggingface.co/Bedovyy/Anima-GGUF/tree/main
-    - gguf Anima2: https://huggingface.co/JusteLeo/Anima2-GGUF/tree/main
-- Download vae
-    - safetensors: https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/vae
-- Download Qwen3-0.6B-Base
-    - safetensors: https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/text_encoders
-    - gguf: https://huggingface.co/mradermacher/Qwen3-0.6B-Base-GGUF/tree/main
-
-## Examples
-
-```sh
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\anima-preview.safetensors --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_06b_base.safetensors  -p "a lovely cat holding a sign says 'anima.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu --diffusion-fa
-```
-
-<img alt="anima image example" src="../assets/anima/example.png" />

docs/backend.md

@@ -1,122 +0,0 @@
-# Backend selection
-
-`stable-diffusion.cpp` has two backend assignments:
-
-- `--backend` selects the runtime backend used to execute model graphs.
-- `--params-backend` selects the backend used to allocate model parameters.
-
-If `--params-backend` is not set, parameters use the same backend as their module runtime backend.
-
-## Syntax
-
-A backend assignment can be a single backend name:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend cpu
-```
-
-This applies to every module that does not have a more specific assignment.
-
-Assignments can also target individual modules:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend te=cpu,vae=cuda0,diffusion=vulkan0
-```
-
-The same syntax is used for parameter placement:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend cuda0 --params-backend te=cpu,vae=cpu
-```
-
-Module names are case-insensitive. Hyphens and underscores in module names are ignored, so `clip_vision`, `clip-vision`, and `clipvision` are equivalent.
-
-`all=`, `default=`, and `*=` can be used to set the default backend inside a mixed assignment:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend all=cuda0,te=cpu
-```
-
-## Modules
-
-| Module | Purpose | Accepted names |
-| --- | --- | --- |
-| `diffusion` | UNet, DiT, MMDiT, Flux, Wan, Qwen Image, and other diffusion models | `diffusion`, `model`, `unet`, `dit` |
-| `te` | Text encoders and conditioners | `te`, `clip`, `text`, `textencoder`, `textencoders`, `conditioner`, `cond`, `llm`, `t5`, `t5xxl` |
-| `clip_vision` | CLIP vision encoder | `clip_vision`, `clipvision`, `clip-vision`, `vision` |
-| `vae` | VAE and TAE | `vae`, `firststage`, `autoencoder`, `tae` |
-| `controlnet` | ControlNet | `controlnet`, `control` |
-| `photomaker` | PhotoMaker ID encoder and PhotoMaker LoRA | `photomaker`, `photomakerid`, `pmid`, `photo` |
-| `upscaler` | ESRGAN upscaler | `upscaler`, `esrgan`, `hires` |
-
-`te` is the preferred module name for text encoders. `clip` is kept as an accepted alias because many existing commands and model names use CLIP terminology.
-
-## Backend names
-
-Backend names are resolved against the GGML backend device list. Matching is case-insensitive and accepts exact names or unique prefixes, so common values include names such as:
-
-- `cpu`
-- `cuda0`
-- `vulkan0`
-- `metal`
-
-The special values `auto`, `default`, and an empty backend name select the default backend. The default preference is GPU, then integrated GPU, then CPU.
-
-The special value `gpu` selects the first GPU backend, falling back to the first integrated GPU backend.
-
-## Runtime backend vs. parameter backend
-
-The runtime backend controls where graph execution runs. The parameter backend controls where model weights are allocated.
-
-For example:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend cuda0 --params-backend cpu
-```
-
-This runs all modules on `cuda0`, but stores parameters in CPU RAM. During execution, parameters are moved to the runtime backend as needed.
-
-Per-module assignments can be mixed:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend diffusion=cuda0,te=cpu,vae=cpu --params-backend diffusion=cuda0,te=cpu,vae=cpu
-```
-
-This keeps text encoding and VAE execution on CPU while the diffusion model runs on GPU.
-
-## Backend sharing and lifetime
-
-Backends are managed by `SDBackendManager`.
-
-Within one manager, backend instances are cached by resolved backend device name. If multiple modules request the same backend, they share the same `ggml_backend_t`.
-
-For example:
-
-```shell
---backend te=cpu,vae=cpu
-```
-
-uses one shared CPU backend for both `te` and `vae` runtime execution.
-
-Runtime and parameter assignments also share the same backend cache. If `--backend diffusion=cuda0` and `--params-backend diffusion=cuda0` resolve to the same device, both use the same backend instance.
-
-`SDBackendManager` owns the backend instances and frees them when the context or upscaler is destroyed. Model runners receive non-owning runtime and parameter backend pointers and do not free them.
-
-## Compatibility flags
-
-The older CPU placement flags are still supported:
-
-- `--clip-on-cpu`
-- `--vae-on-cpu`
-- `--control-net-cpu`
-- `--offload-to-cpu`
-
-`--clip-on-cpu`, `--vae-on-cpu`, and `--control-net-cpu` affect runtime backend assignment only when `--backend` is not set. They map to `te=cpu`, `vae=cpu`, and `controlnet=cpu`.
-
-`--offload-to-cpu` affects parameter backend assignment only when `--params-backend` is not set. It is equivalent to:
-
-```shell
---params-backend cpu
-```
-
-Explicit `--backend` and `--params-backend` assignments are preferred for new commands.

docs/build.md

@@ -16,26 +16,6 @@ git submodule init
 git submodule update
 ```
 
-## WebP and WebM Support in Examples
-
-The example applications (`examples/cli` and `examples/server`) use `libwebp` to support WebP image I/O, and `examples/cli` can also use `libwebm` for `.webm` video output. Both are enabled by default. WebM output currently reuses `libwebp` to encode each frame as VP8 before muxing with `libwebm`.
-
-If you do not want WebP/WebM support, you can disable them at configure time:
-
-```shell
-mkdir build && cd build
-cmake .. -DSD_WEBP=OFF -DSD_WEBM=OFF
-cmake --build . --config Release
-```
-
-If the submodules are not available, you can also link against system packages instead:
-
-```shell
-mkdir build && cd build
-cmake .. -DSD_USE_SYSTEM_WEBP=ON -DSD_USE_SYSTEM_WEBM=ON
-cmake --build . --config Release
-```
-
 ## Build (CPU only)
 
 If you don't have a GPU or CUDA installed, you can build a CPU-only version.

docs/caching.md

@@ -11,7 +11,6 @@ Caching methods accelerate diffusion inference by reusing intermediate computati
 | `dbcache` | DiT models | Block-level L1 residual threshold |
 | `taylorseer` | DiT models | Taylor series approximation |
 | `cache-dit` | DiT models | Combined DBCache + TaylorSeer |
-| `spectrum` | UNET and DiT models | Chebyshev + Taylor output forecasting |
 
 ### UCache (UNET Models)
 
@@ -80,7 +79,7 @@ Uses Taylor series approximation to predict block outputs:
 Combines DBCache and TaylorSeer:
 
 ```bash
---cache-mode cache-dit
+--cache-mode cache-dit --cache-preset fast
 ```
 
 #### Parameters
@@ -92,6 +91,14 @@ Combines DBCache and TaylorSeer:
 | `threshold` | L1 residual difference threshold | 0.08 |
 | `warmup` | Steps before caching starts | 8 |
 
+#### Presets
+
+Available presets: `slow`, `medium`, `fast`, `ultra` (or `s`, `m`, `f`, `u`).
+
+```bash
+--cache-mode cache-dit --cache-preset fast
+```
+
 #### SCM Options
 
 Steps Computation Mask controls which steps can be cached:
@@ -111,26 +118,6 @@ Mask values: `1` = compute, `0` = can cache.
 --scm-policy dynamic
 ```
 
-### Spectrum (UNET and DiT Models)
-
-Spectrum uses Chebyshev polynomial fitting blended with Taylor extrapolation to predict denoised outputs, skipping entire forward passes. Based on the paper [Spectrum: Adaptive Spectral Feature Forecasting for Efficient Diffusion Sampling](https://github.com/tingyu215/Spectrum).
-
-```bash
-sd-cli -m model.safetensors -p "a cat" --cache-mode spectrum
-```
-
-#### Parameters
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `w` | Chebyshev vs Taylor blend weight (0=Taylor, 1=Chebyshev) | 0.40 |
-| `m` | Chebyshev polynomial degree | 3 |
-| `lam` | Ridge regression regularization | 1.0 |
-| `window` | Initial window size (compute every N steps) | 2 |
-| `flex` | Window growth per computed step after warmup | 0.50 |
-| `warmup` | Steps to always compute before caching starts | 4 |
-| `stop` | Stop caching at this fraction of total steps | 0.9 |
-
 ### Performance Tips
 
 - Start with default thresholds and adjust based on output quality

docs/distilled_sd.md

@@ -1,8 +1,8 @@
-# Running distilled models: SSD1B, Vega and SDx.x with tiny U-Nets
+# Running distilled models: SSD1B and SDx.x with tiny U-Nets
 
 ## Preface 
 
-These models feature a reduced U-Net architecture. Unlike standard SDXL models, the SSD-1B and Vega U-Net contains only one middle block and fewer attention layers in its up- and down-blocks, resulting in significantly smaller file sizes. Using these models can reduce inference time by more than 33%. For more details, refer to Segmind's paper: https://arxiv.org/abs/2401.02677v1.
+These models feature a reduced U-Net architecture. Unlike standard SDXL models, the SSD-1B U-Net contains only one middle block and fewer attention layers in its up- and down-blocks, resulting in significantly smaller file sizes. Using these models can reduce inference time by more than 33%. For more details, refer to Segmind's paper: https://arxiv.org/abs/2401.02677v1.
 Similarly, SD1.x- and SD2.x-style models with a tiny U-Net consist of only 6 U-Net blocks, leading to very small files and time savings of up to 50%. For more information, see the paper: https://arxiv.org/pdf/2305.15798.pdf.
 
 ## SSD1B
@@ -17,17 +17,7 @@ Useful LoRAs are also available:
  * https://huggingface.co/seungminh/lora-swarovski-SSD-1B/resolve/main/pytorch_lora_weights.safetensors
  * https://huggingface.co/kylielee505/mylcmlorassd/resolve/main/pytorch_lora_weights.safetensors
 
-## Vega
-
-Segmind's Vega model is available online here:
-
- * https://huggingface.co/segmind/Segmind-Vega/resolve/main/segmind-vega.safetensors
- 
-VegaRT is an example for an LCM-LoRA:
-
- * https://huggingface.co/segmind/Segmind-VegaRT/resolve/main/pytorch_lora_weights.safetensors
-
-Both files can be used out-of-the-box, unlike the models described in next sections.
+These files can be used out-of-the-box, unlike the models described in the next section.
 
 
 ## SD1.x, SD2.x with tiny U-Nets
@@ -87,32 +77,23 @@ pipe.save_pretrained("segmindtiny-sd", safe_serialization=True)
 ```bash
 python convert_diffusers_to_original_stable_diffusion.py \
       --model_path  ./segmindtiny-sd \
-      --checkpoint_path ./segmind_tiny-sd.safetensors  --half --use_safetensors
+      --checkpoint_path ./segmind_tiny-sd.ckpt --half
 ```
 
-The file segmind_tiny-sd.safetensors will be generated and is now ready for use with sd.cpp. You can follow a similar process for the other models mentioned above.
+The file segmind_tiny-sd.ckpt will be generated and is now ready for use with sd.cpp. You can follow a similar process for the other models mentioned above.
 
 
-### SDXS-512-DreamShaper
+### Another available .ckpt file:
 
-Another very tiny and **incredibly fast**  model is SDXS by IDKiro et al.  The authors refer to it as *"Real-Time One-Step Latent Diffusion Models with Image Conditions"*. For details read the paper: https://arxiv.org/pdf/2403.16627 . Once again the authors removed some more blocks of U-Net part and unlike other SD1 models they use an adjusted _AutoEncoderTiny_ instead of default _AutoEncoderKL_ for the VAE part.
-##### Some ready-to-run SDXS-512 model files are available online, such as:
+ * https://huggingface.co/ClashSAN/small-sd/resolve/main/tinySDdistilled.ckpt
 
-* https://huggingface.co/akleine/sdxs-512
-* https://huggingface.co/concedo/sdxs-512-tinySDdistilled-GGUF
+To use this file, you must first adjust its non-contiguous tensors:
 
-##### Run the model as follows:
-```bash
-~/stable-diffusion.cpp/build/bin/sd-cli -m sdxs.safetensors -p "portrait of a lovely cat" \
-  --cfg-scale 1 --steps 1
+```python
+import torch
+ckpt = torch.load("tinySDdistilled.ckpt", map_location=torch.device('cpu'))
+for key, value in ckpt['state_dict'].items():
+    if isinstance(value, torch.Tensor):
+        ckpt['state_dict'][key] = value.contiguous()
+torch.save(ckpt, "tinySDdistilled_fixed.ckpt")
 ```
-Both options: ``` --cfg-scale 1 ``` and  ``` --steps 1 ``` are mandatory here.
-
-### SDXS-512-0.9
-
-Even though the name "SDXS-512-0.9" is similar to "SDXS-512-DreamShaper", it is *completely different* but also **incredibly fast**. Sometimes it is preferred, so try it yourself.
-##### Download a ready-to-run file from here:
-
-* https://huggingface.co/akleine/sdxs-09
-
-For the use of this model, both options ``` --cfg-scale 1 ``` and ``` --steps 1 ``` are again absolutely necessary.

docs/ernie_image.md

@@ -1,35 +0,0 @@
-# How to Use
-
-You can run ERNIE-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or even less.
-
-## Download weights
-
-- Download ERNIE-Image-Turbo
-    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/diffusion_models
-    - gguf: https://huggingface.co/unsloth/ERNIE-Image-Turbo-GGUF/tree/main
-- Download ERNIE-Image
-    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/diffusion_models
-    - gguf: https://huggingface.co/unsloth/ERNIE-Image-GGUF/tree/main
-- Download vae
-    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/vae
-- Download ministral 3b
-    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/text_encoders
-    - gguf: https://huggingface.co/unsloth/Ministral-3-3B-Instruct-2512-GGUF/tree/main
-
-## Examples
-
-### ERNIE-Image-Turbo
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\ernie-image-turbo.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 8 -v --offload-to-cpu --diffusion-fa
-```
-
-<img width="256" alt="ERNIE-Image Turbo example" src="../assets/ernie_image/turbo_example.png" />
-
-### ERNIE-Image
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\ernie-image-UD-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa
-```
-
-<img width="256" alt="ERNIE-Image example" src="../assets/ernie_image/example.png" />

docs/esrgan.md

@@ -1,6 +1,6 @@
 ## Using ESRGAN to upscale results
 
-You can use ESRGAN—such as the model [RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth)—to upscale the generated images and improve their overall resolution and clarity.
+You can use ESRGAN to upscale the generated images. At the moment, only the [RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth) model is supported. Support for more models of this architecture will be added soon.
 
 - Specify the model path using the `--upscale-model PATH` parameter. example:
 

docs/flux2.md

@@ -1,96 +1,21 @@
 # How to Use
 
-## Flux.2-dev
-
-### Download weights
+## Download weights
 
 - Download FLUX.2-dev
     - gguf: https://huggingface.co/city96/FLUX.2-dev-gguf/tree/main
 - Download vae
     - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main
-- Download FLUX.2-small-decoder (full_encoder_small_decoder.safetensors) as an alternative VAE option
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-small-decoder/tree/main
 - Download Mistral-Small-3.2-24B-Instruct-2506-GGUF
     - gguf: https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/tree/main
 
-### Examples
+## Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux2-dev-Q4_K_S.gguf --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf -r .\kontext_input.png -p "change 'flux.cpp' to 'flux2-dev.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux2-dev-Q4_K_S.gguf --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf -r .\kontext_input.png -p "change 'flux.cpp' to 'flux2-dev.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu
 ```
 
 <img alt="flux2 example" src="../assets/flux2/example.png" />
 
-## Flux.2 klein 4B / Flux.2 klein base 4B
 
-### Download weights
 
-- Download FLUX.2-klein-4B
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-4B
-    - gguf: https://huggingface.co/leejet/FLUX.2-klein-4B-GGUF/tree/main
-- Download FLUX.2-klein-base-4B
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4B
-    - gguf: https://huggingface.co/leejet/FLUX.2-klein-base-4B-GGUF/tree/main
-- Download vae
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main
-- Download FLUX.2-small-decoder (full_encoder_small_decoder.safetensors) as an alternative VAE option
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-small-decoder/tree/main
-- Download Qwen3 4b
-    - safetensors: https://huggingface.co/Comfy-Org/flux2-klein-4B/tree/main/split_files/text_encoders
-    - gguf: https://huggingface.co/unsloth/Qwen3-4B-GGUF/tree/main
-
-### Examples
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu --diffusion-fa
-```
-
-<img alt="flux2-klein-4b" src="../assets/flux2/flux2-klein-4b.png" />
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_4b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu --steps 4
-```
-
-<img alt="flux2-klein-4b-edit" src="../assets/flux2/flux2-klein-4b-edit.png" />
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-base-4b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu --diffusion-fa
-```
-
-<img alt="flux2-klein-base-4b" src="../assets/flux2/flux2-klein-base-4b.png" />
-
-## Flux.2 klein 9B / Flux.2 klein base 9B
-
-### Download weights
-
-- Download FLUX.2-klein-9B
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-9B
-    - gguf: https://huggingface.co/leejet/FLUX.2-klein-9B-GGUF/tree/main
-- Download FLUX.2-klein-base-9B
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B
-    - gguf: https://huggingface.co/leejet/FLUX.2-klein-base-9B-GGUF/tree/main
-- Download vae
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main
-- Download Qwen3 8B
-    - safetensors: https://huggingface.co/Comfy-Org/flux2-klein-9B/tree/main/split_files/text_encoders
-    - gguf: https://huggingface.co/unsloth/Qwen3-8B-GGUF/tree/main
-
-### Examples
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu --diffusion-fa
-```
-
-<img alt="flux2-klein-9b" src="../assets/flux2/flux2-klein-9b.png" />
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_8b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu --steps 4
-```
-
-<img alt="flux2-klein-9b-edit" src="../assets/flux2/flux2-klein-9b-edit.png" />
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\flux-2-klein-base-9b.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu --diffusion-fa
-```
-
-<img alt="flux2-klein-base-9b" src="../assets/flux2/flux2-klein-base-9b.png" />

docs/hidream_o1_image.md

@@ -1,20 +0,0 @@
-# How to Use
-
-## Download weights
-
-- Download HiDream-O1-Image-Dev
-    - safetensors: https://huggingface.co/Comfy-Org/HiDream-O1-Image/tree/main/checkpoints
-- Download HiDream-O1-Image
-    - safetensors: https://huggingface.co/Comfy-Org/HiDream-O1-Image/tree/main/checkpoints
-
-## Examples
-
-### HiDream-O1-Image-Dev
-
-```
-.\bin\Release\sd-cli.exe -m  ..\..\ComfyUI\models\diffusion_models\hidream_o1_image_dev_bf16.safetensors -p "a lovely cat holding a sign says 
-'hidream o1 cpp'" --cfg-scale 1.0  -v -H 1024 -W 1024
-```
-
-<img width="256" alt="HiDream-O1-Image-Dev example" src="../assets/hidream-o1/dev_example.png" />
-

docs/hipBLAS_on_Windows.md

@@ -26,12 +26,12 @@ Fortunately, `AMD` provides complete help documentation, you can use the help do
 
 Then we must set `ROCM` as environment variables before running cmake.
 
-Usually if you install according to the official tutorial and do not modify the ROCM path, then there is a high probability that it is here `C:\Program Files\AMD\ROCm\7.1.1\bin`
+Usually if you install according to the official tutorial and do not modify the ROCM path, then there is a high probability that it is here `C:\Program Files\AMD\ROCm\5.5\bin`
 
 This is what I use to set the clang:
 ```Commandline
-set CC=C:\Program Files\AMD\ROCm\7.1.1\bin\clang.exe
-set CXX=C:\Program Files\AMD\ROCm\7.1.1\bin\clang++.exe
+set CC=C:\Program Files\AMD\ROCm\5.5\bin\clang.exe
+set CXX=C:\Program Files\AMD\ROCm\5.5\bin\clang++.exe
 ```
 
 ## Ninja
@@ -46,7 +46,7 @@ set ninja=C:\Program Files\ninja\ninja.exe
 ## Building stable-diffusion.cpp
 
 The thing different from the regular CPU build is `-DSD_HIPBLAS=ON` ,
-`-G "Ninja"`, `-DCMAKE_C_COMPILER=clang`, `-DCMAKE_CXX_COMPILER=clang++`, `-DAMDGPU_TARGETS=gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032`
+`-G "Ninja"`, `-DCMAKE_C_COMPILER=clang`, `-DCMAKE_CXX_COMPILER=clang++`, `-DAMDGPU_TARGETS=gfx1100`
 
 >**Notice**: check the `clang` and `clang++` information:
 ```Commandline
@@ -59,29 +59,26 @@ If you see like this, we can continue:
 clang version 17.0.0 (git@github.amd.com:Compute-Mirrors/llvm-project e3201662d21c48894f2156d302276eb1cf47c7be)
 Target: x86_64-pc-windows-msvc
 Thread model: posix
-InstalledDir: C:\Program Files\AMD\ROCm\7.1.1\bin
+InstalledDir: C:\Program Files\AMD\ROCm\5.5\bin
 ```
 
 ```
 clang version 17.0.0 (git@github.amd.com:Compute-Mirrors/llvm-project e3201662d21c48894f2156d302276eb1cf47c7be)
 Target: x86_64-pc-windows-msvc
 Thread model: posix
-InstalledDir: C:\Program Files\AMD\ROCm\7.1.1\bin
+InstalledDir: C:\Program Files\AMD\ROCm\5.5\bin
 ```
 
->**Notice** that the GPU targets are now compatible with multiple GPU architectures (ROCm 7.1.1 targets). You can change them to match your GPU architecture. Click here to see your architecture [LLVM Target](https://rocm.docs.amd.com/en/latest/release/windows_support.html#windows-supported-gpus)
+>**Notice** that the `gfx1100` is the GPU architecture of my GPU, you can change it to your GPU architecture. Click here to see your architecture [LLVM Target](https://rocm.docs.amd.com/en/latest/release/windows_support.html#windows-supported-gpus)
 
-Examples:
-- AMD Radeon™ RX 7900 XTX Graphics: `gfx1100`
-- AMD Radeon™ RX 7900 XT Graphics: `gfx1101`
-- AMD Radeon™ RX 7900 GRE Graphics: `gfx1102`
+My GPU is AMD Radeon™ RX 7900 XTX Graphics, so I set it to `gfx1100`.
 
 option:
 
 ```commandline
 mkdir build
 cd build
-cmake .. -G "Ninja" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS="gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
+cmake .. -G "Ninja" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS=gfx1100
 cmake --build . --config Release
 ```
 

docs/ovis_image.md

@@ -13,7 +13,7 @@
 ## Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ovis_image-Q4_0.gguf --vae ..\..\ComfyUI\models\vae\ae.sft  --llm ..\..\ComfyUI\models\text_encoders\ovis_2.5.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ovis_image-Q4_0.gguf --vae ..\..\ComfyUI\models\vae\ae.sft  --llm ..\..\ComfyUI\models\text_encoders\ovis_2.5.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu
 ```
 
 <img alt="ovis image example" src="../assets/ovis_image/example.png" />

docs/performance.md

@@ -1,22 +1,3 @@
-## Use Flash Attention to save memory and improve speed.
-
-Enabling flash attention for the diffusion model reduces memory usage by varying amounts of MB.
-eg.:
- - flux 768x768 ~600mb
- - SD2 768x768 ~1400mb
-
-For most backends, it slows things down, but for cuda it generally speeds it up too.
-At the moment, it is only supported for some models and some backends (like cpu, cuda/rocm, metal).
-
-Run by adding `--diffusion-fa` to the arguments and watch for:
-```
-[INFO ] stable-diffusion.cpp:312  - Using flash attention in the diffusion model
-```
-and the compute buffer shrink in the debug log:
-```
-[DEBUG] ggml_extend.hpp:1004 - flux compute buffer size: 650.00 MB(VRAM)
-```
-
 ## Offload weights to the CPU to save VRAM without reducing generation speed.
 
 Using `--offload-to-cpu` allows you to offload weights to the CPU, saving VRAM without reducing generation speed.

docs/qwen_image.md

@@ -14,7 +14,7 @@
 ## Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\qwen-image-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf  -p '一个穿着"QWEN"标志的T恤的中国美女正拿着黑色的马克笔面相镜头微笑。她身后的玻璃板上手写体写着 “一、Qwen-Image的技术路线： 探索视觉生成基础模型的极限，开创理解与生成一体化的未来。二、Qwen-Image的模型特色：1、复杂文字渲染。支持中英渲染、自动布局； 2、精准图像编辑。支持文字编辑、物体增减、风格变换。三、Qwen-Image的未来愿景：赋能专业内容创作、助力生成式AI发展。”' --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu -H 1024 -W 1024 --diffusion-fa --flow-shift 3
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\qwen-image-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf  -p '一个穿着"QWEN"标志的T恤的中国美女正拿着黑色的马克笔面相镜头微笑。她身后的玻璃板上手写体写着 “一、Qwen-Image的技术路线： 探索视觉生成基础模型的极限，开创理解与生成一体化的未来。二、Qwen-Image的模型特色：1、复杂文字渲染。支持中英渲染、自动布局； 2、精准图像编辑。支持文字编辑、物体增减、风格变换。三、Qwen-Image的未来愿景：赋能专业内容创作、助力生成式AI发展。”' --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu -H 1024 -W 1024 --flow-shift 3
 ```
 
 <img alt="qwen example" src="../assets/qwen/example.png" />

docs/qwen_image_edit.md

@@ -23,7 +23,7 @@
 ### Qwen Image Edit
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Qwen_Image_Edit-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --diffusion-fa --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'" --seed 1118877715456453
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Qwen_Image_Edit-Q8_0.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'" --seed 1118877715456453
 ```
 
 <img alt="qwen_image_edit" src="../assets/qwen/qwen_image_edit.png" />
@@ -32,7 +32,7 @@
 ### Qwen Image Edit 2509
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Qwen-Image-Edit-2509-Q4_K_S.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf --llm_vision ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct.mmproj-Q8_0.gguf --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --diffusion-fa --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'Qwen Image Edit 2509'"
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\Qwen-Image-Edit-2509-Q4_K_S.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf --llm_vision ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct.mmproj-Q8_0.gguf --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'Qwen Image Edit 2509'"
 ```
 
 <img alt="qwen_image_edit_2509" src="../assets/qwen/qwen_image_edit_2509.png" />
@@ -42,7 +42,7 @@
 To use the new Qwen Image Edit 2511 mode, the  `--qwen-image-zero-cond-t` flag must be enabled; otherwise, image editing quality will degrade significantly.
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\qwen-image-edit-2511-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --diffusion-fa --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'"  --qwen-image-zero-cond-t
+.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\qwen-image-edit-2511-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\qwen_image_vae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'"  --qwen-image-zero-cond-t
 ```
 
 <img alt="qwen_image_edit_2509" src="../assets/qwen/qwen_image_edit_2511.png" />

docs/troubleshooting.md

@@ -0,0 +1,3 @@
+## Try `--disable-fa`
+
+By default, **stable-diffusion.cpp** uses Flash Attention to improve generation speed and optimize GPU memory usage. However, on some backends, Flash Attention may cause unexpected issues, such as generating completely black images. In such cases, you can try disabling Flash Attention by using `--disable-fa`.

docs/z_image.md

@@ -7,9 +7,6 @@ You can run Z-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or ev
 - Download Z-Image-Turbo
     - safetensors: https://huggingface.co/Comfy-Org/z_image_turbo/tree/main/split_files/diffusion_models
     - gguf: https://huggingface.co/leejet/Z-Image-Turbo-GGUF/tree/main
-- Download Z-Image
-    - safetensors: https://huggingface.co/Comfy-Org/z_image/tree/main/split_files/diffusion_models
-    - gguf: https://huggingface.co/unsloth/Z-Image-GGUF/tree/main
 - Download vae
     - safetensors: https://huggingface.co/black-forest-labs/FLUX.1-schnell/tree/main
 - Download Qwen3 4b
@@ -18,22 +15,12 @@ You can run Z-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or ev
 
 ## Examples
 
-### Z-Image-Turbo
-
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  z_image_turbo-Q3_K.gguf --vae ..\..\ComfyUI\models\vae\ae.sft  --llm ..\..\ComfyUI\models\text_encoders\Qwen3-4B-Instruct-2507-Q4_K_M.gguf -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 1.0 -v --offload-to-cpu --diffusion-fa -H 1024 -W 512
+.\bin\Release\sd-cli.exe --diffusion-model  z_image_turbo-Q3_K.gguf --vae ..\..\ComfyUI\models\vae\ae.sft  --llm ..\..\ComfyUI\models\text_encoders\Qwen3-4B-Instruct-2507-Q4_K_M.gguf -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 1.0 -v --offload-to-cpu -H 1024 -W 512
 ```
 
 <img width="256" alt="z-image example" src="../assets/z_image/q3_K.png" />
 
-### Z-Image-Base
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\z_image_bf16.safetensors --vae ..\..\ComfyUI\models\vae\ae.sft  --llm ..\..\ComfyUI\models\text_encoders\qwen_3_4b.safetensors -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa -H 1024 -W 512
-```
-
-<img width="256" alt="z-image example" src="../assets/z_image/base_bf16.png" />
-
 ## Comparison of Different Quantization Types
 
 | bf16 | q8_0 | q6_K | q5_0 | q4_K | q4_0 | q3_K | q2_K|

easycache.hpp

@@ -1,15 +1,10 @@
-#ifndef __EASYCACHE_HPP__
-#define __EASYCACHE_HPP__
-
 #include <cmath>
 #include <limits>
 #include <unordered_map>
 #include <vector>
 
-#include "condition_cache_utils.hpp"
 #include "denoiser.hpp"
 #include "ggml_extend.hpp"
-#include "tensor.hpp"
 
 struct EasyCacheConfig {
     bool enabled          = false;
@@ -24,15 +19,15 @@ struct EasyCacheCacheEntry {
 
 struct EasyCacheState {
     EasyCacheConfig config;
-    Denoiser* denoiser           = nullptr;
-    float start_sigma            = std::numeric_limits<float>::max();
-    float end_sigma              = 0.0f;
-    bool initialized             = false;
-    bool initial_step            = true;
-    bool skip_current_step       = false;
-    bool step_active             = false;
-    const void* anchor_condition = nullptr;
-    std::unordered_map<const void*, EasyCacheCacheEntry> cache_diffs;
+    Denoiser* denoiser                  = nullptr;
+    float start_sigma                   = std::numeric_limits<float>::max();
+    float end_sigma                     = 0.0f;
+    bool initialized                    = false;
+    bool initial_step                   = true;
+    bool skip_current_step              = false;
+    bool step_active                    = false;
+    const SDCondition* anchor_condition = nullptr;
+    std::unordered_map<const SDCondition*, EasyCacheCacheEntry> cache_diffs;
     std::vector<float> prev_input;
     std::vector<float> prev_output;
     float output_prev_norm                = 0.0f;
@@ -125,30 +120,41 @@ struct EasyCacheState {
         return enabled() && step_active && skip_current_step;
     }
 
-    bool has_cache(const void* cond) const {
+    bool has_cache(const SDCondition* cond) const {
         auto it = cache_diffs.find(cond);
         return it != cache_diffs.end() && !it->second.diff.empty();
     }
 
-    void update_cache(const void* cond, const sd::Tensor<float>& input, const sd::Tensor<float>& output) {
+    void update_cache(const SDCondition* cond, ggml_tensor* input, ggml_tensor* output) {
         EasyCacheCacheEntry& entry = cache_diffs[cond];
-        sd::store_condition_cache_diff(&entry.diff, input, output);
+        size_t ne                  = static_cast<size_t>(ggml_nelements(output));
+        entry.diff.resize(ne);
+        float* out_data = (float*)output->data;
+        float* in_data  = (float*)input->data;
+        for (size_t i = 0; i < ne; ++i) {
+            entry.diff[i] = out_data[i] - in_data[i];
+        }
     }
 
-    void apply_cache(const void* cond, const sd::Tensor<float>& input, sd::Tensor<float>* output) {
+    void apply_cache(const SDCondition* cond, ggml_tensor* input, ggml_tensor* output) {
         auto it = cache_diffs.find(cond);
         if (it == cache_diffs.end() || it->second.diff.empty()) {
             return;
         }
-        sd::apply_condition_cache_diff(it->second.diff, input, output);
+        copy_ggml_tensor(output, input);
+        float* out_data                = (float*)output->data;
+        const std::vector<float>& diff = it->second.diff;
+        for (size_t i = 0; i < diff.size(); ++i) {
+            out_data[i] += diff[i];
+        }
     }
 
-    bool before_condition(const void* cond,
-                          const sd::Tensor<float>& input,
-                          sd::Tensor<float>* output,
+    bool before_condition(const SDCondition* cond,
+                          ggml_tensor* input,
+                          ggml_tensor* output,
                           float sigma,
                           int step_index) {
-        if (!enabled() || step_index < 0 || output == nullptr) {
+        if (!enabled() || step_index < 0) {
             return false;
         }
         if (step_index != current_step_index) {
@@ -175,12 +181,12 @@ struct EasyCacheState {
         if (!has_prev_input || !has_prev_output || !has_cache(cond)) {
             return false;
         }
-        size_t ne = static_cast<size_t>(input.numel());
+        size_t ne = static_cast<size_t>(ggml_nelements(input));
         if (prev_input.size() != ne) {
             return false;
         }
-        const float* input_data = input.data();
-        last_input_change       = 0.0f;
+        float* input_data = (float*)input->data;
+        last_input_change = 0.0f;
         for (size_t i = 0; i < ne; ++i) {
             last_input_change += std::fabs(input_data[i] - prev_input[i]);
         }
@@ -205,7 +211,7 @@ struct EasyCacheState {
         return false;
     }
 
-    void after_condition(const void* cond, const sd::Tensor<float>& input, const sd::Tensor<float>& output) {
+    void after_condition(const SDCondition* cond, ggml_tensor* input, ggml_tensor* output) {
         if (!step_is_active()) {
             return;
         }
@@ -214,16 +220,16 @@ struct EasyCacheState {
             return;
         }
 
-        size_t ne            = static_cast<size_t>(input.numel());
-        const float* in_data = input.data();
+        size_t ne      = static_cast<size_t>(ggml_nelements(input));
+        float* in_data = (float*)input->data;
         prev_input.resize(ne);
         for (size_t i = 0; i < ne; ++i) {
             prev_input[i] = in_data[i];
         }
         has_prev_input = true;
 
-        const float* out_data = output.data();
-        float output_change   = 0.0f;
+        float* out_data     = (float*)output->data;
+        float output_change = 0.0f;
         if (has_prev_output && prev_output.size() == ne) {
             for (size_t i = 0; i < ne; ++i) {
                 output_change += std::fabs(out_data[i] - prev_output[i]);
@@ -256,6 +262,4 @@ struct EasyCacheState {
         cumulative_change_rate = 0.0f;
         has_last_input_change  = false;
     }
-};
-
-#endif
+};

esrgan.hpp

@@ -27,11 +27,11 @@ public:
         blocks["conv5"] = std::shared_ptr<GGMLBlock>(new Conv2d(num_feat + 4 * num_grow_ch, num_feat, {3, 3}, {1, 1}, {1, 1}));
     }
 
-    ggml_tensor* lrelu(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* lrelu(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         return ggml_leaky_relu(ctx->ggml_ctx, x, 0.2f, true);
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         // x: [n, num_feat, h, w]
         // return: [n, num_feat, h, w]
 
@@ -51,7 +51,7 @@ public:
         x_cat      = ggml_concat(ctx->ggml_ctx, x_cat, x4, 2);
         auto x5    = conv5->forward(ctx, x_cat);
 
-        x5 = ggml_add(ctx->ggml_ctx, ggml_ext_scale(ctx->ggml_ctx, x5, 0.2f), x);
+        x5 = ggml_add(ctx->ggml_ctx, ggml_scale(ctx->ggml_ctx, x5, 0.2f), x);
         return x5;
     }
 };
@@ -64,7 +64,7 @@ public:
         blocks["rdb3"] = std::shared_ptr<GGMLBlock>(new ResidualDenseBlock(num_feat, num_grow_ch));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         // x: [n, num_feat, h, w]
         // return: [n, num_feat, h, w]
 
@@ -76,7 +76,7 @@ public:
         out      = rdb2->forward(ctx, out);
         out      = rdb3->forward(ctx, out);
 
-        out = ggml_add(ctx->ggml_ctx, ggml_ext_scale(ctx->ggml_ctx, out, 0.2f), x);
+        out = ggml_add(ctx->ggml_ctx, ggml_scale(ctx->ggml_ctx, out, 0.2f), x);
         return out;
     }
 };
@@ -112,11 +112,11 @@ public:
     int get_scale() { return scale; }
     int get_num_block() { return num_block; }
 
-    ggml_tensor* lrelu(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* lrelu(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         return ggml_leaky_relu(ctx->ggml_ctx, x, 0.2f, true);
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         // x: [n, num_in_ch, h, w]
         // return: [n, num_out_ch, h*scale, w*scale]
         auto conv_first = std::dynamic_pointer_cast<Conv2d>(blocks["conv_first"]);
@@ -124,33 +124,27 @@ public:
         auto conv_hr    = std::dynamic_pointer_cast<Conv2d>(blocks["conv_hr"]);
         auto conv_last  = std::dynamic_pointer_cast<Conv2d>(blocks["conv_last"]);
 
-        auto feat = conv_first->forward(ctx, x);
-        sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.prelude", "feat");
+        auto feat      = conv_first->forward(ctx, x);
         auto body_feat = feat;
         for (int i = 0; i < num_block; i++) {
             std::string name = "body." + std::to_string(i);
             auto block       = std::dynamic_pointer_cast<RRDB>(blocks[name]);
 
             body_feat = block->forward(ctx, body_feat);
-            sd::ggml_graph_cut::mark_graph_cut(body_feat, "esrgan.body." + std::to_string(i), "feat");
         }
         body_feat = conv_body->forward(ctx, body_feat);
         feat      = ggml_add(ctx->ggml_ctx, feat, body_feat);
-        sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.body.out", "feat");
         // upsample
         if (scale >= 2) {
             auto conv_up1 = std::dynamic_pointer_cast<Conv2d>(blocks["conv_up1"]);
             feat          = lrelu(ctx, conv_up1->forward(ctx, ggml_upscale(ctx->ggml_ctx, feat, 2, GGML_SCALE_MODE_NEAREST)));
-            sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.up1", "feat");
             if (scale == 4) {
                 auto conv_up2 = std::dynamic_pointer_cast<Conv2d>(blocks["conv_up2"]);
                 feat          = lrelu(ctx, conv_up2->forward(ctx, ggml_upscale(ctx->ggml_ctx, feat, 2, GGML_SCALE_MODE_NEAREST)));
-                sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.up2", "feat");
             }
         }
         // for all scales
         auto out = conv_last->forward(ctx, lrelu(ctx, conv_hr->forward(ctx, feat)));
-        sd::ggml_graph_cut::mark_graph_cut(out, "esrgan.final", "out");
         return out;
     }
 };
@@ -161,10 +155,10 @@ struct ESRGAN : public GGMLRunner {
     int tile_size = 128;  // avoid cuda OOM for 4gb VRAM
 
     ESRGAN(ggml_backend_t backend,
-           ggml_backend_t params_backend,
+           bool offload_params_to_cpu,
            int tile_size                                  = 128,
            const String2TensorStorage& tensor_storage_map = {})
-        : GGMLRunner(backend, params_backend) {
+        : GGMLRunner(backend, offload_params_to_cpu) {
         this->tile_size = tile_size;
     }
 
@@ -347,25 +341,28 @@ struct ESRGAN : public GGMLRunner {
         return success;
     }
 
-    ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor) {
+    struct ggml_cgraph* build_graph(struct ggml_tensor* x) {
         if (!rrdb_net)
             return nullptr;
         constexpr int kGraphNodes = 1 << 16;  // 65k
-        ggml_cgraph* gf           = new_graph_custom(kGraphNodes);
-        ggml_tensor* x            = make_input(x_tensor);
+        struct ggml_cgraph* gf    = new_graph_custom(kGraphNodes);
+        x                         = to_backend(x);
 
-        auto runner_ctx  = get_context();
-        ggml_tensor* out = rrdb_net->forward(&runner_ctx, x);
+        auto runner_ctx         = get_context();
+        struct ggml_tensor* out = rrdb_net->forward(&runner_ctx, x);
         ggml_build_forward_expand(gf, out);
         return gf;
     }
 
-    sd::Tensor<float> compute(const int n_threads,
-                              const sd::Tensor<float>& x) {
-        auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x); };
-        auto result    = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
-        return result;
+    bool compute(const int n_threads,
+                 struct ggml_tensor* x,
+                 ggml_tensor** output,
+                 ggml_context* output_ctx = nullptr) {
+        auto get_graph = [&]() -> struct ggml_cgraph* {
+            return build_graph(x);
+        };
+        return GGMLRunner::compute(get_graph, n_threads, false, output, output_ctx);
     }
 };
 
-#endif  // __ESRGAN_HPP__
+#endif  // __ESRGAN_HPP__

examples/cli/CMakeLists.txt

@@ -1,20 +1,6 @@
 set(TARGET sd-cli)
 
-add_executable(${TARGET}
-    ../common/common.cpp
-    ../common/log.cpp
-    ../common/media_io.cpp
-    image_metadata.cpp
-    main.cpp
-)
+add_executable(${TARGET} main.cpp)
 install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE stable-diffusion zip ${CMAKE_THREAD_LIBS_INIT})
-if(SD_WEBP)
-    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBP)
-    target_link_libraries(${TARGET} PRIVATE webp libwebpmux)
-endif()
-if(SD_WEBM)
-    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBM)
-    target_link_libraries(${TARGET} PRIVATE webm)
-endif()
-target_compile_features(${TARGET} PUBLIC c_std_11 cxx_std_17)
+target_link_libraries(${TARGET} PRIVATE stable-diffusion ${CMAKE_THREAD_LIBS_INIT})
+target_compile_features(${TARGET} PUBLIC c_std_11 cxx_std_17)

examples/cli/README.md

@@ -4,29 +4,20 @@
 usage: ./bin/sd-cli  [options]
 
 CLI Options:
-  -o, --output <string>         path to write result image to. you can use printf-style %d format specifiers for image
-                                sequences (default: ./output.png) (eg. output_%03d.png). Single-file video outputs
-                                support .avi, .webm, and animated .webp
-  --image <string>              path to the image to inspect (for metadata mode)
-  --metadata-format <string>    metadata output format, one of [text, json] (default: text)
-  --preview-path <string>       path to write preview image to (default: ./preview.png). Multi-frame previews support
-                                .avi, .webm, and animated .webp
-  --preview-interval <int>      interval in denoising steps between consecutive updates of the image preview file
-                                (default is 1, meaning updating at every step)
-  --output-begin-idx <int>      starting index for output image sequence, must be non-negative (default 0 if specified
-                                %d in output path, 1 otherwise)
-  --canny                       apply canny preprocessor (edge detection)
-  --convert-name                convert tensor name (for convert mode)
-  -v, --verbose                 print extra info
-  --color                       colors the logging tags according to level
-  --taesd-preview-only          prevents usage of taesd for decoding the final image. (for use with --preview tae)
-  --preview-noisy               enables previewing noisy inputs of the models rather than the denoised outputs
-  --metadata-raw                include raw hex previews for unparsed metadata payloads
-  --metadata-brief              truncate long metadata text values in text output
-  --metadata-all                include structural/container entries such as IHDR, IDAT, and non-metadata JPEG segments
-  -M, --mode                    run mode, one of [img_gen, vid_gen, upscale, convert, metadata], default: img_gen
-  --preview                     preview method. must be one of the following [none, proj, tae, vae] (default is none)
-  -h, --help                    show this help message and exit
+  -o, --output <string>       path to write result image to. you can use printf-style %d format specifiers for image sequences (default: ./output.png) (eg. output_%03d.png)
+  --output-begin-idx <int>    starting index for output image sequence, must be non-negative (default 0 if specified %d in output path, 1 otherwise)
+  --preview-path <string>     path to write preview image to (default: ./preview.png)
+  --preview-interval <int>    interval in denoising steps between consecutive updates of the image preview file (default is 1, meaning updating at
+                              every step)
+  --canny                     apply canny preprocessor (edge detection)
+  --convert-name              convert tensor name (for convert mode)
+  -v, --verbose               print extra info
+  --color                     colors the logging tags according to level
+  --taesd-preview-only        prevents usage of taesd for decoding the final image. (for use with --preview tae)
+  --preview-noisy             enables previewing noisy inputs of the models rather than the denoised outputs
+  -M, --mode                  run mode, one of [img_gen, vid_gen, upscale, convert], default: img_gen
+  --preview                   preview method. must be one of the following [none, proj, tae, vae] (default is none)
+  -h, --help                  show this help message and exit
 
 Context Options:
   -m, --model <string>                     path to full model
@@ -34,8 +25,7 @@ Context Options:
   --clip_g <string>                        path to the clip-g text encoder
   --clip_vision <string>                   path to the clip-vision encoder
   --t5xxl <string>                         path to the t5xxl text encoder
-  --llm <string>                           path to the llm text encoder. For example: (qwenvl2.5 for qwen-image,
-                                           mistral-small3.2 for flux2, ...)
+  --llm <string>                           path to the llm text encoder. For example: (qwenvl2.5 for qwen-image, mistral-small3.2 for flux2, ...)
   --llm_vision <string>                    path to the llm vit
   --qwen2vl <string>                       alias of --llm. Deprecated.
   --qwen2vl_vision <string>                alias of --llm_vision. Deprecated.
@@ -47,45 +37,43 @@ Context Options:
   --control-net <string>                   path to control net model
   --embd-dir <string>                      embeddings directory
   --lora-model-dir <string>                lora model directory
-  --hires-upscalers-dir <string>           highres fix upscaler model directory
   --tensor-type-rules <string>             weight type per tensor pattern (example: "^vae\.=f16,model\.=q8_0")
   --photo-maker <string>                   path to PHOTOMAKER model
   --upscale-model <string>                 path to esrgan model.
-  -t, --threads <int>                      number of threads to use during computation (default: -1). If threads <= 0,
-                                           then threads will be set to the number of CPU physical cores
+  -t, --threads <int>                      number of threads to use during computation (default: -1). If threads <= 0, then threads will be set to the number of
+                                           CPU physical cores
   --chroma-t5-mask-pad <int>               t5 mask pad size of chroma
-  --max-vram <float>                       maximum VRAM budget in GiB for graph-cut segmented execution. 0 disables
-                                           graph splitting; -1 auto-detects free VRAM minus 1 GiB
+  --vae-tile-overlap <float>               tile overlap for vae tiling, in fraction of tile size (default: 0.5)
+  --flow-shift <float>                     shift value for Flow models like SD3.x or WAN (default: auto)
+  --vae-tiling                             process vae in tiles to reduce memory usage
   --force-sdxl-vae-conv-scale              force use of conv scale on sdxl vae
-  --offload-to-cpu                         place the weights in RAM to save VRAM, and automatically load them into VRAM
-                                           when needed
+  --offload-to-cpu                         place the weights in RAM to save VRAM, and automatically load them into VRAM when needed
   --mmap                                   whether to memory-map model
   --control-net-cpu                        keep controlnet in cpu (for low vram)
   --clip-on-cpu                            keep clip in cpu (for low vram)
   --vae-on-cpu                             keep vae in cpu (for low vram)
-  --fa                                     use flash attention
-  --diffusion-fa                           use flash attention in the diffusion model only
+  --disable-fa                             disable flash attention
   --diffusion-conv-direct                  use ggml_conv2d_direct in the diffusion model
   --vae-conv-direct                        use ggml_conv2d_direct in the vae model
   --circular                               enable circular padding for convolutions
   --circularx                              enable circular RoPE wrapping on x-axis (width) only
   --circulary                              enable circular RoPE wrapping on y-axis (height) only
   --chroma-disable-dit-mask                disable dit mask for chroma
-  --qwen-image-zero-cond-t                 enable zero_cond_t for qwen image
   --chroma-enable-t5-mask                  enable t5 mask for chroma
-  --type                                   weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K,
-                                           q4_K). If not specified, the default is the type of the weight file
+  --type                                   weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K, q4_K). If not specified, the default is the
+                                           type of the weight file
   --rng                                    RNG, one of [std_default, cuda, cpu], default: cuda(sd-webui), cpu(comfyui)
   --sampler-rng                            sampler RNG, one of [std_default, cuda, cpu]. If not specified, use --rng
-  --prediction                             prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow,
-                                           flux2_flow]
-  --lora-apply-mode                        the way to apply LoRA, one of [auto, immediately, at_runtime], default is
-                                           auto. In auto mode, if the model weights contain any quantized parameters,
-                                           the at_runtime mode will be used; otherwise, immediately will be used.The
-                                           immediately mode may have precision and compatibility issues with quantized
-                                           parameters, but it usually offers faster inference speed and, in some cases,
-                                           lower memory usage. The at_runtime mode, on the other hand, is exactly the
-                                           opposite.
+  --prediction                             prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow, flux2_flow]
+  --lora-apply-mode                        the way to apply LoRA, one of [auto, immediately, at_runtime], default is auto. In auto mode, if the model weights
+                                           contain any quantized parameters, the at_runtime mode will be used; otherwise,
+                                           immediately will be used.The immediately mode may have precision and
+                                           compatibility issues with quantized parameters, but it usually offers faster inference
+                                           speed and, in some cases, lower memory usage. The at_runtime mode, on the
+                                           other hand, is exactly the opposite.
+  --vae-tile-size                          tile size for vae tiling, format [X]x[Y] (default: 32x32)
+  --vae-relative-tile-size                 relative tile size for vae tiling, format [X]x[Y], in fraction of image size if < 1, in number of tiles per dim if >=1
+                                           (overrides --vae-tile-size)
 
 Generation Options:
   -p, --prompt <string>                    the prompt to render
@@ -94,108 +82,62 @@ Generation Options:
   --end-img <string>                       path to the end image, required by flf2v
   --mask <string>                          path to the mask image
   --control-image <string>                 path to control image, control net
-  --control-video <string>                 path to control video frames, It must be a directory path. The video frames
-                                           inside should be stored as images in lexicographical (character) order. For
-                                           example, if the control video path is `frames`, the directory contain images
-                                           such as 00.png, 01.png, ... etc.
+  --control-video <string>                 path to control video frames, It must be a directory path. The video frames inside should be stored as images in
+                                           lexicographical (character) order. For example, if the control video path is
+                                           `frames`, the directory contain images such as 00.png, 01.png, ... etc.
   --pm-id-images-dir <string>              path to PHOTOMAKER input id images dir
   --pm-id-embed-path <string>              path to PHOTOMAKER v2 id embed
-  --hires-upscaler <string>                highres fix upscaler, Lanczos, Nearest, Latent, Latent (nearest), Latent
-                                           (nearest-exact), Latent (antialiased), Latent (bicubic), Latent (bicubic
-                                           antialiased), or a model name under --hires-upscalers-dir (default: Latent)
-  --extra-sample-args <string>             extra sampler args, key=value list. Currently lcm supports noise_clip_std,
-                                           noise_scale_start, noise_scale_end
   -H, --height <int>                       image height, in pixel space (default: 512)
   -W, --width <int>                        image width, in pixel space (default: 512)
   --steps <int>                            number of sample steps (default: 20)
   --high-noise-steps <int>                 (high noise) number of sample steps (default: -1 = auto)
-  --clip-skip <int>                        ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer
-                                           (default: -1). <= 0 represents unspecified, will be 1 for SD1.x, 2 for SD2.x
+  --clip-skip <int>                        ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1). <= 0 represents unspecified,
+                                           will be 1 for SD1.x, 2 for SD2.x
   -b, --batch-count <int>                  batch count
   --video-frames <int>                     video frames (default: 1)
   --fps <int>                              fps (default: 24)
-  --timestep-shift <int>                   shift timestep for NitroFusion models (default: 0). recommended N for
-                                           NitroSD-Realism around 250 and 500 for NitroSD-Vibrant
+  --timestep-shift <int>                   shift timestep for NitroFusion models (default: 0). recommended N for NitroSD-Realism around 250 and 500 for
+                                           NitroSD-Vibrant
   --upscale-repeats <int>                  Run the ESRGAN upscaler this many times (default: 1)
   --upscale-tile-size <int>                tile size for ESRGAN upscaling (default: 128)
-  --hires-width <int>                      highres fix target width, 0 to use --hires-scale (default: 0)
-  --hires-height <int>                     highres fix target height, 0 to use --hires-scale (default: 0)
-  --hires-steps <int>                      highres fix second pass sample steps, 0 to reuse --steps (default: 0)
-  --hires-upscale-tile-size <int>          highres fix upscaler tile size, reserved for model-backed upscalers (default:
-                                           128)
   --cfg-scale <float>                      unconditional guidance scale: (default: 7.0)
-  --img-cfg-scale <float>                  image guidance scale for inpaint or instruct-pix2pix models: (default: same
-                                           as --cfg-scale)
+  --img-cfg-scale <float>                  image guidance scale for inpaint or instruct-pix2pix models: (default: same as --cfg-scale)
   --guidance <float>                       distilled guidance scale for models with guidance input (default: 3.5)
-  --slg-scale <float>                      skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means
-                                           disabled, a value of 2.5 is nice for sd3.5 medium
+  --slg-scale <float>                      skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means disabled, a value of 2.5 is nice for sd3.5
+                                           medium
   --skip-layer-start <float>               SLG enabling point (default: 0.01)
   --skip-layer-end <float>                 SLG disabling point (default: 0.2)
-  --eta <float>                            noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and
-                                           res_2s; 1 for euler_a, er_sde and dpm++2s_a)
-  --flow-shift <float>                     shift value for Flow models like SD3.x or WAN (default: auto)
+  --eta <float>                            eta in DDIM, only for DDIM and TCD (default: 0)
   --high-noise-cfg-scale <float>           (high noise) unconditional guidance scale: (default: 7.0)
-  --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or instruct-pix2pix models
-                                           (default: same as --cfg-scale)
-  --high-noise-guidance <float>            (high noise) distilled guidance scale for models with guidance input
-                                           (default: 3.5)
-  --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default:
-                                           0)
+  --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or instruct-pix2pix models (default: same as --cfg-scale)
+  --high-noise-guidance <float>            (high noise) distilled guidance scale for models with guidance input (default: 3.5)
+  --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default: 0)
   --high-noise-skip-layer-start <float>    (high noise) SLG enabling point (default: 0.01)
   --high-noise-skip-layer-end <float>      (high noise) SLG disabling point (default: 0.2)
-  --high-noise-eta <float>                 (high noise) noise multiplier (default: 0 for ddim_trailing, tcd,
-                                           res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)
+  --high-noise-eta <float>                 (high noise) eta in DDIM, only for DDIM and TCD (default: 0)
   --strength <float>                       strength for noising/unnoising (default: 0.75)
-  --pm-style-strength <float>              
-  --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full
-                                           destruction of information in init image
-  --moe-boundary <float>                   timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if
-                                           `--high-noise-steps` is set to -1
+  --pm-style-strength <float>
+  --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image
+  --moe-boundary <float>                   timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if `--high-noise-steps` is set to -1
   --vace-strength <float>                  wan vace strength
-  --vae-tile-overlap <float>               tile overlap for vae tiling, in fraction of tile size (default: 0.5)
-  --hires-scale <float>                    highres fix scale when target size is not set (default: 2.0)
-  --hires-denoising-strength <float>       highres fix second pass denoising strength (default: 0.7)
-  --increase-ref-index                     automatically increase the indices of references images based on the order
-                                           they are listed (starting with 1).
+  --increase-ref-index                     automatically increase the indices of references images based on the order they are listed (starting with 1).
   --disable-auto-resize-ref-image          disable auto resize of ref images
-  --disable-image-metadata                 do not embed generation metadata on image files
-  --vae-tiling                             process vae in tiles to reduce memory usage
-  --hires                                  enable highres fix
   -s, --seed                               RNG seed (default: 42, use random seed for < 0)
-  --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m,
-                                           dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s,
-                                           er_sde, euler_cfg_pp, euler_a_cfg_pp] (default: euler for Flux/SD3/Wan, euler_a otherwise)
-  --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a,
-                                           dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep,
-                                           res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for Flux/SD3/Wan, euler_a otherwise
-  --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits,
-                                           smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent], default:
-                                           discrete
-  --sigmas                                 custom sigma values for the sampler, comma-separated (e.g.,
-                                           "14.61,7.8,3.5,0.0").
+  --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing,
+                                           tcd] (default: euler for Flux/SD3/Wan, euler_a otherwise)
+  --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm,
+                                           ddim_trailing, tcd] default: euler for Flux/SD3/Wan, euler_a otherwise
+  --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, smoothstep, sgm_uniform, simple,
+                                           kl_optimal, lcm], default: discrete
+  --sigmas                                 custom sigma values for the sampler, comma-separated (e.g., "14.61,7.8,3.5,0.0").
   --skip-layers                            layers to skip for SLG steps (default: [7,8,9])
   --high-noise-skip-layers                 (high noise) layers to skip for SLG steps (default: [7,8,9])
   -r, --ref-image                          reference image for Flux Kontext models (can be used multiple times)
-  --cache-mode                             caching method: 'easycache' (DiT), 'ucache' (UNET),
-                                           'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT
-                                           Chebyshev+Taylor forecasting)
+  --cache-mode                             caching method: 'easycache' (DiT), 'ucache' (UNET), 'dbcache'/'taylorseer'/'cache-dit' (DiT block-level)
   --cache-option                           named cache params (key=value format, comma-separated). easycache/ucache:
-                                           threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit:
-                                           Fn=,Bn=,threshold=,warmup=; spectrum: w=,m=,lam=,window=,flex=,warmup=,stop=.
-                                           Examples: "threshold=0.25" or "threshold=1.5,reset=0"
-  --scm-mask                               SCM steps mask for cache-dit: comma-separated 0/1 (e.g.,
-                                           "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache
+                                           threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit: Fn=,Bn=,threshold=,warmup=. Examples:
+                                           "threshold=0.25" or "threshold=1.5,reset=0"
+  --cache-preset                           cache-dit preset: 'slow'/'s', 'medium'/'m', 'fast'/'f', 'ultra'/'u'
+  --scm-mask                               SCM steps mask for cache-dit: comma-separated 0/1 (e.g., "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache
   --scm-policy                             SCM policy: 'dynamic' (default) or 'static'
-  --vae-tile-size                          tile size for vae tiling, format [X]x[Y] (default: 32x32)
-  --vae-relative-tile-size                 relative tile size for vae tiling, format [X]x[Y], in fraction of image size
-                                           if < 1, in number of tiles per dim if >=1 (overrides --vae-tile-size)
-```
-
-Metadata mode inspects PNG/JPEG container metadata without loading any model:
-
-```bash
-./bin/sd-cli -M metadata --image ./output.png
-./bin/sd-cli -M metadata --image ./output.jpg --metadata-format json
-./bin/sd-cli -M metadata --image ./output.png --metadata-raw
-./bin/sd-cli -M metadata --image ./output.png --metadata-all
 ```

examples/cli/avi_writer.h

@@ -0,0 +1,217 @@
+#ifndef __AVI_WRITER_H__
+#define __AVI_WRITER_H__
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+#include "stable-diffusion.h"
+
+#ifndef INCLUDE_STB_IMAGE_WRITE_H
+#include "stb_image_write.h"
+#endif
+
+typedef struct {
+    uint32_t offset;
+    uint32_t size;
+} avi_index_entry;
+
+// Write 32-bit little-endian integer
+void write_u32_le(FILE* f, uint32_t val) {
+    fwrite(&val, 4, 1, f);
+}
+
+// Write 16-bit little-endian integer
+void write_u16_le(FILE* f, uint16_t val) {
+    fwrite(&val, 2, 1, f);
+}
+
+/**
+ * Create an MJPG AVI file from an array of sd_image_t images.
+ * Images are encoded to JPEG using stb_image_write.
+ *
+ * @param filename Output AVI file name.
+ * @param images Array of input images.
+ * @param num_images Number of images in the array.
+ * @param fps Frames per second for the video.
+ * @param quality JPEG quality (0-100).
+ * @return 0 on success, -1 on failure.
+ */
+int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality = 90) {
+    if (num_images == 0) {
+        fprintf(stderr, "Error: Image array is empty.\n");
+        return -1;
+    }
+
+    FILE* f = fopen(filename, "wb");
+    if (!f) {
+        perror("Error opening file for writing");
+        return -1;
+    }
+
+    uint32_t width    = images[0].width;
+    uint32_t height   = images[0].height;
+    uint32_t channels = images[0].channel;
+    if (channels != 3 && channels != 4) {
+        fprintf(stderr, "Error: Unsupported channel count: %u\n", channels);
+        fclose(f);
+        return -1;
+    }
+
+    // --- RIFF AVI Header ---
+    fwrite("RIFF", 4, 1, f);
+    long riff_size_pos = ftell(f);
+    write_u32_le(f, 0);  // Placeholder for file size
+    fwrite("AVI ", 4, 1, f);
+
+    // 'hdrl' LIST (header list)
+    fwrite("LIST", 4, 1, f);
+    write_u32_le(f, 4 + 8 + 56 + 8 + 4 + 8 + 56 + 8 + 40);
+    fwrite("hdrl", 4, 1, f);
+
+    // 'avih' chunk (AVI main header)
+    fwrite("avih", 4, 1, f);
+    write_u32_le(f, 56);
+    write_u32_le(f, 1000000 / fps);       // Microseconds per frame
+    write_u32_le(f, 0);                   // Max bytes per second
+    write_u32_le(f, 0);                   // Padding granularity
+    write_u32_le(f, 0x110);               // Flags (HASINDEX | ISINTERLEAVED)
+    write_u32_le(f, num_images);          // Total frames
+    write_u32_le(f, 0);                   // Initial frames
+    write_u32_le(f, 1);                   // Number of streams
+    write_u32_le(f, width * height * 3);  // Suggested buffer size
+    write_u32_le(f, width);
+    write_u32_le(f, height);
+    write_u32_le(f, 0);  // Reserved
+    write_u32_le(f, 0);  // Reserved
+    write_u32_le(f, 0);  // Reserved
+    write_u32_le(f, 0);  // Reserved
+
+    // 'strl' LIST (stream list)
+    fwrite("LIST", 4, 1, f);
+    write_u32_le(f, 4 + 8 + 56 + 8 + 40);
+    fwrite("strl", 4, 1, f);
+
+    // 'strh' chunk (stream header)
+    fwrite("strh", 4, 1, f);
+    write_u32_le(f, 56);
+    fwrite("vids", 4, 1, f);              // Stream type: video
+    fwrite("MJPG", 4, 1, f);              // Codec: Motion JPEG
+    write_u32_le(f, 0);                   // Flags
+    write_u16_le(f, 0);                   // Priority
+    write_u16_le(f, 0);                   // Language
+    write_u32_le(f, 0);                   // Initial frames
+    write_u32_le(f, 1);                   // Scale
+    write_u32_le(f, fps);                 // Rate
+    write_u32_le(f, 0);                   // Start
+    write_u32_le(f, num_images);          // Length
+    write_u32_le(f, width * height * 3);  // Suggested buffer size
+    write_u32_le(f, (uint32_t)-1);        // Quality
+    write_u32_le(f, 0);                   // Sample size
+    write_u16_le(f, 0);                   // rcFrame.left
+    write_u16_le(f, 0);                   // rcFrame.top
+    write_u16_le(f, 0);                   // rcFrame.right
+    write_u16_le(f, 0);                   // rcFrame.bottom
+
+    // 'strf' chunk (stream format: BITMAPINFOHEADER)
+    fwrite("strf", 4, 1, f);
+    write_u32_le(f, 40);
+    write_u32_le(f, 40);  // biSize
+    write_u32_le(f, width);
+    write_u32_le(f, height);
+    write_u16_le(f, 1);                   // biPlanes
+    write_u16_le(f, 24);                  // biBitCount
+    fwrite("MJPG", 4, 1, f);              // biCompression (FOURCC)
+    write_u32_le(f, width * height * 3);  // biSizeImage
+    write_u32_le(f, 0);                   // XPelsPerMeter
+    write_u32_le(f, 0);                   // YPelsPerMeter
+    write_u32_le(f, 0);                   // Colors used
+    write_u32_le(f, 0);                   // Colors important
+
+    // 'movi' LIST (video frames)
+    // long movi_list_pos = ftell(f);
+    fwrite("LIST", 4, 1, f);
+    long movi_size_pos = ftell(f);
+    write_u32_le(f, 0);  // Placeholder for movi size
+    fwrite("movi", 4, 1, f);
+
+    avi_index_entry* index = (avi_index_entry*)malloc(sizeof(avi_index_entry) * num_images);
+    if (!index) {
+        fclose(f);
+        return -1;
+    }
+
+    // Encode and write each frame as JPEG
+    struct {
+        uint8_t* buf;
+        size_t size;
+    } jpeg_data;
+
+    for (int i = 0; i < num_images; i++) {
+        jpeg_data.buf  = nullptr;
+        jpeg_data.size = 0;
+
+        // Callback function to collect JPEG data into memory
+        auto write_to_buf = [](void* context, void* data, int size) {
+            auto jd = (decltype(jpeg_data)*)context;
+            jd->buf = (uint8_t*)realloc(jd->buf, jd->size + size);
+            memcpy(jd->buf + jd->size, data, size);
+            jd->size += size;
+        };
+
+        // Encode to JPEG in memory
+        stbi_write_jpg_to_func(
+            write_to_buf,
+            &jpeg_data,
+            images[i].width,
+            images[i].height,
+            channels,
+            images[i].data,
+            quality);
+
+        // Write '00dc' chunk (video frame)
+        fwrite("00dc", 4, 1, f);
+        write_u32_le(f, (uint32_t)jpeg_data.size);
+        index[i].offset = ftell(f) - 8;
+        index[i].size   = (uint32_t)jpeg_data.size;
+        fwrite(jpeg_data.buf, 1, jpeg_data.size, f);
+
+        // Align to even byte size
+        if (jpeg_data.size % 2)
+            fputc(0, f);
+
+        free(jpeg_data.buf);
+    }
+
+    // Finalize 'movi' size
+    long cur_pos   = ftell(f);
+    long movi_size = cur_pos - movi_size_pos - 4;
+    fseek(f, movi_size_pos, SEEK_SET);
+    write_u32_le(f, movi_size);
+    fseek(f, cur_pos, SEEK_SET);
+
+    // Write 'idx1' index
+    fwrite("idx1", 4, 1, f);
+    write_u32_le(f, num_images * 16);
+    for (int i = 0; i < num_images; i++) {
+        fwrite("00dc", 4, 1, f);
+        write_u32_le(f, 0x10);
+        write_u32_le(f, index[i].offset);
+        write_u32_le(f, index[i].size);
+    }
+
+    // Finalize RIFF size
+    cur_pos        = ftell(f);
+    long file_size = cur_pos - riff_size_pos - 4;
+    fseek(f, riff_size_pos, SEEK_SET);
+    write_u32_le(f, file_size);
+    fseek(f, cur_pos, SEEK_SET);
+
+    fclose(f);
+    free(index);
+
+    return 0;
+}
+
+#endif  // __AVI_WRITER_H__

examples/cli/image_metadata.h

@@ -1,21 +0,0 @@
-#pragma once
-
-#include <iosfwd>
-#include <string>
-
-enum class MetadataOutputFormat {
-    TEXT,
-    JSON,
-};
-
-struct MetadataReadOptions {
-    MetadataOutputFormat output_format = MetadataOutputFormat::TEXT;
-    bool include_raw                   = false;
-    bool brief                         = false;
-    bool include_structural            = false;
-};
-
-bool print_image_metadata(const std::string& image_path,
-                          const MetadataReadOptions& options,
-                          std::ostream& out,
-                          std::string& error);

examples/cli/main.cpp

@@ -15,12 +15,9 @@
 // #include "preprocessing.hpp"
 #include "stable-diffusion.h"
 
-#include "common/common.h"
-#include "common/media_io.h"
-#include "common/resource_owners.hpp"
-#include "image_metadata.h"
+#include "common/common.hpp"
 
-namespace fs = std::filesystem;
+#include "avi_writer.h"
 
 const char* previews_str[] = {
     "none",
@@ -35,8 +32,6 @@ struct SDCliParams {
     SDMode mode             = IMG_GEN;
     std::string output_path = "output.png";
     int output_begin_idx    = -1;
-    std::string image_path;
-    std::string metadata_format = "text";
 
     bool verbose          = false;
     bool canny_preprocess = false;
@@ -49,9 +44,6 @@ struct SDCliParams {
     bool taesd_preview       = false;
     bool preview_noisy       = false;
     bool color               = false;
-    bool metadata_raw        = false;
-    bool metadata_brief      = false;
-    bool metadata_all        = false;
 
     bool normal_exit = false;
 
@@ -61,19 +53,11 @@ struct SDCliParams {
         options.string_options = {
             {"-o",
              "--output",
-             "path to write result image to. you can use printf-style %d format specifiers for image sequences (default: ./output.png) (eg. output_%03d.png). Single-file video outputs support .avi, .webm, and animated .webp",
+             "path to write result image to. you can use printf-style %d format specifiers for image sequences (default: ./output.png) (eg. output_%03d.png)",
              &output_path},
-            {"",
-             "--image",
-             "path to the image to inspect (for metadata mode)",
-             &image_path},
-            {"",
-             "--metadata-format",
-             "metadata output format, one of [text, json] (default: text)",
-             &metadata_format},
             {"",
              "--preview-path",
-             "path to write preview image to (default: ./preview.png). Multi-frame previews support .avi, .webm, and animated .webp",
+             "path to write preview image to (default: ./preview.png)",
              &preview_path},
         };
 
@@ -113,18 +97,6 @@ struct SDCliParams {
              "--preview-noisy",
              "enables previewing noisy inputs of the models rather than the denoised outputs",
              true, &preview_noisy},
-            {"",
-             "--metadata-raw",
-             "include raw hex previews for unparsed metadata payloads",
-             true, &metadata_raw},
-            {"",
-             "--metadata-brief",
-             "truncate long metadata text values in text output",
-             true, &metadata_brief},
-            {"",
-             "--metadata-all",
-             "include structural/container entries such as IHDR, IDAT, and non-metadata JPEG segments",
-             true, &metadata_all},
 
         };
 
@@ -177,7 +149,7 @@ struct SDCliParams {
         options.manual_options = {
             {"-M",
              "--mode",
-             "run mode, one of [img_gen, vid_gen, upscale, convert, metadata], default: img_gen",
+             "run mode, one of [img_gen, vid_gen, upscale, convert], default: img_gen",
              on_mode_arg},
             {"",
              "--preview",
@@ -192,7 +164,12 @@ struct SDCliParams {
         return options;
     };
 
-    bool resolve() {
+    bool process_and_check() {
+        if (output_path.length() == 0) {
+            LOG_ERROR("error: the following arguments are required: output_path");
+            return false;
+        }
+
         if (mode == CONVERT) {
             if (output_path == "output.png") {
                 output_path = "output.gguf";
@@ -201,43 +178,11 @@ struct SDCliParams {
         return true;
     }
 
-    bool validate() {
-        if (mode != METADATA) {
-            if (output_path.length() == 0) {
-                LOG_ERROR("error: the following arguments are required: output_path");
-                return false;
-            }
-        } else {
-            if (image_path.empty()) {
-                LOG_ERROR("error: metadata mode needs an image path (--image)");
-                return false;
-            }
-            if (metadata_format != "text" && metadata_format != "json") {
-                LOG_ERROR("error: invalid metadata format %s, must be one of [text, json]",
-                          metadata_format.c_str());
-                return false;
-            }
-        }
-        return true;
-    }
-
-    bool resolve_and_validate() {
-        if (!resolve()) {
-            return false;
-        }
-        if (!validate()) {
-            return false;
-        }
-        return true;
-    }
-
     std::string to_string() const {
         std::ostringstream oss;
         oss << "SDCliParams {\n"
             << "  mode: " << modes_str[mode] << ",\n"
             << "  output_path: \"" << output_path << "\",\n"
-            << "  image_path: \"" << image_path << "\",\n"
-            << "  metadata_format: \"" << metadata_format << "\",\n"
             << "  verbose: " << (verbose ? "true" : "false") << ",\n"
             << "  color: " << (color ? "true" : "false") << ",\n"
             << "  canny_preprocess: " << (canny_preprocess ? "true" : "false") << ",\n"
@@ -247,10 +192,7 @@ struct SDCliParams {
             << "  preview_path: \"" << preview_path << "\",\n"
             << "  preview_fps: " << preview_fps << ",\n"
             << "  taesd_preview: " << (taesd_preview ? "true" : "false") << ",\n"
-            << "  preview_noisy: " << (preview_noisy ? "true" : "false") << ",\n"
-            << "  metadata_raw: " << (metadata_raw ? "true" : "false") << ",\n"
-            << "  metadata_brief: " << (metadata_brief ? "true" : "false") << ",\n"
-            << "  metadata_all: " << (metadata_all ? "true" : "false") << "\n"
+            << "  preview_noisy: " << (preview_noisy ? "true" : "false") << "\n"
             << "}";
         return oss.str();
     }
@@ -275,27 +217,78 @@ void parse_args(int argc, const char** argv, SDCliParams& cli_params, SDContextP
         exit(cli_params.normal_exit ? 0 : 1);
     }
 
-    bool valid = cli_params.resolve_and_validate();
-    if (valid && cli_params.mode != METADATA) {
-        valid = ctx_params.resolve_and_validate(cli_params.mode) &&
-                gen_params.resolve_and_validate(cli_params.mode,
-                                                ctx_params.lora_model_dir,
-                                                ctx_params.hires_upscalers_dir);
-    }
-
-    if (!valid) {
+    if (!cli_params.process_and_check() ||
+        !ctx_params.process_and_check(cli_params.mode) ||
+        !gen_params.process_and_check(cli_params.mode, ctx_params.lora_model_dir)) {
         print_usage(argc, argv, options_vec);
         exit(1);
     }
 }
 
+std::string get_image_params(const SDCliParams& cli_params, const SDContextParams& ctx_params, const SDGenerationParams& gen_params, int64_t seed) {
+    std::string parameter_string = gen_params.prompt_with_lora + "\n";
+    if (gen_params.negative_prompt.size() != 0) {
+        parameter_string += "Negative prompt: " + gen_params.negative_prompt + "\n";
+    }
+    parameter_string += "Steps: " + std::to_string(gen_params.sample_params.sample_steps) + ", ";
+    parameter_string += "CFG scale: " + std::to_string(gen_params.sample_params.guidance.txt_cfg) + ", ";
+    if (gen_params.sample_params.guidance.slg.scale != 0 && gen_params.skip_layers.size() != 0) {
+        parameter_string += "SLG scale: " + std::to_string(gen_params.sample_params.guidance.txt_cfg) + ", ";
+        parameter_string += "Skip layers: [";
+        for (const auto& layer : gen_params.skip_layers) {
+            parameter_string += std::to_string(layer) + ", ";
+        }
+        parameter_string += "], ";
+        parameter_string += "Skip layer start: " + std::to_string(gen_params.sample_params.guidance.slg.layer_start) + ", ";
+        parameter_string += "Skip layer end: " + std::to_string(gen_params.sample_params.guidance.slg.layer_end) + ", ";
+    }
+    parameter_string += "Guidance: " + std::to_string(gen_params.sample_params.guidance.distilled_guidance) + ", ";
+    parameter_string += "Eta: " + std::to_string(gen_params.sample_params.eta) + ", ";
+    parameter_string += "Seed: " + std::to_string(seed) + ", ";
+    parameter_string += "Size: " + std::to_string(gen_params.width) + "x" + std::to_string(gen_params.height) + ", ";
+    parameter_string += "Model: " + sd_basename(ctx_params.model_path) + ", ";
+    parameter_string += "RNG: " + std::string(sd_rng_type_name(ctx_params.rng_type)) + ", ";
+    if (ctx_params.sampler_rng_type != RNG_TYPE_COUNT) {
+        parameter_string += "Sampler RNG: " + std::string(sd_rng_type_name(ctx_params.sampler_rng_type)) + ", ";
+    }
+    parameter_string += "Sampler: " + std::string(sd_sample_method_name(gen_params.sample_params.sample_method));
+    if (!gen_params.custom_sigmas.empty()) {
+        parameter_string += ", Custom Sigmas: [";
+        for (size_t i = 0; i < gen_params.custom_sigmas.size(); ++i) {
+            std::ostringstream oss;
+            oss << std::fixed << std::setprecision(4) << gen_params.custom_sigmas[i];
+            parameter_string += oss.str() + (i == gen_params.custom_sigmas.size() - 1 ? "" : ", ");
+        }
+        parameter_string += "]";
+    } else if (gen_params.sample_params.scheduler != SCHEDULER_COUNT) {  // Only show schedule if not using custom sigmas
+        parameter_string += " " + std::string(sd_scheduler_name(gen_params.sample_params.scheduler));
+    }
+    parameter_string += ", ";
+    for (const auto& te : {ctx_params.clip_l_path, ctx_params.clip_g_path, ctx_params.t5xxl_path, ctx_params.llm_path, ctx_params.llm_vision_path}) {
+        if (!te.empty()) {
+            parameter_string += "TE: " + sd_basename(te) + ", ";
+        }
+    }
+    if (!ctx_params.diffusion_model_path.empty()) {
+        parameter_string += "Unet: " + sd_basename(ctx_params.diffusion_model_path) + ", ";
+    }
+    if (!ctx_params.vae_path.empty()) {
+        parameter_string += "VAE: " + sd_basename(ctx_params.vae_path) + ", ";
+    }
+    if (gen_params.clip_skip != -1) {
+        parameter_string += "Clip skip: " + std::to_string(gen_params.clip_skip) + ", ";
+    }
+    parameter_string += "Version: stable-diffusion.cpp";
+    return parameter_string;
+}
+
 void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
     SDCliParams* cli_params = (SDCliParams*)data;
     log_print(level, log, cli_params->verbose, cli_params->color);
 }
 
 bool load_images_from_dir(const std::string dir,
-                          std::vector<SDImageOwner>& images,
+                          std::vector<sd_image_t>& images,
                           int expected_width  = 0,
                           int expected_height = 0,
                           int max_image_num   = 0,
@@ -322,7 +315,7 @@ bool load_images_from_dir(const std::string dir,
         std::string ext  = entry.path().extension().string();
         std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
 
-        if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".bmp" || ext == ".webp") {
+        if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".bmp") {
             LOG_DEBUG("load image %zu from '%s'", images.size(), path.c_str());
             int width             = 0;
             int height            = 0;
@@ -332,12 +325,12 @@ bool load_images_from_dir(const std::string dir,
                 return false;
             }
 
-            images.emplace_back(sd_image_t{(uint32_t)width,
-                                           (uint32_t)height,
-                                           3,
-                                           image_buffer});
+            images.push_back({(uint32_t)width,
+                              (uint32_t)height,
+                              3,
+                              image_buffer});
 
-            if (max_image_num > 0 && static_cast<int>(images.size()) >= max_image_num) {
+            if (max_image_num > 0 && images.size() >= max_image_num) {
                 break;
             }
         }
@@ -352,17 +345,9 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy,
     // is_noisy is set to true if the preview corresponds to noisy latents, false if it's denoised latents
     // unused in this app, it will either be always noisy or always denoised here
     if (frame_count == 1) {
-        if (!write_image_to_file(cli_params->preview_path,
-                                 image->data,
-                                 image->width,
-                                 image->height,
-                                 image->channel)) {
-            LOG_ERROR("save preview image to '%s' failed", cli_params->preview_path.c_str());
-        }
+        stbi_write_png(cli_params->preview_path.c_str(), image->width, image->height, image->channel, image->data, 0);
     } else {
-        if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps) != 0) {
-            LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str());
-        }
+        create_mjpg_avi_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps);
     }
 }
 
@@ -409,19 +394,12 @@ bool save_results(const SDCliParams& cli_params,
 
     fs::path base_path = out_path;
     fs::path ext       = out_path.has_extension() ? out_path.extension() : fs::path{};
+    if (!ext.empty())
+        base_path.replace_extension();
 
     std::string ext_lower = ext.string();
     std::transform(ext_lower.begin(), ext_lower.end(), ext_lower.begin(), ::tolower);
-    const EncodedImageFormat output_format = encoded_image_format_from_path(out_path.string());
-    if (!ext.empty()) {
-        if (output_format == EncodedImageFormat::JPEG ||
-            output_format == EncodedImageFormat::PNG ||
-            output_format == EncodedImageFormat::WEBP ||
-            ext_lower == ".avi" ||
-            ext_lower == ".webm") {
-            base_path.replace_extension();
-        }
-    }
+    bool is_jpg = (ext_lower == ".jpg" || ext_lower == ".jpeg" || ext_lower == ".jpe");
 
     int output_begin_idx = cli_params.output_begin_idx;
     if (output_begin_idx < 0) {
@@ -431,50 +409,42 @@ bool save_results(const SDCliParams& cli_params,
     auto write_image = [&](const fs::path& path, int idx) {
         const sd_image_t& img = results[idx];
         if (!img.data)
-            return false;
+            return;
 
-        const int64_t metadata_seed = cli_params.mode == VID_GEN ? gen_params.seed : gen_params.seed + idx;
-        std::string params          = gen_params.embed_image_metadata
-                                          ? get_image_params(ctx_params, gen_params, metadata_seed, cli_params.mode)
-                                          : "";
-        const bool ok               = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, 90);
+        std::string params = get_image_params(cli_params, ctx_params, gen_params, gen_params.seed + idx);
+        int ok             = 0;
+        if (is_jpg) {
+            ok = stbi_write_jpg(path.string().c_str(), img.width, img.height, img.channel, img.data, 90, params.c_str());
+        } else {
+            ok = stbi_write_png(path.string().c_str(), img.width, img.height, img.channel, img.data, 0, params.c_str());
+        }
         LOG_INFO("save result image %d to '%s' (%s)", idx, path.string().c_str(), ok ? "success" : "failure");
-        return ok;
     };
 
-    int sucessful_reults = 0;
-
     if (std::regex_search(cli_params.output_path, format_specifier_regex)) {
-        if (output_format == EncodedImageFormat::UNKNOWN)
+        if (!is_jpg && ext_lower != ".png")
             ext = ".png";
         fs::path pattern = base_path;
         pattern += ext;
 
         for (int i = 0; i < num_results; ++i) {
             fs::path img_path = format_frame_idx(pattern.string(), output_begin_idx + i);
-            if (write_image(img_path, i)) {
-                sucessful_reults++;
-            }
+            write_image(img_path, i);
         }
-        LOG_INFO("%d/%d images saved", sucessful_reults, num_results);
-        return sucessful_reults != 0;
+        return true;
     }
 
     if (cli_params.mode == VID_GEN && num_results > 1) {
-        if (ext_lower != ".avi" && ext_lower != ".webp" && ext_lower != ".webm")
+        if (ext_lower != ".avi")
             ext = ".avi";
         fs::path video_path = base_path;
         video_path += ext;
-        if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps) == 0) {
-            LOG_INFO("save result video to '%s'", video_path.string().c_str());
-            return true;
-        } else {
-            LOG_ERROR("Failed to save result video to '%s'", video_path.string().c_str());
-            return false;
-        }
+        create_mjpg_avi_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps);
+        LOG_INFO("save result MJPG AVI video to '%s'", video_path.string().c_str());
+        return true;
     }
 
-    if (output_format == EncodedImageFormat::UNKNOWN)
+    if (!is_jpg && ext_lower != ".png")
         ext = ".png";
 
     for (int i = 0; i < num_results; ++i) {
@@ -483,12 +453,10 @@ bool save_results(const SDCliParams& cli_params,
             img_path += "_" + std::to_string(output_begin_idx + i);
         }
         img_path += ext;
-        if (write_image(img_path, i)) {
-            sucessful_reults++;
-        }
+        write_image(img_path, i);
     }
-    LOG_INFO("%d/%d images saved", sucessful_reults, num_results);
-    return sucessful_reults != 0;
+
+    return true;
 }
 
 int main(int argc, const char* argv[]) {
@@ -502,27 +470,6 @@ int main(int argc, const char* argv[]) {
     SDGenerationParams gen_params;
 
     parse_args(argc, argv, cli_params, ctx_params, gen_params);
-    sd_set_log_callback(sd_log_cb, (void*)&cli_params);
-    log_verbose = cli_params.verbose;
-    log_color   = cli_params.color;
-
-    if (cli_params.mode == METADATA) {
-        MetadataReadOptions options;
-        options.output_format      = cli_params.metadata_format == "json"
-                                         ? MetadataOutputFormat::JSON
-                                         : MetadataOutputFormat::TEXT;
-        options.include_raw        = cli_params.metadata_raw;
-        options.brief              = cli_params.metadata_brief;
-        options.include_structural = cli_params.metadata_all;
-
-        std::string error;
-        if (!print_image_metadata(cli_params.image_path, options, std::cout, error)) {
-            LOG_ERROR("%s", error.c_str());
-            return 1;
-        }
-        return 0;
-    }
-
     if (gen_params.video_frames > 4) {
         size_t last_dot_pos   = cli_params.preview_path.find_last_of(".");
         std::string base_path = cli_params.preview_path;
@@ -540,6 +487,9 @@ int main(int argc, const char* argv[]) {
     if (cli_params.preview_method == PREVIEW_PROJ)
         cli_params.preview_fps /= 4;
 
+    sd_set_log_callback(sd_log_cb, (void*)&cli_params);
+    log_verbose = cli_params.verbose;
+    log_color   = cli_params.color;
     sd_set_preview_callback(step_callback,
                             cli_params.preview_method,
                             cli_params.preview_interval,
@@ -575,86 +525,94 @@ int main(int argc, const char* argv[]) {
         }
     }
 
-    bool vae_decode_only = true;
+    bool vae_decode_only     = true;
+    sd_image_t init_image    = {(uint32_t)gen_params.width, (uint32_t)gen_params.height, 3, nullptr};
+    sd_image_t end_image     = {(uint32_t)gen_params.width, (uint32_t)gen_params.height, 3, nullptr};
+    sd_image_t control_image = {(uint32_t)gen_params.width, (uint32_t)gen_params.height, 3, nullptr};
+    sd_image_t mask_image    = {(uint32_t)gen_params.width, (uint32_t)gen_params.height, 1, nullptr};
+    std::vector<sd_image_t> ref_images;
+    std::vector<sd_image_t> pmid_images;
+    std::vector<sd_image_t> control_frames;
 
-    auto load_image_and_update_size = [&](const std::string& path,
-                                          SDImageOwner& image,
-                                          bool resize_image    = true,
-                                          int expected_channel = 3) -> bool {
-        int expected_width  = 0;
-        int expected_height = 0;
-        if (resize_image && gen_params.width_and_height_are_set()) {
-            expected_width  = gen_params.width;
-            expected_height = gen_params.height;
+    auto release_all_resources = [&]() {
+        free(init_image.data);
+        free(end_image.data);
+        free(control_image.data);
+        free(mask_image.data);
+        for (auto image : ref_images) {
+            free(image.data);
+            image.data = nullptr;
         }
-
-        if (!load_sd_image_from_file(image.put(), path.c_str(), expected_width, expected_height, expected_channel)) {
-            LOG_ERROR("load image from '%s' failed", path.c_str());
-            return false;
+        ref_images.clear();
+        for (auto image : pmid_images) {
+            free(image.data);
+            image.data = nullptr;
         }
-
-        gen_params.set_width_and_height_if_unset(image.get().width, image.get().height);
-        return true;
+        pmid_images.clear();
+        for (auto image : control_frames) {
+            free(image.data);
+            image.data = nullptr;
+        }
+        control_frames.clear();
     };
 
     if (gen_params.init_image_path.size() > 0) {
         vae_decode_only = false;
-        if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image)) {
+
+        int width       = 0;
+        int height      = 0;
+        init_image.data = load_image_from_file(gen_params.init_image_path.c_str(), width, height, gen_params.width, gen_params.height);
+        if (init_image.data == nullptr) {
+            LOG_ERROR("load image from '%s' failed", gen_params.init_image_path.c_str());
+            release_all_resources();
             return 1;
         }
     }
 
     if (gen_params.end_image_path.size() > 0) {
         vae_decode_only = false;
-        if (!load_image_and_update_size(gen_params.end_image_path, gen_params.end_image)) {
-            return 1;
-        }
-    }
 
-    if (gen_params.ref_image_paths.size() > 0) {
-        vae_decode_only = false;
-        gen_params.ref_images.clear();
-        for (auto& path : gen_params.ref_image_paths) {
-            SDImageOwner ref_image({0, 0, 3, nullptr});
-            if (!load_image_and_update_size(path, ref_image, false)) {
-                return 1;
-            }
-            gen_params.ref_images.push_back(std::move(ref_image));
+        int width      = 0;
+        int height     = 0;
+        end_image.data = load_image_from_file(gen_params.end_image_path.c_str(), width, height, gen_params.width, gen_params.height);
+        if (end_image.data == nullptr) {
+            LOG_ERROR("load image from '%s' failed", gen_params.end_image_path.c_str());
+            release_all_resources();
+            return 1;
         }
     }
 
     if (gen_params.mask_image_path.size() > 0) {
-        if (!load_sd_image_from_file(gen_params.mask_image.put(),
-                                     gen_params.mask_image_path.c_str(),
-                                     gen_params.get_resolved_width(),
-                                     gen_params.get_resolved_height(),
-                                     1)) {
+        int c           = 0;
+        int width       = 0;
+        int height      = 0;
+        mask_image.data = load_image_from_file(gen_params.mask_image_path.c_str(), width, height, gen_params.width, gen_params.height, 1);
+        if (mask_image.data == nullptr) {
             LOG_ERROR("load image from '%s' failed", gen_params.mask_image_path.c_str());
+            release_all_resources();
             return 1;
         }
     } else {
-        sd_image_t generated_mask = {0, 0, 1, nullptr};
-        generated_mask.data       = (uint8_t*)malloc(gen_params.get_resolved_width() * gen_params.get_resolved_height());
-        if (generated_mask.data == nullptr) {
+        mask_image.data = (uint8_t*)malloc(gen_params.width * gen_params.height);
+        memset(mask_image.data, 255, gen_params.width * gen_params.height);
+        if (mask_image.data == nullptr) {
             LOG_ERROR("malloc mask image failed");
+            release_all_resources();
             return 1;
         }
-        generated_mask.width  = gen_params.get_resolved_width();
-        generated_mask.height = gen_params.get_resolved_height();
-        memset(generated_mask.data, 255, gen_params.get_resolved_width() * gen_params.get_resolved_height());
-        gen_params.mask_image.reset(generated_mask);
     }
 
     if (gen_params.control_image_path.size() > 0) {
-        if (!load_sd_image_from_file(gen_params.control_image.put(),
-                                     gen_params.control_image_path.c_str(),
-                                     gen_params.get_resolved_width(),
-                                     gen_params.get_resolved_height())) {
+        int width          = 0;
+        int height         = 0;
+        control_image.data = load_image_from_file(gen_params.control_image_path.c_str(), width, height, gen_params.width, gen_params.height);
+        if (control_image.data == nullptr) {
             LOG_ERROR("load image from '%s' failed", gen_params.control_image_path.c_str());
+            release_all_resources();
             return 1;
         }
         if (cli_params.canny_preprocess) {  // apply preprocessor
-            preprocess_canny(gen_params.control_image.get(),
+            preprocess_canny(control_image,
                              0.08f,
                              0.08f,
                              0.8f,
@@ -663,26 +621,44 @@ int main(int argc, const char* argv[]) {
         }
     }
 
+    if (gen_params.ref_image_paths.size() > 0) {
+        vae_decode_only = false;
+        for (auto& path : gen_params.ref_image_paths) {
+            int width             = 0;
+            int height            = 0;
+            uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height);
+            if (image_buffer == nullptr) {
+                LOG_ERROR("load image from '%s' failed", path.c_str());
+                release_all_resources();
+                return 1;
+            }
+            ref_images.push_back({(uint32_t)width,
+                                  (uint32_t)height,
+                                  3,
+                                  image_buffer});
+        }
+    }
+
     if (!gen_params.control_video_path.empty()) {
-        gen_params.control_frames.clear();
         if (!load_images_from_dir(gen_params.control_video_path,
-                                  gen_params.control_frames,
-                                  gen_params.get_resolved_width(),
-                                  gen_params.get_resolved_height(),
+                                  control_frames,
+                                  gen_params.width,
+                                  gen_params.height,
                                   gen_params.video_frames,
                                   cli_params.verbose)) {
+            release_all_resources();
             return 1;
         }
     }
 
     if (!gen_params.pm_id_images_dir.empty()) {
-        gen_params.pm_id_images.clear();
         if (!load_images_from_dir(gen_params.pm_id_images_dir,
-                                  gen_params.pm_id_images,
+                                  pmid_images,
                                   0,
                                   0,
                                   0,
                                   cli_params.verbose)) {
+            release_all_resources();
             return 1;
         }
     }
@@ -691,67 +667,119 @@ int main(int argc, const char* argv[]) {
         vae_decode_only = false;
     }
 
-    if (gen_params.hires_enabled &&
-        (gen_params.resolved_hires_upscaler == SD_HIRES_UPSCALER_MODEL ||
-         gen_params.resolved_hires_upscaler == SD_HIRES_UPSCALER_LANCZOS ||
-         gen_params.resolved_hires_upscaler == SD_HIRES_UPSCALER_NEAREST)) {
-        vae_decode_only = false;
-    }
-
     sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(vae_decode_only, true, cli_params.taesd_preview);
 
-    SDImageVec results;
-    int num_results = 0;
+    sd_image_t* results = nullptr;
+    int num_results     = 0;
 
     if (cli_params.mode == UPSCALE) {
         num_results = 1;
-        results.push_back(gen_params.init_image.release());
+        results     = (sd_image_t*)calloc(num_results, sizeof(sd_image_t));
+        if (results == nullptr) {
+            LOG_INFO("failed to allocate results array");
+            release_all_resources();
+            return 1;
+        }
+
+        results[0]      = init_image;
+        init_image.data = nullptr;
     } else {
-        SDCtxPtr sd_ctx(new_sd_ctx(&sd_ctx_params));
+        sd_ctx_t* sd_ctx = new_sd_ctx(&sd_ctx_params);
 
         if (sd_ctx == nullptr) {
             LOG_INFO("new_sd_ctx_t failed");
+            release_all_resources();
             return 1;
         }
 
         if (gen_params.sample_params.sample_method == SAMPLE_METHOD_COUNT) {
-            gen_params.sample_params.sample_method = sd_get_default_sample_method(sd_ctx.get());
+            gen_params.sample_params.sample_method = sd_get_default_sample_method(sd_ctx);
         }
 
         if (gen_params.high_noise_sample_params.sample_method == SAMPLE_METHOD_COUNT) {
-            gen_params.high_noise_sample_params.sample_method = sd_get_default_sample_method(sd_ctx.get());
+            gen_params.high_noise_sample_params.sample_method = sd_get_default_sample_method(sd_ctx);
         }
 
         if (gen_params.sample_params.scheduler == SCHEDULER_COUNT) {
-            gen_params.sample_params.scheduler = sd_get_default_scheduler(sd_ctx.get(), gen_params.sample_params.sample_method);
+            gen_params.sample_params.scheduler = sd_get_default_scheduler(sd_ctx, gen_params.sample_params.sample_method);
         }
 
         if (cli_params.mode == IMG_GEN) {
-            sd_img_gen_params_t img_gen_params = gen_params.to_sd_img_gen_params_t();
+            sd_img_gen_params_t img_gen_params = {
+                gen_params.lora_vec.data(),
+                static_cast<uint32_t>(gen_params.lora_vec.size()),
+                gen_params.prompt.c_str(),
+                gen_params.negative_prompt.c_str(),
+                gen_params.clip_skip,
+                init_image,
+                ref_images.data(),
+                (int)ref_images.size(),
+                gen_params.auto_resize_ref_image,
+                gen_params.increase_ref_index,
+                mask_image,
+                gen_params.width,
+                gen_params.height,
+                gen_params.sample_params,
+                gen_params.strength,
+                gen_params.seed,
+                gen_params.batch_count,
+                control_image,
+                gen_params.control_strength,
+                {
+                    pmid_images.data(),
+                    (int)pmid_images.size(),
+                    gen_params.pm_id_embed_path.c_str(),
+                    gen_params.pm_style_strength,
+                },  // pm_params
+                ctx_params.vae_tiling_params,
+                gen_params.cache_params,
+            };
 
+            results     = generate_image(sd_ctx, &img_gen_params);
             num_results = gen_params.batch_count;
-            results.adopt(generate_image(sd_ctx.get(), &img_gen_params), num_results);
         } else if (cli_params.mode == VID_GEN) {
-            sd_vid_gen_params_t vid_gen_params = gen_params.to_sd_vid_gen_params_t();
-            sd_image_t* generated_video        = generate_video(sd_ctx.get(), &vid_gen_params, &num_results);
-            results.adopt(generated_video, num_results);
+            sd_vid_gen_params_t vid_gen_params = {
+                gen_params.lora_vec.data(),
+                static_cast<uint32_t>(gen_params.lora_vec.size()),
+                gen_params.prompt.c_str(),
+                gen_params.negative_prompt.c_str(),
+                gen_params.clip_skip,
+                init_image,
+                end_image,
+                control_frames.data(),
+                (int)control_frames.size(),
+                gen_params.width,
+                gen_params.height,
+                gen_params.sample_params,
+                gen_params.high_noise_sample_params,
+                gen_params.moe_boundary,
+                gen_params.strength,
+                gen_params.seed,
+                gen_params.video_frames,
+                gen_params.vace_strength,
+                ctx_params.vae_tiling_params,
+                gen_params.cache_params,
+            };
+
+            results = generate_video(sd_ctx, &vid_gen_params, &num_results);
         }
 
-        if (!results) {
+        if (results == nullptr) {
             LOG_ERROR("generate failed");
+            free_sd_ctx(sd_ctx);
             return 1;
         }
+
+        free_sd_ctx(sd_ctx);
     }
 
     int upscale_factor = 4;  // unused for RealESRGAN_x4plus_anime_6B.pth
     if (ctx_params.esrgan_path.size() > 0 && gen_params.upscale_repeats > 0) {
-        UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(ctx_params.esrgan_path.c_str(),
-                                                     ctx_params.offload_params_to_cpu,
-                                                     ctx_params.diffusion_conv_direct,
-                                                     ctx_params.n_threads,
-                                                     gen_params.upscale_tile_size,
-                                                     ctx_params.backend.c_str(),
-                                                     ctx_params.params_backend.c_str()));
+        upscaler_ctx_t* upscaler_ctx = new_upscaler_ctx(ctx_params.esrgan_path.c_str(),
+                                                        ctx_params.offload_params_to_cpu,
+                                                        ctx_params.diffusion_conv_direct,
+                                                        ctx_params.n_threads,
+                                                        gen_params.upscale_tile_size);
 
         if (upscaler_ctx == nullptr) {
             LOG_ERROR("new_upscaler_ctx failed");
@@ -760,24 +788,32 @@ int main(int argc, const char* argv[]) {
                 if (results[i].data == nullptr) {
                     continue;
                 }
-                SDImageOwner current_image(results[i]);
-                results[i] = {0, 0, 0, nullptr};
+                sd_image_t current_image = results[i];
                 for (int u = 0; u < gen_params.upscale_repeats; ++u) {
-                    SDImageOwner upscaled_image(upscale(upscaler_ctx.get(), current_image.get(), upscale_factor));
-                    if (upscaled_image.get().data == nullptr) {
+                    sd_image_t upscaled_image = upscale(upscaler_ctx, current_image, upscale_factor);
+                    if (upscaled_image.data == nullptr) {
                         LOG_ERROR("upscale failed");
                         break;
                     }
-                    current_image = std::move(upscaled_image);
+                    free(current_image.data);
+                    current_image = upscaled_image;
                 }
-                results[i] = current_image.release();  // Set the final upscaled image as the result
+                results[i] = current_image;  // Set the final upscaled image as the result
             }
         }
     }
 
-    if (!save_results(cli_params, ctx_params, gen_params, results.data(), num_results)) {
+    if (!save_results(cli_params, ctx_params, gen_params, results, num_results)) {
         return 1;
     }
 
+    for (int i = 0; i < num_results; i++) {
+        free(results[i].data);
+        results[i].data = nullptr;
+    }
+    free(results);
+
+    release_all_resources();
+
     return 0;
 }

examples/common/common.h

@@ -1,266 +0,0 @@
-#ifndef __EXAMPLES_COMMON_COMMON_H__
-#define __EXAMPLES_COMMON_COMMON_H__
-
-#include <cmath>
-#include <cstdint>
-#include <functional>
-#include <map>
-#include <string>
-#include <vector>
-
-#include "log.h"
-#include "resource_owners.hpp"
-#include "stable-diffusion.h"
-
-#define SAFE_STR(s) ((s) ? (s) : "")
-#define BOOL_STR(b) ((b) ? "true" : "false")
-
-extern const char* const modes_str[];
-#define SD_ALL_MODES_STR "img_gen, vid_gen, convert, upscale, metadata"
-
-enum SDMode {
-    IMG_GEN,
-    VID_GEN,
-    CONVERT,
-    UPSCALE,
-    METADATA,
-    MODE_COUNT
-};
-
-struct StringOption {
-    std::string short_name;
-    std::string long_name;
-    std::string desc;
-    std::string* target;
-};
-
-struct IntOption {
-    std::string short_name;
-    std::string long_name;
-    std::string desc;
-    int* target;
-};
-
-struct FloatOption {
-    std::string short_name;
-    std::string long_name;
-    std::string desc;
-    float* target;
-};
-
-struct BoolOption {
-    std::string short_name;
-    std::string long_name;
-    std::string desc;
-    bool keep_true;
-    bool* target;
-};
-
-struct ManualOption {
-    std::string short_name;
-    std::string long_name;
-    std::string desc;
-    std::function<int(int argc, const char** argv, int index)> cb;
-};
-
-struct ArgOptions {
-    std::vector<StringOption> string_options;
-    std::vector<IntOption> int_options;
-    std::vector<FloatOption> float_options;
-    std::vector<BoolOption> bool_options;
-    std::vector<ManualOption> manual_options;
-
-    static std::string wrap_text(const std::string& text, size_t width, size_t indent);
-    void print() const;
-};
-
-bool parse_options(int argc, const char** argv, const std::vector<ArgOptions>& options_list);
-bool decode_base64_image(const std::string& encoded_input,
-                         int target_channels,
-                         int expected_width,
-                         int expected_height,
-                         SDImageOwner& out_image);
-
-struct SDContextParams {
-    int n_threads = -1;
-    std::string model_path;
-    std::string clip_l_path;
-    std::string clip_g_path;
-    std::string clip_vision_path;
-    std::string t5xxl_path;
-    std::string llm_path;
-    std::string llm_vision_path;
-    std::string diffusion_model_path;
-    std::string high_noise_diffusion_model_path;
-    std::string vae_path;
-    std::string taesd_path;
-    std::string esrgan_path;
-    std::string control_net_path;
-    std::string embedding_dir;
-    std::string photo_maker_path;
-    sd_type_t wtype = SD_TYPE_COUNT;
-    std::string tensor_type_rules;
-    std::string lora_model_dir = ".";
-    std::string hires_upscalers_dir;
-
-    std::map<std::string, std::string> embedding_map;
-    std::vector<sd_embedding_t> embedding_vec;
-
-    rng_type_t rng_type         = CUDA_RNG;
-    rng_type_t sampler_rng_type = RNG_TYPE_COUNT;
-    bool offload_params_to_cpu  = false;
-    float max_vram              = 0.f;
-    std::string backend;
-    std::string params_backend;
-    bool enable_mmap           = false;
-    bool control_net_cpu       = false;
-    bool clip_on_cpu           = false;
-    bool vae_on_cpu            = false;
-    bool flash_attn            = false;
-    bool diffusion_flash_attn  = false;
-    bool diffusion_conv_direct = false;
-    bool vae_conv_direct       = false;
-
-    bool circular   = false;
-    bool circular_x = false;
-    bool circular_y = false;
-
-    bool chroma_use_dit_mask = true;
-    bool chroma_use_t5_mask  = false;
-    int chroma_t5_mask_pad   = 1;
-
-    bool qwen_image_zero_cond_t = false;
-
-    prediction_t prediction           = PREDICTION_COUNT;
-    lora_apply_mode_t lora_apply_mode = LORA_APPLY_AUTO;
-
-    bool force_sdxl_vae_conv_scale = false;
-
-    float flow_shift = INFINITY;
-    ArgOptions get_options();
-    void build_embedding_map();
-    bool resolve(SDMode mode);
-    bool validate(SDMode mode);
-    bool resolve_and_validate(SDMode mode);
-    std::string to_string() const;
-    sd_ctx_params_t to_sd_ctx_params_t(bool vae_decode_only, bool free_params_immediately, bool taesd_preview);
-};
-
-struct SDGenerationParams {
-    // User-facing input fields.
-    std::string prompt;
-    std::string negative_prompt;
-    int clip_skip              = -1;  // <= 0 represents unspecified
-    int width                  = -1;
-    int height                 = -1;
-    int batch_count            = 1;
-    int64_t seed               = 42;
-    float strength             = 0.75f;
-    float control_strength     = 0.9f;
-    bool auto_resize_ref_image = true;
-    bool increase_ref_index    = false;
-    bool embed_image_metadata  = true;
-
-    std::string init_image_path;
-    std::string end_image_path;
-    std::string mask_image_path;
-    std::string control_image_path;
-    std::vector<std::string> ref_image_paths;
-    std::string control_video_path;
-
-    sd_sample_params_t sample_params;
-    sd_sample_params_t high_noise_sample_params;
-    std::string extra_sample_args;
-    std::string high_noise_extra_sample_args;
-    std::vector<int> skip_layers            = {7, 8, 9};
-    std::vector<int> high_noise_skip_layers = {7, 8, 9};
-
-    std::vector<float> custom_sigmas;
-
-    std::string cache_mode;
-    std::string cache_option;
-    std::string scm_mask;
-    bool scm_policy_dynamic = true;
-    sd_cache_params_t cache_params{};
-
-    float moe_boundary                   = 0.875f;
-    int video_frames                     = 1;
-    int fps                              = 16;
-    float vace_strength                  = 1.f;
-    sd_tiling_params_t vae_tiling_params = {false, 0, 0, 0.5f, 0.0f, 0.0f};
-
-    std::string pm_id_images_dir;
-    std::string pm_id_embed_path;
-    float pm_style_strength = 20.f;
-
-    int upscale_repeats   = 1;
-    int upscale_tile_size = 128;
-
-    bool hires_enabled         = false;
-    std::string hires_upscaler = "Latent";
-    std::string hires_upscaler_model_path;
-    float hires_scale              = 2.f;
-    int hires_width                = 0;
-    int hires_height               = 0;
-    int hires_steps                = 0;
-    float hires_denoising_strength = 0.7f;
-    int hires_upscale_tile_size    = 128;
-
-    std::map<std::string, float> lora_map;
-    std::map<std::string, float> high_noise_lora_map;
-
-    // Derived and normalized fields.
-    std::string prompt_with_lora;  // for metadata record only
-    std::vector<sd_lora_t> lora_vec;
-    sd_hires_upscaler_t resolved_hires_upscaler;
-
-    // Owned execution payload.
-    SDImageOwner init_image;
-    SDImageOwner end_image;
-    std::vector<SDImageOwner> ref_images;
-    SDImageOwner mask_image;
-    SDImageOwner control_image;
-    std::vector<SDImageOwner> pm_id_images;
-    std::vector<SDImageOwner> control_frames;
-
-    // Backing storage for sd_img_gen_params_t view fields.
-    std::vector<sd_image_t> ref_image_views;
-    std::vector<sd_image_t> pm_id_image_views;
-    std::vector<sd_image_t> control_frame_views;
-
-    SDGenerationParams();
-    SDGenerationParams(const SDGenerationParams& other)                = default;
-    SDGenerationParams& operator=(const SDGenerationParams& other)     = default;
-    SDGenerationParams(SDGenerationParams&& other) noexcept            = default;
-    SDGenerationParams& operator=(SDGenerationParams&& other) noexcept = default;
-    ArgOptions get_options();
-    bool from_json_str(const std::string& json_str,
-                       const std::function<std::string(const std::string&)>& lora_path_resolver = {});
-    bool initialize_cache_params();
-    void extract_and_remove_lora(const std::string& lora_model_dir);
-    bool width_and_height_are_set() const;
-    void set_width_and_height_if_unset(int w, int h);
-    int get_resolved_width() const;
-    int get_resolved_height() const;
-    bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false);
-    bool validate(SDMode mode);
-    bool resolve_and_validate(SDMode mode,
-                              const std::string& lora_model_dir,
-                              const std::string& hires_upscalers_dir,
-                              bool strict = false);
-    sd_img_gen_params_t to_sd_img_gen_params_t();
-    sd_vid_gen_params_t to_sd_vid_gen_params_t();
-    std::string to_string() const;
-};
-
-std::string version_string();
-std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
-                                            const SDGenerationParams& gen_params,
-                                            int64_t seed,
-                                            SDMode mode = IMG_GEN);
-std::string get_image_params(const SDContextParams& ctx_params,
-                             const SDGenerationParams& gen_params,
-                             int64_t seed,
-                             SDMode mode = IMG_GEN);
-
-#endif  // __EXAMPLES_COMMON_COMMON_H__

examples/common/log.cpp

@@ -1,115 +0,0 @@
-#include "log.h"
-
-#include <vector>
-
-bool log_verbose = false;
-bool log_color   = false;
-
-std::string sd_basename(const std::string& path) {
-    size_t pos = path.find_last_of('/');
-    if (pos != std::string::npos) {
-        return path.substr(pos + 1);
-    }
-    pos = path.find_last_of('\\');
-    if (pos != std::string::npos) {
-        return path.substr(pos + 1);
-    }
-    return path;
-}
-
-void print_utf8(FILE* stream, const char* utf8) {
-    if (!utf8) {
-        return;
-    }
-
-#ifdef _WIN32
-    HANDLE h = (stream == stderr)
-                   ? GetStdHandle(STD_ERROR_HANDLE)
-                   : GetStdHandle(STD_OUTPUT_HANDLE);
-
-    DWORD mode;
-    BOOL is_console = GetConsoleMode(h, &mode);
-
-    if (is_console) {
-        int wlen = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0);
-        if (wlen <= 0) {
-            return;
-        }
-
-        std::vector<wchar_t> wbuf(static_cast<size_t>(wlen));
-
-        MultiByteToWideChar(CP_UTF8, 0, utf8, -1, wbuf.data(), wlen);
-
-        DWORD written;
-        WriteConsoleW(h, wbuf.data(), wlen - 1, &written, NULL);
-    } else {
-        DWORD written;
-        WriteFile(h, utf8, (DWORD)strlen(utf8), &written, NULL);
-    }
-#else
-    fputs(utf8, stream);
-#endif
-}
-
-void log_print(enum sd_log_level_t level, const char* log, bool verbose, bool color) {
-    int tag_color;
-    const char* level_str;
-    FILE* out_stream = (level == SD_LOG_ERROR) ? stderr : stdout;
-
-    if (!log || (!verbose && level <= SD_LOG_DEBUG)) {
-        return;
-    }
-
-    switch (level) {
-        case SD_LOG_DEBUG:
-            tag_color = 37;
-            level_str = "DEBUG";
-            break;
-        case SD_LOG_INFO:
-            tag_color = 34;
-            level_str = "INFO";
-            break;
-        case SD_LOG_WARN:
-            tag_color = 35;
-            level_str = "WARN";
-            break;
-        case SD_LOG_ERROR:
-            tag_color = 31;
-            level_str = "ERROR";
-            break;
-        default:
-            tag_color = 33;
-            level_str = "?????";
-            break;
-    }
-
-    if (color) {
-        fprintf(out_stream, "\033[%d;1m[%-5s]\033[0m ", tag_color, level_str);
-    } else {
-        fprintf(out_stream, "[%-5s] ", level_str);
-    }
-    print_utf8(out_stream, log);
-    fflush(out_stream);
-}
-
-void example_log_printf(sd_log_level_t level, const char* file, int line, const char* format, ...) {
-    constexpr size_t LOG_BUFFER_SIZE = 4096;
-
-    va_list args;
-    va_start(args, format);
-
-    static char log_buffer[LOG_BUFFER_SIZE + 1];
-    int written = snprintf(log_buffer, LOG_BUFFER_SIZE, "%s:%-4d - ", sd_basename(file).c_str(), line);
-
-    if (written >= 0 && written < static_cast<int>(LOG_BUFFER_SIZE)) {
-        vsnprintf(log_buffer + written, LOG_BUFFER_SIZE - written, format, args);
-    }
-    size_t len = strlen(log_buffer);
-    if (len == 0 || log_buffer[len - 1] != '\n') {
-        strncat(log_buffer, "\n", LOG_BUFFER_SIZE - len);
-    }
-
-    log_print(level, log_buffer, log_verbose, log_color);
-
-    va_end(args);
-}

examples/common/log.h

@@ -1,32 +0,0 @@
-#ifndef __EXAMPLE_LOG_H__
-#define __EXAMPLE_LOG_H__
-
-#include <cstdarg>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <string>
-
-#if defined(_WIN32)
-#ifndef NOMINMAX
-#define NOMINMAX
-#endif
-#include <windows.h>
-#endif  // _WIN32
-
-#include "stable-diffusion.h"
-
-extern bool log_verbose;
-extern bool log_color;
-
-std::string sd_basename(const std::string& path);
-void print_utf8(FILE* stream, const char* utf8);
-void log_print(sd_log_level_t level, const char* log, bool verbose, bool color);
-void example_log_printf(sd_log_level_t level, const char* file, int line, const char* format, ...);
-
-#define LOG_DEBUG(format, ...) example_log_printf(SD_LOG_DEBUG, __FILE__, __LINE__, format, ##__VA_ARGS__)
-#define LOG_INFO(format, ...) example_log_printf(SD_LOG_INFO, __FILE__, __LINE__, format, ##__VA_ARGS__)
-#define LOG_WARN(format, ...) example_log_printf(SD_LOG_WARN, __FILE__, __LINE__, format, ##__VA_ARGS__)
-#define LOG_ERROR(format, ...) example_log_printf(SD_LOG_ERROR, __FILE__, __LINE__, format, ##__VA_ARGS__)
-
-#endif  // __EXAMPLE_LOG_H__

examples/common/media_io.h

@@ -1,101 +0,0 @@
-#ifndef __MEDIA_IO_H__
-#define __MEDIA_IO_H__
-
-#include <cstdint>
-#include <string>
-#include <vector>
-
-#include "stable-diffusion.h"
-
-enum class EncodedImageFormat {
-    JPEG,
-    PNG,
-    WEBP,
-    UNKNOWN,
-};
-
-EncodedImageFormat encoded_image_format_from_path(const std::string& path);
-
-std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
-                                            const uint8_t* image,
-                                            int width,
-                                            int height,
-                                            int channels,
-                                            const std::string& parameters = "",
-                                            int quality                   = 90);
-
-bool write_image_to_file(const std::string& path,
-                         const uint8_t* image,
-                         int width,
-                         int height,
-                         int channels,
-                         const std::string& parameters = "",
-                         int quality                   = 90);
-
-uint8_t* load_image_from_file(const char* image_path,
-                              int& width,
-                              int& height,
-                              int expected_width   = 0,
-                              int expected_height  = 0,
-                              int expected_channel = 3);
-
-bool load_sd_image_from_file(sd_image_t* image,
-                             const char* image_path,
-                             int expected_width   = 0,
-                             int expected_height  = 0,
-                             int expected_channel = 3);
-
-uint8_t* load_image_from_memory(const char* image_bytes,
-                                int len,
-                                int& width,
-                                int& height,
-                                int expected_width   = 0,
-                                int expected_height  = 0,
-                                int expected_channel = 3);
-
-int create_mjpg_avi_from_sd_images(const char* filename,
-                                   sd_image_t* images,
-                                   int num_images,
-                                   int fps,
-                                   int quality = 90);
-std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images,
-                                                              int num_images,
-                                                              int fps,
-                                                              int quality = 90);
-
-#ifdef SD_USE_WEBP
-int create_animated_webp_from_sd_images(const char* filename,
-                                        sd_image_t* images,
-                                        int num_images,
-                                        int fps,
-                                        int quality = 90);
-std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(sd_image_t* images,
-                                                                   int num_images,
-                                                                   int fps,
-                                                                   int quality = 90);
-#endif
-
-#ifdef SD_USE_WEBM
-int create_webm_from_sd_images(const char* filename,
-                               sd_image_t* images,
-                               int num_images,
-                               int fps,
-                               int quality = 90);
-std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images,
-                                                          int num_images,
-                                                          int fps,
-                                                          int quality = 90);
-#endif
-
-int create_video_from_sd_images(const char* filename,
-                                sd_image_t* images,
-                                int num_images,
-                                int fps,
-                                int quality = 90);
-std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& output_format,
-                                                           sd_image_t* images,
-                                                           int num_images,
-                                                           int fps,
-                                                           int quality = 90);
-
-#endif  // __MEDIA_IO_H__

examples/common/resource_owners.hpp

@@ -1,236 +0,0 @@
-#ifndef __EXAMPLE_RESOURCE_OWNERS_H__
-#define __EXAMPLE_RESOURCE_OWNERS_H__
-
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <memory>
-#include <utility>
-#include <vector>
-
-#include "stable-diffusion.h"
-
-struct FreeDeleter {
-    void operator()(void* ptr) const {
-        free(ptr);
-    }
-};
-
-struct FileCloser {
-    void operator()(FILE* file) const {
-        if (file != nullptr) {
-            fclose(file);
-        }
-    }
-};
-
-struct SDCtxDeleter {
-    void operator()(sd_ctx_t* ctx) const {
-        if (ctx != nullptr) {
-            free_sd_ctx(ctx);
-        }
-    }
-};
-
-struct UpscalerCtxDeleter {
-    void operator()(upscaler_ctx_t* ctx) const {
-        if (ctx != nullptr) {
-            free_upscaler_ctx(ctx);
-        }
-    }
-};
-
-template <typename T>
-using FreeUniquePtr = std::unique_ptr<T, FreeDeleter>;
-
-using FilePtr        = std::unique_ptr<FILE, FileCloser>;
-using SDCtxPtr       = std::unique_ptr<sd_ctx_t, SDCtxDeleter>;
-using UpscalerCtxPtr = std::unique_ptr<upscaler_ctx_t, UpscalerCtxDeleter>;
-
-class SDImageOwner {
-private:
-    static sd_image_t copy_image(const sd_image_t& image) {
-        if (image.data == nullptr) {
-            return {image.width, image.height, image.channel, nullptr};
-        }
-
-        const size_t byte_count = static_cast<size_t>(image.width) * image.height * image.channel;
-        uint8_t* raw_copy       = static_cast<uint8_t*>(malloc(byte_count));
-        if (raw_copy == nullptr) {
-            return {0, 0, 0, nullptr};
-        }
-
-        std::memcpy(raw_copy, image.data, byte_count);
-        return {image.width, image.height, image.channel, raw_copy};
-    }
-
-    sd_image_t image_ = {0, 0, 0, nullptr};
-
-public:
-    SDImageOwner() = default;
-    explicit SDImageOwner(sd_image_t image)
-        : image_(image) {
-    }
-
-    SDImageOwner(const SDImageOwner& other)
-        : image_(copy_image(other.image_)) {
-    }
-
-    SDImageOwner& operator=(const SDImageOwner& other) {
-        if (this != &other) {
-            reset(copy_image(other.image_));
-        }
-        return *this;
-    }
-
-    SDImageOwner(SDImageOwner&& other) noexcept
-        : image_(other.release()) {
-    }
-
-    SDImageOwner& operator=(SDImageOwner&& other) noexcept {
-        if (this != &other) {
-            reset();
-            image_ = other.release();
-        }
-        return *this;
-    }
-
-    ~SDImageOwner() {
-        reset();
-    }
-
-    sd_image_t* put() {
-        if (image_.data != nullptr) {
-            free(image_.data);
-            image_.data = nullptr;
-        }
-        image_.width   = 0;
-        image_.height  = 0;
-        image_.channel = 0;
-        return &image_;
-    }
-
-    sd_image_t& get() {
-        return image_;
-    }
-
-    const sd_image_t& get() const {
-        return image_;
-    }
-
-    sd_image_t release() {
-        sd_image_t image = image_;
-        image_           = {0, 0, 0, nullptr};
-        return image;
-    }
-
-    void reset(sd_image_t image = {0, 0, 0, nullptr}) {
-        if (image_.data != nullptr) {
-            free(image_.data);
-        }
-        image_ = image;
-    }
-};
-
-class SDImageVec {
-private:
-    std::vector<sd_image_t> images_;
-
-public:
-    SDImageVec() = default;
-
-    SDImageVec(const SDImageVec&)            = delete;
-    SDImageVec& operator=(const SDImageVec&) = delete;
-
-    SDImageVec(SDImageVec&& other) noexcept
-        : images_(std::move(other.images_)) {
-    }
-
-    SDImageVec& operator=(SDImageVec&& other) noexcept {
-        if (this != &other) {
-            clear();
-            images_ = std::move(other.images_);
-        }
-        return *this;
-    }
-
-    ~SDImageVec() {
-        clear();
-    }
-
-    void push_back(sd_image_t image) {
-        images_.push_back(image);
-    }
-
-    void push_back(SDImageOwner&& image) {
-        images_.push_back(image.release());
-    }
-
-    void reserve(size_t count) {
-        images_.reserve(count);
-    }
-
-    void adopt(sd_image_t* images, int count) {
-        clear();
-        if (images == nullptr || count <= 0) {
-            free(images);
-            return;
-        }
-
-        images_.reserve(static_cast<size_t>(count));
-        for (int i = 0; i < count; ++i) {
-            images_.push_back(images[i]);
-        }
-        free(images);
-    }
-
-    size_t size() const {
-        return images_.size();
-    }
-
-    bool empty() const {
-        return images_.empty();
-    }
-
-    int count() const {
-        return static_cast<int>(images_.size());
-    }
-
-    explicit operator bool() const {
-        return !images_.empty();
-    }
-
-    sd_image_t* data() {
-        return images_.data();
-    }
-
-    const sd_image_t* data() const {
-        return images_.data();
-    }
-
-    sd_image_t& operator[](size_t index) {
-        return images_[index];
-    }
-
-    const sd_image_t& operator[](size_t index) const {
-        return images_[index];
-    }
-
-    std::vector<sd_image_t>& raw() {
-        return images_;
-    }
-
-    const std::vector<sd_image_t>& raw() const {
-        return images_;
-    }
-
-    void clear() {
-        for (sd_image_t& image : images_) {
-            free(image.data);
-            image.data = nullptr;
-        }
-        images_.clear();
-    }
-};
-
-#endif  // __EXAMPLE_RESOURCE_OWNERS_H__

examples/server/CMakeLists.txt

@@ -1,104 +1,6 @@
 set(TARGET sd-server)
 
-option(SD_SERVER_BUILD_FRONTEND "Build server frontend with pnpm" ON)
-
-set(FRONTEND_DIR "${CMAKE_CURRENT_SOURCE_DIR}/frontend")
-set(GENERATED_HTML_HEADER "${FRONTEND_DIR}/dist/gen_index_html.h")
-
-set(HAVE_FRONTEND_BUILD OFF)
-
-if(SD_SERVER_BUILD_FRONTEND AND EXISTS "${FRONTEND_DIR}")
-    if(WIN32)
-        find_program(PNPM_EXECUTABLE NAMES pnpm.cmd pnpm)
-    else()
-        find_program(PNPM_EXECUTABLE NAMES pnpm)
-    endif()
-
-    if(PNPM_EXECUTABLE)
-        message(STATUS "Frontend dir found: ${FRONTEND_DIR}")
-        message(STATUS "pnpm found: ${PNPM_EXECUTABLE}")
-
-        set(HAVE_FRONTEND_BUILD ON)
-
-        add_custom_target(${TARGET}_frontend_install
-            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" install
-            WORKING_DIRECTORY "${FRONTEND_DIR}"
-            COMMENT "Installing frontend dependencies"
-            VERBATIM
-        )
-
-        add_custom_target(${TARGET}_frontend_build
-            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" run build
-            WORKING_DIRECTORY "${FRONTEND_DIR}"
-            COMMENT "Building frontend"
-            VERBATIM
-        )
-
-        add_custom_target(${TARGET}_frontend_header
-            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" run build:header
-            WORKING_DIRECTORY "${FRONTEND_DIR}"
-            COMMENT "Generating gen_index_html.h"
-            VERBATIM
-        )
-
-        add_dependencies(${TARGET}_frontend_build ${TARGET}_frontend_install)
-        add_dependencies(${TARGET}_frontend_header ${TARGET}_frontend_build)
-
-        add_custom_target(${TARGET}_frontend
-            DEPENDS ${TARGET}_frontend_header
-        )
-
-        set_source_files_properties("${GENERATED_HTML_HEADER}" PROPERTIES GENERATED TRUE)
-    else()
-        if(EXISTS "${GENERATED_HTML_HEADER}")
-            message(STATUS "pnpm not found; using pre-built frontend header detected at ${GENERATED_HTML_HEADER}")
-            set(HAVE_FRONTEND_BUILD ON)
-            add_custom_target(${TARGET}_frontend)
-        else()
-            message(WARNING "pnpm not found; frontend build disabled.")
-        endif()
-    endif()
-else()
-    message(STATUS "Frontend disabled or directory not found: ${FRONTEND_DIR}")
-endif()
-
-add_executable(${TARGET}
-    ../common/common.cpp
-    ../common/log.cpp
-    ../common/media_io.cpp
-    main.cpp
-    runtime.cpp
-    async_jobs.cpp
-    routes_index.cpp
-    routes_openai.cpp
-    routes_sdapi.cpp
-    routes_sdcpp.cpp
-)
-
-if(HAVE_FRONTEND_BUILD)
-    add_dependencies(${TARGET} ${TARGET}_frontend)
-    target_sources(${TARGET} PRIVATE "${GENERATED_HTML_HEADER}")
-    target_include_directories(${TARGET} PRIVATE "${FRONTEND_DIR}/dist")
-    target_compile_definitions(${TARGET} PRIVATE HAVE_INDEX_HTML)
-    message(STATUS "HAVE_INDEX_HTML enabled")
-else()
-    message(STATUS "HAVE_INDEX_HTML disabled")
-endif()
-
+add_executable(${TARGET} main.cpp)
 install(TARGETS ${TARGET} RUNTIME)
 target_link_libraries(${TARGET} PRIVATE stable-diffusion ${CMAKE_THREAD_LIBS_INIT})
-if(SD_WEBP)
-    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBP)
-    target_link_libraries(${TARGET} PRIVATE webp libwebpmux)
-endif()
-if(SD_WEBM)
-    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBM)
-    target_link_libraries(${TARGET} PRIVATE webm)
-endif()
-
-# due to httplib; it contains a pragma for MSVC, but other things need explicit flags
-if(WIN32 AND NOT MSVC)
-    target_link_libraries(${TARGET} PRIVATE ws2_32)
-endif()
-
-target_compile_features(${TARGET} PUBLIC c_std_11 cxx_std_17)
+target_compile_features(${TARGET} PUBLIC c_std_11 cxx_std_17)

examples/server/README.md

@@ -1,134 +1,15 @@
-# Example
-
-The following example starts `sd-server` with a standalone diffusion model, VAE, and LLM text encoder:
-
-```
-.\bin\Release\sd-server.exe --diffusion-model  ..\models\diffusion_models\z_image_turbo_bf16.safetensors --vae ..\models\vae\ae.sft  --llm ..\models\text_encoders\qwen_3_4b.safetensors --diffusion-fa --offload-to-cpu -v --cfg-scale 1.0
-```
-
-What this example does:
-
-* `--diffusion-model` selects the standalone diffusion model
-* `--vae` selects the VAE decoder
-* `--llm` selects the text encoder / language model used by this pipeline
-* `--diffusion-fa` enables flash attention in the diffusion model
-* `--offload-to-cpu` reduces VRAM pressure by keeping weights in RAM when possible
-* `-v` enables verbose logging
-* `--cfg-scale 1.0` sets the default CFG scale for generation
-
-After the server starts successfully:
-
-* the web UI is available at `http://127.0.0.1:1234/`
-* the native async API is available under `/sdcpp/v1/...`
-* the compatibility APIs are available under `/v1/...` and `/sdapi/v1/...`
-
-If you want to use a different host or port, pass:
-
-```bash
---listen-ip <ip> --listen-port <port>
-```
-
-# Frontend
-
-## Build with Frontend
-
-The server can optionally build the web frontend and embed it into the binary as `gen_index_html.h`.
-
-### Requirements
-
-Install the following tools:
-
-* **Node.js** ≥ 20
-  https://nodejs.org/
-
-* **pnpm** ≥ 10
-  Install via npm:
-
-```bash
-npm install -g pnpm
-```
-
-Verify installation:
-
-```bash
-node -v
-pnpm -v
-```
-
-### Install frontend dependencies
-
-Go to the frontend directory and install dependencies:
-
-```bash
-cd examples/server/frontend
-pnpm install
-```
-
-### Build the server with CMake
-
-Enable the frontend build option when configuring CMake:
-
-```bash
-cmake -B build -DSD_SERVER_BUILD_FRONTEND=ON
-cmake --build build --config Release
-```
-
-If `pnpm` is available, the build system will automatically run:
-
-```
-pnpm run build
-pnpm run build:header
-```
-
-and embed the generated frontend into the server binary.
-
-## Frontend Repository
-
-The web frontend is maintained in a **separate repository**, https://github.com/leejet/sdcpp-webui.
-
-If you want to modify the UI or frontend logic, please submit pull requests to the **frontend repository**.
-
-This repository (`stable-diffusion.cpp`) only vendors the frontend periodically. Changes from the frontend repo are synchronized:
-
-* approximately **every 1–2 weeks**, or
-* when there are **major frontend updates**
-
-Because of this, frontend changes will **not appear here immediately** after being merged upstream.
-
-## Using an external frontend
-
-By default, the server uses the **embedded frontend** generated during the build (`gen_index_html.h`).
-
-You can also serve a custom frontend file instead of the embedded one by using:
-
-```bash
---serve-html-path <path-to-index.html>
-```
-
-For example:
-
-```bash
-sd-server --serve-html-path ./index.html
-```
-
-In this case, the server will load and serve the specified `index.html` file instead of the embedded frontend. This is useful when:
-
-* developing or testing frontend changes
-* using a custom UI
-* avoiding rebuilding the binary after frontend modifications
-
 # Run
 
 ```
 usage: ./bin/sd-server  [options]
 
 Svr Options:
-  -l, --listen-ip <string>      server listen ip (default: 127.0.0.1)
-  --serve-html-path <string>    path to HTML file to serve at root (optional)
-  --listen-port <int>           server listen port (default: 1234)
-  -v, --verbose                 print extra info
-  --color                       colors the logging tags according to level
-  -h, --help                    show this help message and exit
+  -l, --listen-ip <string>    server listen ip (default: 127.0.0.1)
+  --listen-port <int>         server listen port (default: 1234)
+  --serve-html-path <string>  path to HTML file to serve at root (optional)
+  -v, --verbose               print extra info
+  --color                     colors the logging tags according to level
+  -h, --help                  show this help message and exit
 
 Context Options:
   -m, --model <string>                     path to full model
@@ -136,8 +17,7 @@ Context Options:
   --clip_g <string>                        path to the clip-g text encoder
   --clip_vision <string>                   path to the clip-vision encoder
   --t5xxl <string>                         path to the t5xxl text encoder
-  --llm <string>                           path to the llm text encoder. For example: (qwenvl2.5 for qwen-image,
-                                           mistral-small3.2 for flux2, ...)
+  --llm <string>                           path to the llm text encoder. For example: (qwenvl2.5 for qwen-image, mistral-small3.2 for flux2, ...)
   --llm_vision <string>                    path to the llm vit
   --qwen2vl <string>                       alias of --llm. Deprecated.
   --qwen2vl_vision <string>                alias of --llm_vision. Deprecated.
@@ -149,45 +29,43 @@ Context Options:
   --control-net <string>                   path to control net model
   --embd-dir <string>                      embeddings directory
   --lora-model-dir <string>                lora model directory
-  --hires-upscalers-dir <string>           highres fix upscaler model directory
   --tensor-type-rules <string>             weight type per tensor pattern (example: "^vae\.=f16,model\.=q8_0")
   --photo-maker <string>                   path to PHOTOMAKER model
   --upscale-model <string>                 path to esrgan model.
-  -t, --threads <int>                      number of threads to use during computation (default: -1). If threads <= 0,
-                                           then threads will be set to the number of CPU physical cores
+  -t, --threads <int>                      number of threads to use during computation (default: -1). If threads <= 0, then threads will be set to the number of
+                                           CPU physical cores
   --chroma-t5-mask-pad <int>               t5 mask pad size of chroma
-  --max-vram <float>                       maximum VRAM budget in GiB for graph-cut segmented execution. 0 disables
-                                           graph splitting; -1 auto-detects free VRAM minus 1 GiB
+  --vae-tile-overlap <float>               tile overlap for vae tiling, in fraction of tile size (default: 0.5)
+  --flow-shift <float>                     shift value for Flow models like SD3.x or WAN (default: auto)
+  --vae-tiling                             process vae in tiles to reduce memory usage
   --force-sdxl-vae-conv-scale              force use of conv scale on sdxl vae
-  --offload-to-cpu                         place the weights in RAM to save VRAM, and automatically load them into VRAM
-                                           when needed
-  --mmap                                   whether to memory-map model
+  --offload-to-cpu                         place the weights in RAM to save VRAM, and automatically load them into VRAM when needed
   --control-net-cpu                        keep controlnet in cpu (for low vram)
   --clip-on-cpu                            keep clip in cpu (for low vram)
   --vae-on-cpu                             keep vae in cpu (for low vram)
-  --fa                                     use flash attention
-  --diffusion-fa                           use flash attention in the diffusion model only
+  --mmap                                   whether to memory-map model
+  --disable-fa                             disable flash attention
   --diffusion-conv-direct                  use ggml_conv2d_direct in the diffusion model
   --vae-conv-direct                        use ggml_conv2d_direct in the vae model
   --circular                               enable circular padding for convolutions
   --circularx                              enable circular RoPE wrapping on x-axis (width) only
   --circulary                              enable circular RoPE wrapping on y-axis (height) only
   --chroma-disable-dit-mask                disable dit mask for chroma
-  --qwen-image-zero-cond-t                 enable zero_cond_t for qwen image
   --chroma-enable-t5-mask                  enable t5 mask for chroma
-  --type                                   weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K,
-                                           q4_K). If not specified, the default is the type of the weight file
+  --type                                   weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K, q4_K). If not specified, the default is the
+                                           type of the weight file
   --rng                                    RNG, one of [std_default, cuda, cpu], default: cuda(sd-webui), cpu(comfyui)
   --sampler-rng                            sampler RNG, one of [std_default, cuda, cpu]. If not specified, use --rng
-  --prediction                             prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow,
-                                           flux2_flow]
-  --lora-apply-mode                        the way to apply LoRA, one of [auto, immediately, at_runtime], default is
-                                           auto. In auto mode, if the model weights contain any quantized parameters,
-                                           the at_runtime mode will be used; otherwise, immediately will be used.The
-                                           immediately mode may have precision and compatibility issues with quantized
-                                           parameters, but it usually offers faster inference speed and, in some cases,
-                                           lower memory usage. The at_runtime mode, on the other hand, is exactly the
-                                           opposite.
+  --prediction                             prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow, flux2_flow]
+  --lora-apply-mode                        the way to apply LoRA, one of [auto, immediately, at_runtime], default is auto. In auto mode, if the model weights
+                                           contain any quantized parameters, the at_runtime mode will be used; otherwise,
+                                           immediately will be used.The immediately mode may have precision and
+                                           compatibility issues with quantized parameters, but it usually offers faster inference
+                                           speed and, in some cases, lower memory usage. The at_runtime mode, on the
+                                           other hand, is exactly the opposite.
+  --vae-tile-size                          tile size for vae tiling, format [X]x[Y] (default: 32x32)
+  --vae-relative-tile-size                 relative tile size for vae tiling, format [X]x[Y], in fraction of image size if < 1, in number of tiles per dim if >=1
+                                           (overrides --vae-tile-size)
 
 Default Generation Options:
   -p, --prompt <string>                    the prompt to render
@@ -196,99 +74,62 @@ Default Generation Options:
   --end-img <string>                       path to the end image, required by flf2v
   --mask <string>                          path to the mask image
   --control-image <string>                 path to control image, control net
-  --control-video <string>                 path to control video frames, It must be a directory path. The video frames
-                                           inside should be stored as images in lexicographical (character) order. For
-                                           example, if the control video path is `frames`, the directory contain images
-                                           such as 00.png, 01.png, ... etc.
+  --control-video <string>                 path to control video frames, It must be a directory path. The video frames inside should be stored as images in
+                                           lexicographical (character) order. For example, if the control video path is
+                                           `frames`, the directory contain images such as 00.png, 01.png, ... etc.
   --pm-id-images-dir <string>              path to PHOTOMAKER input id images dir
   --pm-id-embed-path <string>              path to PHOTOMAKER v2 id embed
-  --hires-upscaler <string>                highres fix upscaler, Lanczos, Nearest, Latent, Latent (nearest), Latent
-                                           (nearest-exact), Latent (antialiased), Latent (bicubic), Latent (bicubic
-                                           antialiased), or a model name under --hires-upscalers-dir (default: Latent)
-  --extra-sample-args <string>             extra sampler args, key=value list. Currently lcm supports noise_clip_std,
-                                           noise_scale_start, noise_scale_end
   -H, --height <int>                       image height, in pixel space (default: 512)
   -W, --width <int>                        image width, in pixel space (default: 512)
   --steps <int>                            number of sample steps (default: 20)
   --high-noise-steps <int>                 (high noise) number of sample steps (default: -1 = auto)
-  --clip-skip <int>                        ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer
-                                           (default: -1). <= 0 represents unspecified, will be 1 for SD1.x, 2 for SD2.x
+  --clip-skip <int>                        ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1). <= 0 represents unspecified,
+                                           will be 1 for SD1.x, 2 for SD2.x
   -b, --batch-count <int>                  batch count
   --video-frames <int>                     video frames (default: 1)
   --fps <int>                              fps (default: 24)
-  --timestep-shift <int>                   shift timestep for NitroFusion models (default: 0). recommended N for
-                                           NitroSD-Realism around 250 and 500 for NitroSD-Vibrant
+  --timestep-shift <int>                   shift timestep for NitroFusion models (default: 0). recommended N for NitroSD-Realism around 250 and 500 for
+                                           NitroSD-Vibrant
   --upscale-repeats <int>                  Run the ESRGAN upscaler this many times (default: 1)
   --upscale-tile-size <int>                tile size for ESRGAN upscaling (default: 128)
-  --hires-width <int>                      highres fix target width, 0 to use --hires-scale (default: 0)
-  --hires-height <int>                     highres fix target height, 0 to use --hires-scale (default: 0)
-  --hires-steps <int>                      highres fix second pass sample steps, 0 to reuse --steps (default: 0)
-  --hires-upscale-tile-size <int>          highres fix upscaler tile size, reserved for model-backed upscalers (default:
-                                           128)
   --cfg-scale <float>                      unconditional guidance scale: (default: 7.0)
-  --img-cfg-scale <float>                  image guidance scale for inpaint or instruct-pix2pix models: (default: same
-                                           as --cfg-scale)
+  --img-cfg-scale <float>                  image guidance scale for inpaint or instruct-pix2pix models: (default: same as --cfg-scale)
   --guidance <float>                       distilled guidance scale for models with guidance input (default: 3.5)
-  --slg-scale <float>                      skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means
-                                           disabled, a value of 2.5 is nice for sd3.5 medium
+  --slg-scale <float>                      skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means disabled, a value of 2.5 is nice for sd3.5
+                                           medium
   --skip-layer-start <float>               SLG enabling point (default: 0.01)
   --skip-layer-end <float>                 SLG disabling point (default: 0.2)
-  --eta <float>                            noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and
-                                           res_2s; 1 for euler_a, er_sde and dpm++2s_a)
-  --flow-shift <float>                     shift value for Flow models like SD3.x or WAN (default: auto)
+  --eta <float>                            eta in DDIM, only for DDIM and TCD (default: 0)
   --high-noise-cfg-scale <float>           (high noise) unconditional guidance scale: (default: 7.0)
-  --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or instruct-pix2pix models
-                                           (default: same as --cfg-scale)
-  --high-noise-guidance <float>            (high noise) distilled guidance scale for models with guidance input
-                                           (default: 3.5)
-  --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default:
-                                           0)
+  --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or instruct-pix2pix models (default: same as --cfg-scale)
+  --high-noise-guidance <float>            (high noise) distilled guidance scale for models with guidance input (default: 3.5)
+  --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default: 0)
   --high-noise-skip-layer-start <float>    (high noise) SLG enabling point (default: 0.01)
   --high-noise-skip-layer-end <float>      (high noise) SLG disabling point (default: 0.2)
-  --high-noise-eta <float>                 (high noise) noise multiplier (default: 0 for ddim_trailing, tcd,
-                                           res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)
+  --high-noise-eta <float>                 (high noise) eta in DDIM, only for DDIM and TCD (default: 0)
   --strength <float>                       strength for noising/unnoising (default: 0.75)
-  --pm-style-strength <float>              
-  --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full
-                                           destruction of information in init image
-  --moe-boundary <float>                   timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if
-                                           `--high-noise-steps` is set to -1
+  --pm-style-strength <float>
+  --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image
+  --moe-boundary <float>                   timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if `--high-noise-steps` is set to -1
   --vace-strength <float>                  wan vace strength
-  --vae-tile-overlap <float>               tile overlap for vae tiling, in fraction of tile size (default: 0.5)
-  --hires-scale <float>                    highres fix scale when target size is not set (default: 2.0)
-  --hires-denoising-strength <float>       highres fix second pass denoising strength (default: 0.7)
-  --increase-ref-index                     automatically increase the indices of references images based on the order
-                                           they are listed (starting with 1).
+  --increase-ref-index                     automatically increase the indices of references images based on the order they are listed (starting with 1).
   --disable-auto-resize-ref-image          disable auto resize of ref images
-  --disable-image-metadata                 do not embed generation metadata on image files
-  --vae-tiling                             process vae in tiles to reduce memory usage
-  --hires                                  enable highres fix
   -s, --seed                               RNG seed (default: 42, use random seed for < 0)
-  --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m,
-                                           dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s,
-                                           er_sde, euler_cfg_pp, euler_a_cfg_pp] (default: euler for Flux/SD3/Wan, euler_a otherwise)
-  --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a,
-                                           dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep,
-                                           res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for Flux/SD3/Wan, euler_a otherwise
-  --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits,
-                                           smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent], default:
-                                           discrete
-  --sigmas                                 custom sigma values for the sampler, comma-separated (e.g.,
-                                           "14.61,7.8,3.5,0.0").
+  --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing,
+                                           tcd] (default: euler for Flux/SD3/Wan, euler_a otherwise)
+  --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm,
+                                           ddim_trailing, tcd] default: euler for Flux/SD3/Wan, euler_a otherwise
+  --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, smoothstep, sgm_uniform, simple,
+                                           kl_optimal, lcm], default: discrete
+  --sigmas                                 custom sigma values for the sampler, comma-separated (e.g., "14.61,7.8,3.5,0.0").
   --skip-layers                            layers to skip for SLG steps (default: [7,8,9])
   --high-noise-skip-layers                 (high noise) layers to skip for SLG steps (default: [7,8,9])
   -r, --ref-image                          reference image for Flux Kontext models (can be used multiple times)
-  --cache-mode                             caching method: 'easycache' (DiT), 'ucache' (UNET),
-                                           'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT
-                                           Chebyshev+Taylor forecasting)
+  --cache-mode                             caching method: 'easycache' (DiT), 'ucache' (UNET), 'dbcache'/'taylorseer'/'cache-dit' (DiT block-level)
   --cache-option                           named cache params (key=value format, comma-separated). easycache/ucache:
-                                           threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit:
-                                           Fn=,Bn=,threshold=,warmup=; spectrum: w=,m=,lam=,window=,flex=,warmup=,stop=.
-                                           Examples: "threshold=0.25" or "threshold=1.5,reset=0"
-  --scm-mask                               SCM steps mask for cache-dit: comma-separated 0/1 (e.g.,
-                                           "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache
+                                           threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit: Fn=,Bn=,threshold=,warmup=. Examples:
+                                           "threshold=0.25" or "threshold=1.5,reset=0"
+  --cache-preset                           cache-dit preset: 'slow'/'s', 'medium'/'m', 'fast'/'f', 'ultra'/'u'
+  --scm-mask                               SCM steps mask for cache-dit: comma-separated 0/1 (e.g., "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache
   --scm-policy                             SCM policy: 'dynamic' (default) or 'static'
-  --vae-tile-size                          tile size for vae tiling, format [X]x[Y] (default: 32x32)
-  --vae-relative-tile-size                 relative tile size for vae tiling, format [X]x[Y], in fraction of image size
-                                           if < 1, in number of tiles per dim if >=1 (overrides --vae-tile-size)
 ```

examples/server/async_jobs.cpp

@@ -1,349 +0,0 @@
-// Extracted from main.cpp during server refactor.
-
-#include "async_jobs.h"
-
-#include <iomanip>
-#include <sstream>
-
-#include "common/log.h"
-#include "common/media_io.h"
-#include "common/resource_owners.hpp"
-
-const char* async_job_kind_name(AsyncJobKind kind) {
-    switch (kind) {
-        case AsyncJobKind::ImgGen:
-            return "img_gen";
-        case AsyncJobKind::VidGen:
-            return "vid_gen";
-        default:
-            return "img_gen";
-    }
-}
-
-const char* async_job_status_name(AsyncJobStatus status) {
-    switch (status) {
-        case AsyncJobStatus::Queued:
-            return "queued";
-        case AsyncJobStatus::Generating:
-            return "generating";
-        case AsyncJobStatus::Completed:
-            return "completed";
-        case AsyncJobStatus::Failed:
-            return "failed";
-        case AsyncJobStatus::Cancelled:
-            return "cancelled";
-        default:
-            return "failed";
-    }
-}
-
-void purge_expired_jobs(AsyncJobManager& manager) {
-    const int64_t now = unix_timestamp_now();
-
-    for (auto it = manager.expired_jobs.begin(); it != manager.expired_jobs.end();) {
-        if (it->second <= now) {
-            it = manager.expired_jobs.erase(it);
-        } else {
-            ++it;
-        }
-    }
-
-    for (auto it = manager.jobs.begin(); it != manager.jobs.end();) {
-        const auto& job = it->second;
-        if (job->completed_at == 0) {
-            ++it;
-            continue;
-        }
-
-        int64_t ttl_seconds = job->status == AsyncJobStatus::Completed
-                                  ? manager.completed_ttl_seconds
-                                  : manager.failed_ttl_seconds;
-        if (now - job->completed_at >= ttl_seconds) {
-            manager.expired_jobs[job->id] = now + std::max<int64_t>(ttl_seconds, 60);
-            it                            = manager.jobs.erase(it);
-        } else {
-            ++it;
-        }
-    }
-}
-
-size_t count_pending_jobs(const AsyncJobManager& manager) {
-    size_t pending = 0;
-    for (const auto& entry : manager.jobs) {
-        if (entry.second->status == AsyncJobStatus::Queued ||
-            entry.second->status == AsyncJobStatus::Generating) {
-            ++pending;
-        }
-    }
-    return pending;
-}
-
-std::string make_async_job_id(AsyncJobManager& manager) {
-    std::ostringstream oss;
-    oss << "job_" << std::hex << unix_timestamp_now() << "_" << std::setw(8)
-        << std::setfill('0') << manager.next_id++;
-    return oss.str();
-}
-
-bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job) {
-    auto new_end = std::remove(manager.queue.begin(), manager.queue.end(), job.id);
-    if (new_end == manager.queue.end()) {
-        return false;
-    }
-
-    manager.queue.erase(new_end, manager.queue.end());
-    job.status       = AsyncJobStatus::Cancelled;
-    job.completed_at = unix_timestamp_now();
-    job.result_images_b64.clear();
-    job.result_media_b64.clear();
-    job.result_media_mime_type.clear();
-    job.result_frame_count = 0;
-    job.result_fps         = 0;
-    job.error_code         = "cancelled";
-    job.error_message      = "job cancelled by client";
-    return true;
-}
-
-json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJob& job) {
-    json result;
-    result["id"]             = job.id;
-    result["kind"]           = async_job_kind_name(job.kind);
-    result["status"]         = async_job_status_name(job.status);
-    result["created"]        = job.created_at;
-    result["started"]        = job.started_at == 0 ? json(nullptr) : json(job.started_at);
-    result["completed"]      = job.completed_at == 0 ? json(nullptr) : json(job.completed_at);
-    result["queue_position"] = 0;
-
-    if (job.status == AsyncJobStatus::Queued) {
-        size_t position = 1;
-        for (const auto& queued_id : manager.queue) {
-            if (queued_id == job.id) {
-                result["queue_position"] = position;
-                break;
-            }
-            ++position;
-        }
-    }
-
-    if (job.status == AsyncJobStatus::Completed) {
-        if (job.kind == AsyncJobKind::VidGen) {
-            result["result"] = {
-                {"output_format", job.vid_gen.output_format},
-                {"mime_type", job.result_media_mime_type},
-                {"fps", job.result_fps},
-                {"frame_count", job.result_frame_count},
-                {"b64_json", job.result_media_b64},
-            };
-        } else {
-            json images = json::array();
-            for (size_t i = 0; i < job.result_images_b64.size(); ++i) {
-                images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}});
-            }
-            result["result"] = {
-                {"output_format", job.img_gen.output_format},
-                {"images", images},
-            };
-        }
-        result["error"] = nullptr;
-    } else if (job.status == AsyncJobStatus::Failed ||
-               job.status == AsyncJobStatus::Cancelled) {
-        result["result"] = nullptr;
-        result["error"]  = {
-             {"code",
-             job.error_code.empty()
-                  ? (job.status == AsyncJobStatus::Cancelled ? "cancelled" : "generation_failed")
-                  : job.error_code},
-             {"message", job.error_message},
-        };
-    } else {
-        result["result"] = nullptr;
-        result["error"]  = nullptr;
-    }
-
-    return result;
-}
-
-bool execute_img_gen_job(ServerRuntime& runtime,
-                         AsyncGenerationJob& job,
-                         std::vector<std::string>& output_images,
-                         std::string& error_message) {
-    sd_img_gen_params_t params = job.img_gen.to_sd_img_gen_params_t();
-
-    SDImageVec results;
-
-    {
-        std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
-        sd_image_t* raw_results = generate_image(runtime.sd_ctx, &params);
-        results.adopt(raw_results, params.batch_count);
-    }
-
-    const int num_results = results.count();
-    if (num_results <= 0) {
-        error_message = "generate_image returned no results";
-        return false;
-    }
-
-    EncodedImageFormat encoded_format = EncodedImageFormat::PNG;
-    if (job.img_gen.output_format == "jpeg") {
-        encoded_format = EncodedImageFormat::JPEG;
-    } else if (job.img_gen.output_format == "webp") {
-        encoded_format = EncodedImageFormat::WEBP;
-    }
-
-    for (int i = 0; i < num_results; ++i) {
-        if (results[i].data == nullptr) {
-            continue;
-        }
-
-        const std::string metadata = job.img_gen.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime.ctx_params,
-                                                            job.img_gen.gen_params,
-                                                            job.img_gen.gen_params.seed + i)
-                                         : "";
-        auto image_bytes           = encode_image_to_vector(encoded_format,
-                                                            results[i].data,
-                                                            results[i].width,
-                                                            results[i].height,
-                                                            results[i].channel,
-                                                            metadata,
-                                                            job.img_gen.output_compression);
-        if (image_bytes.empty()) {
-            continue;
-        }
-        output_images.push_back(base64_encode(image_bytes));
-    }
-
-    if (output_images.empty()) {
-        error_message = "generate_image returned empty encoded outputs";
-        return false;
-    }
-
-    return true;
-}
-
-bool execute_vid_gen_job(ServerRuntime& runtime,
-                         AsyncGenerationJob& job,
-                         std::string& output_media_b64,
-                         std::string& output_media_mime_type,
-                         int& output_frame_count,
-                         int& output_fps,
-                         std::string& error_message) {
-    sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t();
-
-    SDImageVec results;
-    int num_results = 0;
-
-    {
-        std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
-        sd_image_t* raw_results = generate_video(runtime.sd_ctx, &params, &num_results);
-        results.adopt(raw_results, num_results);
-    }
-
-    num_results = results.count();
-    if (num_results <= 0) {
-        error_message = "generate_video returned no results";
-        return false;
-    }
-
-    std::vector<uint8_t> video_bytes = create_video_from_sd_images_to_vector(job.vid_gen.output_format,
-                                                                             results.data(),
-                                                                             num_results,
-                                                                             job.vid_gen.gen_params.fps,
-                                                                             job.vid_gen.output_compression);
-    if (video_bytes.empty()) {
-        error_message = "failed to encode generated video container";
-        return false;
-    }
-
-    output_media_b64       = base64_encode(video_bytes);
-    output_media_mime_type = video_mime_type(job.vid_gen.output_format);
-    output_frame_count     = num_results;
-    output_fps             = job.vid_gen.gen_params.fps;
-    return true;
-}
-
-void async_job_worker(ServerRuntime& runtime) {
-    AsyncJobManager& manager = *runtime.async_job_manager;
-
-    while (true) {
-        std::shared_ptr<AsyncGenerationJob> job;
-        {
-            std::unique_lock<std::mutex> lock(manager.mutex);
-            manager.cv.wait(lock, [&]() { return manager.stop || !manager.queue.empty(); });
-
-            if (manager.stop && manager.queue.empty()) {
-                break;
-            }
-
-            purge_expired_jobs(manager);
-            if (manager.queue.empty()) {
-                continue;
-            }
-
-            const std::string job_id = manager.queue.front();
-            manager.queue.pop_front();
-
-            auto it = manager.jobs.find(job_id);
-            if (it == manager.jobs.end()) {
-                continue;
-            }
-
-            job             = it->second;
-            job->status     = AsyncJobStatus::Generating;
-            job->started_at = unix_timestamp_now();
-        }
-
-        std::vector<std::string> output_images;
-        std::string output_media_b64;
-        std::string output_media_mime_type;
-        int output_frame_count = 0;
-        int output_fps         = 0;
-        std::string error_message;
-        bool ok = false;
-
-        if (job->kind == AsyncJobKind::ImgGen) {
-            ok = execute_img_gen_job(runtime, *job, output_images, error_message);
-        } else if (job->kind == AsyncJobKind::VidGen) {
-            ok = execute_vid_gen_job(runtime,
-                                     *job,
-                                     output_media_b64,
-                                     output_media_mime_type,
-                                     output_frame_count,
-                                     output_fps,
-                                     error_message);
-        } else {
-            error_message = "unsupported job kind";
-        }
-
-        {
-            std::lock_guard<std::mutex> lock(manager.mutex);
-            auto it = manager.jobs.find(job->id);
-            if (it == manager.jobs.end()) {
-                continue;
-            }
-
-            job->completed_at = unix_timestamp_now();
-            if (ok) {
-                job->status                 = AsyncJobStatus::Completed;
-                job->result_images_b64      = std::move(output_images);
-                job->result_media_b64       = std::move(output_media_b64);
-                job->result_media_mime_type = std::move(output_media_mime_type);
-                job->result_frame_count     = output_frame_count;
-                job->result_fps             = output_fps;
-                job->error_code.clear();
-                job->error_message.clear();
-            } else {
-                job->status        = AsyncJobStatus::Failed;
-                job->error_code    = "generation_failed";
-                job->error_message = error_message.empty() ? "unknown generation error" : error_message;
-                job->result_images_b64.clear();
-                job->result_media_b64.clear();
-                job->result_media_mime_type.clear();
-                job->result_frame_count = 0;
-                job->result_fps         = 0;
-            }
-
-            purge_expired_jobs(manager);
-        }
-    }
-}

examples/server/async_jobs.h

@@ -1,78 +0,0 @@
-#pragma once
-
-#include <condition_variable>
-#include <cstdint>
-#include <deque>
-#include <memory>
-#include <mutex>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-
-#include "runtime.h"
-
-enum class AsyncJobKind {
-    ImgGen,
-    VidGen,
-};
-
-enum class AsyncJobStatus {
-    Queued,
-    Generating,
-    Completed,
-    Failed,
-    Cancelled,
-};
-
-const char* async_job_kind_name(AsyncJobKind kind);
-const char* async_job_status_name(AsyncJobStatus status);
-
-struct AsyncGenerationJob {
-    std::string id;
-    AsyncJobKind kind     = AsyncJobKind::ImgGen;
-    AsyncJobStatus status = AsyncJobStatus::Queued;
-    int64_t created_at    = unix_timestamp_now();
-    int64_t started_at    = 0;
-    int64_t completed_at  = 0;
-    ImgGenJobRequest img_gen;
-    VidGenJobRequest vid_gen;
-    std::vector<std::string> result_images_b64;
-    std::string result_media_b64;
-    std::string result_media_mime_type;
-    int result_frame_count = 0;
-    int result_fps         = 0;
-    std::string error_code;
-    std::string error_message;
-};
-
-struct AsyncJobManager {
-    std::mutex mutex;
-    std::condition_variable cv;
-    std::unordered_map<std::string, std::shared_ptr<AsyncGenerationJob>> jobs;
-    std::unordered_map<std::string, int64_t> expired_jobs;
-    std::deque<std::string> queue;
-    uint64_t next_id              = 0;
-    bool stop                     = false;
-    size_t max_pending_jobs       = 64;
-    int64_t completed_ttl_seconds = 600;
-    int64_t failed_ttl_seconds    = 600;
-};
-
-void purge_expired_jobs(AsyncJobManager& manager);
-size_t count_pending_jobs(const AsyncJobManager& manager);
-std::string make_async_job_id(AsyncJobManager& manager);
-bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job);
-json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJob& job);
-bool execute_img_gen_job(ServerRuntime& runtime,
-                         AsyncGenerationJob& job,
-                         std::vector<std::string>& output_images,
-                         std::string& error_message);
-bool execute_vid_gen_job(ServerRuntime& runtime,
-                         AsyncGenerationJob& job,
-                         std::string& output_media_b64,
-                         std::string& output_media_mime_type,
-                         int& output_frame_count,
-                         int& output_fps,
-                         std::string& error_message);
-void async_job_worker(ServerRuntime& runtime);

examples/server/main.cpp

@@ -1,25 +1,191 @@
-#include <cstdlib>
+// main.cpp
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <iomanip>
 #include <iostream>
 #include <mutex>
-#include <string>
-#include <thread>
+#include <sstream>
 #include <vector>
 
 #include "httplib.h"
+#include "stable-diffusion.h"
 
-#include "async_jobs.h"
-#include "common/common.h"
-#include "common/resource_owners.hpp"
-#include "routes.h"
-#include "runtime.h"
+#include "common/common.hpp"
 
-#ifdef HAVE_INDEX_HTML
-#include "frontend/dist/gen_index_html.h"
+namespace fs = std::filesystem;
+
+// ----------------------- helpers -----------------------
+static const std::string base64_chars =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    "abcdefghijklmnopqrstuvwxyz"
+    "0123456789+/";
+
+std::string base64_encode(const std::vector<uint8_t>& bytes) {
+    std::string ret;
+    int val = 0, valb = -6;
+    for (uint8_t c : bytes) {
+        val = (val << 8) + c;
+        valb += 8;
+        while (valb >= 0) {
+            ret.push_back(base64_chars[(val >> valb) & 0x3F]);
+            valb -= 6;
+        }
+    }
+    if (valb > -6)
+        ret.push_back(base64_chars[((val << 8) >> (valb + 8)) & 0x3F]);
+    while (ret.size() % 4)
+        ret.push_back('=');
+    return ret;
+}
+
+inline bool is_base64(unsigned char c) {
+    return (isalnum(c) || (c == '+') || (c == '/'));
+}
+
+std::vector<uint8_t> base64_decode(const std::string& encoded_string) {
+    int in_len = static_cast<int>(encoded_string.size());
+    int i      = 0;
+    int j      = 0;
+    int in_    = 0;
+    uint8_t char_array_4[4], char_array_3[3];
+    std::vector<uint8_t> ret;
+
+    while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) {
+        char_array_4[i++] = encoded_string[in_];
+        in_++;
+        if (i == 4) {
+            for (i = 0; i < 4; i++)
+                char_array_4[i] = static_cast<uint8_t>(base64_chars.find(char_array_4[i]));
+
+            char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
+            char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
+            char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
+
+            for (i = 0; i < 3; i++)
+                ret.push_back(char_array_3[i]);
+            i = 0;
+        }
+    }
+
+    if (i) {
+        for (j = i; j < 4; j++)
+            char_array_4[j] = 0;
+
+        for (j = 0; j < 4; j++)
+            char_array_4[j] = static_cast<uint8_t>(base64_chars.find(char_array_4[j]));
+
+        char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
+        char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
+        char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
+
+        for (j = 0; j < i - 1; j++)
+            ret.push_back(char_array_3[j]);
+    }
+
+    return ret;
+}
+
+std::string iso_timestamp_now() {
+    using namespace std::chrono;
+    auto now      = system_clock::now();
+    std::time_t t = system_clock::to_time_t(now);
+    std::tm tm{};
+#ifdef _MSC_VER
+    gmtime_s(&tm, &t);
+#else
+    gmtime_r(&t, &tm);
 #endif
+    std::ostringstream oss;
+    oss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
+    return oss.str();
+}
 
-static void print_usage(const char* argv0, const std::vector<ArgOptions>& options_list) {
+struct SDSvrParams {
+    std::string listen_ip = "127.0.0.1";
+    int listen_port       = 1234;
+    std::string serve_html_path;
+    bool normal_exit = false;
+    bool verbose     = false;
+    bool color       = false;
+
+    ArgOptions get_options() {
+        ArgOptions options;
+
+        options.string_options = {
+            {"-l",
+             "--listen-ip",
+             "server listen ip (default: 127.0.0.1)",
+             &listen_ip},
+            {"",
+             "--serve-html-path",
+             "path to HTML file to serve at root (optional)",
+             &serve_html_path}};
+
+        options.int_options = {
+            {"",
+             "--listen-port",
+             "server listen port (default: 1234)",
+             &listen_port},
+        };
+
+        options.bool_options = {
+            {"-v",
+             "--verbose",
+             "print extra info",
+             true, &verbose},
+            {"",
+             "--color",
+             "colors the logging tags according to level",
+             true, &color},
+        };
+
+        auto on_help_arg = [&](int argc, const char** argv, int index) {
+            normal_exit = true;
+            return -1;
+        };
+
+        options.manual_options = {
+            {"-h",
+             "--help",
+             "show this help message and exit",
+             on_help_arg},
+        };
+        return options;
+    };
+
+    bool process_and_check() {
+        if (listen_ip.empty()) {
+            LOG_ERROR("error: the following arguments are required: listen_ip");
+            return false;
+        }
+
+        if (listen_port < 0 || listen_port > 65535) {
+            LOG_ERROR("error: listen_port should be in the range [0, 65535]");
+            return false;
+        }
+
+        if (!serve_html_path.empty() && !fs::exists(serve_html_path)) {
+            LOG_ERROR("error: serve_html_path file does not exist: %s", serve_html_path.c_str());
+            return false;
+        }
+        return true;
+    }
+
+    std::string to_string() const {
+        std::ostringstream oss;
+        oss << "SDSvrParams {\n"
+            << "  listen_ip: " << listen_ip << ",\n"
+            << "  listen_port: \"" << listen_port << "\",\n"
+            << "  serve_html_path: \"" << serve_html_path << "\",\n"
+            << "}";
+        return oss.str();
+    }
+};
+
+void print_usage(int argc, const char* argv[], const std::vector<ArgOptions>& options_list) {
     std::cout << version_string() << "\n";
-    std::cout << "Usage: " << argv0 << " [options]\n\n";
+    std::cout << "Usage: " << argv[0] << " [options]\n\n";
     std::cout << "Svr Options:\n";
     options_list[0].print();
     std::cout << "\nContext Options:\n";
@@ -28,30 +194,20 @@ static void print_usage(const char* argv0, const std::vector<ArgOptions>& option
     options_list[2].print();
 }
 
-static void parse_args(int argc,
-                       const char** argv,
-                       SDSvrParams& svr_params,
-                       SDContextParams& ctx_params,
-                       SDGenerationParams& default_gen_params) {
-    std::vector<ArgOptions> options_vec = {
-        svr_params.get_options(),
-        ctx_params.get_options(),
-        default_gen_params.get_options(),
-    };
+void parse_args(int argc, const char** argv, SDSvrParams& svr_params, SDContextParams& ctx_params, SDGenerationParams& default_gen_params) {
+    std::vector<ArgOptions> options_vec = {svr_params.get_options(), ctx_params.get_options(), default_gen_params.get_options()};
 
     if (!parse_options(argc, argv, options_vec)) {
-        print_usage(argv[0], options_vec);
+        print_usage(argc, argv, options_vec);
         exit(svr_params.normal_exit ? 0 : 1);
     }
 
     const bool random_seed_requested = default_gen_params.seed < 0;
 
-    if (!svr_params.resolve_and_validate() ||
-        !ctx_params.resolve_and_validate(IMG_GEN) ||
-        !default_gen_params.resolve_and_validate(IMG_GEN,
-                                                 ctx_params.lora_model_dir,
-                                                 ctx_params.hires_upscalers_dir)) {
-        print_usage(argv[0], options_vec);
+    if (!svr_params.process_and_check() ||
+        !ctx_params.process_and_check(IMG_GEN) ||
+        !default_gen_params.process_and_check(IMG_GEN, ctx_params.lora_model_dir)) {
+        print_usage(argc, argv, options_vec);
         exit(1);
     }
 
@@ -60,6 +216,63 @@ static void parse_args(int argc,
     }
 }
 
+std::string extract_and_remove_sd_cpp_extra_args(std::string& text) {
+    std::regex re("<sd_cpp_extra_args>(.*?)</sd_cpp_extra_args>");
+    std::smatch match;
+
+    std::string extracted;
+    if (std::regex_search(text, match, re)) {
+        extracted = match[1].str();
+        text      = std::regex_replace(text, re, "");
+    }
+    return extracted;
+}
+
+enum class ImageFormat { JPEG,
+                         PNG };
+
+std::vector<uint8_t> write_image_to_vector(
+    ImageFormat format,
+    const uint8_t* image,
+    int width,
+    int height,
+    int channels,
+    int quality = 90) {
+    std::vector<uint8_t> buffer;
+
+    auto write_func = [&buffer](void* context, void* data, int size) {
+        uint8_t* src = reinterpret_cast<uint8_t*>(data);
+        buffer.insert(buffer.end(), src, src + size);
+    };
+
+    struct ContextWrapper {
+        decltype(write_func)& func;
+    } ctx{write_func};
+
+    auto c_func = [](void* context, void* data, int size) {
+        auto* wrapper = reinterpret_cast<ContextWrapper*>(context);
+        wrapper->func(context, data, size);
+    };
+
+    int result = 0;
+    switch (format) {
+        case ImageFormat::JPEG:
+            result = stbi_write_jpg_to_func(c_func, &ctx, width, height, channels, image, quality);
+            break;
+        case ImageFormat::PNG:
+            result = stbi_write_png_to_func(c_func, &ctx, width, height, channels, image, width * channels);
+            break;
+        default:
+            throw std::runtime_error("invalid image format");
+    }
+
+    if (!result) {
+        throw std::runtime_error("write imgage to mem failed");
+    }
+
+    return buffer;
+}
+
 void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
     SDSvrParams* svr_params = (SDSvrParams*)data;
     log_print(level, log, svr_params->verbose, svr_params->color);
@@ -86,7 +299,7 @@ int main(int argc, const char** argv) {
     LOG_DEBUG("%s", default_gen_params.to_string().c_str());
 
     sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(false, false, false);
-    SDCtxPtr sd_ctx(new_sd_ctx(&sd_ctx_params));
+    sd_ctx_t* sd_ctx              = new_sd_ctx(&sd_ctx_params);
 
     if (sd_ctx == nullptr) {
         LOG_ERROR("new_sd_ctx_t failed");
@@ -95,26 +308,6 @@ int main(int argc, const char** argv) {
 
     std::mutex sd_ctx_mutex;
 
-    std::vector<LoraEntry> lora_cache;
-    std::mutex lora_mutex;
-    std::vector<UpscalerEntry> upscaler_cache;
-    std::mutex upscaler_mutex;
-    AsyncJobManager async_job_manager;
-    ServerRuntime runtime = {
-        sd_ctx.get(),
-        &sd_ctx_mutex,
-        &svr_params,
-        &ctx_params,
-        &default_gen_params,
-        &lora_cache,
-        &lora_mutex,
-        &upscaler_cache,
-        &upscaler_mutex,
-        &async_job_manager,
-    };
-
-    std::thread async_worker(async_job_worker, std::ref(runtime));
-
     httplib::Server svr;
 
     svr.set_pre_routing_handler([](const httplib::Request& req, httplib::Response& res) {
@@ -134,25 +327,409 @@ int main(int argc, const char** argv) {
         return httplib::Server::HandlerResponse::Unhandled;
     });
 
-    std::string index_html;
-#ifdef HAVE_INDEX_HTML
-    index_html.assign(reinterpret_cast<const char*>(index_html_bytes), index_html_size);
-#else
-    index_html = "Stable Diffusion Server is running";
-#endif
-    register_index_endpoints(svr, svr_params, index_html);
-    register_openai_api_endpoints(svr, runtime);
-    register_sdapi_endpoints(svr, runtime);
-    register_sdcpp_api_endpoints(svr, runtime);
+    // health
+    svr.Get("/", [&](const httplib::Request&, httplib::Response& res) {
+        if (!svr_params.serve_html_path.empty()) {
+            std::ifstream file(svr_params.serve_html_path);
+            if (file) {
+                std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
+                res.set_content(content, "text/html");
+            } else {
+                res.status = 500;
+                res.set_content("Error: Unable to read HTML file", "text/plain");
+            }
+        } else {
+            res.set_content("Stable Diffusion Server is running", "text/plain");
+        }
+    });
 
-    LOG_INFO("listening on: http://%s:%d\n", svr_params.listen_ip.c_str(), svr_params.listen_port);
+    // models endpoint (minimal)
+    svr.Get("/v1/models", [&](const httplib::Request&, httplib::Response& res) {
+        json r;
+        r["data"] = json::array();
+        r["data"].push_back({{"id", "sd-cpp-local"}, {"object", "model"}, {"owned_by", "local"}});
+        res.set_content(r.dump(), "application/json");
+    });
+
+    // core endpoint: /v1/images/generations
+    svr.Post("/v1/images/generations", [&](const httplib::Request& req, httplib::Response& res) {
+        try {
+            if (req.body.empty()) {
+                res.status = 400;
+                res.set_content(R"({"error":"empty body"})", "application/json");
+                return;
+            }
+
+            json j                    = json::parse(req.body);
+            std::string prompt        = j.value("prompt", "");
+            int n                     = std::max(1, j.value("n", 1));
+            std::string size          = j.value("size", "");
+            std::string output_format = j.value("output_format", "png");
+            int output_compression    = j.value("output_compression", 100);
+            int width                 = 512;
+            int height                = 512;
+            if (!size.empty()) {
+                auto pos = size.find('x');
+                if (pos != std::string::npos) {
+                    try {
+                        width  = std::stoi(size.substr(0, pos));
+                        height = std::stoi(size.substr(pos + 1));
+                    } catch (...) {
+                    }
+                }
+            }
+
+            if (prompt.empty()) {
+                res.status = 400;
+                res.set_content(R"({"error":"prompt required"})", "application/json");
+                return;
+            }
+
+            std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(prompt);
+
+            if (output_format != "png" && output_format != "jpeg") {
+                res.status = 400;
+                res.set_content(R"({"error":"invalid output_format, must be one of [png, jpeg]"})", "application/json");
+                return;
+            }
+            if (n <= 0)
+                n = 1;
+            if (n > 8)
+                n = 8;  // safety
+            if (output_compression > 100) {
+                output_compression = 100;
+            }
+            if (output_compression < 0) {
+                output_compression = 0;
+            }
+
+            json out;
+            out["created"]       = iso_timestamp_now();
+            out["data"]          = json::array();
+            out["output_format"] = output_format;
+
+            SDGenerationParams gen_params = default_gen_params;
+            gen_params.prompt             = prompt;
+            gen_params.width              = width;
+            gen_params.height             = height;
+            gen_params.batch_count        = n;
+
+            if (!sd_cpp_extra_args_str.empty() && !gen_params.from_json_str(sd_cpp_extra_args_str)) {
+                res.status = 400;
+                res.set_content(R"({"error":"invalid sd_cpp_extra_args"})", "application/json");
+                return;
+            }
+
+            if (!gen_params.process_and_check(IMG_GEN, "")) {
+                res.status = 400;
+                res.set_content(R"({"error":"invalid params"})", "application/json");
+                return;
+            }
+
+            LOG_DEBUG("%s\n", gen_params.to_string().c_str());
+
+            sd_image_t init_image    = {(uint32_t)gen_params.width, (uint32_t)gen_params.height, 3, nullptr};
+            sd_image_t control_image = {(uint32_t)gen_params.width, (uint32_t)gen_params.height, 3, nullptr};
+            sd_image_t mask_image    = {(uint32_t)gen_params.width, (uint32_t)gen_params.height, 1, nullptr};
+            std::vector<sd_image_t> pmid_images;
+
+            sd_img_gen_params_t img_gen_params = {
+                gen_params.lora_vec.data(),
+                static_cast<uint32_t>(gen_params.lora_vec.size()),
+                gen_params.prompt.c_str(),
+                gen_params.negative_prompt.c_str(),
+                gen_params.clip_skip,
+                init_image,
+                nullptr,
+                0,
+                gen_params.auto_resize_ref_image,
+                gen_params.increase_ref_index,
+                mask_image,
+                gen_params.width,
+                gen_params.height,
+                gen_params.sample_params,
+                gen_params.strength,
+                gen_params.seed,
+                gen_params.batch_count,
+                control_image,
+                gen_params.control_strength,
+                {
+                    pmid_images.data(),
+                    (int)pmid_images.size(),
+                    gen_params.pm_id_embed_path.c_str(),
+                    gen_params.pm_style_strength,
+                },  // pm_params
+                ctx_params.vae_tiling_params,
+                gen_params.cache_params,
+            };
+
+            sd_image_t* results = nullptr;
+            int num_results     = 0;
+
+            {
+                std::lock_guard<std::mutex> lock(sd_ctx_mutex);
+                results     = generate_image(sd_ctx, &img_gen_params);
+                num_results = gen_params.batch_count;
+            }
+
+            for (int i = 0; i < num_results; i++) {
+                if (results[i].data == nullptr) {
+                    continue;
+                }
+                auto image_bytes = write_image_to_vector(output_format == "jpeg" ? ImageFormat::JPEG : ImageFormat::PNG,
+                                                         results[i].data,
+                                                         results[i].width,
+                                                         results[i].height,
+                                                         results[i].channel,
+                                                         output_compression);
+                if (image_bytes.empty()) {
+                    LOG_ERROR("write image to mem failed");
+                    continue;
+                }
+
+                // base64 encode
+                std::string b64 = base64_encode(image_bytes);
+                json item;
+                item["b64_json"] = b64;
+                out["data"].push_back(item);
+            }
+
+            res.set_content(out.dump(), "application/json");
+            res.status = 200;
+
+        } catch (const std::exception& e) {
+            res.status = 500;
+            json err;
+            err["error"]   = "server_error";
+            err["message"] = e.what();
+            res.set_content(err.dump(), "application/json");
+        }
+    });
+
+    svr.Post("/v1/images/edits", [&](const httplib::Request& req, httplib::Response& res) {
+        try {
+            if (!req.is_multipart_form_data()) {
+                res.status = 400;
+                res.set_content(R"({"error":"Content-Type must be multipart/form-data"})", "application/json");
+                return;
+            }
+
+            std::string prompt = req.form.get_field("prompt");
+            if (prompt.empty()) {
+                res.status = 400;
+                res.set_content(R"({"error":"prompt required"})", "application/json");
+                return;
+            }
+
+            std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(prompt);
+
+            size_t image_count = req.form.get_file_count("image[]");
+            if (image_count == 0) {
+                res.status = 400;
+                res.set_content(R"({"error":"at least one image[] required"})", "application/json");
+                return;
+            }
+
+            std::vector<std::vector<uint8_t>> images_bytes;
+            for (size_t i = 0; i < image_count; i++) {
+                auto file = req.form.get_file("image[]", i);
+                images_bytes.emplace_back(file.content.begin(), file.content.end());
+            }
+
+            std::vector<uint8_t> mask_bytes;
+            if (req.form.has_file("mask")) {
+                auto file = req.form.get_file("mask");
+                mask_bytes.assign(file.content.begin(), file.content.end());
+            }
+
+            int n = 1;
+            if (req.form.has_field("n")) {
+                try {
+                    n = std::stoi(req.form.get_field("n"));
+                } catch (...) {
+                }
+            }
+            n = std::clamp(n, 1, 8);
+
+            std::string size = req.form.get_field("size");
+            int width = 512, height = 512;
+            if (!size.empty()) {
+                auto pos = size.find('x');
+                if (pos != std::string::npos) {
+                    try {
+                        width  = std::stoi(size.substr(0, pos));
+                        height = std::stoi(size.substr(pos + 1));
+                    } catch (...) {
+                    }
+                }
+            }
+
+            std::string output_format = "png";
+            if (req.form.has_field("output_format"))
+                output_format = req.form.get_field("output_format");
+            if (output_format != "png" && output_format != "jpeg") {
+                res.status = 400;
+                res.set_content(R"({"error":"invalid output_format, must be one of [png, jpeg]"})", "application/json");
+                return;
+            }
+
+            std::string output_compression_str = req.form.get_field("output_compression");
+            int output_compression             = 100;
+            try {
+                output_compression = std::stoi(output_compression_str);
+            } catch (...) {
+            }
+            if (output_compression > 100) {
+                output_compression = 100;
+            }
+            if (output_compression < 0) {
+                output_compression = 0;
+            }
+
+            SDGenerationParams gen_params = default_gen_params;
+            gen_params.prompt             = prompt;
+            gen_params.width              = width;
+            gen_params.height             = height;
+            gen_params.batch_count        = n;
+
+            if (!sd_cpp_extra_args_str.empty() && !gen_params.from_json_str(sd_cpp_extra_args_str)) {
+                res.status = 400;
+                res.set_content(R"({"error":"invalid sd_cpp_extra_args"})", "application/json");
+                return;
+            }
+
+            if (!gen_params.process_and_check(IMG_GEN, "")) {
+                res.status = 400;
+                res.set_content(R"({"error":"invalid params"})", "application/json");
+                return;
+            }
+
+            LOG_DEBUG("%s\n", gen_params.to_string().c_str());
+
+            sd_image_t init_image    = {(uint32_t)gen_params.width, (uint32_t)gen_params.height, 3, nullptr};
+            sd_image_t control_image = {(uint32_t)gen_params.width, (uint32_t)gen_params.height, 3, nullptr};
+            std::vector<sd_image_t> pmid_images;
+
+            std::vector<sd_image_t> ref_images;
+            ref_images.reserve(images_bytes.size());
+            for (auto& bytes : images_bytes) {
+                int img_w           = width;
+                int img_h           = height;
+                uint8_t* raw_pixels = load_image_from_memory(
+                    reinterpret_cast<const char*>(bytes.data()),
+                    static_cast<int>(bytes.size()),
+                    img_w, img_h,
+                    width, height, 3);
+
+                if (!raw_pixels) {
+                    continue;
+                }
+
+                sd_image_t img{(uint32_t)img_w, (uint32_t)img_h, 3, raw_pixels};
+                ref_images.push_back(img);
+            }
+
+            sd_image_t mask_image = {0};
+            if (!mask_bytes.empty()) {
+                int mask_w        = width;
+                int mask_h        = height;
+                uint8_t* mask_raw = load_image_from_memory(
+                    reinterpret_cast<const char*>(mask_bytes.data()),
+                    static_cast<int>(mask_bytes.size()),
+                    mask_w, mask_h,
+                    width, height, 1);
+                mask_image = {(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw};
+            } else {
+                mask_image.width   = width;
+                mask_image.height  = height;
+                mask_image.channel = 1;
+                mask_image.data    = nullptr;
+            }
+
+            sd_img_gen_params_t img_gen_params = {
+                gen_params.lora_vec.data(),
+                static_cast<uint32_t>(gen_params.lora_vec.size()),
+                gen_params.prompt.c_str(),
+                gen_params.negative_prompt.c_str(),
+                gen_params.clip_skip,
+                init_image,
+                ref_images.data(),
+                (int)ref_images.size(),
+                gen_params.auto_resize_ref_image,
+                gen_params.increase_ref_index,
+                mask_image,
+                gen_params.width,
+                gen_params.height,
+                gen_params.sample_params,
+                gen_params.strength,
+                gen_params.seed,
+                gen_params.batch_count,
+                control_image,
+                gen_params.control_strength,
+                {
+                    pmid_images.data(),
+                    (int)pmid_images.size(),
+                    gen_params.pm_id_embed_path.c_str(),
+                    gen_params.pm_style_strength,
+                },  // pm_params
+                ctx_params.vae_tiling_params,
+                gen_params.cache_params,
+            };
+
+            sd_image_t* results = nullptr;
+            int num_results     = 0;
+
+            {
+                std::lock_guard<std::mutex> lock(sd_ctx_mutex);
+                results     = generate_image(sd_ctx, &img_gen_params);
+                num_results = gen_params.batch_count;
+            }
+
+            json out;
+            out["created"]       = iso_timestamp_now();
+            out["data"]          = json::array();
+            out["output_format"] = output_format;
+
+            for (int i = 0; i < num_results; i++) {
+                if (results[i].data == nullptr)
+                    continue;
+                auto image_bytes = write_image_to_vector(output_format == "jpeg" ? ImageFormat::JPEG : ImageFormat::PNG,
+                                                         results[i].data,
+                                                         results[i].width,
+                                                         results[i].height,
+                                                         results[i].channel,
+                                                         output_compression);
+                std::string b64 = base64_encode(image_bytes);
+                json item;
+                item["b64_json"] = b64;
+                out["data"].push_back(item);
+            }
+
+            res.set_content(out.dump(), "application/json");
+            res.status = 200;
+
+            if (init_image.data) {
+                stbi_image_free(init_image.data);
+            }
+            if (mask_image.data) {
+                stbi_image_free(mask_image.data);
+            }
+            for (auto ref_image : ref_images) {
+                stbi_image_free(ref_image.data);
+            }
+        } catch (const std::exception& e) {
+            res.status = 500;
+            json err;
+            err["error"]   = "server_error";
+            err["message"] = e.what();
+            res.set_content(err.dump(), "application/json");
+        }
+    });
+
+    LOG_INFO("listening on: %s:%d\n", svr_params.listen_ip.c_str(), svr_params.listen_port);
     svr.listen(svr_params.listen_ip, svr_params.listen_port);
 
-    {
-        std::lock_guard<std::mutex> lock(async_job_manager.mutex);
-        async_job_manager.stop = true;
-    }
-    async_job_manager.cv.notify_all();
-    async_worker.join();
+    // cleanup
+    free_sd_ctx(sd_ctx);
     return 0;
 }

examples/server/routes.h

@@ -1,11 +0,0 @@
-#pragma once
-
-#include <string>
-
-#include "httplib.h"
-#include "runtime.h"
-
-void register_index_endpoints(httplib::Server& svr, const SDSvrParams& svr_params, const std::string& index_html);
-void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt);
-void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt);
-void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt);

examples/server/routes_index.cpp

@@ -1,22 +0,0 @@
-#include "routes.h"
-
-#include <fstream>
-#include <iterator>
-
-void register_index_endpoints(httplib::Server& svr, const SDSvrParams& svr_params, const std::string& index_html) {
-    const std::string serve_html_path = svr_params.serve_html_path;
-    svr.Get("/", [serve_html_path, index_html](const httplib::Request&, httplib::Response& res) {
-        if (!serve_html_path.empty()) {
-            std::ifstream file(serve_html_path);
-            if (file) {
-                std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
-                res.set_content(content, "text/html");
-            } else {
-                res.status = 500;
-                res.set_content("Error: Unable to read HTML file", "text/plain");
-            }
-        } else {
-            res.set_content(index_html, "text/html");
-        }
-    });
-}

examples/server/routes_openai.cpp

@@ -1,388 +0,0 @@
-#include "routes.h"
-
-#include <algorithm>
-#include <ctime>
-#include <regex>
-
-#include "common/common.h"
-#include "common/media_io.h"
-#include "common/resource_owners.hpp"
-
-static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) {
-    std::regex re("<sd_cpp_extra_args>(.*?)</sd_cpp_extra_args>");
-    std::smatch match;
-
-    std::string extracted;
-    if (std::regex_search(text, match, re)) {
-        extracted = match[1].str();
-        text      = std::regex_replace(text, re, "");
-    }
-    return extracted;
-}
-
-static bool build_openai_generation_request(const httplib::Request& req,
-                                            ServerRuntime& runtime,
-                                            ImgGenJobRequest& request,
-                                            std::string& error_message) {
-    if (req.body.empty()) {
-        error_message = "empty body";
-        return false;
-    }
-
-    json j                    = json::parse(req.body);
-    std::string prompt        = j.value("prompt", "");
-    int n                     = std::max(1, j.value("n", 1));
-    std::string size          = j.value("size", "");
-    std::string output_format = j.value("output_format", "png");
-    int output_compression    = j.value("output_compression", 100);
-    int width                 = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->width : 512;
-    int height                = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->height : 512;
-    if (!size.empty()) {
-        auto pos = size.find('x');
-        if (pos != std::string::npos) {
-            try {
-                width  = std::stoi(size.substr(0, pos));
-                height = std::stoi(size.substr(pos + 1));
-            } catch (...) {
-            }
-        }
-    }
-
-    if (prompt.empty()) {
-        error_message = "prompt required";
-        return false;
-    }
-
-    request.gen_params = *runtime.default_gen_params;
-    if (!assign_output_options(request, output_format, output_compression, true, error_message)) {
-        return false;
-    }
-
-    request.gen_params.prompt      = prompt;
-    request.gen_params.width       = width;
-    request.gen_params.height      = height;
-    request.gen_params.batch_count = n;
-
-    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
-    if (!sd_cpp_extra_args_str.empty() && !request.gen_params.from_json_str(sd_cpp_extra_args_str)) {
-        error_message = "invalid sd_cpp_extra_args";
-        return false;
-    }
-
-    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
-    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
-        error_message = "invalid params";
-        return false;
-    }
-    return true;
-}
-
-static bool build_openai_edit_request(const httplib::Request& req,
-                                      ServerRuntime& runtime,
-                                      ImgGenJobRequest& request,
-                                      std::string& error_message) {
-    if (!req.is_multipart_form_data()) {
-        error_message = "Content-Type must be multipart/form-data";
-        return false;
-    }
-
-    std::string prompt = req.form.get_field("prompt");
-    if (prompt.empty()) {
-        error_message = "prompt required";
-        return false;
-    }
-
-    size_t image_count    = req.form.get_file_count("image[]");
-    bool has_legacy_image = req.form.has_file("image");
-    if (image_count == 0 && !has_legacy_image) {
-        error_message = "at least one image[] required";
-        return false;
-    }
-
-    std::vector<std::vector<uint8_t>> images_bytes;
-    for (size_t i = 0; i < image_count; ++i) {
-        auto file = req.form.get_file("image[]", i);
-        images_bytes.emplace_back(file.content.begin(), file.content.end());
-    }
-    if (image_count == 0 && has_legacy_image) {
-        auto file = req.form.get_file("image");
-        images_bytes.emplace_back(file.content.begin(), file.content.end());
-    }
-
-    std::vector<uint8_t> mask_bytes;
-    if (req.form.has_file("mask")) {
-        auto file = req.form.get_file("mask");
-        mask_bytes.assign(file.content.begin(), file.content.end());
-    }
-
-    int n = 1;
-    if (req.form.has_field("n")) {
-        try {
-            n = std::stoi(req.form.get_field("n"));
-        } catch (...) {
-        }
-    }
-
-    std::string size = req.form.get_field("size");
-    int width        = -1;
-    int height       = -1;
-    if (!size.empty()) {
-        auto pos = size.find('x');
-        if (pos != std::string::npos) {
-            try {
-                width  = std::stoi(size.substr(0, pos));
-                height = std::stoi(size.substr(pos + 1));
-            } catch (...) {
-            }
-        }
-    }
-
-    std::string output_format = req.form.has_field("output_format")
-                                    ? req.form.get_field("output_format")
-                                    : "png";
-
-    int output_compression = 100;
-    try {
-        output_compression = std::stoi(req.form.get_field("output_compression"));
-    } catch (...) {
-    }
-
-    request.gen_params = *runtime.default_gen_params;
-    if (!assign_output_options(request, output_format, output_compression, false, error_message)) {
-        return false;
-    }
-
-    request.gen_params.prompt      = prompt;
-    request.gen_params.width       = width;
-    request.gen_params.height      = height;
-    request.gen_params.batch_count = n;
-
-    for (auto& bytes : images_bytes) {
-        int img_w           = 0;
-        int img_h           = 0;
-        uint8_t* raw_pixels = load_image_from_memory(
-            reinterpret_cast<const char*>(bytes.data()),
-            static_cast<int>(bytes.size()),
-            img_w, img_h,
-            width, height, 3);
-        if (raw_pixels == nullptr) {
-            continue;
-        }
-
-        SDImageOwner image_owner({(uint32_t)img_w, (uint32_t)img_h, 3, raw_pixels});
-        request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height);
-        request.gen_params.ref_images.push_back(std::move(image_owner));
-    }
-
-    if (!request.gen_params.ref_images.empty()) {
-        request.gen_params.init_image = request.gen_params.ref_images.front();
-    }
-
-    if (!mask_bytes.empty()) {
-        int expected_width  = 0;
-        int expected_height = 0;
-        if (request.gen_params.width_and_height_are_set()) {
-            expected_width  = request.gen_params.width;
-            expected_height = request.gen_params.height;
-        }
-        int mask_w = 0;
-        int mask_h = 0;
-
-        uint8_t* mask_raw = load_image_from_memory(
-            reinterpret_cast<const char*>(mask_bytes.data()),
-            static_cast<int>(mask_bytes.size()),
-            mask_w, mask_h,
-            expected_width, expected_height, 1);
-        request.gen_params.mask_image.reset({(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw});
-        const sd_image_t& mask_image = request.gen_params.mask_image.get();
-        request.gen_params.set_width_and_height_if_unset(mask_image.width, mask_image.height);
-    } else {
-        request.gen_params.mask_image.reset({
-            (uint32_t)request.gen_params.get_resolved_width(),
-            (uint32_t)request.gen_params.get_resolved_height(),
-            1,
-            nullptr,
-        });
-    }
-
-    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
-    if (!sd_cpp_extra_args_str.empty() && !request.gen_params.from_json_str(sd_cpp_extra_args_str)) {
-        error_message = "invalid sd_cpp_extra_args";
-        return false;
-    }
-
-    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
-    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
-        error_message = "invalid params";
-        return false;
-    }
-
-    return true;
-}
-
-static bool execute_sync_img_gen_request(ServerRuntime& runtime,
-                                         ImgGenJobRequest& request,
-                                         SDImageVec& results,
-                                         std::string& error_message) {
-    sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t();
-    int num_results                    = 0;
-
-    {
-        std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
-        sd_image_t* raw_results = generate_image(runtime.sd_ctx, &img_gen_params);
-        num_results             = request.gen_params.batch_count;
-        results.adopt(raw_results, num_results);
-    }
-
-    if (results.empty()) {
-        error_message = "generate_image returned no results";
-        return false;
-    }
-    return true;
-}
-
-void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
-    ServerRuntime* runtime = &rt;
-
-    svr.Get("/v1/models", [runtime](const httplib::Request&, httplib::Response& res) {
-        json r;
-        r["data"] = json::array();
-        r["data"].push_back({{"id", "sd-cpp-local"}, {"object", "model"}, {"owned_by", "local"}});
-        res.set_content(r.dump(), "application/json");
-    });
-
-    svr.Post("/v1/images/generations", [runtime](const httplib::Request& req, httplib::Response& res) {
-        try {
-            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
-                res.status = 400;
-                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
-                return;
-            }
-
-            ImgGenJobRequest request;
-            std::string error_message;
-            if (!build_openai_generation_request(req, *runtime, request, error_message)) {
-                res.status = 400;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
-
-            SDImageVec results;
-            if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) {
-                res.status = 500;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            json out;
-            out["created"]       = static_cast<long long>(std::time(nullptr));
-            out["data"]          = json::array();
-            out["output_format"] = request.output_format;
-
-            for (int i = 0; i < request.gen_params.batch_count; ++i) {
-                if (results[i].data == nullptr) {
-                    continue;
-                }
-                std::string params = request.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime->ctx_params,
-                                                            request.gen_params,
-                                                            request.gen_params.seed + i)
-                                         : "";
-                auto image_bytes   = encode_image_to_vector(request.output_format == "jpeg"
-                                                                ? EncodedImageFormat::JPEG
-                                                            : request.output_format == "webp"
-                                                                ? EncodedImageFormat::WEBP
-                                                                : EncodedImageFormat::PNG,
-                                                          results[i].data,
-                                                          results[i].width,
-                                                          results[i].height,
-                                                          results[i].channel,
-                                                          params,
-                                                          request.output_compression);
-                if (image_bytes.empty()) {
-                    LOG_ERROR("write image to mem failed");
-                    continue;
-                }
-
-                json item;
-                item["b64_json"] = base64_encode(image_bytes);
-                out["data"].push_back(item);
-            }
-
-            res.set_content(out.dump(), "application/json");
-            res.status = 200;
-
-        } catch (const std::exception& e) {
-            res.status = 500;
-            json err;
-            err["error"]   = "server_error";
-            err["message"] = e.what();
-            res.set_content(err.dump(), "application/json");
-        }
-    });
-
-    svr.Post("/v1/images/edits", [runtime](const httplib::Request& req, httplib::Response& res) {
-        try {
-            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
-                res.status = 400;
-                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
-                return;
-            }
-
-            ImgGenJobRequest request;
-            std::string error_message;
-            if (!build_openai_edit_request(req, *runtime, request, error_message)) {
-                res.status = 400;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
-
-            SDImageVec results;
-            if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) {
-                res.status = 500;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            json out;
-            out["created"]       = static_cast<long long>(std::time(nullptr));
-            out["data"]          = json::array();
-            out["output_format"] = request.output_format;
-
-            for (int i = 0; i < request.gen_params.batch_count; ++i) {
-                if (results[i].data == nullptr) {
-                    continue;
-                }
-                std::string params = request.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime->ctx_params,
-                                                            request.gen_params,
-                                                            request.gen_params.seed + i)
-                                         : "";
-                auto image_bytes   = encode_image_to_vector(request.output_format == "jpeg" ? EncodedImageFormat::JPEG : EncodedImageFormat::PNG,
-                                                          results[i].data,
-                                                          results[i].width,
-                                                          results[i].height,
-                                                          results[i].channel,
-                                                          params,
-                                                          request.output_compression);
-                json item;
-                item["b64_json"] = base64_encode(image_bytes);
-                out["data"].push_back(item);
-            }
-
-            res.set_content(out.dump(), "application/json");
-            res.status = 200;
-
-        } catch (const std::exception& e) {
-            res.status = 500;
-            json err;
-            err["error"]   = "server_error";
-            err["message"] = e.what();
-            res.set_content(err.dump(), "application/json");
-        }
-    });
-}

examples/server/routes_sdapi.cpp

@@ -1,473 +0,0 @@
-#include "routes.h"
-
-#include <algorithm>
-#include <cctype>
-#include <cstring>
-#include <regex>
-#include <string_view>
-#include <unordered_map>
-
-#include "common/common.h"
-#include "common/media_io.h"
-#include "common/resource_owners.hpp"
-
-namespace fs = std::filesystem;
-
-static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) {
-    std::regex re("<sd_cpp_extra_args>(.*?)</sd_cpp_extra_args>");
-    std::smatch match;
-
-    std::string extracted;
-    if (std::regex_search(text, match, re)) {
-        extracted = match[1].str();
-        text      = std::regex_replace(text, re, "");
-    }
-    return extracted;
-}
-
-static fs::path resolve_display_model_path(const ServerRuntime& runtime) {
-    const auto& ctx = *runtime.ctx_params;
-    if (!ctx.model_path.empty()) {
-        return fs::path(ctx.model_path);
-    }
-    if (!ctx.diffusion_model_path.empty()) {
-        return fs::path(ctx.diffusion_model_path);
-    }
-    return {};
-}
-
-static std::string lower_ascii(std::string value) {
-    std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
-        return static_cast<char>(std::tolower(c));
-    });
-    return value;
-}
-
-static enum sample_method_t get_sdapi_sample_method(std::string name) {
-    enum sample_method_t result = str_to_sample_method(name.c_str());
-    if (result != SAMPLE_METHOD_COUNT) {
-        return result;
-    }
-
-    name = lower_ascii(name);
-    static const std::unordered_map<std::string_view, sample_method_t> hardcoded{
-        {"euler a", EULER_A_SAMPLE_METHOD},
-        {"k_euler_a", EULER_A_SAMPLE_METHOD},
-        {"euler", EULER_SAMPLE_METHOD},
-        {"k_euler", EULER_SAMPLE_METHOD},
-        {"heun", HEUN_SAMPLE_METHOD},
-        {"k_heun", HEUN_SAMPLE_METHOD},
-        {"dpm2", DPM2_SAMPLE_METHOD},
-        {"k_dpm_2", DPM2_SAMPLE_METHOD},
-        {"lcm", LCM_SAMPLE_METHOD},
-        {"ddim", DDIM_TRAILING_SAMPLE_METHOD},
-        {"dpm++ 2m", DPMPP2M_SAMPLE_METHOD},
-        {"k_dpmpp_2m", DPMPP2M_SAMPLE_METHOD},
-        {"res multistep", RES_MULTISTEP_SAMPLE_METHOD},
-        {"k_res_multistep", RES_MULTISTEP_SAMPLE_METHOD},
-        {"res 2s", RES_2S_SAMPLE_METHOD},
-        {"k_res_2s", RES_2S_SAMPLE_METHOD},
-        {"euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
-        {"k_euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
-        {"euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
-        {"k_euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
-    };
-    auto it = hardcoded.find(name);
-    return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT;
-}
-
-static void assign_solid_mask(SDImageOwner& mask_owner, int width, int height) {
-    const size_t pixel_count = static_cast<size_t>(width) * static_cast<size_t>(height);
-    uint8_t* raw_mask        = static_cast<uint8_t*>(malloc(pixel_count));
-    if (raw_mask == nullptr) {
-        mask_owner.reset({0, 0, 1, nullptr});
-        return;
-    }
-    std::memset(raw_mask, 255, pixel_count);
-    mask_owner.reset({(uint32_t)width, (uint32_t)height, 1, raw_mask});
-}
-
-static bool build_sdapi_img_gen_request(const json& j,
-                                        ServerRuntime& runtime,
-                                        bool img2img,
-                                        ImgGenJobRequest& request,
-                                        std::string& error_message) {
-    std::string prompt          = j.value("prompt", "");
-    std::string negative_prompt = j.value("negative_prompt", "");
-    int width                   = j.value("width", 512);
-    int height                  = j.value("height", 512);
-    int steps                   = j.value("steps", runtime.default_gen_params->sample_params.sample_steps);
-    float cfg_scale             = j.value("cfg_scale", runtime.default_gen_params->sample_params.guidance.txt_cfg);
-    int64_t seed                = j.value("seed", -1);
-    int batch_size              = j.value("batch_size", 1);
-    int clip_skip               = j.value("clip_skip", -1);
-    std::string sampler_name    = j.value("sampler_name", "");
-    std::string scheduler_name  = j.value("scheduler", "");
-
-    if (width <= 0 || height <= 0) {
-        error_message = "width and height must be positive";
-        return false;
-    }
-
-    if (prompt.empty()) {
-        error_message = "prompt required";
-        return false;
-    }
-
-    request.gen_params = *runtime.default_gen_params;
-
-    request.gen_params.prompt                         = prompt;
-    request.gen_params.negative_prompt                = negative_prompt;
-    request.gen_params.seed                           = seed;
-    request.gen_params.sample_params.sample_steps     = steps;
-    request.gen_params.batch_count                    = batch_size;
-    request.gen_params.sample_params.guidance.txt_cfg = cfg_scale;
-    request.gen_params.width                          = j.value("width", -1);
-    request.gen_params.height                         = j.value("height", -1);
-
-    if (!img2img && j.value("enable_hr", false)) {
-        request.gen_params.hires_enabled = true;
-        request.gen_params.hires_scale   = j.value("hr_scale", request.gen_params.hires_scale);
-        request.gen_params.hires_width   = j.value("hr_resize_x", request.gen_params.hires_width);
-        request.gen_params.hires_height  = j.value("hr_resize_y", request.gen_params.hires_height);
-        request.gen_params.hires_steps   = j.value("hr_steps", request.gen_params.hires_steps);
-        request.gen_params.hires_denoising_strength =
-            j.value("denoising_strength", request.gen_params.hires_denoising_strength);
-
-        request.gen_params.hires_upscaler = j.value("hr_upscaler", request.gen_params.hires_upscaler);
-    }
-
-    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
-    if (!sd_cpp_extra_args_str.empty() && !request.gen_params.from_json_str(sd_cpp_extra_args_str)) {
-        error_message = "invalid sd_cpp_extra_args";
-        return false;
-    }
-
-    if (clip_skip > 0) {
-        request.gen_params.clip_skip = clip_skip;
-    }
-
-    enum sample_method_t sample_method = get_sdapi_sample_method(sampler_name);
-    if (sample_method != SAMPLE_METHOD_COUNT) {
-        request.gen_params.sample_params.sample_method = sample_method;
-    }
-
-    enum scheduler_t scheduler = str_to_scheduler(scheduler_name.c_str());
-    if (scheduler != SCHEDULER_COUNT) {
-        request.gen_params.sample_params.scheduler = scheduler;
-    }
-
-    if (j.contains("lora") && j["lora"].is_array()) {
-        request.gen_params.lora_map.clear();
-        request.gen_params.high_noise_lora_map.clear();
-
-        for (const auto& item : j["lora"]) {
-            if (!item.is_object()) {
-                continue;
-            }
-
-            std::string path   = item.value("path", "");
-            float multiplier   = item.value("multiplier", 1.0f);
-            bool is_high_noise = item.value("is_high_noise", false);
-
-            if (path.empty()) {
-                error_message = "lora.path required";
-                return false;
-            }
-
-            std::string fullpath = get_lora_full_path(runtime, path);
-            if (fullpath.empty()) {
-                error_message = "invalid lora path: " + path;
-                return false;
-            }
-
-            if (is_high_noise) {
-                request.gen_params.high_noise_lora_map[fullpath] += multiplier;
-            } else {
-                request.gen_params.lora_map[fullpath] += multiplier;
-            }
-        }
-    }
-
-    if (img2img) {
-        const int expected_width  = request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0;
-        const int expected_height = request.gen_params.width_and_height_are_set() ? request.gen_params.height : 0;
-
-        if (j.contains("init_images") && j["init_images"].is_array() && !j["init_images"].empty()) {
-            if (decode_base64_image(j["init_images"][0].get<std::string>(),
-                                    3,
-                                    expected_width,
-                                    expected_height,
-                                    request.gen_params.init_image)) {
-                const sd_image_t& image = request.gen_params.init_image.get();
-                request.gen_params.set_width_and_height_if_unset(image.width, image.height);
-            }
-        }
-
-        if (j.contains("mask") && j["mask"].is_string()) {
-            if (decode_base64_image(j["mask"].get<std::string>(),
-                                    1,
-                                    expected_width,
-                                    expected_height,
-                                    request.gen_params.mask_image)) {
-                const sd_image_t& image = request.gen_params.mask_image.get();
-                request.gen_params.set_width_and_height_if_unset(image.width, image.height);
-            }
-            sd_image_t& mask_image      = request.gen_params.mask_image.get();
-            bool inpainting_mask_invert = j.value("inpainting_mask_invert", 0) != 0;
-            if (inpainting_mask_invert && mask_image.data != nullptr) {
-                for (uint32_t i = 0; i < mask_image.width * mask_image.height; ++i) {
-                    mask_image.data[i] = 255 - mask_image.data[i];
-                }
-            }
-        } else {
-            const int resolved_width  = request.gen_params.get_resolved_width();
-            const int resolved_height = request.gen_params.get_resolved_height();
-            assign_solid_mask(request.gen_params.mask_image, resolved_width, resolved_height);
-        }
-
-        float denoising_strength = j.value("denoising_strength", -1.f);
-        if (denoising_strength >= 0.f) {
-            request.gen_params.strength = std::min(denoising_strength, 1.0f);
-        }
-    }
-
-    if (j.contains("extra_images") && j["extra_images"].is_array()) {
-        for (const auto& extra_image : j["extra_images"]) {
-            if (!extra_image.is_string()) {
-                continue;
-            }
-            SDImageOwner image_owner;
-            if (decode_base64_image(extra_image.get<std::string>(),
-                                    3,
-                                    request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0,
-                                    request.gen_params.width_and_height_are_set() ? request.gen_params.height : 0,
-                                    image_owner)) {
-                const sd_image_t& image = image_owner.get();
-                request.gen_params.set_width_and_height_if_unset(image.width, image.height);
-                request.gen_params.ref_images.push_back(std::move(image_owner));
-            }
-        }
-    }
-
-    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
-    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
-        error_message = "invalid params";
-        return false;
-    }
-
-    return true;
-}
-
-void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
-    ServerRuntime* runtime = &rt;
-
-    auto sdapi_any2img = [runtime](const httplib::Request& req, httplib::Response& res, bool img2img) {
-        try {
-            if (req.body.empty()) {
-                res.status = 400;
-                res.set_content(R"({"error":"empty body"})", "application/json");
-                return;
-            }
-            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
-                res.status = 400;
-                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
-                return;
-            }
-
-            json j = json::parse(req.body);
-            ImgGenJobRequest request;
-            std::string error_message;
-            if (!build_sdapi_img_gen_request(j, *runtime, img2img, request, error_message)) {
-                res.status = 400;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
-
-            sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t();
-            SDImageVec results;
-            int num_results = 0;
-
-            {
-                std::lock_guard<std::mutex> lock(*runtime->sd_ctx_mutex);
-                sd_image_t* raw_results = generate_image(runtime->sd_ctx, &img_gen_params);
-                num_results             = request.gen_params.batch_count;
-                results.adopt(raw_results, num_results);
-            }
-
-            if (results.empty()) {
-                res.status = 500;
-                res.set_content(R"({"error":"generate_image returned no results"})", "application/json");
-                return;
-            }
-
-            json out;
-            out["images"]     = json::array();
-            out["parameters"] = j;
-            out["info"]       = "";
-
-            for (int i = 0; i < num_results; ++i) {
-                if (results[i].data == nullptr) {
-                    continue;
-                }
-
-                std::string params = request.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime->ctx_params,
-                                                            request.gen_params,
-                                                            request.gen_params.seed + i)
-                                         : "";
-                auto image_bytes   = encode_image_to_vector(EncodedImageFormat::PNG,
-                                                            results[i].data,
-                                                            results[i].width,
-                                                            results[i].height,
-                                                            results[i].channel,
-                                                            params);
-
-                if (image_bytes.empty()) {
-                    LOG_ERROR("write image to mem failed");
-                    continue;
-                }
-
-                out["images"].push_back(base64_encode(image_bytes));
-            }
-
-            res.set_content(out.dump(), "application/json");
-            res.status = 200;
-
-        } catch (const std::exception& e) {
-            res.status = 500;
-            json err;
-            err["error"]   = "server_error";
-            err["message"] = e.what();
-            res.set_content(err.dump(), "application/json");
-        }
-    };
-
-    svr.Post("/sdapi/v1/txt2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) {
-        sdapi_any2img(req, res, false);
-    });
-
-    svr.Post("/sdapi/v1/img2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) {
-        sdapi_any2img(req, res, true);
-    });
-
-    svr.Get("/sdapi/v1/loras", [runtime](const httplib::Request&, httplib::Response& res) {
-        refresh_lora_cache(*runtime);
-
-        json result = json::array();
-        {
-            std::lock_guard<std::mutex> lock(*runtime->lora_mutex);
-            for (const auto& e : *runtime->lora_cache) {
-                json item;
-                item["name"] = e.name;
-                item["path"] = e.path;
-                result.push_back(item);
-            }
-        }
-
-        res.set_content(result.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/upscalers", [runtime](const httplib::Request&, httplib::Response& res) {
-        refresh_upscaler_cache(*runtime);
-
-        auto make_builtin = [](const char* name) {
-            json item;
-            item["name"]       = name;
-            item["model_name"] = nullptr;
-            item["model_path"] = nullptr;
-            item["model_url"]  = nullptr;
-            item["scale"]      = 4;
-            return item;
-        };
-
-        json result = json::array();
-        result.push_back(make_builtin("None"));
-        result.push_back(make_builtin("Lanczos"));
-        result.push_back(make_builtin("Nearest"));
-
-        {
-            std::lock_guard<std::mutex> lock(*runtime->upscaler_mutex);
-            for (const auto& e : *runtime->upscaler_cache) {
-                json item;
-                item["name"]       = e.name;
-                item["model_name"] = e.model_name;
-                item["model_path"] = e.fullpath;
-                item["model_url"]  = nullptr;
-                item["scale"]      = e.scale;
-                result.push_back(item);
-            }
-        }
-
-        res.set_content(result.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/latent-upscale-modes", [](const httplib::Request&, httplib::Response& res) {
-        json result = json::array({
-            {{"name", "Latent"}},
-            {{"name", "Latent (nearest)"}},
-            {{"name", "Latent (nearest-exact)"}},
-            {{"name", "Latent (antialiased)"}},
-            {{"name", "Latent (bicubic)"}},
-            {{"name", "Latent (bicubic antialiased)"}},
-        });
-        res.set_content(result.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/samplers", [runtime](const httplib::Request&, httplib::Response& res) {
-        std::vector<std::string> sampler_names;
-        sampler_names.push_back("default");
-        for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) {
-            sampler_names.push_back(sd_sample_method_name((sample_method_t)i));
-        }
-        json r = json::array();
-        for (auto name : sampler_names) {
-            json entry;
-            entry["name"]    = name;
-            entry["aliases"] = json::array({name});
-            entry["options"] = json::object();
-            r.push_back(entry);
-        }
-        res.set_content(r.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/schedulers", [runtime](const httplib::Request&, httplib::Response& res) {
-        std::vector<std::string> scheduler_names;
-        scheduler_names.push_back("default");
-        for (int i = 0; i < SCHEDULER_COUNT; i++) {
-            scheduler_names.push_back(sd_scheduler_name((scheduler_t)i));
-        }
-        json r = json::array();
-        for (auto name : scheduler_names) {
-            json entry;
-            entry["name"]  = name;
-            entry["label"] = name;
-            r.push_back(entry);
-        }
-        res.set_content(r.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/sd-models", [runtime](const httplib::Request&, httplib::Response& res) {
-        fs::path model_path = resolve_display_model_path(*runtime);
-        json entry;
-        entry["title"]      = model_path.stem();
-        entry["model_name"] = model_path.stem();
-        entry["filename"]   = model_path.filename();
-        entry["hash"]       = "8888888888";
-        entry["sha256"]     = "8888888888888888888888888888888888888888888888888888888888888888";
-        entry["config"]     = nullptr;
-        json r              = json::array();
-        r.push_back(entry);
-        res.set_content(r.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/options", [runtime](const httplib::Request&, httplib::Response& res) {
-        fs::path model_path = resolve_display_model_path(*runtime);
-        json r;
-        r["samples_format"]      = "png";
-        r["sd_model_checkpoint"] = model_path.stem();
-        res.set_content(r.dump(), "application/json");
-    });
-}

examples/server/routes_sdcpp.cpp

@@ -1,588 +0,0 @@
-#include "routes.h"
-
-#include <algorithm>
-#include <cmath>
-#include <filesystem>
-
-#include "async_jobs.h"
-#include "common/common.h"
-
-namespace fs = std::filesystem;
-
-static bool parse_cache_mode(const std::string& mode_str, sd_cache_mode_t& mode_out) {
-    if (mode_str == "disabled") {
-        mode_out = SD_CACHE_DISABLED;
-        return true;
-    }
-    if (mode_str == "easycache") {
-        mode_out = SD_CACHE_EASYCACHE;
-        return true;
-    }
-    if (mode_str == "ucache") {
-        mode_out = SD_CACHE_UCACHE;
-        return true;
-    }
-    if (mode_str == "dbcache") {
-        mode_out = SD_CACHE_DBCACHE;
-        return true;
-    }
-    if (mode_str == "taylorseer") {
-        mode_out = SD_CACHE_TAYLORSEER;
-        return true;
-    }
-    if (mode_str == "cache-dit") {
-        mode_out = SD_CACHE_CACHE_DIT;
-        return true;
-    }
-    if (mode_str == "spectrum") {
-        mode_out = SD_CACHE_SPECTRUM;
-        return true;
-    }
-    return false;
-}
-
-static json finite_number_or_null(float value) {
-    return std::isfinite(value) ? json(value) : json(nullptr);
-}
-
-static const char* capability_scheduler_name(enum scheduler_t scheduler) {
-    return scheduler < SCHEDULER_COUNT ? sd_scheduler_name(scheduler) : "default";
-}
-
-static const char* capability_sample_method_name(enum sample_method_t sample_method) {
-    return sample_method < SAMPLE_METHOD_COUNT ? sd_sample_method_name(sample_method) : "default";
-}
-
-static json make_vae_tiling_json(const sd_tiling_params_t& params) {
-    return {
-        {"enabled", params.enabled},
-        {"tile_size_x", params.tile_size_x},
-        {"tile_size_y", params.tile_size_y},
-        {"target_overlap", params.target_overlap},
-        {"rel_size_x", params.rel_size_x},
-        {"rel_size_y", params.rel_size_y},
-    };
-}
-
-static fs::path resolve_display_model_path(const ServerRuntime& runtime) {
-    const auto& ctx = *runtime.ctx_params;
-    if (!ctx.model_path.empty()) {
-        return fs::path(ctx.model_path);
-    }
-    if (!ctx.diffusion_model_path.empty()) {
-        return fs::path(ctx.diffusion_model_path);
-    }
-    return {};
-}
-
-static json make_sample_params_json(const sd_sample_params_t& sample_params, const std::vector<int>& skip_layers) {
-    const auto& guidance = sample_params.guidance;
-    return {
-        {"scheduler", capability_scheduler_name(sample_params.scheduler)},
-        {"sample_method", capability_sample_method_name(sample_params.sample_method)},
-        {"sample_steps", sample_params.sample_steps},
-        {"eta", finite_number_or_null(sample_params.eta)},
-        {"shifted_timestep", sample_params.shifted_timestep},
-        {"flow_shift", finite_number_or_null(sample_params.flow_shift)},
-        {"guidance",
-         {
-             {"txt_cfg", guidance.txt_cfg},
-             {"img_cfg", finite_number_or_null(guidance.img_cfg)},
-             {"distilled_guidance", guidance.distilled_guidance},
-             {"slg",
-              {
-                  {"layers", skip_layers},
-                  {"layer_start", guidance.slg.layer_start},
-                  {"layer_end", guidance.slg.layer_end},
-                  {"scale", guidance.slg.scale},
-              }},
-         }},
-    };
-}
-
-static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const std::string& output_format) {
-    return {
-        {"prompt", defaults.prompt},
-        {"negative_prompt", defaults.negative_prompt},
-        {"clip_skip", defaults.clip_skip},
-        {"width", defaults.width > 0 ? defaults.width : 512},
-        {"height", defaults.height > 0 ? defaults.height : 512},
-        {"strength", defaults.strength},
-        {"seed", defaults.seed},
-        {"batch_count", defaults.batch_count},
-        {"auto_resize_ref_image", defaults.auto_resize_ref_image},
-        {"increase_ref_index", defaults.increase_ref_index},
-        {"control_strength", defaults.control_strength},
-        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)},
-        {"hires",
-         {
-             {"enabled", defaults.hires_enabled},
-             {"upscaler", defaults.hires_upscaler},
-             {"scale", defaults.hires_scale},
-             {"target_width", defaults.hires_width},
-             {"target_height", defaults.hires_height},
-             {"steps", defaults.hires_steps},
-             {"denoising_strength", defaults.hires_denoising_strength},
-             {"upscale_tile_size", defaults.hires_upscale_tile_size},
-         }},
-        {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)},
-        {"cache_mode", defaults.cache_mode},
-        {"cache_option", defaults.cache_option},
-        {"scm_mask", defaults.scm_mask},
-        {"scm_policy_dynamic", defaults.scm_policy_dynamic},
-        {"output_format", output_format},
-        {"output_compression", 100},
-    };
-}
-
-static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const std::string& output_format) {
-    return {
-        {"prompt", defaults.prompt},
-        {"negative_prompt", defaults.negative_prompt},
-        {"clip_skip", defaults.clip_skip},
-        {"width", defaults.width > 0 ? defaults.width : 512},
-        {"height", defaults.height > 0 ? defaults.height : 512},
-        {"strength", defaults.strength},
-        {"seed", defaults.seed},
-        {"video_frames", defaults.video_frames},
-        {"fps", defaults.fps},
-        {"moe_boundary", defaults.moe_boundary},
-        {"vace_strength", defaults.vace_strength},
-        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)},
-        {"high_noise_sample_params", make_sample_params_json(defaults.high_noise_sample_params, defaults.high_noise_skip_layers)},
-        {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)},
-        {"cache_mode", defaults.cache_mode},
-        {"cache_option", defaults.cache_option},
-        {"scm_mask", defaults.scm_mask},
-        {"scm_policy_dynamic", defaults.scm_policy_dynamic},
-        {"output_format", output_format},
-        {"output_compression", 100},
-    };
-}
-
-static json make_img_gen_features_json() {
-    return {
-        {"init_image", true},
-        {"mask_image", true},
-        {"control_image", true},
-        {"ref_images", true},
-        {"lora", true},
-        {"vae_tiling", true},
-        {"hires", true},
-        {"cache", true},
-        {"cancel_queued", true},
-        {"cancel_generating", false},
-    };
-}
-
-static json make_vid_gen_features_json() {
-    return {
-        {"init_image", true},
-        {"end_image", true},
-        {"control_frames", true},
-        {"high_noise_sample_params", true},
-        {"lora", true},
-        {"vae_tiling", true},
-        {"cache", true},
-        {"cancel_queued", true},
-        {"cancel_generating", false},
-    };
-}
-
-static json make_capabilities_json(ServerRuntime& runtime) {
-    refresh_lora_cache(runtime);
-    refresh_upscaler_cache(runtime);
-
-    AsyncJobManager& manager  = *runtime.async_job_manager;
-    const auto& defaults      = *runtime.default_gen_params;
-    const fs::path model_path = resolve_display_model_path(runtime);
-    const bool supports_img   = runtime_supports_generation_mode(runtime, IMG_GEN);
-    const bool supports_vid   = runtime_supports_generation_mode(runtime, VID_GEN);
-    json samplers             = json::array();
-    json schedulers           = json::array();
-    json image_output_formats = supported_img_output_formats();
-    json video_output_formats = supported_vid_output_formats();
-    json available_loras      = json::array();
-    json available_upscalers  = json::array();
-    json supported_modes      = json::array();
-
-    for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) {
-        samplers.push_back(sd_sample_method_name((sample_method_t)i));
-    }
-
-    for (int i = 0; i < SCHEDULER_COUNT; ++i) {
-        schedulers.push_back(sd_scheduler_name((scheduler_t)i));
-    }
-
-    {
-        std::lock_guard<std::mutex> lock(*runtime.lora_mutex);
-        for (const auto& entry : *runtime.lora_cache) {
-            available_loras.push_back({
-                {"name", entry.name},
-                {"path", entry.path},
-            });
-        }
-    }
-
-    available_upscalers.push_back({
-        {"name", "None"},
-    });
-    available_upscalers.push_back({
-        {"name", "Lanczos"},
-    });
-    available_upscalers.push_back({
-        {"name", "Nearest"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent (nearest)"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent (nearest-exact)"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent (antialiased)"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent (bicubic)"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent (bicubic antialiased)"},
-    });
-    {
-        std::lock_guard<std::mutex> lock(*runtime.upscaler_mutex);
-        for (const auto& entry : *runtime.upscaler_cache) {
-            available_upscalers.push_back({
-                {"name", entry.name},
-            });
-        }
-    }
-
-    if (supports_img) {
-        supported_modes.push_back("img_gen");
-    }
-    if (supports_vid) {
-        supported_modes.push_back("vid_gen");
-    }
-
-    std::string default_img_output_format = "png";
-    std::string default_vid_output_format = "avi";
-    if (!image_output_formats.empty()) {
-        default_img_output_format = image_output_formats[0].get<std::string>();
-    }
-    if (!video_output_formats.empty()) {
-        default_vid_output_format = video_output_formats[0].get<std::string>();
-    }
-
-    json defaults_by_mode       = json::object();
-    json output_formats_by_mode = json::object();
-    json features_by_mode       = json::object();
-    if (supports_img) {
-        defaults_by_mode["img_gen"]       = make_img_gen_defaults_json(defaults, default_img_output_format);
-        output_formats_by_mode["img_gen"] = image_output_formats;
-        features_by_mode["img_gen"]       = make_img_gen_features_json();
-    }
-    if (supports_vid) {
-        defaults_by_mode["vid_gen"]       = make_vid_gen_defaults_json(defaults, default_vid_output_format);
-        output_formats_by_mode["vid_gen"] = video_output_formats;
-        features_by_mode["vid_gen"]       = make_vid_gen_features_json();
-    }
-
-    json top_level_defaults       = json::object();
-    json top_level_output_formats = json::array();
-    json top_level_features       = {
-              {"cancel_queued", true},
-              {"cancel_generating", false},
-    };
-    std::string current_mode = "";
-    if (supports_img) {
-        current_mode             = "img_gen";
-        top_level_defaults       = defaults_by_mode["img_gen"];
-        top_level_output_formats = output_formats_by_mode["img_gen"];
-        top_level_features       = features_by_mode["img_gen"];
-    } else if (supports_vid) {
-        current_mode             = "vid_gen";
-        top_level_defaults       = defaults_by_mode["vid_gen"];
-        top_level_output_formats = output_formats_by_mode["vid_gen"];
-        top_level_features       = features_by_mode["vid_gen"];
-    }
-
-    json result;
-    result["model"] = {
-        {"name", model_path.filename().u8string()},
-        {"stem", model_path.stem().u8string()},
-        {"path", model_path.u8string()},
-    };
-    result["current_mode"]     = current_mode;
-    result["supported_modes"]  = supported_modes;
-    result["defaults"]         = top_level_defaults;
-    result["defaults_by_mode"] = defaults_by_mode;
-    result["limits"]           = {
-                  {"min_width", 64},
-                  {"max_width", 4096},
-                  {"min_height", 64},
-                  {"max_height", 4096},
-                  {"max_batch_count", 8},
-                  {"max_queue_size", manager.max_pending_jobs},
-    };
-    result["samplers"]               = samplers;
-    result["schedulers"]             = schedulers;
-    result["output_formats"]         = top_level_output_formats;
-    result["output_formats_by_mode"] = output_formats_by_mode;
-    result["features"]               = top_level_features;
-    result["features_by_mode"]       = features_by_mode;
-    result["loras"]                  = available_loras;
-    result["upscalers"]              = available_upscalers;
-    return result;
-}
-
-static bool parse_img_gen_request(const json& body,
-                                  ServerRuntime& runtime,
-                                  ImgGenJobRequest& request,
-                                  std::string& error_message) {
-    request.gen_params = *runtime.default_gen_params;
-
-    refresh_lora_cache(runtime);
-    if (!request.gen_params.from_json_str(body.dump(), [&](const std::string& path) {
-            return get_lora_full_path(runtime, path);
-        })) {
-        error_message = "invalid generation parameters";
-        return false;
-    }
-
-    std::string output_format = body.value("output_format", "png");
-    int output_compression    = body.value("output_compression", 100);
-    if (!assign_output_options(request, output_format, output_compression, true, error_message)) {
-        return false;
-    }
-    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
-    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
-        error_message = "invalid generation parameters";
-        return false;
-    }
-    return true;
-}
-
-static bool parse_vid_gen_request(const json& body,
-                                  ServerRuntime& runtime,
-                                  VidGenJobRequest& request,
-                                  std::string& error_message) {
-    request.gen_params = *runtime.default_gen_params;
-
-    refresh_lora_cache(runtime);
-    if (!request.gen_params.from_json_str(body.dump(), [&](const std::string& path) {
-            return get_lora_full_path(runtime, path);
-        })) {
-        error_message = "invalid generation parameters";
-        return false;
-    }
-
-    std::string output_format = body.value("output_format", "webm");
-    int output_compression    = body.value("output_compression", 100);
-    if (!assign_output_options(request, output_format, output_compression, error_message)) {
-        return false;
-    }
-    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
-    if (!request.gen_params.resolve_and_validate(VID_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
-        error_message = "invalid generation parameters";
-        return false;
-    }
-    return true;
-}
-
-void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
-    ServerRuntime* runtime = &rt;
-
-    svr.Get("/sdcpp/v1/capabilities", [runtime](const httplib::Request&, httplib::Response& res) {
-        res.status = 200;
-        res.set_content(make_capabilities_json(*runtime).dump(), "application/json");
-    });
-
-    svr.Post("/sdcpp/v1/img_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
-        try {
-            if (req.body.empty()) {
-                res.status = 400;
-                res.set_content(R"({"error":"empty body"})", "application/json");
-                return;
-            }
-            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
-                res.status = 400;
-                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
-                return;
-            }
-
-            json body = json::parse(req.body);
-            ImgGenJobRequest request;
-            std::string error_message;
-            if (!parse_img_gen_request(body, *runtime, request, error_message)) {
-                res.status = 400;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            AsyncJobManager& manager                = *runtime->async_job_manager;
-            std::shared_ptr<AsyncGenerationJob> job = std::make_shared<AsyncGenerationJob>();
-            job->kind                               = AsyncJobKind::ImgGen;
-            job->status                             = AsyncJobStatus::Queued;
-            job->created_at                         = unix_timestamp_now();
-            job->img_gen                            = std::move(request);
-
-            {
-                std::lock_guard<std::mutex> lock(manager.mutex);
-                purge_expired_jobs(manager);
-                if (count_pending_jobs(manager) >= manager.max_pending_jobs) {
-                    res.status = 429;
-                    res.set_content(R"({"error":"job queue is full"})", "application/json");
-                    return;
-                }
-                job->id               = make_async_job_id(manager);
-                manager.jobs[job->id] = job;
-                manager.queue.push_back(job->id);
-            }
-
-            manager.cv.notify_one();
-
-            json out;
-            out["id"]       = job->id;
-            out["kind"]     = async_job_kind_name(job->kind);
-            out["status"]   = async_job_status_name(job->status);
-            out["created"]  = job->created_at;
-            out["poll_url"] = "/sdcpp/v1/jobs/" + job->id;
-
-            res.status = 202;
-            res.set_content(out.dump(), "application/json");
-        } catch (const json::parse_error& e) {
-            res.status = 400;
-            res.set_content(json({{"error", "invalid json"}, {"message", e.what()}}).dump(), "application/json");
-        } catch (const std::exception& e) {
-            res.status = 500;
-            res.set_content(json({{"error", "server_error"}, {"message", e.what()}}).dump(), "application/json");
-        }
-    });
-
-    svr.Post("/sdcpp/v1/vid_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
-        try {
-            if (req.body.empty()) {
-                res.status = 400;
-                res.set_content(R"({"error":"empty body"})", "application/json");
-                return;
-            }
-            if (!runtime_supports_generation_mode(*runtime, VID_GEN)) {
-                res.status = 400;
-                res.set_content(json({{"error", unsupported_generation_mode_error(VID_GEN)}}).dump(), "application/json");
-                return;
-            }
-
-            json body = json::parse(req.body);
-            VidGenJobRequest request;
-            std::string error_message;
-            if (!parse_vid_gen_request(body, *runtime, request, error_message)) {
-                res.status = 400;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            AsyncJobManager& manager                = *runtime->async_job_manager;
-            std::shared_ptr<AsyncGenerationJob> job = std::make_shared<AsyncGenerationJob>();
-            job->kind                               = AsyncJobKind::VidGen;
-            job->status                             = AsyncJobStatus::Queued;
-            job->created_at                         = unix_timestamp_now();
-            job->vid_gen                            = std::move(request);
-
-            {
-                std::lock_guard<std::mutex> lock(manager.mutex);
-                purge_expired_jobs(manager);
-                if (count_pending_jobs(manager) >= manager.max_pending_jobs) {
-                    res.status = 429;
-                    res.set_content(R"({"error":"job queue is full"})", "application/json");
-                    return;
-                }
-                job->id               = make_async_job_id(manager);
-                manager.jobs[job->id] = job;
-                manager.queue.push_back(job->id);
-            }
-
-            manager.cv.notify_one();
-
-            json out;
-            out["id"]       = job->id;
-            out["kind"]     = async_job_kind_name(job->kind);
-            out["status"]   = async_job_status_name(job->status);
-            out["created"]  = job->created_at;
-            out["poll_url"] = "/sdcpp/v1/jobs/" + job->id;
-
-            res.status = 202;
-            res.set_content(out.dump(), "application/json");
-        } catch (const json::parse_error& e) {
-            res.status = 400;
-            res.set_content(json({{"error", "invalid json"}, {"message", e.what()}}).dump(), "application/json");
-        } catch (const std::exception& e) {
-            res.status = 500;
-            res.set_content(json({{"error", "server_error"}, {"message", e.what()}}).dump(), "application/json");
-        }
-    });
-
-    svr.Get(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+))", [runtime](const httplib::Request& req, httplib::Response& res) {
-        AsyncJobManager& manager = *runtime->async_job_manager;
-        std::lock_guard<std::mutex> lock(manager.mutex);
-        purge_expired_jobs(manager);
-
-        std::string job_id = req.matches[1];
-        auto it            = manager.jobs.find(job_id);
-        if (it == manager.jobs.end()) {
-            if (manager.expired_jobs.find(job_id) != manager.expired_jobs.end()) {
-                res.status = 410;
-                res.set_content(R"({"error":"job expired"})", "application/json");
-            } else {
-                res.status = 404;
-                res.set_content(R"({"error":"job not found"})", "application/json");
-            }
-            return;
-        }
-
-        res.status = 200;
-        res.set_content(make_async_job_json(manager, *it->second).dump(), "application/json");
-    });
-
-    svr.Post(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+)/cancel)", [runtime](const httplib::Request& req, httplib::Response& res) {
-        AsyncJobManager& manager = *runtime->async_job_manager;
-        std::lock_guard<std::mutex> lock(manager.mutex);
-        purge_expired_jobs(manager);
-
-        std::string job_id = req.matches[1];
-        auto it            = manager.jobs.find(job_id);
-        if (it == manager.jobs.end()) {
-            if (manager.expired_jobs.find(job_id) != manager.expired_jobs.end()) {
-                res.status = 410;
-                res.set_content(R"({"error":"job expired"})", "application/json");
-            } else {
-                res.status = 404;
-                res.set_content(R"({"error":"job not found"})", "application/json");
-            }
-            return;
-        }
-
-        auto& job = *it->second;
-        if (job.status == AsyncJobStatus::Queued) {
-            if (!cancel_queued_job(manager, job)) {
-                res.status = 409;
-                res.set_content(R"({"error":"job queue state changed before cancellation"})", "application/json");
-                return;
-            }
-            res.status = 200;
-            res.set_content(make_async_job_json(manager, job).dump(), "application/json");
-            return;
-        }
-
-        if (job.status == AsyncJobStatus::Generating) {
-            res.status = 409;
-            res.set_content(R"({"error":"job is currently generating and cannot be interrupted yet"})", "application/json");
-            return;
-        }
-
-        res.status = 200;
-        res.set_content(make_async_job_json(manager, job).dump(), "application/json");
-    });
-}

examples/server/runtime.cpp

@@ -1,332 +0,0 @@
-#include "runtime.h"
-
-#include <algorithm>
-#include <cctype>
-#include <chrono>
-#include <cstdlib>
-#include <filesystem>
-#include <mutex>
-#include <regex>
-#include <sstream>
-
-#include "common/common.h"
-#include "common/log.h"
-
-namespace fs = std::filesystem;
-
-static std::string lower_ascii(std::string value) {
-    std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
-        return static_cast<char>(std::tolower(c));
-    });
-    return value;
-}
-
-static bool is_supported_model_ext(const fs::path& p) {
-    auto ext = lower_ascii(p.extension().string());
-    return ext == ".gguf" || ext == ".pt" || ext == ".pth" || ext == ".safetensors";
-}
-
-static const std::string k_base64_chars =
-    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-    "abcdefghijklmnopqrstuvwxyz"
-    "0123456789+/";
-
-std::string base64_encode(const std::vector<uint8_t>& bytes) {
-    std::string ret;
-    int val  = 0;
-    int valb = -6;
-    for (uint8_t c : bytes) {
-        val = (val << 8) + c;
-        valb += 8;
-        while (valb >= 0) {
-            ret.push_back(k_base64_chars[(val >> valb) & 0x3F]);
-            valb -= 6;
-        }
-    }
-    if (valb > -6) {
-        ret.push_back(k_base64_chars[((val << 8) >> (valb + 8)) & 0x3F]);
-    }
-    while (ret.size() % 4) {
-        ret.push_back('=');
-    }
-    return ret;
-}
-
-std::string normalize_output_format(std::string output_format) {
-    std::transform(output_format.begin(), output_format.end(), output_format.begin(),
-                   [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
-    return output_format;
-}
-
-std::vector<std::string> supported_img_output_formats(bool allow_webp) {
-    std::vector<std::string> formats = {"png", "jpeg"};
-#ifdef SD_USE_WEBP
-    if (allow_webp) {
-        formats.push_back("webp");
-    }
-#else
-    (void)allow_webp;
-#endif
-    return formats;
-}
-
-std::vector<std::string> supported_vid_output_formats() {
-    std::vector<std::string> formats;
-#ifdef SD_USE_WEBM
-    formats.push_back("webm");
-#endif
-#ifdef SD_USE_WEBP
-    formats.push_back("webp");
-#endif
-    formats.push_back("avi");
-    return formats;
-}
-
-static std::string valid_vid_output_formats_message() {
-    const std::vector<std::string> formats = supported_vid_output_formats();
-
-    std::string message = "invalid output_format, must be one of [";
-    for (size_t i = 0; i < formats.size(); ++i) {
-        if (i > 0) {
-            message += ", ";
-        }
-        message += formats[i];
-    }
-    message += "]";
-    return message;
-}
-
-bool assign_output_options(ImgGenJobRequest& request,
-                           std::string output_format,
-                           int output_compression,
-                           bool allow_webp,
-                           std::string& error_message) {
-    request.output_format      = normalize_output_format(std::move(output_format));
-    request.output_compression = std::clamp(output_compression, 0, 100);
-
-    const std::vector<std::string> valid_formats = supported_img_output_formats(allow_webp);
-    const bool valid_format                      = std::find(valid_formats.begin(),
-                                                             valid_formats.end(),
-                                                             request.output_format) != valid_formats.end();
-    if (!valid_format) {
-        error_message = "invalid output_format, must be one of [";
-        for (size_t i = 0; i < valid_formats.size(); ++i) {
-            if (i > 0) {
-                error_message += ", ";
-            }
-            error_message += valid_formats[i];
-        }
-        error_message += "]";
-        return false;
-    }
-
-    return true;
-}
-
-bool assign_output_options(VidGenJobRequest& request,
-                           std::string output_format,
-                           int output_compression,
-                           std::string& error_message) {
-    request.output_format      = normalize_output_format(std::move(output_format));
-    request.output_compression = std::clamp(output_compression, 0, 100);
-
-    if (request.output_format == "avi") {
-        return true;
-    }
-
-    if (request.output_format == "webm") {
-#ifdef SD_USE_WEBM
-        return true;
-#else
-        error_message = valid_vid_output_formats_message();
-        return false;
-#endif
-    }
-
-    if (request.output_format == "webp") {
-#ifdef SD_USE_WEBP
-        return true;
-#else
-        error_message = valid_vid_output_formats_message();
-        return false;
-#endif
-    }
-
-    error_message = valid_vid_output_formats_message();
-    return false;
-}
-
-std::string video_mime_type(const std::string& output_format) {
-    if (output_format == "webm") {
-        return "video/webm";
-    }
-    if (output_format == "webp") {
-        return "image/webp";
-    }
-    return "video/x-msvideo";
-}
-
-bool runtime_supports_generation_mode(const ServerRuntime& runtime, SDMode mode) {
-    if (mode == VID_GEN) {
-        return sd_ctx_supports_video_generation(runtime.sd_ctx);
-    }
-    if (mode == IMG_GEN) {
-        return sd_ctx_supports_image_generation(runtime.sd_ctx);
-    }
-    return true;
-}
-
-std::string unsupported_generation_mode_error(SDMode mode) {
-    if (mode == VID_GEN) {
-        return "loaded model does not support vid_gen";
-    }
-    if (mode == IMG_GEN) {
-        return "loaded model does not support img_gen";
-    }
-    return "loaded model does not support requested mode";
-}
-
-ArgOptions SDSvrParams::get_options() {
-    ArgOptions options;
-
-    options.string_options = {
-        {"-l", "--listen-ip", "server listen ip (default: 127.0.0.1)", &listen_ip},
-        {"", "--serve-html-path", "path to HTML file to serve at root (optional)", &serve_html_path},
-    };
-
-    options.int_options = {
-        {"", "--listen-port", "server listen port (default: 1234)", &listen_port},
-    };
-
-    options.bool_options = {
-        {"-v", "--verbose", "print extra info", true, &verbose},
-        {"", "--color", "colors the logging tags according to level", true, &color},
-    };
-
-    auto on_help_arg = [&](int, const char**, int) {
-        normal_exit = true;
-        return -1;
-    };
-
-    options.manual_options = {
-        {"-h", "--help", "show this help message and exit", on_help_arg},
-    };
-    return options;
-}
-
-bool SDSvrParams::validate() {
-    if (listen_ip.empty()) {
-        LOG_ERROR("error: the following arguments are required: listen_ip");
-        return false;
-    }
-
-    if (listen_port < 0 || listen_port > 65535) {
-        LOG_ERROR("error: listen_port should be in the range [0, 65535]");
-        return false;
-    }
-
-    if (!serve_html_path.empty() && !fs::exists(serve_html_path)) {
-        LOG_ERROR("error: serve_html_path file does not exist: %s", serve_html_path.c_str());
-        return false;
-    }
-    return true;
-}
-
-bool SDSvrParams::resolve_and_validate() {
-    if (!validate()) {
-        return false;
-    }
-    return true;
-}
-
-std::string SDSvrParams::to_string() const {
-    std::ostringstream oss;
-    oss << "SDSvrParams {\n"
-        << "  listen_ip: " << listen_ip << ",\n"
-        << "  listen_port: \"" << listen_port << "\",\n"
-        << "  serve_html_path: \"" << serve_html_path << "\",\n"
-        << "}";
-    return oss.str();
-}
-
-void refresh_lora_cache(ServerRuntime& rt) {
-    std::vector<LoraEntry> new_cache;
-
-    fs::path lora_dir = rt.ctx_params->lora_model_dir;
-    if (fs::exists(lora_dir) && fs::is_directory(lora_dir)) {
-        for (auto& entry : fs::recursive_directory_iterator(lora_dir)) {
-            if (!entry.is_regular_file()) {
-                continue;
-            }
-            const fs::path& p = entry.path();
-            if (!is_supported_model_ext(p)) {
-                continue;
-            }
-
-            LoraEntry lora_entry;
-            lora_entry.name     = p.stem().u8string();
-            lora_entry.fullpath = p.u8string();
-            std::string rel     = p.lexically_relative(lora_dir).u8string();
-            std::replace(rel.begin(), rel.end(), '\\', '/');
-            lora_entry.path = rel;
-
-            new_cache.push_back(std::move(lora_entry));
-        }
-    }
-
-    std::sort(new_cache.begin(), new_cache.end(), [](const LoraEntry& a, const LoraEntry& b) {
-        return a.path < b.path;
-    });
-
-    {
-        std::lock_guard<std::mutex> lock(*rt.lora_mutex);
-        *rt.lora_cache = std::move(new_cache);
-    }
-}
-
-std::string get_lora_full_path(ServerRuntime& rt, const std::string& path) {
-    std::lock_guard<std::mutex> lock(*rt.lora_mutex);
-    auto it = std::find_if(rt.lora_cache->begin(), rt.lora_cache->end(),
-                           [&](const LoraEntry& entry) { return entry.path == path; });
-    return it != rt.lora_cache->end() ? it->fullpath : "";
-}
-
-void refresh_upscaler_cache(ServerRuntime& rt) {
-    std::vector<UpscalerEntry> new_cache;
-
-    fs::path upscaler_dir = rt.ctx_params->hires_upscalers_dir;
-    if (fs::exists(upscaler_dir) && fs::is_directory(upscaler_dir)) {
-        for (auto& entry : fs::directory_iterator(upscaler_dir)) {
-            if (!entry.is_regular_file()) {
-                continue;
-            }
-            const fs::path& p = entry.path();
-            if (!is_supported_model_ext(p)) {
-                continue;
-            }
-
-            UpscalerEntry upscaler_entry;
-            upscaler_entry.name       = p.stem().u8string();
-            upscaler_entry.fullpath   = fs::absolute(p).lexically_normal().u8string();
-            upscaler_entry.model_name = "ESRGAN_4x";
-            upscaler_entry.path       = p.filename().u8string();
-
-            new_cache.push_back(std::move(upscaler_entry));
-        }
-    }
-
-    std::sort(new_cache.begin(), new_cache.end(), [](const UpscalerEntry& a, const UpscalerEntry& b) {
-        return a.name < b.name;
-    });
-
-    {
-        std::lock_guard<std::mutex> lock(*rt.upscaler_mutex);
-        *rt.upscaler_cache = std::move(new_cache);
-    }
-}
-
-int64_t unix_timestamp_now() {
-    return std::chrono::duration_cast<std::chrono::seconds>(
-               std::chrono::system_clock::now().time_since_epoch())
-        .count();
-}

examples/server/runtime.h

@@ -1,100 +0,0 @@
-#pragma once
-
-#include <algorithm>
-#include <cstdint>
-#include <mutex>
-#include <string>
-#include <vector>
-
-#include <json.hpp>
-#include "common/common.h"
-#include "common/resource_owners.hpp"
-#include "stable-diffusion.h"
-
-using json = nlohmann::json;
-
-struct ArgOptions;
-struct SDContextParams;
-struct AsyncJobManager;
-
-struct SDSvrParams {
-    std::string listen_ip = "127.0.0.1";
-    int listen_port       = 1234;
-    std::string serve_html_path;
-    bool normal_exit = false;
-    bool verbose     = false;
-    bool color       = false;
-
-    ArgOptions get_options();
-    bool validate();
-    bool resolve_and_validate();
-    std::string to_string() const;
-};
-
-struct LoraEntry {
-    std::string name;
-    std::string path;
-    std::string fullpath;
-};
-
-struct UpscalerEntry {
-    std::string name;
-    std::string path;
-    std::string fullpath;
-    std::string model_name;
-    int scale = 4;
-};
-
-struct ServerRuntime {
-    sd_ctx_t* sd_ctx;
-    std::mutex* sd_ctx_mutex;
-    const SDSvrParams* svr_params;
-    const SDContextParams* ctx_params;
-    const SDGenerationParams* default_gen_params;
-    std::vector<LoraEntry>* lora_cache;
-    std::mutex* lora_mutex;
-    std::vector<UpscalerEntry>* upscaler_cache;
-    std::mutex* upscaler_mutex;
-    AsyncJobManager* async_job_manager;
-};
-
-struct ImgGenJobRequest {
-    SDGenerationParams gen_params;
-    std::string output_format = "png";
-    int output_compression    = 100;
-
-    sd_img_gen_params_t to_sd_img_gen_params_t() {
-        return gen_params.to_sd_img_gen_params_t();
-    }
-};
-
-struct VidGenJobRequest {
-    SDGenerationParams gen_params;
-    std::string output_format = "webm";
-    int output_compression    = 100;
-
-    sd_vid_gen_params_t to_sd_vid_gen_params_t() {
-        return gen_params.to_sd_vid_gen_params_t();
-    }
-};
-
-std::string base64_encode(const std::vector<uint8_t>& bytes);
-std::string normalize_output_format(std::string output_format);
-std::vector<std::string> supported_img_output_formats(bool allow_webp = true);
-std::vector<std::string> supported_vid_output_formats();
-bool assign_output_options(ImgGenJobRequest& request,
-                           std::string output_format,
-                           int output_compression,
-                           bool allow_webp,
-                           std::string& error_message);
-bool assign_output_options(VidGenJobRequest& request,
-                           std::string output_format,
-                           int output_compression,
-                           std::string& error_message);
-std::string video_mime_type(const std::string& output_format);
-bool runtime_supports_generation_mode(const ServerRuntime& runtime, SDMode mode);
-std::string unsupported_generation_mode_error(SDMode mode);
-void refresh_lora_cache(ServerRuntime& rt);
-std::string get_lora_full_path(ServerRuntime& rt, const std::string& path);
-void refresh_upscaler_cache(ServerRuntime& rt);
-int64_t unix_timestamp_now();

face_detect.py

@@ -1,88 +1,88 @@
-import os
-import sys
-
-import numpy as np
-import torch
-from diffusers.utils import load_image
-# pip install insightface==0.7.3
-from insightface.app import FaceAnalysis
-from insightface.data import get_image as ins_get_image
-from safetensors.torch import save_file
-
-### 
-# https://github.com/cubiq/ComfyUI_IPAdapter_plus/issues/165#issue-2055829543
-###
-class FaceAnalysis2(FaceAnalysis):
-    # NOTE: allows setting det_size for each detection call.
-    # the model allows it but the wrapping code from insightface
-    # doesn't show it, and people end up loading duplicate models
-    # for different sizes where there is absolutely no need to
-    def get(self, img, max_num=0, det_size=(640, 640)):
-        if det_size is not None:
-            self.det_model.input_size = det_size
-
-        return super().get(img, max_num)
-
-def analyze_faces(face_analysis: FaceAnalysis, img_data: np.ndarray, det_size=(640, 640)):
-    # NOTE: try detect faces, if no faces detected, lower det_size until it does
-    detection_sizes = [None] + [(size, size) for size in range(640, 256, -64)] + [(256, 256)]
-
-    for size in detection_sizes:
-        faces = face_analysis.get(img_data, det_size=size)
-        if len(faces) > 0:
-            return faces
-
-    return []
-
-if __name__ == "__main__":
-    #face_detector = FaceAnalysis2(providers=['CUDAExecutionProvider'], allowed_modules=['detection', 'recognition'])
-    face_detector = FaceAnalysis2(providers=['CPUExecutionProvider'], allowed_modules=['detection', 'recognition'])
-    face_detector.prepare(ctx_id=0, det_size=(640, 640))
-    #input_folder_name = './scarletthead_woman'
-    input_folder_name = sys.argv[1]
-    image_basename_list = os.listdir(input_folder_name)
-    image_path_list = sorted([os.path.join(input_folder_name, basename) for basename in image_basename_list])
-
-    input_id_images = []
-    for image_path in image_path_list:
-        input_id_images.append(load_image(image_path))
-    
-    id_embed_list = []
-    
-    for img in input_id_images:
-        img = np.array(img)
-        img = img[:, :, ::-1]
-        faces = analyze_faces(face_detector, img)
-        if len(faces) > 0:
-            id_embed_list.append(torch.from_numpy((faces[0]['embedding'])))
-    
-    if len(id_embed_list) == 0:
-        raise ValueError(f"No face detected in input image pool")
-    
-    id_embeds = torch.stack(id_embed_list)    
-    
-    # for r in id_embeds:
-    #     print(r)
-    # #torch.save(id_embeds, input_folder_name+'/id_embeds.pt');
-    # weights = dict()
-    # weights["id_embeds"] = id_embeds
-    # save_file(weights, input_folder_name+'/id_embeds.safetensors')
-
-    binary_data = id_embeds.numpy().tobytes()
-    two = 4
-    zero = 0
-    one = 1
-    tensor_name = "id_embeds"
-# Write binary data to a file
-    with open(input_folder_name+'/id_embeds.bin', "wb") as f:
-        f.write(two.to_bytes(4, byteorder='little'))
-        f.write((len(tensor_name)).to_bytes(4, byteorder='little'))
-        f.write(zero.to_bytes(4, byteorder='little'))
-        f.write((id_embeds.shape[1]).to_bytes(4, byteorder='little'))
-        f.write((id_embeds.shape[0]).to_bytes(4, byteorder='little'))
-        f.write(one.to_bytes(4, byteorder='little'))
-        f.write(one.to_bytes(4, byteorder='little'))
-        f.write(tensor_name.encode('ascii'))
-        f.write(binary_data)
-
+import os
+import sys
+
+import numpy as np
+import torch
+from diffusers.utils import load_image
+# pip install insightface==0.7.3
+from insightface.app import FaceAnalysis
+from insightface.data import get_image as ins_get_image
+from safetensors.torch import save_file
+
+### 
+# https://github.com/cubiq/ComfyUI_IPAdapter_plus/issues/165#issue-2055829543
+###
+class FaceAnalysis2(FaceAnalysis):
+    # NOTE: allows setting det_size for each detection call.
+    # the model allows it but the wrapping code from insightface
+    # doesn't show it, and people end up loading duplicate models
+    # for different sizes where there is absolutely no need to
+    def get(self, img, max_num=0, det_size=(640, 640)):
+        if det_size is not None:
+            self.det_model.input_size = det_size
+
+        return super().get(img, max_num)
+
+def analyze_faces(face_analysis: FaceAnalysis, img_data: np.ndarray, det_size=(640, 640)):
+    # NOTE: try detect faces, if no faces detected, lower det_size until it does
+    detection_sizes = [None] + [(size, size) for size in range(640, 256, -64)] + [(256, 256)]
+
+    for size in detection_sizes:
+        faces = face_analysis.get(img_data, det_size=size)
+        if len(faces) > 0:
+            return faces
+
+    return []
+
+if __name__ == "__main__":
+    #face_detector = FaceAnalysis2(providers=['CUDAExecutionProvider'], allowed_modules=['detection', 'recognition'])
+    face_detector = FaceAnalysis2(providers=['CPUExecutionProvider'], allowed_modules=['detection', 'recognition'])
+    face_detector.prepare(ctx_id=0, det_size=(640, 640))
+    #input_folder_name = './scarletthead_woman'
+    input_folder_name = sys.argv[1]
+    image_basename_list = os.listdir(input_folder_name)
+    image_path_list = sorted([os.path.join(input_folder_name, basename) for basename in image_basename_list])
+
+    input_id_images = []
+    for image_path in image_path_list:
+        input_id_images.append(load_image(image_path))
+    
+    id_embed_list = []
+    
+    for img in input_id_images:
+        img = np.array(img)
+        img = img[:, :, ::-1]
+        faces = analyze_faces(face_detector, img)
+        if len(faces) > 0:
+            id_embed_list.append(torch.from_numpy((faces[0]['embedding'])))
+    
+    if len(id_embed_list) == 0:
+        raise ValueError(f"No face detected in input image pool")
+    
+    id_embeds = torch.stack(id_embed_list)    
+    
+    # for r in id_embeds:
+    #     print(r)
+    # #torch.save(id_embeds, input_folder_name+'/id_embeds.pt');
+    # weights = dict()
+    # weights["id_embeds"] = id_embeds
+    # save_file(weights, input_folder_name+'/id_embeds.safetensors')
+
+    binary_data = id_embeds.numpy().tobytes()
+    two = 4
+    zero = 0
+    one = 1
+    tensor_name = "id_embeds"
+# Write binary data to a file
+    with open(input_folder_name+'/id_embeds.bin', "wb") as f:
+        f.write(two.to_bytes(4, byteorder='little'))
+        f.write((len(tensor_name)).to_bytes(4, byteorder='little'))
+        f.write(zero.to_bytes(4, byteorder='little'))
+        f.write((id_embeds.shape[1]).to_bytes(4, byteorder='little'))
+        f.write((id_embeds.shape[0]).to_bytes(4, byteorder='little'))
+        f.write(one.to_bytes(4, byteorder='little'))
+        f.write(one.to_bytes(4, byteorder='little'))
+        f.write(tensor_name.encode('ascii'))
+        f.write(binary_data)
+
     

format-code.sh

@@ -1,6 +1,4 @@
-for f in src/*.cpp src/*.h src/*.hpp src/tokenizers/*.h src/tokenizers/*.cpp src/tokenizers/vocab/*.h src/tokenizers/vocab/*.cpp \
-         src/model_io/*.h src/model_io/*.cpp examples/cli/*.cpp examples/cli/*.h examples/server/*.cpp \
-         examples/common/*.hpp examples/common/*.h examples/common/*.cpp; do
+for f in *.cpp *.h *.hpp examples/cli/*.cpp examples/common/*.hpp examples/cli/*.h examples/server/*.cpp; do
   [[ "$f" == vocab* ]] && continue
   echo "formatting '$f'"
   # if [ "$f" != "stable-diffusion.h" ]; then

gguf_reader.hpp

@@ -1,5 +1,5 @@
-#ifndef __SD_MODEL_IO_GGUF_READER_EXT_H__
-#define __SD_MODEL_IO_GGUF_READER_EXT_H__
+#ifndef __GGUF_READER_HPP__
+#define __GGUF_READER_HPP__
 
 #include <cstdint>
 #include <fstream>
@@ -59,9 +59,6 @@ private:
         if (!safe_read(fin, key_len))
             return false;
 
-        if (key_len > 4096)
-            return false;
-
         std::string key(key_len, '\0');
         if (!safe_read(fin, (char*)key.data(), key_len))
             return false;
@@ -231,4 +228,4 @@ public:
     size_t data_offset() const { return data_offset_; }
 };
 
-#endif  // __SD_MODEL_IO_GGUF_READER_EXT_H__
+#endif  // __GGUF_READER_HPP__

latent-preview.h

@@ -1,300 +1,234 @@
-#include <algorithm>
-#include <cstddef>
-#include <cstdint>
-#include "ggml.h"
-#include "tensor.hpp"
-
-const float wan_21_latent_rgb_proj[16][3] = {
-    {0.015123f, -0.148418f, 0.479828f},
-    {0.003652f, -0.010680f, -0.037142f},
-    {0.212264f, 0.063033f, 0.016779f},
-    {0.232999f, 0.406476f, 0.220125f},
-    {-0.051864f, -0.082384f, -0.069396f},
-    {0.085005f, -0.161492f, 0.010689f},
-    {-0.245369f, -0.506846f, -0.117010f},
-    {-0.151145f, 0.017721f, 0.007207f},
-    {-0.293239f, -0.207936f, -0.421135f},
-    {-0.187721f, 0.050783f, 0.177649f},
-    {-0.013067f, 0.265964f, 0.166578f},
-    {0.028327f, 0.109329f, 0.108642f},
-    {-0.205343f, 0.043991f, 0.148914f},
-    {0.014307f, -0.048647f, -0.007219f},
-    {0.217150f, 0.053074f, 0.319923f},
-    {0.155357f, 0.083156f, 0.064780f}};
-float wan_21_latent_rgb_bias[3] = {-0.270270f, -0.234976f, -0.456853f};
-
-const float wan_22_latent_rgb_proj[48][3] = {
-    {0.017126f, -0.027230f, -0.019257f},
-    {-0.113739f, -0.028715f, -0.022885f},
-    {-0.000106f, 0.021494f, 0.004629f},
-    {-0.013273f, -0.107137f, -0.033638f},
-    {-0.000381f, 0.000279f, 0.025877f},
-    {-0.014216f, -0.003975f, 0.040528f},
-    {0.001638f, -0.000748f, 0.011022f},
-    {0.029238f, -0.006697f, 0.035933f},
-    {0.021641f, -0.015874f, 0.040531f},
-    {-0.101984f, -0.070160f, -0.028855f},
-    {0.033207f, -0.021068f, 0.002663f},
-    {-0.104711f, 0.121673f, 0.102981f},
-    {0.082647f, -0.004991f, 0.057237f},
-    {-0.027375f, 0.031581f, 0.006868f},
-    {-0.045434f, 0.029444f, 0.019287f},
-    {-0.046572f, -0.012537f, 0.006675f},
-    {0.074709f, 0.033690f, 0.025289f},
-    {-0.008251f, -0.002745f, -0.006999f},
-    {0.012685f, -0.061856f, -0.048658f},
-    {0.042304f, -0.007039f, 0.000295f},
-    {-0.007644f, -0.060843f, -0.033142f},
-    {0.159909f, 0.045628f, 0.367541f},
-    {0.095171f, 0.086438f, 0.010271f},
-    {0.006812f, 0.019643f, 0.029637f},
-    {0.003467f, -0.010705f, 0.014252f},
-    {-0.099681f, -0.066272f, -0.006243f},
-    {0.047357f, 0.037040f, 0.000185f},
-    {-0.041797f, -0.089225f, -0.032257f},
-    {0.008928f, 0.017028f, 0.018684f},
-    {-0.042255f, 0.016045f, 0.006849f},
-    {0.011268f, 0.036462f, 0.037387f},
-    {0.011553f, -0.016375f, -0.048589f},
-    {0.046266f, -0.027189f, 0.056979f},
-    {0.009640f, -0.017576f, 0.030324f},
-    {-0.045794f, -0.036083f, -0.010616f},
-    {0.022418f, 0.039783f, -0.032939f},
-    {-0.052714f, -0.015525f, 0.007438f},
-    {0.193004f, 0.223541f, 0.264175f},
-    {-0.059406f, -0.008188f, 0.022867f},
-    {-0.156742f, -0.263791f, -0.007385f},
-    {-0.015717f, 0.016570f, 0.033969f},
-    {0.037969f, 0.109835f, 0.200449f},
-    {-0.000782f, -0.009566f, -0.008058f},
-    {0.010709f, 0.052960f, -0.044195f},
-    {0.017271f, 0.045839f, 0.034569f},
-    {0.009424f, 0.013088f, -0.001714f},
-    {-0.024805f, -0.059378f, -0.033756f},
-    {-0.078293f, 0.029070f, 0.026129f}};
-float wan_22_latent_rgb_bias[3] = {0.013160f, -0.096492f, -0.071323f};
-
-const float flux_latent_rgb_proj[16][3] = {
-    {-0.041168f, 0.019917f, 0.097253f},
-    {0.028096f, 0.026730f, 0.129576f},
-    {0.065618f, -0.067950f, -0.014651f},
-    {-0.012998f, -0.014762f, 0.081251f},
-    {0.078567f, 0.059296f, -0.024687f},
-    {-0.015987f, -0.003697f, 0.005012f},
-    {0.033605f, 0.138999f, 0.068517f},
-    {-0.024450f, -0.063567f, -0.030101f},
-    {-0.040194f, -0.016710f, 0.127185f},
-    {0.112681f, 0.088764f, -0.041940f},
-    {-0.023498f, 0.093664f, 0.025543f},
-    {0.082899f, 0.048320f, 0.007491f},
-    {0.075712f, 0.074139f, 0.081965f},
-    {-0.143501f, 0.018263f, -0.136138f},
-    {-0.025767f, -0.082035f, -0.040023f},
-    {-0.111849f, -0.055589f, -0.032361f}};
-float flux_latent_rgb_bias[3] = {0.024600f, -0.006937f, -0.008089f};
-
-const float flux2_latent_rgb_proj[32][3] = {
-    {0.000736f, -0.008385f, -0.019710f},
-    {-0.001352f, -0.016392f, 0.020693f},
-    {-0.006376f, 0.002428f, 0.036736f},
-    {0.039384f, 0.074167f, 0.119789f},
-    {0.007464f, -0.005705f, -0.004734f},
-    {-0.004086f, 0.005287f, -0.000409f},
-    {-0.032835f, 0.050802f, -0.028120f},
-    {-0.003158f, -0.000835f, 0.000406f},
-    {-0.112840f, -0.084337f, -0.023083f},
-    {0.001462f, -0.006656f, 0.000549f},
-    {-0.009980f, -0.007480f, 0.009702f},
-    {0.032540f, 0.000214f, -0.061388f},
-    {0.011023f, 0.000694f, 0.007143f},
-    {-0.001468f, -0.006723f, -0.001678f},
-    {-0.005921f, -0.010320f, -0.003907f},
-    {-0.028434f, 0.027584f, 0.018457f},
-    {0.014349f, 0.011523f, 0.000441f},
-    {0.009874f, 0.003081f, 0.001507f},
-    {0.002218f, 0.005712f, 0.001563f},
-    {0.053010f, -0.019844f, 0.008683f},
-    {-0.002507f, 0.005384f, 0.000938f},
-    {-0.002177f, -0.011366f, 0.003559f},
-    {-0.000261f, 0.015121f, -0.003240f},
-    {-0.003944f, -0.002083f, 0.005043f},
-    {-0.009138f, 0.011336f, 0.003781f},
-    {0.011429f, 0.003985f, -0.003855f},
-    {0.010518f, -0.005586f, 0.010131f},
-    {0.007883f, 0.002912f, -0.001473f},
-    {-0.003318f, -0.003160f, 0.003684f},
-    {-0.034560f, -0.008740f, 0.012996f},
-    {0.000166f, 0.001079f, -0.012153f},
-    {0.017772f, 0.000937f, -0.011953f}};
-float flux2_latent_rgb_bias[3] = {-0.028738f, -0.098463f, -0.107619f};
-
-// This one was taken straight from
-// https://github.com/Stability-AI/sd3.5/blob/8565799a3b41eb0c7ba976d18375f0f753f56402/sd3_impls.py#L288-L303
-// (MiT Licence)
-const float sd3_latent_rgb_proj[16][3] = {
-    {-0.0645f, 0.0177f, 0.1052f},
-    {0.0028f, 0.0312f, 0.0650f},
-    {0.1848f, 0.0762f, 0.0360f},
-    {0.0944f, 0.0360f, 0.0889f},
-    {0.0897f, 0.0506f, -0.0364f},
-    {-0.0020f, 0.1203f, 0.0284f},
-    {0.0855f, 0.0118f, 0.0283f},
-    {-0.0539f, 0.0658f, 0.1047f},
-    {-0.0057f, 0.0116f, 0.0700f},
-    {-0.0412f, 0.0281f, -0.0039f},
-    {0.1106f, 0.1171f, 0.1220f},
-    {-0.0248f, 0.0682f, -0.0481f},
-    {0.0815f, 0.0846f, 0.1207f},
-    {-0.0120f, -0.0055f, -0.0867f},
-    {-0.0749f, -0.0634f, -0.0456f},
-    {-0.1418f, -0.1457f, -0.1259f},
-};
-float sd3_latent_rgb_bias[3] = {0, 0, 0};
-
-const float sdxl_latent_rgb_proj[4][3] = {
-    {0.258303f, 0.277640f, 0.329699f},
-    {-0.299701f, 0.105446f, 0.014194f},
-    {0.050522f, 0.186163f, -0.143257f},
-    {-0.211938f, -0.149892f, -0.080036f}};
-float sdxl_latent_rgb_bias[3] = {0.144381f, -0.033313f, 0.007061f};
-
-const float sd_latent_rgb_proj[4][3] = {
-    {0.337366f, 0.216344f, 0.257386f},
-    {0.165636f, 0.386828f, 0.046994f},
-    {-0.267803f, 0.237036f, 0.223517f},
-    {-0.178022f, -0.200862f, -0.678514f}};
-float sd_latent_rgb_bias[3] = {-0.017478f, -0.055834f, -0.105825f};
-
-void preview_latent_video(uint8_t* buffer, ggml_tensor* latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size) {
-    size_t buffer_head = 0;
-
-    uint32_t latent_width  = static_cast<uint32_t>(latents->ne[0]);
-    uint32_t latent_height = static_cast<uint32_t>(latents->ne[1]);
-    uint32_t dim           = static_cast<uint32_t>(latents->ne[ggml_n_dims(latents) - 1]);
-    uint32_t frames        = 1;
-    if (ggml_n_dims(latents) == 4) {
-        frames = static_cast<uint32_t>(latents->ne[2]);
-    }
-
-    uint32_t rgb_width  = latent_width * patch_size;
-    uint32_t rgb_height = latent_height * patch_size;
-
-    uint32_t unpatched_dim = dim / (patch_size * patch_size);
-
-    for (uint32_t k = 0; k < frames; k++) {
-        for (uint32_t rgb_x = 0; rgb_x < rgb_width; rgb_x++) {
-            for (uint32_t rgb_y = 0; rgb_y < rgb_height; rgb_y++) {
-                int latent_x = rgb_x / patch_size;
-                int latent_y = rgb_y / patch_size;
-
-                int channel_offset = 0;
-                if (patch_size > 1) {
-                    channel_offset = ((rgb_y % patch_size) * patch_size + (rgb_x % patch_size));
-                }
-
-                size_t latent_id = (latent_x * latents->nb[0] + latent_y * latents->nb[1] + k * latents->nb[2]);
-
-                // should be incremented by 1 for each pixel
-                size_t pixel_id = k * rgb_width * rgb_height + rgb_y * rgb_width + rgb_x;
-
-                float r = 0, g = 0, b = 0;
-                if (latent_rgb_proj != nullptr) {
-                    for (uint32_t d = 0; d < unpatched_dim; d++) {
-                        float value = *(float*)((char*)latents->data + latent_id + (d * patch_size * patch_size + channel_offset) * latents->nb[ggml_n_dims(latents) - 1]);
-                        r += value * latent_rgb_proj[d][0];
-                        g += value * latent_rgb_proj[d][1];
-                        b += value * latent_rgb_proj[d][2];
-                    }
-                } else {
-                    // interpret first 3 channels as RGB
-                    r = *(float*)((char*)latents->data + latent_id + 0 * latents->nb[ggml_n_dims(latents) - 1]);
-                    g = *(float*)((char*)latents->data + latent_id + 1 * latents->nb[ggml_n_dims(latents) - 1]);
-                    b = *(float*)((char*)latents->data + latent_id + 2 * latents->nb[ggml_n_dims(latents) - 1]);
-                }
-                if (latent_rgb_bias != nullptr) {
-                    // bias
-                    r += latent_rgb_bias[0];
-                    g += latent_rgb_bias[1];
-                    b += latent_rgb_bias[2];
-                }
-                // change range
-                r = r * .5f + .5f;
-                g = g * .5f + .5f;
-                b = b * .5f + .5f;
-
-                // clamp rgb values to [0,1] range
-                r = r >= 0 ? r <= 1 ? r : 1 : 0;
-                g = g >= 0 ? g <= 1 ? g : 1 : 0;
-                b = b >= 0 ? b <= 1 ? b : 1 : 0;
-
-                buffer[pixel_id * 3 + 0] = (uint8_t)(r * 255);
-                buffer[pixel_id * 3 + 1] = (uint8_t)(g * 255);
-                buffer[pixel_id * 3 + 2] = (uint8_t)(b * 255);
-            }
-        }
-    }
-}
-
-static inline bool preview_latent_tensor_is_video(const sd::Tensor<float>& latents) {
-    return latents.dim() == 5;
-}
-
-void preview_latent_video(uint8_t* buffer, const sd::Tensor<float>& latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size) {
-    uint32_t latent_width  = static_cast<uint32_t>(latents.shape()[0]);
-    uint32_t latent_height = static_cast<uint32_t>(latents.shape()[1]);
-    bool is_video          = preview_latent_tensor_is_video(latents);
-    uint32_t frames        = is_video ? static_cast<uint32_t>(latents.shape()[2]) : 1;
-    uint32_t dim           = is_video ? static_cast<uint32_t>(latents.shape()[3]) : static_cast<uint32_t>(latents.shape()[2]);
-
-    uint32_t rgb_width     = latent_width * patch_size;
-    uint32_t rgb_height    = latent_height * patch_size;
-    uint32_t unpatched_dim = dim / (patch_size * patch_size);
-
-    for (uint32_t k = 0; k < frames; k++) {
-        for (uint32_t rgb_x = 0; rgb_x < rgb_width; rgb_x++) {
-            for (uint32_t rgb_y = 0; rgb_y < rgb_height; rgb_y++) {
-                uint32_t latent_x = rgb_x / patch_size;
-                uint32_t latent_y = rgb_y / patch_size;
-
-                uint32_t channel_offset = 0;
-                if (patch_size > 1) {
-                    channel_offset = ((rgb_y % patch_size) * patch_size + (rgb_x % patch_size));
-                }
-
-                size_t pixel_id   = k * rgb_width * rgb_height + rgb_y * rgb_width + rgb_x;
-                auto latent_value = [&](uint32_t latent_channel) -> float {
-                    return is_video
-                               ? latents.values()[latent_x + latent_width * (latent_y + latent_height * (k + frames * latent_channel))]
-                               : latents.values()[latent_x + latent_width * (latent_y + latent_height * latent_channel)];
-                };
-
-                float r = 0.f, g = 0.f, b = 0.f;
-                if (latent_rgb_proj != nullptr) {
-                    for (uint32_t d = 0; d < unpatched_dim; d++) {
-                        uint32_t latent_channel = d * patch_size * patch_size + channel_offset;
-                        float value             = latent_value(latent_channel);
-                        r += value * latent_rgb_proj[d][0];
-                        g += value * latent_rgb_proj[d][1];
-                        b += value * latent_rgb_proj[d][2];
-                    }
-                } else {
-                    r = latent_value(0);
-                    g = latent_value(1);
-                    b = latent_value(2);
-                }
-                if (latent_rgb_bias != nullptr) {
-                    r += latent_rgb_bias[0];
-                    g += latent_rgb_bias[1];
-                    b += latent_rgb_bias[2];
-                }
-                r = std::min(1.0f, std::max(0.0f, r * .5f + .5f));
-                g = std::min(1.0f, std::max(0.0f, g * .5f + .5f));
-                b = std::min(1.0f, std::max(0.0f, b * .5f + .5f));
-
-                buffer[pixel_id * 3 + 0] = (uint8_t)(r * 255);
-                buffer[pixel_id * 3 + 1] = (uint8_t)(g * 255);
-                buffer[pixel_id * 3 + 2] = (uint8_t)(b * 255);
-            }
-        }
-    }
-}
+#include <cstddef>
+#include <cstdint>
+#include "ggml.h"
+
+const float wan_21_latent_rgb_proj[16][3] = {
+    {0.015123f, -0.148418f, 0.479828f},
+    {0.003652f, -0.010680f, -0.037142f},
+    {0.212264f, 0.063033f, 0.016779f},
+    {0.232999f, 0.406476f, 0.220125f},
+    {-0.051864f, -0.082384f, -0.069396f},
+    {0.085005f, -0.161492f, 0.010689f},
+    {-0.245369f, -0.506846f, -0.117010f},
+    {-0.151145f, 0.017721f, 0.007207f},
+    {-0.293239f, -0.207936f, -0.421135f},
+    {-0.187721f, 0.050783f, 0.177649f},
+    {-0.013067f, 0.265964f, 0.166578f},
+    {0.028327f, 0.109329f, 0.108642f},
+    {-0.205343f, 0.043991f, 0.148914f},
+    {0.014307f, -0.048647f, -0.007219f},
+    {0.217150f, 0.053074f, 0.319923f},
+    {0.155357f, 0.083156f, 0.064780f}};
+float wan_21_latent_rgb_bias[3] = {-0.270270f, -0.234976f, -0.456853f};
+
+const float wan_22_latent_rgb_proj[48][3] = {
+    {0.017126f, -0.027230f, -0.019257f},
+    {-0.113739f, -0.028715f, -0.022885f},
+    {-0.000106f, 0.021494f, 0.004629f},
+    {-0.013273f, -0.107137f, -0.033638f},
+    {-0.000381f, 0.000279f, 0.025877f},
+    {-0.014216f, -0.003975f, 0.040528f},
+    {0.001638f, -0.000748f, 0.011022f},
+    {0.029238f, -0.006697f, 0.035933f},
+    {0.021641f, -0.015874f, 0.040531f},
+    {-0.101984f, -0.070160f, -0.028855f},
+    {0.033207f, -0.021068f, 0.002663f},
+    {-0.104711f, 0.121673f, 0.102981f},
+    {0.082647f, -0.004991f, 0.057237f},
+    {-0.027375f, 0.031581f, 0.006868f},
+    {-0.045434f, 0.029444f, 0.019287f},
+    {-0.046572f, -0.012537f, 0.006675f},
+    {0.074709f, 0.033690f, 0.025289f},
+    {-0.008251f, -0.002745f, -0.006999f},
+    {0.012685f, -0.061856f, -0.048658f},
+    {0.042304f, -0.007039f, 0.000295f},
+    {-0.007644f, -0.060843f, -0.033142f},
+    {0.159909f, 0.045628f, 0.367541f},
+    {0.095171f, 0.086438f, 0.010271f},
+    {0.006812f, 0.019643f, 0.029637f},
+    {0.003467f, -0.010705f, 0.014252f},
+    {-0.099681f, -0.066272f, -0.006243f},
+    {0.047357f, 0.037040f, 0.000185f},
+    {-0.041797f, -0.089225f, -0.032257f},
+    {0.008928f, 0.017028f, 0.018684f},
+    {-0.042255f, 0.016045f, 0.006849f},
+    {0.011268f, 0.036462f, 0.037387f},
+    {0.011553f, -0.016375f, -0.048589f},
+    {0.046266f, -0.027189f, 0.056979f},
+    {0.009640f, -0.017576f, 0.030324f},
+    {-0.045794f, -0.036083f, -0.010616f},
+    {0.022418f, 0.039783f, -0.032939f},
+    {-0.052714f, -0.015525f, 0.007438f},
+    {0.193004f, 0.223541f, 0.264175f},
+    {-0.059406f, -0.008188f, 0.022867f},
+    {-0.156742f, -0.263791f, -0.007385f},
+    {-0.015717f, 0.016570f, 0.033969f},
+    {0.037969f, 0.109835f, 0.200449f},
+    {-0.000782f, -0.009566f, -0.008058f},
+    {0.010709f, 0.052960f, -0.044195f},
+    {0.017271f, 0.045839f, 0.034569f},
+    {0.009424f, 0.013088f, -0.001714f},
+    {-0.024805f, -0.059378f, -0.033756f},
+    {-0.078293f, 0.029070f, 0.026129f}};
+float wan_22_latent_rgb_bias[3] = {0.013160f, -0.096492f, -0.071323f};
+
+const float flux_latent_rgb_proj[16][3] = {
+    {-0.041168f, 0.019917f, 0.097253f},
+    {0.028096f, 0.026730f, 0.129576f},
+    {0.065618f, -0.067950f, -0.014651f},
+    {-0.012998f, -0.014762f, 0.081251f},
+    {0.078567f, 0.059296f, -0.024687f},
+    {-0.015987f, -0.003697f, 0.005012f},
+    {0.033605f, 0.138999f, 0.068517f},
+    {-0.024450f, -0.063567f, -0.030101f},
+    {-0.040194f, -0.016710f, 0.127185f},
+    {0.112681f, 0.088764f, -0.041940f},
+    {-0.023498f, 0.093664f, 0.025543f},
+    {0.082899f, 0.048320f, 0.007491f},
+    {0.075712f, 0.074139f, 0.081965f},
+    {-0.143501f, 0.018263f, -0.136138f},
+    {-0.025767f, -0.082035f, -0.040023f},
+    {-0.111849f, -0.055589f, -0.032361f}};
+float flux_latent_rgb_bias[3] = {0.024600f, -0.006937f, -0.008089f};
+
+const float flux2_latent_rgb_proj[32][3] = {
+    {0.000736f, -0.008385f, -0.019710f},
+    {-0.001352f, -0.016392f, 0.020693f},
+    {-0.006376f, 0.002428f, 0.036736f},
+    {0.039384f, 0.074167f, 0.119789f},
+    {0.007464f, -0.005705f, -0.004734f},
+    {-0.004086f, 0.005287f, -0.000409f},
+    {-0.032835f, 0.050802f, -0.028120f},
+    {-0.003158f, -0.000835f, 0.000406f},
+    {-0.112840f, -0.084337f, -0.023083f},
+    {0.001462f, -0.006656f, 0.000549f},
+    {-0.009980f, -0.007480f, 0.009702f},
+    {0.032540f, 0.000214f, -0.061388f},
+    {0.011023f, 0.000694f, 0.007143f},
+    {-0.001468f, -0.006723f, -0.001678f},
+    {-0.005921f, -0.010320f, -0.003907f},
+    {-0.028434f, 0.027584f, 0.018457f},
+    {0.014349f, 0.011523f, 0.000441f},
+    {0.009874f, 0.003081f, 0.001507f},
+    {0.002218f, 0.005712f, 0.001563f},
+    {0.053010f, -0.019844f, 0.008683f},
+    {-0.002507f, 0.005384f, 0.000938f},
+    {-0.002177f, -0.011366f, 0.003559f},
+    {-0.000261f, 0.015121f, -0.003240f},
+    {-0.003944f, -0.002083f, 0.005043f},
+    {-0.009138f, 0.011336f, 0.003781f},
+    {0.011429f, 0.003985f, -0.003855f},
+    {0.010518f, -0.005586f, 0.010131f},
+    {0.007883f, 0.002912f, -0.001473f},
+    {-0.003318f, -0.003160f, 0.003684f},
+    {-0.034560f, -0.008740f, 0.012996f},
+    {0.000166f, 0.001079f, -0.012153f},
+    {0.017772f, 0.000937f, -0.011953f}};
+float flux2_latent_rgb_bias[3] = {-0.028738f, -0.098463f, -0.107619f};
+
+// This one was taken straight from
+// https://github.com/Stability-AI/sd3.5/blob/8565799a3b41eb0c7ba976d18375f0f753f56402/sd3_impls.py#L288-L303
+// (MiT Licence)
+const float sd3_latent_rgb_proj[16][3] = {
+    {-0.0645f, 0.0177f, 0.1052f},
+    {0.0028f, 0.0312f, 0.0650f},
+    {0.1848f, 0.0762f, 0.0360f},
+    {0.0944f, 0.0360f, 0.0889f},
+    {0.0897f, 0.0506f, -0.0364f},
+    {-0.0020f, 0.1203f, 0.0284f},
+    {0.0855f, 0.0118f, 0.0283f},
+    {-0.0539f, 0.0658f, 0.1047f},
+    {-0.0057f, 0.0116f, 0.0700f},
+    {-0.0412f, 0.0281f, -0.0039f},
+    {0.1106f, 0.1171f, 0.1220f},
+    {-0.0248f, 0.0682f, -0.0481f},
+    {0.0815f, 0.0846f, 0.1207f},
+    {-0.0120f, -0.0055f, -0.0867f},
+    {-0.0749f, -0.0634f, -0.0456f},
+    {-0.1418f, -0.1457f, -0.1259f},
+};
+float sd3_latent_rgb_bias[3] = {0, 0, 0};
+
+const float sdxl_latent_rgb_proj[4][3] = {
+    {0.258303f, 0.277640f, 0.329699f},
+    {-0.299701f, 0.105446f, 0.014194f},
+    {0.050522f, 0.186163f, -0.143257f},
+    {-0.211938f, -0.149892f, -0.080036f}};
+float sdxl_latent_rgb_bias[3] = {0.144381f, -0.033313f, 0.007061f};
+
+const float sd_latent_rgb_proj[4][3] = {
+    {0.337366f, 0.216344f, 0.257386f},
+    {0.165636f, 0.386828f, 0.046994f},
+    {-0.267803f, 0.237036f, 0.223517f},
+    {-0.178022f, -0.200862f, -0.678514f}};
+float sd_latent_rgb_bias[3] = {-0.017478f, -0.055834f, -0.105825f};
+
+void preview_latent_video(uint8_t* buffer, struct ggml_tensor* latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size) {
+    size_t buffer_head = 0;
+
+    uint32_t latent_width  = static_cast<uint32_t>(latents->ne[0]);
+    uint32_t latent_height = static_cast<uint32_t>(latents->ne[1]);
+    uint32_t dim           = static_cast<uint32_t>(latents->ne[ggml_n_dims(latents) - 1]);
+    uint32_t frames        = 1;
+    if (ggml_n_dims(latents) == 4) {
+        frames = static_cast<uint32_t>(latents->ne[2]);
+    }
+
+    uint32_t rgb_width  = latent_width * patch_size;
+    uint32_t rgb_height = latent_height * patch_size;
+
+    uint32_t unpatched_dim = dim / (patch_size * patch_size);
+
+    for (uint32_t k = 0; k < frames; k++) {
+        for (uint32_t rgb_x = 0; rgb_x < rgb_width; rgb_x++) {
+            for (uint32_t rgb_y = 0; rgb_y < rgb_height; rgb_y++) {
+                int latent_x = rgb_x / patch_size;
+                int latent_y = rgb_y / patch_size;
+
+                int channel_offset = 0;
+                if (patch_size > 1) {
+                    channel_offset = ((rgb_y % patch_size) * patch_size + (rgb_x % patch_size));
+                }
+
+                size_t latent_id = (latent_x * latents->nb[0] + latent_y * latents->nb[1] + k * latents->nb[2]);
+
+                // should be incremented by 1 for each pixel
+                size_t pixel_id = k * rgb_width * rgb_height + rgb_y * rgb_width + rgb_x;
+
+                float r = 0, g = 0, b = 0;
+                if (latent_rgb_proj != nullptr) {
+                    for (uint32_t d = 0; d < unpatched_dim; d++) {
+                        float value = *(float*)((char*)latents->data + latent_id + (d * patch_size * patch_size + channel_offset) * latents->nb[ggml_n_dims(latents) - 1]);
+                        r += value * latent_rgb_proj[d][0];
+                        g += value * latent_rgb_proj[d][1];
+                        b += value * latent_rgb_proj[d][2];
+                    }
+                } else {
+                    // interpret first 3 channels as RGB
+                    r = *(float*)((char*)latents->data + latent_id + 0 * latents->nb[ggml_n_dims(latents) - 1]);
+                    g = *(float*)((char*)latents->data + latent_id + 1 * latents->nb[ggml_n_dims(latents) - 1]);
+                    b = *(float*)((char*)latents->data + latent_id + 2 * latents->nb[ggml_n_dims(latents) - 1]);
+                }
+                if (latent_rgb_bias != nullptr) {
+                    // bias
+                    r += latent_rgb_bias[0];
+                    g += latent_rgb_bias[1];
+                    b += latent_rgb_bias[2];
+                }
+                // change range
+                r = r * .5f + .5f;
+                g = g * .5f + .5f;
+                b = b * .5f + .5f;
+
+                // clamp rgb values to [0,1] range
+                r = r >= 0 ? r <= 1 ? r : 1 : 0;
+                g = g >= 0 ? g <= 1 ? g : 1 : 0;
+                b = b >= 0 ? b <= 1 ? b : 1 : 0;
+
+                buffer[pixel_id * 3 + 0] = (uint8_t)(r * 255);
+                buffer[pixel_id * 3 + 1] = (uint8_t)(g * 255);
+                buffer[pixel_id * 3 + 2] = (uint8_t)(b * 255);
+            }
+        }
+    }
+}

lora.hpp

@@ -9,7 +9,7 @@
 struct LoraModel : public GGMLRunner {
     std::string lora_id;
     float multiplier = 1.0f;
-    std::unordered_map<std::string, ggml_tensor*> lora_tensors;
+    std::unordered_map<std::string, struct ggml_tensor*> lora_tensors;
     std::map<ggml_tensor*, ggml_tensor*> original_tensor_to_final_tensor;
     std::set<std::string> applied_lora_tensors;
     std::string file_path;
@@ -22,11 +22,10 @@ struct LoraModel : public GGMLRunner {
 
     LoraModel(const std::string& lora_id,
               ggml_backend_t backend,
-              ggml_backend_t params_backend,
               const std::string& file_path = "",
               std::string prefix           = "",
               SDVersion version            = VERSION_COUNT)
-        : lora_id(lora_id), file_path(file_path), GGMLRunner(backend, params_backend) {
+        : lora_id(lora_id), file_path(file_path), GGMLRunner(backend, false) {
         prefix = "lora." + prefix;
         if (!model_loader.init_from_file_and_convert_name(file_path, prefix, version)) {
             load_failed = true;
@@ -77,13 +76,13 @@ struct LoraModel : public GGMLRunner {
         }
 
         for (const auto& pair : tensors_to_create) {
-            const auto& name   = pair.first;
-            const auto& ts     = pair.second;
-            ggml_tensor* real  = ggml_new_tensor(params_ctx,
-                                                 ts.type,
-                                                 ts.n_dims,
-                                                 ts.ne);
-            lora_tensors[name] = real;
+            const auto& name         = pair.first;
+            const auto& ts           = pair.second;
+            struct ggml_tensor* real = ggml_new_tensor(params_ctx,
+                                                       ts.type,
+                                                       ts.n_dims,
+                                                       ts.ne);
+            lora_tensors[name]       = real;
         }
 
         alloc_params_buffer();
@@ -130,7 +129,7 @@ struct LoraModel : public GGMLRunner {
         }
     }
 
-    ggml_tensor* get_lora_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_backend_t backend) {
+    ggml_tensor* get_lora_weight_diff(const std::string& model_tensor_name, ggml_context* ctx) {
         ggml_tensor* updown = nullptr;
         int index           = 0;
         while (true) {
@@ -153,17 +152,17 @@ struct LoraModel : public GGMLRunner {
 
             auto iter = lora_tensors.find(lora_up_name);
             if (iter != lora_tensors.end()) {
-                lora_up = ggml_ext_cast_f32(ctx, backend, iter->second);
+                lora_up = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(lora_mid_name);
             if (iter != lora_tensors.end()) {
-                lora_mid = ggml_ext_cast_f32(ctx, backend, iter->second);
+                lora_mid = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(lora_down_name);
             if (iter != lora_tensors.end()) {
-                lora_down = ggml_ext_cast_f32(ctx, backend, iter->second);
+                lora_down = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             if (lora_up == nullptr || lora_down == nullptr) {
@@ -196,7 +195,7 @@ struct LoraModel : public GGMLRunner {
             scale_value *= multiplier;
 
             auto curr_updown = ggml_ext_merge_lora(ctx, lora_down, lora_up, lora_mid);
-            curr_updown      = ggml_ext_scale(ctx, curr_updown, scale_value, true);
+            curr_updown      = ggml_scale_inplace(ctx, curr_updown, scale_value);
 
             if (updown == nullptr) {
                 updown = curr_updown;
@@ -209,7 +208,7 @@ struct LoraModel : public GGMLRunner {
         return updown;
     }
 
-    ggml_tensor* get_raw_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_backend_t backend) {
+    ggml_tensor* get_raw_weight_diff(const std::string& model_tensor_name, ggml_context* ctx) {
         ggml_tensor* updown = nullptr;
         int index           = 0;
         while (true) {
@@ -226,7 +225,7 @@ struct LoraModel : public GGMLRunner {
 
             auto iter = lora_tensors.find(diff_name);
             if (iter != lora_tensors.end()) {
-                curr_updown = ggml_ext_cast_f32(ctx, backend, iter->second);
+                curr_updown = ggml_ext_cast_f32(ctx, iter->second);
             } else {
                 break;
             }
@@ -236,7 +235,7 @@ struct LoraModel : public GGMLRunner {
             float scale_value = 1.0f;
             scale_value *= multiplier;
 
-            curr_updown = ggml_ext_scale(ctx, curr_updown, scale_value, true);
+            curr_updown = ggml_scale_inplace(ctx, curr_updown, scale_value);
 
             if (updown == nullptr) {
                 updown = curr_updown;
@@ -249,7 +248,7 @@ struct LoraModel : public GGMLRunner {
         return updown;
     }
 
-    ggml_tensor* get_loha_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_backend_t backend) {
+    ggml_tensor* get_loha_weight_diff(const std::string& model_tensor_name, ggml_context* ctx) {
         ggml_tensor* updown = nullptr;
         int index           = 0;
         while (true) {
@@ -277,33 +276,33 @@ struct LoraModel : public GGMLRunner {
 
             auto iter = lora_tensors.find(hada_1_down_name);
             if (iter != lora_tensors.end()) {
-                hada_1_down = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_1_down = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(hada_1_up_name);
             if (iter != lora_tensors.end()) {
-                hada_1_up = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_1_up = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(hada_1_mid_name);
             if (iter != lora_tensors.end()) {
-                hada_1_mid = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_1_mid = ggml_ext_cast_f32(ctx, iter->second);
                 hada_1_up  = ggml_cont(ctx, ggml_transpose(ctx, hada_1_up));
             }
 
             iter = lora_tensors.find(hada_2_down_name);
             if (iter != lora_tensors.end()) {
-                hada_2_down = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_2_down = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(hada_2_up_name);
             if (iter != lora_tensors.end()) {
-                hada_2_up = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_2_up = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(hada_2_mid_name);
             if (iter != lora_tensors.end()) {
-                hada_2_mid = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_2_mid = ggml_ext_cast_f32(ctx, iter->second);
                 hada_2_up  = ggml_cont(ctx, ggml_transpose(ctx, hada_2_up));
             }
 
@@ -338,10 +337,10 @@ struct LoraModel : public GGMLRunner {
             }
             scale_value *= multiplier;
 
-            ggml_tensor* updown_1 = ggml_ext_merge_lora(ctx, hada_1_down, hada_1_up, hada_1_mid);
-            ggml_tensor* updown_2 = ggml_ext_merge_lora(ctx, hada_2_down, hada_2_up, hada_2_mid);
-            auto curr_updown      = ggml_mul_inplace(ctx, updown_1, updown_2);
-            curr_updown           = ggml_ext_scale(ctx, curr_updown, scale_value, true);
+            struct ggml_tensor* updown_1 = ggml_ext_merge_lora(ctx, hada_1_down, hada_1_up, hada_1_mid);
+            struct ggml_tensor* updown_2 = ggml_ext_merge_lora(ctx, hada_2_down, hada_2_up, hada_2_mid);
+            auto curr_updown             = ggml_mul_inplace(ctx, updown_1, updown_2);
+            curr_updown                  = ggml_scale_inplace(ctx, curr_updown, scale_value);
             if (updown == nullptr) {
                 updown = curr_updown;
             } else {
@@ -352,7 +351,7 @@ struct LoraModel : public GGMLRunner {
         return updown;
     }
 
-    ggml_tensor* get_lokr_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_backend_t backend) {
+    ggml_tensor* get_lokr_weight_diff(const std::string& model_tensor_name, ggml_context* ctx) {
         ggml_tensor* updown = nullptr;
         int index           = 0;
         while (true) {
@@ -379,24 +378,24 @@ struct LoraModel : public GGMLRunner {
 
             auto iter = lora_tensors.find(lokr_w1_name);
             if (iter != lora_tensors.end()) {
-                lokr_w1 = ggml_ext_cast_f32(ctx, backend, iter->second);
+                lokr_w1 = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(lokr_w2_name);
             if (iter != lora_tensors.end()) {
-                lokr_w2 = ggml_ext_cast_f32(ctx, backend, iter->second);
+                lokr_w2 = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             int64_t rank = 1;
             if (lokr_w1 == nullptr) {
                 iter = lora_tensors.find(lokr_w1_a_name);
                 if (iter != lora_tensors.end()) {
-                    lokr_w1_a = ggml_ext_cast_f32(ctx, backend, iter->second);
+                    lokr_w1_a = ggml_ext_cast_f32(ctx, iter->second);
                 }
 
                 iter = lora_tensors.find(lokr_w1_b_name);
                 if (iter != lora_tensors.end()) {
-                    lokr_w1_b = ggml_ext_cast_f32(ctx, backend, iter->second);
+                    lokr_w1_b = ggml_ext_cast_f32(ctx, iter->second);
                 }
 
                 if (lokr_w1_a == nullptr || lokr_w1_b == nullptr) {
@@ -411,12 +410,12 @@ struct LoraModel : public GGMLRunner {
             if (lokr_w2 == nullptr) {
                 iter = lora_tensors.find(lokr_w2_a_name);
                 if (iter != lora_tensors.end()) {
-                    lokr_w2_a = ggml_ext_cast_f32(ctx, backend, iter->second);
+                    lokr_w2_a = ggml_ext_cast_f32(ctx, iter->second);
                 }
 
                 iter = lora_tensors.find(lokr_w2_b_name);
                 if (iter != lora_tensors.end()) {
-                    lokr_w2_b = ggml_ext_cast_f32(ctx, backend, iter->second);
+                    lokr_w2_b = ggml_ext_cast_f32(ctx, iter->second);
                 }
 
                 if (lokr_w2_a == nullptr || lokr_w2_b == nullptr) {
@@ -457,7 +456,7 @@ struct LoraModel : public GGMLRunner {
             scale_value *= multiplier;
 
             auto curr_updown = ggml_ext_kronecker(ctx, lokr_w1, lokr_w2);
-            curr_updown      = ggml_ext_scale(ctx, curr_updown, scale_value, true);
+            curr_updown      = ggml_scale_inplace(ctx, curr_updown, scale_value);
 
             if (updown == nullptr) {
                 updown = curr_updown;
@@ -469,23 +468,23 @@ struct LoraModel : public GGMLRunner {
         return updown;
     }
 
-    ggml_tensor* get_weight_diff(const std::string& model_tensor_name, ggml_backend_t backend, ggml_context* ctx, ggml_tensor* model_tensor, bool with_lora_and_lokr = true) {
+    ggml_tensor* get_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_tensor* model_tensor, bool with_lora = true) {
         // lora
         ggml_tensor* diff = nullptr;
-        if (with_lora_and_lokr) {
-            diff = get_lora_weight_diff(model_tensor_name, ctx, backend);
+        if (with_lora) {
+            diff = get_lora_weight_diff(model_tensor_name, ctx);
         }
         // diff
         if (diff == nullptr) {
-            diff = get_raw_weight_diff(model_tensor_name, ctx, backend);
+            diff = get_raw_weight_diff(model_tensor_name, ctx);
         }
         // loha
         if (diff == nullptr) {
-            diff = get_loha_weight_diff(model_tensor_name, ctx, backend);
+            diff = get_loha_weight_diff(model_tensor_name, ctx);
         }
         // lokr
-        if (diff == nullptr && with_lora_and_lokr) {
-            diff = get_lokr_weight_diff(model_tensor_name, ctx, backend);
+        if (diff == nullptr) {
+            diff = get_lokr_weight_diff(model_tensor_name, ctx);
         }
         if (diff != nullptr) {
             if (ggml_nelements(diff) < ggml_nelements(model_tensor)) {
@@ -503,7 +502,6 @@ struct LoraModel : public GGMLRunner {
     }
 
     ggml_tensor* get_out_diff(ggml_context* ctx,
-                              ggml_backend_t backend,
                               ggml_tensor* x,
                               WeightAdapter::ForwardParams forward_params,
                               const std::string& model_tensor_name) {
@@ -516,108 +514,6 @@ struct LoraModel : public GGMLRunner {
             } else {
                 key = model_tensor_name + "." + std::to_string(index);
             }
-            bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D;
-
-            std::string lokr_w1_name   = "lora." + key + ".lokr_w1";
-            std::string lokr_w1_a_name = "lora." + key + ".lokr_w1_a";
-            // if either of these is found, then we have a lokr lora
-            auto iter   = lora_tensors.find(lokr_w1_name);
-            auto iter_a = lora_tensors.find(lokr_w1_a_name);
-            if (iter != lora_tensors.end() || iter_a != lora_tensors.end()) {
-                std::string lokr_w1_b_name = "lora." + key + ".lokr_w1_b";
-                std::string lokr_w2_name   = "lora." + key + ".lokr_w2";
-                std::string lokr_w2_a_name = "lora." + key + ".lokr_w2_a";
-                std::string lokr_w2_b_name = "lora." + key + ".lokr_w2_b";
-                std::string alpha_name     = "lora." + key + ".alpha";
-
-                ggml_tensor* lokr_w1   = nullptr;
-                ggml_tensor* lokr_w1_a = nullptr;
-                ggml_tensor* lokr_w1_b = nullptr;
-                ggml_tensor* lokr_w2   = nullptr;
-                ggml_tensor* lokr_w2_a = nullptr;
-                ggml_tensor* lokr_w2_b = nullptr;
-
-                if (iter != lora_tensors.end()) {
-                    lokr_w1 = iter->second;
-                }
-                iter = iter_a;
-                if (iter != lora_tensors.end()) {
-                    lokr_w1_a = iter->second;
-                }
-                iter = lora_tensors.find(lokr_w1_b_name);
-                if (iter != lora_tensors.end()) {
-                    lokr_w1_b = iter->second;
-                }
-
-                iter = lora_tensors.find(lokr_w2_name);
-                if (iter != lora_tensors.end()) {
-                    lokr_w2 = iter->second;
-                    if (is_conv2d && lokr_w2->type != GGML_TYPE_F16) {
-                        lokr_w2 = ggml_cast(ctx, lokr_w2, GGML_TYPE_F16);
-                    }
-                }
-                iter = lora_tensors.find(lokr_w2_a_name);
-                if (iter != lora_tensors.end()) {
-                    lokr_w2_a = iter->second;
-                    if (is_conv2d && lokr_w2_a->type != GGML_TYPE_F16) {
-                        lokr_w2_a = ggml_cast(ctx, lokr_w2_a, GGML_TYPE_F16);
-                    }
-                }
-                iter = lora_tensors.find(lokr_w2_b_name);
-                if (iter != lora_tensors.end()) {
-                    lokr_w2_b = iter->second;
-                    if (is_conv2d && lokr_w2_b->type != GGML_TYPE_F16) {
-                        lokr_w2_b = ggml_cast(ctx, lokr_w2_b, GGML_TYPE_F16);
-                    }
-                }
-
-                int rank = 1;
-                if (lokr_w1_b) {
-                    rank = (int)lokr_w1_b->ne[ggml_n_dims(lokr_w1_b) - 1];
-                }
-                if (lokr_w2_b) {
-                    rank = (int)lokr_w2_b->ne[ggml_n_dims(lokr_w2_b) - 1];
-                }
-
-                float scale_value = 1.0f;
-                iter              = lora_tensors.find(alpha_name);
-                if (iter != lora_tensors.end()) {
-                    float alpha = ggml_ext_backend_tensor_get_f32(iter->second);
-                    scale_value = alpha / rank;
-                    applied_lora_tensors.insert(alpha_name);
-                }
-
-                if (rank == 1) {
-                    scale_value = 1.0f;
-                }
-                scale_value *= multiplier;
-
-                auto curr_out_diff = ggml_ext_lokr_forward(ctx, backend, x, lokr_w1, lokr_w1_a, lokr_w1_b, lokr_w2, lokr_w2_a, lokr_w2_b, is_conv2d, forward_params.conv2d, scale_value);
-                if (out_diff == nullptr) {
-                    out_diff = curr_out_diff;
-                } else {
-                    out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0);
-                }
-
-                if (lokr_w1)
-                    applied_lora_tensors.insert(lokr_w1_name);
-                if (lokr_w1_a)
-                    applied_lora_tensors.insert(lokr_w1_a_name);
-                if (lokr_w1_b)
-                    applied_lora_tensors.insert(lokr_w1_b_name);
-                if (lokr_w2)
-                    applied_lora_tensors.insert(lokr_w2_name);
-                if (lokr_w2_a)
-                    applied_lora_tensors.insert(lokr_w2_name);
-                if (lokr_w2_b)
-                    applied_lora_tensors.insert(lokr_w2_b_name);
-                applied_lora_tensors.insert(alpha_name);
-
-                index++;
-                continue;
-            }
-
-            // not a lokr, normal lora path
 
             std::string lora_down_name = "lora." + key + ".lora_down";
             std::string lora_up_name   = "lora." + key + ".lora_up";
@@ -629,7 +525,9 @@ struct LoraModel : public GGMLRunner {
             ggml_tensor* lora_mid  = nullptr;
             ggml_tensor* lora_down = nullptr;
 
-            iter = lora_tensors.find(lora_up_name);
+            bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D;
+
+            auto iter = lora_tensors.find(lora_up_name);
             if (iter != lora_tensors.end()) {
                 lora_up = iter->second;
                 if (is_conv2d && lora_up->type != GGML_TYPE_F16) {
@@ -736,7 +634,7 @@ struct LoraModel : public GGMLRunner {
                                       forward_params.conv2d.scale);
             }
 
-            auto curr_out_diff = ggml_ext_scale(ctx, lx, scale_value, true);
+            auto curr_out_diff = ggml_scale_inplace(ctx, lx, scale_value);
 
             if (out_diff == nullptr) {
                 out_diff = curr_out_diff;
@@ -749,9 +647,9 @@ struct LoraModel : public GGMLRunner {
         return out_diff;
     }
 
-    ggml_cgraph* build_lora_graph(const std::map<std::string, ggml_tensor*>& model_tensors, SDVersion version) {
+    struct ggml_cgraph* build_lora_graph(const std::map<std::string, ggml_tensor*>& model_tensors, SDVersion version) {
         size_t lora_graph_size = LORA_GRAPH_BASE_SIZE + lora_tensors.size() * 10;
-        ggml_cgraph* gf        = ggml_new_graph_custom(compute_ctx, lora_graph_size, false);
+        struct ggml_cgraph* gf = ggml_new_graph_custom(compute_ctx, lora_graph_size, false);
 
         preprocess_lora_tensors(model_tensors);
 
@@ -763,7 +661,7 @@ struct LoraModel : public GGMLRunner {
             ggml_tensor* model_tensor     = it.second;
 
             // lora
-            ggml_tensor* diff = get_weight_diff(model_tensor_name, runtime_backend, compute_ctx, model_tensor);
+            ggml_tensor* diff = get_weight_diff(model_tensor_name, compute_ctx, model_tensor);
             if (diff == nullptr) {
                 continue;
             }
@@ -776,7 +674,7 @@ struct LoraModel : public GGMLRunner {
 
             ggml_tensor* final_tensor;
             if (model_tensor->type != GGML_TYPE_F32 && model_tensor->type != GGML_TYPE_F16) {
-                final_tensor = ggml_ext_cast_f32(compute_ctx, runtime_backend, model_tensor);
+                final_tensor = ggml_ext_cast_f32(compute_ctx, model_tensor);
                 final_tensor = ggml_add_inplace(compute_ctx, final_tensor, diff);
                 final_tensor = ggml_cpy(compute_ctx, final_tensor, model_tensor);
             } else {
@@ -790,11 +688,11 @@ struct LoraModel : public GGMLRunner {
         return gf;
     }
 
-    void apply(std::map<std::string, ggml_tensor*> model_tensors, SDVersion version, int n_threads) {
-        auto get_graph = [&]() -> ggml_cgraph* {
+    void apply(std::map<std::string, struct ggml_tensor*> model_tensors, SDVersion version, int n_threads) {
+        auto get_graph = [&]() -> struct ggml_cgraph* {
             return build_lora_graph(model_tensors, version);
         };
-        GGMLRunner::compute<float>(get_graph, n_threads, false, true);
+        GGMLRunner::compute(get_graph, n_threads, false);
         stat();
         for (auto item : original_tensor_to_final_tensor) {
             ggml_tensor* original_tensor = item.first;
@@ -843,35 +741,34 @@ public:
         : lora_models(lora_models) {
     }
 
-    ggml_tensor* patch_weight(ggml_context* ctx, ggml_backend_t backend, ggml_tensor* weight, const std::string& weight_name, bool with_lora_and_lokr) {
+    ggml_tensor* patch_weight(ggml_context* ctx, ggml_tensor* weight, const std::string& weight_name, bool with_lora) {
         for (auto& lora_model : lora_models) {
-            ggml_tensor* diff = lora_model->get_weight_diff(weight_name, backend, ctx, weight, with_lora_and_lokr);
+            ggml_tensor* diff = lora_model->get_weight_diff(weight_name, ctx, weight, with_lora);
             if (diff == nullptr) {
                 continue;
             }
 
             if (weight->type != GGML_TYPE_F32 && weight->type != GGML_TYPE_F16) {
-                weight = ggml_ext_cast_f32(ctx, backend, weight);
+                weight = ggml_ext_cast_f32(ctx, weight);
             }
             weight = ggml_add(ctx, weight, diff);
         }
         return weight;
     }
 
-    ggml_tensor* patch_weight(ggml_context* ctx, ggml_backend_t backend, ggml_tensor* weight, const std::string& weight_name) override {
-        return patch_weight(ctx, backend, weight, weight_name, true);
+    ggml_tensor* patch_weight(ggml_context* ctx, ggml_tensor* weight, const std::string& weight_name) override {
+        return patch_weight(ctx, weight, weight_name, true);
     }
 
     ggml_tensor* forward_with_lora(ggml_context* ctx,
-                                   ggml_backend_t backend,
                                    ggml_tensor* x,
                                    ggml_tensor* w,
                                    ggml_tensor* b,
                                    const std::string& prefix,
                                    WeightAdapter::ForwardParams forward_params) override {
-        w = patch_weight(ctx, backend, w, prefix + "weight", false);
+        w = patch_weight(ctx, w, prefix + "weight", false);
         if (b) {
-            b = patch_weight(ctx, backend, b, prefix + "bias", false);
+            b = patch_weight(ctx, b, prefix + "bias", false);
         }
         ggml_tensor* out;
         if (forward_params.op_type == ForwardParams::op_type_t::OP_LINEAR) {
@@ -893,7 +790,7 @@ public:
                                    forward_params.conv2d.scale);
         }
         for (auto& lora_model : lora_models) {
-            ggml_tensor* out_diff = lora_model->get_out_diff(ctx, backend, x, forward_params, prefix + "weight");
+            ggml_tensor* out_diff = lora_model->get_out_diff(ctx, x, forward_params, prefix + "weight");
             if (out_diff == nullptr) {
                 continue;
             }

ltxv.hpp

@@ -1,7 +1,8 @@
 #ifndef __LTXV_HPP__
 #define __LTXV_HPP__
 
-#include "common_block.hpp"
+#include "common.hpp"
+#include "ggml_extend.hpp"
 
 namespace LTXV {
 
@@ -26,9 +27,9 @@ namespace LTXV {
                                                                      bias));
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* x,
-                             bool causal = true) {
+        struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                    struct ggml_tensor* x,
+                                    bool causal = true) {
             // x: [N*IC, ID, IH, IW]
             // result: [N*OC, OD, OH, OW]
             auto conv = std::dynamic_pointer_cast<Conv3d>(blocks["conv"]);

mmdit.hpp

@@ -27,13 +27,13 @@ public:
         blocks["fc2"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_features, out_features, bias));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         // x: [N, n_token, in_features]
         auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
         auto fc2 = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
 
         x = fc1->forward(ctx, x);
-        x = ggml_ext_gelu(ctx->ggml_ctx, x, true);
+        x = ggml_gelu_inplace(ctx->ggml_ctx, x);
         x = fc2->forward(ctx, x);
         return x;
     }
@@ -72,7 +72,7 @@ public:
                                                                bias));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         // x: [N, C, H, W]
         // return: [N, H*W, embed_dim]
         auto proj = std::dynamic_pointer_cast<Conv2d>(blocks["proj"]);
@@ -111,7 +111,7 @@ public:
         blocks["mlp.2"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, out_channels, true, true));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* t) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* t) {
         // t: [N, ]
         // return: [N, hidden_size]
         auto mlp_0 = std::dynamic_pointer_cast<Linear>(blocks["mlp.0"]);
@@ -135,7 +135,7 @@ public:
         blocks["mlp.2"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, hidden_size, true, true));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         // x: [N, input_dim]
         // return: [N, hidden_size]
         auto mlp_0 = std::dynamic_pointer_cast<Linear>(blocks["mlp.0"]);
@@ -175,7 +175,7 @@ public:
         }
     }
 
-    std::vector<ggml_tensor*> pre_attention(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    std::vector<struct ggml_tensor*> pre_attention(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         auto qkv_proj = std::dynamic_pointer_cast<Linear>(blocks["qkv"]);
 
         auto qkv         = qkv_proj->forward(ctx, x);
@@ -198,7 +198,7 @@ public:
         return {q, k, v};
     }
 
-    ggml_tensor* post_attention(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* post_attention(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         GGML_ASSERT(!pre_only);
 
         auto proj = std::dynamic_pointer_cast<Linear>(blocks["proj"]);
@@ -208,19 +208,19 @@ public:
     }
 
     // x: [N, n_token, dim]
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* x) {
         auto qkv = pre_attention(ctx, x);
-        x        = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled);  // [N, n_token, dim]
-        x        = post_attention(ctx, x);                                                                                                           // [N, n_token, dim]
+        x        = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, false, ctx->flash_attn_enabled);  // [N, n_token, dim]
+        x        = post_attention(ctx, x);                                                                                                                  // [N, n_token, dim]
         return x;
     }
 };
 
-__STATIC_INLINE__ ggml_tensor* modulate(ggml_context* ctx,
-                                        ggml_tensor* x,
-                                        ggml_tensor* shift,
-                                        ggml_tensor* scale) {
+__STATIC_INLINE__ struct ggml_tensor* modulate(struct ggml_context* ctx,
+                                               struct ggml_tensor* x,
+                                               struct ggml_tensor* shift,
+                                               struct ggml_tensor* scale) {
     // x: [N, L, C]
     // scale: [N, C]
     // shift: [N, C]
@@ -274,8 +274,8 @@ public:
     }
 
     std::tuple<std::vector<ggml_tensor*>, std::vector<ggml_tensor*>, std::vector<ggml_tensor*>> pre_attention_x(GGMLRunnerContext* ctx,
-                                                                                                                ggml_tensor* x,
-                                                                                                                ggml_tensor* c) {
+                                                                                                                struct ggml_tensor* x,
+                                                                                                                struct ggml_tensor* c) {
         GGML_ASSERT(self_attn);
         // x: [N, n_token, hidden_size]
         // c: [N, hidden_size]
@@ -284,19 +284,23 @@ public:
         auto attn2              = std::dynamic_pointer_cast<SelfAttention>(blocks["attn2"]);
         auto adaLN_modulation_1 = std::dynamic_pointer_cast<Linear>(blocks["adaLN_modulation.1"]);
 
-        int n_mods = 9;
-        auto m     = adaLN_modulation_1->forward(ctx, ggml_silu(ctx->ggml_ctx, c));  // [N, n_mods * hidden_size]
-        auto m_vec = ggml_ext_chunk(ctx->ggml_ctx, m, n_mods, 0);
+        int64_t n_mods = 9;
+        auto m         = adaLN_modulation_1->forward(ctx, ggml_silu(ctx->ggml_ctx, c));         // [N, n_mods * hidden_size]
+        m              = ggml_reshape_3d(ctx->ggml_ctx, m, c->ne[0], n_mods, c->ne[1]);         // [N, n_mods, hidden_size]
+        m              = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, m, 0, 2, 1, 3));  // [n_mods, N, hidden_size]
 
-        auto shift_msa  = m_vec[0];  // [N, hidden_size]
-        auto scale_msa  = m_vec[1];  // [N, hidden_size]
-        auto gate_msa   = m_vec[2];  // [N, hidden_size]
-        auto shift_mlp  = m_vec[3];  // [N, hidden_size]
-        auto scale_mlp  = m_vec[4];  // [N, hidden_size]
-        auto gate_mlp   = m_vec[5];  // [N, hidden_size]
-        auto shift_msa2 = m_vec[6];  // [N, hidden_size]
-        auto scale_msa2 = m_vec[7];  // [N, hidden_size]
-        auto gate_msa2  = m_vec[8];  // [N, hidden_size]
+        int64_t offset = m->nb[1] * m->ne[1];
+        auto shift_msa = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 0);  // [N, hidden_size]
+        auto scale_msa = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 1);  // [N, hidden_size]
+        auto gate_msa  = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 2);  // [N, hidden_size]
+
+        auto shift_mlp = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 3);  // [N, hidden_size]
+        auto scale_mlp = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 4);  // [N, hidden_size]
+        auto gate_mlp  = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 5);  // [N, hidden_size]
+
+        auto shift_msa2 = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 6);  // [N, hidden_size]
+        auto scale_msa2 = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 7);  // [N, hidden_size]
+        auto gate_msa2  = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 8);  // [N, hidden_size]
 
         auto x_norm = norm1->forward(ctx, x);
 
@@ -309,29 +313,31 @@ public:
         return {qkv, qkv2, {x, gate_msa, shift_mlp, scale_mlp, gate_mlp, gate_msa2}};
     }
 
-    std::pair<std::vector<ggml_tensor*>, std::vector<ggml_tensor*>> pre_attention(GGMLRunnerContext* ctx,
-                                                                                  ggml_tensor* x,
-                                                                                  ggml_tensor* c) {
+    std::pair<std::vector<struct ggml_tensor*>, std::vector<struct ggml_tensor*>> pre_attention(GGMLRunnerContext* ctx,
+                                                                                                struct ggml_tensor* x,
+                                                                                                struct ggml_tensor* c) {
         // x: [N, n_token, hidden_size]
         // c: [N, hidden_size]
         auto norm1              = std::dynamic_pointer_cast<LayerNorm>(blocks["norm1"]);
         auto attn               = std::dynamic_pointer_cast<SelfAttention>(blocks["attn"]);
         auto adaLN_modulation_1 = std::dynamic_pointer_cast<Linear>(blocks["adaLN_modulation.1"]);
 
-        int n_mods = 6;
+        int64_t n_mods = 6;
         if (pre_only) {
             n_mods = 2;
         }
-        auto m     = adaLN_modulation_1->forward(ctx, ggml_silu(ctx->ggml_ctx, c));  // [N, n_mods * hidden_size]
-        auto m_vec = ggml_ext_chunk(ctx->ggml_ctx, m, n_mods, 0);
+        auto m = adaLN_modulation_1->forward(ctx, ggml_silu(ctx->ggml_ctx, c));         // [N, n_mods * hidden_size]
+        m      = ggml_reshape_3d(ctx->ggml_ctx, m, c->ne[0], n_mods, c->ne[1]);         // [N, n_mods, hidden_size]
+        m      = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, m, 0, 2, 1, 3));  // [n_mods, N, hidden_size]
 
-        auto shift_msa = m_vec[0];  // [N, hidden_size]
-        auto scale_msa = m_vec[1];  // [N, hidden_size]
+        int64_t offset = m->nb[1] * m->ne[1];
+        auto shift_msa = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 0);  // [N, hidden_size]
+        auto scale_msa = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 1);  // [N, hidden_size]
         if (!pre_only) {
-            auto gate_msa  = m_vec[2];  // [N, hidden_size]
-            auto shift_mlp = m_vec[3];  // [N, hidden_size]
-            auto scale_mlp = m_vec[4];  // [N, hidden_size]
-            auto gate_mlp  = m_vec[5];  // [N, hidden_size]
+            auto gate_msa  = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 2);  // [N, hidden_size]
+            auto shift_mlp = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 3);  // [N, hidden_size]
+            auto scale_mlp = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 4);  // [N, hidden_size]
+            auto gate_mlp  = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 5);  // [N, hidden_size]
 
             auto attn_in = modulate(ctx->ggml_ctx, norm1->forward(ctx, x), shift_msa, scale_msa);
 
@@ -346,15 +352,15 @@ public:
         }
     }
 
-    ggml_tensor* post_attention_x(GGMLRunnerContext* ctx,
-                                  ggml_tensor* attn_out,
-                                  ggml_tensor* attn2_out,
-                                  ggml_tensor* x,
-                                  ggml_tensor* gate_msa,
-                                  ggml_tensor* shift_mlp,
-                                  ggml_tensor* scale_mlp,
-                                  ggml_tensor* gate_mlp,
-                                  ggml_tensor* gate_msa2) {
+    struct ggml_tensor* post_attention_x(GGMLRunnerContext* ctx,
+                                         struct ggml_tensor* attn_out,
+                                         struct ggml_tensor* attn2_out,
+                                         struct ggml_tensor* x,
+                                         struct ggml_tensor* gate_msa,
+                                         struct ggml_tensor* shift_mlp,
+                                         struct ggml_tensor* scale_mlp,
+                                         struct ggml_tensor* gate_mlp,
+                                         struct ggml_tensor* gate_msa2) {
         // attn_out: [N, n_token, hidden_size]
         // x: [N, n_token, hidden_size]
         // gate_msa: [N, hidden_size]
@@ -384,13 +390,13 @@ public:
         return x;
     }
 
-    ggml_tensor* post_attention(GGMLRunnerContext* ctx,
-                                ggml_tensor* attn_out,
-                                ggml_tensor* x,
-                                ggml_tensor* gate_msa,
-                                ggml_tensor* shift_mlp,
-                                ggml_tensor* scale_mlp,
-                                ggml_tensor* gate_mlp) {
+    struct ggml_tensor* post_attention(GGMLRunnerContext* ctx,
+                                       struct ggml_tensor* attn_out,
+                                       struct ggml_tensor* x,
+                                       struct ggml_tensor* gate_msa,
+                                       struct ggml_tensor* shift_mlp,
+                                       struct ggml_tensor* scale_mlp,
+                                       struct ggml_tensor* gate_mlp) {
         // attn_out: [N, n_token, hidden_size]
         // x: [N, n_token, hidden_size]
         // gate_msa: [N, hidden_size]
@@ -416,9 +422,9 @@ public:
         return x;
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x,
-                         ggml_tensor* c) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* x,
+                                struct ggml_tensor* c) {
         // x: [N, n_token, hidden_size]
         // c: [N, hidden_size]
         // return: [N, n_token, hidden_size]
@@ -433,8 +439,8 @@ public:
             auto qkv2          = std::get<1>(qkv_intermediates);
             auto intermediates = std::get<2>(qkv_intermediates);
 
-            auto attn_out  = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled);     // [N, n_token, dim]
-            auto attn2_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv2[0], qkv2[1], qkv2[2], num_heads, nullptr, false, ctx->flash_attn_enabled);  // [N, n_token, dim]
+            auto attn_out  = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, false, ctx->flash_attn_enabled);     // [N, n_token, dim]
+            auto attn2_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv2[0], qkv2[1], qkv2[2], num_heads, nullptr, false, false, ctx->flash_attn_enabled);  // [N, n_token, dim]
             x              = post_attention_x(ctx,
                                               attn_out,
                                               attn2_out,
@@ -450,7 +456,7 @@ public:
             auto qkv               = qkv_intermediates.first;
             auto intermediates     = qkv_intermediates.second;
 
-            auto attn_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled);  // [N, n_token, dim]
+            auto attn_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, false, ctx->flash_attn_enabled);  // [N, n_token, dim]
             x             = post_attention(ctx,
                                            attn_out,
                                            intermediates[0],
@@ -463,11 +469,11 @@ public:
     }
 };
 
-__STATIC_INLINE__ std::pair<ggml_tensor*, ggml_tensor*>
+__STATIC_INLINE__ std::pair<struct ggml_tensor*, struct ggml_tensor*>
 block_mixing(GGMLRunnerContext* ctx,
-             ggml_tensor* context,
-             ggml_tensor* x,
-             ggml_tensor* c,
+             struct ggml_tensor* context,
+             struct ggml_tensor* x,
+             struct ggml_tensor* c,
              std::shared_ptr<DismantledBlock> context_block,
              std::shared_ptr<DismantledBlock> x_block) {
     // context: [N, n_context, hidden_size]
@@ -489,29 +495,31 @@ block_mixing(GGMLRunnerContext* ctx,
         x_qkv                    = x_qkv_intermediates.first;
         x_intermediates          = x_qkv_intermediates.second;
     }
-    std::vector<ggml_tensor*> qkv;
+    std::vector<struct ggml_tensor*> qkv;
     for (int i = 0; i < 3; i++) {
         qkv.push_back(ggml_concat(ctx->ggml_ctx, context_qkv[i], x_qkv[i], 1));
     }
 
-    auto attn = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], x_block->num_heads, nullptr, false, ctx->flash_attn_enabled);  // [N, n_context + n_token, hidden_size]
-
+    auto attn         = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], x_block->num_heads, nullptr, false, false, ctx->flash_attn_enabled);  // [N, n_context + n_token, hidden_size]
+    attn              = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, attn, 0, 2, 1, 3));                                                                          // [n_context + n_token, N, hidden_size]
     auto context_attn = ggml_view_3d(ctx->ggml_ctx,
                                      attn,
                                      attn->ne[0],
+                                     attn->ne[1],
                                      context->ne[1],
-                                     attn->ne[2],
                                      attn->nb[1],
                                      attn->nb[2],
-                                     0);  // [N, n_context, hidden_size]
+                                     0);                                                                  // [n_context, N, hidden_size]
+    context_attn      = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, context_attn, 0, 2, 1, 3));  // [N, n_context, hidden_size]
     auto x_attn       = ggml_view_3d(ctx->ggml_ctx,
                                      attn,
                                      attn->ne[0],
+                                     attn->ne[1],
                                      x->ne[1],
-                                     attn->ne[2],
                                      attn->nb[1],
                                      attn->nb[2],
-                                     context->ne[1] * attn->nb[1]);  // [N, n_token, hidden_size]
+                                     attn->nb[2] * context->ne[1]);                                 // [n_token, N, hidden_size]
+    x_attn            = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x_attn, 0, 2, 1, 3));  // [N, n_token, hidden_size]
 
     if (!context_block->pre_only) {
         context = context_block->post_attention(ctx,
@@ -526,7 +534,7 @@ block_mixing(GGMLRunnerContext* ctx,
     }
 
     if (x_block->self_attn) {
-        auto attn2 = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, x_qkv2[0], x_qkv2[1], x_qkv2[2], x_block->num_heads, nullptr, false, ctx->flash_attn_enabled);  // [N, n_token, hidden_size]
+        auto attn2 = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, x_qkv2[0], x_qkv2[1], x_qkv2[2], x_block->num_heads, nullptr, false, false, ctx->flash_attn_enabled);  // [N, n_token, hidden_size]
 
         x = x_block->post_attention_x(ctx,
                                       x_attn,
@@ -563,10 +571,10 @@ public:
         blocks["x_block"]       = std::shared_ptr<GGMLBlock>(new DismantledBlock(hidden_size, num_heads, mlp_ratio, qk_norm, qkv_bias, false, self_attn_x));
     }
 
-    std::pair<ggml_tensor*, ggml_tensor*> forward(GGMLRunnerContext* ctx,
-                                                  ggml_tensor* context,
-                                                  ggml_tensor* x,
-                                                  ggml_tensor* c) {
+    std::pair<struct ggml_tensor*, struct ggml_tensor*> forward(GGMLRunnerContext* ctx,
+                                                                struct ggml_tensor* context,
+                                                                struct ggml_tensor* x,
+                                                                struct ggml_tensor* c) {
         auto context_block = std::dynamic_pointer_cast<DismantledBlock>(blocks["context_block"]);
         auto x_block       = std::dynamic_pointer_cast<DismantledBlock>(blocks["x_block"]);
 
@@ -586,9 +594,9 @@ public:
         blocks["adaLN_modulation.1"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, 2 * hidden_size));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x,
-                         ggml_tensor* c) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* x,
+                                struct ggml_tensor* c) {
         // x: [N, n_token, hidden_size]
         // c: [N, hidden_size]
         // return: [N, n_token, patch_size * patch_size * out_channels]
@@ -596,10 +604,13 @@ public:
         auto linear             = std::dynamic_pointer_cast<Linear>(blocks["linear"]);
         auto adaLN_modulation_1 = std::dynamic_pointer_cast<Linear>(blocks["adaLN_modulation.1"]);
 
-        auto m     = adaLN_modulation_1->forward(ctx, ggml_silu(ctx->ggml_ctx, c));  // [N, 2 * hidden_size]
-        auto m_vec = ggml_ext_chunk(ctx->ggml_ctx, m, 2, 0);
-        auto shift = m_vec[0];  // [N, hidden_size]
-        auto scale = m_vec[1];  // [N, hidden_size]
+        auto m = adaLN_modulation_1->forward(ctx, ggml_silu(ctx->ggml_ctx, c));         // [N, 2 * hidden_size]
+        m      = ggml_reshape_3d(ctx->ggml_ctx, m, c->ne[0], 2, c->ne[1]);              // [N, 2, hidden_size]
+        m      = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, m, 0, 2, 1, 3));  // [2, N, hidden_size]
+
+        int64_t offset = m->nb[1] * m->ne[1];
+        auto shift     = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 0);  // [N, hidden_size]
+        auto scale     = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 1);  // [N, hidden_size]
 
         x = modulate(ctx->ggml_ctx, norm_final->forward(ctx, x), shift, scale);
         x = linear->forward(ctx, x);
@@ -626,7 +637,7 @@ protected:
     int64_t hidden_size;
     std::string qk_norm;
 
-    void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") override {
+    void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") override {
         enum ggml_type wtype = GGML_TYPE_F32;
         params["pos_embed"]  = ggml_new_tensor_3d(ctx, wtype, hidden_size, num_patchs, 1);
     }
@@ -705,8 +716,8 @@ public:
         blocks["final_layer"] = std::shared_ptr<GGMLBlock>(new FinalLayer(hidden_size, patch_size, out_channels));
     }
 
-    ggml_tensor*
-    cropped_pos_embed(ggml_context* ctx,
+    struct ggml_tensor*
+    cropped_pos_embed(struct ggml_context* ctx,
                       int64_t h,
                       int64_t w) {
         auto pos_embed = params["pos_embed"];
@@ -745,11 +756,33 @@ public:
         return spatial_pos_embed;
     }
 
-    ggml_tensor* forward_core_with_concat(GGMLRunnerContext* ctx,
-                                          ggml_tensor* x,
-                                          ggml_tensor* c_mod,
-                                          ggml_tensor* context,
-                                          std::vector<int> skip_layers = std::vector<int>()) {
+    struct ggml_tensor* unpatchify(struct ggml_context* ctx,
+                                   struct ggml_tensor* x,
+                                   int64_t h,
+                                   int64_t w) {
+        // x: [N, H*W, patch_size * patch_size * C]
+        // return: [N, C, H, W]
+        int64_t n = x->ne[2];
+        int64_t c = out_channels;
+        int64_t p = patch_size;
+        h         = (h + 1) / p;
+        w         = (w + 1) / p;
+
+        GGML_ASSERT(h * w == x->ne[1]);
+
+        x = ggml_reshape_4d(ctx, x, c, p * p, w * h, n);       // [N, H*W, P*P, C]
+        x = ggml_cont(ctx, ggml_permute(ctx, x, 2, 0, 1, 3));  // [N, C, H*W, P*P]
+        x = ggml_reshape_4d(ctx, x, p, p, w, h * c * n);       // [N*C*H, W, P, P]
+        x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));  // [N*C*H, P, W, P]
+        x = ggml_reshape_4d(ctx, x, p * w, p * h, c, n);       // [N, C, H*P, W*P]
+        return x;
+    }
+
+    struct ggml_tensor* forward_core_with_concat(GGMLRunnerContext* ctx,
+                                                 struct ggml_tensor* x,
+                                                 struct ggml_tensor* c_mod,
+                                                 struct ggml_tensor* context,
+                                                 std::vector<int> skip_layers = std::vector<int>()) {
         // x: [N, H*W, hidden_size]
         // context: [N, n_context, d_context]
         // c: [N, hidden_size]
@@ -767,8 +800,6 @@ public:
             auto context_x = block->forward(ctx, context, x, c_mod);
             context        = context_x.first;
             x              = context_x.second;
-            sd::ggml_graph_cut::mark_graph_cut(context, "mmdit.joint_blocks." + std::to_string(i), "context");
-            sd::ggml_graph_cut::mark_graph_cut(x, "mmdit.joint_blocks." + std::to_string(i), "x");
         }
 
         x = final_layer->forward(ctx, x, c_mod);  // (N, T, patch_size ** 2 * out_channels)
@@ -776,12 +807,12 @@ public:
         return x;
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x,
-                         ggml_tensor* t,
-                         ggml_tensor* y               = nullptr,
-                         ggml_tensor* context         = nullptr,
-                         std::vector<int> skip_layers = std::vector<int>()) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* x,
+                                struct ggml_tensor* t,
+                                struct ggml_tensor* y        = nullptr,
+                                struct ggml_tensor* context  = nullptr,
+                                std::vector<int> skip_layers = std::vector<int>()) {
         // Forward pass of DiT.
         // x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)
         // t: (N,) tensor of diffusion timesteps
@@ -791,11 +822,11 @@ public:
         auto x_embedder = std::dynamic_pointer_cast<PatchEmbed>(blocks["x_embedder"]);
         auto t_embedder = std::dynamic_pointer_cast<TimestepEmbedder>(blocks["t_embedder"]);
 
-        int64_t W = x->ne[0];
-        int64_t H = x->ne[1];
+        int64_t w = x->ne[0];
+        int64_t h = x->ne[1];
 
         auto patch_embed = x_embedder->forward(ctx, x);                      // [N, H*W, hidden_size]
-        auto pos_embed   = cropped_pos_embed(ctx->ggml_ctx, H, W);           // [1, H*W, hidden_size]
+        auto pos_embed   = cropped_pos_embed(ctx->ggml_ctx, h, w);           // [1, H*W, hidden_size]
         x                = ggml_add(ctx->ggml_ctx, patch_embed, pos_embed);  // [N, H*W, hidden_size]
 
         auto c = t_embedder->forward(ctx, t);  // [N, hidden_size]
@@ -811,15 +842,10 @@ public:
 
             context = context_embedder->forward(ctx, context);  // [N, L, D] aka [N, L, 1536]
         }
-        sd::ggml_graph_cut::mark_graph_cut(x, "mmdit.prelude", "x");
-        sd::ggml_graph_cut::mark_graph_cut(c, "mmdit.prelude", "c");
-        if (context != nullptr) {
-            sd::ggml_graph_cut::mark_graph_cut(context, "mmdit.prelude", "context");
-        }
 
         x = forward_core_with_concat(ctx, x, c, context, skip_layers);  // (N, H*W, patch_size ** 2 * out_channels)
 
-        x = DiT::unpatchify_and_crop(ctx->ggml_ctx, x, H, W, patch_size, patch_size, /*patch_last*/ false);  // [N, C, H, W]
+        x = unpatchify(ctx->ggml_ctx, x, h, w);  // [N, C, H, W]
 
         return x;
     }
@@ -828,10 +854,10 @@ struct MMDiTRunner : public GGMLRunner {
     MMDiT mmdit;
 
     MMDiTRunner(ggml_backend_t backend,
-                ggml_backend_t params_backend,
+                bool offload_params_to_cpu,
                 const String2TensorStorage& tensor_storage_map = {},
                 const std::string prefix                       = "")
-        : GGMLRunner(backend, params_backend), mmdit(tensor_storage_map) {
+        : GGMLRunner(backend, offload_params_to_cpu), mmdit(tensor_storage_map) {
         mmdit.init(params_ctx, tensor_storage_map, prefix);
     }
 
@@ -839,93 +865,89 @@ struct MMDiTRunner : public GGMLRunner {
         return "mmdit";
     }
 
-    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string prefix) {
+    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
         mmdit.get_param_tensors(tensors, prefix);
     }
 
-    ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
-                             const sd::Tensor<float>& timesteps_tensor,
-                             const sd::Tensor<float>& context_tensor = {},
-                             const sd::Tensor<float>& y_tensor       = {},
-                             std::vector<int> skip_layers            = std::vector<int>()) {
-        ggml_cgraph* gf = new_graph_custom(MMDIT_GRAPH_SIZE);
+    struct ggml_cgraph* build_graph(struct ggml_tensor* x,
+                                    struct ggml_tensor* timesteps,
+                                    struct ggml_tensor* context,
+                                    struct ggml_tensor* y,
+                                    std::vector<int> skip_layers = std::vector<int>()) {
+        struct ggml_cgraph* gf = new_graph_custom(MMDIT_GRAPH_SIZE);
 
-        ggml_tensor* x         = make_input(x_tensor);
-        ggml_tensor* timesteps = make_input(timesteps_tensor);
-        ggml_tensor* context   = make_optional_input(context_tensor);
-        ggml_tensor* y         = make_optional_input(y_tensor);
+        x         = to_backend(x);
+        context   = to_backend(context);
+        y         = to_backend(y);
+        timesteps = to_backend(timesteps);
 
-        auto runner_ctx  = get_context();
-        ggml_tensor* out = mmdit.forward(&runner_ctx,
-                                         x,
-                                         timesteps,
-                                         y,
-                                         context,
-                                         skip_layers);
+        auto runner_ctx         = get_context();
+        struct ggml_tensor* out = mmdit.forward(&runner_ctx,
+                                                x,
+                                                timesteps,
+                                                y,
+                                                context,
+                                                skip_layers);
 
         ggml_build_forward_expand(gf, out);
 
         return gf;
     }
 
-    sd::Tensor<float> compute(int n_threads,
-                              const sd::Tensor<float>& x,
-                              const sd::Tensor<float>& timesteps,
-                              const sd::Tensor<float>& context = {},
-                              const sd::Tensor<float>& y       = {},
-                              std::vector<int> skip_layers     = std::vector<int>()) {
+    bool compute(int n_threads,
+                 struct ggml_tensor* x,
+                 struct ggml_tensor* timesteps,
+                 struct ggml_tensor* context,
+                 struct ggml_tensor* y,
+                 struct ggml_tensor** output     = nullptr,
+                 struct ggml_context* output_ctx = nullptr,
+                 std::vector<int> skip_layers    = std::vector<int>()) {
         // x: [N, in_channels, h, w]
         // timesteps: [N, ]
         // context: [N, max_position, hidden_size]([N, 154, 4096]) or [1, max_position, hidden_size]
         // y: [N, adm_in_channels] or [1, adm_in_channels]
-        auto get_graph = [&]() -> ggml_cgraph* {
+        auto get_graph = [&]() -> struct ggml_cgraph* {
             return build_graph(x, timesteps, context, y, skip_layers);
         };
 
-        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
+        return GGMLRunner::compute(get_graph, n_threads, false, output, output_ctx);
     }
 
     void test() {
-        ggml_init_params params;
+        struct ggml_init_params params;
         params.mem_size   = static_cast<size_t>(10 * 1024 * 1024);  // 10 MB
         params.mem_buffer = nullptr;
         params.no_alloc   = false;
 
-        ggml_context* ctx = ggml_init(params);
-        GGML_ASSERT(ctx != nullptr);
+        struct ggml_context* work_ctx = ggml_init(params);
+        GGML_ASSERT(work_ctx != nullptr);
 
         {
             // cpu f16: pass
             // cpu f32: pass
             // cuda f16: pass
             // cuda f32: pass
-            sd::Tensor<float> x({128, 128, 16, 1});
+            auto x = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, 128, 128, 16, 1);
             std::vector<float> timesteps_vec(1, 999.f);
-            auto timesteps = sd::Tensor<float>::from_vector(timesteps_vec);
-            x.fill_(0.01f);
+            auto timesteps = vector_to_ggml_tensor(work_ctx, timesteps_vec);
+            ggml_set_f32(x, 0.01f);
             // print_ggml_tensor(x);
 
-            sd::Tensor<float> context({4096, 154, 1});
-            context.fill_(0.01f);
+            auto context = ggml_new_tensor_3d(work_ctx, GGML_TYPE_F32, 4096, 154, 1);
+            ggml_set_f32(context, 0.01f);
             // print_ggml_tensor(context);
 
-            sd::Tensor<float> y({2048, 1});
-            y.fill_(0.01f);
+            auto y = ggml_new_tensor_2d(work_ctx, GGML_TYPE_F32, 2048, 1);
+            ggml_set_f32(y, 0.01f);
             // print_ggml_tensor(y);
 
-            sd::Tensor<float> out;
+            struct ggml_tensor* out = nullptr;
 
-            int64_t t0   = ggml_time_ms();
-            auto out_opt = compute(8,
-                                   x,
-                                   timesteps,
-                                   context,
-                                   y);
-            int64_t t1   = ggml_time_ms();
+            int64_t t0 = ggml_time_ms();
+            compute(8, x, timesteps, context, y, &out, work_ctx);
+            int64_t t1 = ggml_time_ms();
 
-            GGML_ASSERT(!out_opt.empty());
-            out = std::move(out_opt);
-            print_sd_tensor(out);
+            print_ggml_tensor(out);
             LOG_DEBUG("mmdit test done in %lldms", t1 - t0);
         }
     }
@@ -934,7 +956,7 @@ struct MMDiTRunner : public GGMLRunner {
         // ggml_backend_t backend    = ggml_backend_cuda_init(0);
         ggml_backend_t backend             = ggml_backend_cpu_init();
         ggml_type model_data_type          = GGML_TYPE_F16;
-        std::shared_ptr<MMDiTRunner> mmdit = std::make_shared<MMDiTRunner>(backend, backend);
+        std::shared_ptr<MMDiTRunner> mmdit = std::make_shared<MMDiTRunner>(backend, false);
         {
             LOG_INFO("loading from '%s'", file_path.c_str());
 

model.h

@@ -5,13 +5,20 @@
 #include <map>
 #include <memory>
 #include <set>
+#include <sstream>
 #include <string>
+#include <tuple>
+#include <utility>
 #include <vector>
 
 #include "ggml-backend.h"
 #include "ggml.h"
-#include "model_io/tensor_storage.h"
+#include "gguf.h"
+#include "json.hpp"
 #include "ordered_map.hpp"
+#include "zip.h"
+
+#define SD_MAX_DIMS 5
 
 enum SDVersion {
     VERSION_SD1,
@@ -21,12 +28,9 @@ enum SDVersion {
     VERSION_SD2,
     VERSION_SD2_INPAINT,
     VERSION_SD2_TINY_UNET,
-    VERSION_SDXS_512_DS,
-    VERSION_SDXS_09,
     VERSION_SDXL,
     VERSION_SDXL_INPAINT,
     VERSION_SDXL_PIX2PIX,
-    VERSION_SDXL_VEGA,
     VERSION_SDXL_SSD1B,
     VERSION_SVD,
     VERSION_SD3,
@@ -39,32 +43,28 @@ enum SDVersion {
     VERSION_WAN2_2_I2V,
     VERSION_WAN2_2_TI2V,
     VERSION_QWEN_IMAGE,
-    VERSION_ANIMA,
     VERSION_FLUX2,
-    VERSION_FLUX2_KLEIN,
-    VERSION_HIDREAM_O1,
     VERSION_Z_IMAGE,
     VERSION_OVIS_IMAGE,
-    VERSION_ERNIE_IMAGE,
     VERSION_COUNT,
 };
 
 static inline bool sd_version_is_sd1(SDVersion version) {
-    if (version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX || version == VERSION_SD1_TINY_UNET || version == VERSION_SDXS_512_DS) {
+    if (version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX || version == VERSION_SD1_TINY_UNET) {
         return true;
     }
     return false;
 }
 
 static inline bool sd_version_is_sd2(SDVersion version) {
-    if (version == VERSION_SD2 || version == VERSION_SD2_INPAINT || version == VERSION_SD2_TINY_UNET || version == VERSION_SDXS_09) {
+    if (version == VERSION_SD2 || version == VERSION_SD2_INPAINT || version == VERSION_SD2_TINY_UNET) {
         return true;
     }
     return false;
 }
 
 static inline bool sd_version_is_sdxl(SDVersion version) {
-    if (version == VERSION_SDXL || version == VERSION_SDXL_INPAINT || version == VERSION_SDXL_PIX2PIX || version == VERSION_SDXL_SSD1B || version == VERSION_SDXL_VEGA) {
+    if (version == VERSION_SDXL || version == VERSION_SDXL_INPAINT || version == VERSION_SDXL_PIX2PIX || version == VERSION_SDXL_SSD1B) {
         return true;
     }
     return false;
@@ -99,7 +99,7 @@ static inline bool sd_version_is_flux(SDVersion version) {
 }
 
 static inline bool sd_version_is_flux2(SDVersion version) {
-    if (version == VERSION_FLUX2 || version == VERSION_FLUX2_KLEIN) {
+    if (version == VERSION_FLUX2) {
         return true;
     }
     return false;
@@ -119,13 +119,6 @@ static inline bool sd_version_is_qwen_image(SDVersion version) {
     return false;
 }
 
-static inline bool sd_version_is_anima(SDVersion version) {
-    if (version == VERSION_ANIMA) {
-        return true;
-    }
-    return false;
-}
-
 static inline bool sd_version_is_z_image(SDVersion version) {
     if (version == VERSION_Z_IMAGE) {
         return true;
@@ -133,20 +126,6 @@ static inline bool sd_version_is_z_image(SDVersion version) {
     return false;
 }
 
-static inline bool sd_version_is_ernie_image(SDVersion version) {
-    if (version == VERSION_ERNIE_IMAGE) {
-        return true;
-    }
-    return false;
-}
-
-static inline bool sd_version_uses_flux2_vae(SDVersion version) {
-    if (sd_version_is_flux2(version) || sd_version_is_ernie_image(version)) {
-        return true;
-    }
-    return false;
-}
-
 static inline bool sd_version_is_inpaint(SDVersion version) {
     if (version == VERSION_SD1_INPAINT ||
         version == VERSION_SD2_INPAINT ||
@@ -164,10 +143,7 @@ static inline bool sd_version_is_dit(SDVersion version) {
         sd_version_is_sd3(version) ||
         sd_version_is_wan(version) ||
         sd_version_is_qwen_image(version) ||
-        version == VERSION_HIDREAM_O1 ||
-        sd_version_is_anima(version) ||
-        sd_version_is_z_image(version) ||
-        sd_version_is_ernie_image(version)) {
+        sd_version_is_z_image(version)) {
         return true;
     }
     return false;
@@ -190,40 +166,135 @@ enum PMVersion {
     PM_VERSION_2,
 };
 
+struct TensorStorage {
+    std::string name;
+    ggml_type type          = GGML_TYPE_F32;
+    ggml_type expected_type = GGML_TYPE_COUNT;
+    bool is_f8_e4m3         = false;
+    bool is_f8_e5m2         = false;
+    bool is_f64             = false;
+    bool is_i64             = false;
+    int64_t ne[SD_MAX_DIMS] = {1, 1, 1, 1, 1};
+    int n_dims              = 0;
+
+    size_t file_index = 0;
+    int index_in_zip  = -1;  // >= means stored in a zip file
+    uint64_t offset   = 0;   // offset in file
+
+    TensorStorage() = default;
+
+    TensorStorage(std::string name, ggml_type type, const int64_t* ne, int n_dims, size_t file_index, size_t offset = 0)
+        : name(std::move(name)), type(type), n_dims(n_dims), file_index(file_index), offset(offset) {
+        for (int i = 0; i < n_dims; i++) {
+            this->ne[i] = ne[i];
+        }
+    }
+
+    int64_t nelements() const {
+        int64_t n = 1;
+        for (int i = 0; i < SD_MAX_DIMS; i++) {
+            n *= ne[i];
+        }
+        return n;
+    }
+
+    int64_t nbytes() const {
+        return nelements() * ggml_type_size(type) / ggml_blck_size(type);
+    }
+
+    int64_t nbytes_to_read() const {
+        if (is_f8_e4m3 || is_f8_e5m2) {
+            return nbytes() / 2;
+        } else if (is_f64 || is_i64) {
+            return nbytes() * 2;
+        } else {
+            return nbytes();
+        }
+    }
+
+    void unsqueeze() {
+        if (n_dims == 2) {
+            n_dims = 4;
+            ne[3]  = ne[1];
+            ne[2]  = ne[0];
+            ne[1]  = 1;
+            ne[0]  = 1;
+        }
+    }
+
+    std::vector<TensorStorage> chunk(size_t n) {
+        std::vector<TensorStorage> chunks;
+        uint64_t chunk_size = nbytes_to_read() / n;
+        // printf("%d/%d\n", chunk_size, nbytes_to_read());
+        reverse_ne();
+        for (size_t i = 0; i < n; i++) {
+            TensorStorage chunk_i = *this;
+            chunk_i.ne[0]         = ne[0] / n;
+            chunk_i.offset        = offset + i * chunk_size;
+            chunk_i.reverse_ne();
+            chunks.push_back(chunk_i);
+        }
+        reverse_ne();
+        return chunks;
+    }
+
+    void reverse_ne() {
+        int64_t new_ne[SD_MAX_DIMS] = {1, 1, 1, 1, 1};
+        for (int i = 0; i < n_dims; i++) {
+            new_ne[i] = ne[n_dims - 1 - i];
+        }
+        for (int i = 0; i < n_dims; i++) {
+            ne[i] = new_ne[i];
+        }
+    }
+
+    std::string to_string() const {
+        std::stringstream ss;
+        const char* type_name = ggml_type_name(type);
+        if (is_f8_e4m3) {
+            type_name = "f8_e4m3";
+        } else if (is_f8_e5m2) {
+            type_name = "f8_e5m2";
+        } else if (is_f64) {
+            type_name = "f64";
+        } else if (is_i64) {
+            type_name = "i64";
+        }
+        ss << name << " | " << type_name << " | ";
+        ss << n_dims << " [";
+        for (int i = 0; i < SD_MAX_DIMS; i++) {
+            ss << ne[i];
+            if (i != SD_MAX_DIMS - 1) {
+                ss << ", ";
+            }
+        }
+        ss << "]";
+        return ss.str();
+    }
+};
+
+typedef std::function<bool(const TensorStorage&, ggml_tensor**)> on_new_tensor_cb_t;
+
 typedef OrderedMap<std::string, TensorStorage> String2TensorStorage;
-using TensorTypeRules = std::vector<std::pair<std::string, ggml_type>>;
-
-TensorTypeRules parse_tensor_type_rules(const std::string& tensor_type_rules);
-
-class MmapWrapper;
-
-struct ModelFileData {
-    std::string path;
-    std::vector<TensorStorage> tensors;
-    std::shared_ptr<MmapWrapper> mmapped;
-    std::shared_ptr<struct ggml_backend_buffer> mmbuffer;
-    bool is_zip;
-};
-
-struct MmapTensorStore {
-    std::shared_ptr<MmapWrapper> mmapped;
-    std::shared_ptr<struct ggml_backend_buffer> mmbuffer;
-};
 
 class ModelLoader {
 protected:
     SDVersion version_ = VERSION_COUNT;
     std::vector<std::string> file_paths_;
-    std::vector<ModelFileData> file_data;
-    bool model_files_processed = false;
     String2TensorStorage tensor_storage_map;
 
     void add_tensor_storage(const TensorStorage& tensor_storage);
 
+    bool parse_data_pkl(uint8_t* buffer,
+                        size_t buffer_size,
+                        zip_t* zip,
+                        std::string dir,
+                        size_t file_index,
+                        const std::string prefix);
+
     bool init_from_gguf_file(const std::string& file_path, const std::string& prefix = "");
     bool init_from_safetensors_file(const std::string& file_path, const std::string& prefix = "");
-    bool init_from_torch_zip_file(const std::string& file_path, const std::string& prefix = "");
-    bool init_from_torch_legacy_file(const std::string& file_path, const std::string& prefix = "");
+    bool init_from_ckpt_file(const std::string& file_path, const std::string& prefix = "");
     bool init_from_diffusers_file(const std::string& file_path, const std::string& prefix = "");
 
 public:
@@ -239,12 +310,8 @@ public:
     std::map<ggml_type, uint32_t> get_vae_wtype_stat();
     String2TensorStorage& get_tensor_storage_map() { return tensor_storage_map; }
     void set_wtype_override(ggml_type wtype, std::string tensor_type_rules = "");
-    void process_model_files(bool enable_mmap = false, bool writable_mmap = true);
-    std::vector<MmapTensorStore> mmap_tensors(std::map<std::string, ggml_tensor*>& tensors,
-                                              std::set<std::string> ignore_tensors = {},
-                                              bool writable                        = true);
     bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads = 0, bool use_mmap = false);
-    bool load_tensors(std::map<std::string, ggml_tensor*>& tensors,
+    bool load_tensors(std::map<std::string, struct ggml_tensor*>& tensors,
                       std::set<std::string> ignore_tensors = {},
                       int n_threads                        = 0,
                       bool use_mmap                        = false);
@@ -257,9 +324,17 @@ public:
         return names;
     }
 
+    bool save_to_gguf_file(const std::string& file_path, ggml_type type, const std::string& tensor_type_rules);
     bool tensor_should_be_converted(const TensorStorage& tensor_storage, ggml_type type);
     int64_t get_params_mem_size(ggml_backend_t backend, ggml_type type = GGML_TYPE_COUNT);
     ~ModelLoader() = default;
+
+    static std::string load_merges();
+    static std::string load_qwen2_merges();
+    static std::string load_mistral_merges();
+    static std::string load_mistral_vocab_json();
+    static std::string load_t5_tokenizer_json();
+    static std::string load_umt5_tokenizer_json();
 };
 
 #endif  // __MODEL_H__

name_conversion.cpp

@@ -653,14 +653,6 @@ std::string convert_diffusers_dit_to_original_lumina2(std::string name) {
     return name;
 }
 
-std::string convert_other_dit_to_original_anima(std::string name) {
-    static const std::string anima_net_prefix = "net.";
-    if (!starts_with(name, anima_net_prefix)) {
-        name = anima_net_prefix + name;
-    }
-    return name;
-}
-
 std::string convert_diffusion_model_name(std::string name, std::string prefix, SDVersion version) {
     if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
         name = convert_diffusers_unet_to_original_sd1(name);
@@ -672,8 +664,6 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S
         name = convert_diffusers_dit_to_original_flux(name);
     } else if (sd_version_is_z_image(version)) {
         name = convert_diffusers_dit_to_original_lumina2(name);
-    } else if (sd_version_is_anima(version)) {
-        name = convert_other_dit_to_original_anima(name);
     }
     return name;
 }
@@ -852,7 +842,6 @@ std::string convert_sep_to_dot(std::string name) {
         "conv_in",
         "conv_out",
         "lora_down",
-        "lora_mid",
         "lora_up",
         "diff_b",
         "hada_w1_a",
@@ -1008,13 +997,10 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
     if (is_lora) {
         std::map<std::string, std::string> lora_suffix_map = {
             {".lora_down.weight", ".weight.lora_down"},
-            {".lora_mid.weight", ".weight.lora_mid"},
             {".lora_up.weight", ".weight.lora_up"},
             {".lora.down.weight", ".weight.lora_down"},
-            {".lora.mid.weight", ".weight.lora_mid"},
             {".lora.up.weight", ".weight.lora_up"},
             {"_lora.down.weight", ".weight.lora_down"},
-            {"_lora.mid.weight", ".weight.lora_mid"},
             {"_lora.up.weight", ".weight.lora_up"},
             {".lora_A.weight", ".weight.lora_down"},
             {".lora_B.weight", ".weight.lora_up"},
@@ -1120,11 +1106,7 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
         for (const auto& prefix : first_stage_model_prefix_vec) {
             if (starts_with(name, prefix)) {
                 name = convert_first_stage_model_name(name.substr(prefix.size()), prefix);
-                if (version == VERSION_SDXS_512_DS || version == VERSION_SDXS_09) {
-                    name = "tae." + name;
-                } else {
-                    name = prefix + name;
-                }
+                name = prefix + name;
                 break;
             }
         }

pmid.hpp

@@ -21,19 +21,19 @@ public:
         blocks["layernorm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(in_dim));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
         // x: [N, channels, h, w]
 
         auto fc1        = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
         auto fc2        = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
         auto layer_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["layernorm"]);
 
-        ggml_tensor* r = x;
+        struct ggml_tensor* r = x;
         // x = ggml_ext_layer_norm(ctx, x, ln_w, ln_b);
         x = layer_norm->forward(ctx, x);
         // x = ggml_add(ctx, ggml_mul_mat(ctx, fc1_w, x),  fc1_b);
         x = fc1->forward(ctx, x);
-        x = ggml_ext_gelu(ctx->ggml_ctx, x, true);
+        x = ggml_gelu_inplace(ctx->ggml_ctx, x);
         x = fc2->forward(ctx, x);
         // x = ggml_add(ctx, ggml_mul_mat(ctx, fc2_w, x),  fc2_b);
         if (use_residue)
@@ -54,8 +54,8 @@ public:
         blocks["1"]   = std::shared_ptr<GGMLBlock>(new Mlp(dim, inner_dim, dim, false));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* x) {
         auto norm = std::dynamic_pointer_cast<LayerNorm>(blocks["0"]);
         auto ff   = std::dynamic_pointer_cast<Mlp>(blocks["1"]);
 
@@ -81,9 +81,9 @@ public:
         blocks["to_out"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, dim, false));
     }
 
-    ggml_tensor* reshape_tensor(ggml_context* ctx,
-                                ggml_tensor* x,
-                                int heads) {
+    struct ggml_tensor* reshape_tensor(struct ggml_context* ctx,
+                                       struct ggml_tensor* x,
+                                       int heads) {
         int64_t ne[4];
         for (int i = 0; i < 4; ++i)
             ne[i] = x->ne[i];
@@ -92,17 +92,17 @@ public:
         return x;
     }
 
-    std::vector<ggml_tensor*> chunk_half(ggml_context* ctx,
-                                         ggml_tensor* x) {
+    std::vector<struct ggml_tensor*> chunk_half(struct ggml_context* ctx,
+                                                struct ggml_tensor* x) {
         auto tlo = ggml_view_4d(ctx, x, x->ne[0] / 2, x->ne[1], x->ne[2], x->ne[3], x->nb[1], x->nb[2], x->nb[3], 0);
         auto tli = ggml_view_4d(ctx, x, x->ne[0] / 2, x->ne[1], x->ne[2], x->ne[3], x->nb[1], x->nb[2], x->nb[3], x->nb[0] * x->ne[0] / 2);
         return {ggml_cont(ctx, tlo),
                 ggml_cont(ctx, tli)};
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x,
-                         ggml_tensor* latents) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* x,
+                                struct ggml_tensor* latents) {
         // x (torch.Tensor): image features
         //     shape (b, n1, D)
         // latent (torch.Tensor): latent features
@@ -129,8 +129,8 @@ public:
         k             = reshape_tensor(ctx->ggml_ctx, k, heads);
         v             = reshape_tensor(ctx->ggml_ctx, v, heads);
         scale         = 1.f / sqrt(sqrt((float)dim_head));
-        k             = ggml_ext_scale(ctx->ggml_ctx, k, scale, true);
-        q             = ggml_ext_scale(ctx->ggml_ctx, q, scale, true);
+        k             = ggml_scale_inplace(ctx->ggml_ctx, k, scale);
+        q             = ggml_scale_inplace(ctx->ggml_ctx, q, scale);
         // auto weight = ggml_mul_mat(ctx, q, k);
         auto weight = ggml_mul_mat(ctx->ggml_ctx, k, q);  // NOTE order of mul is opposite to pytorch
 
@@ -176,9 +176,9 @@ public:
         }
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* latents,
-                         ggml_tensor* x) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* latents,
+                                struct ggml_tensor* x) {
         // x: [N, channels, h, w]
         auto proj_in  = std::dynamic_pointer_cast<Linear>(blocks["proj_in"]);
         auto proj_out = std::dynamic_pointer_cast<Linear>(blocks["proj_out"]);
@@ -225,19 +225,19 @@ public:
             4));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x,
-                         ggml_tensor* last_hidden_state) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* x,
+                                struct ggml_tensor* last_hidden_state) {
         // x: [N, channels, h, w]
         auto token_proj          = std::dynamic_pointer_cast<Mlp>(blocks["token_proj"]);
         auto token_norm          = std::dynamic_pointer_cast<LayerNorm>(blocks["token_norm"]);
         auto perceiver_resampler = std::dynamic_pointer_cast<FacePerceiverResampler>(blocks["perceiver_resampler"]);
 
-        x                = token_proj->forward(ctx, x);
-        int64_t nel      = ggml_nelements(x);
-        x                = ggml_reshape_3d(ctx->ggml_ctx, x, cross_attention_dim, num_tokens, nel / (cross_attention_dim * num_tokens));
-        x                = token_norm->forward(ctx, x);
-        ggml_tensor* out = perceiver_resampler->forward(ctx, x, last_hidden_state);
+        x                       = token_proj->forward(ctx, x);
+        int64_t nel             = ggml_nelements(x);
+        x                       = ggml_reshape_3d(ctx->ggml_ctx, x, cross_attention_dim, num_tokens, nel / (cross_attention_dim * num_tokens));
+        x                       = token_norm->forward(ctx, x);
+        struct ggml_tensor* out = perceiver_resampler->forward(ctx, x, last_hidden_state);
         if (use_residul)
             out = ggml_add(ctx->ggml_ctx, x, out);
         return out;
@@ -256,9 +256,9 @@ public:
         blocks["layer_norm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(embed_dim));
     }
 
-    ggml_tensor* fuse_fn(GGMLRunnerContext* ctx,
-                         ggml_tensor* prompt_embeds,
-                         ggml_tensor* id_embeds) {
+    struct ggml_tensor* fuse_fn(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* prompt_embeds,
+                                struct ggml_tensor* id_embeds) {
         auto mlp1       = std::dynamic_pointer_cast<FuseBlock>(blocks["mlp1"]);
         auto mlp2       = std::dynamic_pointer_cast<FuseBlock>(blocks["mlp2"]);
         auto layer_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm"]);
@@ -273,24 +273,24 @@ public:
         return stacked_id_embeds;
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* prompt_embeds,
-                         ggml_tensor* id_embeds,
-                         ggml_tensor* class_tokens_mask,
-                         ggml_tensor* class_tokens_mask_pos,
-                         ggml_tensor* left,
-                         ggml_tensor* right) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* prompt_embeds,
+                                struct ggml_tensor* id_embeds,
+                                struct ggml_tensor* class_tokens_mask,
+                                struct ggml_tensor* class_tokens_mask_pos,
+                                struct ggml_tensor* left,
+                                struct ggml_tensor* right) {
         // x: [N, channels, h, w]
 
-        ggml_tensor* valid_id_embeds = id_embeds;
+        struct ggml_tensor* valid_id_embeds = id_embeds;
         // # slice out the image token embeddings
         ggml_set_name(class_tokens_mask_pos, "class_tokens_mask_pos");
         ggml_set_name(prompt_embeds, "prompt_embeds");
-        ggml_tensor* image_token_embeds = ggml_get_rows(ctx->ggml_ctx, prompt_embeds, class_tokens_mask_pos);
+        struct ggml_tensor* image_token_embeds = ggml_get_rows(ctx->ggml_ctx, prompt_embeds, class_tokens_mask_pos);
         ggml_set_name(image_token_embeds, "image_token_embeds");
-        valid_id_embeds                = ggml_reshape_2d(ctx->ggml_ctx, valid_id_embeds, valid_id_embeds->ne[0],
-                                                         ggml_nelements(valid_id_embeds) / valid_id_embeds->ne[0]);
-        ggml_tensor* stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds);
+        valid_id_embeds                       = ggml_reshape_2d(ctx->ggml_ctx, valid_id_embeds, valid_id_embeds->ne[0],
+                                                                ggml_nelements(valid_id_embeds) / valid_id_embeds->ne[0]);
+        struct ggml_tensor* stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds);
 
         if (left && right) {
             stacked_id_embeds = ggml_concat(ctx->ggml_ctx, left, stacked_id_embeds, 1);
@@ -301,10 +301,10 @@ public:
             stacked_id_embeds = ggml_concat(ctx->ggml_ctx, stacked_id_embeds, right, 1);
         }
 
-        class_tokens_mask                  = ggml_cont(ctx->ggml_ctx, ggml_transpose(ctx->ggml_ctx, class_tokens_mask));
-        class_tokens_mask                  = ggml_repeat(ctx->ggml_ctx, class_tokens_mask, prompt_embeds);
-        prompt_embeds                      = ggml_mul(ctx->ggml_ctx, prompt_embeds, class_tokens_mask);
-        ggml_tensor* updated_prompt_embeds = ggml_add(ctx->ggml_ctx, prompt_embeds, stacked_id_embeds);
+        class_tokens_mask                         = ggml_cont(ctx->ggml_ctx, ggml_transpose(ctx->ggml_ctx, class_tokens_mask));
+        class_tokens_mask                         = ggml_repeat(ctx->ggml_ctx, class_tokens_mask, prompt_embeds);
+        prompt_embeds                             = ggml_mul(ctx->ggml_ctx, prompt_embeds, class_tokens_mask);
+        struct ggml_tensor* updated_prompt_embeds = ggml_add(ctx->ggml_ctx, prompt_embeds, stacked_id_embeds);
         ggml_set_name(updated_prompt_embeds, "updated_prompt_embeds");
         return updated_prompt_embeds;
     }
@@ -317,22 +317,22 @@ struct PhotoMakerIDEncoderBlock : public CLIPVisionModelProjection {
         blocks["fuse_module"]         = std::shared_ptr<GGMLBlock>(new FuseModule(2048));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* id_pixel_values,
-                         ggml_tensor* prompt_embeds,
-                         ggml_tensor* class_tokens_mask,
-                         ggml_tensor* class_tokens_mask_pos,
-                         ggml_tensor* left,
-                         ggml_tensor* right) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* id_pixel_values,
+                                struct ggml_tensor* prompt_embeds,
+                                struct ggml_tensor* class_tokens_mask,
+                                struct ggml_tensor* class_tokens_mask_pos,
+                                struct ggml_tensor* left,
+                                struct ggml_tensor* right) {
         // x: [N, channels, h, w]
         auto vision_model        = std::dynamic_pointer_cast<CLIPVisionModel>(blocks["vision_model"]);
         auto visual_projection   = std::dynamic_pointer_cast<CLIPProjection>(blocks["visual_projection"]);
         auto visual_projection_2 = std::dynamic_pointer_cast<Linear>(blocks["visual_projection_2"]);
         auto fuse_module         = std::dynamic_pointer_cast<FuseModule>(blocks["fuse_module"]);
 
-        ggml_tensor* shared_id_embeds = vision_model->forward(ctx, id_pixel_values);          // [N, hidden_size]
-        ggml_tensor* id_embeds        = visual_projection->forward(ctx, shared_id_embeds);    // [N, proj_dim(768)]
-        ggml_tensor* id_embeds_2      = visual_projection_2->forward(ctx, shared_id_embeds);  // [N, 1280]
+        struct ggml_tensor* shared_id_embeds = vision_model->forward(ctx, id_pixel_values);          // [N, hidden_size]
+        struct ggml_tensor* id_embeds        = visual_projection->forward(ctx, shared_id_embeds);    // [N, proj_dim(768)]
+        struct ggml_tensor* id_embeds_2      = visual_projection_2->forward(ctx, shared_id_embeds);  // [N, 1280]
 
         id_embeds   = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, id_embeds, 2, 0, 1, 3));
         id_embeds_2 = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, id_embeds_2, 2, 0, 1, 3));
@@ -340,12 +340,12 @@ struct PhotoMakerIDEncoderBlock : public CLIPVisionModelProjection {
         id_embeds = ggml_concat(ctx->ggml_ctx, id_embeds, id_embeds_2, 2);  // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right
         id_embeds = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, id_embeds, 1, 2, 0, 3));
 
-        ggml_tensor* updated_prompt_embeds = fuse_module->forward(ctx,
-                                                                  prompt_embeds,
-                                                                  id_embeds,
-                                                                  class_tokens_mask,
-                                                                  class_tokens_mask_pos,
-                                                                  left, right);
+        struct ggml_tensor* updated_prompt_embeds = fuse_module->forward(ctx,
+                                                                         prompt_embeds,
+                                                                         id_embeds,
+                                                                         class_tokens_mask,
+                                                                         class_tokens_mask_pos,
+                                                                         left, right);
         return updated_prompt_embeds;
     }
 };
@@ -365,29 +365,29 @@ struct PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock : public CLIPVisionMo
                                                                                         num_tokens));
     }
 
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* id_pixel_values,
-                         ggml_tensor* prompt_embeds,
-                         ggml_tensor* class_tokens_mask,
-                         ggml_tensor* class_tokens_mask_pos,
-                         ggml_tensor* id_embeds,
-                         ggml_tensor* left,
-                         ggml_tensor* right) {
+    struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                struct ggml_tensor* id_pixel_values,
+                                struct ggml_tensor* prompt_embeds,
+                                struct ggml_tensor* class_tokens_mask,
+                                struct ggml_tensor* class_tokens_mask_pos,
+                                struct ggml_tensor* id_embeds,
+                                struct ggml_tensor* left,
+                                struct ggml_tensor* right) {
         // x: [N, channels, h, w]
         auto vision_model      = std::dynamic_pointer_cast<CLIPVisionModel>(blocks["vision_model"]);
         auto fuse_module       = std::dynamic_pointer_cast<FuseModule>(blocks["fuse_module"]);
         auto qformer_perceiver = std::dynamic_pointer_cast<QFormerPerceiver>(blocks["qformer_perceiver"]);
 
-        // ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values);          // [N, hidden_size]
-        ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values, false);  // [N, hidden_size]
-        id_embeds                      = qformer_perceiver->forward(ctx, id_embeds, last_hidden_state);
+        // struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values);          // [N, hidden_size]
+        struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values, false);  // [N, hidden_size]
+        id_embeds                             = qformer_perceiver->forward(ctx, id_embeds, last_hidden_state);
 
-        ggml_tensor* updated_prompt_embeds = fuse_module->forward(ctx,
-                                                                  prompt_embeds,
-                                                                  id_embeds,
-                                                                  class_tokens_mask,
-                                                                  class_tokens_mask_pos,
-                                                                  left, right);
+        struct ggml_tensor* updated_prompt_embeds = fuse_module->forward(ctx,
+                                                                         prompt_embeds,
+                                                                         id_embeds,
+                                                                         class_tokens_mask,
+                                                                         class_tokens_mask_pos,
+                                                                         left, right);
         return updated_prompt_embeds;
     }
 };
@@ -411,13 +411,13 @@ public:
 
 public:
     PhotoMakerIDEncoder(ggml_backend_t backend,
-                        ggml_backend_t params_backend,
+                        bool offload_params_to_cpu,
                         const String2TensorStorage& tensor_storage_map,
                         const std::string prefix,
                         SDVersion version = VERSION_SDXL,
                         PMVersion pm_v    = PM_VERSION_1,
                         float sty         = 20.f)
-        : GGMLRunner(backend, params_backend),
+        : GGMLRunner(backend, offload_params_to_cpu),
           version(version),
           pm_version(pm_v),
           style_strength(sty) {
@@ -436,17 +436,18 @@ public:
         return pm_version;
     }
 
-    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string prefix) {
+    void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
         if (pm_version == PM_VERSION_1)
             id_encoder.get_param_tensors(tensors, prefix);
         else if (pm_version == PM_VERSION_2)
             id_encoder2.get_param_tensors(tensors, prefix);
     }
 
-    ggml_cgraph* build_graph(const sd::Tensor<float>& id_pixel_values_tensor,
-                             const sd::Tensor<float>& prompt_embeds_tensor,
-                             std::vector<bool>& class_tokens_mask,
-                             const sd::Tensor<float>& id_embeds_tensor = {}) {
+    struct ggml_cgraph* build_graph(  // struct ggml_allocr* allocr,
+        struct ggml_tensor* id_pixel_values,
+        struct ggml_tensor* prompt_embeds,
+        std::vector<bool>& class_tokens_mask,
+        struct ggml_tensor* id_embeds) {
         ctm.clear();
         ctmf16.clear();
         ctmpos.clear();
@@ -457,20 +458,20 @@ public:
 
         auto runner_ctx = get_context();
 
-        ggml_cgraph* gf = ggml_new_graph(compute_ctx);
-
-        ggml_tensor* id_pixel_values = make_input(id_pixel_values_tensor);
-        ggml_tensor* prompt_embeds   = make_input(prompt_embeds_tensor);
-        ggml_tensor* id_embeds       = make_optional_input(id_embeds_tensor);
+        struct ggml_cgraph* gf = ggml_new_graph(compute_ctx);
 
         int64_t hidden_size = prompt_embeds->ne[0];
         int64_t seq_length  = prompt_embeds->ne[1];
         ggml_type type      = GGML_TYPE_F32;
 
-        ggml_tensor* class_tokens_mask_d = ggml_new_tensor_1d(runner_ctx.ggml_ctx, type, class_tokens_mask.size());
+        struct ggml_tensor* class_tokens_mask_d = ggml_new_tensor_1d(runner_ctx.ggml_ctx, type, class_tokens_mask.size());
 
-        ggml_tensor* left  = nullptr;
-        ggml_tensor* right = nullptr;
+        struct ggml_tensor* id_pixel_values_d = to_backend(id_pixel_values);
+        struct ggml_tensor* prompt_embeds_d   = to_backend(prompt_embeds);
+        struct ggml_tensor* id_embeds_d       = to_backend(id_embeds);
+
+        struct ggml_tensor* left  = nullptr;
+        struct ggml_tensor* right = nullptr;
         for (int i = 0; i < class_tokens_mask.size(); i++) {
             if (class_tokens_mask[i]) {
                 // printf(" 1,");
@@ -494,7 +495,7 @@ public:
             right = ggml_new_tensor_3d(runner_ctx.ggml_ctx, type,
                                        hidden_size, seq_length - ctmpos[ctmpos.size() - 1] - 1, 1);
         }
-        ggml_tensor* class_tokens_mask_pos = ggml_new_tensor_1d(runner_ctx.ggml_ctx, GGML_TYPE_I32, ctmpos.size());
+        struct ggml_tensor* class_tokens_mask_pos = ggml_new_tensor_1d(runner_ctx.ggml_ctx, GGML_TYPE_I32, ctmpos.size());
 
         {
             if (type == GGML_TYPE_F16)
@@ -525,21 +526,21 @@ public:
                 }
             }
         }
-        ggml_tensor* updated_prompt_embeds = nullptr;
+        struct ggml_tensor* updated_prompt_embeds = nullptr;
         if (pm_version == PM_VERSION_1)
             updated_prompt_embeds = id_encoder.forward(&runner_ctx,
-                                                       id_pixel_values,
-                                                       prompt_embeds,
+                                                       id_pixel_values_d,
+                                                       prompt_embeds_d,
                                                        class_tokens_mask_d,
                                                        class_tokens_mask_pos,
                                                        left, right);
         else if (pm_version == PM_VERSION_2)
             updated_prompt_embeds = id_encoder2.forward(&runner_ctx,
-                                                        id_pixel_values,
-                                                        prompt_embeds,
+                                                        id_pixel_values_d,
+                                                        prompt_embeds_d,
                                                         class_tokens_mask_d,
                                                         class_tokens_mask_pos,
-                                                        id_embeds,
+                                                        id_embeds_d,
                                                         left, right);
 
         ggml_build_forward_expand(gf, updated_prompt_embeds);
@@ -547,32 +548,36 @@ public:
         return gf;
     }
 
-    sd::Tensor<float> compute(const int n_threads,
-                              const sd::Tensor<float>& id_pixel_values,
-                              const sd::Tensor<float>& prompt_embeds,
-                              const sd::Tensor<float>& id_embeds,
-                              std::vector<bool>& class_tokens_mask) {
-        auto get_graph = [&]() -> ggml_cgraph* {
+    bool compute(const int n_threads,
+                 struct ggml_tensor* id_pixel_values,
+                 struct ggml_tensor* prompt_embeds,
+                 struct ggml_tensor* id_embeds,
+                 std::vector<bool>& class_tokens_mask,
+                 struct ggml_tensor** updated_prompt_embeds,
+                 ggml_context* output_ctx) {
+        auto get_graph = [&]() -> struct ggml_cgraph* {
+            // return build_graph(compute_allocr, id_pixel_values, prompt_embeds, class_tokens_mask);
             return build_graph(id_pixel_values, prompt_embeds, class_tokens_mask, id_embeds);
         };
 
-        return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, true));
+        // GGMLRunner::compute(get_graph, n_threads, updated_prompt_embeds);
+        return GGMLRunner::compute(get_graph, n_threads, true, updated_prompt_embeds, output_ctx);
     }
 };
 
 struct PhotoMakerIDEmbed : public GGMLRunner {
-    std::map<std::string, ggml_tensor*> tensors;
+    std::map<std::string, struct ggml_tensor*> tensors;
     std::string file_path;
     ModelLoader* model_loader;
     bool load_failed = false;
     bool applied     = false;
 
     PhotoMakerIDEmbed(ggml_backend_t backend,
-                      ggml_backend_t params_backend,
+                      bool offload_params_to_cpu,
                       ModelLoader* ml,
                       const std::string& file_path = "",
                       const std::string& prefix    = "")
-        : file_path(file_path), GGMLRunner(backend, params_backend), model_loader(ml) {
+        : file_path(file_path), GGMLRunner(backend, offload_params_to_cpu), model_loader(ml) {
         if (!model_loader->init_from_file_and_convert_name(file_path, prefix)) {
             load_failed = true;
         }
@@ -601,11 +606,11 @@ struct PhotoMakerIDEmbed : public GGMLRunner {
             }
             if (dry_run) {
                 std::lock_guard<std::mutex> lock(tensor_mutex);
-                ggml_tensor* real = ggml_new_tensor(params_ctx,
-                                                    tensor_storage.type,
-                                                    tensor_storage.n_dims,
-                                                    tensor_storage.ne);
-                tensors[name]     = real;
+                struct ggml_tensor* real = ggml_new_tensor(params_ctx,
+                                                           tensor_storage.type,
+                                                           tensor_storage.n_dims,
+                                                           tensor_storage.ne);
+                tensors[name]            = real;
             } else {
                 auto real   = tensors[name];
                 *dst_tensor = real;
@@ -624,8 +629,8 @@ struct PhotoMakerIDEmbed : public GGMLRunner {
         return true;
     }
 
-    ggml_tensor* get() {
-        std::map<std::string, ggml_tensor*>::iterator pos;
+    struct ggml_tensor* get() {
+        std::map<std::string, struct ggml_tensor*>::iterator pos;
         pos = tensors.find("pmid.id_embeds");
         if (pos != tensors.end())
             return pos->second;

preprocessing.hpp

@@ -0,0 +1,226 @@
+#ifndef __PREPROCESSING_HPP__
+#define __PREPROCESSING_HPP__
+
+#include "ggml_extend.hpp"
+#define M_PI_ 3.14159265358979323846f
+
+void convolve(struct ggml_tensor* input, struct ggml_tensor* output, struct ggml_tensor* kernel, int padding) {
+    struct ggml_init_params params;
+    params.mem_size                 = 80 * input->ne[0] * input->ne[1];  // 20M for 512x512
+    params.mem_buffer               = nullptr;
+    params.no_alloc                 = false;
+    struct ggml_context* ctx0       = ggml_init(params);
+    struct ggml_tensor* kernel_fp16 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F16, kernel->ne[0], kernel->ne[1], 1, 1);
+    ggml_fp32_to_fp16_row((float*)kernel->data, (ggml_fp16_t*)kernel_fp16->data, ggml_nelements(kernel));
+    ggml_tensor* h  = ggml_conv_2d(ctx0, kernel_fp16, input, 1, 1, padding, padding, 1, 1);
+    ggml_cgraph* gf = ggml_new_graph(ctx0);
+    ggml_build_forward_expand(gf, ggml_cpy(ctx0, h, output));
+    ggml_graph_compute_with_ctx(ctx0, gf, 1);
+    ggml_free(ctx0);
+}
+
+void gaussian_kernel(struct ggml_tensor* kernel) {
+    int ks_mid   = static_cast<int>(kernel->ne[0] / 2);
+    float sigma  = 1.4f;
+    float normal = 1.f / (2.0f * M_PI_ * powf(sigma, 2.0f));
+    for (int y = 0; y < kernel->ne[0]; y++) {
+        float gx = static_cast<float>(-ks_mid + y);
+        for (int x = 0; x < kernel->ne[1]; x++) {
+            float gy = static_cast<float>(-ks_mid + x);
+            float k_ = expf(-((gx * gx + gy * gy) / (2.0f * powf(sigma, 2.0f)))) * normal;
+            ggml_ext_tensor_set_f32(kernel, k_, x, y);
+        }
+    }
+}
+
+void grayscale(struct ggml_tensor* rgb_img, struct ggml_tensor* grayscale) {
+    for (int iy = 0; iy < rgb_img->ne[1]; iy++) {
+        for (int ix = 0; ix < rgb_img->ne[0]; ix++) {
+            float r    = ggml_ext_tensor_get_f32(rgb_img, ix, iy);
+            float g    = ggml_ext_tensor_get_f32(rgb_img, ix, iy, 1);
+            float b    = ggml_ext_tensor_get_f32(rgb_img, ix, iy, 2);
+            float gray = 0.2989f * r + 0.5870f * g + 0.1140f * b;
+            ggml_ext_tensor_set_f32(grayscale, gray, ix, iy);
+        }
+    }
+}
+
+void prop_hypot(struct ggml_tensor* x, struct ggml_tensor* y, struct ggml_tensor* h) {
+    int n_elements = static_cast<int>(ggml_nelements(h));
+    float* dx      = (float*)x->data;
+    float* dy      = (float*)y->data;
+    float* dh      = (float*)h->data;
+    for (int i = 0; i < n_elements; i++) {
+        dh[i] = sqrtf(dx[i] * dx[i] + dy[i] * dy[i]);
+    }
+}
+
+void prop_arctan2(struct ggml_tensor* x, struct ggml_tensor* y, struct ggml_tensor* h) {
+    int n_elements = static_cast<int>(ggml_nelements(h));
+    float* dx      = (float*)x->data;
+    float* dy      = (float*)y->data;
+    float* dh      = (float*)h->data;
+    for (int i = 0; i < n_elements; i++) {
+        dh[i] = atan2f(dy[i], dx[i]);
+    }
+}
+
+void normalize_tensor(struct ggml_tensor* g) {
+    int n_elements = static_cast<int>(ggml_nelements(g));
+    float* dg      = (float*)g->data;
+    float max      = -INFINITY;
+    for (int i = 0; i < n_elements; i++) {
+        max = dg[i] > max ? dg[i] : max;
+    }
+    max = 1.0f / max;
+    for (int i = 0; i < n_elements; i++) {
+        dg[i] *= max;
+    }
+}
+
+void non_max_supression(struct ggml_tensor* result, struct ggml_tensor* G, struct ggml_tensor* D) {
+    for (int iy = 1; iy < result->ne[1] - 1; iy++) {
+        for (int ix = 1; ix < result->ne[0] - 1; ix++) {
+            float angle = ggml_ext_tensor_get_f32(D, ix, iy) * 180.0f / M_PI_;
+            angle       = angle < 0.0f ? angle += 180.0f : angle;
+            float q     = 1.0f;
+            float r     = 1.0f;
+
+            // angle 0
+            if ((0 >= angle && angle < 22.5f) || (157.5f >= angle && angle <= 180)) {
+                q = ggml_ext_tensor_get_f32(G, ix, iy + 1);
+                r = ggml_ext_tensor_get_f32(G, ix, iy - 1);
+            }
+            // angle 45
+            else if (22.5f >= angle && angle < 67.5f) {
+                q = ggml_ext_tensor_get_f32(G, ix + 1, iy - 1);
+                r = ggml_ext_tensor_get_f32(G, ix - 1, iy + 1);
+            }
+            // angle 90
+            else if (67.5f >= angle && angle < 112.5) {
+                q = ggml_ext_tensor_get_f32(G, ix + 1, iy);
+                r = ggml_ext_tensor_get_f32(G, ix - 1, iy);
+            }
+            // angle 135
+            else if (112.5 >= angle && angle < 157.5f) {
+                q = ggml_ext_tensor_get_f32(G, ix - 1, iy - 1);
+                r = ggml_ext_tensor_get_f32(G, ix + 1, iy + 1);
+            }
+
+            float cur = ggml_ext_tensor_get_f32(G, ix, iy);
+            if ((cur >= q) && (cur >= r)) {
+                ggml_ext_tensor_set_f32(result, cur, ix, iy);
+            } else {
+                ggml_ext_tensor_set_f32(result, 0.0f, ix, iy);
+            }
+        }
+    }
+}
+
+void threshold_hystersis(struct ggml_tensor* img, float high_threshold, float low_threshold, float weak, float strong) {
+    int n_elements = static_cast<int>(ggml_nelements(img));
+    float* imd     = (float*)img->data;
+    float max      = -INFINITY;
+    for (int i = 0; i < n_elements; i++) {
+        max = imd[i] > max ? imd[i] : max;
+    }
+    float ht = max * high_threshold;
+    float lt = ht * low_threshold;
+    for (int i = 0; i < n_elements; i++) {
+        float img_v = imd[i];
+        if (img_v >= ht) {  // strong pixel
+            imd[i] = strong;
+        } else if (img_v <= ht && img_v >= lt) {  // strong pixel
+            imd[i] = weak;
+        }
+    }
+
+    for (int iy = 0; iy < img->ne[1]; iy++) {
+        for (int ix = 0; ix < img->ne[0]; ix++) {
+            if (ix >= 3 && ix <= img->ne[0] - 3 && iy >= 3 && iy <= img->ne[1] - 3) {
+                ggml_ext_tensor_set_f32(img, ggml_ext_tensor_get_f32(img, ix, iy), ix, iy);
+            } else {
+                ggml_ext_tensor_set_f32(img, 0.0f, ix, iy);
+            }
+        }
+    }
+
+    // hysteresis
+    for (int iy = 1; iy < img->ne[1] - 1; iy++) {
+        for (int ix = 1; ix < img->ne[0] - 1; ix++) {
+            float imd_v = ggml_ext_tensor_get_f32(img, ix, iy);
+            if (imd_v == weak) {
+                if (ggml_ext_tensor_get_f32(img, ix + 1, iy - 1) == strong || ggml_ext_tensor_get_f32(img, ix + 1, iy) == strong ||
+                    ggml_ext_tensor_get_f32(img, ix, iy - 1) == strong || ggml_ext_tensor_get_f32(img, ix, iy + 1) == strong ||
+                    ggml_ext_tensor_get_f32(img, ix - 1, iy - 1) == strong || ggml_ext_tensor_get_f32(img, ix - 1, iy) == strong) {
+                    ggml_ext_tensor_set_f32(img, strong, ix, iy);
+                } else {
+                    ggml_ext_tensor_set_f32(img, 0.0f, ix, iy);
+                }
+            }
+        }
+    }
+}
+
+bool preprocess_canny(sd_image_t img, float high_threshold, float low_threshold, float weak, float strong, bool inverse) {
+    struct ggml_init_params params;
+    params.mem_size               = static_cast<size_t>(40 * img.width * img.height);  // 10MB for 512x512
+    params.mem_buffer             = nullptr;
+    params.no_alloc               = false;
+    struct ggml_context* work_ctx = ggml_init(params);
+
+    if (!work_ctx) {
+        LOG_ERROR("ggml_init() failed");
+        return false;
+    }
+
+    float kX[9] = {
+        -1, 0, 1,
+        -2, 0, 2,
+        -1, 0, 1};
+
+    float kY[9] = {
+        1, 2, 1,
+        0, 0, 0,
+        -1, -2, -1};
+
+    // generate kernel
+    int kernel_size             = 5;
+    struct ggml_tensor* gkernel = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, kernel_size, kernel_size, 1, 1);
+    struct ggml_tensor* sf_kx   = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, 3, 3, 1, 1);
+    memcpy(sf_kx->data, kX, ggml_nbytes(sf_kx));
+    struct ggml_tensor* sf_ky = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, 3, 3, 1, 1);
+    memcpy(sf_ky->data, kY, ggml_nbytes(sf_ky));
+    gaussian_kernel(gkernel);
+    struct ggml_tensor* image      = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, img.width, img.height, 3, 1);
+    struct ggml_tensor* image_gray = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, img.width, img.height, 1, 1);
+    struct ggml_tensor* iX         = ggml_dup_tensor(work_ctx, image_gray);
+    struct ggml_tensor* iY         = ggml_dup_tensor(work_ctx, image_gray);
+    struct ggml_tensor* G          = ggml_dup_tensor(work_ctx, image_gray);
+    struct ggml_tensor* tetha      = ggml_dup_tensor(work_ctx, image_gray);
+    sd_image_to_ggml_tensor(img, image);
+    grayscale(image, image_gray);
+    convolve(image_gray, image_gray, gkernel, 2);
+    convolve(image_gray, iX, sf_kx, 1);
+    convolve(image_gray, iY, sf_ky, 1);
+    prop_hypot(iX, iY, G);
+    normalize_tensor(G);
+    prop_arctan2(iX, iY, tetha);
+    non_max_supression(image_gray, G, tetha);
+    threshold_hystersis(image_gray, high_threshold, low_threshold, weak, strong);
+    // to RGB channels
+    for (uint32_t iy = 0; iy < img.height; iy++) {
+        for (uint32_t ix = 0; ix < img.width; ix++) {
+            float gray = ggml_ext_tensor_get_f32(image_gray, ix, iy);
+            gray       = inverse ? 1.0f - gray : gray;
+            ggml_ext_tensor_set_f32(image, gray, ix, iy);
+            ggml_ext_tensor_set_f32(image, gray, ix, iy, 1);
+            ggml_ext_tensor_set_f32(image, gray, ix, iy, 2);
+        }
+    }
+    ggml_tensor_to_sd_image(image, img.data);
+    ggml_free(work_ctx);
+    return true;
+}
+
+#endif  // __PREPROCESSING_HPP__

qwen_image.hpp

@@ -3,8 +3,9 @@
 
 #include <memory>
 
-#include "common_block.hpp"
+#include "common.hpp"
 #include "flux.hpp"
+#include "ggml_extend.hpp"
 
 namespace Qwen {
     constexpr int QWEN_IMAGE_GRAPH_SIZE = 20480;
@@ -26,9 +27,9 @@ namespace Qwen {
             blocks["linear_2"] = std::shared_ptr<GGMLBlock>(new Linear(time_embed_dim, out_dim, sample_proj_bias));
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* sample,
-                             ggml_tensor* condition = nullptr) {
+        struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                    struct ggml_tensor* sample,
+                                    struct ggml_tensor* condition = nullptr) {
             if (condition != nullptr) {
                 auto cond_proj = std::dynamic_pointer_cast<Linear>(blocks["cond_proj"]);
                 sample         = ggml_add(ctx->ggml_ctx, sample, cond_proj->forward(ctx, condition));
@@ -49,8 +50,8 @@ namespace Qwen {
             blocks["timestep_embedder"] = std::shared_ptr<GGMLBlock>(new TimestepEmbedding(256, embedding_dim));
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* timesteps) {
+        struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                    struct ggml_tensor* timesteps) {
             // timesteps: [N,]
             // return: [N, embedding_dim]
             auto timestep_embedder = std::dynamic_pointer_cast<TimestepEmbedding>(blocks["timestep_embedder"]);
@@ -95,7 +96,9 @@ namespace Qwen {
 
             float scale         = 1.f / 32.f;
             bool force_prec_f32 = false;
-
+#ifdef SD_USE_VULKAN
+            force_prec_f32 = true;
+#endif
             // The purpose of the scale here is to prevent NaN issues in certain situations.
             // For example when using CUDA but the weights are k-quants (not all prompts).
             blocks["to_out.0"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, out_dim, out_bias, false, force_prec_f32, scale));
@@ -105,10 +108,10 @@ namespace Qwen {
         }
 
         std::pair<ggml_tensor*, ggml_tensor*> forward(GGMLRunnerContext* ctx,
-                                                      ggml_tensor* img,
-                                                      ggml_tensor* txt,
-                                                      ggml_tensor* pe,
-                                                      ggml_tensor* mask = nullptr) {
+                                                      struct ggml_tensor* img,
+                                                      struct ggml_tensor* txt,
+                                                      struct ggml_tensor* pe,
+                                                      struct ggml_tensor* mask = nullptr) {
             // img: [N, n_img_token, hidden_size]
             // txt: [N, n_txt_token, hidden_size]
             // pe: [n_img_token + n_txt_token, d_head/2, 2, 2]
@@ -122,10 +125,6 @@ namespace Qwen {
             auto to_v     = std::dynamic_pointer_cast<Linear>(blocks["to_v"]);
             auto to_out_0 = std::dynamic_pointer_cast<Linear>(blocks["to_out.0"]);
 
-            if (sd_backend_is(ctx->backend, "Vulkan")) {
-                to_out_0->set_force_prec_f32(true);
-            }
-
             auto norm_added_q = std::dynamic_pointer_cast<UnaryBlock>(blocks["norm_added_q"]);
             auto norm_added_k = std::dynamic_pointer_cast<UnaryBlock>(blocks["norm_added_k"]);
 
@@ -163,25 +162,26 @@ namespace Qwen {
             auto k = ggml_concat(ctx->ggml_ctx, txt_k, img_k, 2);  // [N, n_txt_token + n_img_token, n_head, d_head]
             auto v = ggml_concat(ctx->ggml_ctx, txt_v, img_v, 2);  // [N, n_txt_token + n_img_token, n_head, d_head]
 
-            auto attn         = Rope::attention(ctx, q, k, v, pe, mask, (1.0f / 128.f));  // [N, n_txt_token + n_img_token, n_head*d_head]
+            auto attn         = Rope::attention(ctx, q, k, v, pe, mask, (1.0f / 128.f));                  // [N, n_txt_token + n_img_token, n_head*d_head]
+            attn              = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, attn, 0, 2, 1, 3));  // [n_txt_token + n_img_token, N, hidden_size]
             auto txt_attn_out = ggml_view_3d(ctx->ggml_ctx,
                                              attn,
                                              attn->ne[0],
+                                             attn->ne[1],
                                              txt->ne[1],
-                                             attn->ne[2],
                                              attn->nb[1],
                                              attn->nb[2],
-                                             0);  // [N, n_txt_token, n_head*d_head]
+                                             0);                                                                  // [n_txt_token, N, hidden_size]
+            txt_attn_out      = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, txt_attn_out, 0, 2, 1, 3));  // [N, n_txt_token, hidden_size]
             auto img_attn_out = ggml_view_3d(ctx->ggml_ctx,
                                              attn,
                                              attn->ne[0],
+                                             attn->ne[1],
                                              img->ne[1],
-                                             attn->ne[2],
                                              attn->nb[1],
                                              attn->nb[2],
-                                             txt->ne[1] * attn->nb[1]);  // [N, n_img_token, n_head*d_head]
-            img_attn_out      = ggml_cont(ctx->ggml_ctx, img_attn_out);
-            txt_attn_out      = ggml_cont(ctx->ggml_ctx, txt_attn_out);
+                                             attn->nb[2] * txt->ne[1]);                                           // [n_img_token, N, hidden_size]
+            img_attn_out      = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, img_attn_out, 0, 2, 1, 3));  // [N, n_img_token, hidden_size]
 
             img_attn_out = to_out_0->forward(ctx, img_attn_out);
             txt_attn_out = to_add_out->forward(ctx, txt_attn_out);
@@ -213,7 +213,7 @@ namespace Qwen {
 
             blocks["txt_norm1"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim, eps, false));
             blocks["txt_norm2"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim, eps, false));
-            blocks["txt_mlp"]   = std::shared_ptr<GGMLBlock>(new FeedForward(dim, dim, 4, FeedForward::Activation::GELU, true));
+            blocks["txt_mlp"]   = std::shared_ptr<GGMLBlock>(new FeedForward(dim, dim, 4, FeedForward::Activation::GELU));
 
             blocks["attn"] = std::shared_ptr<GGMLBlock>(new QwenImageAttention(dim,
                                                                                attention_head_dim,
@@ -251,11 +251,11 @@ namespace Qwen {
         }
 
         virtual std::pair<ggml_tensor*, ggml_tensor*> forward(GGMLRunnerContext* ctx,
-                                                              ggml_tensor* img,
-                                                              ggml_tensor* txt,
-                                                              ggml_tensor* t_emb,
-                                                              ggml_tensor* pe,
-                                                              ggml_tensor* modulate_index = nullptr) {
+                                                              struct ggml_tensor* img,
+                                                              struct ggml_tensor* txt,
+                                                              struct ggml_tensor* t_emb,
+                                                              struct ggml_tensor* pe,
+                                                              struct ggml_tensor* modulate_index = nullptr) {
             // img: [N, n_img_token, hidden_size]
             // txt: [N, n_txt_token, hidden_size]
             // pe: [n_img_token + n_txt_token, d_head/2, 2, 2]
@@ -327,9 +327,9 @@ namespace Qwen {
             blocks["linear"] = std::shared_ptr<GGMLBlock>(new Linear(conditioning_embedding_dim, embedding_dim * 2, bias));
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* x,
-                             ggml_tensor* c) {
+        struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                    struct ggml_tensor* x,
+                                    struct ggml_tensor* c) {
             // x: [N, n_token, hidden_size]
             // c: [N, hidden_size]
             // return: [N, n_token, patch_size * patch_size * out_channels]
@@ -391,12 +391,75 @@ namespace Qwen {
             blocks["proj_out"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, params.patch_size * params.patch_size * params.out_channels));
         }
 
-        ggml_tensor* forward_orig(GGMLRunnerContext* ctx,
-                                  ggml_tensor* x,
-                                  ggml_tensor* timestep,
-                                  ggml_tensor* context,
-                                  ggml_tensor* pe,
-                                  ggml_tensor* modulate_index = nullptr) {
+        struct ggml_tensor* pad_to_patch_size(GGMLRunnerContext* ctx,
+                                              struct ggml_tensor* x) {
+            int64_t W = x->ne[0];
+            int64_t H = x->ne[1];
+
+            int pad_h = (params.patch_size - H % params.patch_size) % params.patch_size;
+            int pad_w = (params.patch_size - W % params.patch_size) % params.patch_size;
+            x         = ggml_ext_pad(ctx->ggml_ctx, x, pad_w, pad_h, 0, 0, ctx->circular_x_enabled, ctx->circular_y_enabled);
+            return x;
+        }
+
+        struct ggml_tensor* patchify(struct ggml_context* ctx,
+                                     struct ggml_tensor* x) {
+            // x: [N, C, H, W]
+            // return: [N, h*w, C * patch_size * patch_size]
+            int64_t N = x->ne[3];
+            int64_t C = x->ne[2];
+            int64_t H = x->ne[1];
+            int64_t W = x->ne[0];
+            int64_t p = params.patch_size;
+            int64_t h = H / params.patch_size;
+            int64_t w = W / params.patch_size;
+
+            GGML_ASSERT(h * p == H && w * p == W);
+
+            x = ggml_reshape_4d(ctx, x, p, w, p, h * C * N);       // [N*C*h, p, w, p]
+            x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));  // [N*C*h, w, p, p]
+            x = ggml_reshape_4d(ctx, x, p * p, w * h, C, N);       // [N, C, h*w, p*p]
+            x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));  // [N, h*w, C, p*p]
+            x = ggml_reshape_3d(ctx, x, p * p * C, w * h, N);      // [N, h*w, C*p*p]
+            return x;
+        }
+
+        struct ggml_tensor* process_img(GGMLRunnerContext* ctx,
+                                        struct ggml_tensor* x) {
+            x = pad_to_patch_size(ctx, x);
+            x = patchify(ctx->ggml_ctx, x);
+            return x;
+        }
+
+        struct ggml_tensor* unpatchify(struct ggml_context* ctx,
+                                       struct ggml_tensor* x,
+                                       int64_t h,
+                                       int64_t w) {
+            // x: [N, h*w, C*patch_size*patch_size]
+            // return: [N, C, H, W]
+            int64_t N = x->ne[2];
+            int64_t C = x->ne[0] / params.patch_size / params.patch_size;
+            int64_t H = h * params.patch_size;
+            int64_t W = w * params.patch_size;
+            int64_t p = params.patch_size;
+
+            GGML_ASSERT(C * p * p == x->ne[0]);
+
+            x = ggml_reshape_4d(ctx, x, p * p, C, w * h, N);       // [N, h*w, C, p*p]
+            x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));  // [N, C, h*w, p*p]
+            x = ggml_reshape_4d(ctx, x, p, p, w, h * C * N);       // [N*C*h, w, p, p]
+            x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));  // [N*C*h, p, w, p]
+            x = ggml_reshape_4d(ctx, x, W, H, C, N);               // [N, C, h*p, w*p]
+
+            return x;
+        }
+
+        struct ggml_tensor* forward_orig(GGMLRunnerContext* ctx,
+                                         struct ggml_tensor* x,
+                                         struct ggml_tensor* timestep,
+                                         struct ggml_tensor* context,
+                                         struct ggml_tensor* pe,
+                                         struct ggml_tensor* modulate_index = nullptr) {
             auto time_text_embed = std::dynamic_pointer_cast<QwenTimestepProjEmbeddings>(blocks["time_text_embed"]);
             auto txt_norm        = std::dynamic_pointer_cast<RMSNorm>(blocks["txt_norm"]);
             auto img_in          = std::dynamic_pointer_cast<Linear>(blocks["img_in"]);
@@ -406,15 +469,12 @@ namespace Qwen {
 
             auto t_emb = time_text_embed->forward(ctx, timestep);
             if (params.zero_cond_t) {
-                auto t_emb_0 = time_text_embed->forward(ctx, ggml_ext_zeros_like(ctx->ggml_ctx, timestep));
+                auto t_emb_0 = time_text_embed->forward(ctx, ggml_ext_zeros(ctx->ggml_ctx, timestep->ne[0], timestep->ne[1], timestep->ne[2], timestep->ne[3]));
                 t_emb        = ggml_concat(ctx->ggml_ctx, t_emb, t_emb_0, 1);
             }
             auto img = img_in->forward(ctx, x);
             auto txt = txt_norm->forward(ctx, context);
             txt      = txt_in->forward(ctx, txt);
-            sd::ggml_graph_cut::mark_graph_cut(img, "qwen_image.prelude", "img");
-            sd::ggml_graph_cut::mark_graph_cut(txt, "qwen_image.prelude", "txt");
-            // sd::ggml_graph_cut::mark_graph_cut(t_emb, "qwen_image.prelude", "t_emb");
 
             for (int i = 0; i < params.num_layers; i++) {
                 auto block = std::dynamic_pointer_cast<QwenImageTransformerBlock>(blocks["transformer_blocks." + std::to_string(i)]);
@@ -422,8 +482,6 @@ namespace Qwen {
                 auto result = block->forward(ctx, img, txt, t_emb, pe, modulate_index);
                 img         = result.first;
                 txt         = result.second;
-                sd::ggml_graph_cut::mark_graph_cut(img, "qwen_image.transformer_blocks." + std::to_string(i), "img");
-                sd::ggml_graph_cut::mark_graph_cut(txt, "qwen_image.transformer_blocks." + std::to_string(i), "txt");
             }
 
             if (params.zero_cond_t) {
@@ -436,13 +494,13 @@ namespace Qwen {
             return img;
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* x,
-                             ggml_tensor* timestep,
-                             ggml_tensor* context,
-                             ggml_tensor* pe,
-                             std::vector<ggml_tensor*> ref_latents = {},
-                             ggml_tensor* modulate_index           = nullptr) {
+        struct ggml_tensor* forward(GGMLRunnerContext* ctx,
+                                    struct ggml_tensor* x,
+                                    struct ggml_tensor* timestep,
+                                    struct ggml_tensor* context,
+                                    struct ggml_tensor* pe,
+                                    std::vector<ggml_tensor*> ref_latents = {},
+                                    struct ggml_tensor* modulate_index    = nullptr) {
             // Forward pass of DiT.
             // x: [N, C, H, W]
             // timestep: [N,]
@@ -455,16 +513,19 @@ namespace Qwen {
             int64_t C = x->ne[2];
             int64_t N = x->ne[3];
 
-            auto img           = DiT::pad_and_patchify(ctx, x, params.patch_size, params.patch_size);
+            auto img           = process_img(ctx, x);
             int64_t img_tokens = img->ne[1];
 
             if (ref_latents.size() > 0) {
                 for (ggml_tensor* ref : ref_latents) {
-                    ref = DiT::pad_and_patchify(ctx, ref, params.patch_size, params.patch_size);
+                    ref = process_img(ctx, ref);
                     img = ggml_concat(ctx->ggml_ctx, img, ref, 1);
                 }
             }
 
+            int64_t h_len = ((H + (params.patch_size / 2)) / params.patch_size);
+            int64_t w_len = ((W + (params.patch_size / 2)) / params.patch_size);
+
             auto out = forward_orig(ctx, img, timestep, context, pe, modulate_index);  // [N, h_len*w_len, ph*pw*C]
 
             if (out->ne[1] > img_tokens) {
@@ -473,7 +534,11 @@ namespace Qwen {
                 out = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, out, 0, 2, 1, 3));  // [N, h*w, C * patch_size * patch_size]
             }
 
-            out = DiT::unpatchify_and_crop(ctx->ggml_ctx, out, H, W, params.patch_size, params.patch_size);  // [N, C, H, W]
+            out = unpatchify(ctx->ggml_ctx, out, h_len, w_len);  // [N, C, H + pad_h, W + pad_w]
+
+            // slice
+            out = ggml_ext_slice(ctx->ggml_ctx, out, 1, 0, H);  // [N, C, H, W + pad_w]
+            out = ggml_ext_slice(ctx->ggml_ctx, out, 0, 0, W);  // [N, C, H, W]
 
             return out;
         }
@@ -488,12 +553,12 @@ namespace Qwen {
         SDVersion version;
 
         QwenImageRunner(ggml_backend_t backend,
-                        ggml_backend_t params_backend,
+                        bool offload_params_to_cpu,
                         const String2TensorStorage& tensor_storage_map = {},
                         const std::string prefix                       = "",
                         SDVersion version                              = VERSION_QWEN_IMAGE,
                         bool zero_cond_t                               = false)
-            : GGMLRunner(backend, params_backend) {
+            : GGMLRunner(backend, offload_params_to_cpu) {
             qwen_image_params.num_layers  = 0;
             qwen_image_params.zero_cond_t = zero_cond_t;
             for (auto pair : tensor_storage_map) {
@@ -528,25 +593,24 @@ namespace Qwen {
             return "qwen_image";
         }
 
-        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string prefix) {
+        void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
             qwen_image.get_param_tensors(tensors, prefix);
         }
 
-        ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
-                                 const sd::Tensor<float>& timesteps_tensor,
-                                 const sd::Tensor<float>& context_tensor,
-                                 const std::vector<sd::Tensor<float>>& ref_latents_tensor = {},
-                                 bool increase_ref_index                                  = false) {
-            ggml_cgraph* gf        = new_graph_custom(QWEN_IMAGE_GRAPH_SIZE);
-            ggml_tensor* x         = make_input(x_tensor);
-            ggml_tensor* timesteps = make_input(timesteps_tensor);
+        struct ggml_cgraph* build_graph(struct ggml_tensor* x,
+                                        struct ggml_tensor* timesteps,
+                                        struct ggml_tensor* context,
+                                        std::vector<ggml_tensor*> ref_latents = {},
+                                        bool increase_ref_index               = false) {
             GGML_ASSERT(x->ne[3] == 1);
-            GGML_ASSERT(!context_tensor.empty());
-            ggml_tensor* context = make_input(context_tensor);
-            std::vector<ggml_tensor*> ref_latents;
-            ref_latents.reserve(ref_latents_tensor.size());
-            for (const auto& ref_latent_tensor : ref_latents_tensor) {
-                ref_latents.push_back(make_input(ref_latent_tensor));
+            struct ggml_cgraph* gf = new_graph_custom(QWEN_IMAGE_GRAPH_SIZE);
+
+            x         = to_backend(x);
+            context   = to_backend(context);
+            timesteps = to_backend(timesteps);
+
+            for (int i = 0; i < ref_latents.size(); i++) {
+                ref_latents[i] = to_backend(ref_latents[i]);
             }
 
             pe_vec      = Rope::gen_qwen_image_pe(static_cast<int>(x->ne[1]),
@@ -595,72 +659,67 @@ namespace Qwen {
 
             auto runner_ctx = get_context();
 
-            ggml_tensor* out = qwen_image.forward(&runner_ctx,
-                                                  x,
-                                                  timesteps,
-                                                  context,
-                                                  pe,
-                                                  ref_latents,
-                                                  modulate_index);
+            struct ggml_tensor* out = qwen_image.forward(&runner_ctx,
+                                                         x,
+                                                         timesteps,
+                                                         context,
+                                                         pe,
+                                                         ref_latents,
+                                                         modulate_index);
 
             ggml_build_forward_expand(gf, out);
 
             return gf;
         }
 
-        sd::Tensor<float> compute(int n_threads,
-                                  const sd::Tensor<float>& x,
-                                  const sd::Tensor<float>& timesteps,
-                                  const sd::Tensor<float>& context,
-                                  const std::vector<sd::Tensor<float>>& ref_latents = {},
-                                  bool increase_ref_index                           = false) {
+        bool compute(int n_threads,
+                     struct ggml_tensor* x,
+                     struct ggml_tensor* timesteps,
+                     struct ggml_tensor* context,
+                     std::vector<ggml_tensor*> ref_latents = {},
+                     bool increase_ref_index               = false,
+                     struct ggml_tensor** output           = nullptr,
+                     struct ggml_context* output_ctx       = nullptr) {
             // x: [N, in_channels, h, w]
             // timesteps: [N, ]
             // context: [N, max_position, hidden_size]
-            auto get_graph = [&]() -> ggml_cgraph* {
+            auto get_graph = [&]() -> struct ggml_cgraph* {
                 return build_graph(x, timesteps, context, ref_latents, increase_ref_index);
             };
 
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
+            return GGMLRunner::compute(get_graph, n_threads, false, output, output_ctx);
         }
 
         void test() {
-            ggml_init_params params;
+            struct ggml_init_params params;
             params.mem_size   = static_cast<size_t>(1024 * 1024) * 1024;  // 1GB
             params.mem_buffer = nullptr;
             params.no_alloc   = false;
 
-            ggml_context* ctx = ggml_init(params);
-            GGML_ASSERT(ctx != nullptr);
+            struct ggml_context* work_ctx = ggml_init(params);
+            GGML_ASSERT(work_ctx != nullptr);
 
             {
-                // auto x = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 16, 16, 16, 1);
+                // auto x = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, 16, 16, 16, 1);
                 // ggml_set_f32(x, 0.01f);
-                auto x = sd::load_tensor_from_file_as_tensor<float>("./qwen_image_x.bin");
-                print_sd_tensor(x);
+                auto x = load_tensor_from_file(work_ctx, "./qwen_image_x.bin");
+                print_ggml_tensor(x);
 
                 std::vector<float> timesteps_vec(1, 1000.f);
-                auto timesteps = sd::Tensor<float>::from_vector(timesteps_vec);
+                auto timesteps = vector_to_ggml_tensor(work_ctx, timesteps_vec);
 
-                // auto context = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 3584, 256, 1);
+                // auto context = ggml_new_tensor_3d(work_ctx, GGML_TYPE_F32, 3584, 256, 1);
                 // ggml_set_f32(context, 0.01f);
-                auto context = sd::load_tensor_from_file_as_tensor<float>("./qwen_image_context.bin");
-                print_sd_tensor(context);
+                auto context = load_tensor_from_file(work_ctx, "./qwen_image_context.bin");
+                print_ggml_tensor(context);
 
-                sd::Tensor<float> out;
+                struct ggml_tensor* out = nullptr;
 
-                int64_t t0   = ggml_time_ms();
-                auto out_opt = compute(8,
-                                       x,
-                                       timesteps,
-                                       context,
-                                       {},
-                                       false);
-                int64_t t1   = ggml_time_ms();
+                int64_t t0 = ggml_time_ms();
+                compute(8, x, timesteps, context, {}, false, &out, work_ctx);
+                int64_t t1 = ggml_time_ms();
 
-                GGML_ASSERT(!out_opt.empty());
-                out = std::move(out_opt);
-                print_sd_tensor(out);
+                print_ggml_tensor(out);
                 LOG_DEBUG("qwen_image test done in %lldms", t1 - t0);
             }
         }
@@ -686,7 +745,7 @@ namespace Qwen {
             }
 
             std::shared_ptr<QwenImageRunner> qwen_image = std::make_shared<QwenImageRunner>(backend,
-                                                                                            backend,
+                                                                                            false,
                                                                                             tensor_storage_map,
                                                                                             "model.diffusion_model",
                                                                                             VERSION_QWEN_IMAGE);

rope.hpp

@@ -7,11 +7,6 @@
 #include "ggml_extend.hpp"
 
 namespace Rope {
-    enum class EmbedNDLayout {
-        Matrix,
-        ErnieImage,
-    };
-
     template <class T>
     __STATIC_INLINE__ std::vector<T> linspace(T start, T end, int num) {
         std::vector<T> result(num);
@@ -48,7 +43,7 @@ namespace Rope {
 
     __STATIC_INLINE__ std::vector<std::vector<float>> rope(const std::vector<float>& pos,
                                                            int dim,
-                                                           float theta,
+                                                           int theta,
                                                            const std::vector<int>& axis_wrap_dims = {}) {
         assert(dim % 2 == 0);
         int half_dim = dim / 2;
@@ -172,10 +167,9 @@ namespace Rope {
 
     __STATIC_INLINE__ std::vector<float> embed_nd(const std::vector<std::vector<float>>& ids,
                                                   int bs,
-                                                  const std::vector<float>& axis_thetas,
+                                                  int theta,
                                                   const std::vector<int>& axes_dim,
-                                                  const std::vector<std::vector<int>>& wrap_dims = {},
-                                                  EmbedNDLayout layout                           = EmbedNDLayout::Matrix) {
+                                                  const std::vector<std::vector<int>>& wrap_dims = {}) {
         std::vector<std::vector<float>> trans_ids = transpose(ids);
         size_t pos_len                            = ids.size() / bs;
         size_t num_axes                           = axes_dim.size();
@@ -194,12 +188,8 @@ namespace Rope {
             if (!wrap_dims.empty() && i < (int)wrap_dims.size()) {
                 axis_wrap_dims = wrap_dims[i];
             }
-            float axis_theta = 10000.0f;
-            if (!axis_thetas.empty()) {
-                axis_theta = axis_thetas[std::min(i, axis_thetas.size() - 1)];
-            }
             std::vector<std::vector<float>> rope_emb =
-                rope(trans_ids[i], axes_dim[i], axis_theta, axis_wrap_dims);  // [bs*pos_len, axes_dim[i]/2 * 2 * 2]
+                rope(trans_ids[i], axes_dim[i], theta, axis_wrap_dims);  // [bs*pos_len, axes_dim[i]/2 * 2 * 2]
             for (int b = 0; b < bs; ++b) {
                 for (int j = 0; j < pos_len; ++j) {
                     for (int k = 0; k < rope_emb[0].size(); ++k) {
@@ -210,37 +200,9 @@ namespace Rope {
             offset += rope_emb[0].size();
         }
 
-        if (layout == EmbedNDLayout::ErnieImage) {
-            int head_dim = emb_dim * 2;
-            std::vector<float> ernie_emb(bs * pos_len * head_dim * 2, 0.0f);
-            for (size_t pos_idx = 0; pos_idx < bs * pos_len; ++pos_idx) {
-                for (int i = 0; i < emb_dim; ++i) {
-                    float cos_val             = emb[pos_idx][4 * i];
-                    float sin_val             = emb[pos_idx][4 * i + 2];
-                    size_t cos_offset         = pos_idx * head_dim + 2 * i;
-                    size_t sin_offset         = bs * pos_len * head_dim + cos_offset;
-                    ernie_emb[cos_offset]     = cos_val;
-                    ernie_emb[cos_offset + 1] = cos_val;
-                    ernie_emb[sin_offset]     = sin_val;
-                    ernie_emb[sin_offset + 1] = sin_val;
-                }
-            }
-            return ernie_emb;
-        }
-
         return flatten(emb);
     }
 
-    __STATIC_INLINE__ std::vector<float> embed_nd(const std::vector<std::vector<float>>& ids,
-                                                  int bs,
-                                                  float theta,
-                                                  const std::vector<int>& axes_dim,
-                                                  const std::vector<std::vector<int>>& wrap_dims = {},
-                                                  EmbedNDLayout layout                           = EmbedNDLayout::Matrix) {
-        std::vector<float> axis_thetas(axes_dim.size(), theta);
-        return embed_nd(ids, bs, axis_thetas, axes_dim, wrap_dims, layout);
-    }
-
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_refs_ids(int patch_size,
                                                                    int bs,
                                                                    int axes_dim_num,
@@ -370,7 +332,7 @@ namespace Rope {
                 }
             }
         }
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
+        return embed_nd(ids, bs, theta, axes_dim, wrap_dims);
     }
 
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_qwen_image_ids(int h,
@@ -459,75 +421,7 @@ namespace Rope {
                 }
             }
         }
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
-    }
-
-    __STATIC_INLINE__ std::vector<std::vector<float>> gen_ernie_image_ids(int h,
-                                                                          int w,
-                                                                          int patch_size,
-                                                                          int bs,
-                                                                          int context_len) {
-        int h_len = h / patch_size;
-        int w_len = w / patch_size;
-
-        std::vector<std::vector<float>> img_ids(h_len * w_len, std::vector<float>(3, 0.0f));
-        std::vector<float> h_ids = linspace<float>(0.f, static_cast<float>(h_len - 1), h_len);
-        std::vector<float> w_ids = linspace<float>(0.f, static_cast<float>(w_len - 1), w_len);
-        for (int i = 0; i < h_len; ++i) {
-            for (int j = 0; j < w_len; ++j) {
-                img_ids[i * w_len + j][0] = static_cast<float>(context_len);
-                img_ids[i * w_len + j][1] = h_ids[i];
-                img_ids[i * w_len + j][2] = w_ids[j];
-            }
-        }
-
-        std::vector<std::vector<float>> img_ids_repeated(bs * img_ids.size(), std::vector<float>(3, 0.0f));
-        for (int i = 0; i < bs; ++i) {
-            for (int j = 0; j < static_cast<int>(img_ids.size()); ++j) {
-                img_ids_repeated[i * img_ids.size() + j] = img_ids[j];
-            }
-        }
-
-        std::vector<std::vector<float>> txt_ids(bs * context_len, std::vector<float>(3, 0.0f));
-        for (int i = 0; i < bs; ++i) {
-            for (int j = 0; j < context_len; ++j) {
-                txt_ids[i * context_len + j][0] = static_cast<float>(j);
-            }
-        }
-
-        return concat_ids(img_ids_repeated, txt_ids, bs);
-    }
-
-    __STATIC_INLINE__ std::vector<float> gen_ernie_image_pe(int h,
-                                                            int w,
-                                                            int patch_size,
-                                                            int bs,
-                                                            int context_len,
-                                                            int theta,
-                                                            bool circular_h,
-                                                            bool circular_w,
-                                                            const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids = gen_ernie_image_ids(h, w, patch_size, bs, context_len);
-        std::vector<std::vector<int>> wrap_dims;
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            int h_len = h / patch_size;
-            int w_len = w / patch_size;
-            if (h_len > 0 && w_len > 0) {
-                size_t pos_len = ids.size() / bs;
-                wrap_dims.assign(axes_dim.size(), std::vector<int>(pos_len, 0));
-                const size_t img_tokens = static_cast<size_t>(h_len) * static_cast<size_t>(w_len);
-                for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                    if (circular_h) {
-                        wrap_dims[1][token_i] = h_len;
-                    }
-                    if (circular_w) {
-                        wrap_dims[2][token_i] = w_len;
-                    }
-                }
-            }
-        }
-
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims, EmbedNDLayout::ErnieImage);
+        return embed_nd(ids, bs, theta, axes_dim, wrap_dims);
     }
 
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_vid_ids(int t,
@@ -581,7 +475,7 @@ namespace Rope {
                                                     int theta,
                                                     const std::vector<int>& axes_dim) {
         std::vector<std::vector<float>> ids = gen_vid_ids(t, h, w, pt, ph, pw, bs);
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim);
+        return embed_nd(ids, bs, theta, axes_dim);
     }
 
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_qwen2vl_ids(int grid_h,
@@ -617,7 +511,7 @@ namespace Rope {
                                                         int theta,
                                                         const std::vector<int>& axes_dim) {
         std::vector<std::vector<float>> ids = gen_qwen2vl_ids(grid_h, grid_w, merge_size, window_index);
-        return embed_nd(ids, 1, static_cast<float>(theta), axes_dim);
+        return embed_nd(ids, 1, theta, axes_dim);
     }
 
     __STATIC_INLINE__ int bound_mod(int a, int m) {
@@ -690,13 +584,13 @@ namespace Rope {
             }
         }
 
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
+        return embed_nd(ids, bs, theta, axes_dim, wrap_dims);
     }
 
-    __STATIC_INLINE__ ggml_tensor* apply_rope(ggml_context* ctx,
-                                              ggml_tensor* x,
-                                              ggml_tensor* pe,
-                                              bool rope_interleaved = true) {
+    __STATIC_INLINE__ struct ggml_tensor* apply_rope(struct ggml_context* ctx,
+                                                     struct ggml_tensor* x,
+                                                     struct ggml_tensor* pe,
+                                                     bool rope_interleaved = true) {
         // x: [N, L, n_head, d_head]
         // pe: [L, d_head/2, 2, 2], [[cos, -sin], [sin, cos]]
         int64_t d_head = x->ne[0];
@@ -734,21 +628,21 @@ namespace Rope {
         return x_out;
     }
 
-    __STATIC_INLINE__ ggml_tensor* attention(GGMLRunnerContext* ctx,
-                                             ggml_tensor* q,
-                                             ggml_tensor* k,
-                                             ggml_tensor* v,
-                                             ggml_tensor* pe,
-                                             ggml_tensor* mask,
-                                             float kv_scale        = 1.0f,
-                                             bool rope_interleaved = true) {
+    __STATIC_INLINE__ struct ggml_tensor* attention(GGMLRunnerContext* ctx,
+                                                    struct ggml_tensor* q,
+                                                    struct ggml_tensor* k,
+                                                    struct ggml_tensor* v,
+                                                    struct ggml_tensor* pe,
+                                                    struct ggml_tensor* mask,
+                                                    float kv_scale        = 1.0f,
+                                                    bool rope_interleaved = true) {
         // q,k,v: [N, L, n_head, d_head]
         // pe: [L, d_head/2, 2, 2]
         // return: [N, L, n_head*d_head]
         q = apply_rope(ctx->ggml_ctx, q, pe, rope_interleaved);  // [N*n_head, L, d_head]
         k = apply_rope(ctx->ggml_ctx, k, pe, rope_interleaved);  // [N*n_head, L, d_head]
 
-        auto x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, v->ne[1], mask, true, ctx->flash_attn_enabled, kv_scale);  // [N, L, n_head*d_head]
+        auto x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, v->ne[1], mask, false, true, ctx->flash_attn_enabled, kv_scale);  // [N, L, n_head*d_head]
         return x;
     }
 };  // namespace Rope

src/anima.hpp

@@ -1,689 +0,0 @@
-#ifndef __ANIMA_HPP__
-#define __ANIMA_HPP__
-
-#include <cmath>
-#include <memory>
-#include <utility>
-#include <vector>
-
-#include "common_block.hpp"
-#include "flux.hpp"
-#include "rope.hpp"
-
-namespace Anima {
-    constexpr int ANIMA_GRAPH_SIZE = 65536;
-
-    __STATIC_INLINE__ ggml_tensor* apply_gate(ggml_context* ctx,
-                                              ggml_tensor* x,
-                                              ggml_tensor* gate) {
-        gate = ggml_reshape_3d(ctx, gate, gate->ne[0], 1, gate->ne[1]);  // [N, 1, C]
-        return ggml_mul(ctx, x, gate);
-    }
-
-    struct XEmbedder : public GGMLBlock {
-    public:
-        XEmbedder(int64_t in_dim, int64_t out_dim) {
-            blocks["proj.1"] = std::make_shared<Linear>(in_dim, out_dim, false);
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
-            auto proj = std::dynamic_pointer_cast<Linear>(blocks["proj.1"]);
-            return proj->forward(ctx, x);
-        }
-    };
-
-    struct TimestepEmbedder : public GGMLBlock {
-    public:
-        TimestepEmbedder(int64_t in_dim, int64_t out_dim) {
-            blocks["1.linear_1"] = std::make_shared<Linear>(in_dim, in_dim, false);
-            blocks["1.linear_2"] = std::make_shared<Linear>(in_dim, out_dim, false);
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
-            auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["1.linear_1"]);
-            auto linear_2 = std::dynamic_pointer_cast<Linear>(blocks["1.linear_2"]);
-
-            x = linear_1->forward(ctx, x);
-            x = ggml_silu_inplace(ctx->ggml_ctx, x);
-            x = linear_2->forward(ctx, x);
-            return x;
-        }
-    };
-
-    struct AdaLayerNormZero : public GGMLBlock {
-    protected:
-        int64_t in_features;
-
-    public:
-        AdaLayerNormZero(int64_t in_features, int64_t hidden_features = 256)
-            : in_features(in_features) {
-            blocks["norm"] = std::make_shared<LayerNorm>(in_features, 1e-6f, false, false);
-            blocks["1"]    = std::make_shared<Linear>(in_features, hidden_features, false);
-            blocks["2"]    = std::make_shared<Linear>(hidden_features, 3 * in_features, false);
-        }
-
-        std::pair<ggml_tensor*, ggml_tensor*> forward(GGMLRunnerContext* ctx,
-                                                      ggml_tensor* hidden_states,
-                                                      ggml_tensor* embedded_timestep,
-                                                      ggml_tensor* temb = nullptr) {
-            auto norm     = std::dynamic_pointer_cast<LayerNorm>(blocks["norm"]);
-            auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["1"]);
-            auto linear_2 = std::dynamic_pointer_cast<Linear>(blocks["2"]);
-
-            auto emb = ggml_silu(ctx->ggml_ctx, embedded_timestep);
-            emb      = linear_1->forward(ctx, emb);
-            emb      = linear_2->forward(ctx, emb);  // [N, 3*C]
-
-            if (temb != nullptr) {
-                emb = ggml_add(ctx->ggml_ctx, emb, temb);
-            }
-
-            auto emb_chunks = ggml_ext_chunk(ctx->ggml_ctx, emb, 3, 0);
-            auto shift      = emb_chunks[0];
-            auto scale      = emb_chunks[1];
-            auto gate       = emb_chunks[2];
-
-            auto x = norm->forward(ctx, hidden_states);
-            x      = Flux::modulate(ctx->ggml_ctx, x, shift, scale);
-
-            return {x, gate};
-        }
-    };
-
-    struct AdaLayerNorm : public GGMLBlock {
-    protected:
-        int64_t embedding_dim;
-
-    public:
-        AdaLayerNorm(int64_t in_features, int64_t hidden_features = 256)
-            : embedding_dim(in_features) {
-            blocks["norm"] = std::make_shared<LayerNorm>(in_features, 1e-6f, false, false);
-            blocks["1"]    = std::make_shared<Linear>(in_features, hidden_features, false);
-            blocks["2"]    = std::make_shared<Linear>(hidden_features, 2 * in_features, false);
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* hidden_states,
-                             ggml_tensor* embedded_timestep,
-                             ggml_tensor* temb = nullptr) {
-            auto norm     = std::dynamic_pointer_cast<LayerNorm>(blocks["norm"]);
-            auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["1"]);
-            auto linear_2 = std::dynamic_pointer_cast<Linear>(blocks["2"]);
-
-            auto emb = ggml_silu(ctx->ggml_ctx, embedded_timestep);
-            emb      = linear_1->forward(ctx, emb);
-            emb      = linear_2->forward(ctx, emb);  // [N, 2*C]
-
-            if (temb != nullptr) {
-                auto temb_2c = ggml_view_2d(ctx->ggml_ctx, temb, 2 * embedding_dim, temb->ne[1], temb->nb[1], 0);
-                emb          = ggml_add(ctx->ggml_ctx, emb, temb_2c);
-            }
-
-            auto emb_chunks = ggml_ext_chunk(ctx->ggml_ctx, emb, 2, 0);
-            auto shift      = emb_chunks[0];
-            auto scale      = emb_chunks[1];
-
-            auto x = norm->forward(ctx, hidden_states);
-            x      = Flux::modulate(ctx->ggml_ctx, x, shift, scale);
-            return x;
-        }
-    };
-
-    struct AnimaAttention : public GGMLBlock {
-    protected:
-        int64_t num_heads;
-        int64_t head_dim;
-        std::string out_proj_name;
-
-    public:
-        AnimaAttention(int64_t query_dim,
-                       int64_t context_dim,
-                       int64_t num_heads,
-                       int64_t head_dim,
-                       const std::string& out_proj_name = "output_proj")
-            : num_heads(num_heads), head_dim(head_dim), out_proj_name(out_proj_name) {
-            int64_t inner_dim = num_heads * head_dim;
-
-            blocks["q_proj"]            = std::make_shared<Linear>(query_dim, inner_dim, false);
-            blocks["k_proj"]            = std::make_shared<Linear>(context_dim, inner_dim, false);
-            blocks["v_proj"]            = std::make_shared<Linear>(context_dim, inner_dim, false);
-            blocks["q_norm"]            = std::make_shared<RMSNorm>(head_dim, 1e-6f);
-            blocks["k_norm"]            = std::make_shared<RMSNorm>(head_dim, 1e-6f);
-            blocks[this->out_proj_name] = std::make_shared<Linear>(inner_dim, query_dim, false);
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* hidden_states,
-                             ggml_tensor* encoder_hidden_states = nullptr,
-                             ggml_tensor* pe_q                  = nullptr,
-                             ggml_tensor* pe_k                  = nullptr) {
-            if (encoder_hidden_states == nullptr) {
-                encoder_hidden_states = hidden_states;
-            }
-
-            auto q_proj   = std::dynamic_pointer_cast<Linear>(blocks["q_proj"]);
-            auto k_proj   = std::dynamic_pointer_cast<Linear>(blocks["k_proj"]);
-            auto v_proj   = std::dynamic_pointer_cast<Linear>(blocks["v_proj"]);
-            auto q_norm   = std::dynamic_pointer_cast<RMSNorm>(blocks["q_norm"]);
-            auto k_norm   = std::dynamic_pointer_cast<RMSNorm>(blocks["k_norm"]);
-            auto out_proj = std::dynamic_pointer_cast<Linear>(blocks[out_proj_name]);
-
-            auto q = q_proj->forward(ctx, hidden_states);
-            auto k = k_proj->forward(ctx, encoder_hidden_states);
-            auto v = v_proj->forward(ctx, encoder_hidden_states);
-
-            int64_t N   = q->ne[2];
-            int64_t L_q = q->ne[1];
-            int64_t L_k = k->ne[1];
-
-            auto q4 = ggml_reshape_4d(ctx->ggml_ctx, q, head_dim, num_heads, L_q, N);  // [N, L_q, H, D]
-            auto k4 = ggml_reshape_4d(ctx->ggml_ctx, k, head_dim, num_heads, L_k, N);  // [N, L_k, H, D]
-            auto v4 = ggml_reshape_4d(ctx->ggml_ctx, v, head_dim, num_heads, L_k, N);  // [N, L_k, H, D]
-
-            q4 = q_norm->forward(ctx, q4);
-            k4 = k_norm->forward(ctx, k4);
-
-            ggml_tensor* attn_out = nullptr;
-            if (pe_q != nullptr || pe_k != nullptr) {
-                if (pe_q == nullptr) {
-                    pe_q = pe_k;
-                }
-                if (pe_k == nullptr) {
-                    pe_k = pe_q;
-                }
-                auto q_rope = Rope::apply_rope(ctx->ggml_ctx, q4, pe_q, false);
-                auto k_rope = Rope::apply_rope(ctx->ggml_ctx, k4, pe_k, false);
-                attn_out    = ggml_ext_attention_ext(ctx->ggml_ctx,
-                                                     ctx->backend,
-                                                     q_rope,
-                                                     k_rope,
-                                                     v4,
-                                                     num_heads,
-                                                     nullptr,
-                                                     true,
-                                                     ctx->flash_attn_enabled);
-            } else {
-                auto q_flat = ggml_reshape_3d(ctx->ggml_ctx, q4, head_dim * num_heads, L_q, N);
-                auto k_flat = ggml_reshape_3d(ctx->ggml_ctx, k4, head_dim * num_heads, L_k, N);
-                attn_out    = ggml_ext_attention_ext(ctx->ggml_ctx,
-                                                     ctx->backend,
-                                                     q_flat,
-                                                     k_flat,
-                                                     v,
-                                                     num_heads,
-                                                     nullptr,
-                                                     false,
-                                                     ctx->flash_attn_enabled);
-            }
-
-            return out_proj->forward(ctx, attn_out);
-        }
-    };
-
-    struct AnimaMLP : public GGMLBlock {
-    public:
-        AnimaMLP(int64_t dim, int64_t hidden_dim) {
-            blocks["layer1"] = std::make_shared<Linear>(dim, hidden_dim, false);
-            blocks["layer2"] = std::make_shared<Linear>(hidden_dim, dim, false);
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
-            auto layer1 = std::dynamic_pointer_cast<Linear>(blocks["layer1"]);
-            auto layer2 = std::dynamic_pointer_cast<Linear>(blocks["layer2"]);
-
-            x = layer1->forward(ctx, x);
-            x = ggml_ext_gelu(ctx->ggml_ctx, x, true);
-            x = layer2->forward(ctx, x);
-            return x;
-        }
-    };
-
-    struct AdapterMLP : public GGMLBlock {
-    public:
-        AdapterMLP(int64_t dim, int64_t hidden_dim) {
-            blocks["0"] = std::make_shared<Linear>(dim, hidden_dim, true);
-            blocks["2"] = std::make_shared<Linear>(hidden_dim, dim, true);
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
-            auto layer0 = std::dynamic_pointer_cast<Linear>(blocks["0"]);
-            auto layer2 = std::dynamic_pointer_cast<Linear>(blocks["2"]);
-
-            x = layer0->forward(ctx, x);
-            x = ggml_ext_gelu(ctx->ggml_ctx, x, true);
-            x = layer2->forward(ctx, x);
-            return x;
-        }
-    };
-
-    struct LLMAdapterBlock : public GGMLBlock {
-    public:
-        LLMAdapterBlock(int64_t model_dim = 1024, int64_t source_dim = 1024, int64_t num_heads = 16, int64_t head_dim = 64) {
-            blocks["norm_self_attn"]  = std::make_shared<RMSNorm>(model_dim, 1e-6f);
-            blocks["self_attn"]       = std::make_shared<AnimaAttention>(model_dim, model_dim, num_heads, head_dim, "o_proj");
-            blocks["norm_cross_attn"] = std::make_shared<RMSNorm>(model_dim, 1e-6f);
-            blocks["cross_attn"]      = std::make_shared<AnimaAttention>(model_dim, source_dim, num_heads, head_dim, "o_proj");
-            blocks["norm_mlp"]        = std::make_shared<RMSNorm>(model_dim, 1e-6f);
-            blocks["mlp"]             = std::make_shared<AdapterMLP>(model_dim, model_dim * 4);
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* x,
-                             ggml_tensor* context,
-                             ggml_tensor* target_pe,
-                             ggml_tensor* context_pe) {
-            auto norm_self_attn  = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_self_attn"]);
-            auto self_attn       = std::dynamic_pointer_cast<AnimaAttention>(blocks["self_attn"]);
-            auto norm_cross_attn = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_cross_attn"]);
-            auto cross_attn      = std::dynamic_pointer_cast<AnimaAttention>(blocks["cross_attn"]);
-            auto norm_mlp        = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_mlp"]);
-            auto mlp             = std::dynamic_pointer_cast<AdapterMLP>(blocks["mlp"]);
-
-            auto h = norm_self_attn->forward(ctx, x);
-            h      = self_attn->forward(ctx, h, nullptr, target_pe, target_pe);
-            x      = ggml_add(ctx->ggml_ctx, x, h);
-
-            h = norm_cross_attn->forward(ctx, x);
-            h = cross_attn->forward(ctx, h, context, target_pe, context_pe);
-            x = ggml_add(ctx->ggml_ctx, x, h);
-
-            h = norm_mlp->forward(ctx, x);
-            h = mlp->forward(ctx, h);
-            x = ggml_add(ctx->ggml_ctx, x, h);
-
-            return x;
-        }
-    };
-
-    struct LLMAdapter : public GGMLBlock {
-    protected:
-        int num_layers;
-
-    public:
-        LLMAdapter(int64_t source_dim = 1024,
-                   int64_t target_dim = 1024,
-                   int64_t model_dim  = 1024,
-                   int num_layers     = 6,
-                   int num_heads      = 16)
-            : num_layers(num_layers) {
-            int64_t head_dim = model_dim / num_heads;
-
-            blocks["embed"] = std::make_shared<Embedding>(32128, target_dim);
-            for (int i = 0; i < num_layers; i++) {
-                blocks["blocks." + std::to_string(i)] =
-                    std::make_shared<LLMAdapterBlock>(model_dim, source_dim, num_heads, head_dim);
-            }
-            blocks["out_proj"] = std::make_shared<Linear>(model_dim, target_dim, true);
-            blocks["norm"]     = std::make_shared<RMSNorm>(target_dim, 1e-6f);
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* source_hidden_states,
-                             ggml_tensor* target_input_ids,
-                             ggml_tensor* target_pe,
-                             ggml_tensor* source_pe) {
-            GGML_ASSERT(target_input_ids != nullptr);
-            if (ggml_n_dims(target_input_ids) == 1) {
-                target_input_ids = ggml_reshape_2d(ctx->ggml_ctx, target_input_ids, target_input_ids->ne[0], 1);
-            }
-
-            auto embed    = std::dynamic_pointer_cast<Embedding>(blocks["embed"]);
-            auto out_proj = std::dynamic_pointer_cast<Linear>(blocks["out_proj"]);
-            auto norm     = std::dynamic_pointer_cast<RMSNorm>(blocks["norm"]);
-
-            auto x = embed->forward(ctx, target_input_ids);  // [N, target_len, target_dim]
-
-            for (int i = 0; i < num_layers; i++) {
-                auto block = std::dynamic_pointer_cast<LLMAdapterBlock>(blocks["blocks." + std::to_string(i)]);
-                x          = block->forward(ctx, x, source_hidden_states, target_pe, source_pe);
-            }
-
-            x = out_proj->forward(ctx, x);
-            x = norm->forward(ctx, x);
-            return x;
-        }
-    };
-
-    struct TransformerBlock : public GGMLBlock {
-    public:
-        TransformerBlock(int64_t hidden_size,
-                         int64_t text_embed_dim,
-                         int64_t num_heads,
-                         int64_t head_dim,
-                         int64_t mlp_ratio      = 4,
-                         int64_t adaln_lora_dim = 256) {
-            blocks["adaln_modulation_self_attn"]  = std::make_shared<AdaLayerNormZero>(hidden_size, adaln_lora_dim);
-            blocks["self_attn"]                   = std::make_shared<AnimaAttention>(hidden_size, hidden_size, num_heads, head_dim);
-            blocks["adaln_modulation_cross_attn"] = std::make_shared<AdaLayerNormZero>(hidden_size, adaln_lora_dim);
-            blocks["cross_attn"]                  = std::make_shared<AnimaAttention>(hidden_size, text_embed_dim, num_heads, head_dim);
-            blocks["adaln_modulation_mlp"]        = std::make_shared<AdaLayerNormZero>(hidden_size, adaln_lora_dim);
-            blocks["mlp"]                         = std::make_shared<AnimaMLP>(hidden_size, hidden_size * mlp_ratio);
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* hidden_states,
-                             ggml_tensor* encoder_hidden_states,
-                             ggml_tensor* embedded_timestep,
-                             ggml_tensor* temb,
-                             ggml_tensor* image_pe) {
-            auto norm1 = std::dynamic_pointer_cast<AdaLayerNormZero>(blocks["adaln_modulation_self_attn"]);
-            auto attn1 = std::dynamic_pointer_cast<AnimaAttention>(blocks["self_attn"]);
-            auto norm2 = std::dynamic_pointer_cast<AdaLayerNormZero>(blocks["adaln_modulation_cross_attn"]);
-            auto attn2 = std::dynamic_pointer_cast<AnimaAttention>(blocks["cross_attn"]);
-            auto norm3 = std::dynamic_pointer_cast<AdaLayerNormZero>(blocks["adaln_modulation_mlp"]);
-            auto mlp   = std::dynamic_pointer_cast<AnimaMLP>(blocks["mlp"]);
-
-            auto [normed1, gate1] = norm1->forward(ctx, hidden_states, embedded_timestep, temb);
-            auto h                = attn1->forward(ctx, normed1, nullptr, image_pe, image_pe);
-            hidden_states         = ggml_add(ctx->ggml_ctx, hidden_states, apply_gate(ctx->ggml_ctx, h, gate1));
-
-            auto [normed2, gate2] = norm2->forward(ctx, hidden_states, embedded_timestep, temb);
-            h                     = attn2->forward(ctx, normed2, encoder_hidden_states, nullptr, nullptr);
-            hidden_states         = ggml_add(ctx->ggml_ctx, hidden_states, apply_gate(ctx->ggml_ctx, h, gate2));
-
-            auto [normed3, gate3] = norm3->forward(ctx, hidden_states, embedded_timestep, temb);
-            h                     = mlp->forward(ctx, normed3);
-            hidden_states         = ggml_add(ctx->ggml_ctx, hidden_states, apply_gate(ctx->ggml_ctx, h, gate3));
-
-            return hidden_states;
-        }
-    };
-
-    struct FinalLayer : public GGMLBlock {
-    protected:
-        int64_t hidden_size;
-        int64_t patch_size;
-        int64_t out_channels;
-
-    public:
-        FinalLayer(int64_t hidden_size, int64_t patch_size, int64_t out_channels)
-            : hidden_size(hidden_size), patch_size(patch_size), out_channels(out_channels) {
-            blocks["adaln_modulation"] = std::make_shared<AdaLayerNorm>(hidden_size, 256);
-            blocks["linear"]           = std::make_shared<Linear>(hidden_size, patch_size * patch_size * out_channels, false);
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* hidden_states,
-                             ggml_tensor* embedded_timestep,
-                             ggml_tensor* temb) {
-            auto adaln  = std::dynamic_pointer_cast<AdaLayerNorm>(blocks["adaln_modulation"]);
-            auto linear = std::dynamic_pointer_cast<Linear>(blocks["linear"]);
-
-            hidden_states = adaln->forward(ctx, hidden_states, embedded_timestep, temb);
-            hidden_states = linear->forward(ctx, hidden_states);
-            return hidden_states;
-        }
-    };
-
-    struct AnimaNet : public GGMLBlock {
-    public:
-        int64_t in_channels       = 16;
-        int64_t out_channels      = 16;
-        int64_t hidden_size       = 2048;
-        int64_t text_embed_dim    = 1024;
-        int64_t num_heads         = 16;
-        int64_t head_dim          = 128;
-        int patch_size            = 2;
-        int64_t num_layers        = 28;
-        std::vector<int> axes_dim = {44, 42, 42};
-        int theta                 = 10000;
-
-    public:
-        AnimaNet() = default;
-        explicit AnimaNet(int64_t num_layers)
-            : num_layers(num_layers) {
-            blocks["x_embedder"]       = std::make_shared<XEmbedder>((in_channels + 1) * patch_size * patch_size, hidden_size);
-            blocks["t_embedder"]       = std::make_shared<TimestepEmbedder>(hidden_size, hidden_size * 3);
-            blocks["t_embedding_norm"] = std::make_shared<RMSNorm>(hidden_size, 1e-6f);
-            for (int i = 0; i < num_layers; i++) {
-                blocks["blocks." + std::to_string(i)] = std::make_shared<TransformerBlock>(hidden_size,
-                                                                                           text_embed_dim,
-                                                                                           num_heads,
-                                                                                           head_dim);
-            }
-            blocks["final_layer"] = std::make_shared<FinalLayer>(hidden_size, patch_size, out_channels);
-            blocks["llm_adapter"] = std::make_shared<LLMAdapter>(1024, 1024, 1024, 6, 16);
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* x,
-                             ggml_tensor* timestep,
-                             ggml_tensor* encoder_hidden_states,
-                             ggml_tensor* image_pe,
-                             ggml_tensor* t5_ids       = nullptr,
-                             ggml_tensor* t5_weights   = nullptr,
-                             ggml_tensor* adapter_q_pe = nullptr,
-                             ggml_tensor* adapter_k_pe = nullptr) {
-            GGML_ASSERT(x->ne[3] == 1);
-
-            auto x_embedder       = std::dynamic_pointer_cast<XEmbedder>(blocks["x_embedder"]);
-            auto t_embedder       = std::dynamic_pointer_cast<TimestepEmbedder>(blocks["t_embedder"]);
-            auto t_embedding_norm = std::dynamic_pointer_cast<RMSNorm>(blocks["t_embedding_norm"]);
-            auto final_layer      = std::dynamic_pointer_cast<FinalLayer>(blocks["final_layer"]);
-            auto llm_adapter      = std::dynamic_pointer_cast<LLMAdapter>(blocks["llm_adapter"]);
-
-            int64_t W = x->ne[0];
-            int64_t H = x->ne[1];
-
-            auto padding_mask = ggml_ext_zeros(ctx->ggml_ctx, x->ne[0], x->ne[1], 1, x->ne[3]);
-            x                 = ggml_concat(ctx->ggml_ctx, x, padding_mask, 2);  // [N, C + 1, H, W]
-
-            x = DiT::pad_and_patchify(ctx, x, patch_size, patch_size);  // [N, h*w, (C+1)*ph*pw]
-
-            x = x_embedder->forward(ctx, x);
-
-            auto timestep_proj     = ggml_ext_timestep_embedding(ctx->ggml_ctx, timestep, static_cast<int>(hidden_size));
-            auto temb              = t_embedder->forward(ctx, timestep_proj);
-            auto embedded_timestep = t_embedding_norm->forward(ctx, timestep_proj);
-
-            if (t5_ids != nullptr) {
-                auto adapted_context = llm_adapter->forward(ctx, encoder_hidden_states, t5_ids, adapter_q_pe, adapter_k_pe);
-                if (t5_weights != nullptr) {
-                    auto w = t5_weights;
-                    if (ggml_n_dims(w) == 1) {
-                        w = ggml_reshape_3d(ctx->ggml_ctx, w, 1, w->ne[0], 1);
-                    }
-                    w               = ggml_repeat_4d(ctx->ggml_ctx, w, adapted_context->ne[0], adapted_context->ne[1], adapted_context->ne[2], 1);
-                    adapted_context = ggml_mul(ctx->ggml_ctx, adapted_context, w);
-                }
-                if (adapted_context->ne[1] < 512) {
-                    auto pad_ctx    = ggml_ext_zeros(ctx->ggml_ctx,
-                                                     adapted_context->ne[0],
-                                                     512 - adapted_context->ne[1],
-                                                     adapted_context->ne[2],
-                                                     1);
-                    adapted_context = ggml_concat(ctx->ggml_ctx, adapted_context, pad_ctx, 1);
-                } else if (adapted_context->ne[1] > 512) {
-                    adapted_context = ggml_ext_slice(ctx->ggml_ctx, adapted_context, 1, 0, 512);
-                }
-                encoder_hidden_states = adapted_context;
-            }
-
-            sd::ggml_graph_cut::mark_graph_cut(x, "anima.prelude", "x");
-            sd::ggml_graph_cut::mark_graph_cut(embedded_timestep, "anima.prelude", "embedded_timestep");
-            sd::ggml_graph_cut::mark_graph_cut(temb, "anima.prelude", "temb");
-            sd::ggml_graph_cut::mark_graph_cut(encoder_hidden_states, "anima.prelude", "context");
-
-            for (int i = 0; i < num_layers; i++) {
-                auto block = std::dynamic_pointer_cast<TransformerBlock>(blocks["blocks." + std::to_string(i)]);
-                x          = block->forward(ctx, x, encoder_hidden_states, embedded_timestep, temb, image_pe);
-                sd::ggml_graph_cut::mark_graph_cut(x, "anima.blocks." + std::to_string(i), "x");
-            }
-
-            x = final_layer->forward(ctx, x, embedded_timestep, temb);  // [N, h*w, ph*pw*C]
-
-            x = DiT::unpatchify_and_crop(ctx->ggml_ctx, x, H, W, patch_size, patch_size, false);  // [N, C, H, W]
-
-            return x;
-        }
-    };
-
-    struct AnimaRunner : public GGMLRunner {
-    public:
-        std::vector<float> image_pe_vec;
-        std::vector<float> adapter_q_pe_vec;
-        std::vector<float> adapter_k_pe_vec;
-        AnimaNet net;
-
-        AnimaRunner(ggml_backend_t backend,
-                    ggml_backend_t params_backend,
-                    const String2TensorStorage& tensor_storage_map = {},
-                    const std::string prefix                       = "model.diffusion_model")
-            : GGMLRunner(backend, params_backend) {
-            int64_t num_layers    = 0;
-            std::string layer_tag = prefix + ".net.blocks.";
-            for (const auto& kv : tensor_storage_map) {
-                const std::string& tensor_name = kv.first;
-                size_t pos                     = tensor_name.find(layer_tag);
-                if (pos == std::string::npos) {
-                    continue;
-                }
-                size_t start = pos + layer_tag.size();
-                size_t end   = tensor_name.find('.', start);
-                if (end == std::string::npos) {
-                    continue;
-                }
-                int64_t layer_id = atoll(tensor_name.substr(start, end - start).c_str());
-                num_layers       = std::max(num_layers, layer_id + 1);
-            }
-            if (num_layers <= 0) {
-                num_layers = 28;
-            }
-            LOG_INFO("anima net layers: %" PRId64, num_layers);
-
-            net = AnimaNet(num_layers);
-            net.init(params_ctx, tensor_storage_map, prefix + ".net");
-        }
-
-        std::string get_desc() override {
-            return "anima";
-        }
-
-        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string prefix) {
-            net.get_param_tensors(tensors, prefix + ".net");
-        }
-
-        static std::vector<float> gen_1d_rope_pe_vec(int64_t seq_len, int dim, float theta = 10000.f) {
-            std::vector<float> pos(seq_len);
-            for (int64_t i = 0; i < seq_len; i++) {
-                pos[i] = static_cast<float>(i);
-            }
-            auto rope_emb = Rope::rope(pos, dim, theta);
-            return Rope::flatten(rope_emb);
-        }
-
-        static float calc_ntk_factor(float extrapolation_ratio, int axis_dim) {
-            if (extrapolation_ratio == 1.0f || axis_dim <= 2) {
-                return 1.0f;
-            }
-            return std::pow(extrapolation_ratio, static_cast<float>(axis_dim) / static_cast<float>(axis_dim - 2));
-        }
-
-        static std::vector<float> gen_anima_image_pe_vec(int bs,
-                                                         int h,
-                                                         int w,
-                                                         int patch_size,
-                                                         int theta,
-                                                         const std::vector<int>& axes_dim,
-                                                         float h_extrapolation_ratio,
-                                                         float w_extrapolation_ratio,
-                                                         float t_extrapolation_ratio) {
-            static const std::vector<ggml_tensor*> empty_ref_latents;
-            auto ids = Rope::gen_flux_ids(h,
-                                          w,
-                                          patch_size,
-                                          bs,
-                                          static_cast<int>(axes_dim.size()),
-                                          0,
-                                          {},
-                                          empty_ref_latents,
-                                          false,
-                                          1.0f);
-
-            std::vector<float> axis_thetas = {
-                static_cast<float>(theta) * calc_ntk_factor(t_extrapolation_ratio, axes_dim[0]),
-                static_cast<float>(theta) * calc_ntk_factor(h_extrapolation_ratio, axes_dim[1]),
-                static_cast<float>(theta) * calc_ntk_factor(w_extrapolation_ratio, axes_dim[2]),
-            };
-            return Rope::embed_nd(ids, bs, axis_thetas, axes_dim);
-        }
-
-        ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
-                                 const sd::Tensor<float>& timesteps_tensor,
-                                 const sd::Tensor<float>& context_tensor    = {},
-                                 const sd::Tensor<int32_t>& t5_ids_tensor   = {},
-                                 const sd::Tensor<float>& t5_weights_tensor = {}) {
-            ggml_tensor* x          = make_input(x_tensor);
-            ggml_tensor* timesteps  = make_input(timesteps_tensor);
-            ggml_tensor* context    = make_optional_input(context_tensor);
-            ggml_tensor* t5_ids     = make_optional_input(t5_ids_tensor);
-            ggml_tensor* t5_weights = make_optional_input(t5_weights_tensor);
-            GGML_ASSERT(x->ne[3] == 1);
-            ggml_cgraph* gf = new_graph_custom(ANIMA_GRAPH_SIZE);
-
-            int64_t pad_h = (net.patch_size - x->ne[1] % net.patch_size) % net.patch_size;
-            int64_t pad_w = (net.patch_size - x->ne[0] % net.patch_size) % net.patch_size;
-            int64_t h_pad = x->ne[1] + pad_h;
-            int64_t w_pad = x->ne[0] + pad_w;
-
-            image_pe_vec          = gen_anima_image_pe_vec(1,
-                                                           static_cast<int>(h_pad),
-                                                           static_cast<int>(w_pad),
-                                                           static_cast<int>(net.patch_size),
-                                                           net.theta,
-                                                           net.axes_dim,
-                                                           4.0f,
-                                                           4.0f,
-                                                           1.0f);
-            int64_t image_pos_len = static_cast<int64_t>(image_pe_vec.size()) / (2 * 2 * (net.head_dim / 2));
-            auto image_pe         = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, net.head_dim / 2, image_pos_len);
-            set_backend_tensor_data(image_pe, image_pe_vec.data());
-
-            ggml_tensor* adapter_q_pe = nullptr;
-            ggml_tensor* adapter_k_pe = nullptr;
-            if (t5_ids != nullptr) {
-                int64_t target_len = t5_ids->ne[0];
-                int64_t source_len = context->ne[1];
-
-                adapter_q_pe_vec = gen_1d_rope_pe_vec(target_len, 64, 10000.f);
-                adapter_k_pe_vec = gen_1d_rope_pe_vec(source_len, 64, 10000.f);
-
-                int64_t target_pos_len = static_cast<int64_t>(adapter_q_pe_vec.size()) / (2 * 2 * 32);
-                int64_t source_pos_len = static_cast<int64_t>(adapter_k_pe_vec.size()) / (2 * 2 * 32);
-
-                adapter_q_pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, 32, target_pos_len);
-                adapter_k_pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, 32, source_pos_len);
-                set_backend_tensor_data(adapter_q_pe, adapter_q_pe_vec.data());
-                set_backend_tensor_data(adapter_k_pe, adapter_k_pe_vec.data());
-            }
-
-            auto runner_ctx = get_context();
-            auto out        = net.forward(&runner_ctx,
-                                          x,
-                                          timesteps,
-                                          context,
-                                          image_pe,
-                                          t5_ids,
-                                          t5_weights,
-                                          adapter_q_pe,
-                                          adapter_k_pe);
-
-            ggml_build_forward_expand(gf, out);
-            return gf;
-        }
-
-        sd::Tensor<float> compute(int n_threads,
-                                  const sd::Tensor<float>& x,
-                                  const sd::Tensor<float>& timesteps,
-                                  const sd::Tensor<float>& context    = {},
-                                  const sd::Tensor<int32_t>& t5_ids   = {},
-                                  const sd::Tensor<float>& t5_weights = {}) {
-            auto get_graph = [&]() -> ggml_cgraph* {
-                return build_graph(x, timesteps, context, t5_ids, t5_weights);
-            };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
-        }
-    };
-}  // namespace Anima
-
-#endif  // __ANIMA_HPP__

src/clip.hpp

@@ -1,582 +0,0 @@
-#ifndef __CLIP_HPP__
-#define __CLIP_HPP__
-
-#include "ggml_extend.hpp"
-#include "model.h"
-#include "tokenizers/clip_tokenizer.h"
-
-/*================================================ FrozenCLIPEmbedder ================================================*/
-
-// Ref: https://github.com/huggingface/transformers/blob/main/src/transformers/models/clip/modeling_clip.py
-
-struct CLIPMLP : public GGMLBlock {
-protected:
-    bool use_gelu;
-
-public:
-    CLIPMLP(int64_t d_model, int64_t intermediate_size) {
-        blocks["fc1"] = std::shared_ptr<GGMLBlock>(new Linear(d_model, intermediate_size));
-        blocks["fc2"] = std::shared_ptr<GGMLBlock>(new Linear(intermediate_size, d_model));
-
-        if (d_model == 1024 || d_model == 1280) {  // SD 2.x
-            use_gelu = true;
-        } else {  // SD 1.x
-            use_gelu = false;
-        }
-    }
-
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
-        // x: [N, n_token, d_model]
-        auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
-        auto fc2 = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
-
-        x = fc1->forward(ctx, x);
-        if (use_gelu) {
-            x = ggml_ext_gelu(ctx->ggml_ctx, x, true);
-        } else {
-            x = ggml_ext_gelu_quick(ctx->ggml_ctx, x, true);
-        }
-        x = fc2->forward(ctx, x);
-        return x;
-    }
-};
-
-struct CLIPLayer : public GGMLBlock {
-protected:
-    int64_t d_model;  // hidden_size/embed_dim
-    int64_t n_head;
-    int64_t intermediate_size;
-
-public:
-    CLIPLayer(int64_t d_model,
-              int64_t n_head,
-              int64_t intermediate_size,
-              bool proj_in = false)
-        : d_model(d_model),
-          n_head(n_head),
-          intermediate_size(intermediate_size) {
-        blocks["self_attn"] = std::shared_ptr<GGMLBlock>(new MultiheadAttention(d_model, n_head, true, true, proj_in));
-
-        blocks["layer_norm1"] = std::shared_ptr<GGMLBlock>(new LayerNorm(d_model));
-        blocks["layer_norm2"] = std::shared_ptr<GGMLBlock>(new LayerNorm(d_model));
-
-        blocks["mlp"] = std::shared_ptr<GGMLBlock>(new CLIPMLP(d_model, intermediate_size));
-    }
-
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* mask = nullptr) {
-        // x: [N, n_token, d_model]
-        auto self_attn   = std::dynamic_pointer_cast<MultiheadAttention>(blocks["self_attn"]);
-        auto layer_norm1 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm1"]);
-        auto layer_norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm2"]);
-        auto mlp         = std::dynamic_pointer_cast<CLIPMLP>(blocks["mlp"]);
-
-        x = ggml_add(ctx->ggml_ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask));
-        x = ggml_add(ctx->ggml_ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x)));
-        return x;
-    }
-};
-
-struct CLIPEncoder : public GGMLBlock {
-protected:
-    int n_layer;
-
-public:
-    CLIPEncoder(int n_layer,
-                int64_t d_model,
-                int64_t n_head,
-                int64_t intermediate_size,
-                bool proj_in = false)
-        : n_layer(n_layer) {
-        for (int i = 0; i < n_layer; i++) {
-            std::string name = "layers." + std::to_string(i);
-            blocks[name]     = std::shared_ptr<GGMLBlock>(new CLIPLayer(d_model, n_head, intermediate_size, proj_in));
-        }
-    }
-
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* x,
-                         ggml_tensor* mask                   = nullptr,
-                         int clip_skip                       = -1,
-                         const std::string& graph_cut_prefix = "") {
-        // x: [N, n_token, d_model]
-        int layer_idx = n_layer - 1;
-        // LOG_DEBUG("clip_skip %d", clip_skip);
-        if (clip_skip > 0) {
-            layer_idx = n_layer - clip_skip;
-        }
-
-        for (int i = 0; i < n_layer; i++) {
-            // LOG_DEBUG("layer %d", i);
-            if (i == layer_idx + 1) {
-                break;
-            }
-            std::string name = "layers." + std::to_string(i);
-            auto layer       = std::dynamic_pointer_cast<CLIPLayer>(blocks[name]);
-            x                = layer->forward(ctx, x, mask);  // [N, n_token, d_model]
-            if (!graph_cut_prefix.empty()) {
-                sd::ggml_graph_cut::mark_graph_cut(x, graph_cut_prefix + ".layers." + std::to_string(i), "x");
-            }
-            // LOG_DEBUG("layer %d", i);
-        }
-        return x;
-    }
-};
-
-class CLIPEmbeddings : public GGMLBlock {
-protected:
-    int64_t embed_dim;
-    int64_t vocab_size;
-    int64_t num_positions;
-    bool force_clip_f32;
-
-    void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
-        enum ggml_type token_wtype = GGML_TYPE_F32;
-        if (!force_clip_f32) {
-            token_wtype = get_type(prefix + "token_embedding.weight", tensor_storage_map, GGML_TYPE_F32);
-            if (!support_get_rows(token_wtype)) {
-                token_wtype = GGML_TYPE_F32;
-            }
-        }
-        enum ggml_type position_wtype       = GGML_TYPE_F32;
-        params["token_embedding.weight"]    = ggml_new_tensor_2d(ctx, token_wtype, embed_dim, vocab_size);
-        params["position_embedding.weight"] = ggml_new_tensor_2d(ctx, position_wtype, embed_dim, num_positions);
-    }
-
-public:
-    CLIPEmbeddings(int64_t embed_dim,
-                   int64_t vocab_size    = 49408,
-                   int64_t num_positions = 77,
-                   bool force_clip_f32   = false)
-        : embed_dim(embed_dim),
-          vocab_size(vocab_size),
-          num_positions(num_positions),
-          force_clip_f32(force_clip_f32) {
-    }
-
-    ggml_tensor* get_token_embed_weight() {
-        return params["token_embedding.weight"];
-    }
-
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* input_ids,
-                         ggml_tensor* custom_embed_weight) {
-        // input_ids: [N, n_token]
-        auto token_embed_weight    = params["token_embedding.weight"];
-        auto position_embed_weight = params["position_embedding.weight"];
-
-        GGML_ASSERT(input_ids->ne[0] == position_embed_weight->ne[1]);
-        input_ids            = ggml_reshape_3d(ctx->ggml_ctx, input_ids, input_ids->ne[0], 1, input_ids->ne[1]);
-        auto token_embedding = ggml_get_rows(ctx->ggml_ctx, custom_embed_weight != nullptr ? custom_embed_weight : token_embed_weight, input_ids);
-        token_embedding      = ggml_reshape_3d(ctx->ggml_ctx, token_embedding, token_embedding->ne[0], token_embedding->ne[1], token_embedding->ne[3]);
-
-        // token_embedding + position_embedding
-        auto x = ggml_add(ctx->ggml_ctx,
-                          token_embedding,
-                          position_embed_weight);  // [N, n_token, embed_dim]
-        return x;
-    }
-};
-
-class CLIPVisionEmbeddings : public GGMLBlock {
-protected:
-    int64_t embed_dim;
-    int num_channels;
-    int patch_size;
-    int image_size;
-    int num_patches;
-    int64_t num_positions;
-
-    void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
-        enum ggml_type patch_wtype    = GGML_TYPE_F16;
-        enum ggml_type class_wtype    = GGML_TYPE_F32;
-        enum ggml_type position_wtype = GGML_TYPE_F32;
-
-        params["patch_embedding.weight"]    = ggml_new_tensor_4d(ctx, patch_wtype, patch_size, patch_size, num_channels, embed_dim);
-        params["class_embedding"]           = ggml_new_tensor_1d(ctx, class_wtype, embed_dim);
-        params["position_embedding.weight"] = ggml_new_tensor_2d(ctx, position_wtype, embed_dim, num_positions);
-    }
-
-public:
-    CLIPVisionEmbeddings(int64_t embed_dim,
-                         int num_channels = 3,
-                         int patch_size   = 14,
-                         int image_size   = 224)
-        : embed_dim(embed_dim),
-          num_channels(num_channels),
-          patch_size(patch_size),
-          image_size(image_size) {
-        num_patches   = (image_size / patch_size) * (image_size / patch_size);
-        num_positions = num_patches + 1;
-    }
-
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* pixel_values) {
-        // pixel_values: [N, num_channels, image_size, image_size]
-        // return: [N, num_positions, embed_dim]
-        GGML_ASSERT(pixel_values->ne[0] == image_size && pixel_values->ne[1] == image_size && pixel_values->ne[2] == num_channels);
-
-        auto patch_embed_weight    = params["patch_embedding.weight"];
-        auto class_embed_weight    = params["class_embedding"];
-        auto position_embed_weight = params["position_embedding.weight"];
-
-        // concat(patch_embedding, class_embedding) + position_embedding
-        ggml_tensor* patch_embedding;
-        int64_t N       = pixel_values->ne[3];
-        patch_embedding = ggml_ext_conv_2d(ctx->ggml_ctx, pixel_values, patch_embed_weight, nullptr, patch_size, patch_size);  // [N, embed_dim, image_size // pacht_size, image_size // pacht_size]
-        patch_embedding = ggml_reshape_3d(ctx->ggml_ctx, patch_embedding, num_patches, embed_dim, N);                          // [N, embed_dim, num_patches]
-        patch_embedding = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, patch_embedding, 1, 0, 2, 3));                  // [N, num_patches, embed_dim]
-        patch_embedding = ggml_reshape_4d(ctx->ggml_ctx, patch_embedding, 1, embed_dim, num_patches, N);                       // [N, num_patches, embed_dim, 1]
-
-        ggml_tensor* class_embedding = ggml_new_tensor_2d(ctx->ggml_ctx, GGML_TYPE_F32, embed_dim, N);
-        class_embedding              = ggml_repeat(ctx->ggml_ctx, class_embed_weight, class_embedding);      // [N, embed_dim]
-        class_embedding              = ggml_reshape_4d(ctx->ggml_ctx, class_embedding, 1, embed_dim, 1, N);  // [N, 1, embed_dim, 1]
-
-        ggml_tensor* x = ggml_concat(ctx->ggml_ctx, class_embedding, patch_embedding, 2);  // [N, num_positions, embed_dim, 1]
-        x              = ggml_reshape_3d(ctx->ggml_ctx, x, embed_dim, num_positions, N);   // [N, num_positions, embed_dim]
-        x              = ggml_add(ctx->ggml_ctx, x, position_embed_weight);
-        return x;  // [N, num_positions, embed_dim]
-    }
-};
-
-// OPENAI_CLIP_VIT_L_14: https://huggingface.co/openai/clip-vit-large-patch14/blob/main/config.json
-// OPEN_CLIP_VIT_H_14: https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K/blob/main/config.json
-// OPEN_CLIP_VIT_BIGG_14: https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k/blob/main/config.json (CLIPTextModelWithProjection)
-
-enum CLIPVersion {
-    OPENAI_CLIP_VIT_L_14,   // SD 1.x and SDXL
-    OPEN_CLIP_VIT_H_14,     // SD 2.x
-    OPEN_CLIP_VIT_BIGG_14,  // SDXL
-};
-
-class CLIPTextModel : public GGMLBlock {
-protected:
-    void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
-        if (version == OPEN_CLIP_VIT_BIGG_14) {
-            enum ggml_type wtype      = GGML_TYPE_F32;
-            params["text_projection"] = ggml_new_tensor_2d(ctx, wtype, projection_dim, hidden_size);
-        }
-    }
-
-public:
-    CLIPVersion version = OPENAI_CLIP_VIT_L_14;
-    // network hparams
-    int32_t vocab_size        = 49408;
-    int32_t n_token           = 77;  // max_position_embeddings
-    int32_t hidden_size       = 768;
-    int32_t intermediate_size = 3072;
-    int32_t n_head            = 12;
-    int32_t n_layer           = 12;    // num_hidden_layers
-    int32_t projection_dim    = 1280;  // only for OPEN_CLIP_VIT_BIGG_14
-    bool with_final_ln        = true;
-
-    CLIPTextModel(CLIPVersion version = OPENAI_CLIP_VIT_L_14,
-                  bool with_final_ln  = true,
-                  bool force_clip_f32 = false,
-                  bool proj_in        = false)
-        : version(version), with_final_ln(with_final_ln) {
-        if (version == OPEN_CLIP_VIT_H_14) {
-            hidden_size       = 1024;
-            intermediate_size = 4096;
-            n_head            = 16;
-            n_layer           = 24;
-        } else if (version == OPEN_CLIP_VIT_BIGG_14) {  // CLIPTextModelWithProjection
-            hidden_size       = 1280;
-            intermediate_size = 5120;
-            n_head            = 20;
-            n_layer           = 32;
-        }
-
-        blocks["embeddings"]       = std::shared_ptr<GGMLBlock>(new CLIPEmbeddings(hidden_size, vocab_size, n_token, force_clip_f32));
-        blocks["encoder"]          = std::shared_ptr<GGMLBlock>(new CLIPEncoder(n_layer, hidden_size, n_head, intermediate_size, proj_in));
-        blocks["final_layer_norm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size));
-    }
-
-    ggml_tensor* get_token_embed_weight() {
-        auto embeddings = std::dynamic_pointer_cast<CLIPEmbeddings>(blocks["embeddings"]);
-        return embeddings->get_token_embed_weight();
-    }
-
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* input_ids,
-                         ggml_tensor* tkn_embeddings,
-                         ggml_tensor* mask    = nullptr,
-                         size_t max_token_idx = 0,
-                         bool return_pooled   = false,
-                         int clip_skip        = -1) {
-        // input_ids: [N, n_token]
-        auto embeddings       = std::dynamic_pointer_cast<CLIPEmbeddings>(blocks["embeddings"]);
-        auto encoder          = std::dynamic_pointer_cast<CLIPEncoder>(blocks["encoder"]);
-        auto final_layer_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["final_layer_norm"]);
-
-        auto x = embeddings->forward(ctx, input_ids, tkn_embeddings);  // [N, n_token, hidden_size]
-        sd::ggml_graph_cut::mark_graph_cut(x, "clip_text.prelude", "x");
-        x = encoder->forward(ctx, x, mask, return_pooled ? -1 : clip_skip, "clip_text");
-        if (return_pooled || with_final_ln) {
-            x = final_layer_norm->forward(ctx, x);
-        }
-
-        if (return_pooled) {
-            auto text_projection = params["text_projection"];
-            ggml_tensor* pooled  = ggml_view_1d(ctx->ggml_ctx, x, hidden_size, x->nb[1] * max_token_idx);
-            if (text_projection != nullptr) {
-                pooled = ggml_ext_linear(ctx->ggml_ctx, pooled, text_projection, nullptr);
-            } else {
-                LOG_DEBUG("identity projection");
-            }
-            return pooled;  // [hidden_size, 1, 1]
-        }
-
-        return x;  // [N, n_token, hidden_size]
-    }
-};
-
-class CLIPVisionModel : public GGMLBlock {
-public:
-    // network hparams
-    int32_t num_channels      = 3;
-    int32_t patch_size        = 14;
-    int32_t image_size        = 224;
-    int32_t num_positions     = 257;  // (image_size / patch_size)^2 + 1
-    int32_t hidden_size       = 1024;
-    int32_t intermediate_size = 4096;
-    int32_t n_head            = 16;
-    int32_t n_layer           = 24;
-
-public:
-    CLIPVisionModel(CLIPVersion version = OPENAI_CLIP_VIT_L_14, bool proj_in = false) {
-        if (version == OPEN_CLIP_VIT_H_14) {
-            hidden_size       = 1280;
-            intermediate_size = 5120;
-            n_head            = 16;
-            n_layer           = 32;
-        } else if (version == OPEN_CLIP_VIT_BIGG_14) {
-            hidden_size       = 1664;
-            intermediate_size = 8192;
-            n_head            = 16;
-            n_layer           = 48;
-        }
-
-        blocks["embeddings"]     = std::shared_ptr<GGMLBlock>(new CLIPVisionEmbeddings(hidden_size, num_channels, patch_size, image_size));
-        blocks["pre_layernorm"]  = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size));
-        blocks["encoder"]        = std::shared_ptr<GGMLBlock>(new CLIPEncoder(n_layer, hidden_size, n_head, intermediate_size, proj_in));
-        blocks["post_layernorm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size));
-    }
-
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* pixel_values,
-                         bool return_pooled = true,
-                         int clip_skip      = -1) {
-        // pixel_values: [N, num_channels, image_size, image_size]
-        auto embeddings     = std::dynamic_pointer_cast<CLIPVisionEmbeddings>(blocks["embeddings"]);
-        auto pre_layernorm  = std::dynamic_pointer_cast<LayerNorm>(blocks["pre_layernorm"]);
-        auto encoder        = std::dynamic_pointer_cast<CLIPEncoder>(blocks["encoder"]);
-        auto post_layernorm = std::dynamic_pointer_cast<LayerNorm>(blocks["post_layernorm"]);
-
-        auto x = embeddings->forward(ctx, pixel_values);  // [N, num_positions, embed_dim]
-        x      = pre_layernorm->forward(ctx, x);
-        sd::ggml_graph_cut::mark_graph_cut(x, "clip_vision.prelude", "x");
-        x = encoder->forward(ctx, x, nullptr, clip_skip, "clip_vision");
-
-        auto last_hidden_state = x;
-
-        x = post_layernorm->forward(ctx, x);  // [N, n_token, hidden_size]
-
-        GGML_ASSERT(x->ne[3] == 1);
-        if (return_pooled) {
-            ggml_tensor* pooled = ggml_cont(ctx->ggml_ctx, ggml_view_2d(ctx->ggml_ctx, x, x->ne[0], x->ne[2], x->nb[2], 0));
-            return pooled;  // [N, hidden_size]
-        } else {
-            // return x;  // [N, n_token, hidden_size]
-            return last_hidden_state;  // [N, n_token, hidden_size]
-        }
-    }
-};
-
-class CLIPProjection : public UnaryBlock {
-protected:
-    int64_t in_features;
-    int64_t out_features;
-    bool transpose_weight;
-
-    void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
-        enum ggml_type wtype = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
-        if (transpose_weight) {
-            params["weight"] = ggml_new_tensor_2d(ctx, wtype, out_features, in_features);
-        } else {
-            params["weight"] = ggml_new_tensor_2d(ctx, wtype, in_features, out_features);
-        }
-    }
-
-public:
-    CLIPProjection(int64_t in_features,
-                   int64_t out_features,
-                   bool transpose_weight = false)
-        : in_features(in_features),
-          out_features(out_features),
-          transpose_weight(transpose_weight) {}
-
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
-        ggml_tensor* w = params["weight"];
-        if (transpose_weight) {
-            w = ggml_cont(ctx->ggml_ctx, ggml_transpose(ctx->ggml_ctx, w));
-        }
-        return ggml_ext_linear(ctx->ggml_ctx, x, w, nullptr);
-    }
-};
-
-class CLIPVisionModelProjection : public GGMLBlock {
-public:
-    int32_t hidden_size    = 1024;
-    int32_t projection_dim = 768;
-    int32_t image_size     = 224;
-
-public:
-    CLIPVisionModelProjection(CLIPVersion version   = OPENAI_CLIP_VIT_L_14,
-                              bool transpose_proj_w = false,
-                              bool proj_in          = false) {
-        if (version == OPEN_CLIP_VIT_H_14) {
-            hidden_size    = 1280;
-            projection_dim = 1024;
-        } else if (version == OPEN_CLIP_VIT_BIGG_14) {
-            hidden_size = 1664;
-        }
-
-        blocks["vision_model"]      = std::shared_ptr<GGMLBlock>(new CLIPVisionModel(version, proj_in));
-        blocks["visual_projection"] = std::shared_ptr<GGMLBlock>(new CLIPProjection(hidden_size, projection_dim, transpose_proj_w));
-    }
-
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* pixel_values,
-                         bool return_pooled = true,
-                         int clip_skip      = -1) {
-        // pixel_values: [N, num_channels, image_size, image_size]
-        // return: [N, projection_dim] if return_pooled else [N, n_token, hidden_size]
-        auto vision_model      = std::dynamic_pointer_cast<CLIPVisionModel>(blocks["vision_model"]);
-        auto visual_projection = std::dynamic_pointer_cast<CLIPProjection>(blocks["visual_projection"]);
-
-        auto x = vision_model->forward(ctx, pixel_values, return_pooled, clip_skip);  // [N, hidden_size] or [N, n_token, hidden_size]
-
-        if (return_pooled) {
-            x = visual_projection->forward(ctx, x);  // [N, projection_dim]
-        }
-
-        return x;
-    }
-};
-
-struct CLIPTextModelRunner : public GGMLRunner {
-    CLIPTextModel model;
-
-    std::vector<float> attention_mask_vec;
-
-    CLIPTextModelRunner(ggml_backend_t backend,
-                        ggml_backend_t params_backend,
-                        const String2TensorStorage& tensor_storage_map,
-                        const std::string prefix,
-                        CLIPVersion version = OPENAI_CLIP_VIT_L_14,
-                        bool with_final_ln  = true,
-                        bool force_clip_f32 = false)
-        : GGMLRunner(backend, params_backend) {
-        bool proj_in = false;
-        for (const auto& [name, tensor_storage] : tensor_storage_map) {
-            if (!starts_with(name, prefix)) {
-                continue;
-            }
-            if (contains(name, "self_attn.in_proj")) {
-                proj_in = true;
-                break;
-            }
-        }
-        model = CLIPTextModel(version, with_final_ln, force_clip_f32, proj_in);
-        model.init(params_ctx, tensor_storage_map, prefix);
-    }
-
-    std::string get_desc() override {
-        return "clip";
-    }
-
-    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string prefix) {
-        model.get_param_tensors(tensors, prefix);
-    }
-
-    ggml_tensor* forward(GGMLRunnerContext* ctx,
-                         ggml_tensor* input_ids,
-                         ggml_tensor* embeddings,
-                         ggml_tensor* mask,
-                         size_t max_token_idx = 0,
-                         bool return_pooled   = false,
-                         int clip_skip        = -1) {
-        size_t N       = input_ids->ne[1];
-        size_t n_token = input_ids->ne[0];
-        if (input_ids->ne[0] > model.n_token) {
-            GGML_ASSERT(input_ids->ne[0] % model.n_token == 0);
-            input_ids = ggml_reshape_2d(ctx->ggml_ctx, input_ids, model.n_token, input_ids->ne[0] / model.n_token);
-        }
-
-        return model.forward(ctx, input_ids, embeddings, mask, max_token_idx, return_pooled, clip_skip);
-    }
-
-    ggml_cgraph* build_graph(const sd::Tensor<int32_t>& input_ids_tensor,
-                             int num_custom_embeddings    = 0,
-                             void* custom_embeddings_data = nullptr,
-                             size_t max_token_idx         = 0,
-                             bool return_pooled           = false,
-                             int clip_skip                = -1) {
-        ggml_cgraph* gf        = new_graph_custom(2048);
-        ggml_tensor* input_ids = make_input(input_ids_tensor);
-
-        ggml_tensor* embeddings = nullptr;
-
-        if (num_custom_embeddings > 0 && custom_embeddings_data != nullptr) {
-            auto token_embed_weight = model.get_token_embed_weight();
-            auto custom_embeddings  = ggml_new_tensor_2d(compute_ctx,
-                                                         token_embed_weight->type,
-                                                         model.hidden_size,
-                                                         num_custom_embeddings);
-            set_backend_tensor_data(custom_embeddings, custom_embeddings_data);
-
-            // concatenate custom embeddings
-            embeddings = ggml_concat(compute_ctx, token_embed_weight, custom_embeddings, 1);
-        }
-
-        int n_tokens = static_cast<int>(input_ids->ne[0]);
-        attention_mask_vec.resize(n_tokens * n_tokens);
-        for (int i0 = 0; i0 < n_tokens; i0++) {
-            for (int i1 = 0; i1 < n_tokens; i1++) {
-                float value = 0.f;
-                if (i0 > i1) {
-                    value = -INFINITY;
-                }
-                attention_mask_vec[i1 * n_tokens + i0] = value;
-            }
-        }
-        auto attention_mask = ggml_new_tensor_2d(compute_ctx, GGML_TYPE_F32, n_tokens, n_tokens);
-        set_backend_tensor_data(attention_mask, attention_mask_vec.data());
-
-        auto runner_ctx = get_context();
-
-        ggml_tensor* hidden_states = forward(&runner_ctx, input_ids, embeddings, attention_mask, max_token_idx, return_pooled, clip_skip);
-
-        ggml_build_forward_expand(gf, hidden_states);
-
-        return gf;
-    }
-
-    sd::Tensor<float> compute(const int n_threads,
-                              const sd::Tensor<int32_t>& input_ids,
-                              int num_custom_embeddings,
-                              void* custom_embeddings_data,
-                              size_t max_token_idx,
-                              bool return_pooled,
-                              int clip_skip) {
-        auto get_graph = [&]() -> ggml_cgraph* {
-            return build_graph(input_ids, num_custom_embeddings, custom_embeddings_data, max_token_idx, return_pooled, clip_skip);
-        };
-        auto result = GGMLRunner::compute<float>(get_graph, n_threads, true);
-        if (return_pooled) {
-            return take_or_empty(std::move(result));
-        }
-        return restore_trailing_singleton_dims(std::move(result), 3);
-    }
-};
-
-#endif  // __CLIP_HPP__

src/common_dit.hpp

@@ -1,108 +0,0 @@
-#ifndef __COMMON_DIT_HPP__
-#define __COMMON_DIT_HPP__
-
-#include "ggml_extend.hpp"
-
-namespace DiT {
-    inline ggml_tensor* patchify(ggml_context* ctx,
-                                 ggml_tensor* x,
-                                 int pw,
-                                 int ph,
-                                 bool patch_last = true) {
-        // x: [N, C, H, W]
-        // return: [N, h*w, C*ph*pw] if patch_last else [N, h*w, ph*pw*C]
-        int64_t N = x->ne[3];
-        int64_t C = x->ne[2];
-        int64_t H = x->ne[1];
-        int64_t W = x->ne[0];
-        int64_t h = H / ph;
-        int64_t w = W / pw;
-
-        GGML_ASSERT(h * ph == H && w * pw == W);
-
-        x = ggml_reshape_4d(ctx, x, pw, w, ph, h * C * N);     // [N*C*h, ph, w, pw]
-        x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));  // [N*C*h, w, ph, pw]
-        x = ggml_reshape_4d(ctx, x, pw * ph, w * h, C, N);     // [N, C, h*w, ph*pw]
-        if (patch_last) {
-            x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));  // [N, h*w, C, ph*pw]
-            x = ggml_reshape_3d(ctx, x, pw * ph * C, w * h, N);    // [N, h*w, C*ph*pw]
-        } else {
-            x = ggml_cont(ctx, ggml_ext_torch_permute(ctx, x, 2, 0, 1, 3));  // [N, h*w, C, ph*pw]
-            x = ggml_reshape_3d(ctx, x, C * pw * ph, w * h, N);              // [N, h*w, ph*pw*C]
-        }
-        return x;
-    }
-
-    inline ggml_tensor* unpatchify(ggml_context* ctx,
-                                   ggml_tensor* x,
-                                   int64_t h,
-                                   int64_t w,
-                                   int ph,
-                                   int pw,
-                                   bool patch_last = true) {
-        // x: [N, h*w, C*ph*pw] if patch_last else [N, h*w, ph*pw*C]
-        // return: [N, C, H, W]
-        int64_t N = x->ne[2];
-        int64_t C = x->ne[0] / ph / pw;
-        int64_t H = h * ph;
-        int64_t W = w * pw;
-
-        GGML_ASSERT(C * ph * pw == x->ne[0]);
-
-        if (patch_last) {
-            x = ggml_reshape_4d(ctx, x, pw * ph, C, w * h, N);     // [N, h*w, C, ph*pw]
-            x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));  // [N, C, h*w, ph*pw]
-        } else {
-            x = ggml_reshape_4d(ctx, x, C, pw * ph, w * h, N);     // [N, h*w, ph*pw, C]
-            x = ggml_cont(ctx, ggml_permute(ctx, x, 2, 0, 1, 3));  // [N, C, h*w, ph*pw]
-        }
-
-        x = ggml_reshape_4d(ctx, x, pw, ph, w, h * C * N);     // [N*C*h, w, ph, pw]
-        x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));  // [N*C*h, ph, w, pw]
-        x = ggml_reshape_4d(ctx, x, W, H, C, N);               // [N, C, h*ph, w*pw]
-
-        return x;
-    }
-
-    inline ggml_tensor* pad_to_patch_size(GGMLRunnerContext* ctx,
-                                          ggml_tensor* x,
-                                          int ph,
-                                          int pw) {
-        int64_t W = x->ne[0];
-        int64_t H = x->ne[1];
-
-        int pad_h = (ph - H % ph) % ph;
-        int pad_w = (pw - W % pw) % pw;
-        x         = ggml_ext_pad(ctx->ggml_ctx, x, pad_w, pad_h, 0, 0, ctx->circular_x_enabled, ctx->circular_y_enabled);
-        return x;
-    }
-
-    inline ggml_tensor* pad_and_patchify(GGMLRunnerContext* ctx,
-                                         ggml_tensor* x,
-                                         int ph,
-                                         int pw,
-                                         bool patch_last = true) {
-        x = pad_to_patch_size(ctx, x, ph, pw);
-        x = patchify(ctx->ggml_ctx, x, ph, pw, patch_last);
-        return x;
-    }
-
-    inline ggml_tensor* unpatchify_and_crop(ggml_context* ctx,
-                                            ggml_tensor* x,
-                                            int64_t H,
-                                            int64_t W,
-                                            int ph,
-                                            int pw,
-                                            bool patch_last = true) {
-        int pad_h = (ph - H % ph) % ph;
-        int pad_w = (pw - W % pw) % pw;
-        int64_t h = ((H + pad_h) / ph);
-        int64_t w = ((W + pad_w) / pw);
-        x         = unpatchify(ctx, x, h, w, ph, pw, patch_last);  // [N, C, H + pad_h, W + pad_w]
-        x         = ggml_ext_slice(ctx, x, 1, 0, H);               // [N, C, H, W + pad_w]
-        x         = ggml_ext_slice(ctx, x, 0, 0, W);               // [N, C, H, W]
-        return x;
-    }
-}  // namespace DiT
-
-#endif  // __COMMON_DIT_HPP__

src/condition_cache_utils.hpp

@@ -1,64 +0,0 @@
-#ifndef __CONDITION_CACHE_UTILS_HPP__
-#define __CONDITION_CACHE_UTILS_HPP__
-
-#include <vector>
-
-#include "tensor.hpp"
-
-namespace sd {
-
-    inline bool store_condition_cache_diff(std::vector<float>* diff,
-                                           const sd::Tensor<float>& input,
-                                           const sd::Tensor<float>& output) {
-        if (diff == nullptr || input.empty() || output.empty()) {
-            return false;
-        }
-
-        size_t input_size  = static_cast<size_t>(input.numel());
-        size_t output_size = static_cast<size_t>(output.numel());
-        if (input_size == 0 || input_size != output_size) {
-            diff->clear();
-            return false;
-        }
-
-        const float* input_data  = input.data();
-        const float* output_data = output.data();
-        if (input_data == nullptr || output_data == nullptr) {
-            diff->clear();
-            return false;
-        }
-
-        diff->resize(output_size);
-        for (size_t i = 0; i < output_size; ++i) {
-            (*diff)[i] = output_data[i] - input_data[i];
-        }
-        return true;
-    }
-
-    inline bool apply_condition_cache_diff(const std::vector<float>& diff,
-                                           const sd::Tensor<float>& input,
-                                           sd::Tensor<float>* output) {
-        if (output == nullptr || input.empty() || diff.empty()) {
-            return false;
-        }
-
-        size_t input_size = static_cast<size_t>(input.numel());
-        if (input_size == 0 || diff.size() != input_size) {
-            return false;
-        }
-
-        *output            = input;
-        float* output_data = output->data();
-        if (output_data == nullptr) {
-            return false;
-        }
-
-        for (size_t i = 0; i < input_size; ++i) {
-            output_data[i] += diff[i];
-        }
-        return true;
-    }
-
-}  // namespace sd
-
-#endif  // __CONDITION_CACHE_UTILS_HPP__

src/convert.cpp

@@ -1,138 +0,0 @@
-#include <cstring>
-#include <mutex>
-#include <regex>
-#include <vector>
-
-#include "model.h"
-#include "model_io/gguf_io.h"
-#include "model_io/safetensors_io.h"
-#include "util.h"
-
-#include "ggml-cpu.h"
-
-static ggml_type get_export_tensor_type(ModelLoader& model_loader,
-                                        const TensorStorage& tensor_storage,
-                                        ggml_type type,
-                                        const TensorTypeRules& tensor_type_rules) {
-    const std::string& name = tensor_storage.name;
-    ggml_type tensor_type   = tensor_storage.type;
-    ggml_type dst_type      = type;
-
-    for (const auto& tensor_type_rule : tensor_type_rules) {
-        std::regex pattern(tensor_type_rule.first);
-        if (std::regex_search(name, pattern)) {
-            dst_type = tensor_type_rule.second;
-            break;
-        }
-    }
-
-    if (model_loader.tensor_should_be_converted(tensor_storage, dst_type)) {
-        tensor_type = dst_type;
-    }
-
-    return tensor_type;
-}
-
-static bool load_tensors_for_export(ModelLoader& model_loader,
-                                    ggml_context* ggml_ctx,
-                                    ggml_type type,
-                                    const TensorTypeRules& tensor_type_rules,
-                                    std::vector<TensorWriteInfo>& tensors) {
-    std::mutex tensor_mutex;
-    auto on_new_tensor_cb = [&](const TensorStorage& tensor_storage, ggml_tensor** dst_tensor) -> bool {
-        const std::string& name = tensor_storage.name;
-        ggml_type tensor_type   = get_export_tensor_type(model_loader, tensor_storage, type, tensor_type_rules);
-
-        std::lock_guard<std::mutex> lock(tensor_mutex);
-        ggml_tensor* tensor = ggml_new_tensor(ggml_ctx, tensor_type, tensor_storage.n_dims, tensor_storage.ne);
-        if (tensor == nullptr) {
-            LOG_ERROR("ggml_new_tensor failed");
-            return false;
-        }
-        ggml_set_name(tensor, name.c_str());
-
-        if (!tensor->data) {
-            GGML_ASSERT(ggml_nelements(tensor) == 0);
-            // Avoid crashing writers by setting a dummy pointer for zero-sized tensors.
-            LOG_DEBUG("setting dummy pointer for zero-sized tensor %s", name.c_str());
-            tensor->data = ggml_get_mem_buffer(ggml_ctx);
-        }
-
-        TensorWriteInfo write_info;
-        write_info.tensor = tensor;
-        write_info.n_dims = tensor_storage.n_dims;
-        for (int i = 0; i < tensor_storage.n_dims; ++i) {
-            write_info.ne[i] = tensor_storage.ne[i];
-        }
-
-        *dst_tensor = tensor;
-        tensors.push_back(std::move(write_info));
-
-        return true;
-    };
-
-    bool success = model_loader.load_tensors(on_new_tensor_cb);
-    LOG_INFO("load tensors done");
-    return success;
-}
-
-bool convert(const char* input_path,
-             const char* vae_path,
-             const char* output_path,
-             sd_type_t output_type,
-             const char* tensor_type_rules,
-             bool convert_name) {
-    ModelLoader model_loader;
-
-    if (!model_loader.init_from_file(input_path)) {
-        LOG_ERROR("init model loader from file failed: '%s'", input_path);
-        return false;
-    }
-
-    if (vae_path != nullptr && strlen(vae_path) > 0) {
-        if (!model_loader.init_from_file(vae_path, "vae.")) {
-            LOG_ERROR("init model loader from file failed: '%s'", vae_path);
-            return false;
-        }
-    }
-    if (convert_name) {
-        model_loader.convert_tensors_name();
-    }
-
-    ggml_type type             = (ggml_type)output_type;
-    bool output_is_safetensors = ends_with(output_path, ".safetensors");
-    TensorTypeRules type_rules = parse_tensor_type_rules(tensor_type_rules);
-
-    auto backend    = ggml_backend_cpu_init();
-    size_t mem_size = 1 * 1024 * 1024;  // for padding
-    mem_size += model_loader.get_tensor_storage_map().size() * ggml_tensor_overhead();
-    mem_size += model_loader.get_params_mem_size(backend, type);
-    LOG_INFO("model tensors mem size: %.2fMB", mem_size / 1024.f / 1024.f);
-    ggml_context* ggml_ctx = ggml_init({mem_size, nullptr, false});
-
-    if (ggml_ctx == nullptr) {
-        LOG_ERROR("ggml_init failed for converter");
-        ggml_backend_free(backend);
-        return false;
-    }
-
-    std::vector<TensorWriteInfo> tensors;
-    bool success = load_tensors_for_export(model_loader, ggml_ctx, type, type_rules, tensors);
-    ggml_backend_free(backend);
-
-    std::string error;
-    if (success) {
-        if (output_is_safetensors) {
-            success = write_safetensors_file(output_path, tensors, &error);
-        } else {
-            success = write_gguf_file(output_path, tensors, &error);
-        }
-    }
-
-    if (!success && !error.empty()) {
-        LOG_ERROR("%s", error.c_str());
-    }
-
-    ggml_free(ggml_ctx);
-    return success;
-}