feat: enable hip INT8 tensorwise matmul and convrot (#2071)

Co-authored-by: leejet <leejet714@gmail.com>

.github/workflows/close-organization-fork-prs.yml

@@ -0,0 +1,61 @@
+name: Close PRs from organization forks
+
+on:
+  pull_request_target:
+    types: [opened, reopened]
+
+permissions:
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: false
+
+jobs:
+  close-organization-fork-pr:
+    if: >-
+      github.event.pull_request.head.repo.owner.type == 'Organization' &&
+      github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Explain the contribution policy and close the PR
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const { data: pr } = await github.rest.pulls.get({
+              ...context.repo,
+              pull_number: context.issue.number,
+            });
+            const headRepo = pr.head.repo;
+            if (pr.state !== 'open' || !headRepo ||
+                headRepo.id === pr.base.repo.id || headRepo.owner.type !== 'Organization') {
+              return;
+            }
+
+            const marker = '<!-- organization-fork-policy -->';
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              ...context.repo,
+              issue_number: pr.number,
+              per_page: 100,
+            });
+            const alreadyExplained = comments.some(comment =>
+              comment.user?.login === 'github-actions[bot]' && comment.body?.includes(marker));
+            if (!alreadyExplained) {
+              await github.rest.issues.createComment({
+                ...context.repo,
+                issue_number: pr.number,
+                body: [
+                  marker,
+                  'This repository requires contributions from forks to use a personal fork with **Allow edits from maintainers** enabled.',
+                  'GitHub does not support this option for organization-owned forks, so this PR is being closed automatically.',
+                  'Please open a new PR from a fork in your personal GitHub account and enable **Allow edits from maintainers** so maintainers can help update the branch.',
+                  'See [the GitHub documentation](https://docs.github.com/en/pull-requests/how-tos/work-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork).',
+                ].join('\n\n'),
+              });
+            }
+            await github.rest.pulls.update({
+              ...context.repo,
+              pull_number: pr.number,
+              state: 'closed',
+            });

CONTRIBUTING.md

@@ -12,6 +12,10 @@ If you want to update a third-party dependency, please open an issue first inste
 
 ## Pull Requests
 
+When contributing from a fork, use a fork under your personal GitHub account and enable **Allow edits from maintainers**. This lets maintainers make follow-up fixes directly on the PR branch.
+
+PRs from organization-owned forks are automatically closed when opened or reopened because GitHub does not support this maintainer-edit option for those forks. Submit the changes from a personal fork instead. See [GitHub's documentation](https://docs.github.com/en/pull-requests/how-tos/work-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
+
 Keep each PR focused on one clear change. Large or overly complex PRs are harder to review and may not be merged.
 
 Do not include test code or test scripts in commits or PRs. Keep them local and report verification results in the PR description.

README.md

@@ -64,6 +64,8 @@ API and command-line option may change frequently.***
     - [HiDream-O1-Image](./docs/hidream_o1_image.md)
     - [Ideogram4](./docs/ideogram4.md)
     - [LLaDA-Image](./docs/llada_image.md)
+    - [Ming-Image Design](./docs/ming_image.md)
+    - [PixArt](./docs/pixart.md)
   - [Image Edit Models](./docs/edit.md)
     - [FLUX.1-Kontext-dev](./docs/kontext.md)
     - [Qwen Image Edit series](./docs/qwen_image_edit.md)

docker/Dockerfile

@@ -40,4 +40,6 @@ RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \
     printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \
     chmod +x /sd-cli /sd-server
 
+WORKDIR /sd.cpp
+
 ENTRYPOINT [ "/sd-cli" ]

docker/Dockerfile.cuda

@@ -57,4 +57,6 @@ RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \
     printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \
     chmod +x /sd-cli /sd-server
 
+WORKDIR /sd.cpp
+
 ENTRYPOINT [ "/sd-cli" ]

docker/Dockerfile.musa

@@ -42,4 +42,6 @@ RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \
     printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \
     chmod +x /sd-cli /sd-server
 
+WORKDIR /sd.cpp
+
 ENTRYPOINT [ "/sd-cli" ]

docker/Dockerfile.sycl

@@ -29,4 +29,6 @@ FROM intel/oneapi-basekit:${SYCL_VERSION}-devel-ubuntu24.04 AS runtime
 COPY --from=build /sd.cpp/build/bin/sd-cli /sd-cli
 COPY --from=build /sd.cpp/build/bin/sd-server /sd-server
 
+WORKDIR /sd.cpp
+
 ENTRYPOINT [ "/sd-cli" ]

docker/Dockerfile.vulkan

@@ -41,4 +41,6 @@ RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \
     printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \
     chmod +x /sd-cli /sd-server
 
+WORKDIR /sd.cpp
+
 ENTRYPOINT [ "/sd-cli" ]

docs/backend.md

@@ -156,8 +156,14 @@ the runner's graph-cut capacity checks.
 
 Runtime capacity checks also leave 512 MiB of currently free device memory for
 backend scratch buffers and pipelines, including with explicit backend assignments.
-They cap stale free-memory reports by the device's total memory minus tracked
-resident allocations and reject reports that exceed the device's total memory.
+They cap free-memory reports by the device's total memory minus tracked
+resident allocations. Vulkan reports exceeding total memory are rejected because
+its heap-budget subtraction can underflow. Other backends use the cap instead of
+treating such reports as zero free memory. Failed checks log the reported free and
+total memory alongside tracked weight and runtime allocations.
+With `--mmap`, device-backed mappings count toward these budgets at their full
+mapped-file size, once per device buffer even when multiple parameter blocks
+share it. Mappings retained in the loader cache continue to count.
 
 Components are considered in `diffusion`, `te`, `vae` order so that repeatedly
 used diffusion weights have priority. Each component's weights use the first

docs/caching.md

@@ -2,6 +2,16 @@
 
 Caching methods accelerate diffusion inference by reusing intermediate computations when changes between steps are small.
 
+### Conditioning Cache
+
+Conditioning results are cached per model context using an LRU cache. The default
+capacity is **0 (disabled) for `sd-cli`** and **4 entries for `sd-server` and the C
+API**. Set `--conditioning-cache-size N` to change the limit; `0` disables caching.
+For example, `sd-cli -m model.safetensors -p "a cat" --conditioning-cache-size 4`
+enables the cache in the CLI. The C API option is
+`sd_ctx_params_t::conditioning_cache_size`, initialized by `sd_ctx_params_init()`.
+This cache is independent of the diffusion-step `--cache-mode` options below.
+
 ### Cache Modes
 
 | Mode | Target | Description |

docs/edit.md

@@ -26,6 +26,9 @@ Stable-diffusion.spp also supports basic Unet-based editing models like instruct
 
 ## Configuring Reference Modes (`--ref-image-args`)
 
+For a one-time input transform before reference presets and model processing,
+including cropping, padding, and resizing algorithms, see [Image preprocessing](./image_preprocessing.md).
+
 Different DiT-based editing models require different configurations to process reference images correctly (e.g., whether to use a Vision Language Model (VLM) encoder or pass VAE-encoded images directly to the DiT).
 
 To simplify this, we provide **Presets**. By default, the system automatically selects the best preset based on the model architecture. However, you can override this using the `--ref-image-args` argument.

docs/esrgan.md

@@ -2,6 +2,8 @@
 
 You can use ESRGAN—such as the model [RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth)—to upscale the generated images and improve their overall resolution and clarity.
 
+RGBA images, including Qwen Image 2.1 output, keep their alpha channel during model upscaling and hires fix. ESRGAN processes the RGB channels; the alpha channel is resized with bilinear interpolation and recombined with the upscaled image.
+
 - Specify the model path using the `--upscale-model PATH` parameter. example:
 
 ```bash

docs/image_preprocessing.md

@@ -0,0 +1,173 @@
+# Image preprocessing
+
+Use `--image-preprocess` to transform each image input once, before generation:
+
+```sh
+sd-cli ... \
+  --image-preprocess "target=init,mode=crop-resize,filter=lanczos,antialias=true" \
+  --image-preprocess "target=mask,filter=nearest-exact" \
+  --image-preprocess "target=ref,index=0,mode=fit-pad,width=768,height=768,filter=bicubic"
+```
+
+CLI and server image loaders decode at the original resolution. The generation
+entry point merges input defaults with user rules and prepares one transformed
+image per input. The original pipeline then consumes those images, including
+its mandatory canvas adaptation, reference resizing, and encoder preprocessing.
+
+```text
+native-resolution image
+  -> input defaults + user overrides
+  -> one input transform
+  -> original generation pipeline and model-specific processing
+```
+
+These rules do not override internal VAE, CLIP/VLM, ControlNet, or pixel-patch preprocessing.
+`--ref-image-args` retains its existing meaning and runs after this input transform.
+
+## Inputs and defaults
+
+| `target` | Input | Default geometry | Indexed? |
+| --- | --- | --- | --- |
+| `init` | img2img image or video first frame | Center crop to the generation aspect ratio, then resize | No |
+| `end` | Video last frame | Center crop, then resize | No |
+| `mask` | Inpainting mask | Inherit init geometry; otherwise center crop, then resize | No |
+| `control` | Control image | Center crop, then resize | No |
+| `ref` | Reference images | Preserve source dimensions | Yes |
+| `ip-adapter` | IP-Adapter image | Preserve source dimensions | No |
+| `id` | PhotoMaker identity images | Preserve source dimensions | Yes |
+| `control-frame` | Control video frames | Center crop, then resize | Yes |
+
+Canvas defaults use the aligned generation dimensions. Reference, IP-Adapter,
+and identity inputs use their original dimensions unless overridden. Default
+resampling is nearest for images and nearest-exact for masks.
+
+These defaults are shared by CLI, server, and C API. Moving geometry out of
+the loaders replaces the previous CLI/server BOX/sRGB resizing, so default
+pixels are not guaranteed to match earlier builds.
+
+Reference video and audio preprocessing are outside these image rules.
+Preprocessing options apply to `img_gen` and `vid_gen`, not standalone upscale
+or ADetailer mode. ADetailer clears the user's rules for its internal crops.
+
+## Rules
+
+Rules are comma-separated `key=value` lists. Repeat the CLI option or separate
+rules with semicolons. Every rule requires a `target` and at least one option.
+Rule syntax and input compatibility are checked when image/video generation
+starts. Unknown keys, invalid values, duplicate keys in a rule, missing images,
+and out-of-range indices cause generation to fail with an error log.
+
+Omit `index` to configure every image of that type; otherwise use a zero-based
+index. CLI directory inputs follow filename order. Indexed rules override
+type-wide rules field by field, regardless of order. At equal specificity,
+the last value for a field wins. `auto` selects the input preset.
+
+| `mode` | Input transform |
+| --- | --- |
+| `auto` | Use the input's default geometry |
+| `none` | Keep source dimensions without resizing, cropping, or padding |
+| `stretch` | Resize to the target dimensions |
+| `crop` | Crop a target-sized rectangle without resizing; fail if the source is too small |
+| `crop-resize` | Crop to the target aspect ratio, then resize |
+| `fit-pad` | Fit the entire image inside the target dimensions, preserving aspect ratio, then pad |
+
+`width` and `height` must be specified together as positive integers. They
+override the input transform's dimensions, not the generation or encoder size.
+For a native-size preset, specifying dimensions without a mode selects stretch.
+`mode=none` with explicit dimensions different from the source is contradictory
+and is rejected.
+
+`anchor=center|top|bottom|left|right` selects crop/padding placement.
+`pad_color=#RRGGBB` or `#RRGGBBAA` selects padding, defaulting to opaque black.
+A grayscale mask uses the first color component.
+
+`filter=auto|nearest|nearest-exact|bilinear|bicubic|lanczos` selects resampling.
+`antialias=auto|true|false` enables antialiasing automatically for filtered
+downscaling; explicit true requires bilinear, bicubic, or Lanczos.
+Filtered RGBA resizing uses premultiplied alpha.
+
+`canny=true|false` enables edge detection for any supported image target,
+defaulting to `false`. It runs once after geometry, before the original
+generation pipeline, including with `mode=none`. Grayscale, grayscale-alpha,
+RGB, and RGBA inputs are supported; alpha is preserved.
+
+Each input has its own Canny setting. Indexed rules can enable or disable it
+for individual references, identity images, or video control frames.
+
+```sh
+--image-preprocess "target=init,mode=fit-pad,canny=true"
+--image-preprocess "target=ref,index=0,mode=none,canny=true"
+--image-preprocess "target=control-frame,index=2,canny=true"
+```
+
+Init and mask sources must have the same dimensions. The mask inherits the
+init crop, resize, and padding coordinates, while retaining its own filter,
+padding value, and Canny setting. Conflicting mask geometry is rejected. An
+omitted mask remains absent until the original pipeline creates its default mask.
+
+## Downstream behavior
+
+`mode=none` only skips the input geometry transform. For example:
+
+```sh
+--image-preprocess "target=init,mode=none" \
+--image-preprocess "target=ref,mode=none"
+```
+
+The init image is still adapted to the generation canvas by the original
+pipeline. Reference images still follow `--ref-image-args` and model-specific
+resizing. CLIP retains its fixed input dimensions and normalization. HiDream-O1
+retains its original pixel-reference and visual preprocessing.
+
+Existing sharing between consumers is preserved: for example, Wan img2video
+uses the same adapted first frame for VAE conditioning and CLIP. High-resolution
+passes reuse the prepared images and apply their original size adaptation;
+they do not apply the user's crop a second time.
+
+To disable reference resizing before VAE encoding, use
+`--ref-image-args "resize_before_vae=false"` or the server field
+`"ref_image_args": "resize_before_vae=false"`. This is separate from
+`target=ref,mode=none`, which only skips input geometry. Model constraints
+still apply.
+
+## Server requests
+
+Native image/video requests and SDAPI accept `image_preprocess` as a string or
+an array of rule strings:
+
+```json
+{
+  "image_preprocess": [
+    "target=init,mode=fit-pad,filter=bicubic",
+    "target=mask,filter=nearest-exact",
+    "target=ref,index=0,mode=none"
+  ]
+}
+```
+
+OpenAI-compatible requests accept it through
+`<sd_cpp_extra_args>{...}</sd_cpp_extra_args>` in the prompt.
+Request rules replace server-default rules. Generation metadata records the
+user rules; image encodings and channel conventions are unchanged.
+
+## C API
+
+Set `image_preprocess` on the existing image/video generation parameters.
+The `generate_image()` and `generate_video()` signatures are unchanged:
+
+```c
+sd_img_gen_params_t params;
+sd_img_gen_params_init(&params);
+/* Set prompt, original-resolution input images, and generation options. */
+params.image_preprocess.rules = "target=init,mode=crop-resize,filter=lanczos;"
+                                "target=mask,filter=nearest-exact";
+bool ok = generate_image(ctx, &params, &images, &count);
+```
+
+Both generation parameter initializers set `image_preprocess.rules` to `NULL`,
+selecting input presets. Rule strings are borrowed for the synchronous call.
+The library owns temporary transformed pixels; caller images and arrays are
+not modified. Add `canny=true` to the desired target's rule in
+`image_preprocess.rules` to enable Canny.
+
+The parameter structs have grown; applications and bindings must be rebuilt.

docs/int8_convrot.md

@@ -84,17 +84,18 @@ The floating-point output is reconstructed as
 Y[r, o] ~= A[r, o] * s_x[r] * s_w[o] + b[o]
 ```
 
-The packed runtime activation tensor contains the I8 activation rows and their floating-point row scales. Linear layers that share the same input and convrot group size reuse this packed tensor, avoiding repeated rotation and activation quantization within the graph.
+The packed runtime activation tensor contains the I8 activation rows and their floating-point row scales. Linear layers with an input scale of `1` that share the same input and convrot group size reuse this packed tensor, avoiding repeated rotation and activation quantization within the graph. For other input scales, activations are scaled before explicit convrot quantization, and the linear output is unscaled before adding bias.
 
 ## Backend support
 
 - CPU provides the portable regular Hadamard, activation quantization, INT8 matrix multiplication, and scale restoration implementations.
 - NVIDIA CUDA devices with compute capability 7.5 or newer use the native accelerated path. For H256, CUDA fuses the rotation, row-wise maximum reduction, and activation quantization. It uses cuBLAS for I8 x I8 to I32 GEMM and a CUDA kernel for scale restoration and bias addition.
-- Vulkan and other GPU backends do not currently have dedicated INT8 convrot kernels. They use the backend scheduler to fall back to CPU, which is expected to be substantially slower than the CUDA path.
+- AMD HIP devices in the CDNA, RDNA3 (including RDNA3.5), and RDNA4 families use the same INT8/H256 kernels with hipBLAS for I8 x I8 to I32 GEMM. Other AMD architectures fall back to CPU for INT8 matrix multiplication.
+- Vulkan provides native H256 activation quantization and INT8 matrix multiplication when the build and device support accelerated packed INT8 dot products. Unsupported configurations and GPU backends without these kernels use the backend scheduler to fall back to CPU.
 
 LoRA adapters are applied at runtime without modifying the INT8 weights. The INT8 convrot path computes the base linear output, while LoRA, LoHa, LoKr, and raw weight-difference adapters compute their output corrections from the original, unrotated activation and add them to the base output. `--lora-apply-mode auto` selects this path for models containing INT8 tensorwise weights. If `immediately` is requested, sd.cpp falls back to runtime application because merging an adapter would require dequantizing and rotating its weight update, then recalculating the per-row scales and requantizing the result.
 
-The dedicated CUDA convrot activation path currently requires a group size of `256`; other supported group sizes use CPU execution.
+The dedicated CUDA and HIP convrot activation paths currently require a group size of `256`; other supported group sizes use CPU execution.
 
 ## Example
 

docs/mage_flow.md

@@ -4,20 +4,8 @@
 
 ## Download weights
 
-- Download Mage-Flow
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow/tree/main/transformer
-- Download Mage-Flow-Base
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow-Base/tree/main/transformer
-- Download Mage-Flow-Turbo
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow-Turbo/tree/main/transformer
-- Download Mage-Flow-Edit
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow-Edit/tree/main/transformer
-- Download Mage-Flow-Edit-Turbo
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow-Edit-Turbo/tree/main/transformer
-- Download Mage-Flow-Edit-Base
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow-Edit-Base/tree/main/transformer
-- Download Mage-Flow vae
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow/tree/main/vae
+- Download Mage-Flow diffusion from https://huggingface.co/Comfy-Org/Mage-Flow/tree/main/diffusion_models
+- Download Mage-Flow vae from https://huggingface.co/Comfy-Org/Mage-Flow/tree/main/vae
 - Download Qwen3-VL 4B
     - safetensors: https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders
     - gguf: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main

docs/ming_image.md

@@ -0,0 +1,24 @@
+# Ming-Image
+
+[Ming-Image](https://github.com/inclusionAI/Ming-Image) 0.1 Design uses a 6B diffusion transformer (DiT), Ling-mini-2.0 for text conditioning, and the Ming-Image VAE. Text-to-image generation with RGBA output is supported.
+
+## Download weights
+
+- Download Ming-Image 0.1 Design DiT
+    - safetensors: https://huggingface.co/Comfy-Org/Ming-Image/tree/main/diffusion_models
+- Download Ling-mini-2.0 BF16
+    - safetensors: https://huggingface.co/Comfy-Org/Ming-Image/tree/main/text_encoders
+- Download Ming-Image VAE
+    - safetensors: https://huggingface.co/Comfy-Org/Ming-Image/tree/main/vae
+- Download Ling tokenizer
+    - tokenizer.json: https://huggingface.co/inclusionAI/Ming-Image-0.1-Design/blob/main/mllm/tokenizer.json
+
+The example below uses `ming_image_0.1_design_bf16.safetensors` for the DiT. You can also use `ming_image_0.1_design_int8_convrot.safetensors` with [INT8 convrot support](int8_convrot.md). Use the BF16 text encoder.
+
+## Text-to-image
+
+Pass the Ling `tokenizer.json` with `--tokenizer` and use the matching Ming-Image VAE. Image dimensions must be multiples of 16. Save the output as PNG to preserve the alpha channel.
+
+```bash
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\ming_image_0.1_design_bf16.safetensors --llm ..\models\text_encoders\ming_image_0.1_ling_mini_2.0_bf16.safetensors --vae ..\models\vae\ming_image_vae_bf16.safetensors --tokenizer ..\models\text_encoders\tokenizer.json -p "A cheerful orange cat sticker, transparent background" --width 1024 --height 1024 --steps 12 --cfg-scale 1 --sampling-method euler --diffusion-fa -v --offload-to-cpu -o ming_image.png
+```

docs/performance.md

@@ -21,6 +21,54 @@ CPU fallback. It excludes weights and cache buffers. Within a runner lifecycle,
 the summary is printed only on the first graph or when backend capacities or the
 segment count change.
 
+## Use VAE tiling to reduce encode and decode memory usage.
+
+`--vae-tiling` enables spatial tiling for both VAE encoding and decoding. The
+default tile size is 256x256 **image pixels**, independent of the VAE scale factor:
+
+```shell
+--vae-tiling --vae-tile-size 256x256 --vae-tile-overlap 0.5
+```
+
+`--vae-tile-size` accepts one size or `WIDTHxHEIGHT`. A zero dimension uses the
+256-pixel default. Sizes are rounded down to a multiple of the VAE scale factor
+and capped at the current input dimensions. Explicit sizes below four latent
+pixels per axis (or the full axis when it is smaller) are rejected. Encoding and
+decoding use the same spatial sizes, without an additional encoding multiplier.
+Inputs that fit within a tile are processed as one tile.
+
+For a 512x512 image with the default 50% overlap, both encoding and decoding use
+3x3 tiles. A 256-pixel tile corresponds to 32 latent pixels for an 8x VAE, 16 for
+a 16x VAE, and 8 for a 32x VAE. Smaller tiles reduce each graph's memory demand,
+but overlapping work can increase processing time and tiling can affect image
+quality, especially during encoding. Use larger tiles when more context is needed.
+
+`--vae-relative-tile-size` overrides the absolute size on each axis with a positive
+value. Values up to and including 1 specify a fraction of the current input size;
+values greater than 1 specify a target number of tiles per axis, accounting for
+overlap. For example, `0.5x0.5` uses half the width and height in both encode and
+decode. The target overlap is clamped to 0 through 0.5 and the actual overlap is
+adjusted to fit the image. Size and overlap options require `--vae-tiling`.
+
+**Migration:** `--vae-tile-size` and the C/JSON fields `tile_size_w` and
+`tile_size_h` now use image pixels instead of latent units. The C/JSON fields
+`tile_size_x/y` have been renamed to `tile_size_w/h`, and `rel_size_x/y` to
+`rel_size_w/h`. The command-line option names are unchanged. For example, an old
+decode tile size of 32 corresponds to 256 pixels for an 8x VAE or 512 pixels for a
+16x VAE. Encoding no longer enlarges explicit or relative tile sizes.
+
+The main VAE decode path retries allocation failures with smaller tiles, even
+without `--vae-tiling`. Supported video VAEs first try temporal tiling; spatial
+retries use at most 256-pixel tiles initially and then halve the effective tile
+dimensions down to the minimum size. Each spatial retry must reduce the effective
+tile size. These runtime adjustments do not change the caller's parameters.
+Execution failures are not retried, and encoding has no automatic OOM retry.
+
+`--temporal-tiling` remains independent of spatial tiling. MiniMax H3 always uses
+spatial tiling (256x256 pixels and 25% overlap by default) and its own temporal
+windows. With `--vae-tiling`, its overlap follows `--vae-tile-overlap`; explicit
+spatial sizes are honored.
+
 ## Offload weights to the CPU to save VRAM without reducing generation speed.
 
 Using `--offload-to-cpu` allows you to offload weights to the CPU, saving VRAM without reducing generation speed.
@@ -63,6 +111,12 @@ See [backend selection](./backend.md) for full syntax.
 
 When a graph has cut markers and its missing weights plus incremental compute workspace exceed the available device headroom, it runs its fixed segment list in order. A reusable monolithic compute buffer is not counted as a new allocation. An explicit `--max-vram` budget deducts already-resident managed weights and compute/cache buffers registered by every runner sharing the device, so later graph runs remain segmented when the full graph exceeds the budget. The current segment's weights are pinned during compute, and the next parameter-bearing segment is prefetched when the device supports asynchronous transfer. No opt-in streaming flag is required.
 
+When choosing between monolithic and segmented execution, the runner requires
+an additional 128 MiB of headroom in both available device memory and any explicit
+managed budget. This planning headroom absorbs small allocation estimate changes;
+subsequent capacity checks can consume it while still preserving the 512 MiB device
+scratch reserve and respecting the managed budget.
+
 - `--max-vram <GiB>` optionally lowers the live-memory limit. A positive value is a managed per-device budget, `0` uses the device's current free memory without an explicit budget, and a negative value snapshots free memory at startup while reserving that many GiB (`--max-vram -1` reserves about 1 GiB). Driver contexts and unrelated external allocations remain outside the managed budget.
 - `--disable-prefetch` disables asynchronous next-segment prefetch while retaining synchronous loading, eviction, and segmented execution.
 - `--disable-segmented-compute` forces monolithic graph execution for diagnostics or compatibility, even when the automatic memory check would select segments.

docs/pixart.md

@@ -0,0 +1,46 @@
+# How to Use
+
+You can run PixArt-α / PixArt-Σ with stable-diffusion.cpp.
+
+PixArt is a DiT-based text-to-image model family conditioned by a T5-XXL text
+encoder and a 4-channel VAE: SDXL-style for PixArt-Σ and SD1.x-style for PixArt-α.
+
+## Download weights
+
+- Download the transformer (diffusion model)
+    - PixArt-Σ XL-2 1024-MS: https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/transformer
+    - PixArt-α XL-2 1024-MS: https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS/tree/main/transformer
+- Download the T5-XXL text encoder
+    - safetensors: https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/text_encoder
+- Download the VAE
+    - PixArt-Σ: https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/vae
+    - PixArt-α: https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS/tree/main/vae
+    - Use the VAE matching the checkpoint's latent space. For TAE decoding or
+      preview, use TAESDXL for PixArt-Σ and TAESD for PixArt-α.
+- Tokenizer: the T5 vocabulary is embedded; no extra tokenizer file is needed.
+
+## Examples
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\pixart_sigma_xl2_1024_ms.safetensors --t5xxl ..\models\text_encoders\t5xxl.safetensors --vae ..\models\vae\pixart_vae.safetensors -p "a lovely cat" --cfg-scale 4.5 -W 1024 -H 1024 --steps 20 -v
+```
+
+## Notes
+
+- The VAE scaling factor defaults to `0.13025` for PixArt-Σ. PixArt-α
+  checkpoints with resolution micro-condition weights use `0.18215`.
+  PixArt-α 512 has the same tensor layout as PixArt-Σ, so it requires an
+  explicit override: `--model-args "pixart_vae_scale_factor=0.18215"`.
+  This argument can also override the scale for other compatible checkpoints.
+- PixArt-Σ checkpoints compute 2D sincos positional embeddings at runtime;
+  the trained grid is 64x64 patches with an interpolation scale of 2.
+  For checkpoints trained at a different resolution, the positional embedding
+  parameters can be adjusted via model args:
+  `--model-args "pixart_pos_embed_base_size=<trained grid>,pixart_interpolation_scale=<scale>"`
+  (e.g. `pixart_pos_embed_base_size=32,pixart_interpolation_scale=1,pixart_vae_scale_factor=0.18215` for
+  PixArt-α XL-2 512).
+- Checkpoints carrying resolution/aspect-ratio micro-condition weights are
+  detected but those conditions are not applied yet; a warning is logged and
+  generation proceeds with the timestep embedding only.
+- The transformer predicts 8 channels (noise + learned variance); only the
+  noise half is used for sampling, matching the reference implementation.

docs/qwen_image_2.1.md

@@ -23,7 +23,7 @@ Run the following commands from the build directory. Use image dimensions divisi
 ### Text to image
 
 ```powershell
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\qwen_image_2.1_int8_convrot.safetensors --vae ..\models\vae\qwen_image_2.1_vae_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf -p "a lovely cat holding a sign says 'qwen2.1.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu -o qwen_image_2.1.png
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\qwen_image_2.1_int8_convrot.safetensors --vae ..\models\vae\qwen_image_2.1_vae_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf -p "a lovely cat holding a sign says 'qwen2.1.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu --fa -o qwen_image_2.1.png
 ```
 
 <img alt="Qwen Image 2.1 example" src="../assets/qwen/qwen_image_2.1.png" />
@@ -35,7 +35,53 @@ To use GGUF diffusion weights, set `--diffusion-model` to the path of a file suc
 Pass the reference image with `-r` and describe the edit in `-p`. Vision weights are required; the example below loads them separately with `--llm_vision`.
 
 ```powershell
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\qwen_image_2.1_int8_convrot.safetensors --vae ..\models\vae\qwen_image_2.1_vae_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\Qwen3VL-8B-Instruct-mmproj-BF16.gguf -r ..\assets\qwen\qwen_image_2.1.png -p "change 'qwen2.1.cpp' to 'sd.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu -o qwen_image_2.1_edit.png
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\qwen_image_2.1_int8_convrot.safetensors --vae ..\models\vae\qwen_image_2.1_vae_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\Qwen3VL-8B-Instruct-mmproj-BF16.gguf -r ..\assets\qwen\qwen_image_2.1.png -p "change 'qwen2.1.cpp' to 'sd.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu --fa -o qwen_image_2.1_edit.png
 ```
 
 For multiple reference images, repeat `-r` in the desired order, for example `-r first.png -r second.png`.
+
+### Prefix cache
+
+By default, the first denoising call for each fixed condition saves the text and reference-image keys and values from every transformer layer. Later calls only compute the target-image tokens. Positive and negative conditions use separate caches, which are released when sampling ends.
+
+Set `qwen_image_2_1_prefix_cache_type` in `--model-args` to `auto` or a type name using the same parser and case-sensitive names as `--type`:
+
+- `auto` (default): use FP16 only when Flash Attention is enabled, Sage Attention is disabled, the attention scale is unchanged, and every attention operation in the cache-writing or cache-reading graph selects Flash Attention after backend support checks. If an operation falls back, rebuild the prefix in FP32 before executing and keep FP32 for the rest of that sampling run.
+- `f32`: always store FP32 keys and values.
+- `f16`: always store FP16 keys and values, including with ordinary attention or custom attention scaling. This saves cache memory but can introduce additional rounding error.
+- Other types, such as `bf16`, `q4_1`, `q5_0`, `q5_1`, `q8_0`, `q4_K`, `q6_K`, `iq4_nl`, and `iq4_xs`: use the requested storage type if the ggml build provides runtime conversion to and from FP32. Quantization is lossy and must be selected explicitly; `auto` never selects a quantized type.
+
+Cache data is packed into contiguous rows of `hidden_size` elements before conversion, so 256-element quantization blocks work with the model's 128-element attention heads without padding. The type's block size must divide `hidden_size`. Unknown types, types lacking runtime conversion (for example `q8_1` and several IQ formats), and incompatible block sizes are ignored with a warning, leaving the previous setting or the default `auto` unchanged.
+
+For example, use `--model-args qwen_image_2_1_prefix_cache_type=q8_0` to enable 8-bit cache storage. Cached keys and values are converted back to the attention input type before concatenating with the current target tokens. This reduces persistent cache memory; attention working buffers still use floating-point values, and conversion adds work on each step. Backends without the required conversion operations use the existing CPU fallback.
+
+For the default 32-layer model, a prefix of 4096 tokens takes approximately the following memory per condition, excluding weights, working buffers, and allocation overhead:
+
+| Cache type | Memory |
+| --- | ---: |
+| `f32` | 4 GiB |
+| `f16` | 2 GiB |
+| `q8_0` | 1.0625 GiB |
+| `q4_0` | 0.5625 GiB |
+
+The runner accounts for the cache when checking the memory budget. If a cached execution runs out of memory, it releases the prefix caches, disables caching for the rest of that sampling run, and retries the full sequence once. Per-step conditioning extensions currently use the full-sequence path.
+
+Disable this optimization with `--model-args qwen_image_2_1_prefix_cache=false`. It reuses step-independent activations; numerical results can still differ slightly because the matrix sizes change.
+
+### Alpha channel
+
+This model supports alpha channel output. As the model determines whether to output a regular image or with transparency through the prompt, according to [official recommendation](https://github.com/QwenLM/Qwen-Image-2.1#transparent-image-generation-rgba), use the following prompt format for better results:
+
+> `This is an RGBA image with transparency. <your description>. The image has alpha channel and the background is transparent.`
+
+Since transparency is decided by the prompt rather than by the input or an explicit switch, the same format applies equally to editing, whether or not the reference image itself has an alpha channel. Note that alpha is kept only in `.png` and `.webp` outputs; saving as `.jpg` drops the transparency.
+
+Here are some examples ran with Q6_K quantization:
+| Input | Prompt | Output |
+| --- | --- | --- |
+| ![Qwen Image 2.1 alpha input example 1](../assets/qwen/qwen-image-2.1-alpha-in1.png) | This is an RGBA image with transparency. Replace the text "BLOOM" with "Qwen Image 2.1", keeping the same font of the original text. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 1](../assets/qwen/qwen-image-2.1-alpha-out1.png) |
+| ![Qwen Image 2.1 alpha input example 2](../assets/logo.png) | This is an RGBA image with transparency. Remove the background of the image, keeping only the text and cat. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 2](../assets/qwen/qwen-image-2.1-alpha-out2.png) |
+
+### Other features
+
+Other features of the model could be found on the [model card from QwenLM/Qwen-Image-2.1 repo](https://github.com/QwenLM/Qwen-Image-2.1), including 2 finetuned prompt rewriting Qwen3.5-9B model.

docs/wan.md

@@ -40,6 +40,9 @@ Wan models require `-M vid_gen`, including single-frame generation. `--video-fra
             - safetensors: https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/tree/main/split_files/diffusion_models
             - gguf: https://huggingface.co/QuantStack/Wan2.2-S2V-14B-GGUF/tree/main
             - int8_convrot safetensors: https://huggingface.co/noctrex/Wan2.2-S2V-14B-int8_convrot
+        - Wan2.2 VACE-Fun A14B
+            - safetensors: https://huggingface.co/alibaba-pai/Wan2.2-VACE-Fun-A14B
+            - gguf: https://huggingface.co/QuantStack/Wan2.2-VACE-Fun-A14B-GGUF/tree/main
 - Download vae
     - wan_2.1_vae (for all the wan model except Wan2.2 TI2V 5B)
         - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/vae/wan_2.1_vae.safetensors
@@ -256,3 +259,25 @@ ffmpeg -i ..\..\ComfyUI\input\post+depth.mp4 -qscale:v 1 -vf fps=8 post+depth\fr
 ```
 
 <video src=../assets/wan/Wan2.1_14B_vace_v2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
+
+### Wan2.2 VACE-Fun A14B
+
+VACE-Fun runs as a MoE pair: `--diffusion-model` takes the low-noise expert and
+`--high-noise-diffusion-model` the high-noise one. Reference-to-video uses `-i`
+for the reference image, same as Wan2.1 VACE.
+
+#### R2V
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.2-VACE-Fun-A14B-low-noise-Q8_0.gguf --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-VACE-Fun-A14B-high-noise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu
+```
+
+<video src=../assets/wan/Wan2.2_A14B_vace_r2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
+
+#### T2V
+
+Same command without `-i` (VACE context is synthesized from an empty control
+video, like Wan2.1 VACE t2v).
+
+> On GPUs with ~12 GB VRAM, VACE also needs `--vae-tiling` — the control-video
+> encode can exceed the budget otherwise.

examples/cli/README.md

@@ -14,6 +14,12 @@ equivalent to `--log-level verbose`. If repeated, the last logging option wins.
 For direct image repair or automatic post-generation YOLOv8 detection followed by cropped inpainting, see
 [ADetailer](../../docs/adetailer.md).
 
+Use repeatable `--image-preprocess` rules to select resizing, cropping, padding,
+and resampling separately for each image input. Add `canny=true` to any input
+rule for edge detection. See
+[Image preprocessing](../../docs/image_preprocessing.md) for input selectors,
+input defaults, downstream model processing, mask alignment, and examples.
+
 Metadata mode inspects PNG/JPEG container metadata without loading any model:
 
 ```bash

examples/cli/main.cpp

@@ -41,7 +41,6 @@ struct SDCliParams {
     std::string metadata_format = "text";
 
     sd_log_level_t log_level = SD_LOG_INFO;
-    bool canny_preprocess    = false;
     bool convert_name        = false;
 
     preview_t preview_method = PREVIEW_NONE;
@@ -107,10 +106,6 @@ struct SDCliParams {
         };
 
         options.bool_options = {
-            {"",
-             "--canny",
-             "apply canny preprocessor (edge detection)",
-             true, &canny_preprocess},
             {"",
              "--convert-name",
              "convert tensor name (for convert mode)",
@@ -268,7 +263,6 @@ struct SDCliParams {
             << "  metadata_format: \"" << metadata_format << "\",\n"
             << "  log_level: " << log_level_name(log_level) << ",\n"
             << "  color: " << (color ? "true" : "false") << ",\n"
-            << "  canny_preprocess: " << (canny_preprocess ? "true" : "false") << ",\n"
             << "  convert_name: " << (convert_name ? "true" : "false") << ",\n"
             << "  preview_method: " << previews_str[preview_method] << ",\n"
             << "  preview_interval: " << preview_interval << ",\n"
@@ -328,9 +322,7 @@ void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
 
 bool load_images_from_dir(const std::string dir,
                           std::vector<SDImageOwner>& images,
-                          int expected_width  = 0,
-                          int expected_height = 0,
-                          int max_image_num   = 0) {
+                          int max_image_num = 0) {
     if (!fs::exists(dir) || !fs::is_directory(dir)) {
         LOG_ERROR("'%s' is not a valid directory\n", dir.c_str());
         return false;
@@ -357,7 +349,8 @@ bool load_images_from_dir(const std::string dir,
             LOG_VERBOSE("load image %zu from '%s'", images.size(), path.c_str());
             int width             = 0;
             int height            = 0;
-            uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, expected_width, expected_height);
+            int loaded_channel    = 0;
+            uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, loaded_channel, 0, 0);
             if (image_buffer == nullptr) {
                 LOG_ERROR("load image from '%s' failed", path.c_str());
                 return false;
@@ -365,7 +358,7 @@ bool load_images_from_dir(const std::string dir,
 
             images.emplace_back(sd_image_t{(uint32_t)width,
                                            (uint32_t)height,
-                                           3,
+                                           (uint32_t)loaded_channel,
                                            image_buffer});
 
             if (max_image_num > 0 && static_cast<int>(images.size()) >= max_image_num) {
@@ -651,10 +644,11 @@ int main(int argc, const char* argv[]) {
 
     SDCliParams cli_params;
     SDContextParams ctx_params;
+    ctx_params.conditioning_cache_size = 0;
     SDGenerationParams gen_params;
 
-    parse_args(argc, argv, cli_params, ctx_params, gen_params);
     sd_set_log_callback(sd_log_cb, (void*)&cli_params);
+    parse_args(argc, argv, cli_params, ctx_params, gen_params);
 
     if (cli_params.mode == METADATA) {
         MetadataReadOptions options;
@@ -750,16 +744,8 @@ int main(int argc, const char* argv[]) {
 
     auto load_image_and_update_size = [&](const std::string& path,
                                           SDImageOwner& image,
-                                          bool resize_image    = true,
                                           int expected_channel = 3) -> bool {
-        int expected_width  = 0;
-        int expected_height = 0;
-        if (resize_image && gen_params.width_and_height_are_set()) {
-            expected_width  = gen_params.width;
-            expected_height = gen_params.height;
-        }
-
-        if (!load_sd_image_from_file(image.put(), path.c_str(), expected_width, expected_height, expected_channel)) {
+        if (!load_sd_image_from_file(image.put(), path.c_str(), 0, 0, expected_channel)) {
             LOG_ERROR("load image from '%s' failed", path.c_str());
             return false;
         }
@@ -781,7 +767,8 @@ int main(int argc, const char* argv[]) {
     };
 
     if (gen_params.init_image_path.size() > 0) {
-        if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image)) {
+        const bool native_init = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER;
+        if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image, native_init ? 0 : 3)) {
             return 1;
         }
     }
@@ -795,8 +782,8 @@ int main(int argc, const char* argv[]) {
     if (gen_params.ref_image_paths.size() > 0) {
         gen_params.ref_images.clear();
         for (auto& path : gen_params.ref_image_paths) {
-            SDImageOwner ref_image({0, 0, 3, nullptr});
-            if (!load_image_and_update_size(path, ref_image, false)) {
+            SDImageOwner ref_image({0, 0, 0, nullptr});
+            if (!load_image_and_update_size(path, ref_image, 0)) {
                 return 1;
             }
             gen_params.ref_images.push_back(std::move(ref_image));
@@ -837,41 +824,22 @@ int main(int argc, const char* argv[]) {
     if (gen_params.mask_image_path.size() > 0) {
         if (!load_sd_image_from_file(gen_params.mask_image.put(),
                                      gen_params.mask_image_path.c_str(),
-                                     gen_params.get_resolved_width(),
-                                     gen_params.get_resolved_height(),
+                                     0,
+                                     0,
                                      1)) {
             LOG_ERROR("load image from '%s' failed", gen_params.mask_image_path.c_str());
             return 1;
         }
-    } else {
-        sd_image_t generated_mask = {0, 0, 1, nullptr};
-        generated_mask.data       = (uint8_t*)malloc(gen_params.get_resolved_width() * gen_params.get_resolved_height());
-        if (generated_mask.data == nullptr) {
-            LOG_ERROR("malloc mask image failed");
-            return 1;
-        }
-        generated_mask.width  = gen_params.get_resolved_width();
-        generated_mask.height = gen_params.get_resolved_height();
-        memset(generated_mask.data, 255, gen_params.get_resolved_width() * gen_params.get_resolved_height());
-        gen_params.mask_image.reset(generated_mask);
     }
 
     if (gen_params.control_image_path.size() > 0) {
         if (!load_sd_image_from_file(gen_params.control_image.put(),
                                      gen_params.control_image_path.c_str(),
-                                     gen_params.get_resolved_width(),
-                                     gen_params.get_resolved_height())) {
+                                     0,
+                                     0)) {
             LOG_ERROR("load image from '%s' failed", gen_params.control_image_path.c_str());
             return 1;
         }
-        if (cli_params.canny_preprocess) {  // apply preprocessor
-            preprocess_canny(gen_params.control_image.get(),
-                             0.08f,
-                             0.08f,
-                             0.8f,
-                             1.0f,
-                             false);
-        }
     }
 
     if (gen_params.ip_adapter_image_path.size() > 0) {
@@ -888,8 +856,6 @@ int main(int argc, const char* argv[]) {
         gen_params.control_frames.clear();
         if (!load_images_from_dir(gen_params.control_video_path,
                                   gen_params.control_frames,
-                                  gen_params.get_resolved_width(),
-                                  gen_params.get_resolved_height(),
                                   gen_params.video_frames)) {
             return 1;
         }
@@ -898,10 +864,7 @@ int main(int argc, const char* argv[]) {
     if (!gen_params.pm_id_images_dir.empty()) {
         gen_params.pm_id_images.clear();
         if (!load_images_from_dir(gen_params.pm_id_images_dir,
-                                  gen_params.pm_id_images,
-                                  0,
-                                  0,
-                                  0)) {
+                                  gen_params.pm_id_images)) {
             return 1;
         }
     }

examples/common/common.cpp

@@ -518,7 +518,8 @@ ArgOptions SDContextParams::get_options() {
         {"",
          "--model-args",
          "extra model args, key=value list. Supports chroma_use_dit_mask, chroma_use_t5_mask, "
-         "chroma_t5_mask_pad, qwen_image_zero_cond_t",
+         "chroma_t5_mask_pad, qwen_image_zero_cond_t, qwen_image_2_1_prefix_cache, "
+         "qwen_image_2_1_prefix_cache_type (auto or a type name from --type)",
          (int)',',
          &model_args},
         {"",
@@ -571,6 +572,10 @@ ArgOptions SDContextParams::get_options() {
          "number of threads to use during computation (default: -1). "
          "If threads <= 0, then threads will be set to the number of CPU physical cores",
          &n_threads},
+        {"",
+         "--conditioning-cache-size",
+         "maximum number of conditioning results cached per model context (default: " + std::to_string(conditioning_cache_size) + ", 0 disables caching)",
+         &conditioning_cache_size},
     };
 
     options.bool_options = {
@@ -822,6 +827,10 @@ bool SDContextParams::resolve(SDMode mode) {
 }
 
 bool SDContextParams::validate(SDMode mode) {
+    if (conditioning_cache_size < 0) {
+        LOG_ERROR("error: conditioning-cache-size must be non-negative");
+        return false;
+    }
     if (mode == CONVERT) {
         const bool has_convert_input = model_path.length() != 0 ||
                                        clip_l_path.length() != 0 ||
@@ -898,6 +907,7 @@ std::string SDContextParams::to_string() const {
     std::ostringstream oss;
     oss << "SDContextParams {\n"
         << "  n_threads: " << n_threads << ",\n"
+        << "  conditioning_cache_size: " << conditioning_cache_size << ",\n"
         << "  model_path: \"" << model_path << "\",\n"
         << "  clip_l_path: \"" << clip_l_path << "\",\n"
         << "  clip_g_path: \"" << clip_g_path << "\",\n"
@@ -992,6 +1002,7 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) {
     sd_ctx_params.pulid_weights_path              = pulid_weights_path.c_str();
     sd_ctx_params.tensor_type_rules               = tensor_type_rules.c_str();
     sd_ctx_params.n_threads                       = n_threads;
+    sd_ctx_params.conditioning_cache_size         = conditioning_cache_size;
     sd_ctx_params.wtype                           = wtype;
     sd_ctx_params.rng_type                        = rng_type;
     sd_ctx_params.sampler_rng_type                = sampler_rng_type;
@@ -1128,6 +1139,9 @@ ArgOptions SDGenerationParams::get_options() {
          "Key-value list to set up the way the reference images are processed (empty = auto-detect from model weigths)",
          (int)',',
          &ref_image_args},
+        {"", "--image-preprocess",
+         "Image preprocessing rule: target=init|end|mask|control|ref|ip-adapter|id|control-frame,index=N,mode=auto|none|stretch|crop|crop-resize|fit-pad,filter=auto|nearest|nearest-exact|bilinear|bicubic|lanczos,antialias=auto|true|false,width=W,height=H,anchor=center|top|bottom|left|right,pad_color=#RRGGBB[AA],canny=true|false. Repeat for multiple rules.",
+         (int)';', &image_preprocess},
     };
 
     options.int_options = {
@@ -1308,11 +1322,6 @@ ArgOptions SDGenerationParams::get_options() {
          "automatically increase the indices of references images based on the order they are listed (starting with 1).",
          true,
          &increase_ref_index},
-        {"",
-         "--disable-auto-resize-ref-image",
-         "disable auto resize of ref images",
-         false,
-         &auto_resize_ref_image},
         {"",
          "--circular",
          "enable circular padding on both axes for tileable output",
@@ -1332,7 +1341,7 @@ ArgOptions SDGenerationParams::get_options() {
          &embed_image_metadata},
         {"",
          "--vae-tiling",
-         "process vae in tiles to reduce memory usage",
+         "process vae encode and decode in spatial tiles to reduce memory usage (default: 256x256 image pixels)",
          true,
          &vae_tiling_params.enabled},
         {"",
@@ -1596,12 +1605,12 @@ ArgOptions SDGenerationParams::get_options() {
         size_t x_pos              = tile_size_str.find('x');
         try {
             if (x_pos != std::string::npos) {
-                std::string tile_x_str        = tile_size_str.substr(0, x_pos);
-                std::string tile_y_str        = tile_size_str.substr(x_pos + 1);
-                vae_tiling_params.tile_size_x = std::stoi(tile_x_str);
-                vae_tiling_params.tile_size_y = std::stoi(tile_y_str);
+                std::string tile_w_str        = tile_size_str.substr(0, x_pos);
+                std::string tile_h_str        = tile_size_str.substr(x_pos + 1);
+                vae_tiling_params.tile_size_w = std::stoi(tile_w_str);
+                vae_tiling_params.tile_size_h = std::stoi(tile_h_str);
             } else {
-                vae_tiling_params.tile_size_x = vae_tiling_params.tile_size_y = std::stoi(tile_size_str);
+                vae_tiling_params.tile_size_w = vae_tiling_params.tile_size_h = std::stoi(tile_size_str);
             }
         } catch (const std::invalid_argument&) {
             return -1;
@@ -1619,12 +1628,12 @@ ArgOptions SDGenerationParams::get_options() {
         size_t x_pos             = rel_size_str.find('x');
         try {
             if (x_pos != std::string::npos) {
-                std::string rel_x_str        = rel_size_str.substr(0, x_pos);
-                std::string rel_y_str        = rel_size_str.substr(x_pos + 1);
-                vae_tiling_params.rel_size_x = std::stof(rel_x_str);
-                vae_tiling_params.rel_size_y = std::stof(rel_y_str);
+                std::string rel_w_str        = rel_size_str.substr(0, x_pos);
+                std::string rel_h_str        = rel_size_str.substr(x_pos + 1);
+                vae_tiling_params.rel_size_w = std::stof(rel_w_str);
+                vae_tiling_params.rel_size_h = std::stof(rel_h_str);
             } else {
-                vae_tiling_params.rel_size_x = vae_tiling_params.rel_size_y = std::stof(rel_size_str);
+                vae_tiling_params.rel_size_w = vae_tiling_params.rel_size_h = std::stof(rel_size_str);
             }
         } catch (const std::invalid_argument&) {
             return -1;
@@ -1754,11 +1763,11 @@ ArgOptions SDGenerationParams::get_options() {
          on_scm_policy_arg},
         {"",
          "--vae-tile-size",
-         "tile size for vae tiling, format [X]x[Y] (default: 32x32)",
+         "tile size for vae encode and decode in image pixels, format [W]x[H] or [S] (default: 256x256; requires --vae-tiling)",
          on_tile_size_arg},
         {"",
          "--vae-relative-tile-size",
-         "relative tile size for vae tiling, format [X]x[Y], in fraction of image size if < 1, in number of tiles per dim if >=1 (overrides --vae-tile-size)",
+         "relative tile size for vae encode and decode, format [W]x[H] or [S]: <=1 is a dimension fraction, >1 a target tile count (overrides --vae-tile-size; requires --vae-tiling)",
          on_relative_tile_size_arg},
         {"",
          "--prompt-file",
@@ -1848,28 +1857,28 @@ bool decode_base64_image(const std::string& encoded_input,
         return false;
     }
 
-    int decoded_width  = 0;
-    int decoded_height = 0;
-    uint8_t* raw_data  = load_image_from_memory(reinterpret_cast<const char*>(image_bytes.data()),
-                                                static_cast<int>(image_bytes.size()),
-                                                decoded_width,
-                                                decoded_height,
-                                                expected_width,
-                                                expected_height,
-                                                target_channels);
+    int decoded_width    = 0;
+    int decoded_height   = 0;
+    int resolved_channel = target_channels;
+    uint8_t* raw_data    = load_image_from_memory(reinterpret_cast<const char*>(image_bytes.data()),
+                                                  static_cast<int>(image_bytes.size()),
+                                                  decoded_width,
+                                                  decoded_height,
+                                                  resolved_channel,
+                                                  expected_width,
+                                                  expected_height,
+                                                  target_channels);
     if (raw_data == nullptr) {
         return false;
     }
 
-    out_image.reset({(uint32_t)decoded_width, (uint32_t)decoded_height, (uint32_t)target_channels, raw_data});
+    out_image.reset({(uint32_t)decoded_width, (uint32_t)decoded_height, (uint32_t)resolved_channel, raw_data});
     return true;
 }
 
 static bool parse_image_json_field(const json& parent,
                                    const char* key,
                                    int channels,
-                                   int expected_width,
-                                   int expected_height,
                                    SDImageOwner& out_image) {
     if (!parent.contains(key)) {
         return true;
@@ -1881,14 +1890,12 @@ static bool parse_image_json_field(const json& parent,
     if (!parent.at(key).is_string()) {
         return false;
     }
-    return decode_base64_image(parent.at(key).get<std::string>(), channels, expected_width, expected_height, out_image);
+    return decode_base64_image(parent.at(key).get<std::string>(), channels, 0, 0, out_image);
 }
 
 static bool parse_image_array_json_field(const json& parent,
                                          const char* key,
                                          int channels,
-                                         int expected_width,
-                                         int expected_height,
                                          std::vector<SDImageOwner>& out_images) {
     if (!parent.contains(key)) {
         return true;
@@ -1907,7 +1914,7 @@ static bool parse_image_array_json_field(const json& parent,
             return false;
         }
         SDImageOwner image;
-        if (!decode_base64_image(item.get<std::string>(), channels, expected_width, expected_height, image)) {
+        if (!decode_base64_image(item.get<std::string>(), channels, 0, 0, image)) {
             return false;
         }
         out_images.push_back(std::move(image));
@@ -2006,6 +2013,29 @@ static bool resolve_model_file_from_dir(const std::string& model_name,
     return false;
 }
 
+bool SDGenerationParams::parse_image_preprocess_json(const std::string& json_str) {
+    const auto value = json::parse(json_str, nullptr, false);
+    std::string rules;
+    if (value.is_string()) {
+        rules = value.get<std::string>();
+    } else if (value.is_array()) {
+        for (const auto& item : value) {
+            if (!item.is_string()) {
+                LOG_ERROR("image_preprocess must contain rule strings");
+                return false;
+            }
+            if (!rules.empty())
+                rules += ";";
+            rules += item.get<std::string>();
+        }
+    } else {
+        LOG_ERROR("image_preprocess must be a string or array of strings");
+        return false;
+    }
+    image_preprocess = std::move(rules);
+    return true;
+}
+
 bool SDGenerationParams::from_json_str(
     const std::string& json_str,
     const std::function<std::string(const std::string&)>& lora_path_resolver) {
@@ -2017,6 +2047,9 @@ bool SDGenerationParams::from_json_str(
         return false;
     }
 
+    if (j.contains("image_preprocess") && !parse_image_preprocess_json(j["image_preprocess"].dump()))
+        return false;
+
     auto load_if_exists = [&](const char* key, auto& out) {
         if (j.contains(key)) {
             using T = std::decay_t<decltype(out)>;
@@ -2054,6 +2087,7 @@ bool SDGenerationParams::from_json_str(
     load_if_exists("cache_mode", cache_mode);
     load_if_exists("cache_option", cache_option);
     load_if_exists("scm_mask", scm_mask);
+    load_if_exists("ref_image_args", ref_image_args);
 
     load_if_exists("clip_skip", clip_skip);
     load_if_exists("width", width);
@@ -2071,7 +2105,6 @@ bool SDGenerationParams::from_json_str(
     load_if_exists("moe_boundary", moe_boundary);
     load_if_exists("vace_strength", vace_strength);
 
-    load_if_exists("auto_resize_ref_image", auto_resize_ref_image);
     load_if_exists("increase_ref_index", increase_ref_index);
     load_if_exists("embed_image_metadata", embed_image_metadata);
 
@@ -2191,20 +2224,20 @@ bool SDGenerationParams::from_json_str(
         if (tiling_json.contains("temporal_tiling") && tiling_json["temporal_tiling"].is_boolean()) {
             vae_tiling_params.temporal_tiling = tiling_json["temporal_tiling"];
         }
-        if (tiling_json.contains("tile_size_x") && tiling_json["tile_size_x"].is_number_integer()) {
-            vae_tiling_params.tile_size_x = tiling_json["tile_size_x"];
+        if (tiling_json.contains("tile_size_w") && tiling_json["tile_size_w"].is_number_integer()) {
+            vae_tiling_params.tile_size_w = tiling_json["tile_size_w"];
         }
-        if (tiling_json.contains("tile_size_y") && tiling_json["tile_size_y"].is_number_integer()) {
-            vae_tiling_params.tile_size_y = tiling_json["tile_size_y"];
+        if (tiling_json.contains("tile_size_h") && tiling_json["tile_size_h"].is_number_integer()) {
+            vae_tiling_params.tile_size_h = tiling_json["tile_size_h"];
         }
         if (tiling_json.contains("target_overlap") && tiling_json["target_overlap"].is_number()) {
             vae_tiling_params.target_overlap = tiling_json["target_overlap"];
         }
-        if (tiling_json.contains("rel_size_x") && tiling_json["rel_size_x"].is_number()) {
-            vae_tiling_params.rel_size_x = tiling_json["rel_size_x"];
+        if (tiling_json.contains("rel_size_w") && tiling_json["rel_size_w"].is_number()) {
+            vae_tiling_params.rel_size_w = tiling_json["rel_size_w"];
         }
-        if (tiling_json.contains("rel_size_y") && tiling_json["rel_size_y"].is_number()) {
-            vae_tiling_params.rel_size_y = tiling_json["rel_size_y"];
+        if (tiling_json.contains("rel_size_h") && tiling_json["rel_size_h"].is_number()) {
+            vae_tiling_params.rel_size_h = tiling_json["rel_size_h"];
         }
         if (tiling_json.contains("extra_tiling_args") && tiling_json["extra_tiling_args"].is_string()) {
             extra_tiling_args = tiling_json["extra_tiling_args"].get<std::string>();
@@ -2215,32 +2248,23 @@ bool SDGenerationParams::from_json_str(
         LOG_ERROR("invalid lora");
         return false;
     }
-    if (!parse_image_json_field(j, "init_image", 3, width, height, init_image)) {
-        LOG_ERROR("invalid init_image");
+    auto load_image = [&](const char* key, int channels, SDImageOwner& image) {
+        if (!parse_image_json_field(j, key, channels, image)) {
+            LOG_ERROR("invalid %s", key);
+            return false;
+        }
+        return true;
+    };
+    if (!load_image("init_image", 0, init_image) ||
+        !load_image("end_image", 3, end_image) ||
+        !load_image("mask_image", 1, mask_image) ||
+        !load_image("control_image", 3, control_image) ||
+        !load_image("ip_adapter_image", 3, ip_adapter_image)) {
         return false;
     }
-    if (!parse_image_json_field(j, "end_image", 3, width, height, end_image)) {
-        LOG_ERROR("invalid end_image");
-        return false;
-    }
-    if (!parse_image_array_json_field(j, "ref_images", 3, width, height, ref_images)) {
-        LOG_ERROR("invalid ref_images");
-        return false;
-    }
-    if (!parse_image_array_json_field(j, "control_frames", 3, width, height, control_frames)) {
-        LOG_ERROR("invalid control_frames");
-        return false;
-    }
-    if (!parse_image_json_field(j, "mask_image", 1, width, height, mask_image)) {
-        LOG_ERROR("invalid mask_image");
-        return false;
-    }
-    if (!parse_image_json_field(j, "control_image", 3, width, height, control_image)) {
-        LOG_ERROR("invalid control_image");
-        return false;
-    }
-    if (!parse_image_json_field(j, "ip_adapter_image", 3, width, height, ip_adapter_image)) {
-        LOG_ERROR("invalid ip_adapter_image");
+    if (!parse_image_array_json_field(j, "ref_images", 0, ref_images) ||
+        !parse_image_array_json_field(j, "control_frames", 3, control_frames)) {
+        LOG_ERROR("invalid input image array");
         return false;
     }
 
@@ -2484,6 +2508,10 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s
 }
 
 bool SDGenerationParams::validate(SDMode mode) {
+    if (!image_preprocess.empty() && mode != IMG_GEN && mode != VID_GEN) {
+        LOG_ERROR("--image-preprocess requires img_gen or vid_gen mode");
+        return false;
+    }
     if (batch_count <= 0) {
         LOG_ERROR("error: batch_count must be greater than 0");
         return false;
@@ -2659,14 +2687,6 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() {
         pulid_id_weight,
     };
 
-    if (!auto_resize_ref_image) {
-        if (!ref_image_args.empty()) {
-            ref_image_args += ",";
-        }
-        ref_image_args += "resize_before_vae=0";
-        LOG_WARN("Notice: --disable-auto-resize-ref-image is deprecated. Use --ref-image-args \"resize_before_vae=off\" instead.");
-    }
-
     if (increase_ref_index) {
         if (!ref_image_args.empty()) {
             ref_image_args += ",";
@@ -2714,6 +2734,7 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() {
     params.hires.custom_sigmas_count = static_cast<int>(hires_custom_sigmas.size());
     params.circular_x                = circular || circular_x;
     params.circular_y                = circular || circular_y;
+    params.image_preprocess          = {image_preprocess.c_str()};
     return params;
 }
 
@@ -2816,6 +2837,7 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
     params.hires.custom_sigmas_count = static_cast<int>(hires_custom_sigmas.size());
     params.circular_x                = circular || circular_x;
     params.circular_y                = circular || circular_y;
+    params.image_preprocess          = {image_preprocess.c_str()};
     return params;
 }
 
@@ -2872,7 +2894,8 @@ std::string SDGenerationParams::to_string() const {
         << "  ref_video_audio_paths: " << vec_str_to_string(ref_video_audio_paths) << ",\n"
         << "  ref_audio_paths: " << vec_str_to_string(ref_audio_paths) << ",\n"
         << "  control_video_path: \"" << control_video_path << "\",\n"
-        << "  auto_resize_ref_image: " << (auto_resize_ref_image ? "true" : "false") << ",\n"
+        << "  image_preprocess: " << image_preprocess << ",\n"
+        << "  ref_image_args: " << ref_image_args << ",\n"
         << "  increase_ref_index: " << (increase_ref_index ? "true" : "false") << ",\n"
         << "  pm_id_images_dir: \"" << pm_id_images_dir << "\",\n"
         << "  pm_id_embed_path: \"" << pm_id_embed_path << "\",\n"
@@ -2911,11 +2934,11 @@ std::string SDGenerationParams::to_string() const {
         << "  vae_tiling_params: { "
         << vae_tiling_params.enabled << ", "
         << vae_tiling_params.temporal_tiling << ", "
-        << vae_tiling_params.tile_size_x << ", "
-        << vae_tiling_params.tile_size_y << ", "
+        << vae_tiling_params.tile_size_w << ", "
+        << vae_tiling_params.tile_size_h << ", "
         << vae_tiling_params.target_overlap << ", "
-        << vae_tiling_params.rel_size_x << ", "
-        << vae_tiling_params.rel_size_y << ", "
+        << vae_tiling_params.rel_size_w << ", "
+        << vae_tiling_params.rel_size_h << ", "
         << "\"" << extra_tiling_args << "\" },\n"
         << "}";
     return oss.str();
@@ -3023,12 +3046,13 @@ std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
     set_json_basename_if_not_empty(models, "control_net", ctx_params.control_net_path);
     root["models"] = std::move(models);
 
-    root["clip_skip"]             = gen_params.clip_skip;
-    root["strength"]              = gen_params.strength;
-    root["control_strength"]      = gen_params.control_strength;
-    root["ip_adapter_strength"]   = gen_params.ip_adapter_strength;
-    root["auto_resize_ref_image"] = gen_params.auto_resize_ref_image;
-    root["increase_ref_index"]    = gen_params.increase_ref_index;
+    root["clip_skip"]           = gen_params.clip_skip;
+    root["strength"]            = gen_params.strength;
+    root["control_strength"]    = gen_params.control_strength;
+    root["ip_adapter_strength"] = gen_params.ip_adapter_strength;
+    root["ref_image_args"]      = gen_params.ref_image_args;
+    root["image_preprocess"]    = gen_params.image_preprocess;
+    root["increase_ref_index"]  = gen_params.increase_ref_index;
     if (mode == VID_GEN) {
         root["video"] = {
             {"frame_count", gen_params.video_frames},
@@ -3116,11 +3140,11 @@ std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
         root["vae_tiling"] = {
             {"enabled", gen_params.vae_tiling_params.enabled},
             {"temporal_tiling", gen_params.vae_tiling_params.temporal_tiling},
-            {"tile_size_x", gen_params.vae_tiling_params.tile_size_x},
-            {"tile_size_y", gen_params.vae_tiling_params.tile_size_y},
+            {"tile_size_w", gen_params.vae_tiling_params.tile_size_w},
+            {"tile_size_h", gen_params.vae_tiling_params.tile_size_h},
             {"target_overlap", gen_params.vae_tiling_params.target_overlap},
-            {"rel_size_x", gen_params.vae_tiling_params.rel_size_x},
-            {"rel_size_y", gen_params.vae_tiling_params.rel_size_y},
+            {"rel_size_w", gen_params.vae_tiling_params.rel_size_w},
+            {"rel_size_h", gen_params.vae_tiling_params.rel_size_h},
             {"extra_tiling_args", gen_params.extra_tiling_args},
         };
     }

examples/common/common.h

@@ -116,7 +116,8 @@ bool decode_base64_image(const std::string& encoded_input,
                          SDImageOwner& out_image);
 
 struct SDContextParams {
-    int n_threads = -1;
+    int n_threads               = -1;
+    int conditioning_cache_size = 4;
     std::string model_path;
     std::string clip_l_path;
     std::string clip_g_path;
@@ -200,18 +201,17 @@ struct SDGenerationParams {
     std::string ad_prompt;
     std::string ad_negative_prompt;
     std::string extra_ad_args;
-    int clip_skip              = -1;  // <= 0 represents unspecified
-    int width                  = -1;
-    int height                 = -1;
-    int batch_count            = 1;
-    int qwen_image_layers      = 3;
-    int64_t seed               = 42;
-    float strength             = 0.75f;
-    float control_strength     = 0.9f;
-    float ip_adapter_strength  = 1.0f;
-    bool auto_resize_ref_image = true;
-    bool increase_ref_index    = false;
-    bool embed_image_metadata  = true;
+    int clip_skip             = -1;  // <= 0 represents unspecified
+    int width                 = -1;
+    int height                = -1;
+    int batch_count           = 1;
+    int qwen_image_layers     = 3;
+    int64_t seed              = 42;
+    float strength            = 0.75f;
+    float control_strength    = 0.9f;
+    float ip_adapter_strength = 1.0f;
+    bool increase_ref_index   = false;
+    bool embed_image_metadata = true;
 
     std::string init_image_path;
     std::string end_image_path;
@@ -247,6 +247,7 @@ struct SDGenerationParams {
     std::string extra_tiling_args;
 
     std::string ref_image_args;
+    std::string image_preprocess;
 
     std::string pm_id_images_dir;
     std::string pm_id_embed_path;
@@ -310,6 +311,7 @@ struct SDGenerationParams {
     ArgOptions get_options();
     bool from_json_str(const std::string& json_str,
                        const std::function<std::string(const std::string&)>& lora_path_resolver = {});
+    bool parse_image_preprocess_json(const std::string& json_str);
     bool initialize_cache_params();
     void extract_and_remove_lora(const std::string& lora_model_dir);
     bool width_and_height_are_set() const;

examples/common/media_io.cpp

@@ -261,6 +261,10 @@ uint8_t* decode_webp_image_to_buffer(const uint8_t* data,
     height               = features.height;
     source_channel_count = features.has_alpha ? 4 : 3;
 
+    if (expected_channel == 0) {
+        expected_channel = source_channel_count;
+    }
+
     const size_t pixel_count = static_cast<size_t>(width) * static_cast<size_t>(height);
 
     if (expected_channel == 1) {
@@ -481,7 +485,8 @@ uint8_t* load_image_common(bool from_memory,
                            int& height,
                            int expected_width,
                            int expected_height,
-                           int expected_channel) {
+                           int expected_channel,
+                           int& out_channel) {
     const char* image_path;
     FreeUniquePtr<uint8_t> image_buffer;
     int source_channel_count = 0;
@@ -538,6 +543,32 @@ uint8_t* load_image_common(bool from_memory,
         LOG_ERROR("load image from '%s' failed", image_path);
         return nullptr;
     }
+    if (expected_channel == 0) {
+        expected_channel = source_channel_count == 2 ? 4 : (source_channel_count == 1 ? 3 : source_channel_count);
+        if (expected_channel != source_channel_count) {
+            FreeUniquePtr<uint8_t> promoted((uint8_t*)malloc((size_t)width * height * expected_channel));
+            if (promoted == nullptr) {
+                LOG_ERROR("error: allocate memory for channel promotion, image_path = %s", image_path);
+                return nullptr;
+            }
+            const size_t pixel_count = (size_t)width * (size_t)height;
+            for (size_t i = 0; i < pixel_count; ++i) {
+                if (source_channel_count == 1) {
+                    promoted.get()[i * 3 + 0] = image_buffer.get()[i];
+                    promoted.get()[i * 3 + 1] = image_buffer.get()[i];
+                    promoted.get()[i * 3 + 2] = image_buffer.get()[i];
+                } else {
+                    promoted.get()[i * 4 + 0] = image_buffer.get()[i * 2];
+                    promoted.get()[i * 4 + 1] = image_buffer.get()[i * 2];
+                    promoted.get()[i * 4 + 2] = image_buffer.get()[i * 2];
+                    promoted.get()[i * 4 + 3] = image_buffer.get()[i * 2 + 1];
+                }
+            }
+            image_buffer         = std::move(promoted);
+            source_channel_count = expected_channel;
+        }
+    }
+    // stb reports the source channel count even when it converts the output.
     if (source_channel_count < expected_channel) {
         fprintf(stderr,
                 "the number of channels for the input image must be >= %d,"
@@ -597,7 +628,7 @@ uint8_t* load_image_common(bool from_memory,
         }
         stbir_resize(image_buffer.get(), width, height, 0,
                      resized_image_buffer.get(), expected_width, expected_height, 0, STBIR_TYPE_UINT8,
-                     expected_channel, STBIR_ALPHA_CHANNEL_NONE, 0,
+                     expected_channel, expected_channel == 4 ? 3 : STBIR_ALPHA_CHANNEL_NONE, 0,
                      STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP,
                      STBIR_FILTER_BOX, STBIR_FILTER_BOX,
                      STBIR_COLORSPACE_SRGB, nullptr);
@@ -605,6 +636,7 @@ uint8_t* load_image_common(bool from_memory,
         height       = expected_height;
         image_buffer = std::move(resized_image_buffer);
     }
+    out_channel = expected_channel;
     return image_buffer.release();
 }
 
@@ -777,10 +809,11 @@ bool write_image_to_file(const std::string& path,
 uint8_t* load_image_from_file(const char* image_path,
                               int& width,
                               int& height,
+                              int& out_channel,
                               int expected_width,
                               int expected_height,
                               int expected_channel) {
-    return load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel);
+    return load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel, out_channel);
 }
 
 bool load_sd_image_from_file(sd_image_t* image,
@@ -790,13 +823,14 @@ bool load_sd_image_from_file(sd_image_t* image,
                              int expected_channel) {
     int width;
     int height;
-    image->data = load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel);
+    int resolved_channel = expected_channel;
+    image->data          = load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel, resolved_channel);
     if (image->data == nullptr) {
         return false;
     }
     image->width   = width;
     image->height  = height;
-    image->channel = expected_channel;
+    image->channel = resolved_channel;
     return true;
 }
 
@@ -804,10 +838,11 @@ uint8_t* load_image_from_memory(const char* image_bytes,
                                 int len,
                                 int& width,
                                 int& height,
+                                int& out_channel,
                                 int expected_width,
                                 int expected_height,
                                 int expected_channel) {
-    return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel);
+    return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel, out_channel);
 }
 
 static void append_avi_metadata(std::vector<uint8_t>& data, const std::string& parameters) {

examples/common/media_io.h

@@ -32,9 +32,12 @@ bool write_image_to_file(const std::string& path,
                          const std::string& parameters = "",
                          int quality                   = 90);
 
+// expected_channel == 0 preserves native channels (grayscale -> RGB, gray+alpha -> RGBA).
+// out_channel receives the output channel count.
 uint8_t* load_image_from_file(const char* image_path,
                               int& width,
                               int& height,
+                              int& out_channel,
                               int expected_width   = 0,
                               int expected_height  = 0,
                               int expected_channel = 3);
@@ -49,6 +52,7 @@ uint8_t* load_image_from_memory(const char* image_bytes,
                                 int len,
                                 int& width,
                                 int& height,
+                                int& out_channel,
                                 int expected_width   = 0,
                                 int expected_height  = 0,
                                 int expected_channel = 3);

examples/server/CMakeLists.txt

@@ -21,21 +21,21 @@ if(SD_SERVER_BUILD_FRONTEND AND EXISTS "${FRONTEND_DIR}")
         set(HAVE_FRONTEND_BUILD ON)
 
         add_custom_target(${TARGET}_frontend_install
-            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" install
+            COMMAND "${PNPM_EXECUTABLE}" --ignore-workspace -C "${FRONTEND_DIR}" install
             WORKING_DIRECTORY "${FRONTEND_DIR}"
             COMMENT "Installing frontend dependencies"
             VERBATIM
         )
 
         add_custom_target(${TARGET}_frontend_build
-            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" run build
+            COMMAND "${PNPM_EXECUTABLE}" --ignore-workspace -C "${FRONTEND_DIR}" run build
             WORKING_DIRECTORY "${FRONTEND_DIR}"
             COMMENT "Building frontend"
             VERBATIM
         )
 
         add_custom_target(${TARGET}_frontend_header
-            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" run build:header
+            COMMAND "${PNPM_EXECUTABLE}" --ignore-workspace -C "${FRONTEND_DIR}" run build:header
             WORKING_DIRECTORY "${FRONTEND_DIR}"
             COMMENT "Generating gen_index_html.h"
             VERBATIM

examples/server/api.md

@@ -56,6 +56,7 @@ Current endpoints include:
 - `GET /sdcpp/v1/jobs/{id}`
 - `POST /sdcpp/v1/jobs/{id}/cancel`
 - `POST /sdcpp/v1/vid_gen`
+- `POST /sdcpp/v1/upscale`
 
 ## `sd_cpp_extra_args`
 
@@ -148,6 +149,19 @@ Native extension fields:
 
 - any `sdcpp API` fields embedded through `sd_cpp_extra_args` inside `prompt`
 
+Uploaded images are decoded at their original dimensions. The first decoded
+image establishes the generation dimensions if `size` is omitted. Input
+geometry follows `image_preprocess`: references preserve their dimensions by
+default, while init and mask use the generation canvas preset.
+
+Reference encoding then follows model presets and `ref_image_args`. To skip
+input geometry for references and disable resizing before VAE encoding, include
+this in `prompt`:
+
+```text
+edit this image <sd_cpp_extra_args>{"image_preprocess":"target=ref,mode=none","ref_image_args":"resize_before_vae=false"}</sd_cpp_extra_args>
+```
+
 Response fields:
 
 | Field | Type | Notes |
@@ -421,7 +435,8 @@ Top-level fields:
 | `samplers` | `array<string>` | Available sampling methods |
 | `schedulers` | `array<string>` | Available schedulers |
 | `loras` | `array<object>` | Available LoRA entries |
-| `upscalers` | `array<object>` | Available model-backed highres upscalers |
+| `upscalers` | `array<object>` | Available highres upscalers, built-in and model-backed |
+| `upscale` | `boolean` | Whether a compatible RGB ESRGAN model is available for `POST /sdcpp/v1/upscale` |
 | `limits` | `object` | Shared queue and size limits |
 
 `model`
@@ -463,6 +478,8 @@ Shared nested fields:
 | Field | Type | Notes |
 | --- | --- | --- |
 | `upscalers[].name` | `string` | Built-in name or model stem; use this value in `hires.upscaler` |
+| `upscalers[].model` | `boolean` | True for a model-backed upscaler, false for a built-in scaling filter |
+| `upscalers[].image_upscale` | `boolean` | Whether this model can be selected by `POST /sdcpp/v1/upscale`; false for latent upscalers and built-in filters |
 
 Built-in entries include `None`, `Lanczos`, `Nearest`, `Latent`, `Latent (nearest)`, `Latent (nearest-exact)`, `Latent (antialiased)`, `Latent (bicubic)`, and `Latent (bicubic antialiased)`. Model-backed entries are scanned from the top level of `--hires-upscalers-dir`; subdirectories are not scanned.
 
@@ -476,6 +493,8 @@ Built-in entries include `None`, `Lanczos`, `Nearest`, `Latent`, `Latent (neares
 | `limits.max_height` | `integer` |
 | `limits.max_batch_count` | `integer` |
 | `limits.max_queue_size` | `integer` |
+| `limits.max_upscale_width` | `integer` |
+| `limits.max_upscale_height` | `integer` |
 
 Shared default fields used by both `img_gen` and `vid_gen`:
 
@@ -505,11 +524,11 @@ Shared default fields used by both `img_gen` and `vid_gen`:
 | `vae_tiling_params` | `object` |
 | `vae_tiling_params.enabled` | `boolean` |
 | `vae_tiling_params.temporal_tiling` | `boolean` |
-| `vae_tiling_params.tile_size_x` | `integer` |
-| `vae_tiling_params.tile_size_y` | `integer` |
+| `vae_tiling_params.tile_size_w` | `integer` |
+| `vae_tiling_params.tile_size_h` | `integer` |
 | `vae_tiling_params.target_overlap` | `number` |
-| `vae_tiling_params.rel_size_x` | `number` |
-| `vae_tiling_params.rel_size_y` | `number` |
+| `vae_tiling_params.rel_size_w` | `number` |
+| `vae_tiling_params.rel_size_h` | `number` |
 | `vae_tiling_params.extra_tiling_args` | `string` |
 | `cache_mode` | `string` |
 | `cache_option` | `string` |
@@ -518,6 +537,8 @@ Shared default fields used by both `img_gen` and `vid_gen`:
 | `output_format` | `string` |
 | `output_compression` | `integer` |
 
+`vae_tiling_params.tile_size_w` and `tile_size_h` are in **image pixels**, with `0` selecting the 256-pixel default. Both encode and decode use these sizes without an encoding multiplier. Positive `rel_size_w`/`rel_size_h` values override the corresponding absolute size: values up to 1 are dimension fractions, and values greater than 1 are target tile counts. Set `enabled` to use spatial tiling. Sizes are aligned down to the VAE scale factor and capped at the input dimensions; explicit sizes below the minimum supported tile size are rejected. These fields previously used latent units; see [VAE tiling](../../docs/performance.md#use-vae-tiling-to-reduce-encode-and-decode-memory-usage) for migration and OOM retry behavior.
+
 `vae_tiling_params.extra_tiling_args` accepts a key=value list. Supported video VAEs accept `temporal_tile_frames` (alias `temporal_tile_size`, default `4`) and `temporal_tile_overlap` (default `1`).
 LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEHV use overlap blending. MiniMax H3 keeps its model-specific fixed temporal windows because its latent-to-frame mapping is non-linear.
 
@@ -526,7 +547,7 @@ LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEH
 | Field | Type |
 | --- | --- |
 | `batch_count` | `integer` |
-| `auto_resize_ref_image` | `boolean` |
+| `ref_image_args` | `string` |
 | `increase_ref_index` | `boolean` |
 | `control_strength` | `number` |
 | `ip_adapter_strength` | `number` |
@@ -628,6 +649,52 @@ Typical status codes:
 - `404 Not Found`
 - `410 Gone`
 
+#### `POST /sdcpp/v1/upscale`
+
+Runs one RGB ESRGAN upscaler over an image, with no generation involved. Latent upscaler models remain available for hires generation but cannot be used here.
+
+This is the HTTP equivalent of `sd-cli -M upscale`: no diffusion model, text
+encoder or sampling is used, so it is fast enough to answer synchronously and
+does not create a job.
+
+Request fields:
+
+| Field | Type | Notes |
+| --- | --- | --- |
+| `image` | `string` | Required. Base64 or data URL image |
+| `upscaler` | `string` | A name from `upscalers` with `image_upscale: true`; the first compatible entry when omitted |
+| `repeats` | `integer` | Run the upscaler this many times, 1 to 4 (default `1`) |
+| `tile_size` | `integer` | Tile size, defaulting to the server's `--upscale-tile-size` |
+| `output_format` | `string` | `png`, `jpeg`, or `webp` when built with WebP support (default `png`); unsupported formats return 400 |
+| `output_compression` | `integer` | Range is clamped to `0..100` |
+
+Response fields:
+
+| Field | Type | Notes |
+| --- | --- | --- |
+| `images` | `array<object>` | One image |
+| `images[].index` | `integer` | |
+| `images[].b64_json` | `string` | Base64-encoded image bytes |
+| `upscaler` | `string` | The upscaler actually used |
+| `scale` | `integer` | The model's scale factor |
+| `repeats` | `integer` | How many times it was run |
+| `width` | `integer` | Result width |
+| `height` | `integer` | Result height |
+| `output_format` | `string` | Final encoded image format |
+
+Typical status codes:
+
+- `200 OK`
+- `400 Bad Request` (invalid request, unsupported output format, unreadable image, incompatible upscaler, or output dimensions exceeding the limit)
+- `500 Internal Server Error`
+
+Notes:
+
+- Final output dimensions, including all repeats, must not exceed 8192 pixels on either axis (`limits.max_upscale_width` and `limits.max_upscale_height`). Requests exceeding this bound are rejected before upscaling.
+- The upscaler models are three-channel; alpha is not preserved.
+- The request holds the generation context lock, so an upscale and a
+  generation never run on the device at the same time.
+
 #### `POST /sdcpp/v1/jobs/{id}/cancel`
 
 Attempts to cancel an accepted job.
@@ -653,7 +720,7 @@ Example:
   "strength": 0.75,
   "seed": -1,
   "batch_count": 1,
-  "auto_resize_ref_image": true,
+  "ref_image_args": "",
   "increase_ref_index": false,
   "control_strength": 0.9,
   "ip_adapter_strength": 1.0,
@@ -702,11 +769,11 @@ Example:
   "vae_tiling_params": {
     "enabled": false,
     "temporal_tiling": false,
-    "tile_size_x": 0,
-    "tile_size_y": 0,
+    "tile_size_w": 0,
+    "tile_size_h": 0,
     "target_overlap": 0.5,
-    "rel_size_x": 0.0,
-    "rel_size_y": 0.0,
+    "rel_size_w": 0.0,
+    "rel_size_h": 0.0,
     "extra_tiling_args": ""
   },
 
@@ -728,6 +795,17 @@ Example:
 
 ### Image Encoding Rules
 
+Native image/video requests and SDAPI accept `image_preprocess` as a rule string
+or array of rule strings. OpenAI-compatible requests can supply it in
+`sd_cpp_extra_args`. See [Image preprocessing](../../docs/image_preprocessing.md)
+for one-time input geometry, native-resolution decoding, mask alignment, and
+`canny=true` for edge detection on any supported image input.
+
+Image generation also accepts `ref_image_args` as a string (for example,
+`"resize_before_vae=false"`) in native and SDAPI requests, or through
+`sd_cpp_extra_args` in OpenAI-compatible requests. It controls downstream
+reference encoding and is independent of input geometry rules.
+
 Any image field accepts:
 
 - a raw base64 string, or
@@ -735,12 +813,15 @@ Any image field accepts:
 
 Channel expectations:
 
-- `init_image`: 3 channels
-- `ref_images[]`: 3 channels
+- `init_image`: native channels (3 or 4); alpha is preserved and applied per model
+- `ref_images[]`: native channels (3 or 4); alpha is preserved and applied per model
 - `control_image`: 3 channels
 - `ip_adapter_image`: 3 channels
 - `mask_image`: 1 channel
 
+Models that support RGBA (e.g. Qwen-Image 2.1) use the alpha channel of `init_image`
+and `ref_images[]`. RGB-only models drop it, so sending RGBA is safe for every model.
+
 If omitted or null:
 
 - single-image fields map to an empty `sd_image_t`
@@ -760,7 +841,8 @@ Top-level scalar fields:
 | `strength` | `number` |
 | `seed` | `integer` |
 | `batch_count` | `integer` |
-| `auto_resize_ref_image` | `boolean` |
+| `ref_image_args` | `string` |
+| `image_preprocess` | `string \| array<string>` |
 | `increase_ref_index` | `boolean` |
 | `control_strength` | `number` |
 | `ip_adapter_strength` | `number` |
@@ -820,11 +902,11 @@ Other native fields:
 | `vae_tiling_params` | `object` |
 | `vae_tiling_params.enabled` | `boolean` |
 | `vae_tiling_params.temporal_tiling` | `boolean` |
-| `vae_tiling_params.tile_size_x` | `integer` |
-| `vae_tiling_params.tile_size_y` | `integer` |
+| `vae_tiling_params.tile_size_w` | `integer` |
+| `vae_tiling_params.tile_size_h` | `integer` |
 | `vae_tiling_params.target_overlap` | `number` |
-| `vae_tiling_params.rel_size_x` | `number` |
-| `vae_tiling_params.rel_size_y` | `number` |
+| `vae_tiling_params.rel_size_w` | `number` |
+| `vae_tiling_params.rel_size_h` | `number` |
 | `vae_tiling_params.extra_tiling_args` | `string` |
 | `cache_mode` | `string` |
 | `cache_option` | `string` |
@@ -1035,11 +1117,11 @@ Example:
   "vae_tiling_params": {
     "enabled": false,
     "temporal_tiling": false,
-    "tile_size_x": 0,
-    "tile_size_y": 0,
+    "tile_size_w": 0,
+    "tile_size_h": 0,
     "target_overlap": 0.5,
-    "rel_size_x": 0.0,
-    "rel_size_y": 0.0,
+    "rel_size_w": 0.0,
+    "rel_size_h": 0.0,
     "extra_tiling_args": ""
   },
 
@@ -1160,11 +1242,11 @@ Other native fields:
 | `vae_tiling_params` | `object` |
 | `vae_tiling_params.enabled` | `boolean` |
 | `vae_tiling_params.temporal_tiling` | `boolean` |
-| `vae_tiling_params.tile_size_x` | `integer` |
-| `vae_tiling_params.tile_size_y` | `integer` |
+| `vae_tiling_params.tile_size_w` | `integer` |
+| `vae_tiling_params.tile_size_h` | `integer` |
 | `vae_tiling_params.target_overlap` | `number` |
-| `vae_tiling_params.rel_size_x` | `number` |
-| `vae_tiling_params.rel_size_y` | `number` |
+| `vae_tiling_params.rel_size_w` | `number` |
+| `vae_tiling_params.rel_size_h` | `number` |
 | `vae_tiling_params.extra_tiling_args` | `string` |
 | `cache_mode` | `string` |
 | `cache_option` | `string` |

examples/server/main.cpp

@@ -76,9 +76,9 @@ int main(int argc, const char** argv) {
     SDSvrParams svr_params;
     SDContextParams ctx_params;
     SDGenerationParams default_gen_params;
-    parse_args(argc, argv, svr_params, ctx_params, default_gen_params);
 
     sd_set_log_callback(sd_log_cb, (void*)&svr_params);
+    parse_args(argc, argv, svr_params, ctx_params, default_gen_params);
 
     LOG_VERBOSE("version: %s", version_string().c_str());
     LOG_VERBOSE("%s", sd_get_system_info());

examples/server/routes_openai.cpp

@@ -157,42 +157,46 @@ static bool build_openai_edit_request(const httplib::Request& req,
     request.gen_params.height      = height;
     request.gen_params.batch_count = n;
 
+    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
     for (auto& bytes : images_bytes) {
-        int img_w           = 0;
-        int img_h           = 0;
-        uint8_t* raw_pixels = load_image_from_memory(
-            reinterpret_cast<const char*>(bytes.data()),
-            static_cast<int>(bytes.size()),
-            img_w, img_h,
-            width, height, 3);
+        int img_w            = 0;
+        int img_h            = 0;
+        int resolved_channel = 0;
+        uint8_t* raw_pixels  = load_image_from_memory(
+             reinterpret_cast<const char*>(bytes.data()),
+             static_cast<int>(bytes.size()),
+             img_w, img_h, resolved_channel,
+             0, 0,
+             0);
         if (raw_pixels == nullptr) {
             continue;
         }
 
-        SDImageOwner image_owner({(uint32_t)img_w, (uint32_t)img_h, 3, raw_pixels});
+        const bool is_first_ref_image = request.gen_params.ref_images.empty();
+        SDImageOwner image_owner({(uint32_t)img_w, (uint32_t)img_h, (uint32_t)resolved_channel, raw_pixels});
         request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height);
+
+        if (is_first_ref_image) {
+            request.gen_params.init_image = image_owner;
+            if (request.gen_params.init_image.get().data == nullptr) {
+                error_message = "could not allocate init image";
+                return false;
+            }
+        }
+
         request.gen_params.ref_images.push_back(std::move(image_owner));
     }
 
-    if (!request.gen_params.ref_images.empty()) {
-        request.gen_params.init_image = request.gen_params.ref_images.front();
-    }
-
     if (!mask_bytes.empty()) {
-        int expected_width  = 0;
-        int expected_height = 0;
-        if (request.gen_params.width_and_height_are_set()) {
-            expected_width  = request.gen_params.width;
-            expected_height = request.gen_params.height;
-        }
-        int mask_w = 0;
-        int mask_h = 0;
+        int mask_w       = 0;
+        int mask_h       = 0;
+        int mask_channel = 0;
 
         uint8_t* mask_raw = load_image_from_memory(
             reinterpret_cast<const char*>(mask_bytes.data()),
             static_cast<int>(mask_bytes.size()),
-            mask_w, mask_h,
-            expected_width, expected_height, 1);
+            mask_w, mask_h, mask_channel,
+            0, 0, 1);
         request.gen_params.mask_image.reset({(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw});
         const sd_image_t& mask_image = request.gen_params.mask_image.get();
         request.gen_params.set_width_and_height_if_unset(mask_image.width, mask_image.height);
@@ -205,7 +209,6 @@ static bool build_openai_edit_request(const httplib::Request& req,
         });
     }
 
-    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
     if (!sd_cpp_extra_args_str.empty() && !request.gen_params.from_json_str(sd_cpp_extra_args_str)) {
         error_message = "invalid sd_cpp_extra_args";
         return false;

examples/server/routes_sdapi.cpp

@@ -80,17 +80,6 @@ static enum sample_method_t get_sdapi_sample_method(std::string name) {
     return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT;
 }
 
-static void assign_solid_mask(SDImageOwner& mask_owner, int width, int height) {
-    const size_t pixel_count = static_cast<size_t>(width) * static_cast<size_t>(height);
-    uint8_t* raw_mask        = static_cast<uint8_t*>(malloc(pixel_count));
-    if (raw_mask == nullptr) {
-        mask_owner.reset({0, 0, 1, nullptr});
-        return;
-    }
-    std::memset(raw_mask, 255, pixel_count);
-    mask_owner.reset({(uint32_t)width, (uint32_t)height, 1, raw_mask});
-}
-
 static bool build_sdapi_img_gen_request(const json& j,
                                         ServerRuntime& runtime,
                                         bool img2img,
@@ -193,15 +182,25 @@ static bool build_sdapi_img_gen_request(const json& j,
         }
     }
 
-    if (img2img) {
-        const int expected_width  = request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0;
-        const int expected_height = request.gen_params.width_and_height_are_set() ? request.gen_params.height : 0;
+    if (j.contains("ref_image_args")) {
+        if (!j["ref_image_args"].is_string()) {
+            error_message = "ref_image_args must be a string";
+            return false;
+        }
+        request.gen_params.ref_image_args = j["ref_image_args"].get<std::string>();
+    }
 
+    if (j.contains("image_preprocess") && !request.gen_params.parse_image_preprocess_json(j["image_preprocess"].dump())) {
+        error_message = "invalid image_preprocess";
+        return false;
+    }
+
+    if (img2img) {
         if (j.contains("init_images") && j["init_images"].is_array() && !j["init_images"].empty()) {
             if (decode_base64_image(j["init_images"][0].get<std::string>(),
-                                    3,
-                                    expected_width,
-                                    expected_height,
+                                    0,
+                                    0,
+                                    0,
                                     request.gen_params.init_image)) {
                 const sd_image_t& image = request.gen_params.init_image.get();
                 request.gen_params.set_width_and_height_if_unset(image.width, image.height);
@@ -211,8 +210,8 @@ static bool build_sdapi_img_gen_request(const json& j,
         if (j.contains("mask") && j["mask"].is_string()) {
             if (decode_base64_image(j["mask"].get<std::string>(),
                                     1,
-                                    expected_width,
-                                    expected_height,
+                                    0,
+                                    0,
                                     request.gen_params.mask_image)) {
                 const sd_image_t& image = request.gen_params.mask_image.get();
                 request.gen_params.set_width_and_height_if_unset(image.width, image.height);
@@ -225,9 +224,7 @@ static bool build_sdapi_img_gen_request(const json& j,
                 }
             }
         } else {
-            const int resolved_width  = request.gen_params.get_resolved_width();
-            const int resolved_height = request.gen_params.get_resolved_height();
-            assign_solid_mask(request.gen_params.mask_image, resolved_width, resolved_height);
+            request.gen_params.mask_image.reset({0, 0, 1, nullptr});
         }
 
         float denoising_strength = j.value("denoising_strength", -1.f);
@@ -243,9 +240,8 @@ static bool build_sdapi_img_gen_request(const json& j,
             }
             SDImageOwner image_owner;
             if (decode_base64_image(extra_image.get<std::string>(),
-                                    3,
-                                    request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0,
-                                    request.gen_params.width_and_height_are_set() ? request.gen_params.height : 0,
+                                    0,
+                                    0, 0,
                                     image_owner)) {
                 const sd_image_t& image = image_owner.get();
                 request.gen_params.set_width_and_height_if_unset(image.width, image.height);

examples/server/routes_sdcpp.cpp

@@ -3,12 +3,33 @@
 #include <algorithm>
 #include <cmath>
 #include <filesystem>
+#include <limits>
 
 #include "async_jobs.h"
 #include "common/common.h"
+#include "common/media_io.h"
+#include "common/resource_owners.hpp"
 
 namespace fs = std::filesystem;
 
+static constexpr uint32_t k_max_upscale_dimension = 8192;
+
+static bool valid_upscale_dimensions(const sd_image_t& image, int factor, int repeats) {
+    if (image.width == 0 || image.height == 0 || factor < 1 || repeats < 1 || repeats > 4) {
+        return false;
+    }
+    uint32_t width  = image.width;
+    uint32_t height = image.height;
+    for (int i = 0; i < repeats; ++i) {
+        if (width > k_max_upscale_dimension / factor || height > k_max_upscale_dimension / factor) {
+            return false;
+        }
+        width *= factor;
+        height *= factor;
+    }
+    return true;
+}
+
 static bool parse_cache_mode(const std::string& mode_str, sd_cache_mode_t& mode_out) {
     if (mode_str == "disabled") {
         mode_out = SD_CACHE_DISABLED;
@@ -57,11 +78,11 @@ static json make_vae_tiling_json(const sd_tiling_params_t& params) {
     return {
         {"enabled", params.enabled},
         {"temporal_tiling", params.temporal_tiling},
-        {"tile_size_x", params.tile_size_x},
-        {"tile_size_y", params.tile_size_y},
+        {"tile_size_w", params.tile_size_w},
+        {"tile_size_h", params.tile_size_h},
         {"target_overlap", params.target_overlap},
-        {"rel_size_x", params.rel_size_x},
-        {"rel_size_y", params.rel_size_y},
+        {"rel_size_w", params.rel_size_w},
+        {"rel_size_h", params.rel_size_h},
         {"extra_tiling_args", params.extra_tiling_args ? params.extra_tiling_args : ""},
     };
 }
@@ -127,7 +148,8 @@ static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const
         {"seed", defaults.seed},
         {"batch_count", defaults.batch_count},
         {"qwen_image_layers", defaults.qwen_image_layers},
-        {"auto_resize_ref_image", defaults.auto_resize_ref_image},
+        {"ref_image_args", defaults.ref_image_args},
+        {"image_preprocess", defaults.image_preprocess},
         {"increase_ref_index", defaults.increase_ref_index},
         {"control_strength", defaults.control_strength},
         {"ip_adapter_strength", defaults.ip_adapter_strength},
@@ -153,6 +175,7 @@ static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const
         {"strength", defaults.strength},
         {"seed", defaults.seed},
         {"video_frames", defaults.video_frames},
+        {"image_preprocess", defaults.image_preprocess},
         {"fps", defaults.fps},
         {"moe_boundary", defaults.moe_boundary},
         {"vace_strength", defaults.vace_strength},
@@ -239,37 +262,59 @@ static json make_capabilities_json(ServerRuntime& runtime) {
 
     available_upscalers.push_back({
         {"name", "None"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Lanczos"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Nearest"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent (nearest)"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent (nearest-exact)"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent (antialiased)"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent (bicubic)"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent (bicubic antialiased)"},
+        {"model", false},
+        {"image_upscale", false},
     });
+    bool have_upscaler_models = false;
     {
         std::lock_guard<std::mutex> lock(*runtime.upscaler_mutex);
         for (const auto& entry : *runtime.upscaler_cache) {
             available_upscalers.push_back({
                 {"name", entry.name},
+                {"model", true},
+                {"image_upscale", entry.image_upscale_factor > 0},
             });
+            have_upscaler_models = have_upscaler_models || entry.image_upscale_factor > 0;
         }
     }
 
@@ -339,6 +384,8 @@ static json make_capabilities_json(ServerRuntime& runtime) {
                   {"max_height", 4096},
                   {"max_batch_count", 8},
                   {"max_queue_size", manager.max_pending_jobs},
+                  {"max_upscale_width", k_max_upscale_dimension},
+                  {"max_upscale_height", k_max_upscale_dimension},
     };
     result["samplers"]               = samplers;
     result["schedulers"]             = schedulers;
@@ -348,6 +395,7 @@ static json make_capabilities_json(ServerRuntime& runtime) {
     result["features_by_mode"]       = features_by_mode;
     result["loras"]                  = available_loras;
     result["upscalers"]              = available_upscalers;
+    result["upscale"]                = have_upscaler_models;
     return result;
 }
 
@@ -413,6 +461,171 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         res.set_content(make_capabilities_json(*runtime).dump(), "application/json");
     });
 
+    svr.Post("/sdcpp/v1/upscale", [runtime](const httplib::Request& req, httplib::Response& res) {
+        try {
+            if (req.body.empty()) {
+                res.status = 400;
+                res.set_content(R"({"error":"empty body"})", "application/json");
+                return;
+            }
+            json body = json::parse(req.body);
+            if (!body.is_object()) {
+                res.status = 400;
+                res.set_content(R"({"error":"body must be an object"})", "application/json");
+                return;
+            }
+            for (const char* key : {"repeats", "tile_size", "output_compression"}) {
+                if (!body.contains(key)) {
+                    continue;
+                }
+                const auto& value = body[key];
+                const bool valid  = value.is_number_unsigned()
+                                        ? value.get<uint64_t>() <= static_cast<uint64_t>(std::numeric_limits<int>::max())
+                                        : value.is_number_integer() && value.get<int64_t>() >= std::numeric_limits<int>::min() &&
+                                             value.get<int64_t>() <= std::numeric_limits<int>::max();
+                if (!valid) {
+                    res.status = 400;
+                    res.set_content(json({{"error", std::string(key) + " must be a 32-bit integer"}}).dump(), "application/json");
+                    return;
+                }
+            }
+            ImgGenJobRequest output_options;
+            std::string error_message;
+            if (!assign_output_options(output_options,
+                                       body.value("output_format", std::string("png")),
+                                       body.value("output_compression", 100),
+                                       true,
+                                       error_message)) {
+                res.status = 400;
+                res.set_content(json({{"error", error_message}}).dump(), "application/json");
+                return;
+            }
+            const int tile_size      = std::max(32, body.value("tile_size", runtime->default_gen_params->upscale_tile_size));
+            const int repeats        = std::clamp(body.value("repeats", 1), 1, 4);
+            const std::string wanted = body.value("upscaler", std::string());
+
+            const std::string encoded = body.value("image", std::string());
+            if (encoded.empty()) {
+                res.status = 400;
+                res.set_content(R"({"error":"image is required"})", "application/json");
+                return;
+            }
+            SDImageOwner input;
+            if (!decode_base64_image(encoded, 3, 0, 0, input) || input.get().data == nullptr) {
+                res.status = 400;
+                res.set_content(R"({"error":"image could not be read"})", "application/json");
+                return;
+            }
+
+            refresh_upscaler_cache(*runtime);
+            int model_scale = 0;
+            std::string model_path;
+            std::string used_name;
+            {
+                std::lock_guard<std::mutex> lock(*runtime->upscaler_mutex);
+                for (const auto& entry : *runtime->upscaler_cache) {
+                    if (entry.image_upscale_factor > 0 && (wanted.empty() || entry.name == wanted)) {
+                        model_path  = entry.fullpath;
+                        used_name   = entry.name;
+                        model_scale = entry.image_upscale_factor;
+                        break;
+                    }
+                }
+            }
+            if (model_path.empty()) {
+                res.status = 400;
+                res.set_content(json({{"error", wanted.empty()
+                                                    ? std::string("no RGB ESRGAN upscaler models are available; "
+                                                                  "start the server with --hires-upscalers-dir")
+                                                    : "no compatible image upscaler called " + wanted}})
+                                    .dump(),
+                                "application/json");
+                return;
+            }
+
+            if (!valid_upscale_dimensions(input.get(), model_scale, repeats)) {
+                res.status = 400;
+                res.set_content(R"({"error":"upscaled dimensions must not exceed 8192 x 8192"})", "application/json");
+                return;
+            }
+
+            // One GPU: an upscale must not run while a generation is using it.
+            std::lock_guard<std::mutex> ctx_lock(*runtime->sd_ctx_mutex);
+            UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(model_path.c_str(),
+                                                         runtime->ctx_params->diffusion_conv_direct,
+                                                         runtime->ctx_params->n_threads,
+                                                         tile_size,
+                                                         runtime->ctx_params->backend.c_str(),
+                                                         runtime->ctx_params->params_backend.c_str()));
+            if (upscaler_ctx == nullptr) {
+                res.status = 500;
+                res.set_content(R"({"error":"the upscaler model could not be loaded"})", "application/json");
+                return;
+            }
+            const int factor = get_upscale_factor(upscaler_ctx.get());
+            // The model file may have changed since its metadata was cached.
+            if (!valid_upscale_dimensions(input.get(), factor, repeats)) {
+                res.status = 400;
+                res.set_content(R"({"error":"upscaled dimensions must not exceed 8192 x 8192"})", "application/json");
+                return;
+            }
+
+            SDImageOwner current(input.release());
+            for (int i = 0; i < repeats; ++i) {
+                sd_image_t* out_images = nullptr;
+                int out_count          = 0;
+                if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &out_images, &out_count) ||
+                    out_count <= 0 || out_images[0].data == nullptr) {
+                    free_sd_images(out_images, out_count);
+                    res.status = 500;
+                    res.set_content(R"({"error":"upscale failed"})", "application/json");
+                    return;
+                }
+                sd_image_t produced = out_images[0];
+                out_images[0]       = {0, 0, 0, nullptr};
+                free_sd_images(out_images, out_count);
+                current.reset(produced);
+            }
+
+            const std::string& format = output_options.output_format;
+            const int compression     = output_options.output_compression;
+            const sd_image_t result   = current.get();
+            auto image_bytes          = encode_image_to_vector(format == "jpeg"   ? EncodedImageFormat::JPEG
+                                                               : format == "webp" ? EncodedImageFormat::WEBP
+                                                                                  : EncodedImageFormat::PNG,
+                                                      result.data,
+                                                      result.width,
+                                                      result.height,
+                                                      result.channel,
+                                                      "",
+                                                      compression);
+            if (image_bytes.empty()) {
+                res.status = 500;
+                res.set_content(R"({"error":"the result could not be encoded"})", "application/json");
+                return;
+            }
+
+            json out;
+            out["upscaler"]      = used_name;
+            out["scale"]         = factor;
+            out["repeats"]       = repeats;
+            out["width"]         = result.width;
+            out["height"]        = result.height;
+            out["output_format"] = format;
+            json images          = json::array();
+            images.push_back({{"index", 0}, {"b64_json", base64_encode(image_bytes)}});
+            out["images"] = std::move(images);
+            res.set_content(out.dump(), "application/json");
+            res.status = 200;
+        } catch (const json::exception& e) {
+            res.status = 400;
+            res.set_content(json({{"error", "invalid request"}, {"message", e.what()}}).dump(), "application/json");
+        } catch (const std::exception& e) {
+            res.status = 500;
+            res.set_content(json({{"error", std::string("server_error: ") + e.what()}}).dump(), "application/json");
+        }
+    });
+
     svr.Post("/sdcpp/v1/img_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
         try {
             if (req.body.empty()) {

examples/server/runtime.cpp

@@ -255,27 +255,48 @@ void refresh_lora_cache(ServerRuntime& rt) {
     std::vector<LoraEntry> new_cache;
 
     fs::path lora_dir = rt.ctx_params->lora_model_dir;
-    if (fs::exists(lora_dir) && fs::is_directory(lora_dir)) {
-        for (auto& entry : fs::recursive_directory_iterator(lora_dir, fs::directory_options::skip_permission_denied)) {
-            if (!entry.is_regular_file()) {
-                continue;
-            }
-            const fs::path& p = entry.path();
-            if (!is_supported_model_ext(p)) {
-                continue;
-            }
+    std::error_code ec;
+    if (fs::exists(lora_dir, ec) && !ec && fs::is_directory(lora_dir, ec) && !ec) {
+        try {
+            auto it = fs::recursive_directory_iterator(
+                lora_dir,
+                fs::directory_options::skip_permission_denied,
+                ec);
+            auto end = fs::recursive_directory_iterator();
+            while (!ec && it != end) {
+                std::error_code entry_ec;
+                bool is_reg = it->is_regular_file(entry_ec);
+                if (entry_ec || !is_reg) {
+                    it.increment(ec);
+                    continue;
+                }
+                const fs::path& p = it->path();
+                if (!is_supported_model_ext(p)) {
+                    it.increment(ec);
+                    continue;
+                }
 
-            LoraEntry lora_entry;
-            lora_entry.name     = p.stem().u8string();
-            lora_entry.fullpath = p.u8string();
-            std::string rel     = p.lexically_relative(lora_dir).u8string();
-            std::replace(rel.begin(), rel.end(), '\\', '/');
-            lora_entry.path = rel;
+                LoraEntry lora_entry;
+                lora_entry.name     = p.stem().u8string();
+                lora_entry.fullpath = p.u8string();
+                std::string rel     = p.lexically_relative(lora_dir).u8string();
+                std::replace(rel.begin(), rel.end(), '\\', '/');
+                lora_entry.path = rel;
 
-            new_cache.push_back(std::move(lora_entry));
+                new_cache.push_back(std::move(lora_entry));
+                it.increment(ec);
+            }
+        } catch (const std::exception& e) {
+            LOG_WARN("error while scanning lora directory '%s': %s", lora_dir.string().c_str(), e.what());
+            return;
         }
     }
 
+    if (ec) {
+        LOG_WARN("error while scanning lora directory '%s': %s", lora_dir.string().c_str(), ec.message().c_str());
+        return;
+    }
+
     std::sort(new_cache.begin(), new_cache.end(), [](const LoraEntry& a, const LoraEntry& b) {
         return a.path < b.path;
     });
@@ -295,28 +316,78 @@ std::string get_lora_full_path(ServerRuntime& rt, const std::string& path) {
 
 void refresh_upscaler_cache(ServerRuntime& rt) {
     std::vector<UpscalerEntry> new_cache;
+    std::vector<UpscalerEntry> previous_cache;
+    {
+        std::lock_guard<std::mutex> lock(*rt.upscaler_mutex);
+        previous_cache = *rt.upscaler_cache;
+    }
 
     fs::path upscaler_dir = rt.ctx_params->hires_upscalers_dir;
-    if (fs::exists(upscaler_dir) && fs::is_directory(upscaler_dir)) {
-        for (auto& entry : fs::directory_iterator(upscaler_dir)) {
-            if (!entry.is_regular_file()) {
-                continue;
-            }
-            const fs::path& p = entry.path();
-            if (!is_supported_model_ext(p)) {
-                continue;
-            }
+    std::error_code ec;
+    if (fs::exists(upscaler_dir, ec) && !ec && fs::is_directory(upscaler_dir, ec) && !ec) {
+        try {
+            auto it = fs::directory_iterator(
+                upscaler_dir,
+                fs::directory_options::skip_permission_denied,
+                ec);
+            auto end = fs::directory_iterator();
+            while (!ec && it != end) {
+                std::error_code entry_ec;
+                bool is_reg = it->is_regular_file(entry_ec);
+                if (entry_ec || !is_reg) {
+                    it.increment(ec);
+                    continue;
+                }
+                const fs::path& p = it->path();
+                if (!is_supported_model_ext(p)) {
+                    it.increment(ec);
+                    continue;
+                }
 
-            UpscalerEntry upscaler_entry;
-            upscaler_entry.name       = p.stem().u8string();
-            upscaler_entry.fullpath   = fs::absolute(p).lexically_normal().u8string();
-            upscaler_entry.model_name = "ESRGAN_4x";
-            upscaler_entry.path       = p.filename().u8string();
+                UpscalerEntry upscaler_entry;
+                upscaler_entry.name = p.stem().u8string();
+                std::error_code abs_ec;
+                fs::path abs_path         = fs::absolute(p, abs_ec);
+                upscaler_entry.fullpath   = (abs_ec ? p : abs_path.lexically_normal()).u8string();
+                upscaler_entry.model_name = "ESRGAN_4x";
+                upscaler_entry.path       = p.filename().u8string();
 
-            new_cache.push_back(std::move(upscaler_entry));
+                std::error_code size_ec;
+                upscaler_entry.file_size = it->file_size(size_ec);
+                if (size_ec) {
+                    it.increment(ec);
+                    continue;
+                }
+
+                std::error_code time_ec;
+                upscaler_entry.last_modified        = it->last_write_time(time_ec);
+                auto previous                       = std::find_if(previous_cache.begin(), previous_cache.end(), [&](const UpscalerEntry& cached) {
+                    return cached.fullpath == upscaler_entry.fullpath &&
+                           cached.file_size == upscaler_entry.file_size &&
+                           (!time_ec && cached.last_modified == upscaler_entry.last_modified);
+                });
+                upscaler_entry.image_upscale_factor = previous != previous_cache.end()
+                                                          ? previous->image_upscale_factor
+                                                          : get_upscaler_model_scale(upscaler_entry.fullpath.c_str());
+                if (upscaler_entry.image_upscale_factor > 0) {
+                    upscaler_entry.scale      = upscaler_entry.image_upscale_factor;
+                    upscaler_entry.model_name = "ESRGAN_" + std::to_string(upscaler_entry.scale) + "x";
+                }
+
+                new_cache.push_back(std::move(upscaler_entry));
+                it.increment(ec);
+            }
+        } catch (const std::exception& e) {
+            LOG_WARN("error while scanning upscalers directory '%s': %s", upscaler_dir.string().c_str(), e.what());
+            return;
         }
     }
 
+    if (ec) {
+        LOG_WARN("error while scanning upscalers directory '%s': %s", upscaler_dir.string().c_str(), ec.message().c_str());
+        return;
+    }
+
     std::sort(new_cache.begin(), new_cache.end(), [](const UpscalerEntry& a, const UpscalerEntry& b) {
         return a.name < b.name;
     });

examples/server/runtime.h

@@ -2,6 +2,7 @@
 
 #include <algorithm>
 #include <cstdint>
+#include <filesystem>
 #include <mutex>
 #include <string>
 #include <vector>
@@ -43,6 +44,9 @@ struct UpscalerEntry {
     std::string fullpath;
     std::string model_name;
     int scale = 4;
+    int image_upscale_factor = 0;
+    uintmax_t file_size      = 0;
+    std::filesystem::file_time_type last_modified;
 };
 
 struct ServerRuntime {

include/stable-diffusion.h

@@ -173,11 +173,13 @@ enum lora_apply_mode_t {
 typedef struct {
     bool enabled;
     bool temporal_tiling;
-    int tile_size_x;
-    int tile_size_y;
+    // Spatial tile dimensions in image pixels for both encode and decode; 0 uses 256.
+    int tile_size_w;
+    int tile_size_h;
     float target_overlap;
-    float rel_size_x;
-    float rel_size_y;
+    // Positive values override tile_size: <= 1 is a dimension fraction, > 1 a target tile count.
+    float rel_size_w;
+    float rel_size_h;
     const char* extra_tiling_args;
 } sd_tiling_params_t;
 
@@ -247,6 +249,7 @@ typedef struct {
     float attn_scale;                // Override flash-attention K/V scaling; 0 keeps the model default
     const char* tokenizer;           // tokenizer.json path or main=FILE,clip-l=FILE,clip-g=FILE assignments; required for PiD and Lens
     bool sage_attn;
+    int conditioning_cache_size;  // Maximum cached conditioning entries per context; 0 disables caching (default: 4)
 } sd_ctx_params_t;
 
 typedef struct {
@@ -263,6 +266,11 @@ typedef struct {
     uint8_t* data;
 } sd_image_t;
 
+typedef struct {
+    // Semicolon-separated target=...,key=value rules. NULL preserves defaults.
+    const char* rules;
+} sd_image_preprocess_params_t;
+
 typedef struct {
     sd_image_t* frames;
     int frame_count;
@@ -410,6 +418,7 @@ typedef struct {
     int qwen_image_layers;
     bool circular_x;
     bool circular_y;
+    sd_image_preprocess_params_t image_preprocess;
 } sd_img_gen_params_t;
 
 typedef struct {
@@ -443,6 +452,7 @@ typedef struct {
     sd_hires_params_t hires;
     bool circular_x;
     bool circular_y;
+    sd_image_preprocess_params_t image_preprocess;
 } sd_vid_gen_params_t;
 
 typedef struct sd_ctx_t sd_ctx_t;
@@ -550,6 +560,8 @@ SD_API bool upscale(upscaler_ctx_t* upscaler_ctx,
                     int* num_images_out);
 
 SD_API int get_upscale_factor(upscaler_ctx_t* upscaler_ctx);
+// Reads model metadata only; returns 0 if the file is not a recognized RGB ESRGAN model.
+SD_API int get_upscaler_model_scale(const char* model_path);
 
 typedef struct adetailer_ctx_t adetailer_ctx_t;
 

src/conditioning/conditioner.hpp

@@ -16,6 +16,7 @@
 #include "model/te/clip.hpp"
 #include "model/te/llada_image_te.hpp"
 #include "model/te/llm.hpp"
+#include "model/te/ming_image_te.hpp"
 #include "model/te/t5.hpp"
 #include "model_loader.h"
 #include "tokenizers/sensenova_u1_tokenizer.h"
@@ -2219,7 +2220,10 @@ struct LLMEmbedder : public Conditioner {
                                           false,
                                           deepstack_image_embeds,
                                           image_grids);
-        GGML_ASSERT(!hidden_states.empty());
+        if (hidden_states.empty()) {
+            LOG_ERROR("LLM prompt encoding failed");
+            return {};
+        }
         hidden_states = apply_token_weights(std::move(hidden_states), weights);
         GGML_ASSERT(hidden_states.shape()[1] > prompt_template_encode_start_idx);
 
@@ -3163,6 +3167,66 @@ struct LLMEmbedder : public Conditioner {
             int64_t tag_count    = static_cast<int64_t>(tags.size());
             result.c_token_types = sd::Tensor<int32_t>({tag_count}, std::move(tags));
         }
+
+        return result;
+    }
+};
+
+struct MingImageEmbedder : public Conditioner {
+    std::shared_ptr<Tokenizer> tokenizer;
+    std::shared_ptr<MingImageTE::MingImageTextRunner> text_model;
+    const std::string prefix = "text_encoders.llm";
+
+    MingImageEmbedder(ggml_backend_t backend, const String2TensorStorage& tensors, std::shared_ptr<RunnerWeightManager> weight_manager, const TokenizerConfig& tokenizers) {
+        if (!tokenizers.has(TokenizerConfig::MAIN)) {
+            throw std::runtime_error("Ming-Image requires the Ling tokenizer.json; pass --tokenizer FILE");
+        }
+        text_model = std::make_shared<MingImageTE::MingImageTextRunner>(backend, tensors, prefix, weight_manager);
+        tokenizer  = tokenizers.create(TokenizerConfig::MAIN, text_model->config.backbone.vocab_size, 156895);
+    }
+
+    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+        text_model->get_param_tensors(tensors, prefix);
+    }
+    void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& ops) override {
+        text_model->get_param_tensor_ops(ops);
+    }
+    void get_layer_split_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+        text_model->get_param_tensors(tensors, prefix);
+    }
+    void set_flash_attention_enabled(bool enabled) override { text_model->set_flash_attention_enabled(enabled); }
+    void set_max_graph_vram_bytes(size_t bytes) override { text_model->set_max_graph_vram_bytes(bytes); }
+    void set_runtime_backends(const std::vector<ggml_backend_t>& backends) override { text_model->set_runtime_backends(backends); }
+    void set_graph_cut_layer_split_enabled(bool enabled) override { text_model->set_graph_cut_layer_split_enabled(enabled); }
+    void set_graph_cut_layer_split_backend_vram_limits(const std::vector<size_t>& limits) override { text_model->set_graph_cut_layer_split_backend_vram_limits(limits); }
+    void set_scale_overrides(float linear, float attention) override { text_model->set_scale_overrides(linear, attention); }
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override { text_model->set_weight_adapter(adapter); }
+    void runner_end() override { text_model->runner_end(); }
+
+    SDCondition get_learned_condition(int n_threads, const ConditionerParams& input) override {
+        if (input.ref_images != nullptr && !input.ref_images->empty()) {
+            LOG_ERROR("Ming-Image currently supports text-to-image only");
+            return {};
+        }
+        std::string prompt =
+            "<role>SYSTEM</role>你是一个友好的AI助手。\n\ndetailed thinking off<|role_end|>"
+            "<role>HUMAN</role>" +
+            input.text + "<|role_end|><role>ASSISTANT</role>";
+        std::vector<int> tokens;
+        if (!tokenizer->encode(prompt, tokens, nullptr)) {
+            return {};
+        }
+        if (tokens.size() + 258 > 32768) {
+            LOG_ERROR("Ming-Image prompt exceeds the text encoder context length");
+            return {};
+        }
+        auto output = text_model->compute(n_threads, tokens);
+        if (output.empty()) {
+            return {};
+        }
+        SDCondition result;
+        result.c_crossattn = sd::ops::slice(sd::ops::slice(output, 1, 0, 256), 0, 0, text_model->config.caption_dim);
+        result.extra_c_crossattns.push_back(sd::ops::slice(output, 1, 256, output.shape()[1]));
         return result;
     }
 };

src/conditioning/conditioning_cache.h

@@ -0,0 +1,107 @@
+#ifndef __SD_CONDITIONING_CONDITIONING_CACHE_H__
+#define __SD_CONDITIONING_CONDITIONING_CACHE_H__
+
+#include <algorithm>
+#include <list>
+#include <tuple>
+
+#include "conditioning/conditioner.hpp"
+
+class ConditioningCache {
+    struct Entry {
+        ConditionerParams params;
+        std::vector<sd::Tensor<float>> ref_images;
+        std::vector<MiniMaxH3PresentationItem> references;
+        SDCondition condition;
+
+        Entry(const ConditionerParams& input, const SDCondition& output)
+            : params(input), condition(output) {
+            // Request-owned reference pointers must not outlive the request.
+            if (input.ref_images != nullptr) {
+                ref_images        = *input.ref_images;
+                params.ref_images = &ref_images;
+            }
+            if (input.minimax_h3_references != nullptr) {
+                references                   = *input.minimax_h3_references;
+                params.minimax_h3_references = &references;
+            }
+        }
+
+        Entry(const Entry&)            = delete;
+        Entry& operator=(const Entry&) = delete;
+    };
+
+    size_t capacity_ = 4;
+    std::list<Entry> entries_;
+
+    static bool same_images(const std::vector<sd::Tensor<float>>& a,
+                            const std::vector<sd::Tensor<float>>& b) {
+        return std::equal(a.begin(), a.end(), b.begin(), b.end(),
+                          [](const sd::Tensor<float>& x, const sd::Tensor<float>& y) {
+                              return x.shape() == y.shape() && x.values() == y.values();
+                          });
+    }
+
+    static bool same_params(const ConditionerParams& a, const ConditionerParams& b) {
+        const auto fields = [](const ConditionerParams& p) {
+            const auto& r = p.ref_image_params;
+            return std::tie(p.text, p.clip_skip, p.width, p.height, p.zero_out_masked,
+                            r.pass_to_vlm, r.pass_to_dit, r.ref_index_mode,
+                            r.force_ref_timestep_zero, r.resize_before_vae, r.vae_input_max_pixels,
+                            r.vlm_resize_mode, r.vlm_min_size, r.vlm_max_size, r.resize_vae_to_target);
+        };
+        if (fields(a) != fields(b) ||
+            (a.ref_images == nullptr) != (b.ref_images == nullptr) ||
+            (a.minimax_h3_references == nullptr) != (b.minimax_h3_references == nullptr)) {
+            return false;
+        }
+        if (a.ref_images != nullptr && !same_images(*a.ref_images, *b.ref_images)) {
+            return false;
+        }
+        if (a.minimax_h3_references != nullptr &&
+            !std::equal(a.minimax_h3_references->begin(), a.minimax_h3_references->end(),
+                        b.minimax_h3_references->begin(), b.minimax_h3_references->end(),
+                        [](const MiniMaxH3PresentationItem& x, const MiniMaxH3PresentationItem& y) {
+                            return x.kind == y.kind && x.timestamps == y.timestamps && same_images(x.frames, y.frames);
+                        })) {
+            return false;
+        }
+        return true;
+    }
+
+public:
+    void set_capacity(size_t capacity) {
+        capacity_ = capacity;
+        while (entries_.size() > capacity_) {
+            entries_.pop_back();
+        }
+    }
+
+    void clear() {
+        entries_.clear();
+    }
+
+    SDCondition get(Conditioner& conditioner, int n_threads, const ConditionerParams& params) {
+        if (capacity_ == 0) {
+            return conditioner.get_learned_condition(n_threads, params);
+        }
+        for (auto it = entries_.begin(); it != entries_.end(); ++it) {
+            if (same_params(it->params, params)) {
+                entries_.splice(entries_.begin(), entries_, it);
+                LOG_INFO("conditioning cache hit");
+                return entries_.front().condition;
+            }
+        }
+        auto condition = conditioner.get_learned_condition(n_threads, params);
+        if (!condition.empty()) {
+            if (entries_.size() == capacity_) {
+                entries_.pop_back();
+            }
+            entries_.emplace_front(params, condition);
+            LOG_VERBOSE("conditioning cache stored (%zu/%zu)", entries_.size(), capacity_);
+        }
+        return condition;
+    }
+};
+
+#endif  // __SD_CONDITIONING_CONDITIONING_CACHE_H__

src/core/backend_fit.cpp

@@ -478,27 +478,48 @@ namespace sd::backend_fit {
         return true;
     }
 
-    bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params, bool prefer_temporal_tiling) {
+    bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params,
+                                         bool prefer_temporal_tiling,
+                                         ggml_status status,
+                                         int latent_tile_size_w,
+                                         int latent_tile_size_h,
+                                         int scale_factor) {
+        // Execution failures can leave the device unusable; tiling only helps with allocation failures.
+        if (status != GGML_STATUS_ALLOC_FAILED) {
+            return false;
+        }
         const char* retry_mode = nullptr;
         if (prefer_temporal_tiling && !tiling_params.temporal_tiling) {
             tiling_params.temporal_tiling = true;
             retry_mode                    = tiling_params.enabled ? "spatial+temporal" : "temporal";
-        } else if (!tiling_params.enabled) {
-            tiling_params.enabled    = true;
-            tiling_params.rel_size_x = 0.5f;
-            tiling_params.rel_size_y = 0.5f;
-            if (tiling_params.tile_size_x <= 0) {
-                tiling_params.tile_size_x = 256;
-            }
-            if (tiling_params.tile_size_y <= 0) {
-                tiling_params.tile_size_y = 256;
-            }
-            retry_mode = tiling_params.temporal_tiling ? "spatial+temporal" : "spatial";
         } else {
-            return false;
+            if (latent_tile_size_w <= 0 || latent_tile_size_h <= 0 || scale_factor <= 0) {
+                return false;
+            }
+            auto smaller_tile = [&](int size) {
+                int next_size = size / 2;
+                if (!tiling_params.enabled) {
+                    next_size = std::min(next_size, 256 / scale_factor);
+                }
+                return std::min(size, std::max(4, next_size));
+            };
+            const int tile_size_w = smaller_tile(latent_tile_size_w);
+            const int tile_size_h = smaller_tile(latent_tile_size_h);
+            if (tile_size_w == latent_tile_size_w && tile_size_h == latent_tile_size_h) {
+                return false;
+            }
+            tiling_params.enabled     = true;
+            tiling_params.rel_size_w  = 0.0f;
+            tiling_params.rel_size_h  = 0.0f;
+            tiling_params.tile_size_w = tile_size_w * scale_factor;
+            tiling_params.tile_size_h = tile_size_h * scale_factor;
+            retry_mode                = tiling_params.temporal_tiling ? "spatial+temporal" : "spatial";
+            LOG_WARN("Reducing VAE decode tiles from %dx%d to %dx%d image pixels",
+                     latent_tile_size_w * scale_factor, latent_tile_size_h * scale_factor,
+                     tiling_params.tile_size_w, tiling_params.tile_size_h);
         }
 
-        LOG_WARN("VAE decode failed (likely out of memory); retrying with %s tiling",
+        LOG_WARN("VAE decode ran out of memory; retrying with %s tiling",
                  retry_mode);
         return true;
     }

src/core/backend_fit.h

@@ -16,7 +16,11 @@ namespace sd::backend_fit {
                               std::string& params_spec);
 
     bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params,
-                                         bool prefer_temporal_tiling);
+                                         bool prefer_temporal_tiling,
+                                         ggml_status status,
+                                         int latent_tile_size_w,
+                                         int latent_tile_size_h,
+                                         int scale_factor);
 
 }  // namespace sd::backend_fit
 

src/core/ggml_extend.cpp

@@ -255,14 +255,20 @@ ggml_tensor* ggml_ext_linear_i8_tensorwise(ggml_context* ctx,
     }
 
     ggml_tensor* fused_bias = scale == 1.f ? b : nullptr;
+    auto mul_mat            = [&](ggml_tensor* input) {
+        if (input->type == GGML_TYPE_F32 && convrot_group_size > 0) {
+            input = ggml_quantize_i8_convrot(ctx, input, convrot_group_size);
+        }
+        return ggml_mul_mat_i8_tensorwise(ctx, w, input, weight_scale, fused_bias, convrot_group_size);
+    };
     if (x->ne[2] * x->ne[3] > 1024) {
         int64_t ne2 = x->ne[2];
         int64_t ne3 = x->ne[3];
         x           = ggml_reshape_2d(ctx, x, x->ne[0], x->ne[1] * x->ne[2] * x->ne[3]);
-        x           = ggml_mul_mat_i8_tensorwise(ctx, w, x, weight_scale, fused_bias, convrot_group_size);
+        x           = mul_mat(x);
         x           = ggml_reshape_4d(ctx, x, x->ne[0], x->ne[1] / ne2 / ne3, ne2, ne3);
     } else {
-        x = ggml_mul_mat_i8_tensorwise(ctx, w, x, weight_scale, fused_bias, convrot_group_size);
+        x = mul_mat(x);
     }
 
     if (scale != 1.f) {
@@ -465,7 +471,11 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx,
                               int d1,
                               int d2,
                               bool force_prec_f32,
-                              bool direct) {
+                              bool direct,
+                              float scale) {
+    if (scale != 1.f) {
+        x = ggml_ext_scale(ctx, x, scale);
+    }
     if (direct) {
         int64_t OC = w->ne[3] / IC;
         int64_t N  = x->ne[3] / IC;
@@ -502,6 +512,9 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx,
         }
     }
 
+    if (scale != 1.f) {
+        x = ggml_ext_scale(ctx, x, 1.f / scale);
+    }
     if (b != nullptr) {
         b = ggml_reshape_4d(ctx, b, 1, 1, 1, b->ne[0]);  // [OC, 1, 1, 1]
         x = ggml_add_inplace(ctx, x, b);
@@ -623,7 +636,11 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
                                     bool skip_reshape,
                                     bool flash_attn,
                                     float kv_scale,
-                                    bool sage_attn) {  // avoid overflow
+                                    bool sage_attn,
+                                    bool* used_flash_attn) {  // avoid overflow
+    if (used_flash_attn != nullptr) {
+        *used_flash_attn = false;
+    }
     int64_t L_q;
     int64_t L_k;
     int64_t C;
@@ -755,6 +772,9 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
         if (can_use_flash_attn) {
             kqv = build_kqv(q, k, v, mask);
             if (kqv != nullptr) {
+                if (used_flash_attn != nullptr) {
+                    *used_flash_attn = true;
+                }
                 kqv = ggml_view_4d(ctx,
                                    kqv,
                                    d_head,

src/core/ggml_extend.h

@@ -154,7 +154,8 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx,
                               int d1              = 1,
                               int d2              = 1,
                               bool force_prec_f32 = false,
-                              bool direct         = false);
+                              bool direct         = false,
+                              float scale         = 1.f);
 
 // w: [OC，IC, KD, 1 * 1]
 // x: [N, IC, ID, IH*IW]
@@ -217,11 +218,12 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
                                     ggml_tensor* k,
                                     ggml_tensor* v,
                                     int64_t n_head,
-                                    ggml_tensor* mask = nullptr,
-                                    bool skip_reshape = false,
-                                    bool flash_attn   = false,
-                                    float kv_scale    = 1.0f,
-                                    bool sage_attn    = false);
+                                    ggml_tensor* mask     = nullptr,
+                                    bool skip_reshape     = false,
+                                    bool flash_attn       = false,
+                                    float kv_scale        = 1.0f,
+                                    bool sage_attn        = false,
+                                    bool* used_flash_attn = nullptr);
 
 ggml_tensor* ggml_ext_layer_norm(ggml_context* ctx,
                                  ggml_tensor* x,

src/core/ggml_extend_backend.cpp

@@ -13,6 +13,7 @@
 #endif
 
 #include "core/util.h"
+#include "ggml-backend-impl.h"
 #include "ggml-impl.h"
 #include "stable-diffusion.h"
 
@@ -433,6 +434,24 @@ bool sd_backend_is_cpu(ggml_backend_t backend) {
     return dev != nullptr && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU;
 }
 
+ggml_backend_buffer_t sd_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device,
+                                                          void* ptr,
+                                                          size_t size,
+                                                          size_t max_tensor_size) {
+    ggml_backend_buffer_t buffer = ggml_backend_dev_buffer_from_host_ptr(device, ptr, size, max_tensor_size);
+    if (buffer != nullptr && buffer->context == nullptr) {
+        ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(device);
+        if (reg != nullptr && std::strcmp(ggml_backend_reg_name(reg), "Metal") == 0) {
+            // Metal can wrap a failed mapping in a non-null buffer. Its free callback also
+            // dereferences the missing context, so only release the outer buffer.
+            buffer->iface.free_buffer = nullptr;
+            ggml_backend_buffer_free(buffer);
+            return nullptr;
+        }
+    }
+    return buffer;
+}
+
 bool sd_backend_supports_cuda_mma(ggml_backend_t backend) {
 #ifdef SD_USE_CUDA
     if (!sd_backend_is(backend, "CUDA")) {

src/core/ggml_extend_backend.h

@@ -88,6 +88,10 @@ private:
 bool sd_backend_is(ggml_backend_t backend, const std::string& name);
 bool sd_backend_is_cpu(ggml_backend_t backend);
 bool sd_backend_supports_cuda_mma(ggml_backend_t backend);
+ggml_backend_buffer_t sd_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device,
+                                                          void* ptr,
+                                                          size_t size,
+                                                          size_t max_tensor_size);
 ggml_backend_t sd_backend_cpu_init();
 bool sd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads);
 ggml_status sd_backend_graph_compute_with_eval_callback(ggml_backend_t backend,

src/core/ggml_runner.cpp

@@ -21,11 +21,12 @@ ggml_tensor* ggml_ext_attention_ext(GGMLRunnerContext* ctx,
                                     ggml_tensor* mask,
                                     bool skip_reshape,
                                     bool flash_attn,
-                                    float kv_scale) {
+                                    float kv_scale,
+                                    bool* used_flash_attn) {
     if (ctx->attn_scale > 0.f) {
         kv_scale = ctx->attn_scale;
     }
-    return ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, mask, skip_reshape, flash_attn, kv_scale, ctx->sage_attn_enabled);
+    return ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, mask, skip_reshape, flash_attn, kv_scale, ctx->sage_attn_enabled, used_flash_attn);
 }
 
 void GGMLRunner::alloc_params_ctx() {
@@ -515,9 +516,10 @@ GGMLRunner::~GGMLRunner() {
     free_params_ctx();
 }
 
-GGMLRunnerContext GGMLRunner::get_context() {
+GGMLRunnerContext GGMLRunner::get_context(ggml_cgraph* graph) {
     GGMLRunnerContext runner_ctx;
     runner_ctx.ggml_ctx              = compute_ctx;
+    runner_ctx.graph                 = graph;
     runner_ctx.backend               = runtime_backend;
     runner_ctx.flash_attn_enabled    = flash_attn_enabled;
     runner_ctx.sage_attn_enabled     = sage_attn_enabled;
@@ -532,8 +534,8 @@ GGMLRunnerContext GGMLRunner::get_context() {
     runner_ctx.get_cache_tensor      = [this](const std::string& name) {
         return this->get_cache_tensor_by_name(name);
     };
-    runner_ctx.cache_tensor = [this](const std::string& name, ggml_tensor* tensor) {
-        this->cache(name, tensor);
+    runner_ctx.cache_tensor = [this, graph](const std::string& name, ggml_tensor* tensor) {
+        this->cache(name, tensor, graph);
     };
     runner_ctx.set_backend_tensor_data = [this](ggml_tensor* tensor, const void* data) {
         this->set_backend_tensor_data(tensor, data);
@@ -575,7 +577,7 @@ ggml_tensor* GGMLRunner::to_backend(ggml_tensor* tensor) {
     }
 }
 
-void GGMLRunner::cache(const std::string name, ggml_tensor* tensor) {
+void GGMLRunner::cache(const std::string name, ggml_tensor* tensor, ggml_cgraph* graph) {
     if (tensor != nullptr && tensor->view_src != nullptr) {
         tensor = ggml_cont(compute_ctx, tensor);
     }
@@ -583,6 +585,10 @@ void GGMLRunner::cache(const std::string name, ggml_tensor* tensor) {
         ggml_set_output(tensor);
     }
     cache_.stage(name, tensor);
+    if (graph != nullptr && tensor != nullptr) {
+        // Schedule the cache output here so its source can be reused before graph end.
+        ggml_build_forward_expand(graph, tensor);
+    }
 }
 
 std::optional<sd::Tensor<float>> GGMLRunner::compute(get_graph_cb_t get_graph,
@@ -590,6 +596,7 @@ std::optional<sd::Tensor<float>> GGMLRunner::compute(get_graph_cb_t get_graph,
                                                      bool auto_runner_end,
                                                      bool no_return,
                                                      const std::function<bool()>& read_outputs) {
+    last_compute_status_ = GGML_STATUS_FAILED;
     if (graph_active_) {
         LOG_ERROR("%s does not support reentrant graph execution", get_desc().c_str());
         return std::nullopt;
@@ -613,7 +620,9 @@ std::optional<sd::Tensor<float>> GGMLRunner::compute(get_graph_cb_t get_graph,
         GGMLRunner& runner;
         const bool& success;
         ~GraphEndGuard() {
-            runner.workspace_.segment_end();
+            if (!runner.workspace_.segment_end()) {
+                runner.last_compute_status_ = GGML_STATUS_FAILED;
+            }
             runner.cache_.graph_end(false);
             runner.cut_cache_.clear();
             runner.free_compute_ctx();
@@ -641,7 +650,12 @@ std::optional<sd::Tensor<float>> GGMLRunner::compute(get_graph_cb_t get_graph,
     std::optional<sd::Tensor<float>> output;
     try {
         output = execute_graph(graph, n_threads, no_return, read_outputs);
+    } catch (const std::bad_alloc&) {
+        last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
+        LOG_ERROR("%s graph allocation failed", get_desc().c_str());
+        return std::nullopt;
     } catch (const std::exception& error) {
+        last_compute_status_ = GGML_STATUS_FAILED;
         LOG_ERROR("%s graph execution failed on %s: %s", get_desc().c_str(),
                   ggml_backend_name(runtime_backend), error.what());
         return std::nullopt;
@@ -649,6 +663,7 @@ std::optional<sd::Tensor<float>> GGMLRunner::compute(get_graph_cb_t get_graph,
     success = output.has_value();
     if (success) {
         cache_.graph_end(true);
+        last_compute_status_ = GGML_STATUS_SUCCESS;
     }
     return output;
 }
@@ -766,6 +781,7 @@ bool GGMLRunner::execute_segment(ggml_cgraph* graph, int n_threads) {
     }
     workspace_.synchronize();
     if (status != GGML_STATUS_SUCCESS) {
+        last_compute_status_ = status;
         LOG_ERROR("%s compute failed: %s", get_desc().c_str(), ggml_status_to_string(status));
         return false;
     }
@@ -818,13 +834,23 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
     const auto& cached_plan     = resolve_graph_cut_plan(graph);
     const auto full_measurement = measure(graph, cached_plan.compute_buffer_size);
     if (full_measurement.buffers.empty()) {
+        last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
         return std::nullopt;
     }
+    auto fits_monolithic = [&]() {
+        // Planning headroom absorbs allocation estimate drift; execution keeps the normal limits.
+        constexpr size_t planning_headroom = 128ULL * 1024ULL * 1024ULL;
+        auto requests                      = memory_requests(full_measurement.buffers, cache_.pending_bytes(graph));
+        for (auto& request : requests) {
+            request.pending_allocation_bytes = add_bytes(request.pending_allocation_bytes, planning_headroom);
+        }
+        return fits(requests, params);
+    };
     auto manager         = residency_manager.lock();
     const bool segmented = !is_multi_device() && !sd_backend_is_cpu(runtime_backend) &&
                            manager != nullptr && manager->segmented_compute_enabled() &&
                            cached_plan.valid && cached_plan.has_cuts && cached_plan.segments.size() > 1 &&
-                           !fits(memory_requests(full_measurement.buffers, cache_.pending_bytes(graph)), params);
+                           !fits_monolithic();
     ggml_graph_cut::Plan monolithic_plan;
     if (!segmented) {
         monolithic_plan.segments.emplace_back();
@@ -888,7 +914,9 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
             SegmentGraphBindings& bindings;
             ggml_context* context;
             ~SegmentCleanup() {
-                runner.workspace_.segment_end();
+                if (!runner.workspace_.segment_end()) {
+                    runner.last_compute_status_ = GGML_STATUS_FAILED;
+                }
                 bindings.restore();
                 weights.segment_end();
                 ggml_free(context);
@@ -898,6 +926,7 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
 
         auto measurement = segmented ? measure(segment_graph, segment.compute_buffer_size) : full_measurement;
         if (!workspace_.prepare(measurement)) {
+            last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
             return fail_segment("workspace preparation");
         }
         const size_t cut_bytes       = last ? 0 : cut_cache_.estimate_output_bytes(graph, segment);
@@ -912,7 +941,11 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
                 sync_runtime_residency();
                 requests = memory_requests(measurement.buffers, new_cache_bytes);
             }
-            return weights.ensure_segment_capacity(index, requests);
+            const bool ready = weights.ensure_segment_capacity(index, requests);
+            if (!ready && manager != nullptr) {
+                last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
+            }
+            return ready;
         };
         if (!weights.segment_start(index, ensure_capacity)) {
             return fail_segment("weight preparation");
@@ -921,12 +954,17 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
         if (!workspace_.measurement_matches(segment_graph, measurement)) {
             measurement = measure(segment_graph, segment.compute_buffer_size);
         }
-        if (!workspace_.prepare(measurement) || !ensure_capacity()) {
+        if (!workspace_.prepare(measurement)) {
+            last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
+            return fail_segment("workspace preparation");
+        }
+        if (!ensure_capacity()) {
             return fail_segment("workspace capacity check");
         }
         if (!workspace_.allocate(segment_graph, [&](ggml_backend_sched_t scheduler, ggml_cgraph* current) {
                 pin_multi_device_nodes(scheduler, current);
             })) {
+            last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
             return fail_segment("workspace allocation");
         }
         for (const auto& size : measurement.buffers) {
@@ -945,10 +983,16 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
         }
         LOG_DEBUG("%s executing segment %zu/%zu: %s", get_desc().c_str(),
                   index + 1, plan.segments.size(), segment.group_name.c_str());
-        if (!execute_segment(segment_graph, n_threads) ||
-            !cache_.capture(segment_graph) ||
-            !cut_cache_.capture(graph, segment, get_desc().c_str())) {
-            return fail_segment("execution or output caching");
+        if (!execute_segment(segment_graph, n_threads)) {
+            return fail_segment("execution");
+        }
+        auto cache_status = cache_.capture(segment_graph);
+        if (cache_status == GGML_STATUS_SUCCESS) {
+            cache_status = cut_cache_.capture(graph, segment, get_desc().c_str());
+        }
+        if (cache_status != GGML_STATUS_SUCCESS) {
+            last_compute_status_ = cache_status;
+            return fail_segment("output caching");
         }
         sync_runtime_residency();
         if (last) {
@@ -964,6 +1008,7 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
             }
         }
         if (!workspace_.segment_end()) {
+            last_compute_status_ = GGML_STATUS_FAILED;
             return fail_segment("workspace synchronization");
         }
         // Final outputs and their callbacks may still be views of consumed cuts.

src/core/ggml_runner.h

@@ -67,6 +67,7 @@ struct WeightAdapter {
 struct GGMLRunnerContext {
     ggml_backend_t backend                                           = nullptr;
     ggml_context* ggml_ctx                                           = nullptr;
+    ggml_cgraph* graph                                               = nullptr;
     bool flash_attn_enabled                                          = false;
     bool sage_attn_enabled                                           = false;
     float linear_scale                                               = 0.f;
@@ -102,6 +103,12 @@ struct GGMLRunnerContext {
         return get_cache_tensor(name);
     }
 
+    void expand_graph(ggml_tensor* tensor) const {
+        if (graph != nullptr && tensor != nullptr) {
+            ggml_build_forward_expand(graph, tensor);
+        }
+    }
+
     void persist_cache_tensor(const std::string& name, ggml_tensor* tensor) const {
         if (!cache_tensor || tensor == nullptr) {
             return;
@@ -122,15 +129,17 @@ ggml_tensor* ggml_ext_attention_ext(GGMLRunnerContext* ctx,
                                     ggml_tensor* k,
                                     ggml_tensor* v,
                                     int64_t n_head,
-                                    ggml_tensor* mask = nullptr,
-                                    bool skip_reshape = false,
-                                    bool flash_attn   = false,
-                                    float kv_scale    = 1.f);
+                                    ggml_tensor* mask     = nullptr,
+                                    bool skip_reshape     = false,
+                                    bool flash_attn       = false,
+                                    float kv_scale        = 1.f,
+                                    bool* used_flash_attn = nullptr);
 
 struct GGMLRunner {
 private:
     std::map<ggml_backend_t, size_t> logged_compute_bytes_;
-    size_t logged_segment_count_ = 0;
+    size_t logged_segment_count_     = 0;
+    ggml_status last_compute_status_ = GGML_STATUS_SUCCESS;
 
     sd::ComputeWorkspace::Measurement measure(ggml_cgraph* graph, size_t direct_bytes);
     std::vector<DeviceMemoryRequest> memory_requests(const std::vector<sd::BackendBufferSize>& sizes,
@@ -288,7 +297,8 @@ public:
 
     virtual ~GGMLRunner();
 
-    virtual GGMLRunnerContext get_context();
+    // Binding a graph schedules cache outputs at registration instead of graph end.
+    virtual GGMLRunnerContext get_context(ggml_cgraph* graph = nullptr);
 
     void reset_compute_ctx();
 
@@ -323,7 +333,7 @@ public:
 
     ggml_tensor* to_backend(ggml_tensor* tensor);
 
-    void cache(const std::string name, ggml_tensor* tensor);
+    void cache(const std::string name, ggml_tensor* tensor, ggml_cgraph* graph = nullptr);
 
     ggml_tensor* get_cache_tensor_by_name(const std::string& name) {
         return cache_.get(name);
@@ -335,6 +345,8 @@ public:
                                              bool no_return                            = false,
                                              const std::function<bool()>& read_outputs = {});
 
+    ggml_status last_compute_status() const { return last_compute_status_; }
+
     void set_flash_attention_enabled(bool enabled) {
         flash_attn_enabled = enabled;
     }

src/core/runner_cache.cpp

@@ -26,10 +26,13 @@ namespace sd {
 
     std::unique_ptr<CachedTensor> CachedTensor::copy(ggml_backend_t backend,
                                                      const std::string& name,
-                                                     ggml_tensor* source) {
+                                                     ggml_tensor* source,
+                                                     ggml_status& status) {
+        status = GGML_STATUS_FAILED;
         if (ggml_graph_cut::tensor_buffer(source) == nullptr) {
             return nullptr;
         }
+        status         = GGML_STATUS_ALLOC_FAILED;
         auto entry     = std::make_unique<CachedTensor>();
         entry->context = ggml_init({2 * ggml_tensor_overhead(), nullptr, true});
         if (entry->context == nullptr) {
@@ -50,6 +53,7 @@ namespace sd {
         } else {
             ggml_backend_tensor_copy(source, entry->tensor);
         }
+        status = GGML_STATUS_SUCCESS;
         return entry;
     }
 
@@ -106,9 +110,9 @@ namespace sd {
         return pending > SIZE_MAX - committed ? SIZE_MAX : committed + pending;
     }
 
-    bool RunnerCache::capture(ggml_cgraph* graph) {
+    ggml_status RunnerCache::capture(ggml_cgraph* graph) {
         if (outputs_.empty()) {
-            return true;
+            return GGML_STATUS_SUCCESS;
         }
         const auto tensors = cache_graph_tensors(graph);
         for (const auto& output : outputs_) {
@@ -116,14 +120,15 @@ namespace sd {
                 continue;
             }
             GGML_ASSERT(ggml_is_contiguous(output.second));
-            auto entry = CachedTensor::copy(backend_, output.first, output.second);
+            ggml_status status;
+            auto entry = CachedTensor::copy(backend_, output.first, output.second, status);
             if (entry == nullptr) {
-                return false;
+                return status;
             }
             pending_[output.first] = std::move(entry);
         }
         ggml_backend_synchronize(backend_);
-        return true;
+        return GGML_STATUS_SUCCESS;
     }
 
     void RunnerCache::graph_end(bool success) {
@@ -180,9 +185,9 @@ namespace sd {
         }
     }
 
-    bool GraphCutTensorCache::capture(ggml_cgraph* graph,
-                                      const ggml_graph_cut::Segment& segment,
-                                      const char* log_desc) {
+    ggml_status GraphCutTensorCache::capture(ggml_cgraph* graph,
+                                             const ggml_graph_cut::Segment& segment,
+                                             const char* log_desc) {
         size_t copied_bytes = 0;
         size_t copied_count = 0;
         for (int index : segment.output_node_indices) {
@@ -191,10 +196,11 @@ namespace sd {
                 !segment.future_cut_names.count(output->name)) {
                 continue;
             }
-            auto entry = CachedTensor::copy(backend_, output->name, ggml_graph_cut::cache_source_tensor(output));
+            ggml_status status;
+            auto entry = CachedTensor::copy(backend_, output->name, ggml_graph_cut::cache_source_tensor(output), status);
             if (entry == nullptr) {
                 LOG_ERROR("%s failed to capture graph cut tensor: %s", log_desc, output->name);
-                return false;
+                return status;
             }
             const size_t size = ggml_backend_buffer_get_size(entry->buffer);
             copied_bytes      = size > SIZE_MAX - copied_bytes ? SIZE_MAX : copied_bytes + size;
@@ -206,6 +212,6 @@ namespace sd {
             LOG_DEBUG("%s graph cut cache added %6.2f MB (%zu tensors)",
                       log_desc, copied_bytes / (1024.f * 1024.f), copied_count);
         }
-        return true;
+        return GGML_STATUS_SUCCESS;
     }
 }

src/core/runner_cache.h

@@ -20,7 +20,8 @@ namespace sd {
         ~CachedTensor();
         static std::unique_ptr<CachedTensor> copy(ggml_backend_t backend,
                                                   const std::string& name,
-                                                  ggml_tensor* source);
+                                                  ggml_tensor* source,
+                                                  ggml_status& status);
     };
     using CachedTensors = std::map<std::string, std::unique_ptr<CachedTensor>>;
 
@@ -41,7 +42,8 @@ namespace sd {
         const std::map<std::string, ggml_tensor*>& outputs() const { return outputs_; }
         size_t pending_bytes(ggml_cgraph* graph) const;
         size_t resident_bytes(ggml_backend_dev_t device) const;
-        bool capture(ggml_cgraph* graph);
+        bool empty() const { return committed_.empty(); }
+        ggml_status capture(ggml_cgraph* graph);
         void graph_end(bool success);
         void clear();
     };
@@ -57,7 +59,7 @@ namespace sd {
         size_t resident_bytes(ggml_backend_dev_t device) const;
         size_t estimate_output_bytes(ggml_cgraph* graph,
                                      const ggml_graph_cut::Segment& segment) const;
-        bool capture(ggml_cgraph* graph, const ggml_graph_cut::Segment& segment, const char* log_desc);
+        ggml_status capture(ggml_cgraph* graph, const ggml_graph_cut::Segment& segment, const char* log_desc);
         void prune(const std::unordered_set<std::string>& keep_names);
         void clear() { tensors_.clear(); }
     };

src/detailer.cpp

@@ -970,6 +970,7 @@ bool adetail_image(adetailer_ctx_t* context,
         generation.pm_params           = {};
         generation.pulid_params        = {};
         generation.hires.enabled       = false;
+        generation.image_preprocess    = {};
         if (params.steps > 0) {
             generation.sample_params.sample_steps        = params.steps;
             generation.sample_params.custom_sigmas       = nullptr;

src/model.h

@@ -62,6 +62,8 @@ enum SDVersion {
     VERSION_SENSENOVA_U1_5,
     VERSION_LLADA_IMAGE,
     VERSION_ESRGAN,
+    VERSION_PIXART,
+    VERSION_MING_IMAGE,
     VERSION_COUNT,
 };
 
@@ -252,6 +254,10 @@ static inline bool sd_version_is_sensenova_u1(SDVersion version) {
     return version == VERSION_SENSENOVA_U1_5;
 }
 
+static inline bool sd_version_is_pixart(SDVersion version) {
+    return version == VERSION_PIXART;
+}
+
 static inline bool sd_version_supports_video_generation(SDVersion version) {
     return version == VERSION_SVD || sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_lingbot_video(version) || sd_version_is_ltxav(version) || sd_version_is_minimax_h3(version);
 }
@@ -275,7 +281,7 @@ static inline bool sd_version_uses_flux2_vae(SDVersion version) {
 }
 
 static inline bool sd_version_uses_wan_vae(SDVersion version) {
-    if (sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_qwen_image(version) || sd_version_is_krea2(version) || sd_version_is_anima(version)) {
+    if (sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_qwen_image(version) || sd_version_is_krea2(version) || sd_version_is_anima(version) || version == VERSION_MING_IMAGE) {
         return true;
     }
     return false;
@@ -309,6 +315,7 @@ static inline bool sd_version_is_dit(SDVersion version) {
         version == VERSION_HIDREAM_O1 ||
         sd_version_is_anima(version) ||
         sd_version_is_z_image(version) ||
+        version == VERSION_MING_IMAGE ||
         sd_version_is_llada_image(version) ||
         sd_version_is_boogu_image(version) ||
         sd_version_is_ernie_image(version) ||
@@ -320,7 +327,8 @@ static inline bool sd_version_is_dit(SDVersion version) {
         sd_version_is_sefi_image(version) ||
         sd_version_is_krea2(version) ||
         sd_version_is_mage_flow(version) ||
-        sd_version_is_sensenova_u1(version)) {
+        sd_version_is_sensenova_u1(version) ||
+        sd_version_is_pixart(version)) {
         return true;
     }
     return false;

src/model/adapter/lora.hpp

@@ -163,6 +163,27 @@ struct LoraModel : public GGMLRunner {
 
             lora_tensors = std::move(new_lora_tensors);
         }
+
+        std::unordered_map<std::string, ggml_tensor*> new_lora_tensors;
+        for (const auto& [old_name, tensor] : lora_tensors) {
+            std::string new_name = old_name;
+            if (starts_with(old_name, "lora.model.diffusion_model.transformer_blocks.")) {
+                // Qwen Image 2.1 stores the gate before the projection in fused MLP weights.
+                for (const auto& suffix : {std::string(".img_mlp.gate_layer.weight."), std::string(".img_mlp.proj.weight.")}) {
+                    size_t pos = old_name.find(suffix);
+                    if (pos == std::string::npos) {
+                        continue;
+                    }
+                    std::string fused_name = old_name.substr(5, pos - 5) + ".img_mlp.gate_up.weight";
+                    if (model_tensor_names.find(fused_name) != model_tensor_names.end()) {
+                        new_name = "lora." + fused_name + (suffix == ".img_mlp.proj.weight." ? ".1." : ".") + old_name.substr(pos + suffix.size());
+                    }
+                    break;
+                }
+            }
+            new_lora_tensors[new_name] = tensor;
+        }
+        lora_tensors = std::move(new_lora_tensors);
     }
 
     ggml_tensor* get_lora_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_backend_t backend) {

src/model/common/ggml_block.hpp

@@ -1,6 +1,7 @@
 #ifndef __SD_MODEL_COMMON_GGML_BLOCK_HPP__
 #define __SD_MODEL_COMMON_GGML_BLOCK_HPP__
 
+#include <cmath>
 #include <cstdint>
 #include <map>
 #include <memory>
@@ -214,7 +215,7 @@ public:
             if (ctx->backend != nullptr) {
                 ggml_tensor* fp8_matmul = ggml_mul_mat(ctx->ggml_ctx, w, x);
                 if (force_prec_f32) {
-                    ggml_mul_mat_set_prec(fp8_matmul, GGML_PREC_F32);
+                    ggml_prec_set_acc(fp8_matmul, GGML_PREC_F32);
                 }
                 supports_fp8_matmul = ggml_backend_supports_op(ctx->backend, fp8_matmul);
             }
@@ -335,15 +336,75 @@ class Embedding : public UnaryBlock {
 protected:
     int64_t embedding_dim;
     int64_t num_embeddings;
+    bool is_int8_tensorwise     = false;
+    int int8_convrot_group_size = 0;
+
+    static void get_rows_i8(ggml_tensor* dst, int ith, int nth, void* userdata) {
+        const auto* embedding     = static_cast<const Embedding*>(userdata);
+        const int group_size      = embedding->int8_convrot_group_size;
+        const auto* weight        = dst->src[0];
+        const auto* input_ids     = static_cast<const int32_t*>(dst->src[1]->data);
+        const auto* scales        = static_cast<const float*>(dst->src[2]->data);
+        const bool scalar_scale   = ggml_nelements(dst->src[2]) == 1;
+        const float normalization = group_size > 0 ? 1.f / std::sqrt(static_cast<float>(group_size)) : 1.f;
+        for (int64_t row = ith; row < dst->ne[1]; row += nth) {
+            const int32_t token = input_ids[row];
+            GGML_ASSERT(token >= 0 && token < weight->ne[1]);
+            const auto* src   = reinterpret_cast<const int8_t*>(static_cast<const char*>(weight->data) + token * weight->nb[1]);
+            auto* out         = reinterpret_cast<float*>(static_cast<char*>(dst->data) + row * dst->nb[1]);
+            const float scale = scales[scalar_scale ? 0 : token] * normalization;
+            for (int64_t i = 0; i < weight->ne[0]; ++i) {
+                out[i] = static_cast<float>(src[i]) * scale;
+            }
+            // The regular Hadamard rotation is symmetric and its own inverse.
+            for (int stride = 1; stride < group_size; stride *= 4) {
+                for (int64_t base = 0; base < weight->ne[0]; base += 4 * stride) {
+                    for (int j = 0; j < stride; ++j) {
+                        float* values      = out + base + j;
+                        const float a      = values[0];
+                        const float b      = values[stride];
+                        const float c      = values[2 * stride];
+                        const float d      = values[3 * stride];
+                        values[0]          = a + b + c - d;
+                        values[stride]     = a + b - c + d;
+                        values[2 * stride] = a - b + c + d;
+                        values[3 * stride] = -a + b + c + d;
+                    }
+                }
+            }
+        }
+    }
+
     void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map, const std::string prefix = "") override {
-        enum ggml_type wtype = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
-        if (!support_get_rows(wtype)) {
+        auto weight_storage     = tensor_storage_map.find(prefix + "weight");
+        is_int8_tensorwise      = weight_storage != tensor_storage_map.end() && weight_storage->second.is_int8_tensorwise;
+        int8_convrot_group_size = 0;
+        enum ggml_type wtype    = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
+        if (is_int8_tensorwise) {
+            GGML_ASSERT(wtype == GGML_TYPE_I8);
+            auto scale_storage = tensor_storage_map.find(prefix + "weight_scale");
+            GGML_ASSERT(scale_storage != tensor_storage_map.end());
+            const int64_t scale_nelements = scale_storage->second.nelements();
+            GGML_ASSERT(scale_nelements == 1 || scale_nelements == num_embeddings);
+            params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, scale_nelements);
+            if (weight_storage->second.int8_convrot) {
+                int8_convrot_group_size = weight_storage->second.int8_convrot_group_size;
+                int remainder           = int8_convrot_group_size;
+                while (remainder > 1 && remainder % 4 == 0) {
+                    remainder /= 4;
+                }
+                GGML_ASSERT(remainder == 1 && embedding_dim % int8_convrot_group_size == 0);
+            }
+        } else if (!support_get_rows(wtype)) {
             wtype = GGML_TYPE_F32;
         }
         params["weight"] = ggml_new_tensor_2d(ctx, wtype, embedding_dim, num_embeddings);
     }
 
     enum ggml_op param_usage_op(const std::string& name) const override {
+        if (is_int8_tensorwise) {
+            return GGML_OP_CUSTOM;
+        }
         return name == "weight" ? GGML_OP_GET_ROWS : GGML_OP_NONE;
     }
 
@@ -363,9 +424,17 @@ public:
         int64_t n = input_ids->ne[1];
         input_ids = ggml_reshape_1d(ctx->ggml_ctx, input_ids, input_ids->ne[0] * input_ids->ne[1]);
 
-        input_ids      = ggml_reshape_3d(ctx->ggml_ctx, input_ids, input_ids->ne[0], 1, input_ids->ne[1]);
-        auto embedding = ggml_get_rows(ctx->ggml_ctx, weight, input_ids);
-        embedding      = ggml_reshape_3d(ctx->ggml_ctx, embedding, embedding->ne[0], embedding->ne[1] / n, n);
+        ggml_tensor* embedding;
+        if (is_int8_tensorwise) {
+            GGML_ASSERT(input_ids->type == GGML_TYPE_I32);
+            ggml_tensor* args[] = {weight, input_ids, params["weight_scale"]};
+            embedding           = ggml_custom_4d(ctx->ggml_ctx, GGML_TYPE_F32, embedding_dim, input_ids->ne[0], 1, 1,
+                                                 args, 3, get_rows_i8, GGML_N_TASKS_MAX, this);
+        } else {
+            input_ids = ggml_reshape_3d(ctx->ggml_ctx, input_ids, input_ids->ne[0], 1, input_ids->ne[1]);
+            embedding = ggml_get_rows(ctx->ggml_ctx, weight, input_ids);
+        }
+        embedding = ggml_reshape_3d(ctx->ggml_ctx, embedding, embedding->ne[0], embedding->ne[1] / n, n);
 
         // [N, n_token, embedding_dim]
         return embedding;

src/model/common/rope.hpp

@@ -5,6 +5,7 @@
 #include <cassert>
 #include <cmath>
 #include <set>
+#include <utility>
 #include <vector>
 #include "core/ggml_extend.h"
 #include "core/ggml_runner.h"
@@ -16,6 +17,45 @@ namespace Rope {
         ErnieImage,
     };
 
+    struct SpatialRegion {
+        size_t begin;
+        size_t count;
+        float height_period;
+        float width_period;
+        int height_axis = 1;
+        int width_axis  = 2;
+    };
+
+    struct PositionLayout {
+        // Token ranges are relative to one batch item.
+        std::vector<SpatialRegion> images;
+        size_t token_count = 0;
+
+        void append_tokens(size_t count) {
+            token_count += count;
+        }
+
+        void append_image(int height, int width, int frames = 1, float height_step = 1.f, float width_step = 1.f) {
+            size_t count = static_cast<size_t>(height) * width * frames;
+            images.push_back({token_count, count, height * height_step, width * width_step});
+            append_tokens(count);
+        }
+    };
+
+    struct Frequency {
+        size_t axis;
+        float omega;
+    };
+
+    struct Embedding {
+        std::vector<float> values;
+        std::vector<std::vector<float>> ids;
+        PositionLayout positions;
+        std::vector<Frequency> frequencies;
+        EmbedNDLayout layout = EmbedNDLayout::Matrix;
+        int batch_size       = 1;
+    };
+
     enum class RefIndexMode {
         FIXED,
         INCREASE,
@@ -56,40 +96,25 @@ namespace Rope {
         return flat_vec;
     }
 
-    __STATIC_INLINE__ std::vector<std::vector<float>> rope(const std::vector<float>& pos,
-                                                           int dim,
-                                                           float theta,
-                                                           const std::vector<int>& axis_wrap_dims = {}) {
+    __STATIC_INLINE__ std::vector<float> rope_frequencies(int dim, float theta) {
         assert(dim % 2 == 0);
-        int half_dim = dim / 2;
-
+        int half_dim             = dim / 2;
         std::vector<float> scale = linspace(0.f, (dim * 1.f - 2) / dim, half_dim);
-
         std::vector<float> omega(half_dim);
         for (int i = 0; i < half_dim; ++i) {
             omega[i] = 1.0f / ::powf(1.f * theta, scale[i]);
         }
+        return omega;
+    }
 
+    __STATIC_INLINE__ std::vector<std::vector<float>> rope(const std::vector<float>& pos,
+                                                           const std::vector<float>& omega) {
+        int half_dim    = static_cast<int>(omega.size());
         size_t pos_size = pos.size();
         std::vector<std::vector<float>> out(pos_size, std::vector<float>(half_dim));
         for (size_t i = 0; i < pos_size; ++i) {
             for (size_t j = 0; j < half_dim; ++j) {
                 float angle = pos[i] * omega[j];
-                if (!axis_wrap_dims.empty()) {
-                    size_t wrap_size = axis_wrap_dims.size();
-                    // mod batch size since we only store this for one item in the batch
-                    size_t wrap_idx = wrap_size > 0 ? (i % wrap_size) : 0;
-                    int wrap_dim    = axis_wrap_dims[wrap_idx];
-                    if (wrap_dim > 0) {
-                        constexpr float TWO_PI = 6.28318530717958647692f;
-                        float cycles           = omega[j] * wrap_dim / TWO_PI;
-                        // closest periodic harmonic, necessary to ensure things neatly tile
-                        // without this round, things don't tile at the boundaries and you end up
-                        // with the model knowing what is "center"
-                        float rounded = std::round(cycles);
-                        angle         = pos[i] * TWO_PI * rounded / wrap_dim;
-                    }
-                }
 
                 out[i][j] = angle;
             }
@@ -108,6 +133,12 @@ namespace Rope {
         return result;
     }
 
+    __STATIC_INLINE__ std::vector<std::vector<float>> rope(const std::vector<float>& pos,
+                                                           int dim,
+                                                           float theta) {
+        return rope(pos, rope_frequencies(dim, theta));
+    }
+
     // Generate IDs for image patches and text
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_flux_txt_ids(int bs, int context_len, int axes_dim_num, std::set<int> arange_dims) {
         auto txt_ids = std::vector<std::vector<float>>(bs * context_len, std::vector<float>(axes_dim_num, 0.0f));
@@ -136,12 +167,16 @@ namespace Rope {
                                                                        int patch_size,
                                                                        int bs,
                                                                        int axes_dim_num,
-                                                                       int index       = 0,
-                                                                       int h_offset    = 0,
-                                                                       int w_offset    = 0,
-                                                                       bool scale_rope = false) {
+                                                                       int index              = 0,
+                                                                       int h_offset           = 0,
+                                                                       int w_offset           = 0,
+                                                                       bool scale_rope        = false,
+                                                                       PositionLayout* layout = nullptr) {
         int h_len = (h + (patch_size / 2)) / patch_size;
         int w_len = (w + (patch_size / 2)) / patch_size;
+        if (layout) {
+            layout->append_image(h_len, w_len);
+        }
         std::vector<std::vector<float>> img_ids(h_len * w_len, std::vector<float>(axes_dim_num, 0.0));
 
         int h_start = h_offset;
@@ -192,8 +227,8 @@ namespace Rope {
                                                   int bs,
                                                   const std::vector<float>& axis_thetas,
                                                   const std::vector<int>& axes_dim,
-                                                  const std::vector<std::vector<int>>& wrap_dims = {},
-                                                  EmbedNDLayout layout                           = EmbedNDLayout::Matrix) {
+                                                  EmbedNDLayout layout                = EmbedNDLayout::Matrix,
+                                                  std::vector<Frequency>* frequencies = nullptr) {
         std::vector<std::vector<float>> trans_ids = transpose(ids);
         size_t pos_len                            = ids.size() / bs;
         size_t num_axes                           = axes_dim.size();
@@ -205,19 +240,25 @@ namespace Rope {
         for (int d : axes_dim)
             emb_dim += d / 2;
 
+        if (frequencies) {
+            frequencies->clear();
+            frequencies->reserve(emb_dim);
+        }
         std::vector<std::vector<float>> emb(bs * pos_len, std::vector<float>(emb_dim * 2 * 2, 0.0));
         size_t offset = 0;
         for (size_t i = 0; i < num_axes; ++i) {
-            std::vector<int> axis_wrap_dims;
-            if (!wrap_dims.empty() && i < (int)wrap_dims.size()) {
-                axis_wrap_dims = wrap_dims[i];
-            }
             float axis_theta = 10000.0f;
             if (!axis_thetas.empty()) {
                 axis_theta = axis_thetas[std::min(i, axis_thetas.size() - 1)];
             }
+            auto omega = rope_frequencies(axes_dim[i], axis_theta);
+            if (frequencies) {
+                for (float frequency : omega) {
+                    frequencies->push_back({i, frequency});
+                }
+            }
             std::vector<std::vector<float>> rope_emb =
-                rope(trans_ids[i], axes_dim[i], axis_theta, axis_wrap_dims);  // [bs*pos_len, axes_dim[i]/2 * 2 * 2]
+                rope(trans_ids[i], omega);  // [bs*pos_len, axes_dim[i]/2 * 2 * 2]
             for (int b = 0; b < bs; ++b) {
                 for (int j = 0; j < pos_len; ++j) {
                     for (int k = 0; k < rope_emb[0].size(); ++k) {
@@ -253,10 +294,10 @@ namespace Rope {
                                                   int bs,
                                                   float theta,
                                                   const std::vector<int>& axes_dim,
-                                                  const std::vector<std::vector<int>>& wrap_dims = {},
-                                                  EmbedNDLayout layout                           = EmbedNDLayout::Matrix) {
+                                                  EmbedNDLayout layout                = EmbedNDLayout::Matrix,
+                                                  std::vector<Frequency>* frequencies = nullptr) {
         std::vector<float> axis_thetas(axes_dim.size(), theta);
-        return embed_nd(ids, bs, axis_thetas, axes_dim, wrap_dims, layout);
+        return embed_nd(ids, bs, axis_thetas, axes_dim, layout, frequencies);
     }
 
     __STATIC_INLINE__ std::vector<float> embed_interleaved_mrope(const std::vector<std::vector<float>>& ids,
@@ -264,7 +305,7 @@ namespace Rope {
                                                                  float theta,
                                                                  int head_dim,
                                                                  const std::vector<int>& mrope_section,
-                                                                 const std::vector<std::vector<int>>& axis_wrap_dims = {}) {
+                                                                 std::vector<Frequency>* frequencies = nullptr) {
         GGML_ASSERT(bs > 0);
         GGML_ASSERT(head_dim % 2 == 0);
         GGML_ASSERT(mrope_section.size() >= 3);
@@ -273,20 +314,26 @@ namespace Rope {
         size_t pos_len                            = ids.size() / bs;
         int half_dim                              = head_dim / 2;
 
+        auto omega = rope_frequencies(head_dim, theta);
+        if (frequencies) {
+            frequencies->clear();
+            for (float frequency : omega) {
+                frequencies->push_back({0, frequency});
+            }
+        }
         std::vector<std::vector<std::vector<float>>> axis_embs;
         axis_embs.reserve(3);
         for (int axis = 0; axis < 3; ++axis) {
-            std::vector<int> axis_wrap;
-            if (axis < static_cast<int>(axis_wrap_dims.size())) {
-                axis_wrap = axis_wrap_dims[axis];
-            }
-            axis_embs.push_back(rope(trans_ids[axis], head_dim, theta, axis_wrap));
+            axis_embs.push_back(rope(trans_ids[axis], omega));
         }
 
         std::vector<std::vector<float>> emb = axis_embs[0];
         for (int axis = 1; axis < 3; ++axis) {
             int length = std::min<int>(mrope_section[axis] * 3, half_dim);
             for (int freq_idx = axis; freq_idx < length; freq_idx += 3) {
+                if (frequencies) {
+                    (*frequencies)[freq_idx].axis = axis;
+                }
                 for (size_t pos_idx = 0; pos_idx < bs * pos_len; ++pos_idx) {
                     for (int k = 0; k < 4; ++k) {
                         emb[pos_idx][4 * freq_idx + k] = axis_embs[axis][pos_idx][4 * freq_idx + k];
@@ -298,13 +345,13 @@ namespace Rope {
         return flatten(emb);
     }
 
-    __STATIC_INLINE__ std::vector<float> embed_2d_interleaved(int height,
-                                                              int width,
-                                                              int dim,
-                                                              float theta    = 10000.f,
-                                                              float scale    = 16.f,
-                                                              int ref_grid_h = 0,
-                                                              int ref_grid_w = 0) {
+    __STATIC_INLINE__ Embedding embed_2d_interleaved(int height,
+                                                     int width,
+                                                     int dim,
+                                                     float theta    = 10000.f,
+                                                     float scale    = 16.f,
+                                                     int ref_grid_h = 0,
+                                                     int ref_grid_w = 0) {
         assert(dim % 4 == 0);
         int half_dim      = dim / 2;
         int dim_axis      = dim / 2;
@@ -318,6 +365,10 @@ namespace Rope {
             w_ntk       = std::pow(static_cast<float>(width) / static_cast<float>(ref_grid_w), power);
         }
 
+        Embedding result;
+        result.positions.append_image(height, width, 1,
+                                      height > 1 ? scale / (height - 1) : 1.f,
+                                      width > 1 ? scale / (width - 1) : 1.f);
         std::vector<float> x_pos;
         std::vector<float> y_pos;
         x_pos.reserve(static_cast<size_t>(height) * width);
@@ -326,13 +377,20 @@ namespace Rope {
             float y = height == 1 ? 0.f : scale * static_cast<float>(iy) / static_cast<float>(height - 1);
             for (int ix = 0; ix < width; ++ix) {
                 float x = width == 1 ? 0.f : scale * static_cast<float>(ix) / static_cast<float>(width - 1);
+                result.ids.push_back({0.f, y, x});
                 x_pos.push_back(x);
                 y_pos.push_back(y);
             }
         }
 
-        auto x_emb = rope(x_pos, dim_axis, theta * w_ntk);
-        auto y_emb = rope(y_pos, dim_axis, theta * h_ntk);
+        auto x_freq = rope_frequencies(dim_axis, theta * w_ntk);
+        auto y_freq = rope_frequencies(dim_axis, theta * h_ntk);
+        auto x_emb  = rope(x_pos, x_freq);
+        auto y_emb  = rope(y_pos, y_freq);
+        for (int i = 0; i < axis_half_dim; ++i) {
+            result.frequencies.push_back({2, x_freq[i]});
+            result.frequencies.push_back({1, y_freq[i]});
+        }
 
         std::vector<float> out(static_cast<size_t>(height) * width * half_dim * 4);
         for (int pos = 0; pos < height * width; ++pos) {
@@ -348,7 +406,8 @@ namespace Rope {
                 }
             }
         }
-        return out;
+        result.values = std::move(out);
+        return result;
     }
 
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_refs_ids(int patch_size,
@@ -359,7 +418,8 @@ namespace Rope {
                                                                    RefIndexMode ref_index_mode,
                                                                    float ref_index_scale,
                                                                    bool scale_rope,
-                                                                   int base_offset = 0) {
+                                                                   int base_offset        = 0,
+                                                                   PositionLayout* layout = nullptr) {
         std::vector<std::vector<float>> ids;
         int curr_h_offset = 0;
         int curr_w_offset = 0;
@@ -386,7 +446,8 @@ namespace Rope {
                                             static_cast<int>(index * ref_index_scale),
                                             h_offset + base_offset,
                                             w_offset + base_offset,
-                                            scale_rope);
+                                            scale_rope,
+                                            layout);
             ids          = concat_ids(ids, ref_ids, bs);
 
             if (ref_index_mode == RefIndexMode::INCREASE) {
@@ -409,88 +470,53 @@ namespace Rope {
                                                                    const std::vector<ggml_tensor*>& ref_latents,
                                                                    RefIndexMode ref_index_mode,
                                                                    float ref_index_scale,
-                                                                   bool is_longcat) {
+                                                                   bool is_longcat,
+                                                                   PositionLayout* layout = nullptr) {
+        if (layout) {
+            layout->append_tokens(context_len);
+        }
         int x_index = is_longcat ? 1 : 0;
 
         auto txt_ids = is_longcat ? gen_longcat_txt_ids(bs, context_len, axes_dim_num) : gen_flux_txt_ids(bs, context_len, axes_dim_num, txt_arange_dims);
         int offset   = is_longcat ? context_len : 0;
-        auto img_ids = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, x_index, offset, offset);
+        auto img_ids = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, x_index, offset, offset, false, layout);
 
         auto ids = concat_ids(txt_ids, img_ids, bs);
         if (ref_latents.size() > 0) {
-            auto refs_ids = gen_refs_ids(patch_size, bs, axes_dim_num, x_index + 1, ref_latents, ref_index_mode, ref_index_scale, false, offset);
+            auto refs_ids = gen_refs_ids(patch_size, bs, axes_dim_num, x_index + 1, ref_latents, ref_index_mode, ref_index_scale, false, offset, layout);
             ids           = concat_ids(ids, refs_ids, bs);
         }
         return ids;
     }
 
     // Generate flux positional embeddings
-    __STATIC_INLINE__ std::vector<float> gen_flux_pe(int h,
-                                                     int w,
-                                                     int patch_size,
-                                                     int bs,
-                                                     int context_len,
-                                                     std::set<int> txt_arange_dims,
-                                                     const std::vector<ggml_tensor*>& ref_latents,
-                                                     RefIndexMode ref_index_mode,
-                                                     float ref_index_scale,
-                                                     int theta,
-                                                     bool circular_h,
-                                                     bool circular_w,
-                                                     const std::vector<int>& axes_dim,
-                                                     bool is_longcat) {
-        std::vector<std::vector<float>> ids = gen_flux_ids(h,
-                                                           w,
-                                                           patch_size,
-                                                           bs,
-                                                           static_cast<int>(axes_dim.size()),
-                                                           context_len,
-                                                           txt_arange_dims,
-                                                           ref_latents,
-                                                           ref_index_mode,
-                                                           ref_index_scale,
-                                                           is_longcat);
-        std::vector<std::vector<int>> wrap_dims;
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            int h_len = (h + (patch_size / 2)) / patch_size;
-            int w_len = (w + (patch_size / 2)) / patch_size;
-            if (h_len > 0 && w_len > 0) {
-                size_t pos_len = ids.size() / bs;
-                wrap_dims.assign(axes_dim.size(), std::vector<int>(pos_len, 0));
-                size_t cursor           = context_len;  // text first
-                const size_t img_tokens = static_cast<size_t>(h_len) * static_cast<size_t>(w_len);
-                for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                    if (circular_h) {
-                        wrap_dims[1][cursor + token_i] = h_len;
-                    }
-                    if (circular_w) {
-                        wrap_dims[2][cursor + token_i] = w_len;
-                    }
-                }
-                cursor += img_tokens;
-                // reference latents
-                for (ggml_tensor* ref : ref_latents) {
-                    if (ref == nullptr) {
-                        continue;
-                    }
-                    int ref_h         = static_cast<int>(ref->ne[1]);
-                    int ref_w         = static_cast<int>(ref->ne[0]);
-                    int ref_h_l       = (ref_h + (patch_size / 2)) / patch_size;
-                    int ref_w_l       = (ref_w + (patch_size / 2)) / patch_size;
-                    size_t ref_tokens = static_cast<size_t>(ref_h_l) * static_cast<size_t>(ref_w_l);
-                    for (size_t token_i = 0; token_i < ref_tokens; ++token_i) {
-                        if (circular_h) {
-                            wrap_dims[1][cursor + token_i] = ref_h_l;
-                        }
-                        if (circular_w) {
-                            wrap_dims[2][cursor + token_i] = ref_w_l;
-                        }
-                    }
-                    cursor += ref_tokens;
-                }
-            }
-        }
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
+    __STATIC_INLINE__ Embedding gen_flux_pe(int h,
+                                            int w,
+                                            int patch_size,
+                                            int bs,
+                                            int context_len,
+                                            std::set<int> txt_arange_dims,
+                                            const std::vector<ggml_tensor*>& ref_latents,
+                                            RefIndexMode ref_index_mode,
+                                            float ref_index_scale,
+                                            int theta,
+                                            const std::vector<int>& axes_dim,
+                                            bool is_longcat) {
+        Embedding result;
+        result.batch_size = bs;
+        result.ids        = gen_flux_ids(h,
+                                         w,
+                                         patch_size,
+                                         bs,
+                                         static_cast<int>(axes_dim.size()),
+                                         context_len,
+                                         txt_arange_dims,
+                                         ref_latents,
+                                         ref_index_mode,
+                                         ref_index_scale,
+                                         is_longcat, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_vid_ids(int t,
@@ -500,14 +526,18 @@ namespace Rope {
                                                                   int ph,
                                                                   int pw,
                                                                   int bs,
-                                                                  int t_offset    = 0,
-                                                                  int h_offset    = 0,
-                                                                  int w_offset    = 0,
-                                                                  bool scale_rope = false) {
+                                                                  int t_offset           = 0,
+                                                                  int h_offset           = 0,
+                                                                  int w_offset           = 0,
+                                                                  bool scale_rope        = false,
+                                                                  PositionLayout* layout = nullptr) {
         int t_len = (t + (pt / 2)) / pt;
         int h_len = (h + (ph / 2)) / ph;
         int w_len = (w + (pw / 2)) / pw;
 
+        if (layout) {
+            layout->append_image(h_len, w_len, t_len);
+        }
         std::vector<std::vector<float>> vid_ids(t_len * h_len * w_len, std::vector<float>(3, 0.0));
 
         if (scale_rope) {
@@ -573,7 +603,11 @@ namespace Rope {
                                                                          int bs,
                                                                          int context_len,
                                                                          const std::vector<ggml_tensor*>& ref_latents,
-                                                                         RefIndexMode ref_index_mode) {
+                                                                         RefIndexMode ref_index_mode,
+                                                                         PositionLayout* layout = nullptr) {
+        if (layout) {
+            layout->append_tokens(context_len);
+        }
         int h_len        = (h + (patch_size / 2)) / patch_size;
         int w_len        = (w + (patch_size / 2)) / patch_size;
         int txt_id_start = std::max(h_len, w_len) / 2;
@@ -585,90 +619,49 @@ namespace Rope {
             }
         }
         int axes_dim_num = 3;
-        auto img_ids     = gen_vid_ids(t, h, w, 1, patch_size, patch_size, bs, 0, 0, 0, true);
+        auto img_ids     = gen_vid_ids(t, h, w, 1, patch_size, patch_size, bs, 0, 0, 0, true, layout);
         auto ids         = concat_ids(txt_ids_repeated, img_ids, bs);
         if (ref_latents.size() > 0) {
             int ref_start_index = ref_index_mode == RefIndexMode::DECREASE ? 0 : 1;
-            auto refs_ids       = gen_refs_ids(patch_size, bs, axes_dim_num, ref_start_index, ref_latents, ref_index_mode, 1.f, true);
+            auto refs_ids       = gen_refs_ids(patch_size, bs, axes_dim_num, ref_start_index, ref_latents, ref_index_mode, 1.f, true, 0, layout);
             ids                 = concat_ids(ids, refs_ids, bs);
         }
         return ids;
     }
 
     // Generate qwen_image positional embeddings
-    __STATIC_INLINE__ std::vector<float> gen_qwen_image_pe(int t,
-                                                           int h,
-                                                           int w,
-                                                           int patch_size,
-                                                           int bs,
-                                                           int context_len,
-                                                           const std::vector<ggml_tensor*>& ref_latents,
-                                                           RefIndexMode ref_index_mode,
-                                                           int theta,
-                                                           bool circular_h,
-                                                           bool circular_w,
-                                                           const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids = gen_qwen_image_ids(t, h, w, patch_size, bs, context_len, ref_latents, ref_index_mode);
-        std::vector<std::vector<int>> wrap_dims;
-        // This logic simply stores the (pad and patch_adjusted) sizes of images so we can make sure rope correctly tiles
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            int pad_h = (patch_size - (h % patch_size)) % patch_size;
-            int pad_w = (patch_size - (w % patch_size)) % patch_size;
-            int h_len = (h + pad_h) / patch_size;
-            int w_len = (w + pad_w) / patch_size;
-            if (h_len > 0 && w_len > 0) {
-                const size_t total_tokens = ids.size();
-                // Track per-token wrap lengths for the row/column axes so only spatial tokens become periodic.
-                wrap_dims.assign(axes_dim.size(), std::vector<int>(total_tokens / bs, 0));
-                size_t cursor           = context_len;  // ignore text tokens
-                const size_t img_tokens = static_cast<size_t>(t) * static_cast<size_t>(h_len) * static_cast<size_t>(w_len);
-                for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                    if (circular_h) {
-                        wrap_dims[1][cursor + token_i] = h_len;
-                    }
-                    if (circular_w) {
-                        wrap_dims[2][cursor + token_i] = w_len;
-                    }
-                }
-                cursor += img_tokens;
-                // For each reference image, store wrap sizes as well
-                for (ggml_tensor* ref : ref_latents) {
-                    if (ref == nullptr) {
-                        continue;
-                    }
-                    int ref_h           = static_cast<int>(ref->ne[1]);
-                    int ref_w           = static_cast<int>(ref->ne[0]);
-                    int ref_pad_h       = (patch_size - (ref_h % patch_size)) % patch_size;
-                    int ref_pad_w       = (patch_size - (ref_w % patch_size)) % patch_size;
-                    int ref_h_len       = (ref_h + ref_pad_h) / patch_size;
-                    int ref_w_len       = (ref_w + ref_pad_w) / patch_size;
-                    size_t ref_n_tokens = static_cast<size_t>(ref_h_len) * static_cast<size_t>(ref_w_len);
-                    for (size_t token_i = 0; token_i < ref_n_tokens; ++token_i) {
-                        if (circular_h) {
-                            wrap_dims[1][cursor + token_i] = ref_h_len;
-                        }
-                        if (circular_w) {
-                            wrap_dims[2][cursor + token_i] = ref_w_len;
-                        }
-                    }
-                    cursor += ref_n_tokens;
-                }
-            }
-        }
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
+    __STATIC_INLINE__ Embedding gen_qwen_image_pe(int t,
+                                                  int h,
+                                                  int w,
+                                                  int patch_size,
+                                                  int bs,
+                                                  int context_len,
+                                                  const std::vector<ggml_tensor*>& ref_latents,
+                                                  RefIndexMode ref_index_mode,
+                                                  int theta,
+                                                  const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.ids        = gen_qwen_image_ids(t, h, w, patch_size, bs, context_len, ref_latents, ref_index_mode, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_mage_flow_pe(int h,
-                                                          int w,
-                                                          int bs,
-                                                          int context_len,
-                                                          const std::vector<ggml_tensor*>& ref_latents,
-                                                          int theta,
-                                                          const std::vector<int>& axes_dim) {
+    __STATIC_INLINE__ Embedding gen_mage_flow_pe(int h,
+                                                 int w,
+                                                 int bs,
+                                                 int context_len,
+                                                 const std::vector<ggml_tensor*>& ref_latents,
+                                                 int theta,
+                                                 const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.positions.append_tokens(context_len);
         const int axes_dim_num = static_cast<int>(axes_dim.size());
-        auto make_image_ids    = [=](int image_h, int image_w, int image_index) {
+        auto make_image_ids    = [=, &result](int image_h, int image_w, int image_index) {
             std::vector<std::vector<float>> image_ids(static_cast<size_t>(bs) * image_h * image_w,
                                                          std::vector<float>(axes_dim_num, 0.f));
+            result.positions.append_image(image_h, image_w);
             int h_start = -(image_h - image_h / 2);
             int w_start = -(image_w - image_w / 2);
             for (int b = 0; b < bs; ++b) {
@@ -692,15 +685,18 @@ namespace Rope {
                                           static_cast<int>(i + 1));
             ids          = concat_ids(ids, ref_ids, bs);
         }
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim);
+        result.ids    = std::move(ids);
+        result.values = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_lens_ids(int h,
                                                                    int w,
                                                                    int bs,
                                                                    int context_len,
-                                                                   bool scale_rope = true) {
-        auto img_ids_repeated = gen_flux_img_ids(h, w, 1, bs, 3, 0, 0, 0, scale_rope);
+                                                                   bool scale_rope        = true,
+                                                                   PositionLayout* layout = nullptr) {
+        auto img_ids_repeated = gen_flux_img_ids(h, w, 1, bs, 3, 0, 0, 0, scale_rope, layout);
 
         int txt_id_start = scale_rope ? std::max(h / 2, w / 2) : 0;
         auto txt_ids     = linspace<float>(1.f * txt_id_start, 1.f * context_len + txt_id_start, context_len);
@@ -711,44 +707,37 @@ namespace Rope {
             }
         }
 
+        if (layout) {
+            layout->append_tokens(context_len);
+        }
         return concat_ids(img_ids_repeated, txt_ids_repeated, bs);
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_lens_pe(int h,
-                                                     int w,
-                                                     int bs,
-                                                     int context_len,
-                                                     int theta,
-                                                     bool circular_h,
-                                                     bool circular_w,
-                                                     const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids = gen_lens_ids(h, w, bs, context_len, true);
-        std::vector<std::vector<int>> wrap_dims;
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            size_t pos_len = ids.size() / bs;
-            wrap_dims.assign(axes_dim.size(), std::vector<int>(pos_len, 0));
-            const size_t img_tokens = static_cast<size_t>(h) * static_cast<size_t>(w);
-            for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                if (circular_h) {
-                    wrap_dims[1][token_i] = h;
-                }
-                if (circular_w) {
-                    wrap_dims[2][token_i] = w;
-                }
-            }
-        }
-
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
+    __STATIC_INLINE__ Embedding gen_lens_pe(int h,
+                                            int w,
+                                            int bs,
+                                            int context_len,
+                                            int theta,
+                                            const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.ids        = gen_lens_ids(h, w, bs, context_len, true, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_ernie_image_ids(int h,
                                                                           int w,
                                                                           int patch_size,
                                                                           int bs,
-                                                                          int context_len) {
+                                                                          int context_len,
+                                                                          PositionLayout* layout = nullptr) {
         int h_len = h / patch_size;
         int w_len = w / patch_size;
 
+        if (layout) {
+            layout->append_image(h_len, w_len);
+        }
         std::vector<std::vector<float>> img_ids(h_len * w_len, std::vector<float>(3, 0.0f));
         std::vector<float> h_ids = linspace<float>(0.f, static_cast<float>(h_len - 1), h_len);
         std::vector<float> w_ids = linspace<float>(0.f, static_cast<float>(w_len - 1), w_len);
@@ -774,39 +763,25 @@ namespace Rope {
             }
         }
 
+        if (layout) {
+            layout->append_tokens(context_len);
+        }
         return concat_ids(img_ids_repeated, txt_ids, bs);
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_ernie_image_pe(int h,
-                                                            int w,
-                                                            int patch_size,
-                                                            int bs,
-                                                            int context_len,
-                                                            int theta,
-                                                            bool circular_h,
-                                                            bool circular_w,
-                                                            const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids = gen_ernie_image_ids(h, w, patch_size, bs, context_len);
-        std::vector<std::vector<int>> wrap_dims;
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            int h_len = h / patch_size;
-            int w_len = w / patch_size;
-            if (h_len > 0 && w_len > 0) {
-                size_t pos_len = ids.size() / bs;
-                wrap_dims.assign(axes_dim.size(), std::vector<int>(pos_len, 0));
-                const size_t img_tokens = static_cast<size_t>(h_len) * static_cast<size_t>(w_len);
-                for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                    if (circular_h) {
-                        wrap_dims[1][token_i] = h_len;
-                    }
-                    if (circular_w) {
-                        wrap_dims[2][token_i] = w_len;
-                    }
-                }
-            }
-        }
-
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims, EmbedNDLayout::ErnieImage);
+    __STATIC_INLINE__ Embedding gen_ernie_image_pe(int h,
+                                                   int w,
+                                                   int patch_size,
+                                                   int bs,
+                                                   int context_len,
+                                                   int theta,
+                                                   const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.layout     = EmbedNDLayout::ErnieImage;
+        result.ids        = gen_ernie_image_ids(h, w, patch_size, bs, context_len, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     // Generate wan positional embeddings
@@ -905,7 +880,8 @@ namespace Rope {
                                                                       int context_len,
                                                                       int seq_multi_of,
                                                                       const std::vector<ggml_tensor*>& ref_latents,
-                                                                      RefIndexMode ref_index_mode) {
+                                                                      RefIndexMode ref_index_mode,
+                                                                      PositionLayout* layout = nullptr) {
         SD_UNUSED(ref_index_mode);
         int padded_context_len = context_len + bound_mod(context_len, seq_multi_of);
         auto txt_ids           = std::vector<std::vector<float>>(bs * padded_context_len, std::vector<float>(3, 0.0f));
@@ -913,11 +889,17 @@ namespace Rope {
             txt_ids[i][0] = (i % padded_context_len) + 1.f;
         }
 
+        if (layout) {
+            layout->append_tokens(padded_context_len);
+        }
         int axes_dim_num = 3;
         int index        = padded_context_len + 1;
-        auto img_ids     = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, index);
+        auto img_ids     = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, index, 0, 0, false, layout);
 
         int img_pad_len = bound_mod(static_cast<int>(img_ids.size() / bs), seq_multi_of);
+        if (layout) {
+            layout->append_tokens(img_pad_len);
+        }
         if (img_pad_len > 0) {
             std::vector<std::vector<float>> img_pad_ids(bs * img_pad_len, std::vector<float>(3, 0.f));
             img_ids = concat_ids(img_ids, img_pad_ids, bs);
@@ -936,7 +918,8 @@ namespace Rope {
                                                                           int patch_size,
                                                                           int bs,
                                                                           int context_len,
-                                                                          int seq_multi_of) {
+                                                                          int seq_multi_of,
+                                                                          PositionLayout* layout = nullptr) {
         int context_pad_len    = bound_mod(context_len, seq_multi_of);
         int padded_context_len = context_len + context_pad_len;
         auto txt_ids           = std::vector<std::vector<float>>(bs * padded_context_len, std::vector<float>(3, 0.0f));
@@ -947,11 +930,17 @@ namespace Rope {
             }
         }
 
+        if (layout) {
+            layout->append_tokens(padded_context_len);
+        }
         int axes_dim_num = 3;
         int index        = padded_context_len + 1;
-        auto img_ids     = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, index);
+        auto img_ids     = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, index, 0, 0, false, layout);
 
         int img_pad_len = bound_mod(static_cast<int>(img_ids.size() / bs), seq_multi_of);
+        if (layout) {
+            layout->append_tokens(img_pad_len);
+        }
         if (img_pad_len > 0) {
             std::vector<std::vector<float>> img_pad_ids(bs * img_pad_len, std::vector<float>(3, 0.f));
             img_ids = concat_ids(img_ids, img_pad_ids, bs);
@@ -968,7 +957,8 @@ namespace Rope {
                                                                                int patch_size,
                                                                                int context_len,
                                                                                int sigvq_len,
-                                                                               int seq_multi_of) {
+                                                                               int seq_multi_of,
+                                                                               PositionLayout* layout = nullptr) {
         const int context_pad    = bound_mod(context_len, seq_multi_of);
         const int padded_context = context_len + context_pad;
         const int h_len          = (h + (patch_size / 2)) / patch_size;
@@ -994,11 +984,17 @@ namespace Rope {
             cursor += 2;
         }
 
+        if (layout) {
+            layout->append_tokens(cap_ids.size());
+        }
         std::vector<std::vector<float>> img_ids;
         for (int copy = 0; copy < 2; ++copy) {
-            auto ids = gen_flux_img_ids(h, w, patch_size, 1, 3, cap_end_positions[copy]);
+            auto ids = gen_flux_img_ids(h, w, patch_size, 1, 3, cap_end_positions[copy], 0, 0, false, layout);
             img_ids.insert(img_ids.end(), ids.begin(), ids.end());
             img_ids.insert(img_ids.end(), image_pad, std::vector<float>(3, 0.f));
+            if (layout) {
+                layout->append_tokens(image_pad);
+            }
         }
 
         const int sigvq_start = static_cast<int>(cap_ids.size() + img_ids.size()) + 1;
@@ -1016,95 +1012,59 @@ namespace Rope {
         ids.insert(ids.end(), cap_ids.begin(), cap_ids.end());
         ids.insert(ids.end(), img_ids.begin(), img_ids.end());
         ids.insert(ids.end(), sigvq_ids.begin(), sigvq_ids.end());
+        if (layout) {
+            layout->append_tokens(sigvq_ids.size());
+        }
         SD_UNUSED(padded_image);
         return ids;
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_llada_image_edit_pe(int h,
-                                                                 int w,
-                                                                 int patch_size,
-                                                                 int context_len,
-                                                                 int sigvq_len,
-                                                                 int seq_multi_of,
-                                                                 int theta,
-                                                                 const std::vector<int>& axes_dim) {
-        auto ids = gen_llada_image_edit_ids(h, w, patch_size, context_len, sigvq_len, seq_multi_of);
-        return embed_nd(ids, 1, static_cast<float>(theta), axes_dim, {});
+    __STATIC_INLINE__ Embedding gen_llada_image_edit_pe(int h,
+                                                        int w,
+                                                        int patch_size,
+                                                        int context_len,
+                                                        int sigvq_len,
+                                                        int seq_multi_of,
+                                                        int theta,
+                                                        const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = 1;
+        result.ids        = gen_llada_image_edit_ids(h, w, patch_size, context_len, sigvq_len, seq_multi_of, &result.positions);
+        result.values     = embed_nd(result.ids, 1, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_llada_image_pe(int h,
-                                                            int w,
-                                                            int patch_size,
-                                                            int bs,
-                                                            int context_len,
-                                                            int seq_multi_of,
-                                                            int theta,
-                                                            bool circular_h,
-                                                            bool circular_w,
-                                                            const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids = gen_llada_image_ids(h, w, patch_size, bs, context_len, seq_multi_of);
-        std::vector<std::vector<int>> wrap_dims;
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            int pad_h = (patch_size - (h % patch_size)) % patch_size;
-            int pad_w = (patch_size - (w % patch_size)) % patch_size;
-            int h_len = (h + pad_h) / patch_size;
-            int w_len = (w + pad_w) / patch_size;
-            if (h_len > 0 && w_len > 0) {
-                size_t pos_len = ids.size() / bs;
-                wrap_dims.assign(axes_dim.size(), std::vector<int>(pos_len, 0));
-                size_t cursor     = context_len + bound_mod(context_len, seq_multi_of);
-                size_t img_tokens = static_cast<size_t>(h_len) * static_cast<size_t>(w_len);
-                for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                    if (circular_h) {
-                        wrap_dims[1][cursor + token_i] = h_len;
-                    }
-                    if (circular_w) {
-                        wrap_dims[2][cursor + token_i] = w_len;
-                    }
-                }
-            }
-        }
-
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
+    __STATIC_INLINE__ Embedding gen_llada_image_pe(int h,
+                                                   int w,
+                                                   int patch_size,
+                                                   int bs,
+                                                   int context_len,
+                                                   int seq_multi_of,
+                                                   int theta,
+                                                   const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.ids        = gen_llada_image_ids(h, w, patch_size, bs, context_len, seq_multi_of, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     // Generate z_image positional embeddings
-    __STATIC_INLINE__ std::vector<float> gen_z_image_pe(int h,
-                                                        int w,
-                                                        int patch_size,
-                                                        int bs,
-                                                        int context_len,
-                                                        int seq_multi_of,
-                                                        const std::vector<ggml_tensor*>& ref_latents,
-                                                        RefIndexMode ref_index_mode,
-                                                        int theta,
-                                                        bool circular_h,
-                                                        bool circular_w,
-                                                        const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids = gen_z_image_ids(h, w, patch_size, bs, context_len, seq_multi_of, ref_latents, ref_index_mode);
-        std::vector<std::vector<int>> wrap_dims;
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            int pad_h = (patch_size - (h % patch_size)) % patch_size;
-            int pad_w = (patch_size - (w % patch_size)) % patch_size;
-            int h_len = (h + pad_h) / patch_size;
-            int w_len = (w + pad_w) / patch_size;
-            if (h_len > 0 && w_len > 0) {
-                size_t pos_len = ids.size() / bs;
-                wrap_dims.assign(axes_dim.size(), std::vector<int>(pos_len, 0));
-                size_t cursor     = context_len + bound_mod(context_len, seq_multi_of);  // skip text (and its padding)
-                size_t img_tokens = static_cast<size_t>(h_len) * static_cast<size_t>(w_len);
-                for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                    if (circular_h) {
-                        wrap_dims[1][cursor + token_i] = h_len;
-                    }
-                    if (circular_w) {
-                        wrap_dims[2][cursor + token_i] = w_len;
-                    }
-                }
-            }
-        }
-
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
+    __STATIC_INLINE__ Embedding gen_z_image_pe(int h,
+                                               int w,
+                                               int patch_size,
+                                               int bs,
+                                               int context_len,
+                                               int seq_multi_of,
+                                               const std::vector<ggml_tensor*>& ref_latents,
+                                               RefIndexMode ref_index_mode,
+                                               int theta,
+                                               const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.ids        = gen_z_image_ids(h, w, patch_size, bs, context_len, seq_multi_of, ref_latents, ref_index_mode, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     __STATIC_INLINE__ ggml_tensor* apply_rope(ggml_context* ctx,

src/model/common/rope_circular.hpp

@@ -0,0 +1,65 @@
+#ifndef __SD_MODEL_COMMON_ROPE_CIRCULAR_HPP__
+#define __SD_MODEL_COMMON_ROPE_CIRCULAR_HPP__
+
+#include "model/common/rope.hpp"
+
+namespace Rope {
+    __STATIC_INLINE__ void apply_circular(Embedding& embedding, bool circular_x, bool circular_y) {
+        if (!circular_x && !circular_y) {
+            return;
+        }
+
+        GGML_ASSERT(embedding.batch_size > 0);
+        GGML_ASSERT(embedding.ids.size() % embedding.batch_size == 0);
+        size_t pos_len  = embedding.ids.size() / embedding.batch_size;
+        size_t half_dim = embedding.frequencies.size();
+        GGML_ASSERT(embedding.positions.token_count == pos_len);
+        GGML_ASSERT(embedding.values.size() == embedding.ids.size() * half_dim * 4);
+
+        constexpr float TWO_PI = 6.28318530717958647692f;
+        for (const auto& region : embedding.positions.images) {
+            GGML_ASSERT(region.begin <= pos_len && region.count <= pos_len - region.begin);
+            for (size_t j = 0; j < half_dim; ++j) {
+                const auto& frequency = embedding.frequencies[j];
+                float period          = 0.f;
+                if (circular_y && frequency.axis == static_cast<size_t>(region.height_axis)) {
+                    period = region.height_period;
+                } else if (circular_x && frequency.axis == static_cast<size_t>(region.width_axis)) {
+                    period = region.width_period;
+                }
+                if (period <= 0) {
+                    continue;
+                }
+
+                // Quantize to periodic harmonics while preserving the original coordinate offsets.
+                float rounded = std::round(frequency.omega * period / TWO_PI);
+                for (int b = 0; b < embedding.batch_size; ++b) {
+                    size_t begin = b * pos_len + region.begin;
+                    for (size_t i = begin; i < begin + region.count; ++i) {
+                        GGML_ASSERT(frequency.axis < embedding.ids[i].size());
+                        float angle   = embedding.ids[i][frequency.axis] * TWO_PI * rounded / period;
+                        float cos_val = std::cos(angle);
+                        float sin_val = std::sin(angle);
+                        if (embedding.layout == EmbedNDLayout::ErnieImage) {
+                            size_t cos_offset                = (i * half_dim + j) * 2;
+                            size_t sin_offset                = embedding.ids.size() * half_dim * 2 + cos_offset;
+                            embedding.values[cos_offset]     = cos_val;
+                            embedding.values[cos_offset + 1] = cos_val;
+                            embedding.values[sin_offset]     = sin_val;
+                            embedding.values[sin_offset + 1] = sin_val;
+                        } else {
+                            size_t offset                = (i * half_dim + j) * 4;
+                            embedding.values[offset]     = cos_val;
+                            embedding.values[offset + 1] = -sin_val;
+                            embedding.values[offset + 2] = sin_val;
+                            embedding.values[offset + 3] = cos_val;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+}  // namespace Rope
+
+#endif  // __SD_MODEL_COMMON_ROPE_CIRCULAR_HPP__

src/model/diffusion/anima.hpp

@@ -603,34 +603,37 @@ namespace Anima {
             return std::pow(extrapolation_ratio, static_cast<float>(axis_dim) / static_cast<float>(axis_dim - 2));
         }
 
-        static std::vector<float> gen_anima_image_pe_vec(int bs,
-                                                         int h,
-                                                         int w,
-                                                         int patch_size,
-                                                         int theta,
-                                                         const std::vector<int>& axes_dim,
-                                                         float h_extrapolation_ratio,
-                                                         float w_extrapolation_ratio,
-                                                         float t_extrapolation_ratio,
-                                                         const std::vector<ggml_tensor*>& ref_latents) {
-            auto ids = Rope::gen_flux_ids(h,
-                                          w,
-                                          patch_size,
-                                          bs,
-                                          static_cast<int>(axes_dim.size()),
-                                          0,
-                                          {},
-                                          ref_latents,
-                                          Rope::RefIndexMode::FIXED,
-                                          1.0f,
-                                          false);
+        static Rope::Embedding gen_anima_image_pe_vec(int bs,
+                                                      int h,
+                                                      int w,
+                                                      int patch_size,
+                                                      int theta,
+                                                      const std::vector<int>& axes_dim,
+                                                      float h_extrapolation_ratio,
+                                                      float w_extrapolation_ratio,
+                                                      float t_extrapolation_ratio,
+                                                      const std::vector<ggml_tensor*>& ref_latents) {
+            Rope::Embedding result;
+            result.batch_size = bs;
+            result.ids        = Rope::gen_flux_ids(h,
+                                                   w,
+                                                   patch_size,
+                                                   bs,
+                                                   static_cast<int>(axes_dim.size()),
+                                                   0,
+                                                   {},
+                                                   ref_latents,
+                                                   Rope::RefIndexMode::FIXED,
+                                                   1.0f,
+                                                   false, &result.positions);
 
             std::vector<float> axis_thetas = {
                 static_cast<float>(theta) * calc_ntk_factor(t_extrapolation_ratio, axes_dim[0]),
                 static_cast<float>(theta) * calc_ntk_factor(h_extrapolation_ratio, axes_dim[1]),
                 static_cast<float>(theta) * calc_ntk_factor(w_extrapolation_ratio, axes_dim[2]),
             };
-            return Rope::embed_nd(ids, bs, axis_thetas, axes_dim);
+            result.values = Rope::embed_nd(result.ids, bs, axis_thetas, axes_dim, result.layout, &result.frequencies);
+            return result;
         }
 
         ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
@@ -657,16 +660,16 @@ namespace Anima {
             int64_t h_pad = x->ne[1] + pad_h;
             int64_t w_pad = x->ne[0] + pad_w;
 
-            image_pe_vec          = gen_anima_image_pe_vec(1,
-                                                           static_cast<int>(h_pad),
-                                                           static_cast<int>(w_pad),
-                                                           static_cast<int>(config.patch_size),
-                                                           config.theta,
-                                                           config.axes_dim,
-                                                           4.0f,
-                                                           4.0f,
-                                                           1.0f,
-                                                           ref_latents);
+            image_pe_vec          = finish_rope_pe(gen_anima_image_pe_vec(1,
+                                                                          static_cast<int>(h_pad),
+                                                                          static_cast<int>(w_pad),
+                                                                          static_cast<int>(config.patch_size),
+                                                                          config.theta,
+                                                                          config.axes_dim,
+                                                                          4.0f,
+                                                                          4.0f,
+                                                                          1.0f,
+                                                                          ref_latents));
             int64_t image_pos_len = static_cast<int64_t>(image_pe_vec.size()) / (2 * 2 * (config.head_dim / 2));
             auto image_pe         = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.head_dim / 2, image_pos_len);
             set_backend_tensor_data(image_pe, image_pe_vec.data());

src/model/diffusion/boogu.hpp

@@ -720,15 +720,18 @@ namespace Boogu {
         }
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_boogu_pe(int h,
-                                                      int w,
-                                                      int patch_size,
-                                                      int bs,
-                                                      int context_len,
-                                                      const std::vector<ggml_tensor*>& ref_latents,
-                                                      int theta,
-                                                      const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids;
+    __STATIC_INLINE__ Rope::Embedding gen_boogu_pe(int h,
+                                                   int w,
+                                                   int patch_size,
+                                                   int bs,
+                                                   int context_len,
+                                                   const std::vector<ggml_tensor*>& ref_latents,
+                                                   int theta,
+                                                   const std::vector<int>& axes_dim) {
+        Rope::Embedding result;
+        result.batch_size = bs;
+        result.positions.append_tokens(context_len);
+        auto& ids = result.ids;
         ids.reserve(static_cast<size_t>(bs) * context_len);
         for (int b = 0; b < bs; b++) {
             for (int i = 0; i < context_len; i++) {
@@ -741,15 +744,18 @@ namespace Boogu {
         for (ggml_tensor* ref : ref_latents) {
             int ref_h_tokens = patched_token_count(ref->ne[1], patch_size);
             int ref_w_tokens = patched_token_count(ref->ne[0], patch_size);
+            result.positions.append_image(ref_h_tokens, ref_w_tokens);
             append_spatial_ids(ids, bs, pe_shift, ref_h_tokens, ref_w_tokens);
             pe_shift += std::max(ref_h_tokens, ref_w_tokens);
         }
 
         int h_tokens = patched_token_count(h, patch_size);
         int w_tokens = patched_token_count(w, patch_size);
+        result.positions.append_image(h_tokens, w_tokens);
         append_spatial_ids(ids, bs, pe_shift, h_tokens, w_tokens);
 
-        return Rope::embed_nd(ids, bs, static_cast<float>(theta), axes_dim);
+        result.values = Rope::embed_nd(ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     struct BooguImageRunner : public DiffusionModelRunner {
@@ -793,14 +799,14 @@ namespace Boogu {
                 ref_latents.push_back(make_input(ref_latent_tensor));
             }
 
-            pe_vec      = gen_boogu_pe(static_cast<int>(x->ne[1]),
-                                       static_cast<int>(x->ne[0]),
-                                       config.patch_size,
-                                       static_cast<int>(x->ne[3]),
-                                       static_cast<int>(context->ne[1]),
-                                       ref_latents,
-                                       config.theta,
-                                       config.axes_dim);
+            pe_vec      = finish_rope_pe(gen_boogu_pe(static_cast<int>(x->ne[1]),
+                                                      static_cast<int>(x->ne[0]),
+                                                      config.patch_size,
+                                                      static_cast<int>(x->ne[3]),
+                                                      static_cast<int>(context->ne[1]),
+                                                      ref_latents,
+                                                      config.theta,
+                                                      config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());

src/model/diffusion/control.hpp

@@ -376,7 +376,7 @@ struct ControlNet : public GGMLRunner {
             hint = make_input(hint_tensor);
         }
 
-        auto runner_ctx = get_context();
+        auto runner_ctx = get_context(gf);
 
         auto outs = control_net.forward(&runner_ctx,
                                         x,
@@ -389,8 +389,7 @@ struct ControlNet : public GGMLRunner {
         if (guided_hint_input == nullptr && !outs.empty()) {
             guided_hint_output_ggml = outs[0];
             ggml_set_output(guided_hint_output_ggml);
-            cache(guided_hint_cache_name(), guided_hint_output_ggml);
-            ggml_build_forward_expand(gf, guided_hint_output_ggml);
+            runner_ctx.persist_cache_tensor(guided_hint_cache_name(), guided_hint_output_ggml);
         }
 
         control_outputs_ggml.reserve(outs.size() > 0 ? outs.size() - 1 : 0);

src/model/diffusion/ernie_image.hpp

@@ -415,15 +415,13 @@ namespace ErnieImage {
             GGML_ASSERT(!context_tensor.empty());
             ggml_tensor* context = make_input(context_tensor);
 
-            pe_vec      = Rope::gen_ernie_image_pe(static_cast<int>(x->ne[1]),
-                                                   static_cast<int>(x->ne[0]),
-                                                   config.patch_size,
-                                                   static_cast<int>(x->ne[3]),
-                                                   static_cast<int>(context->ne[1]),
-                                                   config.theta,
-                                                   circular_y_enabled,
-                                                   circular_x_enabled,
-                                                   config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_ernie_image_pe(static_cast<int>(x->ne[1]),
+                                                                  static_cast<int>(x->ne[0]),
+                                                                  config.patch_size,
+                                                                  static_cast<int>(x->ne[3]),
+                                                                  static_cast<int>(context->ne[1]),
+                                                                  config.theta,
+                                                                  config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, config.axes_dim_sum, 1, pos_len, 2);
             set_backend_tensor_data(pe, pe_vec.data());

src/model/diffusion/flux.hpp

@@ -1548,20 +1548,18 @@ namespace Flux {
             } else if (version == VERSION_OVIS_IMAGE) {
                 txt_arange_dims = {1, 2};
             }
-            pe_vec      = Rope::gen_flux_pe(static_cast<int>(x->ne[1]),
-                                            static_cast<int>(x->ne[0]),
-                                            config.patch_size,
-                                            static_cast<int>(x->ne[3]),
-                                            static_cast<int>(context->ne[1]),
-                                            txt_arange_dims,
-                                            ref_latents,
-                                            ref_index_mode,
-                                            config.ref_index_scale,
-                                            config.theta,
-                                            circular_y_enabled,
-                                            circular_x_enabled,
-                                            config.axes_dim,
-                                            sd_version_is_longcat(version));
+            pe_vec      = finish_rope_pe(Rope::gen_flux_pe(static_cast<int>(x->ne[1]),
+                                                           static_cast<int>(x->ne[0]),
+                                                           config.patch_size,
+                                                           static_cast<int>(x->ne[3]),
+                                                           static_cast<int>(context->ne[1]),
+                                                           txt_arange_dims,
+                                                           ref_latents,
+                                                           ref_index_mode,
+                                                           config.ref_index_scale,
+                                                           config.theta,
+                                                           config.axes_dim,
+                                                           sd_version_is_longcat(version)));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             // LOG_VERBOSE("pos_len %d", pos_len);
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);

src/model/diffusion/ideogram4.hpp

@@ -149,18 +149,21 @@ namespace Ideogram4 {
         return std::make_shared<Linear>(in_features, out_features, bias);
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_ideogram4_pe(int grid_h,
-                                                          int grid_w,
-                                                          int bs,
-                                                          int context_len,
-                                                          int head_dim,
-                                                          int rope_theta,
-                                                          const std::vector<int>& mrope_section,
-                                                          bool circular_x = false,
-                                                          bool circular_y = false) {
+    __STATIC_INLINE__ Rope::Embedding gen_ideogram4_pe(int grid_h,
+                                                       int grid_w,
+                                                       int bs,
+                                                       int context_len,
+                                                       int head_dim,
+                                                       int rope_theta,
+                                                       const std::vector<int>& mrope_section) {
         GGML_ASSERT(bs == 1);
-        std::vector<std::vector<float>> ids(static_cast<size_t>(bs) * (context_len + grid_h * grid_w),
-                                            std::vector<float>(3, 0.f));
+        Rope::Embedding result;
+        result.batch_size = bs;
+        result.positions.append_tokens(context_len);
+        result.positions.append_image(grid_h, grid_w);
+        result.ids.assign(static_cast<size_t>(bs) * (context_len + grid_h * grid_w),
+                          std::vector<float>(3, 0.f));
+        auto& ids = result.ids;
 
         for (int i = 0; i < context_len; ++i) {
             ids[i] = {static_cast<float>(i), static_cast<float>(i), static_cast<float>(i)};
@@ -175,29 +178,13 @@ namespace Ideogram4 {
             }
         }
 
-        std::vector<std::vector<int>> axis_wrap_dims(3);
-        if (circular_y || circular_x) {
-            size_t total_len = static_cast<size_t>(bs) * (context_len + grid_h * grid_w);
-            axis_wrap_dims[1].assign(total_len, 0);
-            axis_wrap_dims[2].assign(total_len, 0);
-            if (circular_y) {
-                for (size_t idx = static_cast<size_t>(context_len); idx < total_len; ++idx) {
-                    axis_wrap_dims[1][idx] = grid_h;
-                }
-            }
-            if (circular_x) {
-                for (size_t idx = static_cast<size_t>(context_len); idx < total_len; ++idx) {
-                    axis_wrap_dims[2][idx] = grid_w;
-                }
-            }
-        }
-
-        return Rope::embed_interleaved_mrope(ids,
-                                             bs,
-                                             static_cast<float>(rope_theta),
-                                             head_dim,
-                                             mrope_section,
-                                             axis_wrap_dims);
+        result.values = Rope::embed_interleaved_mrope(ids,
+                                                      bs,
+                                                      static_cast<float>(rope_theta),
+                                                      head_dim,
+                                                      mrope_section,
+                                                      &result.frequencies);
+        return result;
     }
 
     class Ideogram4Attention : public GGMLBlock {
@@ -509,15 +496,13 @@ namespace Ideogram4 {
             int64_t head_dim = config.emb_dim / config.num_heads;
 
             auto runner_ctx = get_context();
-            pe_vec          = gen_ideogram4_pe(static_cast<int>(grid_h),
-                                               static_cast<int>(grid_w),
-                                               static_cast<int>(x->ne[3]),
-                                               static_cast<int>(context_len),
-                                               static_cast<int>(head_dim),
-                                               static_cast<int>(config.rope_theta),
-                                               config.mrope_section,
-                                               runner_ctx.circular_x_enabled,
-                                               runner_ctx.circular_y_enabled);
+            pe_vec          = finish_rope_pe(gen_ideogram4_pe(static_cast<int>(grid_h),
+                                                              static_cast<int>(grid_w),
+                                                              static_cast<int>(x->ne[3]),
+                                                              static_cast<int>(context_len),
+                                                              static_cast<int>(head_dim),
+                                                              static_cast<int>(config.rope_theta),
+                                                              config.mrope_section));
             auto pe         = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, head_dim / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());
 

src/model/diffusion/krea2.hpp

@@ -689,23 +689,28 @@ namespace Krea2 {
         }
     };
 
-    __STATIC_INLINE__ std::vector<float> gen_krea2_pe(int h,
-                                                      int w,
-                                                      int patch_size,
-                                                      int bs,
-                                                      int context_len,
-                                                      float theta,
-                                                      const std::vector<int>& axes_dim,
-                                                      const std::vector<ggml_tensor*>& ref_latents,
-                                                      Rope::RefIndexMode ref_index_mode) {
+    __STATIC_INLINE__ Rope::Embedding gen_krea2_pe(int h,
+                                                   int w,
+                                                   int patch_size,
+                                                   int bs,
+                                                   int context_len,
+                                                   float theta,
+                                                   const std::vector<int>& axes_dim,
+                                                   const std::vector<ggml_tensor*>& ref_latents,
+                                                   Rope::RefIndexMode ref_index_mode) {
+        Rope::Embedding result;
+        result.batch_size = bs;
+        result.positions.append_tokens(context_len);
         auto txt_ids = Rope::gen_flux_txt_ids(bs, context_len, 3, {});
-        auto img_ids = Rope::gen_flux_img_ids(h, w, patch_size, bs, 3, 0, 0, 0, false);
+        auto img_ids = Rope::gen_flux_img_ids(h, w, patch_size, bs, 3, 0, 0, 0, false, &result.positions);
         auto ids     = Rope::concat_ids(txt_ids, img_ids, bs);
         if (ref_latents.size() > 0) {
-            auto refs_ids = Rope::gen_refs_ids(patch_size, bs, 3, 1, ref_latents, ref_index_mode, 1.0f, false, 0);
+            auto refs_ids = Rope::gen_refs_ids(patch_size, bs, 3, 1, ref_latents, ref_index_mode, 1.0f, false, 0, &result.positions);
             ids           = Rope::concat_ids(ids, refs_ids, bs);
         }
-        return Rope::embed_nd(ids, bs, theta, axes_dim);
+        result.ids    = std::move(ids);
+        result.values = Rope::embed_nd(result.ids, bs, theta, axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     struct Krea2Runner : public DiffusionModelRunner {
@@ -749,15 +754,15 @@ namespace Krea2 {
                 ref_latents.push_back(make_input(ref_latent_tensor));
             }
 
-            pe_vec      = gen_krea2_pe(static_cast<int>(x->ne[1]),
-                                       static_cast<int>(x->ne[0]),
-                                       config.patch_size,
-                                       static_cast<int>(x->ne[3]),
-                                       static_cast<int>(context->ne[1]),
-                                       config.theta,
-                                       config.axes_dim,
-                                       ref_latents,
-                                       ref_image_params.ref_index_mode);
+            pe_vec      = finish_rope_pe(gen_krea2_pe(static_cast<int>(x->ne[1]),
+                                                      static_cast<int>(x->ne[0]),
+                                                      config.patch_size,
+                                                      static_cast<int>(x->ne[3]),
+                                                      static_cast<int>(context->ne[1]),
+                                                      config.theta,
+                                                      config.axes_dim,
+                                                      ref_latents,
+                                                      ref_image_params.ref_index_mode));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());

src/model/diffusion/lens.hpp

@@ -384,14 +384,12 @@ namespace Lens {
             GGML_ASSERT(!context_tensor.empty());
             ggml_tensor* context = make_input(context_tensor);
 
-            pe_vec      = Rope::gen_lens_pe(static_cast<int>(x->ne[1]),
-                                            static_cast<int>(x->ne[0]),
-                                            static_cast<int>(x->ne[3]),
-                                            static_cast<int>(context->ne[1]),
-                                            config.theta,
-                                            circular_y_enabled,
-                                            circular_x_enabled,
-                                            config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_lens_pe(static_cast<int>(x->ne[1]),
+                                                           static_cast<int>(x->ne[0]),
+                                                           static_cast<int>(x->ne[3]),
+                                                           static_cast<int>(context->ne[1]),
+                                                           config.theta,
+                                                           config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());

src/model/diffusion/llada_image.hpp

@@ -412,16 +412,14 @@ namespace LLaDAImage {
             GGML_ASSERT(!context_tensor.empty());
             ggml_tensor* context = make_input(context_tensor);
 
-            pe_vec      = Rope::gen_llada_image_pe(static_cast<int>(x->ne[1]),
-                                                   static_cast<int>(x->ne[0]),
-                                                   config.patch_size,
-                                                   static_cast<int>(x->ne[3]),
-                                                   static_cast<int>(context->ne[1]),
-                                                   ZImage::SEQ_MULTI_OF,
-                                                   config.theta,
-                                                   circular_y_enabled,
-                                                   circular_x_enabled,
-                                                   config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_llada_image_pe(static_cast<int>(x->ne[1]),
+                                                                  static_cast<int>(x->ne[0]),
+                                                                  config.patch_size,
+                                                                  static_cast<int>(x->ne[3]),
+                                                                  static_cast<int>(context->ne[1]),
+                                                                  ZImage::SEQ_MULTI_OF,
+                                                                  config.theta,
+                                                                  config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());
@@ -461,14 +459,14 @@ namespace LLaDAImage {
             ggml_tensor* source    = make_input(source_tensor);
             GGML_ASSERT(x->ne[3] == 1);
 
-            pe_vec      = Rope::gen_llada_image_edit_pe(static_cast<int>(x->ne[1]),
-                                                        static_cast<int>(x->ne[0]),
-                                                        config.patch_size,
-                                                        static_cast<int>(context->ne[1]),
-                                                   semantic != nullptr ? static_cast<int>(semantic->ne[1]) : 0,
-                                                        ZImage::SEQ_MULTI_OF,
-                                                        config.theta,
-                                                        config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_llada_image_edit_pe(static_cast<int>(x->ne[1]),
+                                                                       static_cast<int>(x->ne[0]),
+                                                                       config.patch_size,
+                                                                       static_cast<int>(context->ne[1]),
+                                                                  semantic != nullptr ? static_cast<int>(semantic->ne[1]) : 0,
+                                                                       ZImage::SEQ_MULTI_OF,
+                                                                       config.theta,
+                                                                       config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());

src/model/diffusion/mage_flow.hpp

@@ -110,13 +110,13 @@ namespace MageFlow {
             }
 
             int batch_size = static_cast<int>(x->ne[3]);
-            pe_vec         = Rope::gen_mage_flow_pe(static_cast<int>(x->ne[1]),
-                                                    static_cast<int>(x->ne[0]),
-                                                    batch_size,
-                                                    static_cast<int>(context->ne[1]),
-                                                    ref_latents,
-                                                    config.theta,
-                                                    config.axes_dim);
+            pe_vec         = finish_rope_pe(Rope::gen_mage_flow_pe(static_cast<int>(x->ne[1]),
+                                                                   static_cast<int>(x->ne[0]),
+                                                                   batch_size,
+                                                                   static_cast<int>(context->ne[1]),
+                                                                   ref_latents,
+                                                                   config.theta,
+                                                                   config.axes_dim));
             int pos_len    = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe        = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());

src/model/diffusion/ming_image.hpp

@@ -0,0 +1,135 @@
+#ifndef __SD_MODEL_DIFFUSION_MING_IMAGE_HPP__
+#define __SD_MODEL_DIFFUSION_MING_IMAGE_HPP__
+
+#include "z_image.hpp"
+
+namespace MingImage {
+    struct MingImageConfig : ZImage::ZImageConfig {
+        bool split_qkv = true;
+
+        static MingImageConfig detect_from_weights(const String2TensorStorage& tensors, const std::string& prefix) {
+            MingImageConfig config;
+            static_cast<ZImage::ZImageConfig&>(config) = ZImage::ZImageConfig::detect_from_weights(tensors, prefix);
+            config.split_qkv                           = tensors.count(prefix + ".layers.0.attention.qkv.weight") == 0;
+            return config;
+        }
+    };
+
+    class MingImageModel : public GGMLBlock {
+        MingImageConfig config;
+
+    public:
+        explicit MingImageModel(const MingImageConfig& config)
+            : config(config) {
+            blocks["x_embedder"]     = std::make_shared<Linear>(config.patch_size * config.patch_size * config.in_channels, config.hidden_size);
+            blocks["t_embedder"]     = std::make_shared<TimestepEmbedder>(1024, 256, std::min<int64_t>(config.hidden_size, 256));
+            blocks["cap_embedder.0"] = std::make_shared<RMSNorm>(config.cap_feat_dim, config.norm_eps);
+            blocks["cap_embedder.1"] = std::make_shared<Linear>(config.cap_feat_dim, config.hidden_size);
+            auto add_blocks          = [&](const std::string& prefix, int64_t count, bool modulation) {
+                for (int64_t i = 0; i < count; ++i) {
+                    blocks[prefix + std::to_string(i)] = std::make_shared<ZImage::JointTransformerBlock>(
+                        static_cast<int>(i), config.hidden_size, config.head_dim, config.num_heads,
+                        config.num_kv_heads, config.multiple_of, config.ffn_dim_multiplier,
+                        config.norm_eps, config.qk_norm, modulation, true, config.split_qkv, 1e-5f);
+                }
+            };
+            add_blocks("noise_refiner.", config.num_refiner_layers, true);
+            add_blocks("context_refiner.", config.num_refiner_layers, false);
+            add_blocks("layers.", config.num_layers, true);
+            blocks["final_layer"] = std::make_shared<ZImage::FinalLayer>(config.hidden_size, config.patch_size, config.out_channels);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* timestep, ggml_tensor* context, ggml_tensor* direct, ggml_tensor* pe) {
+            auto gctx           = ctx->ggml_ctx;
+            const int64_t width = x->ne[0], height = x->ne[1];
+            auto img            = DiT::pad_and_patchify(ctx, x, config.patch_size, config.patch_size, false);
+            img                 = std::dynamic_pointer_cast<Linear>(blocks["x_embedder"])->forward(ctx, img);
+            auto txt            = std::dynamic_pointer_cast<RMSNorm>(blocks["cap_embedder.0"])->forward(ctx, context);
+            txt                 = std::dynamic_pointer_cast<Linear>(blocks["cap_embedder.1"])->forward(ctx, txt);
+            txt                 = ggml_concat(gctx, txt, direct, 1);
+            auto t              = std::dynamic_pointer_cast<TimestepEmbedder>(blocks["t_embedder"])->forward(ctx, timestep);
+            const int64_t n_txt = txt->ne[1], n_img = img->ne[1];
+            auto txt_pe = ggml_ext_slice(gctx, pe, 3, 0, n_txt);
+            auto img_pe = ggml_ext_slice(gctx, pe, 3, n_txt, n_txt + n_img);
+            for (int64_t i = 0; i < config.num_refiner_layers; ++i) {
+                txt = std::dynamic_pointer_cast<ZImage::JointTransformerBlock>(blocks["context_refiner." + std::to_string(i)])->forward(ctx, txt, txt_pe);
+                sd::ggml_graph_cut::mark_graph_cut(txt, "ming_image.context_refiner." + std::to_string(i), "txt");
+            }
+            for (int64_t i = 0; i < config.num_refiner_layers; ++i) {
+                img = std::dynamic_pointer_cast<ZImage::JointTransformerBlock>(blocks["noise_refiner." + std::to_string(i)])->forward(ctx, img, img_pe, nullptr, t);
+                sd::ggml_graph_cut::mark_graph_cut(img, "ming_image.noise_refiner." + std::to_string(i), "img");
+            }
+            auto combined = ggml_concat(gctx, txt, img, 1);
+            for (int64_t i = 0; i < config.num_layers; ++i) {
+                combined = std::dynamic_pointer_cast<ZImage::JointTransformerBlock>(blocks["layers." + std::to_string(i)])->forward(ctx, combined, pe, nullptr, t);
+                sd::ggml_graph_cut::mark_graph_cut(combined, "ming_image.layers." + std::to_string(i), "combined");
+            }
+            img = ggml_ext_slice(gctx, combined, 1, n_txt, n_txt + n_img);
+            img = std::dynamic_pointer_cast<ZImage::FinalLayer>(blocks["final_layer"])->forward(ctx, img, t);
+            img = DiT::unpatchify_and_crop(gctx, img, height, width, config.patch_size, config.patch_size, false);
+            return ggml_scale(gctx, img, -1.f);
+        }
+    };
+
+    struct MingImageRunner : DiffusionModelRunner {
+        MingImageConfig config;
+        MingImageModel model;
+        std::vector<float> pe_values;
+
+        MingImageRunner(ggml_backend_t backend, const String2TensorStorage& tensors, const std::string& prefix, std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+            : DiffusionModelRunner(backend, prefix, weight_manager),
+              config(MingImageConfig::detect_from_weights(tensors, prefix)),
+              model(config) {
+            model.init(params_ctx, tensors, prefix);
+        }
+
+        std::string get_desc() override { return "ming_image"; }
+
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
+            model.get_param_tensors(tensors, prefix);
+        }
+
+        sd::Tensor<float> compute(int n_threads, const DiffusionParams& inputs) override {
+            const auto* extra = diffusion_extra_as<MingImageDiffusionExtra>(inputs);
+            if (inputs.ref_latents != nullptr && !inputs.ref_latents->empty()) {
+                LOG_ERROR("Ming-Image reference-image conditioning is not supported");
+                return {};
+            }
+            if (inputs.context == nullptr || extra->direct_context == nullptr) {
+                LOG_ERROR("Ming-Image requires both query and direct text conditions");
+                return {};
+            }
+            auto graph = [&]() {
+                auto gf      = new_graph_custom(ZImage::Z_IMAGE_GRAPH_SIZE);
+                auto x       = make_input(*inputs.x);
+                auto t       = make_input(*inputs.timesteps);
+                auto context = make_input(*inputs.context);
+                auto direct  = make_input(*extra->direct_context);
+                GGML_ASSERT(x->ne[3] == 1);
+                const int64_t n_txt = context->ne[1] + direct->ne[1];
+                const int64_t n_img = ((x->ne[0] + config.patch_size - 1) / config.patch_size) *
+                                      ((x->ne[1] + config.patch_size - 1) / config.patch_size);
+                auto padded = finish_rope_pe(Rope::gen_z_image_pe(
+                    static_cast<int>(x->ne[1]), static_cast<int>(x->ne[0]), config.patch_size, 1,
+                    static_cast<int>(n_txt), ZImage::SEQ_MULTI_OF, {}, Rope::RefIndexMode::FIXED,
+                    config.theta, config.axes_dim));
+                // Zero-masked alignment tokens cannot affect valid queries. Omit them while
+                // retaining the padded caption length used to position image tokens.
+                const size_t stride      = config.axes_dim_sum * 2;
+                const int64_t padded_txt = n_txt + Rope::bound_mod(static_cast<int>(n_txt), ZImage::SEQ_MULTI_OF);
+                pe_values.assign(padded.begin(), padded.begin() + n_txt * stride);
+                pe_values.insert(pe_values.end(), padded.begin() + padded_txt * stride,
+                                 padded.begin() + (padded_txt + n_img) * stride);
+                auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, n_txt + n_img);
+                set_backend_tensor_data(pe, pe_values.data());
+                auto ctx = get_context();
+                auto out = model.forward(&ctx, x, t, context, direct, pe);
+                ggml_build_forward_expand(gf, out);
+                return gf;
+            };
+            return restore_trailing_singleton_dims(GGMLRunner::compute(graph, n_threads, false), inputs.x->dim());
+        }
+    };
+}
+
+#endif  // __SD_MODEL_DIFFUSION_MING_IMAGE_HPP__

src/model/diffusion/minimax_h3.hpp

@@ -264,6 +264,9 @@ namespace MiniMaxH3 {
             for (int64_t i = 0; i < num_layers; ++i) {
                 auto block = std::dynamic_pointer_cast<TokenRefinerBlock>(blocks["blocks." + std::to_string(i)]);
                 x          = block->forward(ctx, x);
+                sd::ggml_graph_cut::mark_graph_cut(x,
+                                                   "minimax_h3.token_refiner.blocks." + std::to_string(i),
+                                                   "hidden_states");
             }
             return std::dynamic_pointer_cast<RMSNorm>(blocks["final_norm"])->forward(ctx, x);
         }
@@ -527,7 +530,11 @@ namespace MiniMaxH3 {
             GGML_ASSERT(context->ne[0] == config.text_dim);
             auto condition_proj = std::dynamic_pointer_cast<Linear>(blocks["condition_proj"]);
             auto token_refiner  = std::dynamic_pointer_cast<TokenRefiner>(blocks["token_refiner"]);
-            return token_refiner->forward(ctx, condition_proj->forward(ctx, context));
+            auto projected      = condition_proj->forward(ctx, context);
+            sd::ggml_graph_cut::mark_graph_cut(projected,
+                                               "minimax_h3.condition_proj",
+                                               "hidden_states");
+            return token_refiner->forward(ctx, projected);
         }
 
         ggml_tensor* time_embedding(GGMLRunnerContext* ctx,

src/model/diffusion/minit2i.hpp

@@ -154,18 +154,26 @@ namespace MiniT2I {
         return Rope::flatten(Rope::rope(Rope::linspace(0.f, static_cast<float>(length - 1), length), head_dim, 10000.f));
     }
 
-    inline std::vector<float> make_vision_rope(int side, int head_dim) {
+    inline Rope::Embedding make_vision_rope(int side, int head_dim) {
         GGML_ASSERT(head_dim % 4 == 0);
         int dim     = head_dim / 2;
         int quarter = dim / 2;
         int length  = side * side;
+        Rope::Embedding result;
+        result.positions.append_image(side, side);
         std::vector<float> out(static_cast<size_t>(length) * (head_dim / 2) * 4);
         std::vector<float> freqs(quarter);
         for (int i = 0; i < quarter; ++i) {
             freqs[i] = 1.0f / std::pow(10000.0f, static_cast<float>(2 * i) / static_cast<float>(dim));
         }
+        for (int axis : {1, 2}) {
+            for (float frequency : freqs) {
+                result.frequencies.push_back({static_cast<size_t>(axis), frequency});
+            }
+        }
         for (int y = 0; y < side; ++y) {
             for (int x = 0; x < side; ++x) {
+                result.ids.push_back({0.f, static_cast<float>(y), static_cast<float>(x)});
                 int pos     = y * side + x;
                 size_t base = static_cast<size_t>(pos) * (head_dim / 2) * 4;
                 for (int i = 0; i < quarter; ++i) {
@@ -182,7 +190,8 @@ namespace MiniT2I {
                 }
             }
         }
-        return out;
+        result.values = std::move(out);
+        return result;
     }
 
     struct SwiGLUMlp : public GGMLBlock {
@@ -475,6 +484,8 @@ namespace MiniT2I {
         int64_t cached_txt_len                      = -1;
         int64_t cached_hidden_size                  = -1;
         int64_t cached_head_dim                     = -1;
+        bool cached_circular_x                      = false;
+        bool cached_circular_y                      = false;
 
         MiniT2IRunner(ggml_backend_t backend,
                       const String2TensorStorage& tensor_storage_map      = {},
@@ -521,6 +532,8 @@ namespace MiniT2I {
                 cached_txt_len == txt_len &&
                 cached_hidden_size == config.hidden_size &&
                 cached_head_dim == config.head_dim &&
+                cached_circular_x == circular_x_enabled &&
+                cached_circular_y == circular_y_enabled &&
                 cached_pos_embed != nullptr &&
                 cached_txt_pe != nullptr &&
                 cached_joint_pe != nullptr) {
@@ -531,7 +544,7 @@ namespace MiniT2I {
 
             auto pos_embed_vec = make_2d_sincos_pos_embed(static_cast<int>(img_side), static_cast<int>(config.hidden_size));
             auto txt_pe_vec    = make_text_rope(static_cast<int>(txt_len), static_cast<int>(config.head_dim));
-            auto img_pe_vec    = make_vision_rope(static_cast<int>(img_side), static_cast<int>(config.head_dim));
+            auto img_pe_vec    = finish_rope_pe(make_vision_rope(static_cast<int>(img_side), static_cast<int>(config.head_dim)));
             auto joint_pe_vec  = txt_pe_vec;
             joint_pe_vec.insert(joint_pe_vec.end(), img_pe_vec.begin(), img_pe_vec.end());
 
@@ -561,6 +574,8 @@ namespace MiniT2I {
             cached_txt_len     = txt_len;
             cached_hidden_size = config.hidden_size;
             cached_head_dim    = config.head_dim;
+            cached_circular_x  = circular_x_enabled;
+            cached_circular_y  = circular_y_enabled;
         }
 
         ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,

src/model/diffusion/model.hpp

@@ -7,7 +7,7 @@
 
 #include "core/ggml_runner.h"
 #include "core/tensor_ggml.hpp"
-#include "model/common/rope.hpp"
+#include "model/common/rope_circular.hpp"
 #include "model_manager.h"
 
 enum class RefImageResizeMode {
@@ -71,6 +71,8 @@ struct AnimaDiffusionExtra {
 
 struct QwenImage21DiffusionExtra {
     const sd::Tensor<int32_t>* image_slots = nullptr;
+    // Nonzero IDs identify immutable prefix inputs within one sampling run.
+    uint64_t prefix_id = 0;
 };
 
 struct WanDiffusionExtra {
@@ -139,6 +141,10 @@ struct LLaDAImageDiffusionExtra {
     const sd::Tensor<float>* semantic = nullptr;
 };
 
+struct MingImageDiffusionExtra {
+    const sd::Tensor<float>* direct_context = nullptr;
+};
+
 using DiffusionExtraParams = std::variant<std::monostate,
                                           UNetDiffusionExtra,
                                           SkipLayerDiffusionExtra,
@@ -152,7 +158,8 @@ using DiffusionExtraParams = std::variant<std::monostate,
                                           MiniT2IDiffusionExtra,
                                           SenseNovaU1DiffusionExtra,
                                           HunyuanVideoDiffusionExtra,
-                                          LLaDAImageDiffusionExtra>;
+                                          LLaDAImageDiffusionExtra,
+                                          MingImageDiffusionExtra>;
 
 struct DiffusionParams {
     const sd::Tensor<float>* x                        = nullptr;
@@ -182,6 +189,11 @@ struct DiffusionModelRunner : public GGMLRunner {
 protected:
     std::string prefix;
 
+    std::vector<float> finish_rope_pe(Rope::Embedding embedding) {
+        Rope::apply_circular(embedding, circular_x_enabled, circular_y_enabled);
+        return std::move(embedding.values);
+    }
+
 public:
     DiffusionModelRunner(ggml_backend_t backend,
                          const std::string& prefix,

src/model/diffusion/pid.hpp

@@ -135,13 +135,13 @@ namespace Pid {
         return Rope::flatten(Rope::rope(Rope::linspace(0.f, static_cast<float>(length - 1), length), dim, theta));
     }
 
-    inline std::vector<float> make_rope_2d(int height,
-                                           int width,
-                                           int dim,
-                                           float theta    = 10000.f,
-                                           float scale    = 16.f,
-                                           int ref_grid_h = 0,
-                                           int ref_grid_w = 0) {
+    inline Rope::Embedding make_rope_2d(int height,
+                                        int width,
+                                        int dim,
+                                        float theta    = 10000.f,
+                                        float scale    = 16.f,
+                                        int ref_grid_h = 0,
+                                        int ref_grid_w = 0) {
         GGML_ASSERT(dim % 4 == 0);
         return Rope::embed_2d_interleaved(height, width, dim, theta, scale, ref_grid_h, ref_grid_w);
     }
@@ -867,13 +867,13 @@ namespace Pid {
             int64_t Hs = Hp / config.patch_size;
             int64_t Ws = Wp / config.patch_size;
 
-            pos_img_vec  = make_rope_2d(static_cast<int>(Hs),
-                                        static_cast<int>(Ws),
-                                        static_cast<int>(config.hidden_size / config.num_groups),
-                                        10000.f,
-                                        16.f,
-                                        static_cast<int>(config.rope_ref_grid_h),
-                                        static_cast<int>(config.rope_ref_grid_w));
+            pos_img_vec  = finish_rope_pe(make_rope_2d(static_cast<int>(Hs),
+                                                       static_cast<int>(Ws),
+                                                       static_cast<int>(config.hidden_size / config.num_groups),
+                                                       10000.f,
+                                                       16.f,
+                                                       static_cast<int>(config.rope_ref_grid_h),
+                                                       static_cast<int>(config.rope_ref_grid_w)));
             auto pos_img = ggml_new_tensor_4d(compute_ctx,
                                               GGML_TYPE_F32,
                                               2,
@@ -904,13 +904,13 @@ namespace Pid {
                                                 1);
             set_backend_tensor_data(pixel_pos, pixel_pos_vec.data());
 
-            pixel_pos_comp_vec  = make_rope_2d(static_cast<int>(Hs),
-                                               static_cast<int>(Ws),
-                                               static_cast<int>(config.pixel_attn_hidden_size / config.pixel_num_groups),
-                                               10000.f,
-                                               16.f,
-                                               static_cast<int>(config.rope_ref_grid_h),
-                                               static_cast<int>(config.rope_ref_grid_w));
+            pixel_pos_comp_vec  = finish_rope_pe(make_rope_2d(static_cast<int>(Hs),
+                                                              static_cast<int>(Ws),
+                                                              static_cast<int>(config.pixel_attn_hidden_size / config.pixel_num_groups),
+                                                              10000.f,
+                                                              16.f,
+                                                              static_cast<int>(config.rope_ref_grid_h),
+                                                              static_cast<int>(config.rope_ref_grid_w)));
             auto pixel_pos_comp = ggml_new_tensor_4d(compute_ctx,
                                                      GGML_TYPE_F32,
                                                      2,

src/model/diffusion/pixart.hpp

@@ -0,0 +1,399 @@
+#ifndef __SD_MODEL_DIFFUSION_PIXART_HPP__
+#define __SD_MODEL_DIFFUSION_PIXART_HPP__
+
+#include <algorithm>
+#include <cinttypes>
+#include <cmath>
+#include <vector>
+
+#include "core/ggml_extend.h"
+#include "core/ggml_runner.h"
+#include "core/util.h"
+#include "model/common/ggml_block.hpp"
+#include "model/diffusion/dit.hpp"
+#include "model/diffusion/mmdit.hpp"
+#include "model/diffusion/model.hpp"
+#include "model_loader.h"
+
+// Ref: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/pixart_transformer_2d.py
+// Ref: https://github.com/PixArt-alpha/PixArt-sigma
+
+namespace PixArt {
+    constexpr int PIXART_GRAPH_SIZE = 20480;
+    constexpr int ADALN_EMBED_DIM   = 256;
+
+    struct PixArtConfig {
+        int64_t in_channels         = 4;
+        int64_t out_channels        = 8;  // learn_sigma: noise prediction + learned variance
+        int64_t hidden_size         = 1152;
+        int64_t cross_attention_dim = 1152;
+        int64_t caption_channels    = 4096;
+        int64_t num_heads           = 16;
+        int64_t patch_size          = 2;
+        int64_t ffn_dim             = 4608;
+        int64_t pos_embed_base_size = 64;
+        float interpolation_scale   = 2.f;
+        int num_layers              = 28;
+
+        static PixArtConfig detect_from_weights(const String2TensorStorage& weights, const std::string& prefix) {
+            PixArtConfig config;
+            auto find = [&](const std::string& suffix) -> const TensorStorage* {
+                auto it = weights.find(prefix + "." + suffix);
+                return it == weights.end() ? nullptr : &it->second;
+            };
+            if (auto w = find("x_embedder.proj.weight")) {
+                config.hidden_size = w->ne[3];
+                config.in_channels = w->ne[2];
+                config.patch_size  = w->ne[0];
+            }
+            if (auto w = find("final_layer.linear.weight")) {
+                config.out_channels = w->ne[1] / (config.patch_size * config.patch_size);
+            }
+            if (auto w = find("y_embedder.y_proj.fc1.weight")) {
+                config.caption_channels = w->ne[0];
+            }
+            if (auto w = find("blocks.0.cross_attn.kv_linear.weight")) {
+                config.cross_attention_dim = w->ne[0];
+            }
+            if (auto w = find("blocks.0.mlp.fc1.weight")) {
+                config.ffn_dim = w->ne[1];
+            }
+            if (find("csize_embedder.mlp.0.weight") != nullptr) {
+                LOG_WARN("pixart: resolution/aspect-ratio micro conditions are not supported; output may differ from the reference");
+            }
+            int layers                     = 0;
+            const std::string block_prefix = prefix + ".blocks.";
+            for (const auto& [name, _] : weights) {
+                if (starts_with(name, block_prefix)) {
+                    layers = std::max(layers, atoi(name.substr(block_prefix.size()).c_str()) + 1);
+                }
+            }
+            if (layers > 0) {
+                config.num_layers = layers;
+                LOG_VERBOSE("pixart: layers = %d, hidden_size = %" PRId64,
+                            layers, config.hidden_size);
+            }
+            return config;
+        }
+    };
+
+    // Mirrors diffusers get_2d_sincos_pos_embed for a (gh, gw) patch grid.
+    static std::vector<float> gen_2d_sincos_pos_embed(int64_t dim,
+                                                      int64_t gh,
+                                                      int64_t gw,
+                                                      int64_t base_size,
+                                                      float interpolation_scale) {
+        // diffusers: meshgrid(grid_w, grid_h, indexing="xy") -> grid[0]=w, grid[1]=h,
+        // embedding = concat(sincos(w), sincos(h))
+        std::vector<float> out(static_cast<size_t>(gh) * gw * dim);
+        int64_t quarter = dim / 4;
+        for (int64_t h = 0; h < gh; ++h) {
+            float pos_h = static_cast<float>(h) / (static_cast<float>(gh) / base_size) / interpolation_scale;
+            for (int64_t w = 0; w < gw; ++w) {
+                float pos_w  = static_cast<float>(w) / (static_cast<float>(gw) / base_size) / interpolation_scale;
+                float* dst_w = out.data() + (h * gw + w) * dim;
+                float* dst_h = dst_w + dim / 2;
+                for (int64_t i = 0; i < quarter; ++i) {
+                    float omega        = 1.f / powf(10000.f, static_cast<float>(i) / quarter);
+                    dst_w[i]           = sinf(pos_w * omega);
+                    dst_w[i + quarter] = cosf(pos_w * omega);
+                    dst_h[i]           = sinf(pos_h * omega);
+                    dst_h[i + quarter] = cosf(pos_h * omega);
+                }
+            }
+        }
+        return out;
+    }
+
+    class PixArtTimestepEmbedding : public GGMLBlock {
+    public:
+        PixArtTimestepEmbedding(int64_t in_channels, int64_t out_dim) {
+            blocks["mlp.0"] = std::make_shared<Linear>(in_channels, out_dim);
+            blocks["mlp.2"] = std::make_shared<Linear>(out_dim, out_dim);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            x = std::dynamic_pointer_cast<Linear>(blocks["mlp.0"])->forward(ctx, x);
+            x = ggml_silu(ctx->ggml_ctx, x);
+            return std::dynamic_pointer_cast<Linear>(blocks["mlp.2"])->forward(ctx, x);
+        }
+    };
+
+    class PixArtAttention : public GGMLBlock {
+        int64_t num_heads;
+        bool self_attention;
+
+    public:
+        PixArtAttention(int64_t dim, int64_t num_heads, int64_t context_dim, bool self_attention)
+            : num_heads(num_heads), self_attention(self_attention) {
+            if (self_attention) {
+                blocks["qkv"] = std::make_shared<Linear>(dim, 3 * dim);
+            } else {
+                blocks["q_linear"]  = std::make_shared<Linear>(dim, dim);
+                blocks["kv_linear"] = std::make_shared<Linear>(context_dim, 2 * dim);
+            }
+            blocks["proj"] = std::make_shared<Linear>(dim, dim);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* context, ggml_tensor* mask = nullptr) {
+            std::vector<ggml_tensor*> qkv;
+            if (self_attention) {
+                auto projected = std::dynamic_pointer_cast<Linear>(blocks["qkv"])->forward(ctx, x);
+                qkv            = ggml_ext_chunk(ctx->ggml_ctx, projected, 3, 0);
+            } else {
+                auto q     = std::dynamic_pointer_cast<Linear>(blocks["q_linear"])->forward(ctx, x);
+                auto kv    = std::dynamic_pointer_cast<Linear>(blocks["kv_linear"])->forward(ctx, context);
+                auto parts = ggml_ext_chunk(ctx->ggml_ctx, kv, 2, 0);
+                qkv        = {q, parts[0], parts[1]};
+            }
+            auto out = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, mask, false, ctx->flash_attn_enabled);
+            return std::dynamic_pointer_cast<Linear>(blocks["proj"])->forward(ctx, out);
+        }
+    };
+
+    class PixArtBlock : public GGMLBlock {
+        int64_t dim;
+
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         const std::string prefix                       = "") override {
+            ggml_type wtype             = get_type(prefix + "scale_shift_table", tensor_storage_map, GGML_TYPE_F32);
+            params["scale_shift_table"] = ggml_new_tensor_2d(ctx, wtype, dim, 6);
+        }
+
+    public:
+        PixArtBlock(int64_t dim, int64_t num_heads, int64_t context_dim, int64_t ffn_dim)
+            : dim(dim) {
+            blocks["attn"]       = std::make_shared<PixArtAttention>(dim, num_heads, dim, true);
+            blocks["cross_attn"] = std::make_shared<PixArtAttention>(dim, num_heads, context_dim, false);
+            blocks["mlp.fc1"]    = std::make_shared<Linear>(dim, ffn_dim);
+            blocks["mlp.fc2"]    = std::make_shared<Linear>(ffn_dim, dim);
+        }
+
+        static ggml_tensor* norm(ggml_context* ctx, ggml_tensor* x) {
+            return ggml_norm(ctx, x, 1e-6f);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* mod, ggml_tensor* context, ggml_tensor* context_mask) {
+            // x: [N, n_token, dim]
+            // mod: [N, 6 * dim], shared adaLN-single output
+            int64_t N = x->ne[2];
+
+            auto table = params["scale_shift_table"];
+            if (table->type != GGML_TYPE_F32) {
+                table = ggml_cast(ctx->ggml_ctx, table, GGML_TYPE_F32);
+            }
+            table   = ggml_reshape_3d(ctx->ggml_ctx, table, dim, 6, 1);
+            auto m  = ggml_add(ctx->ggml_ctx, ggml_reshape_3d(ctx->ggml_ctx, mod, dim, 6, N), table);
+            auto mv = ggml_ext_chunk(ctx->ggml_ctx, ggml_reshape_2d(ctx->ggml_ctx, ggml_ext_cont(ctx->ggml_ctx, m), dim * 6, N), 6, 0);
+
+            auto attn1 = std::dynamic_pointer_cast<PixArtAttention>(blocks["attn"]);
+            auto attn2 = std::dynamic_pointer_cast<PixArtAttention>(blocks["cross_attn"]);
+            auto proj  = std::dynamic_pointer_cast<Linear>(blocks["mlp.fc1"]);
+            auto fc2   = std::dynamic_pointer_cast<Linear>(blocks["mlp.fc2"]);
+
+            auto gate = [&](ggml_tensor* y, ggml_tensor* g) {
+                g = ggml_reshape_3d(ctx->ggml_ctx, g, dim, 1, N);
+                return ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, y, g));
+            };
+
+            auto h = modulate(ctx->ggml_ctx, norm(ctx->ggml_ctx, x), mv[0], mv[1]);
+            x      = gate(attn1->forward(ctx, h, h), mv[2]);
+            // ada_norm_single: no norm before cross-attention (PixArtMS.py)
+            x = ggml_add(ctx->ggml_ctx, x, attn2->forward(ctx, x, context, context_mask));
+            h = modulate(ctx->ggml_ctx, norm(ctx->ggml_ctx, x), mv[3], mv[4]);
+            h = proj->forward(ctx, h);
+            h = ggml_ext_gelu(ctx->ggml_ctx, h, true);
+            h = fc2->forward(ctx, h);
+            return gate(h, mv[5]);
+        }
+    };
+
+    class PixArtModel : public GGMLBlock {
+        PixArtConfig config;
+
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         const std::string prefix                       = "") override {
+            ggml_type wtype                         = get_type(prefix + "final_layer.scale_shift_table", tensor_storage_map, GGML_TYPE_F32);
+            params["final_layer.scale_shift_table"] = ggml_new_tensor_2d(ctx, wtype, config.hidden_size, 2);
+        }
+
+    public:
+        PixArtModel() = default;
+        PixArtModel(const PixArtConfig& config)
+            : config(config) {
+            blocks["x_embedder.proj"]       = std::make_shared<Conv2d>(config.in_channels,
+                                                                 config.hidden_size,
+                                                                 std::pair<int, int>{static_cast<int>(config.patch_size), static_cast<int>(config.patch_size)},
+                                                                 std::pair<int, int>{static_cast<int>(config.patch_size), static_cast<int>(config.patch_size)});
+            blocks["t_embedder"]            = std::make_shared<PixArtTimestepEmbedding>(ADALN_EMBED_DIM, config.hidden_size);
+            blocks["t_block.1"]             = std::make_shared<Linear>(config.hidden_size, 6 * config.hidden_size);
+            blocks["y_embedder.y_proj.fc1"] = std::make_shared<Linear>(config.caption_channels, config.hidden_size);
+            blocks["y_embedder.y_proj.fc2"] = std::make_shared<Linear>(config.hidden_size, config.cross_attention_dim);
+            for (int i = 0; i < config.num_layers; ++i) {
+                blocks["blocks." + std::to_string(i)] =
+                    std::make_shared<PixArtBlock>(config.hidden_size, config.num_heads, config.cross_attention_dim, config.ffn_dim);
+            }
+            blocks["final_layer.norm_final"] = std::make_shared<LayerNorm>(config.hidden_size, 1e-6f, false);
+            blocks["final_layer.linear"]     = std::make_shared<Linear>(config.hidden_size,
+                                                                    config.patch_size * config.patch_size * config.out_channels);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx,
+                             ggml_tensor* x,
+                             ggml_tensor* timesteps,
+                             ggml_tensor* context,
+                             ggml_tensor* pos_embed,
+                             ggml_tensor* context_mask) {
+            // x: [N, C, H, W] latent, context: [N, n_ctx, caption_channels]
+            int64_t W  = x->ne[0];
+            int64_t H  = x->ne[1];
+            int64_t N  = x->ne[3];
+            int64_t p  = config.patch_size;
+            int64_t wp = W / p;
+            int64_t hp = H / p;
+
+            auto h = std::dynamic_pointer_cast<Conv2d>(blocks["x_embedder.proj"])->forward(ctx, x);  // [N, hidden, hp, wp]
+            h      = ggml_ext_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, h, 1, 2, 0, 3));       // [N, hp, wp, hidden] -> [N, hp*wp, hidden]
+            h      = ggml_reshape_3d(ctx->ggml_ctx, h, config.hidden_size, wp * hp, N);              // [N, hp*wp, hidden]
+            h      = ggml_add(ctx->ggml_ctx, h, pos_embed);
+
+            auto t   = ggml_ext_timestep_embedding(ctx->ggml_ctx, timesteps, ADALN_EMBED_DIM, 10000);
+            auto emb = std::dynamic_pointer_cast<PixArtTimestepEmbedding>(blocks["t_embedder"])->forward(ctx, t);
+
+            auto mod = std::dynamic_pointer_cast<Linear>(blocks["t_block.1"])
+                           ->forward(ctx, ggml_silu(ctx->ggml_ctx, emb));  // [N, 6 * hidden]
+
+            auto ctx_emb = std::dynamic_pointer_cast<Linear>(blocks["y_embedder.y_proj.fc1"])->forward(ctx, context);
+            ctx_emb      = ggml_ext_gelu(ctx->ggml_ctx, ctx_emb, true);
+            ctx_emb      = std::dynamic_pointer_cast<Linear>(blocks["y_embedder.y_proj.fc2"])->forward(ctx, ctx_emb);
+
+            for (int i = 0; i < config.num_layers; ++i) {
+                auto block = std::dynamic_pointer_cast<PixArtBlock>(blocks["blocks." + std::to_string(i)]);
+                h          = block->forward(ctx, h, mod, ctx_emb, context_mask);
+                sd::ggml_graph_cut::mark_graph_cut(h, "pixart.blocks." + std::to_string(i), "h");
+            }
+
+            // scale_shift_table + emb -> (shift, scale) for the affine-free final norm
+            auto tail_table = params["final_layer.scale_shift_table"];
+            if (tail_table->type != GGML_TYPE_F32) {
+                tail_table = ggml_cast(ctx->ggml_ctx, tail_table, GGML_TYPE_F32);
+            }
+            auto ss    = ggml_add(ctx->ggml_ctx,
+                                  ggml_reshape_3d(ctx->ggml_ctx, tail_table, config.hidden_size, 2, 1),
+                                  ggml_reshape_3d(ctx->ggml_ctx, emb, config.hidden_size, 1, N));  // [2, hidden, N]
+            auto parts = ggml_ext_chunk(ctx->ggml_ctx,
+                                        ggml_reshape_2d(ctx->ggml_ctx, ggml_ext_cont(ctx->ggml_ctx, ss), config.hidden_size * 2, N),
+                                        2, 0);
+            h          = std::dynamic_pointer_cast<LayerNorm>(blocks["final_layer.norm_final"])->forward(ctx, h);
+            h          = modulate(ctx->ggml_ctx, h, parts[0], parts[1]);
+            h          = std::dynamic_pointer_cast<Linear>(blocks["final_layer.linear"])->forward(ctx, h);  // [N, hp*wp, p*p*out_ch]
+            h          = DiT::unpatchify(ctx->ggml_ctx, h, hp, wp, static_cast<int>(p), static_cast<int>(p), false);
+            return h;  // [N, out_channels, H, W]
+        }
+    };
+
+    struct PixArtRunner : public DiffusionModelRunner {
+        PixArtConfig config;
+        PixArtModel model;
+        std::vector<float> pos_vec;
+
+        PixArtRunner(ggml_backend_t backend,
+                     const String2TensorStorage& tensor_storage_map      = {},
+                     const std::string prefix                            = "",
+                     std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
+                     const char* model_args                              = nullptr)
+            : DiffusionModelRunner(backend, prefix, weight_manager),
+              config(PixArtConfig::detect_from_weights(tensor_storage_map, prefix)) {
+            for (const auto& [key, value] : parse_key_value_args(model_args, "model arg")) {
+                if (key == "pixart_pos_embed_base_size") {
+                    int parsed = 0;
+                    if (parse_strict_int(value, parsed)) {
+                        config.pos_embed_base_size = parsed;
+                    } else {
+                        LOG_WARN("ignoring invalid PixArt model arg '%s=%s'", key.c_str(), value.c_str());
+                    }
+                } else if (key == "pixart_interpolation_scale") {
+                    float parsed = 0.f;
+                    if (parse_strict_float(value, parsed)) {
+                        config.interpolation_scale = parsed;
+                    } else {
+                        LOG_WARN("ignoring invalid PixArt model arg '%s=%s'", key.c_str(), value.c_str());
+                    }
+                }
+            }
+            model = PixArtModel(config);
+            model.init(params_ctx, tensor_storage_map, prefix);
+        }
+
+        std::string get_desc() override {
+            return "pixart";
+        }
+
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
+            model.get_param_tensors(tensors, prefix);
+        }
+
+        ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
+                                 const sd::Tensor<float>& timesteps_tensor,
+                                 const sd::Tensor<float>& context_tensor,
+                                 const sd::Tensor<float>& mask_tensor) {
+            ggml_cgraph* gf        = new_graph_custom(PIXART_GRAPH_SIZE);
+            ggml_tensor* x         = make_input(x_tensor);
+            ggml_tensor* timesteps = make_input(timesteps_tensor);
+            GGML_ASSERT(x->ne[3] == 1);
+            GGML_ASSERT(!context_tensor.empty());
+            ggml_tensor* context = make_input(context_tensor);
+
+            ggml_tensor* context_mask = nullptr;
+            if (!mask_tensor.empty()) {
+                // additive attention bias over context tokens: 0 keep / -inf discard
+                context_mask = ggml_reshape_4d(compute_ctx, make_input(mask_tensor), mask_tensor.shape()[0], 1, 1, 1);
+            }
+
+            int64_t W  = x->ne[0];
+            int64_t H  = x->ne[1];
+            int64_t wp = W / config.patch_size;
+            int64_t hp = H / config.patch_size;
+
+            pos_vec  = gen_2d_sincos_pos_embed(config.hidden_size, hp, wp,
+                                               config.pos_embed_base_size, config.interpolation_scale);
+            auto pos = ggml_new_tensor_3d(compute_ctx, GGML_TYPE_F32, config.hidden_size, wp * hp, 1);
+            set_backend_tensor_data(pos, pos_vec.data());
+
+            auto runner_ctx  = get_context();
+            ggml_tensor* out = model.forward(&runner_ctx, x, timesteps, context, pos, context_mask);
+            // learn_sigma: keep the noise prediction half of the output channels
+            out = ggml_ext_slice(compute_ctx, out, 2, 0, config.in_channels);
+            ggml_build_forward_expand(gf, out);
+            return gf;
+        }
+
+        sd::Tensor<float> compute(int n_threads,
+                                  const sd::Tensor<float>& x,
+                                  const sd::Tensor<float>& timesteps,
+                                  const sd::Tensor<float>& context,
+                                  const sd::Tensor<float>& context_mask) {
+            auto get_graph = [&]() -> ggml_cgraph* {
+                return build_graph(x, timesteps, context, context_mask);
+            };
+            return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
+        }
+
+        sd::Tensor<float> compute(int n_threads,
+                                  const DiffusionParams& diffusion_params) override {
+            GGML_ASSERT(diffusion_params.x != nullptr);
+            GGML_ASSERT(diffusion_params.timesteps != nullptr);
+            auto context     = tensor_or_empty(diffusion_params.context);
+            auto context_msk = tensor_or_empty(diffusion_params.y);
+            return compute(n_threads,
+                           *diffusion_params.x,
+                           *diffusion_params.timesteps,
+                           context,
+                           context_msk);
+        }
+    };
+}  // namespace PixArt
+
+#endif  // __SD_MODEL_DIFFUSION_PIXART_HPP__

src/model/diffusion/qwen_image.hpp

@@ -635,18 +635,16 @@ namespace Qwen {
                 ref_index_mode = Rope::RefIndexMode::DECREASE;
             }
 
-            pe_vec      = Rope::gen_qwen_image_pe(time_len,
-                                                  static_cast<int>(x->ne[1]),
-                                                  static_cast<int>(x->ne[0]),
-                                                  config.patch_size,
-                                                  batch_size,
-                                                  static_cast<int>(context->ne[1]),
-                                                  ref_latents,
-                                                  ref_index_mode,
-                                                  config.theta,
-                                                  circular_y_enabled,
-                                                  circular_x_enabled,
-                                                  config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_qwen_image_pe(time_len,
+                                                                 static_cast<int>(x->ne[1]),
+                                                                 static_cast<int>(x->ne[0]),
+                                                                 config.patch_size,
+                                                                 batch_size,
+                                                                 static_cast<int>(context->ne[1]),
+                                                                 ref_latents,
+                                                                 ref_index_mode,
+                                                                 config.theta,
+                                                                 config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             // LOG_VERBOSE("pos_len %d", pos_len);
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);

src/model/diffusion/qwen_image_2_1.hpp

@@ -68,6 +68,7 @@ namespace Qwen {
         std::vector<QwenImage21Segment> segments;
         std::vector<std::vector<float>> positions;
         int64_t prefix_length = 0;
+        Rope::PositionLayout rope_layout;
 
         static QwenImage21Layout build(int64_t text_length,
                                        const sd::Tensor<int32_t>& image_slots,
@@ -82,6 +83,7 @@ namespace Qwen {
                 auto [height, width] = image_shapes[index];
                 int64_t start        = static_cast<int64_t>(layout.positions.size());
                 layout.segments.push_back({start, start + height * width, context_start, index});
+                layout.rope_layout.append_image(static_cast<int>(height), static_cast<int>(width));
                 for (int64_t h = 0; h < height; ++h) {
                     for (int64_t w = 0; w < width; ++w) {
                         layout.positions.push_back({static_cast<float>(position),
@@ -106,6 +108,7 @@ namespace Qwen {
                 } else {
                     int64_t start = static_cast<int64_t>(layout.positions.size());
                     layout.segments.push_back({start, start + i - begin, begin, -1});
+                    layout.rope_layout.append_tokens(i - begin);
                     for (int64_t j = begin; j < i; ++j, ++position) {
                         float p = static_cast<float>(position);
                         layout.positions.push_back({p, p, p});
@@ -121,6 +124,20 @@ namespace Qwen {
         }
     };
 
+    struct QwenImage21PrefixCache {
+        enum class Mode {
+            NONE,
+            STORE,
+            REUSE
+        };
+        Mode mode = Mode::NONE;
+        std::string name;
+        std::string cut_group;
+        int64_t prefix_length = 0;
+        ggml_type type        = GGML_TYPE_F32;
+        bool* flash_attn_used = nullptr;
+    };
+
     class QwenImage21ZeroCenterRMSNorm : public RMSNorm {
     public:
         using RMSNorm::RMSNorm;
@@ -160,27 +177,70 @@ namespace Qwen {
             }
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe, const std::vector<QwenImage21Segment>& segments, const std::vector<ggml_tensor*>& masks) {
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe, const std::vector<QwenImage21Segment>& segments, const std::vector<ggml_tensor*>& masks, const QwenImage21PrefixCache& cache) {
             int64_t heads = x->ne[0] / dim_head;
             auto project  = [&](const char* name) {
                 auto h = std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, x);
                 return ggml_reshape_4d(ctx->ggml_ctx, h, dim_head, heads, x->ne[1], x->ne[2]);
             };
-            auto q              = project("to_q");
-            auto k              = project("to_k");
-            auto v              = project("to_v");
-            q                   = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_q"])->forward(ctx, q);
-            k                   = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_k"])->forward(ctx, k);
-            q                   = Rope::apply_rope(ctx->ggml_ctx, q, pe);
-            k                   = Rope::apply_rope(ctx->ggml_ctx, k, pe);
+            auto q = project("to_q");
+            auto k = project("to_k");
+            auto v = project("to_v");
+            q      = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_q"])->forward(ctx, q);
+            k      = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_k"])->forward(ctx, k);
+            q      = Rope::apply_rope(ctx->ggml_ctx, q, pe);
+            k      = Rope::apply_rope(ctx->ggml_ctx, k, pe);
+            if (cache.mode == QwenImage21PrefixCache::Mode::STORE) {
+                // Preserve query-first attention evaluation while writing each layer's
+                // prefix before its full-sequence K/V can accumulate across layers.
+                ctx->expand_graph(q);
+                auto persist = [&](ggml_tensor* tensor, int axis, const char* name) {
+                    auto part = ggml_ext_slice(ctx->ggml_ctx, tensor, axis, 0, cache.prefix_length);
+                    // Pack the contiguous data into wider rows so quantization blocks
+                    // can exceed head_dim without padding or changing element order.
+                    part      = ggml_reshape_2d(ctx->ggml_ctx, part, x->ne[0], cache.prefix_length);
+                    auto copy = ggml_cast(ctx->ggml_ctx, part, cache.type);
+                    // Keep the copy in this layer's segment so graph cuts do not
+                    // retain or recompute the full-sequence K/V in the final segment.
+                    sd::ggml_graph_cut::mark_graph_cut(copy, cache.cut_group, name);
+                    ctx->persist_cache_tensor(cache.name + "." + name, copy);
+                };
+                persist(k, 1, "k");
+                persist(v, 2, "v");
+            }
+            auto attend = [&](ggml_tensor* aq, ggml_tensor* ak, ggml_tensor* av, ggml_tensor* mask) {
+                bool used_flash_attn = false;
+                auto out             = ggml_ext_attention_ext(ctx, aq, ak, av, heads, mask, true, ctx->flash_attn_enabled, 1.f, &used_flash_attn);
+                if (cache.flash_attn_used != nullptr) {
+                    *cache.flash_attn_used &= used_flash_attn;
+                }
+                return out;
+            };
             ggml_tensor* result = nullptr;
-            for (size_t i = 0; i < segments.size(); ++i) {
-                const auto& segment = segments[i];
-                auto sq             = ggml_ext_slice(ctx->ggml_ctx, q, 1, segment.start, segment.end);
-                auto sk             = ggml_ext_slice(ctx->ggml_ctx, k, 1, 0, segment.end);
-                auto sv             = ggml_ext_slice(ctx->ggml_ctx, v, 2, 0, segment.end);
-                auto out            = ggml_ext_attention_ext(ctx, sq, sk, sv, heads, masks[i], true, ctx->flash_attn_enabled);
-                result              = result == nullptr ? out : ggml_concat(ctx->ggml_ctx, result, out, 1);
+            if (cache.mode == QwenImage21PrefixCache::Mode::REUSE) {
+                auto prefix_k = ctx->load_cache_tensor(cache.name + ".k");
+                auto prefix_v = ctx->load_cache_tensor(cache.name + ".v");
+                GGML_ASSERT(prefix_k != nullptr && prefix_v != nullptr);
+                if (prefix_k->type != k->type) {
+                    prefix_k = ggml_cast(ctx->ggml_ctx, prefix_k, k->type);
+                }
+                if (prefix_v->type != v->type) {
+                    prefix_v = ggml_cast(ctx->ggml_ctx, prefix_v, v->type);
+                }
+                prefix_k = ggml_reshape_4d(ctx->ggml_ctx, prefix_k, dim_head, cache.prefix_length, heads, k->ne[3]);
+                prefix_v = ggml_reshape_4d(ctx->ggml_ctx, prefix_v, dim_head, heads, cache.prefix_length, v->ne[3]);
+                k        = ggml_concat(ctx->ggml_ctx, prefix_k, k, 1);
+                v        = ggml_concat(ctx->ggml_ctx, prefix_v, v, 2);
+                result   = attend(q, k, v, nullptr);
+            } else {
+                for (size_t i = 0; i < segments.size(); ++i) {
+                    const auto& segment = segments[i];
+                    auto sq             = ggml_ext_slice(ctx->ggml_ctx, q, 1, segment.start, segment.end);
+                    auto sk             = ggml_ext_slice(ctx->ggml_ctx, k, 1, 0, segment.end);
+                    auto sv             = ggml_ext_slice(ctx->ggml_ctx, v, 2, 0, segment.end);
+                    auto out            = attend(sq, sk, sv, masks[i]);
+                    result              = result == nullptr ? out : ggml_concat(ctx->ggml_ctx, result, out, 1);
+                }
             }
             auto to_out = std::dynamic_pointer_cast<Linear>(blocks["to_out.0"]);
             if (sd_backend_is(ctx->backend, "Vulkan") || sd_backend_is(ctx->backend, "ROCm")) {
@@ -219,13 +279,14 @@ namespace Qwen {
             return ggml_concat(ctx, prefix, target, 1);
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, const std::vector<ggml_tensor*>& modulation, ggml_tensor* pe, const QwenImage21Layout& layout, const std::vector<ggml_tensor*>& masks) {
-            auto h = std::dynamic_pointer_cast<LayerNorm>(blocks["img_norm1"])->forward(ctx, x);
-            h      = modulate(ctx->ggml_ctx, h, modulation[0], layout.prefix_length);
-            h      = std::dynamic_pointer_cast<QwenImage21Attention>(blocks["attn"])->forward(ctx, h, pe, layout.segments, masks);
-            x      = ggml_add(ctx->ggml_ctx, x, modulate(ctx->ggml_ctx, h, modulation[1], layout.prefix_length, true));
-            h      = std::dynamic_pointer_cast<LayerNorm>(blocks["img_norm2"])->forward(ctx, x);
-            h      = modulate(ctx->ggml_ctx, h, modulation[2], layout.prefix_length);
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, const std::vector<ggml_tensor*>& modulation, ggml_tensor* pe, const QwenImage21Layout& layout, const std::vector<ggml_tensor*>& masks, const QwenImage21PrefixCache& cache) {
+            const int64_t prefix_length = cache.mode == QwenImage21PrefixCache::Mode::REUSE ? 0 : layout.prefix_length;
+            auto h                      = std::dynamic_pointer_cast<LayerNorm>(blocks["img_norm1"])->forward(ctx, x);
+            h                           = modulate(ctx->ggml_ctx, h, modulation[0], prefix_length);
+            h                           = std::dynamic_pointer_cast<QwenImage21Attention>(blocks["attn"])->forward(ctx, h, pe, layout.segments, masks, cache);
+            x                           = ggml_add(ctx->ggml_ctx, x, modulate(ctx->ggml_ctx, h, modulation[1], prefix_length, true));
+            h                           = std::dynamic_pointer_cast<LayerNorm>(blocks["img_norm2"])->forward(ctx, x);
+            h                           = modulate(ctx->ggml_ctx, h, modulation[2], prefix_length);
             ggml_tensor* gate;
             auto fused = blocks.find("img_mlp.gate_up");
             if (fused != blocks.end()) {
@@ -239,7 +300,7 @@ namespace Qwen {
             }
             h = ggml_mul(ctx->ggml_ctx, h, ggml_silu(ctx->ggml_ctx, gate));
             h = std::dynamic_pointer_cast<Linear>(blocks["img_mlp.out"])->forward(ctx, h);
-            return ggml_add(ctx->ggml_ctx, x, modulate(ctx->ggml_ctx, h, modulation[3], layout.prefix_length, true));
+            return ggml_add(ctx->ggml_ctx, x, modulate(ctx->ggml_ctx, h, modulation[3], prefix_length, true));
         }
     };
 
@@ -261,7 +322,7 @@ namespace Qwen {
             }
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* timestep, ggml_tensor* context, const std::vector<ggml_tensor*>& refs, ggml_tensor* pe, const QwenImage21Layout& layout, const std::vector<ggml_tensor*>& masks) {
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* timestep, ggml_tensor* context, const std::vector<ggml_tensor*>& refs, ggml_tensor* pe, const QwenImage21Layout& layout, const std::vector<ggml_tensor*>& masks, const QwenImage21PrefixCache& cache) {
             auto time = ggml_concat(ctx->ggml_ctx, timestep, ggml_ext_zeros_like(ctx->ggml_ctx, timestep), 0);
             // Runtime flow timesteps already use the [0, 1000] scale.
             time               = ggml_ext_timestep_embedding(ctx->ggml_ctx, time, 256, 10000, 1.f);
@@ -269,27 +330,37 @@ namespace Qwen {
             time               = ggml_silu(ctx->ggml_ctx, time);
             auto modulation    = std::dynamic_pointer_cast<Linear>(blocks["modulation.1"])->forward(ctx, time);
             auto mod           = ggml_ext_chunk(ctx->ggml_ctx, modulation, 4, 0);
-            auto text          = std::dynamic_pointer_cast<QwenImage21TextProjection>(blocks["txt_in"])->forward(ctx, context);
             auto img_in        = std::dynamic_pointer_cast<Linear>(blocks["img_in"]);
             ggml_tensor* joint = nullptr;
-            for (const auto& segment : layout.segments) {
-                ggml_tensor* h;
-                if (segment.image_index < 0) {
-                    h = ggml_ext_slice(ctx->ggml_ctx, text, 1, segment.context_start,
-                                       segment.context_start + segment.end - segment.start);
-                } else {
-                    auto image = segment.image_index == static_cast<int>(refs.size()) ? x : refs[segment.image_index];
-                    h          = img_in->forward(ctx, DiT::patchify(ctx->ggml_ctx, image, 1, 1));
+            if (cache.mode == QwenImage21PrefixCache::Mode::REUSE) {
+                joint = img_in->forward(ctx, DiT::patchify(ctx->ggml_ctx, x, 1, 1));
+            } else {
+                auto text = std::dynamic_pointer_cast<QwenImage21TextProjection>(blocks["txt_in"])->forward(ctx, context);
+                for (const auto& segment : layout.segments) {
+                    ggml_tensor* h;
+                    if (segment.image_index < 0) {
+                        h = ggml_ext_slice(ctx->ggml_ctx, text, 1, segment.context_start,
+                                           segment.context_start + segment.end - segment.start);
+                    } else {
+                        auto image = segment.image_index == static_cast<int>(refs.size()) ? x : refs[segment.image_index];
+                        h          = img_in->forward(ctx, DiT::patchify(ctx->ggml_ctx, image, 1, 1));
+                    }
+                    joint = joint == nullptr ? h : ggml_concat(ctx->ggml_ctx, joint, h, 1);
                 }
-                joint = joint == nullptr ? h : ggml_concat(ctx->ggml_ctx, joint, h, 1);
             }
             sd::ggml_graph_cut::mark_graph_cut(joint, "qwen_image_2_1.prelude", "joint");
             for (int i = 0; i < config.num_layers; ++i) {
-                auto block = std::dynamic_pointer_cast<QwenImage21TransformerBlock>(blocks["transformer_blocks." + std::to_string(i)]);
-                joint      = block->forward(ctx, joint, mod, pe, layout, masks);
-                sd::ggml_graph_cut::mark_graph_cut(joint, "qwen_image_2_1.transformer_blocks." + std::to_string(i), "joint");
+                const std::string layer = "transformer_blocks." + std::to_string(i);
+                auto layer_cache        = cache;
+                layer_cache.name        = cache.name + "." + std::to_string(i);
+                layer_cache.cut_group   = "qwen_image_2_1." + layer;
+                auto block              = std::dynamic_pointer_cast<QwenImage21TransformerBlock>(blocks[layer]);
+                joint                   = block->forward(ctx, joint, mod, pe, layout, masks, layer_cache);
+                sd::ggml_graph_cut::mark_graph_cut(joint, layer_cache.cut_group, "joint");
+            }
+            if (cache.mode != QwenImage21PrefixCache::Mode::REUSE) {
+                joint = ggml_ext_slice(ctx->ggml_ctx, joint, 1, layout.prefix_length, joint->ne[1]);
             }
-            joint      = ggml_ext_slice(ctx->ggml_ctx, joint, 1, layout.prefix_length, joint->ne[1]);
             auto scale = std::dynamic_pointer_cast<Linear>(blocks["norm_out.linear"])->forward(ctx, ggml_ext_chunk(ctx->ggml_ctx, time, 2, 1)[0]);
             joint      = std::dynamic_pointer_cast<LayerNorm>(blocks["norm_out.norm"])->forward(ctx, joint);
             joint      = ggml_mul(ctx->ggml_ctx, joint, ggml_scale_bias(ctx->ggml_ctx, scale, 1.f, 1.f));
@@ -303,11 +374,61 @@ namespace Qwen {
         QwenImage21Model model;
         std::vector<float> pe_data;
         std::vector<sd::Tensor<float>> mask_data;
+        ggml_type prefix_cache_type = GGML_TYPE_COUNT;
+        bool prefix_cache_enabled   = true;
+        bool prefix_cache_disabled  = false;
+        bool prefix_cache_auto_f32  = false;
 
-        QwenImage21Runner(ggml_backend_t backend, const String2TensorStorage& weights, const std::string& prefix, std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+        static bool supports_prefix_cache_type(ggml_type type) {
+            if (type == GGML_TYPE_F32) {
+                return true;
+            }
+            const auto* traits = ggml_get_type_traits(type);
+            if (traits->from_float_ref == nullptr || traits->to_float == nullptr) {
+                return false;
+            }
+            auto cpu = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
+            if (cpu == nullptr) {
+                return false;
+            }
+            auto ctx = std::unique_ptr<ggml_context, decltype(&ggml_free)>(
+                ggml_init({3 * ggml_tensor_overhead(), nullptr, true}), ggml_free);
+            if (ctx == nullptr) {
+                return false;
+            }
+            // Some reference quantizers have no runtime copy support. Query the
+            // device through the registry so dynamically loaded CPU backends work.
+            auto source  = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, ggml_blck_size(type));
+            auto encoded = ggml_cast(ctx.get(), source, type);
+            auto decoded = ggml_cast(ctx.get(), encoded, GGML_TYPE_F32);
+            return ggml_backend_dev_supports_op(cpu, encoded) && ggml_backend_dev_supports_op(cpu, decoded);
+        }
+
+        QwenImage21Runner(ggml_backend_t backend, const String2TensorStorage& weights, const std::string& prefix, std::shared_ptr<RunnerWeightManager> weight_manager = nullptr, const char* model_args = nullptr)
             : DiffusionModelRunner(backend, prefix, weight_manager),
               config(QwenImage21Config::detect_from_weights(weights, prefix)),
               model(config) {
+            for (const auto& [key, value] : parse_key_value_args(model_args, "model arg")) {
+                if (key == "qwen_image_2_1_prefix_cache" && !parse_strict_bool(value, prefix_cache_enabled)) {
+                    LOG_WARN("ignoring invalid Qwen Image 2.1 model arg '%s=%s'", key.c_str(), value.c_str());
+                } else if (key == "qwen_image_2_1_prefix_cache_type") {
+                    if (value == "auto") {
+                        prefix_cache_type = GGML_TYPE_COUNT;
+                        continue;
+                    }
+                    const auto type = sd_type_to_ggml_type(str_to_sd_type(value.c_str()));
+                    if (type == GGML_TYPE_COUNT) {
+                        LOG_WARN("ignoring unknown Qwen Image 2.1 cache type '%s'", value.c_str());
+                    } else if (!supports_prefix_cache_type(type)) {
+                        LOG_WARN("ignoring Qwen Image 2.1 cache type '%s': runtime conversion to and from F32 is unavailable", value.c_str());
+                    } else if (config.hidden_size % ggml_blck_size(type) != 0) {
+                        LOG_WARN("ignoring Qwen Image 2.1 cache type '%s': block size %" PRId64 " does not divide hidden size %" PRId64,
+                                 value.c_str(), ggml_blck_size(type), config.hidden_size);
+                    } else {
+                        prefix_cache_type = type;
+                    }
+                }
+            }
             model.init(params_ctx, weights, prefix);
         }
 
@@ -317,6 +438,20 @@ namespace Qwen {
             model.get_param_tensors(tensors, prefix);
         }
 
+        bool has_prefix_cache(const QwenImage21PrefixCache& cache) {
+            for (int i = 0; i < config.num_layers; ++i) {
+                const auto name = cache.name + "." + std::to_string(i);
+                auto k          = get_cache_tensor_by_name(name + ".k");
+                auto v          = get_cache_tensor_by_name(name + ".v");
+                if (k == nullptr || v == nullptr || k->type != cache.type || v->type != cache.type ||
+                    k->ne[0] != config.hidden_size || k->ne[1] != cache.prefix_length || k->ne[2] != 1 || k->ne[3] != 1 ||
+                    v->ne[0] != config.hidden_size || v->ne[1] != cache.prefix_length || v->ne[2] != 1 || v->ne[3] != 1) {
+                    return false;
+                }
+            }
+            return true;
+        }
+
         sd::Tensor<float> compute(int n_threads, const DiffusionParams& inputs) override {
             const auto& x       = tensor_or_empty(inputs.x);
             const auto& context = tensor_or_empty(inputs.context);
@@ -345,38 +480,113 @@ namespace Qwen {
                 LOG_ERROR("%s", error.what());
                 return {};
             }
-            pe_data = Rope::embed_nd(layout.positions, 1, 10000.f, config.axes_dim);
-            mask_data.clear();
-            for (const auto& segment : layout.segments) {
-                sd::Tensor<float> mask;
-                if (segment.image_index < 0) {
-                    mask = sd::Tensor<float>::zeros({segment.end, segment.end - segment.start});
-                    for (int64_t q = segment.start; q < segment.end; ++q) {
-                        for (int64_t k = q + 1; k < segment.end; ++k) {
-                            mask[k + segment.end * (q - segment.start)] = -INFINITY;
-                        }
+            if (!runner_started()) {
+                prefix_cache_disabled = false;
+                prefix_cache_auto_f32 = false;
+            }
+            QwenImage21PrefixCache cache;
+            if (prefix_cache_enabled && !prefix_cache_disabled && extra != nullptr && extra->prefix_id != 0 && layout.prefix_length > 0) {
+                cache.name = "qwen_image_2_1.prefix." + std::to_string(extra->prefix_id) +
+                             ".circular." + std::to_string(circular_x_enabled) + std::to_string(circular_y_enabled);
+                cache.prefix_length = layout.prefix_length;
+                if (prefix_cache_type != GGML_TYPE_COUNT) {
+                    cache.type = prefix_cache_type;
+                } else if (!prefix_cache_auto_f32 && flash_attn_enabled && !sage_attn_enabled &&
+                           (attn_scale <= 0.f || attn_scale == 1.f)) {
+                    cache.type = GGML_TYPE_F16;
+                }
+                cache.mode = has_prefix_cache(cache) ? QwenImage21PrefixCache::Mode::REUSE : QwenImage21PrefixCache::Mode::STORE;
+            }
+            bool flash_attn_used = true;
+            auto run             = [&](const QwenImage21PrefixCache& active_cache) {
+                flash_attn_used    = true;
+                auto checked_cache = active_cache;
+                if (prefix_cache_type == GGML_TYPE_COUNT && active_cache.type == GGML_TYPE_F16) {
+                    checked_cache.flash_attn_used = &flash_attn_used;
+                }
+                const bool cached         = active_cache.mode == QwenImage21PrefixCache::Mode::REUSE;
+                const auto first_position = layout.positions.begin() + (cached ? layout.prefix_length : 0);
+                Rope::Embedding embedding;
+                embedding.ids.assign(first_position, layout.positions.end());
+                const size_t offset             = cached ? static_cast<size_t>(layout.prefix_length) : 0;
+                embedding.positions.token_count = embedding.ids.size();
+                for (auto region : layout.rope_layout.images) {
+                    if (region.begin >= offset) {
+                        region.begin -= offset;
+                        embedding.positions.images.push_back(region);
                     }
                 }
-                mask_data.push_back(std::move(mask));
-            }
-            auto build = [&]() {
-                auto graph = new_graph_custom(QWEN_IMAGE_GRAPH_SIZE * 2);
-                auto pe    = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.head_dim / 2, layout.positions.size());
-                set_backend_tensor_data(pe, pe_data.data());
-                std::vector<ggml_tensor*> masks, ref_inputs;
-                for (const auto& mask : mask_data) {
-                    masks.push_back(mask.empty() ? nullptr : make_input(mask));
+                embedding.values = Rope::embed_nd(embedding.ids, 1, 10000.f, config.axes_dim, embedding.layout, &embedding.frequencies);
+                pe_data          = finish_rope_pe(std::move(embedding));
+                mask_data.clear();
+                if (!cached) {
+                    for (const auto& segment : layout.segments) {
+                        sd::Tensor<float> mask;
+                        if (segment.image_index < 0) {
+                            mask = sd::Tensor<float>::zeros({segment.end, segment.end - segment.start});
+                            for (int64_t q = segment.start; q < segment.end; ++q) {
+                                for (int64_t k = q + 1; k < segment.end; ++k) {
+                                    mask[k + segment.end * (q - segment.start)] = -INFINITY;
+                                }
+                            }
+                        }
+                        mask_data.push_back(std::move(mask));
+                    }
                 }
-                for (const auto& ref : refs) {
-                    ref_inputs.push_back(make_input(ref));
-                }
-                auto ctx = get_context();
-                auto out = model.forward(&ctx, make_input(x), make_input(*inputs.timesteps), make_input(context),
-                                         ref_inputs, pe, layout, masks);
-                ggml_build_forward_expand(graph, out);
-                return graph;
+                auto build = [&]() {
+                    auto graph = new_graph_custom(QWEN_IMAGE_GRAPH_SIZE * 2);
+                    auto pe    = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.head_dim / 2,
+                                                                layout.positions.size() - (cached ? layout.prefix_length : 0));
+                    set_backend_tensor_data(pe, pe_data.data());
+                    std::vector<ggml_tensor*> masks, ref_inputs;
+                    for (const auto& mask : mask_data) {
+                        masks.push_back(mask.empty() ? nullptr : make_input(mask));
+                    }
+                    if (!cached) {
+                        for (const auto& ref : refs) {
+                            ref_inputs.push_back(make_input(ref));
+                        }
+                    }
+                    auto ctx = get_context(graph);
+                    auto out = model.forward(&ctx, make_input(x), make_input(*inputs.timesteps), cached ? nullptr : make_input(context),
+                                                         ref_inputs, pe, layout, masks, checked_cache);
+                    if (!flash_attn_used) {
+                        return static_cast<ggml_cgraph*>(nullptr);
+                    }
+                    ggml_build_forward_expand(graph, out);
+                    return graph;
+                };
+                return restore_trailing_singleton_dims(GGMLRunner::compute(build, n_threads, false), x.dim());
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute(build, n_threads, false), x.dim());
+            auto result = run(cache);
+            if (result.empty() && !flash_attn_used) {
+                // Casting an F16 cache back to F32 cannot recover its original values.
+                // Recompute the prefix before executing a graph that falls back from FA.
+                free_cache_ctx_and_buffer();
+                prefix_cache_auto_f32 = true;
+                cache.type            = GGML_TYPE_F32;
+                cache.mode            = QwenImage21PrefixCache::Mode::STORE;
+                LOG_DEBUG("Qwen Image 2.1: Flash Attention unavailable; using F32 prefix caching for this sampling run");
+                result = run(cache);
+            }
+            if (result.empty() && last_compute_status() == GGML_STATUS_ALLOC_FAILED &&
+                (cache.mode != QwenImage21PrefixCache::Mode::NONE || !cache_.empty())) {
+                // The failed graph has ended before persistent inputs are released.
+                free_cache_ctx_and_buffer();
+                prefix_cache_disabled = true;
+                LOG_WARN("Qwen Image 2.1: insufficient memory for prefix caching; retrying without it for this sampling run");
+                return run(QwenImage21PrefixCache{});
+            }
+            if (!result.empty() && cache.mode == QwenImage21PrefixCache::Mode::STORE) {
+                if (!has_prefix_cache(cache)) {
+                    free_cache_ctx_and_buffer();
+                    prefix_cache_disabled = true;
+                    LOG_WARN("Qwen Image 2.1: incomplete prefix cache; disabling it for this sampling run");
+                } else {
+                    LOG_DEBUG("Qwen Image 2.1: cached prefix %" PRIu64 " (%" PRId64 " tokens, %s)", extra->prefix_id, layout.prefix_length, ggml_type_name(cache.type));
+                }
+            }
+            return result;
         }
     };
 }

src/model/diffusion/sensenova_u1.h

@@ -442,16 +442,9 @@ namespace SenseNovaU1 {
                 k = ggml_concat(ctx->ggml_ctx, prefix_k, k, 2);
                 v = ggml_concat(ctx->ggml_ctx, prefix_v, v, 2);
             } else {
-                // Keep dedicated graph outputs alive until the runner copies them
-                // into its persistent cache buffer after graph execution.
-                auto cache_k = ggml_dup_tensor(ctx->ggml_ctx, k);
-                cache_k      = ggml_cpy(ctx->ggml_ctx, k, cache_k);
-                ggml_set_output(cache_k);
-                auto cache_v = ggml_dup_tensor(ctx->ggml_ctx, v);
-                cache_v      = ggml_cpy(ctx->ggml_ctx, v, cache_v);
-                ggml_set_output(cache_v);
-                ctx->persist_cache_tensor(layer_cache + ".k", cache_k);
-                ctx->persist_cache_tensor(layer_cache + ".v", cache_v);
+                ctx->expand_graph(q);
+                ctx->persist_cache_tensor(layer_cache + ".k", k);
+                ctx->persist_cache_tensor(layer_cache + ".v", v);
             }
 
             q = ggml_cont(ctx->ggml_ctx,
@@ -687,7 +680,7 @@ namespace SenseNovaU1 {
             ggml_set_name(attention_mask, "snu15.prefix.attention_mask");
             set_backend_tensor_data(attention_mask, attention_mask_vec.data());
 
-            auto runner_ctx = get_context();
+            auto runner_ctx = get_context(graph);
             auto text_model = model.text_model();
             auto hidden     = text_model->embed(&runner_ctx, ids);
             hidden          = text_model->forward(&runner_ctx,

src/model/diffusion/z_image.hpp

@@ -140,7 +140,8 @@ namespace ZImage {
                        int64_t num_kv_heads,
                        bool qk_norm,
                        bool norm_elementwise_affine = true,
-                       bool split_qkv               = false)
+                       bool split_qkv               = false,
+                       float qk_norm_eps            = 1e-6f)
             : head_dim(head_dim), num_heads(num_heads), num_kv_heads(num_kv_heads), qk_norm(qk_norm), split_qkv(split_qkv) {
             float scale = 1.f;
             if (split_qkv) {
@@ -153,8 +154,8 @@ namespace ZImage {
                 blocks["out"] = std::make_shared<Linear>(num_heads * head_dim, hidden_size, false, false, false, scale);
             }
             if (qk_norm) {
-                blocks["q_norm"] = std::make_shared<RMSNorm>(head_dim, 1e-06f, norm_elementwise_affine);
-                blocks["k_norm"] = std::make_shared<RMSNorm>(head_dim, 1e-06f, norm_elementwise_affine);
+                blocks["q_norm"] = std::make_shared<RMSNorm>(head_dim, qk_norm_eps, norm_elementwise_affine);
+                blocks["k_norm"] = std::make_shared<RMSNorm>(head_dim, qk_norm_eps, norm_elementwise_affine);
             }
         }
 
@@ -318,9 +319,10 @@ namespace ZImage {
                               bool qk_norm,
                               bool modulation              = true,
                               bool norm_elementwise_affine = true,
-                              bool split_qkv               = false)
+                              bool split_qkv               = false,
+                              float qk_norm_eps            = 1e-6f)
             : modulation(modulation) {
-            blocks["attention"]       = std::make_shared<JointAttention>(hidden_size, head_dim, num_heads, num_kv_heads, qk_norm, norm_elementwise_affine, split_qkv);
+            blocks["attention"]       = std::make_shared<JointAttention>(hidden_size, head_dim, num_heads, num_kv_heads, qk_norm, norm_elementwise_affine, split_qkv, qk_norm_eps);
             blocks["feed_forward"]    = std::make_shared<FeedForward>(hidden_size, hidden_size, multiple_of, ffn_dim_multiplier);
             blocks["attention_norm1"] = std::make_shared<RMSNorm>(hidden_size, norm_eps, norm_elementwise_affine);
             blocks["ffn_norm1"]       = std::make_shared<RMSNorm>(hidden_size, norm_eps, norm_elementwise_affine);
@@ -642,18 +644,16 @@ namespace ZImage {
                 ref_latents.push_back(make_input(ref_latent_tensor));
             }
 
-            pe_vec      = Rope::gen_z_image_pe(static_cast<int>(x->ne[1]),
-                                               static_cast<int>(x->ne[0]),
-                                               config.patch_size,
-                                               static_cast<int>(x->ne[3]),
-                                               static_cast<int>(context->ne[1]),
-                                               SEQ_MULTI_OF,
-                                               ref_latents,
-                                               ref_index_mode,
-                                               config.theta,
-                                               circular_y_enabled,
-                                               circular_x_enabled,
-                                               config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_z_image_pe(static_cast<int>(x->ne[1]),
+                                                              static_cast<int>(x->ne[0]),
+                                                              config.patch_size,
+                                                              static_cast<int>(x->ne[3]),
+                                                              static_cast<int>(context->ne[1]),
+                                                              SEQ_MULTI_OF,
+                                                              ref_latents,
+                                                              ref_index_mode,
+                                                              config.theta,
+                                                              config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             // LOG_VERBOSE("pos_len %d", pos_len);
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);

src/model/te/llm.hpp

@@ -50,6 +50,8 @@ namespace LLM {
         GEMMA4_12B,
         GPT_OSS_20B,
         LLADA2_MOE,
+        BAILING_MOE,
+        QWEN2,
         ARCH_COUNT,
     };
 
@@ -64,6 +66,8 @@ namespace LLM {
         "gemma4_12b",
         "gpt_oss_20b",
         "llada2_moe",
+        "bailing_moe",
+        "qwen2",
     };
 
     enum class MLPActivation {
@@ -225,7 +229,7 @@ namespace LLM {
                 config.intermediate_size       = 9216;
                 config.num_layers              = 26;
                 config.vocab_size              = 256000;
-            } else if (arch == LLMArch::LLADA2_MOE) {
+            } else if (arch == LLMArch::LLADA2_MOE || arch == LLMArch::BAILING_MOE) {
                 config.head_dim                = 128;
                 config.num_heads               = 16;
                 config.num_kv_heads            = 4;
@@ -240,7 +244,7 @@ namespace LLM {
                 config.max_position_embeddings = 16384;
                 config.rope_thetas             = {600000.f};
                 config.qkv_fused               = true;
-                config.bidirectional           = true;
+                config.bidirectional           = arch == LLMArch::LLADA2_MOE;
                 config.partial_rotary          = 0.5f;
                 config.num_experts             = 256;
                 config.num_experts_per_tok     = 8;
@@ -250,6 +254,18 @@ namespace LLM {
                 config.n_group                 = 8;
                 config.topk_group              = 4;
                 config.routed_scaling_factor   = 2.5f;
+                if (arch == LLMArch::BAILING_MOE) {
+                    config.vocab_size              = 157184;
+                    config.max_position_embeddings = 32768;
+                }
+            } else if (arch == LLMArch::QWEN2) {
+                config.hidden_size             = 1536;
+                config.intermediate_size       = 8960;
+                config.num_heads               = 12;
+                config.num_kv_heads            = 2;
+                config.vocab_size              = 151936;
+                config.max_position_embeddings = 32768;
+                config.bidirectional           = true;
             } else if (arch == LLMArch::GPT_OSS_20B) {
                 config.head_dim                = 64;
                 config.num_heads               = 64;
@@ -283,22 +299,22 @@ namespace LLM {
                     if (contains(name, "attn.q_proj")) {
                         config.llama_cpp_style = true;
                     }
-                    if (contains(name, "visual.patch_embed.proj.1.weight")) {
+                    if (ends_with(name, "visual.patch_embed.proj.1.weight")) {
                         config.vision.split_patch_embed = true;
                     }
-                    if (contains(name, "visual.patch_embed.proj.0.weight")) {
+                    if (ends_with(name, "visual.patch_embed.proj.0.weight")) {
                         config.vision.patch_size  = static_cast<int>(tensor_storage.ne[0]);
                         config.vision.in_channels = tensor_storage.ne[2];
                         config.vision.hidden_size = tensor_storage.ne[3];
                     }
                     // HF-format checkpoints keep the patch embed unsplit under a single name.
-                    if (contains(name, "visual.patch_embed.proj.weight")) {
+                    if (ends_with(name, "visual.patch_embed.proj.weight")) {
                         config.vision.patch_size = static_cast<int>(tensor_storage.ne[0]);
                     }
                     if (contains(name, "visual.patch_embed.bias") || contains(name, "visual.patch_embed.proj.bias")) {
                         config.vision.hidden_size = tensor_storage.ne[0];
                     }
-                    if (contains(name, "visual.pos_embed.weight")) {
+                    if (ends_with(name, "visual.pos_embed.weight") && tensor_storage.n_dims == 2) {
                         config.vision.hidden_size             = tensor_storage.ne[0];
                         config.vision.num_position_embeddings = static_cast<int>(tensor_storage.ne[1]);
                     }
@@ -332,7 +348,7 @@ namespace LLM {
                         }
                     }
                 }
-                if (contains(name, "embed_tokens.weight")) {
+                if (ends_with(name, "embed_tokens.weight") && tensor_storage.n_dims == 2) {
                     config.hidden_size = tensor_storage.ne[0];
                     config.vocab_size  = tensor_storage.ne[1];
                 }
@@ -471,6 +487,8 @@ namespace LLM {
         int64_t n_group;
         int64_t topk_group;
         float routed_scaling_factor;
+        bool image_router;
+        bool fused_experts = false;
 
         void init_params(ggml_context* ctx,
                          const String2TensorStorage& tensor_storage_map = {},
@@ -488,6 +506,10 @@ namespace LLM {
             // scores and the group sums match.
             params["gate.weight"]      = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, num_experts);
             params["gate.expert_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, num_experts);
+            if (image_router) {
+                params["image_gate.weight"]      = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, num_experts);
+                params["image_gate.expert_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, num_experts);
+            }
 
             ggml_type gate_type = supported_type(get_type(prefix + "experts.gate_proj.weight", tensor_storage_map, GGML_TYPE_F32), hidden_size);
             ggml_type up_type   = supported_type(get_type(prefix + "experts.up_proj.weight", tensor_storage_map, GGML_TYPE_F32), hidden_size);
@@ -506,8 +528,14 @@ namespace LLM {
                 }
             };
 
-            declare_experts("experts.gate_proj.weight", gate_type, hidden_size, moe_intermediate_size);
-            declare_experts("experts.up_proj.weight", up_type, hidden_size, moe_intermediate_size);
+            fused_experts = tensor_storage_map.count(prefix + "experts.gate_up_proj.weight") != 0;
+            if (fused_experts) {
+                auto type = supported_type(get_type(prefix + "experts.gate_up_proj.weight", tensor_storage_map, GGML_TYPE_F32), hidden_size);
+                declare_experts("experts.gate_up_proj.weight", type, hidden_size, 2 * moe_intermediate_size);
+            } else {
+                declare_experts("experts.gate_proj.weight", gate_type, hidden_size, moe_intermediate_size);
+                declare_experts("experts.up_proj.weight", up_type, hidden_size, moe_intermediate_size);
+            }
             declare_experts("experts.down_proj.weight", down_type, moe_intermediate_size, hidden_size);
         }
 
@@ -519,7 +547,8 @@ namespace LLM {
               num_experts_per_tok(config.num_experts_per_tok),
               n_group(config.n_group),
               topk_group(config.topk_group),
-              routed_scaling_factor(config.routed_scaling_factor) {
+              routed_scaling_factor(config.routed_scaling_factor),
+              image_router(config.arch == LLMArch::BAILING_MOE) {
             if (config.num_shared_experts > 0) {
                 blocks["shared_experts"] = std::make_shared<MLP>(config.hidden_size,
                                                                  config.moe_intermediate_size * config.num_shared_experts,
@@ -582,7 +611,7 @@ namespace LLM {
             return ggml_mul_mat_id(ctx->ggml_ctx, w, x, selected_experts);
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* image_mask = nullptr) {
             // x: [N, n_token, hidden_size]
             GGML_ASSERT(num_experts > 0 && num_experts_per_tok > 0);
             GGML_ASSERT(n_group > 0 && topk_group > 0 && num_experts % n_group == 0);
@@ -596,10 +625,21 @@ namespace LLM {
 
             auto logits = ggml_mul_mat(gctx, params["gate.weight"], x);
             logits      = ggml_reshape_2d(gctx, logits, num_experts, n_token_total);
-            auto scores = ggml_sigmoid(gctx, logits);  // [num_experts, tokens]
+            auto bias   = params["gate.expert_bias"];
+            if (image_router) {
+                GGML_ASSERT(image_mask != nullptr);
+                auto mask         = ggml_reshape_2d(gctx, image_mask, 1, n_token_total);
+                auto inverse_mask = ggml_scale_bias(gctx, mask, -1.f, 1.f);
+                auto image_logits = ggml_reshape_2d(gctx, ggml_mul_mat(gctx, params["image_gate.weight"], x), num_experts, n_token_total);
+                logits            = ggml_add(gctx, ggml_mul(gctx, logits, inverse_mask), ggml_mul(gctx, image_logits, mask));
+                bias              = ggml_add(gctx,
+                                             ggml_mul(gctx, ggml_repeat_4d(gctx, bias, num_experts, n_token_total, 1, 1), inverse_mask),
+                                             ggml_mul(gctx, ggml_repeat_4d(gctx, params["image_gate.expert_bias"], num_experts, n_token_total, 1, 1), mask));
+            }
+            auto scores = ggml_sigmoid(gctx, logits);
 
             // The bias steers selection only; the combine weights come from the unbiased scores.
-            auto routing = ggml_add(gctx, scores, params["gate.expert_bias"]);
+            auto routing = ggml_add(gctx, scores, bias);
             routing      = ggml_add(gctx, routing, group_limited_mask(ctx, routing, n_token_total));
 
             auto selected_experts = ggml_argsort_top_k(gctx, routing, (int)num_experts_per_tok);  // [top_k, tokens]
@@ -614,12 +654,17 @@ namespace LLM {
             weights = ggml_scale(gctx, weights, routed_scaling_factor);
             weights = ggml_reshape_3d(gctx, weights, 1, num_experts_per_tok, n_token_total);
 
-            auto xf        = ggml_reshape_3d(gctx, x, hidden_size, 1, n_token_total);
-            auto gate      = expert_linear(ctx, "experts.gate_proj.weight", xf, selected_experts);
-            auto up        = expert_linear(ctx, "experts.up_proj.weight", xf, selected_experts);
-            auto activated = ggml_swiglu_split(gctx, gate, up);
-            auto experts   = expert_linear(ctx, "experts.down_proj.weight", activated, selected_experts);
-            experts        = ggml_mul(gctx, experts, weights);
+            auto xf = ggml_reshape_3d(gctx, x, hidden_size, 1, n_token_total);
+            ggml_tensor* activated;
+            if (fused_experts) {
+                activated = ggml_swiglu(gctx, expert_linear(ctx, "experts.gate_up_proj.weight", xf, selected_experts));
+            } else {
+                auto gate = expert_linear(ctx, "experts.gate_proj.weight", xf, selected_experts);
+                auto up   = expert_linear(ctx, "experts.up_proj.weight", xf, selected_experts);
+                activated = ggml_swiglu_split(gctx, gate, up);
+            }
+            auto experts = expert_linear(ctx, "experts.down_proj.weight", activated, selected_experts);
+            experts      = ggml_mul(gctx, experts, weights);
 
             ggml_tensor* out = nullptr;
             for (int64_t i = 0; i < num_experts_per_tok; ++i) {
@@ -1428,7 +1473,9 @@ namespace LLM {
                              ggml_tensor* x,
                              ggml_tensor* input_pos,
                              ggml_tensor* attention_mask = nullptr,
-                             int rope_index              = 0) {
+                             int rope_index              = 0,
+                             ggml_tensor* rope_cos       = nullptr,
+                             ggml_tensor* rope_sin       = nullptr) {
             // x: [N, n_token, hidden_size]
             int64_t n_token = x->ne[1];
             int64_t N       = x->ne[2];
@@ -1471,13 +1518,28 @@ namespace LLM {
                 v = ggml_rms_norm(ctx->ggml_ctx, v, rms_norm_eps);
             }
 
-            if (arch == LLMArch::MISTRAL_SMALL_3_2) {
+            if (rope_cos != nullptr) {
+                GGML_ASSERT(rope_sin != nullptr);
+                // Bailing video RoPE interleaves spatial frequencies within a partial NEOX head.
+                auto rotate = [&](ggml_tensor* input) {
+                    auto gctx    = ctx->ggml_ctx;
+                    int64_t half = rope_cos->ne[0];
+                    auto first   = ggml_ext_slice(gctx, input, 0, 0, half);
+                    auto second  = ggml_ext_slice(gctx, input, 0, half, 2 * half);
+                    auto left    = ggml_sub(gctx, ggml_mul(gctx, first, rope_cos), ggml_mul(gctx, second, rope_sin));
+                    auto right   = ggml_add(gctx, ggml_mul(gctx, second, rope_cos), ggml_mul(gctx, first, rope_sin));
+                    auto rotated = ggml_concat(gctx, left, right, 0);
+                    return ggml_concat(gctx, rotated, ggml_ext_slice(gctx, input, 0, 2 * half, input->ne[0]), 0);
+                };
+                q = rotate(q);
+                k = rotate(k);
+            } else if (arch == LLMArch::MISTRAL_SMALL_3_2) {
                 q = ggml_rope_ext(ctx->ggml_ctx, q, input_pos, nullptr, 128, GGML_ROPE_TYPE_NORMAL, 8192, 1000000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
                 k = ggml_rope_ext(ctx->ggml_ctx, k, input_pos, nullptr, 128, GGML_ROPE_TYPE_NORMAL, 8192, 1000000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
             } else if (arch == LLMArch::MINISTRAL_3_3B) {
                 q = ggml_rope_ext(ctx->ggml_ctx, q, input_pos, nullptr, 128, GGML_ROPE_TYPE_NEOX, 262144, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
                 k = ggml_rope_ext(ctx->ggml_ctx, k, input_pos, nullptr, 128, GGML_ROPE_TYPE_NEOX, 262144, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
-            } else if (arch == LLMArch::QWEN3) {
+            } else if (arch == LLMArch::QWEN3 || arch == LLMArch::QWEN2) {
                 q = ggml_rope_ext(ctx->ggml_ctx, q, input_pos, nullptr, 128, GGML_ROPE_TYPE_NEOX, 40960, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
                 k = ggml_rope_ext(ctx->ggml_ctx, k, input_pos, nullptr, 128, GGML_ROPE_TYPE_NEOX, 40960, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
             } else if (arch == LLMArch::GPT_OSS_20B) {
@@ -1655,7 +1717,7 @@ namespace LLM {
                 v_attn      = ggml_reshape_3d(ctx->ggml_ctx, v_attn, n_token, head_dim, num_kv_heads * N);
 
                 auto kq = ggml_mul_mat(ctx->ggml_ctx, k, q);
-                ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
+                ggml_prec_set_acc(kq, GGML_PREC_F32);
                 kq = ggml_scale_inplace(ctx->ggml_ctx, kq, 1.0f / std::sqrt(static_cast<float>(head_dim)));
                 if (attention_mask != nullptr) {
                     kq = ggml_add_inplace(ctx->ggml_ctx, kq, attention_mask);
@@ -1724,7 +1786,7 @@ namespace LLM {
             blocks["self_attn"] = std::make_shared<Attention>(config, sliding_attention == 0);
             if (config.arch == LLMArch::GPT_OSS_20B) {
                 blocks["mlp"] = std::make_shared<GPTOSSMLP>(config);
-            } else if (config.arch == LLMArch::LLADA2_MOE && layer_index >= config.first_k_dense_replace) {
+            } else if ((config.arch == LLMArch::LLADA2_MOE || config.arch == LLMArch::BAILING_MOE) && layer_index >= config.first_k_dense_replace) {
                 blocks["mlp"] = std::make_shared<LLaDA2MoEMLP>(config);
             } else {
                 blocks["mlp"] = std::make_shared<MLP>(config.hidden_size,
@@ -1746,7 +1808,10 @@ namespace LLM {
                              ggml_tensor* x,
                              ggml_tensor* input_pos,
                              ggml_tensor* attention_mask         = nullptr,
-                             ggml_tensor* sliding_attention_mask = nullptr) {
+                             ggml_tensor* sliding_attention_mask = nullptr,
+                             ggml_tensor* image_mask             = nullptr,
+                             ggml_tensor* rope_cos               = nullptr,
+                             ggml_tensor* rope_sin               = nullptr) {
             // x: [N, n_token, hidden_size]
             auto self_attn                                  = std::dynamic_pointer_cast<Attention>(blocks["self_attn"]);
             auto input_layernorm                            = std::dynamic_pointer_cast<LLMRMSNorm>(blocks["input_layernorm"]);
@@ -1768,7 +1833,7 @@ namespace LLM {
 
             auto residual = x;
             x             = input_layernorm->forward(ctx, x);
-            x             = self_attn->forward(ctx, x, input_pos, block_attention_mask, rope_index);
+            x             = self_attn->forward(ctx, x, input_pos, block_attention_mask, rope_index, rope_cos, rope_sin);
             if (post_attention_norm != nullptr) {
                 x = post_attention_norm->forward(ctx, x);
             }
@@ -1782,7 +1847,7 @@ namespace LLM {
             } else if (auto moe_mlp = std::dynamic_pointer_cast<LLaDA2MoEMLP>(blocks["mlp"])) {
                 // LLaDA2 is dense for the first first_k_dense_replace layers and MoE afterwards,
                 // so the block type varies per layer rather than per arch.
-                x = moe_mlp->forward(ctx, x);
+                x = moe_mlp->forward(ctx, x, image_mask);
             } else {
                 auto mlp = std::dynamic_pointer_cast<MLP>(blocks["mlp"]);
                 x        = mlp->forward(ctx, x);
@@ -1804,10 +1869,11 @@ namespace LLM {
     protected:
         int64_t num_layers;
         LLMConfig config;
+        std::string graph_cut_prefix;
 
     public:
-        TextModel(const LLMConfig& config)
-            : num_layers(config.num_layers), config(config) {
+        TextModel(const LLMConfig& config, const std::string& graph_cut_prefix = "llm.text")
+            : num_layers(config.num_layers), config(config), graph_cut_prefix(graph_cut_prefix) {
             blocks["embed_tokens"] = std::shared_ptr<GGMLBlock>(new Embedding(config.vocab_size, config.hidden_size));
             for (int i = 0; i < num_layers; i++) {
                 blocks["layers." + std::to_string(i)] = std::shared_ptr<GGMLBlock>(new TransformerBlock(config, i));
@@ -1831,7 +1897,10 @@ namespace LLM {
                                     std::set<int> out_layers,
                                     const std::vector<std::vector<std::pair<int, ggml_tensor*>>>& deepstack_image_embeds = {},
                                     ggml_tensor* sliding_attention_mask                                                  = nullptr,
-                                    bool return_all_hidden_states                                                        = false) {
+                                    bool return_all_hidden_states                                                        = false,
+                                    ggml_tensor* image_mask                                                              = nullptr,
+                                    ggml_tensor* rope_cos                                                                = nullptr,
+                                    ggml_tensor* rope_sin                                                                = nullptr) {
             auto norm = config.final_norm ? std::dynamic_pointer_cast<LLMRMSNorm>(blocks["norm"])
                                           : nullptr;
             std::vector<ggml_tensor*> intermediate_outputs;
@@ -1843,18 +1912,18 @@ namespace LLM {
                 intermediate_outputs.push_back(x);
             }
 
-            sd::ggml_graph_cut::mark_graph_cut(x, "llm.text.prelude", "x");
+            sd::ggml_graph_cut::mark_graph_cut(x, graph_cut_prefix + ".prelude", "x");
             for (int i = 0; i < num_layers; i++) {
                 auto block = std::dynamic_pointer_cast<TransformerBlock>(blocks["layers." + std::to_string(i)]);
 
-                x = block->forward(ctx, x, input_pos, attention_mask, sliding_attention_mask);
+                x = block->forward(ctx, x, input_pos, attention_mask, sliding_attention_mask, image_mask, rope_cos, rope_sin);
                 if (i < static_cast<int>(deepstack_image_embeds.size())) {
                     x = add_deepstack_image_embeds(ctx, x, deepstack_image_embeds[static_cast<size_t>(i)]);
                 }
                 if (return_all_hidden_states || out_layers.size() > 1) {
                     x = ggml_cont(ctx->ggml_ctx, x);
                 }
-                sd::ggml_graph_cut::mark_graph_cut(x, "llm.text.layers." + std::to_string(i), "x");
+                sd::ggml_graph_cut::mark_graph_cut(x, graph_cut_prefix + ".layers." + std::to_string(i), "x");
                 if (return_all_hidden_states) {
                     if (i + 1 < num_layers) {
                         intermediate_outputs.push_back(x);

src/model/te/ming_image_te.hpp

@@ -0,0 +1,159 @@
+#ifndef __SD_MODEL_TE_MING_IMAGE_TE_HPP__
+#define __SD_MODEL_TE_MING_IMAGE_TE_HPP__
+
+#include "llm.hpp"
+
+namespace MingImageTE {
+    struct MingImageTEConfig {
+        LLM::LLMConfig backbone;
+        LLM::LLMConfig connector;
+        int64_t num_queries   = 256;
+        int64_t caption_dim   = 2560;
+        int64_t diffusion_dim = 3840;
+
+        static MingImageTEConfig detect_from_weights(const String2TensorStorage& tensors, const std::string& prefix) {
+            MingImageTEConfig config;
+            for (const auto& entry : tensors) {
+                if (starts_with(entry.first, prefix + ".backbone.") &&
+                    contains(entry.first, ".mlp.experts.") && entry.second.type == GGML_TYPE_I8) {
+                    throw std::runtime_error("Ming-Image INT8/W4A8 text encoder experts are not supported; use the BF16 text encoder");
+                }
+            }
+            bool vision           = false;
+            config.backbone       = LLM::LLMConfig::detect_from_weights(tensors, prefix + ".backbone.", LLM::LLMArch::BAILING_MOE, vision);
+            config.connector      = LLM::LLMConfig::detect_from_weights(tensors, prefix + ".connector.", LLM::LLMArch::QWEN2, vision);
+            const auto query      = tensors.find(prefix + ".query_tokens_dict.16x16");
+            const auto projection = tensors.find(prefix + ".proj_out.weight");
+            const auto direct     = tensors.find(prefix + ".proj_directvlm.1.weight");
+            if (query == tensors.end() || projection == tensors.end() || direct == tensors.end()) {
+                throw std::runtime_error("Ming-Image requires the learned queries, connector and both condition projections");
+            }
+            config.num_queries   = query->second.ne[1];
+            config.caption_dim   = projection->second.ne[1];
+            config.diffusion_dim = direct->second.ne[1];
+            if (config.num_queries != 256 || config.caption_dim != 2560 || config.diffusion_dim != 3840 ||
+                config.backbone.num_layers != 20 || config.backbone.hidden_size != 2048 ||
+                config.connector.num_layers != 28 || config.connector.hidden_size != 1536) {
+                throw std::runtime_error("unsupported Ming-Image text encoder configuration");
+            }
+            LOG_VERBOSE("ming_image_te: queries = %" PRId64 ", caption_dim = %" PRId64 ", diffusion_dim = %" PRId64,
+                        config.num_queries, config.caption_dim, config.diffusion_dim);
+            return config;
+        }
+    };
+
+    struct ConnectorModel : LLM::TextModel {
+        explicit ConnectorModel(const LLM::LLMConfig& config)
+            : LLM::TextModel(config, "ming_image.connector") {
+            blocks.erase("embed_tokens");
+        }
+    };
+
+    class MingImageTextModel : public GGMLBlock {
+        MingImageTEConfig config;
+
+        void init_params(ggml_context* ctx, const String2TensorStorage& tensors = {}, const std::string prefix = "") override {
+            params["query_tokens_dict.16x16"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, config.backbone.hidden_size, config.num_queries);
+        }
+
+    public:
+        explicit MingImageTextModel(const MingImageTEConfig& config)
+            : config(config) {
+            blocks["backbone"]         = std::make_shared<LLM::TextModel>(config.backbone, "ming_image.backbone");
+            blocks["connector"]        = std::make_shared<ConnectorModel>(config.connector);
+            blocks["proj_in"]          = std::make_shared<Linear>(config.backbone.hidden_size, config.connector.hidden_size);
+            blocks["proj_out"]         = std::make_shared<Linear>(config.connector.hidden_size, config.caption_dim);
+            blocks["proj_directvlm.0"] = std::make_shared<RMSNorm>(config.backbone.hidden_size * 3, 1e-5f);
+            blocks["proj_directvlm.1"] = std::make_shared<Linear>(config.backbone.hidden_size * 3, config.diffusion_dim);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* ids, int64_t prompt_length, ggml_tensor* mask, ggml_tensor* image_mask, ggml_tensor* cos, ggml_tensor* sin, ggml_tensor* connector_positions) {
+            auto gctx                 = ctx->ggml_ctx;
+            auto backbone             = std::dynamic_pointer_cast<LLM::TextModel>(blocks["backbone"]);
+            auto connector            = std::dynamic_pointer_cast<ConnectorModel>(blocks["connector"]);
+            auto x                    = backbone->embed(ctx, ids);
+            const int64_t query_start = prompt_length + 1;
+            auto before               = ggml_ext_slice(gctx, x, 1, 0, query_start);
+            auto after                = ggml_ext_slice(gctx, x, 1, query_start + config.num_queries, x->ne[1]);
+            x                         = ggml_concat(gctx, ggml_concat(gctx, before, params["query_tokens_dict.16x16"], 1), after, 1);
+            // HF hidden_states[20] includes final RMSNorm; sd.cpp selects it as num_layers + 1.
+            x            = backbone->forward_embeds(ctx, x, nullptr, mask, {5, 12, 21}, {}, nullptr, false, image_mask, cos, sin);
+            auto direct  = ggml_ext_slice(gctx, x, 1, 0, prompt_length);
+            direct       = std::dynamic_pointer_cast<RMSNorm>(blocks["proj_directvlm.0"])->forward(ctx, direct);
+            direct       = std::dynamic_pointer_cast<Linear>(blocks["proj_directvlm.1"])->forward(ctx, direct);
+            auto queries = ggml_ext_slice(gctx, x, 0, config.backbone.hidden_size * 2, config.backbone.hidden_size * 3);
+            queries      = ggml_ext_slice(gctx, queries, 1, query_start, query_start + config.num_queries);
+            queries      = std::dynamic_pointer_cast<Linear>(blocks["proj_in"])->forward(ctx, queries);
+            queries      = connector->forward_embeds(ctx, queries, connector_positions, nullptr, {});
+            queries      = std::dynamic_pointer_cast<Linear>(blocks["proj_out"])->forward(ctx, queries);
+            queries      = ggml_pad(gctx, queries, static_cast<int>(config.diffusion_dim - config.caption_dim), 0, 0, 0);
+            return ggml_concat(gctx, queries, direct, 1);
+        }
+    };
+
+    struct MingImageTextRunner : GGMLRunner {
+        MingImageTEConfig config;
+        MingImageTextModel model;
+
+        MingImageTextRunner(ggml_backend_t backend, const String2TensorStorage& tensors, const std::string& prefix, std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+            : GGMLRunner(backend, weight_manager), config(MingImageTEConfig::detect_from_weights(tensors, prefix)), model(config) {
+            model.init(params_ctx, tensors, prefix);
+        }
+
+        std::string get_desc() override { return "ming_image_text"; }
+
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) {
+            model.get_param_tensors(tensors, prefix);
+        }
+
+        void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& ops) {
+            model.get_param_tensor_ops(ops);
+        }
+
+        sd::Tensor<float> compute(int n_threads, const std::vector<int>& tokens) {
+            const int64_t prompt_length = tokens.size();
+            const int64_t query_start   = prompt_length + 1;
+            const int64_t total         = prompt_length + config.num_queries + 2;
+            std::vector<int32_t> ids(tokens.begin(), tokens.end());
+            ids.push_back(157158);
+            ids.insert(ids.end(), config.num_queries, 157157);
+            ids.push_back(157159);
+            auto input = sd::Tensor<int32_t>({total}, std::move(ids));
+            sd::Tensor<float> attention_mask({total, total});
+            sd::Tensor<float> image_mask({1, total});
+            sd::Tensor<float> cos({32, 1, total}), sin({32, 1, total});
+            std::vector<int32_t> positions(config.num_queries);
+            std::iota(positions.begin(), positions.end(), 0);
+            auto connector_positions = sd::Tensor<int32_t>({config.num_queries}, std::move(positions));
+            for (int64_t token = 0; token < total; ++token) {
+                const bool query  = token >= query_start && token < query_start + config.num_queries;
+                image_mask[token] = query ? 1.f : 0.f;
+                for (int64_t key = 0; key < total; ++key) {
+                    attention_mask[key + token * total] = key > token ? -INFINITY : 0.f;
+                }
+                // A 16x16 query bank is represented upstream as a [1, 2, 512] image grid,
+                // then spatially merged and centered to [1, 1, 256].
+                int64_t temporal = query ? query_start : (token == total - 1 ? query_start + 1 : token);
+                int64_t width    = query ? token - 127 : temporal;
+                for (int j = 0; j < 32; ++j) {
+                    int64_t position    = query && j < 24 && j % 2 ? width : temporal;
+                    float frequency     = 1.f / std::pow(600000.f, static_cast<float>(2 * j) / 64.f);
+                    float angle         = static_cast<float>(position) * frequency;
+                    cos[token * 32 + j] = std::cos(angle);
+                    sin[token * 32 + j] = std::sin(angle);
+                }
+            }
+            auto graph = [&]() {
+                auto gf  = new_graph_custom(LLM::LLM_GRAPH_SIZE);
+                auto ctx = get_context();
+                auto out = model.forward(&ctx, make_input(input), prompt_length, make_input(attention_mask),
+                                         make_input(image_mask), make_input(cos), make_input(sin), make_input(connector_positions));
+                ggml_build_forward_expand(gf, out);
+                return gf;
+            };
+            return take_or_empty(GGMLRunner::compute(graph, n_threads));
+        }
+    };
+}
+
+#endif  // __SD_MODEL_TE_MING_IMAGE_TE_HPP__

src/model/vae/auto_encoder_kl.hpp

@@ -544,7 +544,7 @@ public:
                        const String2TensorStorage& tensor_storage_map = {},
                        const std::string& prefix                      = "")
         : version(version), decode_only(decode_only), use_video_decoder(use_video_decoder) {
-        if (sd_version_is_dit(version)) {
+        if (sd_version_is_dit(version) && version != VERSION_PIXART) {
             if (sd_version_uses_flux2_vae(version)) {
                 dd_config.z_channels = 32;
                 embed_dim            = 32;
@@ -678,7 +678,7 @@ struct AutoEncoderKL : public VAE {
         if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
             scale_factor = 0.18215f;
             shift_factor = 0.f;
-        } else if (sd_version_is_sdxl(version)) {
+        } else if (sd_version_is_sdxl(version) || sd_version_is_pixart(version)) {
             scale_factor = 0.13025f;
             shift_factor = 0.f;
         } else if (sd_version_is_sd3(version)) {

src/model/vae/ltx_vae.hpp

@@ -1300,7 +1300,7 @@ struct LTXVideoVAE : public VAE {
             feat_map[feat_idx] = get_cache_tensor_by_name(temporal_feat_cache_name(feat_idx));
         }
 
-        auto runner_ctx  = get_context();
+        auto runner_ctx  = get_context(gf);
         int feat_count   = 0;
         ggml_tensor* out = vae.decode_tiled_chunk(&runner_ctx,
                                                   z,
@@ -1313,8 +1313,7 @@ struct LTXVideoVAE : public VAE {
         for (int feat_idx = 0; feat_idx < feat_count && feat_idx < static_cast<int>(feat_map.size()); ++feat_idx) {
             ggml_tensor* feat_cache = feat_map[static_cast<size_t>(feat_idx)];
             if (feat_cache != nullptr) {
-                cache(temporal_feat_cache_name(static_cast<size_t>(feat_idx)), feat_cache);
-                ggml_build_forward_expand(gf, feat_cache);
+                runner_ctx.persist_cache_tensor(temporal_feat_cache_name(static_cast<size_t>(feat_idx)), feat_cache);
             }
         }
 

src/model/vae/minimax_h3_vae.hpp

@@ -556,12 +556,18 @@ namespace MiniMaxH3VAE {
                                    tensor.shape()[3]});
         }
 
-        static sd_tiling_params_t h3_tiling(sd_tiling_params_t params) {
+        sd_tiling_params_t resolve_tiling_params(sd_tiling_params_t params) const override {
+            if (!params.enabled) {
+                params.target_overlap = 0.25f;
+            }
+            if (params.tile_size_w == 0 && params.rel_size_w == 0.f) {
+                params.tile_size_w = 256;
+            }
+            if (params.tile_size_h == 0 && params.rel_size_h == 0.f) {
+                params.tile_size_h = 256;
+            }
             params.enabled         = true;
             params.temporal_tiling = false;
-            params.tile_size_x     = 16;
-            params.tile_size_y     = 16;
-            params.target_overlap  = 0.25f;
             return params;
         }
 
@@ -605,7 +611,7 @@ namespace MiniMaxH3VAE {
                                  bool circular_x = false,
                                  bool circular_y = false) override {
             auto input  = ensure_video_shape(x);
-            auto tiling = h3_tiling(tiling_params);
+            auto tiling = resolve_tiling_params(tiling_params);
             if (input.shape()[2] == 1) {
                 auto encoded = VAE::encode(n_threads, input, tiling, circular_x, circular_y);
                 if (!encoded.empty() && encoded.shape()[2] > 1) {
@@ -646,7 +652,7 @@ namespace MiniMaxH3VAE {
                                  bool circular_y   = false,
                                  bool silent       = false) override {
             auto input  = ensure_video_shape(x);
-            auto tiling = h3_tiling(tiling_params);
+            auto tiling = resolve_tiling_params(tiling_params);
             if (input.shape()[2] == 1) {
                 auto decoded = VAE::decode(n_threads,
                                            input,

src/model/vae/tae.hpp

@@ -701,7 +701,7 @@ public:
         bool use_midblock_gn = false;
         taef2                = sd_version_uses_flux2_vae(version);
 
-        if (sd_version_is_dit(version)) {
+        if (sd_version_is_dit(version) && !sd_version_is_pixart(version)) {
             z_channels = 16;
         }
         if (taef2) {

src/model/vae/vae.hpp

@@ -1,6 +1,9 @@
 #ifndef __SD_MODEL_VAE_VAE_HPP__
 #define __SD_MODEL_VAE_VAE_HPP__
 
+#include <cmath>
+#include <limits>
+
 #include "core/tensor_ggml.hpp"
 #include "model/common/block.hpp"
 #include "model/vae/vae_tiling.hpp"
@@ -117,8 +120,8 @@ protected:
                                     int output_width,
                                     int output_height,
                                     int scale,
-                                    int p_tile_size_x,
-                                    int p_tile_size_y,
+                                    int p_tile_size_w,
+                                    int p_tile_size_h,
                                     float tile_overlap_factor,
                                     bool circular_x,
                                     bool circular_y,
@@ -138,17 +141,28 @@ protected:
             }
             return output_tile;
         };
-        return ::process_tiles_2d(input,
-                                  output_width,
-                                  output_height,
-                                  scale,
-                                  p_tile_size_x,
-                                  p_tile_size_y,
-                                  tile_overlap_factor,
-                                  circular_x,
-                                  circular_y,
-                                  on_processing,
-                                  silent);
+        const bool original_circular_x = circular_x_enabled;
+        const bool original_circular_y = circular_y_enabled;
+        const int64_t latent_width     = decode_graph ? input.shape()[0] : output_width;
+        const int64_t latent_height    = decode_graph ? input.shape()[1] : output_height;
+        circular_x                     = circular_x || original_circular_x;
+        circular_y                     = circular_y || original_circular_y;
+        // Full-width axes wrap in convolutions; split axes wrap between tiles.
+        set_circular_axes(circular_x && p_tile_size_w >= latent_width,
+                          circular_y && p_tile_size_h >= latent_height);
+        auto output = ::process_tiles_2d(input,
+                                         output_width,
+                                         output_height,
+                                         scale,
+                                         p_tile_size_w,
+                                         p_tile_size_h,
+                                         tile_overlap_factor,
+                                         circular_x && p_tile_size_w < latent_width,
+                                         circular_y && p_tile_size_h < latent_height,
+                                         on_processing,
+                                         silent);
+        set_circular_axes(original_circular_x, original_circular_y);
+        return output;
     }
 
 public:
@@ -178,33 +192,48 @@ public:
         return supports_temporal_tiling(VAETemporalDirection::DECODE);
     }
 
-    void get_tile_sizes(int& tile_size_x,
-                        int& tile_size_y,
+    virtual sd_tiling_params_t resolve_tiling_params(sd_tiling_params_t params) const {
+        return params;
+    }
+
+    bool get_tile_sizes(int& tile_size_w,
+                        int& tile_size_h,
                         float& tile_overlap,
                         const sd_tiling_params_t& params,
-                        int64_t latent_x,
-                        int64_t latent_y,
-                        float encoding_factor = 1.0f) {
-        tile_overlap       = std::max(std::min(params.target_overlap, 0.5f), 0.0f);
-        auto get_tile_size = [&](int requested_size, float factor, int64_t latent_size) {
-            const int default_tile_size  = 32;
-            const int min_tile_dimension = 4;
-            int tile_size                = default_tile_size;
-            // factor <= 1 means simple fraction of the latent dimension
-            // factor > 1 means number of tiles across that dimension
-            if (factor > 0.f) {
-                if (factor > 1.0)
-                    factor = 1 / (factor - factor * tile_overlap + tile_overlap);
-                tile_size = static_cast<int>(std::round(latent_size * factor));
-            } else if (requested_size >= min_tile_dimension) {
-                tile_size = requested_size;
+                        int64_t latent_w,
+                        int64_t latent_h) {
+        const auto tiling = resolve_tiling_params(params);
+        if (latent_w <= 0 || latent_h <= 0 ||
+            latent_w > std::numeric_limits<int>::max() || latent_h > std::numeric_limits<int>::max() ||
+            !std::isfinite(tiling.target_overlap)) {
+            LOG_ERROR("invalid VAE tiling dimensions or overlap");
+            return false;
+        }
+        const int scale_factor = get_scale_factor();
+        tile_overlap           = std::max(std::min(tiling.target_overlap, 0.5f), 0.0f);
+        auto get_tile_size     = [&](int requested_size, double factor, int64_t latent_size, int& tile_size) {
+            if (requested_size < 0 || !std::isfinite(factor) || factor < 0.0) {
+                LOG_ERROR("VAE tile sizes and relative sizes must be finite and non-negative");
+                return false;
             }
-            tile_size = static_cast<int>(tile_size * encoding_factor);
-            return std::max(std::min(tile_size, static_cast<int>(latent_size)), min_tile_dimension);
+            const int min_tile_dimension = std::min(4, static_cast<int>(latent_size));
+            double size                  = (requested_size > 0 ? requested_size : 256) / scale_factor;
+            if (factor > 0.0) {
+                if (factor > 1.0) {
+                    factor = 1.0 / (factor * (1.0 - tile_overlap) + tile_overlap);
+                }
+                size = std::floor(static_cast<double>(latent_size) * factor);
+            }
+            if (size < min_tile_dimension && (requested_size > 0 || factor > 0.0)) {
+                LOG_ERROR("VAE tile size must be at least %d image pixels on this axis", min_tile_dimension * scale_factor);
+                return false;
+            }
+            tile_size = static_cast<int>(std::min(static_cast<double>(latent_size), std::max<double>(min_tile_dimension, size)));
+            return true;
         };
 
-        tile_size_x = get_tile_size(params.tile_size_x, params.rel_size_x, latent_x);
-        tile_size_y = get_tile_size(params.tile_size_y, params.rel_size_y, latent_y);
+        return get_tile_size(tiling.tile_size_w, tiling.rel_size_w, latent_w, tile_size_w) &&
+               get_tile_size(tiling.tile_size_h, tiling.rel_size_h, latent_h, tile_size_h);
     }
 
     virtual sd::Tensor<float> encode(int n_threads,
@@ -213,6 +242,7 @@ public:
                                      bool circular_x = false,
                                      bool circular_y = false) {
         int64_t t0              = ggml_time_ms();
+        tiling_params           = resolve_tiling_params(tiling_params);
         sd::Tensor<float> input = x;
         sd::Tensor<float> output;
         if (scale_input) {
@@ -224,21 +254,19 @@ public:
             int64_t W              = input.shape()[0] / scale_factor;
             int64_t H              = input.shape()[1] / scale_factor;
             float tile_overlap;
-            int tile_size_x, tile_size_y;
-            // Image VAE encode is more sensitive to tile boundary context than decode.
-            // Keep the smaller legacy factor for video VAEs, but default image encode
-            // tiles to 64 latent pixels so a 512px SD image is encoded as one tile.
-            const float encode_tile_factor = sd_version_is_minimax_h3(version) ? 1.f : (sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version)) ? 1.30539f
-                                                                                                                                                                                            : 2.0f;
-            get_tile_sizes(tile_size_x, tile_size_y, tile_overlap, tiling_params, W, H, encode_tile_factor);
-            LOG_VERBOSE("VAE Tile size: %dx%d", tile_size_x, tile_size_y);
+            int tile_size_w, tile_size_h;
+            if (!get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params, W, H)) {
+                return {};
+            }
+            LOG_VERBOSE("VAE encode tile size: %dx%d pixels (%dx%d latent)",
+                        tile_size_w * scale_factor, tile_size_h * scale_factor, tile_size_w, tile_size_h);
             output = tiled_compute(input,
                                    n_threads,
                                    static_cast<int>(W),
                                    static_cast<int>(H),
                                    scale_factor,
-                                   tile_size_x,
-                                   tile_size_y,
+                                   tile_size_w,
+                                   tile_size_h,
                                    tile_overlap,
                                    circular_x,
                                    circular_y,
@@ -271,6 +299,7 @@ public:
                                      bool circular_y   = false,
                                      bool silent       = false) {
         int64_t t0              = ggml_time_ms();
+        tiling_params           = resolve_tiling_params(tiling_params);
         sd::Tensor<float> input = x;
         sd::Tensor<float> output;
 
@@ -279,10 +308,13 @@ public:
             int64_t W              = input.shape()[0] * scale_factor;
             int64_t H              = input.shape()[1] * scale_factor;
             float tile_overlap;
-            int tile_size_x, tile_size_y;
-            get_tile_sizes(tile_size_x, tile_size_y, tile_overlap, tiling_params, input.shape()[0], input.shape()[1]);
+            int tile_size_w, tile_size_h;
+            if (!get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params, input.shape()[0], input.shape()[1])) {
+                return {};
+            }
             if (!silent) {
-                LOG_VERBOSE("VAE Tile size: %dx%d", tile_size_x, tile_size_y);
+                LOG_VERBOSE("VAE decode tile size: %dx%d pixels (%dx%d latent)",
+                            tile_size_w * scale_factor, tile_size_h * scale_factor, tile_size_w, tile_size_h);
             }
             output = tiled_compute(
                 input,
@@ -290,8 +322,8 @@ public:
                 static_cast<int>(W),
                 static_cast<int>(H),
                 scale_factor,
-                tile_size_x,
-                tile_size_y,
+                tile_size_w,
+                tile_size_h,
                 tile_overlap,
                 circular_x,
                 circular_y,

src/model/vae/wan_vae.hpp

@@ -24,6 +24,7 @@ namespace WAN {
         std::tuple<int, int, int> padding;
         std::tuple<int, int, int> dilation;
         bool bias;
+        float scale = 1.f;
 
         void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
             auto weight = tensor_storage_map.find(prefix + "weight");
@@ -60,6 +61,10 @@ namespace WAN {
               dilation(std::move(dilation)),
               bias(bias) {}
 
+        void set_scale(float scale_value) {
+            scale = scale_value;
+        }
+
         ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* cache_x = nullptr) {
             // x: [N*IC, ID, IH, IW]
             // result: x: [N*OC, ID, IH, IW]
@@ -82,11 +87,25 @@ namespace WAN {
             }
 
             x = ggml_ext_pad_ext(ctx->ggml_ctx, ctx->backend, x, lp0, rp0, lp1, rp1, lp2, rp2, 0, 0, ctx->circular_x_enabled, ctx->circular_y_enabled);
+            if (w->ne[2] == 1 && x->ne[2] == 1 && x->ne[3] == in_channels) {
+                // One frame through a one-frame-deep kernel is a 2D conv; backends without
+                // im2col_3d (Metal) otherwise fall back to a much slower direct conv_3d.
+                if (!ggml_is_contiguous(x)) {
+                    x = ggml_cont(ctx->ggml_ctx, x);
+                }
+                ggml_tensor* x2 = ggml_reshape_4d(ctx->ggml_ctx, x, x->ne[0], x->ne[1], in_channels, 1);
+                ggml_tensor* w2 = ggml_reshape_4d(ctx->ggml_ctx, w, w->ne[0], w->ne[1], in_channels, out_channels);
+                x2              = ggml_ext_conv_2d(ctx->ggml_ctx, x2, w2, b,
+                                                   std::get<2>(stride), std::get<1>(stride), 0, 0,
+                                                   std::get<2>(dilation), std::get<1>(dilation),
+                                                   ctx->conv2d_direct_enabled, false, false, scale);
+                return ggml_reshape_4d(ctx->ggml_ctx, x2, x2->ne[0], x2->ne[1], 1, out_channels);
+            }
             return ggml_ext_conv_3d(ctx->ggml_ctx, ctx->backend, x, w, b, in_channels,
                                     std::get<2>(stride), std::get<1>(stride), std::get<0>(stride),
                                     0, 0, 0,
                                     std::get<2>(dilation), std::get<1>(dilation), std::get<0>(dilation),
-                                    false, ctx->conv3d_direct_enabled);
+                                    false, ctx->conv3d_direct_enabled, scale);
         }
     };
 
@@ -1065,7 +1084,7 @@ namespace WAN {
 
                 _conv_num     = 34;
                 _enc_conv_num = 26;
-            } else if (version == VERSION_QWEN_IMAGE_LAYERED) {
+            } else if (version == VERSION_QWEN_IMAGE_LAYERED || version == VERSION_MING_IMAGE) {
                 input_channels = 4;
             }
 
@@ -1103,6 +1122,19 @@ namespace WAN {
             } else {
                 blocks["conv2"] = std::shared_ptr<GGMLBlock>(new CausalConv3d(z_dim, z_dim, {1, 1, 1}));
             }
+            if (version == VERSION_QWEN_IMAGE_2_1) {
+                // Keep large VAE activations within the FP16 convolution range.
+                const float conv_scale = 1.f / 128.f;
+                std::vector<GGMLBlock*> all_blocks;
+                get_all_blocks(all_blocks);
+                for (auto block : all_blocks) {
+                    if (auto conv = dynamic_cast<Conv2d*>(block)) {
+                        conv->set_scale(conv_scale);
+                    } else if (auto conv = dynamic_cast<CausalConv3d*>(block)) {
+                        conv->set_scale(conv_scale);
+                    }
+                }
+            }
         }
 
         static ggml_tensor* patchify(ggml_context* ctx,
@@ -1391,11 +1423,17 @@ namespace WAN {
         }
 
         sd::Tensor<float> diffusion_to_vae_latents(const sd::Tensor<float>& latents) override {
+            if (version == VERSION_MING_IMAGE) {
+                return latents / 8.0064f;
+            }
             auto [mean_tensor, std_tensor] = get_latents_mean_std(latents);
             return (latents * std_tensor) / scale_factor + mean_tensor;
         }
 
         sd::Tensor<float> vae_to_diffusion_latents(const sd::Tensor<float>& latents) override {
+            if (version == VERSION_MING_IMAGE) {
+                return latents * 8.0064f;
+            }
             auto [mean_tensor, std_tensor] = get_latents_mean_std(latents);
             return ((latents - mean_tensor) * scale_factor) / std_tensor;
         }
@@ -1429,15 +1467,14 @@ namespace WAN {
 
             ggml_tensor* z = make_input(z_tensor);
 
-            auto runner_ctx = get_context();
+            auto runner_ctx = get_context(gf);
 
             ggml_tensor* out = ae.decode_tiled_chunk(&runner_ctx, z, chunk_idx);
 
             for (size_t feat_idx = 0; feat_idx < ae._feat_map.size(); feat_idx++) {
                 ggml_tensor* feat_cache = ae._feat_map[feat_idx];
                 if (feat_cache != nullptr) {
-                    cache("feat_idx:" + std::to_string(feat_idx), feat_cache);
-                    ggml_build_forward_expand(gf, feat_cache);
+                    runner_ctx.persist_cache_tensor("feat_idx:" + std::to_string(feat_idx), feat_cache);
                 }
             }
 

src/model_io/gguf_io.cpp

@@ -47,10 +47,14 @@ bool read_gguf_file(const std::string& file_path,
     gguf_context* ctx_gguf_ = nullptr;
     ggml_context* ctx_meta_ = nullptr;
 
-    ctx_gguf_ = gguf_init_from_file(file_path.c_str(), {true, &ctx_meta_});
+    GGUFReader gguf_reader;
+    bool probe_ok = gguf_reader.load(file_path);
+
+    if (!probe_ok || !gguf_reader.has_tensors_beyond_ggml_limits()) {
+        ctx_gguf_ = gguf_init_from_file(file_path.c_str(), {true, &ctx_meta_});
+    }
     if (!ctx_gguf_) {
-        GGUFReader gguf_reader;
-        if (!gguf_reader.load(file_path)) {
+        if (!probe_ok && !gguf_reader.load(file_path)) {
             set_error(error, "failed to open '" + file_path + "' with GGUFReader");
             return false;
         }

src/model_io/gguf_reader_ext.h

@@ -35,23 +35,72 @@ enum class GGUFMetadataType : uint32_t {
 class GGUFReader {
 private:
     std::vector<GGUFTensorInfo> tensors_;
+    bool has_wide_tensors_    = false;
+    uint64_t remaining_bytes_ = 0;
     size_t data_offset_;
     size_t alignment_ = 32;  // default alignment is 32
 
     template <typename T>
     bool safe_read(std::ifstream& fin, T& value) {
-        fin.read(reinterpret_cast<char*>(&value), sizeof(T));
-        return fin.good();
+        return safe_read(fin, reinterpret_cast<char*>(&value), sizeof(T));
     }
 
     bool safe_read(std::ifstream& fin, char* buffer, size_t size) {
+        if (size > remaining_bytes_)
+            return false;
         fin.read(buffer, size);
-        return fin.good();
+        if (!fin.good())
+            return false;
+        remaining_bytes_ -= size;
+        return true;
     }
 
-    bool safe_seek(std::ifstream& fin, std::streamoff offset, std::ios::seekdir dir) {
-        fin.seekg(offset, dir);
-        return fin.good();
+    bool safe_skip(std::ifstream& fin, uint64_t count, uint64_t element_size = 1) {
+        if (count > remaining_bytes_ / element_size)
+            return false;
+        uint64_t size = count * element_size;
+        fin.seekg(static_cast<std::streamoff>(size), std::ios::cur);
+        if (!fin.good())
+            return false;
+        remaining_bytes_ -= size;
+        return true;
+    }
+
+    bool skip_metadata_values(std::ifstream& fin, GGUFMetadataType type, uint64_t count) {
+        switch (type) {
+            case GGUFMetadataType::UINT8:
+            case GGUFMetadataType::INT8:
+            case GGUFMetadataType::BOOL:
+                return safe_skip(fin, count);
+
+            case GGUFMetadataType::UINT16:
+            case GGUFMetadataType::INT16:
+                return safe_skip(fin, count, 2);
+
+            case GGUFMetadataType::UINT32:
+            case GGUFMetadataType::INT32:
+            case GGUFMetadataType::FLOAT32:
+                return safe_skip(fin, count, 4);
+
+            case GGUFMetadataType::UINT64:
+            case GGUFMetadataType::INT64:
+            case GGUFMetadataType::FLOAT64:
+                return safe_skip(fin, count, 8);
+
+            case GGUFMetadataType::STRING:
+                if (count > remaining_bytes_ / sizeof(uint64_t))
+                    return false;
+                for (uint64_t i = 0; i < count; i++) {
+                    uint64_t len = 0;
+                    if (!safe_read(fin, len) || !safe_skip(fin, len))
+                        return false;
+                }
+                return true;
+
+            default:
+                LOG_ERROR("Unknown metadata type=%u", static_cast<uint32_t>(type));
+                return false;
+        }
     }
 
     bool read_metadata(std::ifstream& fin) {
@@ -84,52 +133,12 @@ private:
             return true;
         }
 
-        switch (static_cast<GGUFMetadataType>(type)) {
-            case GGUFMetadataType::UINT8:
-            case GGUFMetadataType::INT8:
-            case GGUFMetadataType::BOOL:
-                return safe_seek(fin, 1, std::ios::cur);
-
-            case GGUFMetadataType::UINT16:
-            case GGUFMetadataType::INT16:
-                return safe_seek(fin, 2, std::ios::cur);
-
-            case GGUFMetadataType::UINT32:
-            case GGUFMetadataType::INT32:
-            case GGUFMetadataType::FLOAT32:
-                return safe_seek(fin, 4, std::ios::cur);
-
-            case GGUFMetadataType::UINT64:
-            case GGUFMetadataType::INT64:
-            case GGUFMetadataType::FLOAT64:
-                return safe_seek(fin, 8, std::ios::cur);
-
-            case GGUFMetadataType::STRING: {
-                uint64_t len = 0;
-                if (!safe_read(fin, len))
-                    return false;
-                return safe_seek(fin, len, std::ios::cur);
-            }
-
-            case GGUFMetadataType::ARRAY: {
-                uint32_t elem_type = 0;
-                uint64_t len       = 0;
-                if (!safe_read(fin, elem_type))
-                    return false;
-                if (!safe_read(fin, len))
-                    return false;
-
-                for (uint64_t i = 0; i < len; i++) {
-                    if (!read_metadata(fin))
-                        return false;
-                }
-                return true;
-            }
-
-            default:
-                LOG_ERROR("Unknown metadata type=%u", type);
+        uint64_t count = 1;
+        if (type == static_cast<uint32_t>(GGUFMetadataType::ARRAY)) {
+            if (!safe_read(fin, type) || !safe_read(fin, count))
                 return false;
         }
+        return skip_metadata_values(fin, static_cast<GGUFMetadataType>(type), count);
     }
 
     GGUFTensorInfo read_tensor_info(std::ifstream& fin) {
@@ -154,6 +163,7 @@ private:
         }
 
         if (n_dims > GGML_MAX_DIMS) {
+            has_wide_tensors_ = true;
             for (uint32_t i = GGML_MAX_DIMS; i < n_dims; i++) {
                 info.shape[GGML_MAX_DIMS - 1] *= info.shape[i];  // stack to last dim;
             }
@@ -174,12 +184,20 @@ private:
 
 public:
     bool load(const std::string& file_path) {
-        std::ifstream fin(file_path, std::ios::binary);
+        std::ifstream fin(file_path, std::ios::binary | std::ios::ate);
         if (!fin) {
             LOG_ERROR("failed to open '%s'", file_path.c_str());
             return false;
         }
 
+        std::streamoff file_size = fin.tellg();
+        if (file_size < 0)
+            return false;
+        remaining_bytes_ = static_cast<uint64_t>(file_size);
+        fin.seekg(0, std::ios::beg);
+        if (!fin.good())
+            return false;
+
         // --- Header ---
         char magic[4];
         if (!safe_read(fin, magic, 4) || strncmp(magic, "GGUF", 4) != 0) {
@@ -228,6 +246,8 @@ public:
     }
 
     const std::vector<GGUFTensorInfo>& tensors() const { return tensors_; }
+
+    bool has_tensors_beyond_ggml_limits() const { return has_wide_tensors_; }
     size_t data_offset() const { return data_offset_; }
 };
 

src/model_loader.cpp

@@ -322,7 +322,13 @@ bool ModelLoader::init_from_safetensors_index_file(const std::string& file_path,
     }
 
     for (const std::string& shard_path : shard_paths) {
-        if (!parse_file(shard_path, prefix)) {
+        FileStamp stamp;
+        if (!read_file_stamp(shard_path, stamp)) {
+            return false;
+        }
+        parsed_dependencies_.push_back(stamp);
+        LOG_INFO("load %s using safetensors format", shard_path.c_str());
+        if (!init_from_safetensors_file(shard_path, prefix)) {
             return false;
         }
     }
@@ -518,6 +524,9 @@ SDVersion ModelLoader::get_sd_version() const {
             return VERSION_LLADA_IMAGE;
         }
         if (tensor_storage.name.find("model.diffusion_model.cap_embedder.0.weight") != std::string::npos) {
+            if (tensor_storage_map.find("text_encoders.llm.connector.layers.0.self_attn.q_proj.weight") != tensor_storage_map.end()) {
+                return VERSION_MING_IMAGE;
+            }
             return VERSION_Z_IMAGE;
         }
         if (tensor_storage.name.find("double_stream_layers.0.img_instruct_attn.processor.img_to_q.weight") != std::string::npos) {
@@ -526,7 +535,18 @@ SDVersion ModelLoader::get_sd_version() const {
         if (tensor_storage.name.find("model.diffusion_model.layers.0.adaLN_sa_ln.weight") != std::string::npos) {
             return VERSION_ERNIE_IMAGE;
         }
+        if (tensor_storage.name.find("model.diffusion_model.t_block.1.weight") != std::string::npos &&
+            tensor_storage_map.find("model.diffusion_model.x_embedder.proj.weight") != tensor_storage_map.end() &&
+            tensor_storage_map.find("model.diffusion_model.audio_patchify_proj.weight") == tensor_storage_map.end()) {
+            return VERSION_PIXART;
+        }
         if (tensor_storage.name.find("model.diffusion_model.adaln_single.emb.timestep_embedder.linear_1.bias") != std::string::npos) {
+            // PixArt shares this timestep embedding with LTX-AV.
+            if (tensor_storage_map.find("model.diffusion_model.pos_embed.proj.weight") != tensor_storage_map.end() &&
+                tensor_storage_map.find("model.diffusion_model.adaln_single.linear.weight") != tensor_storage_map.end() &&
+                tensor_storage_map.find("model.diffusion_model.audio_patchify_proj.weight") == tensor_storage_map.end()) {
+                return VERSION_PIXART;
+            }
             return VERSION_LTXAV;
         }
         if (tensor_storage.name.find("model.diffusion_model.video_patch_proj.weight") != std::string::npos &&
@@ -874,7 +894,8 @@ void ModelLoader::process_model_files(bool enable_mmap, bool writable_mmap) {
 
 std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggml_tensor*>& tensors,
                                                        std::set<std::string> ignore_tensors,
-                                                       bool writable_mmap) {
+                                                       bool writable_mmap,
+                                                       ggml_backend_dev_t device) {
     std::set<std::string> names;
     for (const auto& entry : tensors) {
         names.insert(entry.first);
@@ -896,6 +917,39 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
         if (!fdata.mmbuffer)
             continue;
 
+        // Wrapped on first use: a device buffer makes the whole file resident on that device.
+        std::shared_ptr<struct ggml_backend_buffer> file_buffer = device == nullptr ? fdata.mmbuffer : nullptr;
+        bool file_unmappable                                    = false;
+
+        auto buffer_for_file = [&]() -> ggml_backend_buffer_t {
+            if (file_buffer || file_unmappable) {
+                return file_buffer.get();
+            }
+            auto cached = fdata.device_mmbuffers.find(device);
+            if (cached != fdata.device_mmbuffers.end()) {
+                file_buffer = cached->second;
+                return file_buffer.get();
+            }
+            size_t max_tensor_size = 0;
+            for (const auto& ts : fdata.tensors) {
+                max_tensor_size = std::max(max_tensor_size, static_cast<size_t>(ts.nbytes()));
+            }
+            ggml_backend_buffer_t buf = sd_backend_dev_buffer_from_host_ptr(device,
+                                                                            fdata.mmapped->writable_data(),
+                                                                            fdata.mmapped->size(),
+                                                                            max_tensor_size);
+            if (buf == nullptr) {
+                LOG_WARN("mmap: %s cannot map '%s', loading it instead",
+                         ggml_backend_dev_name(device), fdata.path.c_str());
+                file_unmappable = true;
+                return nullptr;
+            }
+            LOG_INFO("mmap: mapped '%s' for %s", fdata.path.c_str(), ggml_backend_dev_name(device));
+            file_buffer                    = std::shared_ptr<struct ggml_backend_buffer>(buf, ggml_backend_buffer_free);
+            fdata.device_mmbuffers[device] = file_buffer;
+            return file_buffer.get();
+        };
+
         const std::vector<TensorStorage>& file_tensors = fdata.tensors;
 
         size_t file_mapped_bytes   = 0;
@@ -944,10 +998,13 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
                 continue;
             }
 
-            ggml_backend_buffer_t buf_mmap = fdata.mmbuffer.get();
-            uint8_t* mmap_data             = static_cast<uint8_t*>(ggml_backend_buffer_get_base(buf_mmap));
-            dst_tensor->buffer             = buf_mmap;
-            dst_tensor->data               = mmap_data + tensor_offset;
+            ggml_backend_buffer_t buf_mmap = buffer_for_file();
+            if (buf_mmap == nullptr) {
+                break;
+            }
+            uint8_t* mmap_data = static_cast<uint8_t*>(ggml_backend_buffer_get_base(buf_mmap));
+            dst_tensor->buffer = buf_mmap;
+            dst_tensor->data   = mmap_data + tensor_offset;
 
             file_mapped_bytes += tensor_size;
             file_mapped_tensors++;
@@ -956,7 +1013,7 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
         if (file_mapped_bytes > 0) {
             mapped_tensors += file_mapped_tensors;
             mapped_bytes += file_mapped_bytes;
-            result.push_back({fdata.mmapped, fdata.mmbuffer});
+            result.push_back({fdata.mmapped, file_buffer});
         }
     }
 
@@ -972,6 +1029,16 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
     return result;
 }
 
+std::vector<ggml_backend_buffer_t> ModelLoader::get_device_mmap_buffers() const {
+    std::vector<ggml_backend_buffer_t> buffers;
+    for (const auto& fdata : file_data) {
+        for (const auto& entry : fdata.device_mmbuffers) {
+            buffers.push_back(entry.second.get());
+        }
+    }
+    return buffers;
+}
+
 bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                                bool enable_mmap,
                                const std::set<std::string>* target_tensor_names,
@@ -1037,10 +1104,12 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
         if (tensors_to_process.empty()) {
             continue;
         }
-        LOG_VERBOSE("loading %zu/%zu tensors from %s",
-                    tensors_to_process.size(),
-                    file_tensors.size(),
-                    file_path.c_str());
+        if (log_progress) {
+            LOG_VERBOSE("loading %zu/%zu tensors from %s",
+                        tensors_to_process.size(),
+                        file_tensors.size(),
+                        file_path.c_str());
+        }
 
         bool is_zip = fdata.is_zip;
 
@@ -1115,6 +1184,11 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                     if (dst_tensor->buffer != nullptr && dst_tensor->buffer == fdata.mmbuffer.get()) {
                         continue;
                     }
+                    if (dst_tensor->buffer != nullptr &&
+                        std::any_of(fdata.device_mmbuffers.begin(), fdata.device_mmbuffers.end(),
+                                    [&](const auto& entry) { return entry.second.get() == dst_tensor->buffer; })) {
+                        continue;
+                    }
 
                     size_t nbytes_to_read = tensor_storage.nbytes_to_read();
 
@@ -1539,6 +1613,9 @@ bool ModelLoader::tensor_should_be_converted(const TensorStorage& tensor_storage
             // Pass, do not convert. For Unet
         } else if (contains(name, "embedding")) {
             // Pass, do not convert embedding
+        } else if (contains(name, "scale_shift_table")) {
+            // Pass, do not convert. adaLN modulation tables (PixArt, LTXV) are sliced
+            // element-wise, which is invalid on quantized block layouts.
         } else if (ends_with(name, "_pad_token")) {
             // Pass, do not convert. LLaDA-Image stores its pad tokens far outside the f16
             // range, so any format with an f16 scale or payload turns them into inf.

src/model_loader.h

@@ -20,6 +20,8 @@ struct ModelFileData {
     std::vector<TensorStorage> tensors;
     std::shared_ptr<MmapWrapper> mmapped;
     std::shared_ptr<struct ggml_backend_buffer> mmbuffer;
+    // mmapped wrapped by devices that can use host memory in place (buffer_from_host_ptr)
+    std::map<ggml_backend_dev_t, std::shared_ptr<struct ggml_backend_buffer>> device_mmbuffers;
     bool is_zip;
 };
 
@@ -120,7 +122,9 @@ public:
     void process_model_files(bool enable_mmap = false, bool writable_mmap = true);
     std::vector<MmapTensorStore> mmap_tensors(std::map<std::string, ggml_tensor*>& tensors,
                                               std::set<std::string> ignore_tensors = {},
-                                              bool writable                        = true);
+                                              bool writable                        = true,
+                                              ggml_backend_dev_t device            = nullptr);
+    std::vector<ggml_backend_buffer_t> get_device_mmap_buffers() const;
     bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                       bool use_mmap                                    = false,
                       const std::set<std::string>* target_tensor_names = nullptr,

src/model_manager.cpp

@@ -79,6 +79,10 @@ static bool device_supports_param_op(ggml_backend_dev_t device,
     if (op == GGML_OP_GET_ROWS) {
         ggml_tensor* indices = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);
         op_tensor            = ggml_get_rows(ctx, weight, indices);
+    } else if (op == GGML_OP_CUSTOM) {
+        op_tensor         = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+        op_tensor->op     = op;
+        op_tensor->src[0] = weight;
     }
     if (op_tensor == nullptr) {
         ggml_free(ctx);
@@ -246,17 +250,13 @@ bool ModelManager::register_param_tensors(ModelComponent component,
         }
         ggml_set_name(tensor, name.c_str());
 
-        auto state            = std::make_unique<TensorState>();
-        state->name           = name;
-        state->tensor         = tensor;
-        state->component      = component;
-        state->source_file    = source_file;
-        state->source_version = source_version;
-        auto source           = sources.find(name);
-        if (source != sources.end()) {
-            state->source     = source->second;
-            state->has_source = true;
-        }
+        auto state             = std::make_unique<TensorState>();
+        state->name            = name;
+        state->tensor          = tensor;
+        state->component       = component;
+        state->source_file     = source_file;
+        state->source_version  = source_version;
+        state->sources         = find_tensor_sources(*state, sources);
         state->residency_mode  = residency_mode;
         state->compute_backend = compute_backend;
         state->params_backend  = params_backend;
@@ -511,7 +511,9 @@ bool ModelManager::stage_tensors_to_compute_backend(const std::vector<TensorStat
             LOG_ERROR("model manager params backend is null for tensor '%s'", state->name.c_str());
             return false;
         }
-        if (state->compute_backend == state->params_backend || state->staged_to_compute_backend) {
+        // Custom CPU operators must retain host weights even when the runner uses a GPU.
+        if (state->usage_op == GGML_OP_CUSTOM ||
+            state->compute_backend == state->params_backend || state->staged_to_compute_backend) {
             continue;
         }
         if (!state->loaded_to_params_backend || state->tensor == nullptr || state->tensor->data == nullptr) {
@@ -757,12 +759,33 @@ bool ModelManager::validate_tensor(const TensorState& state) const {
         return true;
     }
 
-    if (!state.has_source) {
+    if (state.sources.empty()) {
         LOG_ERROR("%s tensor '%s' not in model metadata", model_component_name(state.component), state.name.c_str());
         return false;
     }
 
-    const TensorStorage& tensor_storage = state.source;
+    TensorStorage tensor_storage = state.sources.front();
+    if (state.sources.size() > 1) {
+        const int dim = tensor_storage.n_dims - 1;
+        if (dim < 0 || dim >= GGML_MAX_DIMS) {
+            return false;
+        }
+        tensor_storage.ne[dim] = 0;
+        for (const auto& part : state.sources) {
+            if (part.n_dims != tensor_storage.n_dims || part.ne[dim] < 0 ||
+                part.ne[dim] > state.tensor->ne[dim] - tensor_storage.ne[dim]) {
+                LOG_ERROR("invalid tensor part '%s' for '%s'", part.name.c_str(), state.name.c_str());
+                return false;
+            }
+            for (int i = 0; i < GGML_MAX_DIMS; ++i) {
+                if (i != dim && part.ne[i] != tensor_storage.ne[i]) {
+                    LOG_ERROR("incompatible tensor part '%s' for '%s'", part.name.c_str(), state.name.c_str());
+                    return false;
+                }
+            }
+            tensor_storage.ne[dim] += part.ne[dim];
+        }
+    }
     if (state.tensor->ne[0] != tensor_storage.ne[0] ||
         state.tensor->ne[1] != tensor_storage.ne[1] ||
         state.tensor->ne[2] != tensor_storage.ne[2] ||
@@ -780,44 +803,58 @@ bool ModelManager::validate_tensor(const TensorState& state) const {
 
 bool ModelManager::mmap_params(const std::vector<TensorState*>& states,
                                std::vector<ParamsStorageBlock*>& created_storage_blocks) {
-    std::map<std::string, ggml_tensor*> mmap_candidates;
-    std::map<std::string, TensorState*> mmap_states;
+    // A GPU that computes on mmapped params in place cannot address a CPU buffer, and nothing
+    // stages them for it, so they are mapped through a buffer of that GPU's device.
+    struct MmapGroup {
+        std::map<std::string, ggml_tensor*> candidates;
+        std::map<std::string, TensorState*> states;
+    };
+    std::map<ggml_backend_dev_t, MmapGroup> groups;
     for (TensorState* state : states) {
         if (state == nullptr || !can_mmap_storage(*state) || state->tensor == nullptr ||
             state->tensor->data != nullptr || state->tensor->view_src != nullptr) {
             continue;
         }
-        mmap_candidates[state->name] = state->tensor;
-        mmap_states[state->name]     = state;
-    }
-    if (mmap_candidates.empty()) {
-        return true;
-    }
-
-    auto mmap_store = model_loader_.mmap_tensors(mmap_candidates, {}, writable_mmap_);
-    if (mmap_store.empty()) {
-        return true;
-    }
-
-    auto block                = std::make_unique<ParamsStorageBlock>();
-    block->mmap_tensor_stores = std::move(mmap_store);
-    ParamsStorageBlock* raw   = block.get();
-    for (const auto& pair : mmap_states) {
-        TensorState* state = pair.second;
-        if (state != nullptr && state->tensor != nullptr && state->tensor->data != nullptr) {
-            block->states.push_back(state);
+        ggml_backend_dev_t device = nullptr;
+        if (!sd_backend_is_cpu(state->compute_backend) && !sd_backend_is_cpu(state->params_backend)) {
+            device = ggml_backend_get_device(state->compute_backend);
         }
+        MmapGroup& group              = groups[device];
+        group.candidates[state->name] = state->tensor;
+        group.states[state->name]     = state;
     }
 
-    if (!block->states.empty()) {
-        params_storage_blocks_.push_back(std::move(block));
-        created_storage_blocks.push_back(raw);
+    for (auto& [device, group] : groups) {
+        // Device buffers wrap read-only mappings only; params that LoRAs are merged into in place
+        // are loaded instead.
+        if (device != nullptr && writable_mmap_) {
+            continue;
+        }
+        auto mmap_store = model_loader_.mmap_tensors(group.candidates, {}, writable_mmap_, device);
+        if (mmap_store.empty()) {
+            continue;
+        }
+
+        auto block                = std::make_unique<ParamsStorageBlock>();
+        block->mmap_tensor_stores = std::move(mmap_store);
+        ParamsStorageBlock* raw   = block.get();
+        for (const auto& pair : group.states) {
+            TensorState* state = pair.second;
+            if (state != nullptr && state->tensor != nullptr && state->tensor->data != nullptr) {
+                block->states.push_back(state);
+            }
+        }
+
+        if (!block->states.empty()) {
+            params_storage_blocks_.push_back(std::move(block));
+            created_storage_blocks.push_back(raw);
+        }
     }
     return true;
 }
 
 bool ModelManager::can_mmap_storage(const TensorState& state) const {
-    if (state.source_file != 0 || !enable_mmap_ || state.residency_mode != ResidencyMode::ParamBackend) {
+    if (state.sources.size() > 1 || state.source_file != 0 || !enable_mmap_ || state.residency_mode != ResidencyMode::ParamBackend) {
         return false;
     }
     if (state.compute_backend == nullptr || state.params_backend == nullptr) {
@@ -927,6 +964,62 @@ bool ModelManager::alloc_params_buffers(const std::vector<TensorState*>& states,
     return true;
 }
 
+bool ModelManager::load_tensor_parts(TensorState& state) {
+    auto ctx = std::unique_ptr<ggml_context, decltype(&ggml_free)>(
+        ggml_init({state.sources.size() * ggml_tensor_overhead(), nullptr, true}), ggml_free);
+    if (!ctx) {
+        return false;
+    }
+    const size_t size = ggml_nbytes(state.tensor);
+    std::vector<uint8_t> buffer;
+    void* data = state.tensor->data;
+    if (!ggml_backend_buffer_is_host(state.tensor->buffer)) {
+        buffer.resize(size);
+        data = buffer.data();
+    }
+    std::map<std::string, ggml_tensor*> parts;
+    std::set<std::string> names;
+    size_t offset = 0;
+    for (const auto& source : state.sources) {
+        auto part              = ggml_new_tensor(ctx.get(), state.tensor->type, source.n_dims, source.ne);
+        const size_t part_size = ggml_nbytes(part);
+        if (part_size > size - offset) {
+            return false;
+        }
+        part->data         = static_cast<uint8_t*>(data) + offset;
+        parts[source.name] = part;
+        names.insert(source.name);
+        offset += part_size;
+    }
+    if (offset != size) {
+        return false;
+    }
+    std::set<std::string> loaded;
+    std::mutex mutex;
+    auto callback = [&](const TensorStorage& source, ggml_tensor** dst) {
+        *dst      = nullptr;
+        auto part = parts.find(source.name);
+        if (part != parts.end()) {
+            *dst = part->second;
+            std::lock_guard<std::mutex> lock(mutex);
+            loaded.insert(source.name);
+        }
+        return true;
+    };
+    const bool success = state.source_file == 0
+                             ? model_loader_.load_tensors(callback, enable_mmap_, &names, false)
+                             : model_loader_.load_file_tensors(state.source_file, state.source_version, callback, names, enable_mmap_);
+    if (!success || loaded != names) {
+        return false;
+    }
+    if (!buffer.empty()) {
+        // Upload the assembled tensor once, including for row-split backend buffers.
+        ggml_backend_tensor_set(state.tensor, buffer.data(), 0, size);
+    }
+    state.loaded_to_params_backend = true;
+    return true;
+}
+
 bool ModelManager::load_tensors(const std::vector<TensorState*>& states) {
     using ReadGroup = std::pair<ModelLoader::FileId, SDVersion>;
     using ReadBatch = std::map<std::string, std::vector<TensorState*>>;
@@ -934,6 +1027,12 @@ bool ModelManager::load_tensors(const std::vector<TensorState*>& states) {
     for (auto* state : states) {
         if (state == nullptr)
             continue;
+        if (state->sources.size() > 1) {
+            if (!load_tensor_parts(*state)) {
+                return false;
+            }
+            continue;
+        }
         auto& batches = groups[{state->source_file, state->source_version}];
         // The loader supplies one destination per name; only conflicting types need another batch.
         auto batch = std::find_if(batches.begin(), batches.end(), [&](const ReadBatch& candidate) {
@@ -1309,6 +1408,9 @@ size_t ModelManager::compute_backend_alloc_size(const std::vector<TensorState*>&
         if (state == nullptr || state->tensor == nullptr) {
             continue;
         }
+        if (state->usage_op == GGML_OP_CUSTOM && !sd_backend_is_cpu(state->compute_backend)) {
+            continue;
+        }
         const bool compute_resident =
             state->compute_backend == state->params_backend
                 ? state->loaded_to_params_backend
@@ -1353,15 +1455,16 @@ size_t ModelManager::compute_backend_resident_bytes(ggml_backend_t compute_backe
     }
 
     size_t total_size = 0;
-    auto add_buffer   = [&](ggml_backend_buffer_t buffer) {
-        if (buffer == nullptr || ggml_backend_buffer_is_host(buffer)) {
+    std::unordered_set<ggml_backend_buffer_t> seen;
+    auto add_buffer = [&](ggml_backend_buffer_t buffer) {
+        if (buffer == nullptr || ggml_backend_buffer_is_host(buffer) || !seen.insert(buffer).second) {
             return;
         }
         ggml_backend_buffer_type_t buffer_type = ggml_backend_buffer_get_type(buffer);
         auto split_devices                     = split_buffer_devices_.find(buffer_type);
         const bool on_device                   = split_devices == split_buffer_devices_.end()
-                                                       ? buffer_type != nullptr && ggml_backend_buft_get_device(buffer_type) == compute_device
-                                                       : std::any_of(split_devices->second.begin(), split_devices->second.end(), [&](const auto& entry) {
+                                                     ? buffer_type != nullptr && ggml_backend_buft_get_device(buffer_type) == compute_device
+                                                     : std::any_of(split_devices->second.begin(), split_devices->second.end(), [&](const auto& entry) {
                                          return ggml_backend_get_device(entry.first) == compute_device;
                                      });
         if (!on_device) {
@@ -1371,9 +1474,16 @@ size_t ModelManager::compute_backend_resident_bytes(ggml_backend_t compute_backe
         total_size               = buffer_size > SIZE_MAX - total_size ? SIZE_MAX : total_size + buffer_size;
     };
 
+    // The loader may retain device mappings after their parameter blocks are released.
+    for (ggml_backend_buffer_t buffer : model_loader_.get_device_mmap_buffers()) {
+        add_buffer(buffer);
+    }
     for (const auto& block : params_storage_blocks_) {
         if (block != nullptr) {
             add_buffer(block->buffer);
+            for (const auto& store : block->mmap_tensor_stores) {
+                add_buffer(store.mmbuffer.get());
+            }
         }
     }
     for (const auto& block : compute_staging_blocks_) {
@@ -1613,7 +1723,8 @@ void ModelManager::remove_runtime_owner(uintptr_t owner_id) {
 
 ModelManager::CapacityCheck ModelManager::check_capacity(
     const DeviceMemoryRequest& request,
-    const std::vector<TensorState*>& states) const {
+    const std::vector<TensorState*>& states,
+    bool log_details) const {
     CapacityCheck result;
     if (request.compute_backend == nullptr || sd_backend_is_cpu(request.compute_backend)) {
         return result;
@@ -1631,16 +1742,23 @@ ModelManager::CapacityCheck ModelManager::check_capacity(
         }
         size_t free_bytes = 0, total_bytes = 0;
         ggml_backend_dev_memory(device, &free_bytes, &total_bytes);
+        const size_t weights_resident = compute_backend_resident_bytes(backend);
+        const size_t other_runtime    = other_runtime_resident_bytes(request.owner_id, backend);
+        const size_t resident         = add(weights_resident, add(other_runtime, request.runtime_resident_bytes));
+        if (log_details) {
+            LOG_WARN("model manager memory on %s: reported free %.2f MB / total %.2f MB, tracked weights %.2f MB / other runtime %.2f MB / current runtime %.2f MB",
+                        ggml_backend_name(backend),
+                        free_bytes / (1024.0 * 1024.0), total_bytes / (1024.0 * 1024.0),
+                        weights_resident / (1024.0 * 1024.0), other_runtime / (1024.0 * 1024.0),
+                        request.runtime_resident_bytes / (1024.0 * 1024.0));
+        }
         if (free_bytes == 0 && total_bytes == 0) {
             return SIZE_MAX;
         }
         // Vulkan's heap budget subtraction can underflow when usage exceeds the budget.
-        if (total_bytes > 0 && free_bytes > total_bytes) {
+        if (total_bytes > 0 && free_bytes > total_bytes && sd_backend_is(backend, "Vulkan")) {
             return size_t{0};
         }
-        const size_t resident = add(compute_backend_resident_bytes(backend),
-                                       add(other_runtime_resident_bytes(request.owner_id, backend),
-                                           request.runtime_resident_bytes));
         if (total_bytes > 0) {
             free_bytes = std::min(free_bytes, resident < total_bytes ? total_bytes - resident : 0);
         }
@@ -1786,7 +1904,7 @@ bool ModelManager::ensure_compute_backend_capacity(
         }
     }
 
-    const auto capacity                = check_capacity(request, required_states);
+    const auto capacity                = check_capacity(request, required_states, true);
     const std::string available_device = capacity.available_device_bytes == SIZE_MAX
                                              ? "unknown"
                                              : sd_format("%.2f MB", capacity.available_device_bytes / (1024.0 * 1024.0));

src/model_manager.h

@@ -38,8 +38,7 @@ private:
         std::string name;
         ggml_tensor* tensor      = nullptr;
         ModelComponent component = ModelComponent::Count;
-        TensorStorage source;
-        bool has_source                 = false;
+        std::vector<TensorStorage> sources;
         ModelLoader::FileId source_file = 0;
         SDVersion source_version        = VERSION_COUNT;
 
@@ -138,6 +137,8 @@ private:
     bool apply_loras_to_params(const std::vector<TensorState*>& states);
     bool mmap_params(const std::vector<TensorState*>& states,
                      std::vector<ParamsStorageBlock*>& created_storage_blocks);
+    static std::vector<TensorStorage> find_tensor_sources(const TensorState& state, const String2TensorStorage& sources);
+    bool load_tensor_parts(TensorState& state);
     bool can_mmap_storage(const TensorState& state) const;
     bool alloc_params_buffers(const std::vector<TensorState*>& states,
                               std::vector<ParamsStorageBlock*>& created_storage_blocks);
@@ -157,7 +158,8 @@ private:
         }
     };
     CapacityCheck check_capacity(const DeviceMemoryRequest& request,
-                                 const std::vector<TensorState*>& states) const;
+                                 const std::vector<TensorState*>& states,
+                                 bool log_details = false) const;
 
     ggml_backend_buffer_type_t params_buffer_type_for(const TensorState& state) const;
     ggml_backend_buffer_type_t split_buffer_type_for(const TensorState& state) const;

src/model_manager_files.cpp

@@ -15,6 +15,26 @@ static bool same_tensor_source(const TensorStorage& a, const TensorStorage& b) {
            a.int8_convrot_group_size == b.int8_convrot_group_size;
 }
 
+std::vector<TensorStorage> ModelManager::find_tensor_sources(const TensorState& state, const String2TensorStorage& sources) {
+    auto first = sources.find(state.name);
+    if (first == sources.end()) {
+        return {};
+    }
+    std::vector<TensorStorage> result{first->second};
+    if (state.component == ModelComponent::LoRA ||
+        std::equal(first->second.ne, first->second.ne + GGML_MAX_DIMS, state.tensor->ne)) {
+        return result;
+    }
+    for (size_t i = 1;; ++i) {
+        auto part = sources.find(state.name + "." + std::to_string(i));
+        if (part == sources.end()) {
+            break;
+        }
+        result.push_back(part->second);
+    }
+    return result;
+}
+
 void ModelManager::invalidate_sources(const std::unordered_set<TensorState*>& states) {
     auto affected = states;
     for (const auto& block : params_storage_blocks_) {
@@ -76,20 +96,16 @@ bool ModelManager::set_loader(ModelLoader loader) {
     }
     std::unordered_set<TensorState*> changed;
     for (const auto& state : tensor_states_) {
-        const auto& sources = sources_for(*state);
-        auto source         = sources.find(state->name);
-        const bool found    = source != sources.end();
-        if (found != state->has_source || (found && !same_tensor_source(state->source, source->second)) ||
+        const auto sources = find_tensor_sources(*state, sources_for(*state));
+        if (sources.size() != state->sources.size() ||
+            !std::equal(sources.begin(), sources.end(), state->sources.begin(), same_tensor_source) ||
             (lora_changed && state->component != ModelComponent::LoRA && state->applied_lora_epoch != UINT64_MAX)) {
             changed.insert(state.get());
         }
     }
     invalidate_sources(changed);
     for (auto* state : changed) {
-        const auto& sources = sources_for(*state);
-        auto source         = sources.find(state->name);
-        state->has_source   = source != sources.end();
-        state->source       = state->has_source ? source->second : TensorStorage{};
+        state->sources = find_tensor_sources(*state, sources_for(*state));
     }
     if (lora_changed) {
         ++current_lora_epoch_;
@@ -134,9 +150,8 @@ ModelLoader::FileVersions ModelManager::source_versions(const std::set<ModelComp
             versions[state->source_file] = loader.file_revision(state->source_file);
             continue;
         }
-        auto source = sources.find(state->name);
-        if (source != sources.end()) {
-            versions[source->second.file_id] = source->second.file_revision;
+        for (const auto& source : find_tensor_sources(*state, sources)) {
+            versions[source.file_id] = source.file_revision;
         }
     }
     return versions;

src/model_manager_prefetch.cpp

@@ -210,7 +210,7 @@ WeightPrefetchResult ModelManager::prefetch_params(
             is_optional_missing_tensor(state->name)) {
             continue;
         }
-        if (state->compute_backend == state->params_backend) {
+        if (state->usage_op == GGML_OP_CUSTOM || state->compute_backend == state->params_backend) {
             needs_synchronous_load = needs_synchronous_load ||
                                      !state->loaded_to_params_backend;
             continue;
@@ -275,6 +275,7 @@ bool ModelManager::activate_prefetched_params(
         [&](TensorState* state) {
             return state == nullptr || should_ignore(*state) ||
                    is_optional_missing_tensor(state->name) ||
+                   state->usage_op == GGML_OP_CUSTOM ||
                    state->compute_backend == state->params_backend ||
                    state->staged_to_compute_backend;
         });

src/name_conversion.cpp

@@ -105,7 +105,25 @@ std::string convert_open_clip_to_hf_clip_name(std::string name) {
 
 std::string convert_llada2_moe_te_name(std::string name);
 
-std::string convert_cond_stage_model_name(std::string name, std::string prefix) {
+static std::string convert_ming_image_te_name(std::string name) {
+    if (name == "llm.query_tokens") {
+        name = "llm.query_tokens_dict.16x16";
+    }
+    static const std::vector<std::pair<std::string, std::string>> name_map = {
+        {"thinker.", "backbone."},
+        {"attention.", "self_attn."},
+        {"self_attn.dense.", "self_attn.o_proj."},
+        {"gate.proj.", "gate."},
+    };
+    replace_with_name_map(name, name_map);
+    return name;
+}
+
+std::string convert_cond_stage_model_name(std::string name, std::string prefix, SDVersion version) {
+    if (version == VERSION_MING_IMAGE && prefix == "text_encoders." && starts_with(name, "llm.")) {
+        name = convert_ming_image_te_name(name);
+    }
+
     static const std::vector<std::pair<std::string, std::string>> clip_name_map{
         {"transformer.text_projection.weight", "transformer.text_model.text_projection"},
         {"model.text_projection.weight", "transformer.text_model.text_projection"},
@@ -932,6 +950,79 @@ static bool is_diffusers_controlnet_name(const std::string& name) {
     return false;
 }
 
+static std::string convert_diffusers_dit_to_original_pixart(std::string name) {
+    static const std::vector<std::pair<std::string, std::string>> prefix_map = {
+        {"pos_embed.proj.", "x_embedder.proj."},
+        {"adaln_single.emb.timestep_embedder.linear_1.", "t_embedder.mlp.0."},
+        {"adaln_single.emb.timestep_embedder.linear_2.", "t_embedder.mlp.2."},
+        {"adaln_single.emb.resolution_embedder.linear_1.", "csize_embedder.mlp.0."},
+        {"adaln_single.emb.resolution_embedder.linear_2.", "csize_embedder.mlp.2."},
+        {"adaln_single.emb.aspect_ratio_embedder.linear_1.", "ar_embedder.mlp.0."},
+        {"adaln_single.emb.aspect_ratio_embedder.linear_2.", "ar_embedder.mlp.2."},
+        {"adaln_single.linear.", "t_block.1."},
+        {"caption_projection.linear_1.", "y_embedder.y_proj.fc1."},
+        {"caption_projection.linear_2.", "y_embedder.y_proj.fc2."},
+        {"proj_out.", "final_layer.linear."},
+    };
+    for (const auto& entry : prefix_map) {
+        if (starts_with(name, entry.first)) {
+            return entry.second + name.substr(entry.first.size());
+        }
+    }
+    if (name == "scale_shift_table") {
+        return "final_layer.scale_shift_table";
+    }
+    const std::string block_prefix = "transformer_blocks.";
+    if (!starts_with(name, block_prefix)) {
+        return name;
+    }
+    const size_t block_end = name.find('.', block_prefix.size());
+    if (block_end == std::string::npos) {
+        return name;
+    }
+    const std::string prefix                                                = "blocks." + name.substr(block_prefix.size(), block_end - block_prefix.size()) + ".";
+    name                                                                    = name.substr(block_end + 1);
+    static const std::vector<std::pair<std::string, std::string>> block_map = {
+        {"attn1.to_q.", "attn.qkv."},
+        {"attn1.to_out.0.", "attn.proj."},
+        {"attn2.to_q.", "cross_attn.q_linear."},
+        {"attn2.to_k.", "cross_attn.kv_linear."},
+        {"attn2.to_out.0.", "cross_attn.proj."},
+        {"ff.net.0.proj.", "mlp.fc1."},
+        {"ff.net.2.", "mlp.fc2."},
+    };
+    for (const auto& entry : block_map) {
+        if (starts_with(name, entry.first)) {
+            return prefix + entry.second + name.substr(entry.first.size());
+        }
+    }
+    static const std::vector<std::pair<std::string, std::string>> part_map = {
+        {"attn1.to_k.weight", "attn.qkv.weight.1"},
+        {"attn1.to_k.bias", "attn.qkv.bias.1"},
+        {"attn1.to_v.weight", "attn.qkv.weight.2"},
+        {"attn1.to_v.bias", "attn.qkv.bias.2"},
+        {"attn2.to_v.weight", "cross_attn.kv_linear.weight.1"},
+        {"attn2.to_v.bias", "cross_attn.kv_linear.bias.1"},
+    };
+    for (const auto& entry : part_map) {
+        if (name == entry.first || starts_with(name, entry.first + ".")) {
+            return prefix + entry.second + name.substr(entry.first.size());
+        }
+    }
+    return prefix + name;
+}
+
+static std::string convert_ming_image_dit_name(std::string name) {
+    static const std::vector<std::pair<std::string, std::string>> name_map = {
+        {"all_x_embedder.2-1.", "x_embedder."},
+        {"all_final_layer.2-1.", "final_layer."},
+        {"attention.norm_q.", "attention.q_norm."},
+        {"attention.norm_k.", "attention.k_norm."},
+    };
+    replace_with_name_map(name, name_map);
+    return name;
+}
+
 std::string convert_diffusion_model_name(std::string name, std::string prefix, SDVersion version) {
     if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
         name = convert_diffusers_unet_to_original_sd1(name);
@@ -943,6 +1034,8 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S
         name = convert_diffusers_dit_to_original_flux(name);
     } else if (sd_version_is_hunyuan_video(version)) {
         name = convert_hunyuan_video_to_original_flux(name);
+    } else if (version == VERSION_MING_IMAGE) {
+        name = convert_ming_image_dit_name(name);
     } else if (sd_version_is_z_image(version)) {
         name = convert_diffusers_dit_to_original_lumina2(name);
     } else if (sd_version_is_llada_image(version)) {
@@ -951,6 +1044,8 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S
         name = convert_other_dit_to_original_anima(name);
     } else if (sd_version_is_krea2(version)) {
         name = convert_diffusers_dit_to_original_krea2(name);
+    } else if (sd_version_is_pixart(version)) {
+        name = convert_diffusers_dit_to_original_pixart(name);
     }
     return name;
 }
@@ -1349,9 +1444,8 @@ bool is_first_stage_model_name(const std::string& name) {
 }
 
 static std::string convert_esrgan_tensor_name(std::string name) {
-    static std::unordered_map<std::string, std::string> esrgan_name_map;
-
-    if (esrgan_name_map.empty()) {
+    static const auto esrgan_name_map = [] {
+        std::unordered_map<std::string, std::string> esrgan_name_map;
         esrgan_name_map["model.0."] = "conv_first.";
 
         constexpr int max_num_blocks = 64;
@@ -1377,7 +1471,8 @@ static std::string convert_esrgan_tensor_name(std::string name) {
         esrgan_name_map["model.7."]  = "conv_last.";
         esrgan_name_map["model.8."]  = "conv_hr.";
         esrgan_name_map["model.10."] = "conv_last.";
-    }
+        return esrgan_name_map;
+    }();
 
     replace_with_prefix_map(name, esrgan_name_map);
     return name;
@@ -1604,7 +1699,7 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
     {
         for (const auto& prefix : cond_stage_model_prefix_vec) {
             if (starts_with(name, prefix)) {
-                name = convert_cond_stage_model_name(name.substr(prefix.size()), prefix);
+                name = convert_cond_stage_model_name(name.substr(prefix.size()), prefix, version);
                 name = prefix + name;
                 break;
             }

src/pipeline/diffusion_engine.cpp

@@ -7,6 +7,7 @@
 #include <list>
 #include <mutex>
 #include <set>
+#include <tuple>
 #include <type_traits>
 #include <unordered_set>
 #include <utility>
@@ -29,6 +30,7 @@
 #include "stable-diffusion.h"
 
 #include "conditioning/conditioner.hpp"
+#include "conditioning/conditioning_cache.h"
 #include "core/backend_fit.h"
 #include "extensions/generation_extension.h"
 #include "model/adapter/ip_adapter.hpp"
@@ -103,6 +105,8 @@ const char* model_version_to_str[] = {
     "SenseNova U1.5",
     "LLaDA-Image",
     "ESRGAN",
+    "PixArt",
+    "Ming-Image",
 };
 
 static_assert(VERSION_COUNT == sizeof(model_version_to_str) / sizeof(model_version_to_str[0]),
@@ -123,6 +127,18 @@ void calculate_alphas_cumprod(float* alphas_cumprod,
     }
 }
 
+void calculate_alphas_cumprod_linear_beta(float* alphas_cumprod,
+                                          float beta_start,
+                                          float beta_end,
+                                          int timesteps = TIMESTEPS) {
+    float product = 1.0f;
+    for (int i = 0; i < timesteps; i++) {
+        float beta = beta_start + (beta_end - beta_start) * ((float)i / (timesteps - 1));
+        product *= 1.0f - beta;
+        alphas_cumprod[i] = product;
+    }
+}
+
 template <typename T, typename = void>
 struct has_set_runtime_backends : std::false_type {};
 template <typename T>
@@ -135,6 +151,7 @@ static_assert(std::atomic<sd_cancel_mode_t>::is_always_lock_free,
 
 StableDiffusionGGML::StableDiffusionGGML()
     : rng(std::make_shared<PhiloxRNG>()),
+      conditioning_cache_(std::make_unique<ConditioningCache>()),
       denoiser(std::make_shared<CompVisDenoiser>()) {}
 
 StableDiffusionGGML::~StableDiffusionGGML() = default;
@@ -204,6 +221,8 @@ void StableDiffusionGGML::end_runners() {
 }
 
 bool StableDiffusionGGML::reset_runners(const RunnerGroups& groups) {
+    conditioning_cache_->clear();
+    conditioning_loras_.clear();
     end_runners();
     clear_lora_adapters();
     runtime_lora_models.clear();
@@ -661,6 +680,10 @@ void StableDiffusionGGML::refresh_compvis_denoiser_sigmas() {
     std::vector<float> alphas_cumprod(TIMESTEPS);
     if (file_alphas_cumprod.size() == TIMESTEPS) {
         alphas_cumprod = file_alphas_cumprod;
+    } else if (sd_version_is_pixart(version)) {
+        // PixArt checkpoints train with a linear beta schedule (0.0001 -> 0.02)
+        // instead of the scaled_linear schedule used by SD1.x/SDXL.
+        calculate_alphas_cumprod_linear_beta(alphas_cumprod.data(), 0.0001f, 0.02f);
     } else {
         calculate_alphas_cumprod(alphas_cumprod.data());
     }
@@ -915,6 +938,11 @@ bool StableDiffusionGGML::init(const sd_ctx_params_t* sd_ctx_params) {
             return false;
         }
     }
+    if (sd_ctx_params->conditioning_cache_size < 0) {
+        LOG_ERROR("conditioning_cache_size must be non-negative");
+        return false;
+    }
+    conditioning_cache_->set_capacity(static_cast<size_t>(sd_ctx_params->conditioning_cache_size));
     auto configuration = std::make_unique<ModelConfig>(*sd_ctx_params);
     n_threads          = sd_ctx_params->n_threads;
     tensor_executor    = std::make_unique<sd::ParallelExecutor>(n_threads > 0 ? n_threads : sd_get_num_physical_cores());
@@ -1183,6 +1211,11 @@ bool StableDiffusionGGML::validate_and_load_runners() {
     LOG_VERBOSE("validating model metadata");
 
     std::set<std::string> ignore_tensors;
+    if (version == VERSION_MING_IMAGE) {
+        ignore_tensors.insert("text_encoders.llm.vision.");
+        ignore_tensors.insert("text_encoders.llm.linear_proj.");
+        ignore_tensors.insert("text_encoders.llm.backbone.lm_head.");
+    }
     if (use_tae && !tae_preview_only) {
         ignore_tensors.insert("first_stage_model.");
     }
@@ -1343,6 +1376,7 @@ bool StableDiffusionGGML::build_denoiser() {
                    sd_version_is_anima(version) ||
                    sd_version_is_ernie_image(version) ||
                    sd_version_is_z_image(version) ||
+                   version == VERSION_MING_IMAGE ||
                    sd_version_is_llada_image(version) ||
                    sd_version_is_boogu_image(version) ||
                    sd_version_is_pid(version) ||
@@ -1364,6 +1398,8 @@ bool StableDiffusionGGML::build_denoiser() {
                 default_flow_shift = 3.16f;
             } else if (sd_version_is_mage_flow(version)) {
                 default_flow_shift = 6.f;
+            } else if (version == VERSION_MING_IMAGE) {
+                default_flow_shift = INFINITY;
             } else if (sd_version_is_llada_image(version)) {
                 default_flow_shift = 1.0f;  // unused: LLADA_IMAGE_SCHEDULER builds a fixed grid
             } else {
@@ -1424,6 +1460,8 @@ bool StableDiffusionGGML::build_denoiser() {
             } else if (sd_version_is_minimax_h3(version)) {
                 LOG_INFO("running in MiniMax H3 AV FLOW mode");
                 denoiser = std::make_shared<H3AVFlowDenoiser>(default_flow_shift, 3.f, get_latent_channel());
+            } else if (version == VERSION_MING_IMAGE) {
+                denoiser = std::make_shared<MingImageFlowDenoiser>();
             } else {
                 LOG_INFO("running in FLOW mode");
                 denoiser = std::make_shared<DiscreteFlowDenoiser>();
@@ -1765,8 +1803,19 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun
     int64_t t0 = ggml_time_ms();
     end_runners();
     clear_lora_adapters();
-    if (!model_manager->prepare_lora_sources(all_loras))
+    if (!model_manager->prepare_lora_sources(all_loras)) {
+        conditioning_cache_->clear();
         return false;
+    }
+    if (!std::equal(all_loras.begin(), all_loras.end(),
+                    conditioning_loras_.begin(), conditioning_loras_.end(),
+                    [](const ModelManager::LoraSpec& a, const ModelManager::LoraSpec& b) {
+                        return a.file_id == b.file_id && a.file_revision == b.file_revision &&
+                               a.multiplier == b.multiplier && a.is_high_noise == b.is_high_noise &&
+                               a.tensor_name_prefix_filter == b.tensor_name_prefix_filter;
+                    })) {
+        conditioning_cache_->clear();
+    }
     runtime_lora_models.erase(std::remove_if(runtime_lora_models.begin(), runtime_lora_models.end(), [&](const RuntimeLora& entry) {
                                   return std::none_of(all_loras.begin(), all_loras.end(), [&](const ModelManager::LoraSpec& spec) {
                                       return entry.matches(spec);
@@ -1776,6 +1825,7 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun
     const bool success = apply_lora_immediately ? apply_loras_immediately(all_loras)
                                                 : apply_loras_at_runtime(all_loras);
     if (!success) {
+        conditioning_cache_->clear();
         clear_lora_adapters();
         runtime_lora_models.clear();
         return false;
@@ -1785,9 +1835,14 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun
     if (!all_loras.empty()) {
         LOG_INFO("apply_loras completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
     }
+    conditioning_loras_ = std::move(all_loras);
     return true;
 }
 
+SDCondition StableDiffusionGGML::get_learned_condition(const ConditionerParams& params) {
+    return conditioning_cache_->get(*cond_stage_model, n_threads, params);
+}
+
 void StableDiffusionGGML::reset_generation_extensions() {
     for (auto& extension : generation_extensions) {
         extension->reset_runtime_condition();
@@ -1972,6 +2027,8 @@ void StableDiffusionGGML::preview_image(int step,
         int patch_sz                     = 1;
         const float(*latent_rgb_proj)[3] = nullptr;
         float* latent_rgb_bias           = nullptr;
+        const float* latent_alpha_proj   = nullptr;
+        float latent_alpha_bias          = 1.f;
 
         if (channels == 128) {
             if (sd_version_uses_flux2_vae(version)) {
@@ -1985,6 +2042,16 @@ void StableDiffusionGGML::preview_image(int step,
                 LOG_WARN("No latent to RGB projection known for this model");
                 return;
             }
+        } else if (channels == 64) {
+            if (version == VERSION_QWEN_IMAGE_2_1) {
+                latent_rgb_proj   = qwen21_latent_rgb_proj;
+                latent_rgb_bias   = qwen21_latent_rgb_bias;
+                latent_alpha_proj = qwen21_latent_alpha_proj;
+                latent_alpha_bias = qwen21_latent_alpha_bias;
+            } else {
+                LOG_WARN("No latent to RGB projection known for this model");
+                return;
+            }
         } else if (channels == 48) {
             if (sd_version_is_wan(version)) {
                 latent_rgb_proj = wan_22_latent_rgb_proj;
@@ -2035,13 +2102,14 @@ void StableDiffusionGGML::preview_image(int step,
         uint32_t img_width  = static_cast<uint32_t>(_latents.shape()[0]) * patch_sz;
         uint32_t img_height = static_cast<uint32_t>(_latents.shape()[1]) * patch_sz;
 
-        uint8_t* data = (uint8_t*)malloc(frames * img_width * img_height * 3 * sizeof(uint8_t));
+        uint32_t img_channels = latent_alpha_proj != nullptr ? 4 : 3;
+        uint8_t* data         = (uint8_t*)malloc(frames * img_width * img_height * img_channels * sizeof(uint8_t));
         GGML_ASSERT(data != nullptr);
-        preview_latent_video(data, _latents, latent_rgb_proj, latent_rgb_bias, patch_sz);
+        preview_latent_video(data, _latents, latent_rgb_proj, latent_rgb_bias, patch_sz, latent_alpha_proj, latent_alpha_bias);
         sd_image_t* images = (sd_image_t*)malloc(frames * sizeof(sd_image_t));
         GGML_ASSERT(images != nullptr);
         for (uint32_t i = 0; i < frames; i++) {
-            images[i] = {img_width, img_height, 3, data + i * img_width * img_height * 3};
+            images[i] = {img_width, img_height, img_channels, data + i * img_width * img_height * img_channels};
         }
         step_callback(step, frames, images, is_noisy, step_callback_data);
         free(data);
@@ -2104,7 +2172,7 @@ std::vector<float> StableDiffusionGGML::prepare_sample_timesteps(float sigma,
     if (version == VERSION_HIDREAM_O1) {
         return std::vector<float>{1.0f - (t / static_cast<float>(TIMESTEPS))};
     }
-    if (sd_version_is_z_image(version) || sd_version_is_ideogram4(version)) {
+    if (sd_version_is_z_image(version) || sd_version_is_ideogram4(version) || version == VERSION_MING_IMAGE) {
         return std::vector<float>{1000.f - t};
     }
     return std::vector<float>{t};
@@ -2216,6 +2284,15 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
     };
     RunnerEndOnExit sample_diffusion_runner_end{work_diffusion_model.get()};
 
+    // These inputs are immutable for this sampling run. Extensions may replace or
+    // modify them per step, so those paths need an explicit stability contract first.
+    const bool cache_qwen_prefix = version == VERSION_QWEN_IMAGE_2_1 &&
+                                   std::none_of(generation_extensions.begin(), generation_extensions.end(),
+                                                [](const auto& extension) { return extension->is_enabled(); });
+    using QwenPrefixInputs = std::tuple<const sd::Tensor<float>*, const sd::Tensor<int32_t>*,
+                                        const std::vector<sd::Tensor<float>>*>;
+    std::vector<QwenPrefixInputs> qwen_prefix_inputs;
+
     RunnerEndOnExit sample_control_runner_end{!control_image.empty() && control_net != nullptr ? control_net.get() : nullptr};
 
     const bool apply_denoise_mask = !denoise_mask.empty() &&
@@ -2392,6 +2469,7 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
                                 timesteps_tensor,
                                 cond,
                                 &controls);
+        bool uncond_controls_ready = false;
 
         static const std::vector<sd::Tensor<float>> empty_ref_latents;
         bool uncond_without_ref_latents = !img_uncond.empty() &&
@@ -2447,6 +2525,9 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
                     condition.c_token_types.empty() ? nullptr : &condition.c_token_types,
                     condition.c_vinput_mask.empty() ? nullptr : &condition.c_vinput_mask,
                     condition.c_image_embeds.empty() ? nullptr : &condition.c_image_embeds};
+            } else if (version == VERSION_MING_IMAGE) {
+                diffusion_params.extra = MingImageDiffusionExtra{
+                    condition.extra_c_crossattns.empty() ? nullptr : &condition.extra_c_crossattns[0]};
             } else if (sd_version_is_llada_image(version)) {
                 diffusion_params.extra = LLaDAImageDiffusionExtra{
                     condition.extra_c_crossattns.empty() ? nullptr : &condition.extra_c_crossattns[0]};
@@ -2481,10 +2562,33 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
                 return std::move(cached_output);
             }
 
+            // A re-enabled condition can miss the cache even when the positive pass was reused.
+            if (!uncond_controls_ready && !uncond.empty() &&
+                (&condition == &uncond || &condition == &img_uncond)) {
+                compute_sample_controls(control_image,
+                                        noised_input,
+                                        timesteps_tensor,
+                                        uncond,
+                                        &controls);
+                uncond_controls_ready = true;
+            }
+
             for (const auto& extension : generation_extensions) {
                 extension->before_diffusion(diffusion_params, step);
             }
 
+            if (cache_qwen_prefix) {
+                auto* extra = std::get_if<QwenImage21DiffusionExtra>(&diffusion_params.extra);
+                if (extra != nullptr) {
+                    auto key         = std::make_tuple(diffusion_params.context, extra->image_slots,
+                                               diffusion_params.ref_image_params.pass_to_dit ? diffusion_params.ref_latents : nullptr);
+                    auto entry       = std::find(qwen_prefix_inputs.begin(), qwen_prefix_inputs.end(), key);
+                    extra->prefix_id = static_cast<uint64_t>(entry - qwen_prefix_inputs.begin()) + 1;
+                    if (entry == qwen_prefix_inputs.end()) {
+                        qwen_prefix_inputs.push_back(key);
+                    }
+                }
+            }
             auto output_opt = work_diffusion_model->compute(n_threads, diffusion_params);
             if (output_opt.empty()) {
                 LOG_ERROR("diffusion model compute failed");
@@ -2508,41 +2612,69 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
             }
         }
 
+        float effective_guidance_scale = guidance_schedule.empty()
+                                             ? cfg_scale
+                                             : guidance_schedule[guidance_schedule.size() - 1 - step];
+
+        float image_guidance_scale = img_cfg_scale;
+
+        constexpr float kEpsilon = 1e-5f;
+
+        bool skip_uncond = false;
+        if (!uncond.empty() && !needs_uncond_denoised && !use_apg_guidance) {
+            if (!img_uncond.empty()) {
+                skip_uncond = std::abs(image_guidance_scale - effective_guidance_scale) < kEpsilon;
+            } else {
+                skip_uncond = std::abs(effective_guidance_scale - 1.0f) < kEpsilon;
+            }
+        }
+
+        bool skip_img_uncond = false;
+        if (!img_uncond.empty() && !needs_uncond_denoised && !use_apg_guidance) {
+            if (!uncond.empty()) {
+                skip_img_uncond = std::abs(image_guidance_scale - 1.0f) < kEpsilon;
+            } else {
+                skip_img_uncond = std::abs(effective_guidance_scale - 1.0f) < kEpsilon;
+            }
+        }
+
         cond_out = run_condition(*positive_condition, c_concat_override);
         if (cond_out.empty()) {
             return {};
         }
 
         if (!uncond.empty()) {
-            if (!step_cache.is_step_skipped()) {
-                compute_sample_controls(control_image,
-                                        noised_input,
-                                        timesteps_tensor,
-                                        uncond,
-                                        &controls);
-            }
-            const std::vector<int>* uncond_skip_layers = nullptr;
-            if (is_skiplayer_step && slg_uncond) {
-                LOG_VERBOSE("Skipping layers at uncond step %d\n", step);
-                uncond_skip_layers = &skip_layer_guidance.layers();
-            }
-            uncond_out = run_condition(uncond,
-                                       uncond.c_concat.empty() ? nullptr : &uncond.c_concat,
-                                       uncond_skip_layers,
-                                       nullptr,
-                                       true);
-            if (uncond_out.empty()) {
-                return {};
+            if (!skip_uncond) {
+                const std::vector<int>* uncond_skip_layers = nullptr;
+                if (is_skiplayer_step && slg_uncond) {
+                    LOG_VERBOSE("Skipping layers at uncond step %d\n", step);
+                    uncond_skip_layers = &skip_layer_guidance.layers();
+                }
+                uncond_out = run_condition(uncond,
+                                           uncond.c_concat.empty() ? nullptr : &uncond.c_concat,
+                                           uncond_skip_layers,
+                                           nullptr,
+                                           true);
+                if (uncond_out.empty()) {
+                    return {};
+                }
+            } else {
+                step_cache.invalidate_condition(&uncond);
             }
         }
+
         if (!img_uncond.empty()) {
-            img_uncond_out = run_condition(img_uncond,
-                                           img_uncond.c_concat.empty() ? nullptr : &img_uncond.c_concat,
-                                           nullptr,
-                                           uncond_without_ref_latents ? &empty_ref_latents : nullptr,
-                                           true);
-            if (img_uncond_out.empty()) {
-                return {};
+            if (!skip_img_uncond) {
+                img_uncond_out = run_condition(img_uncond,
+                                               img_uncond.c_concat.empty() ? nullptr : &img_uncond.c_concat,
+                                               nullptr,
+                                               uncond_without_ref_latents ? &empty_ref_latents : nullptr,
+                                               true);
+                if (img_uncond_out.empty()) {
+                    return {};
+                }
+            } else {
+                step_cache.invalidate_condition(&img_uncond);
             }
         }
         sd::guidance::GuidanceInput guidance_input;
@@ -2552,7 +2684,7 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
         guidance_input.pred_uncond     = uncond_out.empty() ? nullptr : &uncond_out;
         guidance_input.pred_img_uncond = img_uncond_out.empty() ? nullptr : &img_uncond_out;
 
-        sd::guidance::GuiderOutput guided = guidance_schedule.empty() ? primary_guidance.forward(guidance_input, {}) : primary_guidance.forward(guidance_input, {}, guidance_schedule[guidance_schedule.size() - 1 - step]);
+        sd::guidance::GuiderOutput guided = primary_guidance.forward(guidance_input, {}, effective_guidance_scale);
         if (guided.pred.empty()) {
             return {};
         }
@@ -2630,7 +2762,7 @@ int StableDiffusionGGML::get_diffusion_model_down_factor() {
     if (sd_version_is_dit(version)) {
         if (sd_version_is_sensenova_u1(version)) {
             down_factor = 32;
-        } else if (version == VERSION_QWEN_IMAGE_2_1 || sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_minimax_h3(version)) {
+        } else if (version == VERSION_QWEN_IMAGE_2_1 || version == VERSION_MING_IMAGE || sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_minimax_h3(version) || sd_version_is_pixart(version)) {
             down_factor = 2;
         } else {
             down_factor = 1;
@@ -2668,6 +2800,8 @@ int StableDiffusionGGML::get_latent_channel() {
             latent_channel = 128;
         } else if (sd_version_is_mage_flow(version)) {
             latent_channel = 128;
+        } else if (sd_version_is_pixart(version)) {
+            latent_channel = 4;
         } else {
             latent_channel = 16;
         }
@@ -2676,7 +2810,7 @@ int StableDiffusionGGML::get_latent_channel() {
 }
 
 int StableDiffusionGGML::get_image_channels() const {
-    return version == VERSION_QWEN_IMAGE_LAYERED || version == VERSION_QWEN_IMAGE_2_1 ? 4 : 3;
+    return version == VERSION_QWEN_IMAGE_LAYERED || version == VERSION_QWEN_IMAGE_2_1 || version == VERSION_MING_IMAGE ? 4 : 3;
 }
 
 int StableDiffusionGGML::get_image_seq_len(int h, int w) {
@@ -2763,13 +2897,26 @@ sd::Tensor<float> StableDiffusionGGML::decode_first_stage(const sd::Tensor<float
         return sd::ops::clamp((x + 1.f) * 0.5f, 0.0f, 1.0f);
     }
     auto latents                      = first_stage_model->diffusion_to_vae_latents(x);
-    auto decoded                      = first_stage_model->decode(n_threads, latents, vae_tiling_params, decode_video, circular_x, circular_y);
-    const bool prefer_temporal_tiling = decode_video && first_stage_model->can_temporal_tile_decode();
-    while (decoded.empty() &&
-           sd::backend_fit::prepare_vae_decode_retry_tiling(vae_tiling_params, prefer_temporal_tiling)) {
-        decoded = first_stage_model->decode(n_threads, latents, vae_tiling_params, decode_video, circular_x, circular_y);
+    auto tiling_params                = first_stage_model->resolve_tiling_params(vae_tiling_params);
+    const bool prefer_temporal_tiling = decode_video && latents.dim() == 5 && latents.shape()[2] > 1 &&
+                                        first_stage_model->can_temporal_tile_decode();
+    for (;;) {
+        int tile_size_w = static_cast<int>(latents.shape()[0]);
+        int tile_size_h = static_cast<int>(latents.shape()[1]);
+        float tile_overlap;
+        if (tiling_params.enabled &&
+            !first_stage_model->get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params,
+                                               latents.shape()[0], latents.shape()[1])) {
+            return {};
+        }
+        auto decoded = first_stage_model->decode(n_threads, latents, tiling_params, decode_video, circular_x, circular_y);
+        if (!decoded.empty() ||
+            !sd::backend_fit::prepare_vae_decode_retry_tiling(tiling_params, prefer_temporal_tiling,
+                                                              first_stage_model->last_compute_status(),
+                                                              tile_size_w, tile_size_h, first_stage_model->get_scale_factor())) {
+            return decoded;
+        }
     }
-    return decoded;
 }
 
 sd::Tensor<float> StableDiffusionGGML::normalize_ltx_video_latents(const sd::Tensor<float>& x) {

src/pipeline/diffusion_engine.h

@@ -26,6 +26,7 @@ class RNG;
 struct Denoiser;
 struct LoraModel;
 struct ConditionerParams;
+class ConditioningCache;
 struct SDCondition;
 struct RefImageParams;
 namespace Wav2Vec2 {
@@ -178,6 +179,8 @@ public:
     std::recursive_mutex execution_mutex;
     std::unique_ptr<ModelConfig> config_;
     RunnerState runner_state_;
+    std::unique_ptr<ConditioningCache> conditioning_cache_;
+    std::vector<ModelManager::LoraSpec> conditioning_loras_;
     bool executing_ = false;
 
     std::shared_ptr<Denoiser> denoiser;
@@ -361,6 +364,8 @@ public:
 
     bool apply_loras(const sd_lora_t* loras, uint32_t lora_count);
 
+    SDCondition get_learned_condition(const ConditionerParams& params);
+
     void reset_generation_extensions();
 
     void prepare_generation_extensions(const sd_pm_params_t& pm_params,

src/pipeline/image.cpp

@@ -10,6 +10,7 @@
 #include "model/vae/vae.hpp"
 #include "request.h"
 #include "runtime/denoiser.hpp"
+#include "runtime/image_preprocess.h"
 #include "upscaler.h"
 
 namespace sd::pipeline {
@@ -34,19 +35,21 @@ namespace sd::pipeline {
             return original_axes;
         }
 
-        int tile_size_x, tile_size_y;
+        int tile_size_w, tile_size_h;
         float overlap;
-        int latent_size_x = request.width / request.vae_scale_factor;
-        int latent_size_y = request.height / request.vae_scale_factor;
-        sd->first_stage_model->get_tile_sizes(tile_size_x,
-                                              tile_size_y,
-                                              overlap,
-                                              sd_img_gen_params->vae_tiling_params,
-                                              latent_size_x,
-                                              latent_size_y);
+        int latent_size_w = request.width / request.vae_scale_factor;
+        int latent_size_h = request.height / request.vae_scale_factor;
+        if (!sd->first_stage_model->get_tile_sizes(tile_size_w,
+                                                   tile_size_h,
+                                                   overlap,
+                                                   sd_img_gen_params->vae_tiling_params,
+                                                   latent_size_w,
+                                                   latent_size_h)) {
+            return original_axes;
+        }
 
-        sd->circular_x = sd->circular_x && (tile_size_x >= latent_size_x);
-        sd->circular_y = sd->circular_y && (tile_size_y >= latent_size_y);
+        sd->circular_x = sd->circular_x && (tile_size_w >= latent_size_w);
+        sd->circular_y = sd->circular_y && (tile_size_h >= latent_size_h);
 
         if (sd->first_stage_model) {
             sd->first_stage_model->set_circular_axes(sd->circular_x, sd->circular_y);
@@ -55,8 +58,8 @@ namespace sd::pipeline {
             sd->preview_vae->set_circular_axes(sd->circular_x, sd->circular_y);
         }
 
-        sd->circular_x = original_axes.circular_x && (tile_size_x < latent_size_x);
-        sd->circular_y = original_axes.circular_y && (tile_size_y < latent_size_y);
+        sd->circular_x = original_axes.circular_x && (tile_size_w < latent_size_w);
+        sd->circular_y = original_axes.circular_y && (tile_size_h < latent_size_h);
 
         return original_axes;
     }
@@ -440,8 +443,7 @@ namespace sd::pipeline {
         sd->compute_ip_adapter_tokens(sd_img_gen_params->ip_adapter_image, sd_img_gen_params->ip_adapter_strength);
         int64_t prepare_start_ms         = ggml_time_ms();
         condition_params.zero_out_masked = false;
-        auto cond                        = sd->cond_stage_model->get_learned_condition(sd->n_threads,
-                                                                                       condition_params);
+        auto cond                        = sd->get_learned_condition(condition_params);
         if (cond.empty()) {
             LOG_ERROR("failed to encode prompt");
             return std::nullopt;
@@ -463,6 +465,11 @@ namespace sd::pipeline {
                 // states with a zeroed prompt mask, so no extra text encode is needed.
                 uncond.c_crossattn = cond.c_crossattn;
                 uncond.c_vector    = sd::Tensor<float>::zeros_like(cond.c_vector);
+            } else if (sd->version == VERSION_MING_IMAGE) {
+                uncond.c_crossattn = sd::Tensor<float>::zeros_like(cond.c_crossattn);
+                for (const auto& extra : cond.extra_c_crossattns) {
+                    uncond.extra_c_crossattns.push_back(sd::Tensor<float>::zeros_like(extra));
+                }
             } else if (sd_version_is_sensenova_u1(sd->version)) {
                 auto* sensenova_conditioner = static_cast<SenseNovaU1Conditioner*>(sd->cond_stage_model.get());
                 uncond                      = sensenova_conditioner->get_unconditional_condition(request->negative_prompt);
@@ -479,8 +486,7 @@ namespace sd::pipeline {
                     // LLaDA-Image CFG keeps the source latent but drops its SigVQ features.
                     condition_params.ref_images = nullptr;
                 }
-                uncond = sd->cond_stage_model->get_learned_condition(sd->n_threads,
-                                                                     condition_params);
+                uncond = sd->get_learned_condition(condition_params);
                 if (uncond.empty()) {
                     LOG_ERROR("failed to encode negative prompt");
                     return std::nullopt;
@@ -508,8 +514,7 @@ namespace sd::pipeline {
                 if (use_ref_latent_img_cfg) {
                     condition_params.ref_images = &empty_ref_images;
                 }
-                img_uncond = sd->cond_stage_model->get_learned_condition(sd->n_threads,
-                                                                         condition_params);
+                img_uncond = sd->get_learned_condition(condition_params);
                 if (img_uncond.empty()) {
                     LOG_ERROR("failed to encode image guidance prompt");
                     return std::nullopt;
@@ -800,6 +805,12 @@ namespace sd::pipeline {
         int64_t t0            = ggml_time_ms();
         sd->vae_tiling_params = sd_img_gen_params->vae_tiling_params;
         GenerationRequest request(sd, sd_img_gen_params);
+        sd::ImagePreprocessor preprocessing(sd_img_gen_params->image_preprocess.rules);
+        sd_img_gen_params_t processed_params = *sd_img_gen_params;
+        if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
+            return false;
+        sd_img_gen_params = &processed_params;
+        request.pm_params = processed_params.pm_params;
         LOG_INFO("generate_image %dx%d", request.width, request.height);
 
         sd->rng->manual_seed(request.seed);

src/pipeline/model_builders.cpp

@@ -1,5 +1,6 @@
 #include "model_builders.h"
 
+#include <cmath>
 #include <cstring>
 #include <utility>
 
@@ -23,11 +24,13 @@
 #include "model/diffusion/llada_image.hpp"
 #include "model/diffusion/ltxv.hpp"
 #include "model/diffusion/mage_flow.hpp"
+#include "model/diffusion/ming_image.hpp"
 #include "model/diffusion/minimax_h3.hpp"
 #include "model/diffusion/minit2i.hpp"
 #include "model/diffusion/mmdit.hpp"
 #include "model/diffusion/model.hpp"
 #include "model/diffusion/pid.hpp"
+#include "model/diffusion/pixart.hpp"
 #include "model/diffusion/qwen_image.hpp"
 #include "model/diffusion/qwen_image_2_1.hpp"
 #include "model/diffusion/sensenova_u1.h"
@@ -291,7 +294,8 @@ namespace sd::model_builders {
                 result.diffusion = std::make_shared<Qwen::QwenImage21Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                              tensor_storage_map,
                                                                              "model.diffusion_model",
-                                                                             weight_manager);
+                                                                             weight_manager,
+                                                                             sd_ctx_params->model_args);
             } else {
                 result.diffusion = std::make_shared<Qwen::QwenImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                            tensor_storage_map,
@@ -300,6 +304,19 @@ namespace sd::model_builders {
                                                                            weight_manager,
                                                                            sd_ctx_params->model_args);
             }
+        } else if (version == VERSION_PIXART) {
+            result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+                                                                  tensor_storage_map,
+                                                                  true,
+                                                                  0,
+                                                                  false,
+                                                                  weight_manager,
+                                                                  sd_ctx_params->model_args);
+            result.diffusion   = std::make_shared<PixArt::PixArtRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+                                                                      tensor_storage_map,
+                                                                      "model.diffusion_model",
+                                                                      weight_manager,
+                                                                      sd_ctx_params->model_args);
         } else if (sd_version_is_mage_flow(version)) {
             result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
                                                                tensor_storage_map,
@@ -358,6 +375,11 @@ namespace sd::model_builders {
                                                                     tensor_storage_map,
                                                                     "model.diffusion_model",
                                                                     weight_manager);
+        } else if (version == VERSION_MING_IMAGE) {
+            result.conditioner = std::make_shared<MingImageEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+                                                                     tensor_storage_map, weight_manager, tokenizers);
+            result.diffusion   = std::make_shared<MingImage::MingImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+                                                                            tensor_storage_map, "model.diffusion_model", weight_manager);
         } else if (sd_version_is_z_image(version)) {
             result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
                                                                tensor_storage_map,
@@ -561,6 +583,23 @@ namespace sd::model_builders {
                                                              false,
                                                              vae_version,
                                                              weight_manager);
+                if (sd_version_is_pixart(version)) {
+                    // Alpha-512 and Sigma share tensor layouts; Alpha-512 needs an explicit scale override.
+                    if (tensor_storage_map.count("model.diffusion_model.csize_embedder.mlp.0.weight") != 0) {
+                        model->scale_factor = 0.18215f;
+                    }
+                    for (const auto& [key, value] : parse_key_value_args(sd_ctx_params->model_args, "model arg")) {
+                        if (key == "pixart_vae_scale_factor") {
+                            float parsed = 0.f;
+                            if (parse_strict_float(value, parsed) && std::isfinite(parsed) && parsed > 0.f) {
+                                model->scale_factor = parsed;
+                            } else {
+                                LOG_WARN("ignoring invalid PixArt model arg '%s=%s'", key.c_str(), value.c_str());
+                            }
+                        }
+                    }
+                    LOG_VERBOSE("pixart: VAE scale factor = %.5f", model->scale_factor);
+                }
                 if (sd_version_is_sdxl(version) &&
                     (strlen(SAFE_STR(sd_ctx_params->vae_path)) == 0 || sd_ctx_params->force_sdxl_vae_conv_scale || options.external_vae_is_invalid)) {
                     float vae_conv_2d_scale = 1.f / 32.f;