feat: adapt LCM for flow models (#1413)

.github/workflows/build.yml

@@ -176,6 +176,7 @@ jobs:
 
   build-and-push-docker-images:
     name: Build and push container images
+    if: ${{ github.event_name != 'pull_request' }}
     runs-on: ubuntu-latest
 
     permissions:

CMakeLists.txt

@@ -11,6 +11,10 @@ endif()
 if (MSVC)
     add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
     add_compile_definitions(_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING)
+    add_compile_options(
+        $<$<COMPILE_LANGUAGE:C>:/MP>
+        $<$<COMPILE_LANGUAGE:CXX>:/MP>
+    )
 endif()
 
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)

README.md

@@ -97,6 +97,7 @@ API and command-line option may change frequently.***
     - `DPM++ 2M`
     - [`DPM++ 2M v2`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/8457)
     - `DPM++ 2S a`
+    - `ER-SDE`
     - [`LCM`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/13952)
 - Cross-platform reproducibility
     - `--rng cuda`, default, consistent with the `stable-diffusion-webui GPU RNG`

docs/distilled_sd.md

@@ -87,51 +87,32 @@ pipe.save_pretrained("segmindtiny-sd", safe_serialization=True)
 ```bash
 python convert_diffusers_to_original_stable_diffusion.py \
       --model_path  ./segmindtiny-sd \
-      --checkpoint_path ./segmind_tiny-sd.ckpt --half
+      --checkpoint_path ./segmind_tiny-sd.safetensors  --half --use_safetensors
 ```
 
-The file segmind_tiny-sd.ckpt will be generated and is now ready for use with sd.cpp. You can follow a similar process for the other models mentioned above.
+The file segmind_tiny-sd.safetensors will be generated and is now ready for use with sd.cpp. You can follow a similar process for the other models mentioned above.
 
 
-##### Another available .ckpt file:
-
- * https://huggingface.co/ClashSAN/small-sd/resolve/main/tinySDdistilled.ckpt
-
-To use this file, you must first adjust its non-contiguous tensors:
-
-```python
-import torch
-ckpt = torch.load("tinySDdistilled.ckpt", map_location=torch.device('cpu'))
-for key, value in ckpt['state_dict'].items():
-    if isinstance(value, torch.Tensor):
-        ckpt['state_dict'][key] = value.contiguous()
-torch.save(ckpt, "tinySDdistilled_fixed.ckpt")
-```
-
-
-### SDXS-512
+### SDXS-512-DreamShaper
 
 Another very tiny and **incredibly fast**  model is SDXS by IDKiro et al.  The authors refer to it as *"Real-Time One-Step Latent Diffusion Models with Image Conditions"*. For details read the paper: https://arxiv.org/pdf/2403.16627 . Once again the authors removed some more blocks of U-Net part and unlike other SD1 models they use an adjusted _AutoEncoderTiny_ instead of default _AutoEncoderKL_ for the VAE part.
+##### Some ready-to-run SDXS-512 model files are available online, such as:
 
-##### 1. Download the diffusers model from  Hugging Face using Python:
-
-```python
-from diffusers import StableDiffusionPipeline
-pipe = StableDiffusionPipeline.from_pretrained("IDKiro/sdxs-512-dreamshaper")
-pipe.save_pretrained(save_directory="sdxs")
-```
-##### 2. Create a safetensors file
-
-```bash
-python convert_diffusers_to_original_stable_diffusion.py \
-    --model_path  sdxs  --checkpoint_path sdxs.safetensors --half --use_safetensors
-```
-
-##### 3. Run the model as follows:
+* https://huggingface.co/akleine/sdxs-512
+* https://huggingface.co/concedo/sdxs-512-tinySDdistilled-GGUF
 
+##### Run the model as follows:
 ```bash
 ~/stable-diffusion.cpp/build/bin/sd-cli -m sdxs.safetensors -p "portrait of a lovely cat" \
   --cfg-scale 1 --steps 1
 ```
+Both options: ``` --cfg-scale 1 ``` and  ``` --steps 1 ``` are mandatory here.
 
-Both options: ``` --cfg-scale 1 ``` and  ``` --steps 1 ``` are mandatory here.                                                 
+### SDXS-512-0.9
+
+Even though the name "SDXS-512-0.9" is similar to "SDXS-512-DreamShaper", it is *completely different* but also **incredibly fast**. Sometimes it is preferred, so try it yourself.
+##### Download a ready-to-run file from here:
+
+* https://huggingface.co/akleine/sdxs-09
+
+For the use of this model, both options ``` --cfg-scale 1 ``` and ``` --steps 1 ``` are again absolutely necessary.

examples/cli/README.md

@@ -114,7 +114,7 @@ Generation Options:
                                            medium
   --skip-layer-start <float>               SLG enabling point (default: 0.01)
   --skip-layer-end <float>                 SLG disabling point (default: 0.2)
-  --eta <float>                            noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a and dpm++2s_a)
+  --eta <float>                            noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)
   --flow-shift <float>                     shift value for Flow models like SD3.x or WAN (default: auto)
   --high-noise-cfg-scale <float>           (high noise) unconditional guidance scale: (default: 7.0)
   --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or instruct-pix2pix models (default: same as --cfg-scale)
@@ -122,7 +122,7 @@ Generation Options:
   --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default: 0)
   --high-noise-skip-layer-start <float>    (high noise) SLG enabling point (default: 0.01)
   --high-noise-skip-layer-end <float>      (high noise) SLG disabling point (default: 0.2)
-  --high-noise-eta <float>                 (high noise) noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a and dpm++2s_a)
+  --high-noise-eta <float>                 (high noise) noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)
   --strength <float>                       strength for noising/unnoising (default: 0.75)
   --pm-style-strength <float>
   --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image
@@ -133,10 +133,10 @@ Generation Options:
   --disable-image-metadata                 do not embed generation metadata on image files
   -s, --seed                               RNG seed (default: 42, use random seed for < 0)
   --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing,
-                                           tcd, res_multistep, res_2s] (default: euler for Flux/SD3/Wan, euler_a
+                                           tcd, res_multistep, res_2s, er_sde] (default: euler for Flux/SD3/Wan, euler_a
                                            otherwise)
   --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm,
-                                           ddim_trailing, tcd, res_multistep, res_2s] default: euler for Flux/SD3/Wan,
+                                           ddim_trailing, tcd, res_multistep, res_2s, er_sde] default: euler for Flux/SD3/Wan,
                                            euler_a otherwise
   --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, smoothstep, sgm_uniform, simple,
                                            kl_optimal, lcm, bong_tangent], default: discrete

examples/common/common.cpp

@@ -855,7 +855,7 @@ ArgOptions SDGenerationParams::get_options() {
          &sample_params.guidance.slg.layer_end},
         {"",
          "--eta",
-         "noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a and dpm++2s_a)",
+         "noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)",
          &sample_params.eta},
         {"",
          "--flow-shift",
@@ -887,7 +887,7 @@ ArgOptions SDGenerationParams::get_options() {
          &high_noise_sample_params.guidance.slg.layer_end},
         {"",
          "--high-noise-eta",
-         "(high noise) noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a and dpm++2s_a)",
+         "(high noise) noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)",
          &high_noise_sample_params.eta},
         {"",
          "--strength",
@@ -1185,12 +1185,12 @@ ArgOptions SDGenerationParams::get_options() {
          on_seed_arg},
         {"",
          "--sampling-method",
-         "sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s] "
+         "sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde] "
          "(default: euler for Flux/SD3/Wan, euler_a otherwise)",
          on_sample_method_arg},
         {"",
          "--high-noise-sampling-method",
-         "(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s]"
+         "(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde]"
          " default: euler for Flux/SD3/Wan, euler_a otherwise",
          on_high_noise_sample_method_arg},
         {"",
@@ -1589,10 +1589,18 @@ bool SDGenerationParams::from_json_str(
         LOG_ERROR("invalid init_image");
         return false;
     }
+    if (!parse_image_json_field(j, "end_image", 3, width, height, end_image)) {
+        LOG_ERROR("invalid end_image");
+        return false;
+    }
     if (!parse_image_array_json_field(j, "ref_images", 3, width, height, ref_images)) {
         LOG_ERROR("invalid ref_images");
         return false;
     }
+    if (!parse_image_array_json_field(j, "control_frames", 3, width, height, control_frames)) {
+        LOG_ERROR("invalid control_frames");
+        return false;
+    }
     if (!parse_image_json_field(j, "mask_image", 1, width, height, mask_image)) {
         LOG_ERROR("invalid mask_image");
         return false;

examples/common/media_io.cpp

@@ -95,6 +95,57 @@ using WebPMuxPtr         = std::unique_ptr<WebPMux, WebPMuxDeleter>;
 using WebPAnimEncoderPtr = std::unique_ptr<WebPAnimEncoder, WebPAnimEncoderDeleter>;
 #endif
 
+#ifdef SD_USE_WEBM
+class MemoryMkvWriter : public mkvmuxer::IMkvWriter {
+public:
+    mkvmuxer::int32 Write(const void* buf, mkvmuxer::uint32 len) override {
+        if (buf == nullptr && len > 0) {
+            return -1;
+        }
+        const size_t end_pos = position_ + static_cast<size_t>(len);
+        if (end_pos > data_.size()) {
+            data_.resize(end_pos);
+        }
+        if (len > 0) {
+            memcpy(data_.data() + position_, buf, len);
+        }
+        position_ = end_pos;
+        return 0;
+    }
+
+    mkvmuxer::int64 Position() const override {
+        return static_cast<mkvmuxer::int64>(position_);
+    }
+
+    mkvmuxer::int32 Position(mkvmuxer::int64 position) override {
+        if (position < 0) {
+            return -1;
+        }
+        const size_t target = static_cast<size_t>(position);
+        if (target > data_.size()) {
+            data_.resize(target);
+        }
+        position_ = target;
+        return 0;
+    }
+
+    bool Seekable() const override {
+        return true;
+    }
+
+    void ElementStartNotify(mkvmuxer::uint64, mkvmuxer::int64) override {
+    }
+
+    const std::vector<uint8_t>& data() const {
+        return data_;
+    }
+
+private:
+    std::vector<uint8_t> data_;
+    size_t position_ = 0;
+};
+#endif
+
 bool read_binary_file_bytes(const char* path, std::vector<uint8_t>& data) {
     std::ifstream fin(fs::path(path), std::ios::binary);
     if (!fin) {
@@ -570,6 +621,32 @@ void write_u16_le(FILE* f, uint16_t val) {
     fwrite(&val, 2, 1, f);
 }
 
+void write_u32_le(std::vector<uint8_t>& data, uint32_t val) {
+    data.push_back(static_cast<uint8_t>(val & 0xFF));
+    data.push_back(static_cast<uint8_t>((val >> 8) & 0xFF));
+    data.push_back(static_cast<uint8_t>((val >> 16) & 0xFF));
+    data.push_back(static_cast<uint8_t>((val >> 24) & 0xFF));
+}
+
+void write_u16_le(std::vector<uint8_t>& data, uint16_t val) {
+    data.push_back(static_cast<uint8_t>(val & 0xFF));
+    data.push_back(static_cast<uint8_t>((val >> 8) & 0xFF));
+}
+
+void patch_u32_le(std::vector<uint8_t>& data, size_t offset, uint32_t val) {
+    if (offset + 4 > data.size()) {
+        return;
+    }
+    data[offset + 0] = static_cast<uint8_t>(val & 0xFF);
+    data[offset + 1] = static_cast<uint8_t>((val >> 8) & 0xFF);
+    data[offset + 2] = static_cast<uint8_t>((val >> 16) & 0xFF);
+    data[offset + 3] = static_cast<uint8_t>((val >> 24) & 0xFF);
+}
+
+void write_fourcc(std::vector<uint8_t>& data, const char* fourcc) {
+    data.insert(data.end(), fourcc, fourcc + 4);
+}
+
 EncodedImageFormat encoded_image_format_from_path(const std::string& path) {
     std::string ext = fs::path(path).extension().string();
     std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
@@ -699,95 +776,96 @@ uint8_t* load_image_from_memory(const char* image_bytes,
     return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel);
 }
 
-int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality) {
+std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality) {
     if (num_images == 0) {
         fprintf(stderr, "Error: Image array is empty.\n");
-        return -1;
+        return {};
     }
 
-    FilePtr file(fopen(filename, "wb"));
-    if (!file) {
-        perror("Error opening file for writing");
-        return -1;
-    }
-    FILE* f = file.get();
-
     uint32_t width    = images[0].width;
     uint32_t height   = images[0].height;
     uint32_t channels = images[0].channel;
     if (channels != 3 && channels != 4) {
         fprintf(stderr, "Error: Unsupported channel count: %u\n", channels);
-        return -1;
+        return {};
     }
 
-    fwrite("RIFF", 4, 1, f);
-    long riff_size_pos = ftell(f);
-    write_u32_le(f, 0);
-    fwrite("AVI ", 4, 1, f);
+    // stb_image_write changes JPEG sampling behavior above quality 90.
+    // MJPG AVI playback is more compatible when we keep the encoder on the
+    // <= 90 path.
+    const int mjpg_quality = std::clamp(quality, 1, 90);
 
-    fwrite("LIST", 4, 1, f);
-    write_u32_le(f, 4 + 8 + 56 + 8 + 4 + 8 + 56 + 8 + 40);
-    fwrite("hdrl", 4, 1, f);
+    std::vector<uint8_t> avi_data;
+    avi_data.reserve(static_cast<size_t>(num_images) * 1024);
 
-    fwrite("avih", 4, 1, f);
-    write_u32_le(f, 56);
-    write_u32_le(f, 1000000 / fps);
-    write_u32_le(f, 0);
-    write_u32_le(f, 0);
-    write_u32_le(f, 0x110);
-    write_u32_le(f, num_images);
-    write_u32_le(f, 0);
-    write_u32_le(f, 1);
-    write_u32_le(f, width * height * 3);
-    write_u32_le(f, width);
-    write_u32_le(f, height);
-    write_u32_le(f, 0);
-    write_u32_le(f, 0);
-    write_u32_le(f, 0);
-    write_u32_le(f, 0);
+    write_fourcc(avi_data, "RIFF");
+    const size_t riff_size_pos = avi_data.size();
+    write_u32_le(avi_data, 0);
+    write_fourcc(avi_data, "AVI ");
 
-    fwrite("LIST", 4, 1, f);
-    write_u32_le(f, 4 + 8 + 56 + 8 + 40);
-    fwrite("strl", 4, 1, f);
+    write_fourcc(avi_data, "LIST");
+    write_u32_le(avi_data, 4 + 8 + 56 + 8 + 4 + 8 + 56 + 8 + 40);
+    write_fourcc(avi_data, "hdrl");
 
-    fwrite("strh", 4, 1, f);
-    write_u32_le(f, 56);
-    fwrite("vids", 4, 1, f);
-    fwrite("MJPG", 4, 1, f);
-    write_u32_le(f, 0);
-    write_u16_le(f, 0);
-    write_u16_le(f, 0);
-    write_u32_le(f, 0);
-    write_u32_le(f, 1);
-    write_u32_le(f, fps);
-    write_u32_le(f, 0);
-    write_u32_le(f, num_images);
-    write_u32_le(f, width * height * 3);
-    write_u32_le(f, (uint32_t)-1);
-    write_u32_le(f, 0);
-    write_u16_le(f, 0);
-    write_u16_le(f, 0);
-    write_u16_le(f, 0);
-    write_u16_le(f, 0);
+    write_fourcc(avi_data, "avih");
+    write_u32_le(avi_data, 56);
+    write_u32_le(avi_data, 1000000 / fps);
+    write_u32_le(avi_data, 0);
+    write_u32_le(avi_data, 0);
+    write_u32_le(avi_data, 0x110);
+    write_u32_le(avi_data, num_images);
+    write_u32_le(avi_data, 0);
+    write_u32_le(avi_data, 1);
+    write_u32_le(avi_data, width * height * 3);
+    write_u32_le(avi_data, width);
+    write_u32_le(avi_data, height);
+    write_u32_le(avi_data, 0);
+    write_u32_le(avi_data, 0);
+    write_u32_le(avi_data, 0);
+    write_u32_le(avi_data, 0);
 
-    fwrite("strf", 4, 1, f);
-    write_u32_le(f, 40);
-    write_u32_le(f, 40);
-    write_u32_le(f, width);
-    write_u32_le(f, height);
-    write_u16_le(f, 1);
-    write_u16_le(f, 24);
-    fwrite("MJPG", 4, 1, f);
-    write_u32_le(f, width * height * 3);
-    write_u32_le(f, 0);
-    write_u32_le(f, 0);
-    write_u32_le(f, 0);
-    write_u32_le(f, 0);
+    write_fourcc(avi_data, "LIST");
+    write_u32_le(avi_data, 4 + 8 + 56 + 8 + 40);
+    write_fourcc(avi_data, "strl");
 
-    fwrite("LIST", 4, 1, f);
-    long movi_size_pos = ftell(f);
-    write_u32_le(f, 0);
-    fwrite("movi", 4, 1, f);
+    write_fourcc(avi_data, "strh");
+    write_u32_le(avi_data, 56);
+    write_fourcc(avi_data, "vids");
+    write_fourcc(avi_data, "MJPG");
+    write_u32_le(avi_data, 0);
+    write_u16_le(avi_data, 0);
+    write_u16_le(avi_data, 0);
+    write_u32_le(avi_data, 0);
+    write_u32_le(avi_data, 1);
+    write_u32_le(avi_data, fps);
+    write_u32_le(avi_data, 0);
+    write_u32_le(avi_data, num_images);
+    write_u32_le(avi_data, width * height * 3);
+    write_u32_le(avi_data, static_cast<uint32_t>(-1));
+    write_u32_le(avi_data, 0);
+    write_u16_le(avi_data, 0);
+    write_u16_le(avi_data, 0);
+    write_u16_le(avi_data, 0);
+    write_u16_le(avi_data, 0);
+
+    write_fourcc(avi_data, "strf");
+    write_u32_le(avi_data, 40);
+    write_u32_le(avi_data, 40);
+    write_u32_le(avi_data, width);
+    write_u32_le(avi_data, height);
+    write_u16_le(avi_data, 1);
+    write_u16_le(avi_data, 24);
+    write_fourcc(avi_data, "MJPG");
+    write_u32_le(avi_data, width * height * 3);
+    write_u32_le(avi_data, 0);
+    write_u32_le(avi_data, 0);
+    write_u32_le(avi_data, 0);
+    write_u32_le(avi_data, 0);
+
+    write_fourcc(avi_data, "LIST");
+    const size_t movi_size_pos = avi_data.size();
+    write_u32_le(avi_data, 0);
+    write_fourcc(avi_data, "movi");
 
     std::vector<avi_index_entry> index(static_cast<size_t>(num_images));
     std::vector<uint8_t> jpeg_data;
@@ -801,55 +879,61 @@ int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int
             buffer->insert(buffer->end(), src, src + size);
         };
 
-        if (!stbi_write_jpg_to_func(write_to_buf, &jpeg_data, images[i].width, images[i].height, channels, images[i].data, quality)) {
+        if (!stbi_write_jpg_to_func(write_to_buf, &jpeg_data, images[i].width, images[i].height, channels, images[i].data, mjpg_quality)) {
             fprintf(stderr, "Error: Failed to encode JPEG frame.\n");
-            return -1;
+            return {};
         }
 
-        fwrite("00dc", 4, 1, f);
-        write_u32_le(f, (uint32_t)jpeg_data.size());
-        index[i].offset = ftell(f) - 8;
-        index[i].size   = (uint32_t)jpeg_data.size();
-        fwrite(jpeg_data.data(), 1, jpeg_data.size(), f);
+        index[i].offset = static_cast<uint32_t>(avi_data.size());
+        write_fourcc(avi_data, "00dc");
+        write_u32_le(avi_data, static_cast<uint32_t>(jpeg_data.size()));
+        index[i].size = (uint32_t)jpeg_data.size();
+        avi_data.insert(avi_data.end(), jpeg_data.begin(), jpeg_data.end());
 
         if (jpeg_data.size() % 2) {
-            fputc(0, f);
+            avi_data.push_back(0);
         }
     }
 
-    long cur_pos   = ftell(f);
-    long movi_size = cur_pos - movi_size_pos - 4;
-    fseek(f, movi_size_pos, SEEK_SET);
-    write_u32_le(f, movi_size);
-    fseek(f, cur_pos, SEEK_SET);
+    const size_t movi_size = avi_data.size() - movi_size_pos - 4;
+    patch_u32_le(avi_data, movi_size_pos, static_cast<uint32_t>(movi_size));
 
-    fwrite("idx1", 4, 1, f);
-    write_u32_le(f, num_images * 16);
+    write_fourcc(avi_data, "idx1");
+    write_u32_le(avi_data, num_images * 16);
     for (int i = 0; i < num_images; i++) {
-        fwrite("00dc", 4, 1, f);
-        write_u32_le(f, 0x10);
-        write_u32_le(f, index[i].offset);
-        write_u32_le(f, index[i].size);
+        write_fourcc(avi_data, "00dc");
+        write_u32_le(avi_data, 0x10);
+        write_u32_le(avi_data, index[i].offset);
+        write_u32_le(avi_data, index[i].size);
     }
 
-    cur_pos        = ftell(f);
-    long file_size = cur_pos - riff_size_pos - 4;
-    fseek(f, riff_size_pos, SEEK_SET);
-    write_u32_le(f, file_size);
-    fseek(f, cur_pos, SEEK_SET);
+    const size_t file_size = avi_data.size() - riff_size_pos - 4;
+    patch_u32_le(avi_data, riff_size_pos, static_cast<uint32_t>(file_size));
 
+    return avi_data;
+}
+
+int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality) {
+    std::vector<uint8_t> avi_data = create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality);
+    if (avi_data.empty()) {
+        return -1;
+    }
+    if (!write_binary_file_bytes(filename, avi_data)) {
+        perror("Error opening file for writing");
+        return -1;
+    }
     return 0;
 }
 
 #ifdef SD_USE_WEBP
-int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality) {
+std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality) {
     if (num_images == 0) {
         fprintf(stderr, "Error: Image array is empty.\n");
-        return -1;
+        return {};
     }
     if (fps <= 0) {
         fprintf(stderr, "Error: FPS must be positive.\n");
-        return -1;
+        return {};
     }
 
     const int width    = static_cast<int>(images[0].width);
@@ -857,14 +941,14 @@ int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images
     const int channels = static_cast<int>(images[0].channel);
     if (channels != 1 && channels != 3 && channels != 4) {
         fprintf(stderr, "Error: Unsupported channel count: %d\n", channels);
-        return -1;
+        return {};
     }
 
     WebPAnimEncoderOptions anim_options;
     WebPConfig config;
     if (!WebPAnimEncoderOptionsInit(&anim_options) || !WebPConfigInit(&config)) {
         fprintf(stderr, "Error: Failed to initialize WebP animation encoder.\n");
-        return -1;
+        return {};
     }
 
     config.quality      = static_cast<float>(quality);
@@ -875,13 +959,13 @@ int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images
     }
     if (!WebPValidateConfig(&config)) {
         fprintf(stderr, "Error: Invalid WebP encoder configuration.\n");
-        return -1;
+        return {};
     }
 
     WebPAnimEncoderPtr enc(WebPAnimEncoderNew(width, height, &anim_options));
     if (enc == nullptr) {
         fprintf(stderr, "Error: Could not create WebPAnimEncoder object.\n");
-        return -1;
+        return {};
     }
 
     const int frame_duration_ms = std::max(1, static_cast<int>(std::lround(1000.0 / static_cast<double>(fps))));
@@ -891,13 +975,13 @@ int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images
         const sd_image_t& image = images[i];
         if (static_cast<int>(image.width) != width || static_cast<int>(image.height) != height) {
             fprintf(stderr, "Error: Frame dimensions do not match.\n");
-            return -1;
+            return {};
         }
 
         WebPPictureGuard picture;
         if (!picture.initialized) {
             fprintf(stderr, "Error: Failed to initialize WebPPicture.\n");
-            return -1;
+            return {};
         }
         picture.picture.use_argb = 1;
         picture.picture.width    = width;
@@ -921,12 +1005,12 @@ int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images
 
         if (!picture_ok) {
             fprintf(stderr, "Error: Failed to import frame into WebPPicture.\n");
-            return -1;
+            return {};
         }
 
         if (!WebPAnimEncoderAdd(enc.get(), &picture.picture, timestamp_ms, &config)) {
             fprintf(stderr, "Error: Failed to add frame to animated WebP: %s\n", WebPAnimEncoderGetError(enc.get()));
-            return -1;
+            return {};
         }
 
         timestamp_ms += frame_duration_ms;
@@ -934,52 +1018,50 @@ int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images
 
     if (!WebPAnimEncoderAdd(enc.get(), nullptr, timestamp_ms, nullptr)) {
         fprintf(stderr, "Error: Failed to finalize animated WebP frames: %s\n", WebPAnimEncoderGetError(enc.get()));
-        return -1;
+        return {};
     }
 
     WebPDataGuard webp_data;
     if (!WebPAnimEncoderAssemble(enc.get(), &webp_data.data)) {
         fprintf(stderr, "Error: Failed to assemble animated WebP: %s\n", WebPAnimEncoderGetError(enc.get()));
-        return -1;
+        return {};
     }
 
-    FilePtr f(fopen(filename, "wb"));
-    if (!f) {
+    return std::vector<uint8_t>(webp_data.data.bytes, webp_data.data.bytes + webp_data.data.size);
+}
+
+int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality) {
+    std::vector<uint8_t> webp_data = create_animated_webp_from_sd_images_to_vector(images, num_images, fps, quality);
+    if (webp_data.empty()) {
+        return -1;
+    }
+    if (!write_binary_file_bytes(filename, webp_data)) {
         perror("Error opening file for writing");
         return -1;
     }
-    if (webp_data.data.size > 0 && fwrite(webp_data.data.bytes, 1, webp_data.data.size, f.get()) != webp_data.data.size) {
-        fprintf(stderr, "Error: Failed to write animated WebP file.\n");
-        return -1;
-    }
-
     return 0;
 }
 #endif
 
 #ifdef SD_USE_WEBM
-int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality) {
+std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality) {
     if (num_images == 0) {
         fprintf(stderr, "Error: Image array is empty.\n");
-        return -1;
+        return {};
     }
     if (fps <= 0) {
         fprintf(stderr, "Error: FPS must be positive.\n");
-        return -1;
+        return {};
     }
 
     const int width  = static_cast<int>(images[0].width);
     const int height = static_cast<int>(images[0].height);
     if (width <= 0 || height <= 0) {
         fprintf(stderr, "Error: Invalid frame dimensions.\n");
-        return -1;
+        return {};
     }
 
-    mkvmuxer::MkvWriter writer;
-    if (!writer.Open(filename)) {
-        fprintf(stderr, "Error: Could not open WebM file for writing.\n");
-        return -1;
-    }
+    MemoryMkvWriter writer;
 
     const int ret = [&]() -> int {
         mkvmuxer::Segment segment;
@@ -1045,30 +1127,63 @@ int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num
         }
         return 0;
     }();
-    writer.Close();
-    return ret;
+    if (ret != 0) {
+        return {};
+    }
+    return writer.data();
+}
+
+int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality) {
+    std::vector<uint8_t> webm_data = create_webm_from_sd_images_to_vector(images, num_images, fps, quality);
+    if (webm_data.empty()) {
+        return -1;
+    }
+    if (!write_binary_file_bytes(filename, webm_data)) {
+        perror("Error opening file for writing");
+        return -1;
+    }
+    return 0;
 }
 #endif
 
-int create_video_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality) {
-    std::string path = filename ? filename : "";
-    auto pos         = path.find_last_of('.');
-    std::string ext  = pos == std::string::npos ? "" : path.substr(pos);
-    for (char& ch : ext) {
-        ch = static_cast<char>(tolower(static_cast<unsigned char>(ch)));
+std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& output_format,
+                                                           sd_image_t* images,
+                                                           int num_images,
+                                                           int fps,
+                                                           int quality) {
+    std::string format = output_format;
+    std::transform(format.begin(), format.end(), format.begin(),
+                   [](unsigned char c) { return static_cast<char>(tolower(c)); });
+    if (!format.empty() && format[0] == '.') {
+        format.erase(format.begin());
     }
 
 #ifdef SD_USE_WEBM
-    if (ext == ".webm") {
-        return create_webm_from_sd_images(filename, images, num_images, fps, quality);
+    if (format == "webm") {
+        return create_webm_from_sd_images_to_vector(images, num_images, fps, quality);
     }
 #endif
 
 #ifdef SD_USE_WEBP
-    if (ext == ".webp") {
-        return create_animated_webp_from_sd_images(filename, images, num_images, fps, quality);
+    if (format == "webp") {
+        return create_animated_webp_from_sd_images_to_vector(images, num_images, fps, quality);
     }
 #endif
 
-    return create_mjpg_avi_from_sd_images(filename, images, num_images, fps, quality);
+    return create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality);
+}
+
+int create_video_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality) {
+    std::string path                = filename ? filename : "";
+    auto pos                        = path.find_last_of('.');
+    std::string ext                 = pos == std::string::npos ? "" : path.substr(pos);
+    std::vector<uint8_t> video_data = create_video_from_sd_images_to_vector(ext, images, num_images, fps, quality);
+    if (video_data.empty()) {
+        return -1;
+    }
+    if (!write_binary_file_bytes(filename, video_data)) {
+        perror("Error opening file for writing");
+        return -1;
+    }
+    return 0;
 }

examples/common/media_io.h

@@ -58,6 +58,10 @@ int create_mjpg_avi_from_sd_images(const char* filename,
                                    int num_images,
                                    int fps,
                                    int quality = 90);
+std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images,
+                                                              int num_images,
+                                                              int fps,
+                                                              int quality = 90);
 
 #ifdef SD_USE_WEBP
 int create_animated_webp_from_sd_images(const char* filename,
@@ -65,6 +69,10 @@ int create_animated_webp_from_sd_images(const char* filename,
                                         int num_images,
                                         int fps,
                                         int quality = 90);
+std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(sd_image_t* images,
+                                                                   int num_images,
+                                                                   int fps,
+                                                                   int quality = 90);
 #endif
 
 #ifdef SD_USE_WEBM
@@ -73,6 +81,10 @@ int create_webm_from_sd_images(const char* filename,
                                int num_images,
                                int fps,
                                int quality = 90);
+std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images,
+                                                          int num_images,
+                                                          int fps,
+                                                          int quality = 90);
 #endif
 
 int create_video_from_sd_images(const char* filename,
@@ -80,5 +92,10 @@ int create_video_from_sd_images(const char* filename,
                                 int num_images,
                                 int fps,
                                 int quality = 90);
+std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& output_format,
+                                                           sd_image_t* images,
+                                                           int num_images,
+                                                           int fps,
+                                                           int quality = 90);
 
 #endif  // __MEDIA_IO_H__

examples/server/README.md

@@ -219,7 +219,7 @@ Default Generation Options:
                                            medium
   --skip-layer-start <float>               SLG enabling point (default: 0.01)
   --skip-layer-end <float>                 SLG disabling point (default: 0.2)
-  --eta <float>                            noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a and dpm++2s_a)
+  --eta <float>                            noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)
   --flow-shift <float>                     shift value for Flow models like SD3.x or WAN (default: auto)
   --high-noise-cfg-scale <float>           (high noise) unconditional guidance scale: (default: 7.0)
   --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or instruct-pix2pix models (default: same as --cfg-scale)
@@ -227,7 +227,7 @@ Default Generation Options:
   --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default: 0)
   --high-noise-skip-layer-start <float>    (high noise) SLG enabling point (default: 0.01)
   --high-noise-skip-layer-end <float>      (high noise) SLG disabling point (default: 0.2)
-  --high-noise-eta <float>                 (high noise) noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a and dpm++2s_a)
+  --high-noise-eta <float>                 (high noise) noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)
   --strength <float>                       strength for noising/unnoising (default: 0.75)
   --pm-style-strength <float>
   --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image
@@ -238,10 +238,10 @@ Default Generation Options:
   --disable-image-metadata                 do not embed generation metadata on image files
   -s, --seed                               RNG seed (default: 42, use random seed for < 0)
   --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing,
-                                           tcd, res_multistep, res_2s] (default: euler for Flux/SD3/Wan, euler_a
+                                           tcd, res_multistep, res_2s, er_sde] (default: euler for Flux/SD3/Wan, euler_a
                                            otherwise)
   --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm,
-                                           ddim_trailing, tcd, res_multistep, res_2s] default: euler for Flux/SD3/Wan,
+                                           ddim_trailing, tcd, res_multistep, res_2s, er_sde] default: euler for Flux/SD3/Wan,
                                            euler_a otherwise
   --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, smoothstep, sgm_uniform, simple,
                                            kl_optimal, lcm, bong_tangent], default: discrete

examples/server/api.md

@@ -9,7 +9,7 @@ The server currently exposes three API families:
 - `sdcpp API` under `/sdcpp/v1/...`
 
 The `sdcpp API` is the native API surface.
-Its request schema is also the canonical schema for `sd_cpp_extra_args`.
+Its request schema is the same schema used by `sd_cpp_extra_args`.
 
 Global LoRA rule:
 
@@ -55,8 +55,6 @@ Current endpoints include:
 - `POST /sdcpp/v1/jobs/{id}/cancel`
 - `POST /sdcpp/v1/vid_gen`
 
-`POST /sdcpp/v1/vid_gen` is currently exposed but returns `501 Not Implemented`.
-
 ## `sd_cpp_extra_args`
 
 `sd_cpp_extra_args` is an extension mechanism for the compatibility APIs.
@@ -79,12 +77,12 @@ Behavior:
 - The JSON block is parsed using the same field rules as the `sdcpp API`.
 - The block is removed from the final prompt before generation.
 
-Intended use:
+Supported use:
 
 - extend `OpenAI API` requests with native `stable-diffusion.cpp` controls
 - extend `sdapi` requests with native `stable-diffusion.cpp` controls
 
-Not intended use:
+Unsupported use:
 
 - do not use `sd_cpp_extra_args` with `/sdcpp/v1/*`
 
@@ -372,20 +370,25 @@ Field types:
 
 Returns frontend-friendly capability metadata.
 
-Typical contents:
+The mode-aware fields are the primary interface. The top-level compatibility fields are deprecated mirrors kept for older clients.
 
-| Field | Type |
-| --- | --- |
-| `model` | `object` |
-| `defaults` | `object` |
-| `loras` | `array<object>` |
-| `samplers` | `array<string>` |
-| `schedulers` | `array<string>` |
-| `output_formats` | `array<string>` |
-| `limits` | `object` |
-| `features` | `object` |
+Top-level fields:
 
-Nested fields currently returned:
+| Field | Type | Notes |
+| --- | --- | --- |
+| `model` | `object` | Loaded model metadata |
+| `current_mode` | `string` | The native generation mode mirrored by top-level compatibility fields |
+| `supported_modes` | `array<string>` | Supported native modes such as `img_gen` or `vid_gen` |
+| `defaults` | `object` | Deprecated compatibility mirror of `defaults_by_mode[current_mode]` |
+| `output_formats` | `array<string>` | Deprecated compatibility mirror of `output_formats_by_mode[current_mode]` |
+| `features` | `object` | Deprecated compatibility mirror of `features_by_mode[current_mode]` |
+| `defaults_by_mode` | `object` | Explicit defaults for each supported mode |
+| `output_formats_by_mode` | `object` | Explicit output formats for each supported mode |
+| `features_by_mode` | `object` | Explicit feature flags for each supported mode |
+| `samplers` | `array<string>` | Available sampling methods |
+| `schedulers` | `array<string>` | Available schedulers |
+| `loras` | `array<object>` | Available LoRA entries |
+| `limits` | `object` | Shared queue and size limits |
 
 `model`
 
@@ -395,50 +398,24 @@ Nested fields currently returned:
 | `model.stem` | `string` |
 | `model.path` | `string` |
 
-`defaults`
+Compatibility rules:
+
+- `defaults`, `output_formats`, and `features` are deprecated compatibility mirrors
+- those three top-level fields always mirror `current_mode`
+- `supported_modes`, `defaults_by_mode`, `output_formats_by_mode`, and `features_by_mode` are the mode-aware fields
+
+Mode-aware objects:
 
 | Field | Type |
 | --- | --- |
-| `defaults.prompt` | `string` |
-| `defaults.negative_prompt` | `string` |
-| `defaults.clip_skip` | `integer` |
-| `defaults.width` | `integer` |
-| `defaults.height` | `integer` |
-| `defaults.strength` | `number` |
-| `defaults.seed` | `integer` |
-| `defaults.batch_count` | `integer` |
-| `defaults.auto_resize_ref_image` | `boolean` |
-| `defaults.increase_ref_index` | `boolean` |
-| `defaults.control_strength` | `number` |
-| `defaults.sample_params` | `object` |
-| `defaults.sample_params.scheduler` | `string` |
-| `defaults.sample_params.sample_method` | `string` |
-| `defaults.sample_params.sample_steps` | `integer` |
-| `defaults.sample_params.eta` | `number \| null` |
-| `defaults.sample_params.shifted_timestep` | `integer` |
-| `defaults.sample_params.flow_shift` | `number \| null` |
-| `defaults.sample_params.guidance` | `object` |
-| `defaults.sample_params.guidance.txt_cfg` | `number` |
-| `defaults.sample_params.guidance.img_cfg` | `number \| null` |
-| `defaults.sample_params.guidance.distilled_guidance` | `number` |
-| `defaults.sample_params.guidance.slg` | `object` |
-| `defaults.sample_params.guidance.slg.layers` | `array<integer>` |
-| `defaults.sample_params.guidance.slg.layer_start` | `number` |
-| `defaults.sample_params.guidance.slg.layer_end` | `number` |
-| `defaults.sample_params.guidance.slg.scale` | `number` |
-| `defaults.vae_tiling_params` | `object` |
-| `defaults.vae_tiling_params.enabled` | `boolean` |
-| `defaults.vae_tiling_params.tile_size_x` | `integer` |
-| `defaults.vae_tiling_params.tile_size_y` | `integer` |
-| `defaults.vae_tiling_params.target_overlap` | `number` |
-| `defaults.vae_tiling_params.rel_size_x` | `number` |
-| `defaults.vae_tiling_params.rel_size_y` | `number` |
-| `defaults.cache_mode` | `string` |
-| `defaults.cache_option` | `string` |
-| `defaults.scm_mask` | `string` |
-| `defaults.scm_policy_dynamic` | `boolean` |
-| `defaults.output_format` | `string` |
-| `defaults.output_compression` | `integer` |
+| `defaults_by_mode.img_gen` | `object` |
+| `defaults_by_mode.vid_gen` | `object` |
+| `output_formats_by_mode.img_gen` | `array<string>` |
+| `output_formats_by_mode.vid_gen` | `array<string>` |
+| `features_by_mode.img_gen` | `object` |
+| `features_by_mode.vid_gen` | `object` |
+
+Shared nested fields:
 
 `loras`
 
@@ -458,19 +435,100 @@ Nested fields currently returned:
 | `limits.max_batch_count` | `integer` |
 | `limits.max_queue_size` | `integer` |
 
-`features`
+Shared default fields used by both `img_gen` and `vid_gen`:
 
 | Field | Type |
 | --- | --- |
-| `features.init_image` | `boolean` |
-| `features.mask_image` | `boolean` |
-| `features.control_image` | `boolean` |
-| `features.ref_images` | `boolean` |
-| `features.lora` | `boolean` |
-| `features.vae_tiling` | `boolean` |
-| `features.cache` | `boolean` |
-| `features.cancel_queued` | `boolean` |
-| `features.cancel_generating` | `boolean` |
+| `prompt` | `string` |
+| `negative_prompt` | `string` |
+| `clip_skip` | `integer` |
+| `width` | `integer` |
+| `height` | `integer` |
+| `strength` | `number` |
+| `seed` | `integer` |
+| `sample_params` | `object` |
+| `sample_params.scheduler` | `string` |
+| `sample_params.sample_method` | `string` |
+| `sample_params.sample_steps` | `integer` |
+| `sample_params.eta` | `number \| null` |
+| `sample_params.shifted_timestep` | `integer` |
+| `sample_params.flow_shift` | `number \| null` |
+| `sample_params.guidance.txt_cfg` | `number` |
+| `sample_params.guidance.img_cfg` | `number \| null` |
+| `sample_params.guidance.distilled_guidance` | `number` |
+| `sample_params.guidance.slg.layers` | `array<integer>` |
+| `sample_params.guidance.slg.layer_start` | `number` |
+| `sample_params.guidance.slg.layer_end` | `number` |
+| `sample_params.guidance.slg.scale` | `number` |
+| `vae_tiling_params` | `object` |
+| `vae_tiling_params.enabled` | `boolean` |
+| `vae_tiling_params.tile_size_x` | `integer` |
+| `vae_tiling_params.tile_size_y` | `integer` |
+| `vae_tiling_params.target_overlap` | `number` |
+| `vae_tiling_params.rel_size_x` | `number` |
+| `vae_tiling_params.rel_size_y` | `number` |
+| `cache_mode` | `string` |
+| `cache_option` | `string` |
+| `scm_mask` | `string` |
+| `scm_policy_dynamic` | `boolean` |
+| `output_format` | `string` |
+| `output_compression` | `integer` |
+
+`img_gen`-specific default fields:
+
+| Field | Type |
+| --- | --- |
+| `batch_count` | `integer` |
+| `auto_resize_ref_image` | `boolean` |
+| `increase_ref_index` | `boolean` |
+| `control_strength` | `number` |
+
+`vid_gen`-specific default fields:
+
+| Field | Type |
+| --- | --- |
+| `video_frames` | `integer` |
+| `fps` | `integer` |
+| `moe_boundary` | `number` |
+| `vace_strength` | `number` |
+| `high_noise_sample_params` | `object` |
+| `high_noise_sample_params.scheduler` | `string` |
+| `high_noise_sample_params.sample_method` | `string` |
+| `high_noise_sample_params.sample_steps` | `integer` |
+| `high_noise_sample_params.eta` | `number \| null` |
+| `high_noise_sample_params.shifted_timestep` | `integer` |
+| `high_noise_sample_params.flow_shift` | `number \| null` |
+| `high_noise_sample_params.guidance.txt_cfg` | `number` |
+| `high_noise_sample_params.guidance.img_cfg` | `number \| null` |
+| `high_noise_sample_params.guidance.distilled_guidance` | `number` |
+| `high_noise_sample_params.guidance.slg.layers` | `array<integer>` |
+| `high_noise_sample_params.guidance.slg.layer_start` | `number` |
+| `high_noise_sample_params.guidance.slg.layer_end` | `number` |
+| `high_noise_sample_params.guidance.slg.scale` | `number` |
+
+Fields returned in `features_by_mode.img_gen`:
+
+- `init_image`
+- `mask_image`
+- `control_image`
+- `ref_images`
+- `lora`
+- `vae_tiling`
+- `cache`
+- `cancel_queued`
+- `cancel_generating`
+
+Fields returned in `features_by_mode.vid_gen`:
+
+- `init_image`
+- `end_image`
+- `control_frames`
+- `high_noise_sample_params`
+- `lora`
+- `vae_tiling`
+- `cache`
+- `cancel_queued`
+- `cancel_generating`
 
 #### `POST /sdcpp/v1/img_gen`
 
@@ -521,9 +579,7 @@ Typical status codes:
 - `409 Conflict`
 - `410 Gone`
 
-### Canonical Request Schema
-
-The `sdcpp API` request body is the canonical native schema.
+### Request Body
 
 Example:
 
@@ -612,7 +668,7 @@ Channel expectations:
 If omitted or null:
 
 - single-image fields map to an empty `sd_image_t`
-- array fields map to `nullptr + count = 0`
+- array fields map to an empty C-style array, represented as `pointer = nullptr` and `count = 0`
 
 ### Field Mapping Summary
 
@@ -686,11 +742,11 @@ HTTP-only output fields:
 | `output_format` | `string` |
 | `output_compression` | `integer` |
 
-### Optional Field Semantics
+### Optional Field Handling
 
-Clients should preserve unset semantics for optional sampling fields.
+Optional sampling fields may be omitted.
 
-If a user has not explicitly provided one of these fields, the client should omit it instead of injecting a guessed fallback:
+When omitted, backend defaults apply to these fields:
 
 - `sample_params.scheduler`
 - `sample_params.sample_method`
@@ -766,29 +822,394 @@ Example cancelled job:
 }
 ```
 
-### Validation and Retention
+### Submission Errors
 
-Recommended behavior:
+`POST /sdcpp/v1/img_gen` may return:
 
-- malformed JSON returns `400`
-- invalid image payloads return `400`
-- invalid parameter structure returns `400`
-- queue full returns `429` or `503`
-- accepted runtime failures transition the job to `failed`
-- unsupported in-progress cancellation may return `409`
+- `202 Accepted` when the job is created
+- `400 Bad Request` for an empty body, unsupported model mode, invalid JSON, or invalid generation parameters
+- `429 Too Many Requests` when the job queue is full
+- `500 Internal Server Error` for unexpected server exceptions during submission
 
-Recommended retention controls:
+### `vid_gen`
 
-- pending job limit
-- completed job TTL
-- failed job TTL
+The following section documents the native async contract for video generation.
 
-### Future `vid_gen`
+#### `POST /sdcpp/v1/vid_gen`
 
-Future `vid_gen` should reuse the same async job model:
+Submits an async video generation job.
 
-- `POST /sdcpp/v1/vid_gen`
-- `GET /sdcpp/v1/jobs/{id}`
-- `POST /sdcpp/v1/jobs/{id}/cancel`
+Successful submission returns `202 Accepted`.
 
-Its request body should mirror `sd_vid_gen_params_t` in the same way that `img_gen` mirrors `sd_img_gen_params_t`.
+Example response:
+
+```json
+{
+  "id": "job_01HTXYZVID",
+  "kind": "vid_gen",
+  "status": "queued",
+  "created": 1775401200,
+  "poll_url": "/sdcpp/v1/jobs/job_01HTXYZVID"
+}
+```
+
+Response fields:
+
+| Field | Type |
+| --- | --- |
+| `id` | `string` |
+| `kind` | `string` |
+| `status` | `string` |
+| `created` | `integer` |
+| `poll_url` | `string` |
+
+### Request Body
+
+Compared with `img_gen`, the `vid_gen` request body:
+
+- `vid_gen` is a single video sequence job, so `batch_count` is not part of the request schema
+- `ref_images`, `mask_image`, `control_image`, `control_strength`, and `embed_image_metadata` are not part of the request schema
+- `vid_gen` adds `end_image`, `control_frames`, `high_noise_sample_params`, `video_frames`, `fps`, `moe_boundary`, and `vace_strength`
+
+Example:
+
+```json
+{
+  "prompt": "a cat walking through a rainy alley",
+  "negative_prompt": "",
+  "clip_skip": -1,
+  "width": 832,
+  "height": 480,
+  "strength": 0.75,
+  "seed": -1,
+  "video_frames": 33,
+  "fps": 16,
+  "moe_boundary": 0.875,
+  "vace_strength": 1.0,
+
+  "init_image": null,
+  "end_image": null,
+  "control_frames": [],
+
+  "sample_params": {
+    "scheduler": "discrete",
+    "sample_method": "euler",
+    "sample_steps": 28,
+    "eta": 1.0,
+    "shifted_timestep": 0,
+    "custom_sigmas": [],
+    "flow_shift": 0.0,
+    "guidance": {
+      "txt_cfg": 7.0,
+      "img_cfg": 7.0,
+      "distilled_guidance": 3.5,
+      "slg": {
+        "layers": [7, 8, 9],
+        "layer_start": 0.01,
+        "layer_end": 0.2,
+        "scale": 0.0
+      }
+    }
+  },
+
+  "high_noise_sample_params": {
+    "scheduler": "discrete",
+    "sample_method": "euler",
+    "sample_steps": -1,
+    "eta": 1.0,
+    "shifted_timestep": 0,
+    "flow_shift": 0.0,
+    "guidance": {
+      "txt_cfg": 7.0,
+      "img_cfg": 7.0,
+      "distilled_guidance": 3.5,
+      "slg": {
+        "layers": [7, 8, 9],
+        "layer_start": 0.01,
+        "layer_end": 0.2,
+        "scale": 0.0
+      }
+    }
+  },
+
+  "lora": [],
+
+  "vae_tiling_params": {
+    "enabled": false,
+    "tile_size_x": 0,
+    "tile_size_y": 0,
+    "target_overlap": 0.5,
+    "rel_size_x": 0.0,
+    "rel_size_y": 0.0
+  },
+
+  "cache_mode": "disabled",
+  "cache_option": "",
+  "scm_mask": "",
+  "scm_policy_dynamic": true,
+
+  "output_format": "webm",
+  "output_compression": 100
+}
+```
+
+### LoRA Rules
+
+- The server only accepts explicit LoRA entries from the `lora` field.
+- Prompt-embedded `<lora:...>` tags are intentionally unsupported.
+- `lora[].is_high_noise` controls whether a LoRA applies only to the high-noise stage.
+
+### Image and Frame Encoding Rules
+
+Any image field accepts:
+
+- a raw base64 string, or
+- a data URL such as `data:image/png;base64,...`
+
+Channel expectations:
+
+- `init_image`: 3 channels
+- `end_image`: 3 channels
+- `control_frames[]`: 3 channels
+
+Frame ordering rules:
+
+- `control_frames[]` order is the conditioning frame order
+- `control_frames[]` is preserved in request order
+
+If omitted or null:
+
+- single-image fields map to an empty `sd_image_t`
+- array fields map to an empty C-style array, represented as `pointer = nullptr` and `count = 0`
+
+### Field Mapping Summary
+
+Top-level scalar fields:
+
+| Field | Type |
+| --- | --- |
+| `prompt` | `string` |
+| `negative_prompt` | `string` |
+| `clip_skip` | `integer` |
+| `width` | `integer` |
+| `height` | `integer` |
+| `strength` | `number` |
+| `seed` | `integer` |
+| `video_frames` | `integer` |
+| `fps` | `integer` |
+| `moe_boundary` | `number` |
+| `vace_strength` | `number` |
+
+Image and frame fields:
+
+| Field | Type |
+| --- | --- |
+| `init_image` | `string \| null` |
+| `end_image` | `string \| null` |
+| `control_frames` | `array<string>` |
+
+LoRA fields:
+
+| Field | Type |
+| --- | --- |
+| `lora[].path` | `string` |
+| `lora[].multiplier` | `number` |
+| `lora[].is_high_noise` | `boolean` |
+
+Sampling fields:
+
+| Field | Type |
+| --- | --- |
+| `sample_params.scheduler` | `string` |
+| `sample_params.sample_method` | `string` |
+| `sample_params.sample_steps` | `integer` |
+| `sample_params.eta` | `number` |
+| `sample_params.shifted_timestep` | `integer` |
+| `sample_params.custom_sigmas` | `array<number>` |
+| `sample_params.flow_shift` | `number` |
+| `sample_params.guidance.txt_cfg` | `number` |
+| `sample_params.guidance.img_cfg` | `number` |
+| `sample_params.guidance.distilled_guidance` | `number` |
+| `sample_params.guidance.slg.layers` | `array<integer>` |
+| `sample_params.guidance.slg.layer_start` | `number` |
+| `sample_params.guidance.slg.layer_end` | `number` |
+| `sample_params.guidance.slg.scale` | `number` |
+
+High-noise sampling fields:
+
+| Field | Type |
+| --- | --- |
+| `high_noise_sample_params.scheduler` | `string` |
+| `high_noise_sample_params.sample_method` | `string` |
+| `high_noise_sample_params.sample_steps` | `integer` |
+| `high_noise_sample_params.eta` | `number` |
+| `high_noise_sample_params.shifted_timestep` | `integer` |
+| `high_noise_sample_params.flow_shift` | `number` |
+| `high_noise_sample_params.guidance.txt_cfg` | `number` |
+| `high_noise_sample_params.guidance.img_cfg` | `number` |
+| `high_noise_sample_params.guidance.distilled_guidance` | `number` |
+| `high_noise_sample_params.guidance.slg.layers` | `array<integer>` |
+| `high_noise_sample_params.guidance.slg.layer_start` | `number` |
+| `high_noise_sample_params.guidance.slg.layer_end` | `number` |
+| `high_noise_sample_params.guidance.slg.scale` | `number` |
+
+Other native fields:
+
+| Field | Type |
+| --- | --- |
+| `vae_tiling_params` | `object` |
+| `cache_mode` | `string` |
+| `cache_option` | `string` |
+| `scm_mask` | `string` |
+| `scm_policy_dynamic` | `boolean` |
+
+HTTP-only output fields:
+
+| Field | Type |
+| --- | --- |
+| `output_format` | `string` |
+| `output_compression` | `integer` |
+
+For `vid_gen`, `output_format` and `output_compression` control container encoding.
+`fps` is request metadata for the generated sequence and is echoed in the completed job result.
+
+Allowed `output_format` values:
+
+- `webm`
+- `webp`
+- `avi`
+
+Output format behavior:
+
+- `output_format` defaults to `webm`
+- `webp` means animated WebP
+- `avi` means MJPG AVI
+- `webm` requires the server to be built with WebM support; otherwise the request returns `400`
+
+### Result Payload
+
+Completed jobs return one encoded container payload, not a list of per-frame images.
+
+Result fields:
+
+- `result.b64_json` contains the whole encoded container file as base64
+- `result.mime_type` identifies the media type
+- `result.output_format` echoes the selected container format
+- `result.fps` echoes the effective playback FPS
+- `result.frame_count` reports the actual decoded frame count used to build the container
+
+Expected MIME types:
+
+| `output_format` | `mime_type` |
+| --- | --- |
+| `webm` | `video/webm` |
+| `webp` | `image/webp` |
+| `avi` | `video/x-msvideo` |
+
+### Optional Field Handling
+
+Optional sampling fields may be omitted.
+
+When omitted, backend defaults apply to these fields:
+
+- `sample_params.scheduler`
+- `sample_params.sample_method`
+- `sample_params.eta`
+- `sample_params.flow_shift`
+- `sample_params.guidance.img_cfg`
+- `high_noise_sample_params.scheduler`
+- `high_noise_sample_params.sample_method`
+- `high_noise_sample_params.eta`
+- `high_noise_sample_params.flow_shift`
+- `high_noise_sample_params.guidance.img_cfg`
+
+`high_noise_sample_params` may also be omitted entirely.
+
+### Frame Count Semantics
+
+`video_frames` is the requested target length, but the current core video path internally normalizes the effective frame count to the largest `4n + 1` value that does not exceed the requested count.
+
+Examples:
+
+- `video_frames = 33` stays `33`
+- `video_frames = 34` becomes `33`
+- `video_frames = 32` becomes `29`
+
+The completed job payload includes the actual decoded `frame_count`.
+
+### Completion Result
+
+Example completed job:
+
+```json
+{
+  "id": "job_01HTXYZVID",
+  "kind": "vid_gen",
+  "status": "completed",
+  "created": 1775401200,
+  "started": 1775401203,
+  "completed": 1775401215,
+  "queue_position": 0,
+  "result": {
+    "output_format": "webm",
+    "mime_type": "video/webm",
+    "fps": 16,
+    "frame_count": 33,
+    "b64_json": "GkXfo59ChoEBQveBAULygQRC84EIQo..."
+  },
+  "error": null
+}
+```
+
+The response returns the encoded `.webm`, animated `.webp`, or `.avi` container payload directly.
+
+### Failure Result
+
+Example failed job:
+
+```json
+{
+  "id": "job_01HTXYZVID",
+  "kind": "vid_gen",
+  "status": "failed",
+  "created": 1775401200,
+  "started": 1775401203,
+  "completed": 1775401204,
+  "queue_position": 0,
+  "result": null,
+  "error": {
+    "code": "generation_failed",
+    "message": "generate_video returned no results"
+  }
+}
+```
+
+### Cancelled Result
+
+Example cancelled job:
+
+```json
+{
+  "id": "job_01HTXYZVID",
+  "kind": "vid_gen",
+  "status": "cancelled",
+  "created": 1775401200,
+  "started": null,
+  "completed": 1775401202,
+  "queue_position": 0,
+  "result": null,
+  "error": {
+    "code": "cancelled",
+    "message": "job cancelled by client"
+  }
+}
+```
+
+### Submission Errors
+
+`POST /sdcpp/v1/vid_gen` may return:
+
+- `202 Accepted` when the job is created
+- `400 Bad Request` for an empty body, unsupported model mode, invalid JSON, invalid generation parameters, or an unsupported output format
+- `429 Too Many Requests` when the job queue is full
+- `500 Internal Server Error` for unexpected server exceptions during submission

examples/server/async_jobs.cpp

@@ -95,8 +95,12 @@ bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job) {
     job.status       = AsyncJobStatus::Cancelled;
     job.completed_at = unix_timestamp_now();
     job.result_images_b64.clear();
-    job.error_code    = "cancelled";
-    job.error_message = "job cancelled by client";
+    job.result_media_b64.clear();
+    job.result_media_mime_type.clear();
+    job.result_frame_count = 0;
+    job.result_fps         = 0;
+    job.error_code         = "cancelled";
+    job.error_message      = "job cancelled by client";
     return true;
 }
 
@@ -122,14 +126,24 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
     }
 
     if (job.status == AsyncJobStatus::Completed) {
-        json images = json::array();
-        for (size_t i = 0; i < job.result_images_b64.size(); ++i) {
-            images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}});
+        if (job.kind == AsyncJobKind::VidGen) {
+            result["result"] = {
+                {"output_format", job.vid_gen.output_format},
+                {"mime_type", job.result_media_mime_type},
+                {"fps", job.result_fps},
+                {"frame_count", job.result_frame_count},
+                {"b64_json", job.result_media_b64},
+            };
+        } else {
+            json images = json::array();
+            for (size_t i = 0; i < job.result_images_b64.size(); ++i) {
+                images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}});
+            }
+            result["result"] = {
+                {"output_format", job.img_gen.output_format},
+                {"images", images},
+            };
         }
-        result["result"] = {
-            {"output_format", job.img_gen.output_format},
-            {"images", images},
-        };
         result["error"] = nullptr;
     } else if (job.status == AsyncJobStatus::Failed ||
                job.status == AsyncJobStatus::Cancelled) {
@@ -156,16 +170,15 @@ bool execute_img_gen_job(ServerRuntime& runtime,
     sd_img_gen_params_t params = job.img_gen.to_sd_img_gen_params_t();
 
     SDImageVec results;
-    int num_results = 0;
 
     {
         std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
         sd_image_t* raw_results = generate_image(runtime.sd_ctx, &params);
-        num_results             = params.batch_count;
-        results.adopt(raw_results, num_results);
+        results.adopt(raw_results, params.batch_count);
     }
 
-    if (results.empty() || num_results <= 0) {
+    const int num_results = results.count();
+    if (num_results <= 0) {
         error_message = "generate_image returned no results";
         return false;
     }
@@ -208,6 +221,47 @@ bool execute_img_gen_job(ServerRuntime& runtime,
     return true;
 }
 
+bool execute_vid_gen_job(ServerRuntime& runtime,
+                         AsyncGenerationJob& job,
+                         std::string& output_media_b64,
+                         std::string& output_media_mime_type,
+                         int& output_frame_count,
+                         int& output_fps,
+                         std::string& error_message) {
+    sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t();
+
+    SDImageVec results;
+    int num_results = 0;
+
+    {
+        std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
+        sd_image_t* raw_results = generate_video(runtime.sd_ctx, &params, &num_results);
+        results.adopt(raw_results, num_results);
+    }
+
+    num_results = results.count();
+    if (num_results <= 0) {
+        error_message = "generate_video returned no results";
+        return false;
+    }
+
+    std::vector<uint8_t> video_bytes = create_video_from_sd_images_to_vector(job.vid_gen.output_format,
+                                                                             results.data(),
+                                                                             num_results,
+                                                                             job.vid_gen.gen_params.fps,
+                                                                             job.vid_gen.output_compression);
+    if (video_bytes.empty()) {
+        error_message = "failed to encode generated video container";
+        return false;
+    }
+
+    output_media_b64       = base64_encode(video_bytes);
+    output_media_mime_type = video_mime_type(job.vid_gen.output_format);
+    output_frame_count     = num_results;
+    output_fps             = job.vid_gen.gen_params.fps;
+    return true;
+}
+
 void async_job_worker(ServerRuntime& runtime) {
     AsyncJobManager& manager = *runtime.async_job_manager;
 
@@ -240,11 +294,23 @@ void async_job_worker(ServerRuntime& runtime) {
         }
 
         std::vector<std::string> output_images;
+        std::string output_media_b64;
+        std::string output_media_mime_type;
+        int output_frame_count = 0;
+        int output_fps         = 0;
         std::string error_message;
         bool ok = false;
 
         if (job->kind == AsyncJobKind::ImgGen) {
             ok = execute_img_gen_job(runtime, *job, output_images, error_message);
+        } else if (job->kind == AsyncJobKind::VidGen) {
+            ok = execute_vid_gen_job(runtime,
+                                     *job,
+                                     output_media_b64,
+                                     output_media_mime_type,
+                                     output_frame_count,
+                                     output_fps,
+                                     error_message);
         } else {
             error_message = "unsupported job kind";
         }
@@ -258,8 +324,12 @@ void async_job_worker(ServerRuntime& runtime) {
 
             job->completed_at = unix_timestamp_now();
             if (ok) {
-                job->status            = AsyncJobStatus::Completed;
-                job->result_images_b64 = std::move(output_images);
+                job->status                 = AsyncJobStatus::Completed;
+                job->result_images_b64      = std::move(output_images);
+                job->result_media_b64       = std::move(output_media_b64);
+                job->result_media_mime_type = std::move(output_media_mime_type);
+                job->result_frame_count     = output_frame_count;
+                job->result_fps             = output_fps;
                 job->error_code.clear();
                 job->error_message.clear();
             } else {
@@ -267,6 +337,10 @@ void async_job_worker(ServerRuntime& runtime) {
                 job->error_code    = "generation_failed";
                 job->error_message = error_message.empty() ? "unknown generation error" : error_message;
                 job->result_images_b64.clear();
+                job->result_media_b64.clear();
+                job->result_media_mime_type.clear();
+                job->result_frame_count = 0;
+                job->result_fps         = 0;
             }
 
             purge_expired_jobs(manager);

examples/server/async_jobs.h

@@ -36,7 +36,12 @@ struct AsyncGenerationJob {
     int64_t started_at    = 0;
     int64_t completed_at  = 0;
     ImgGenJobRequest img_gen;
+    VidGenJobRequest vid_gen;
     std::vector<std::string> result_images_b64;
+    std::string result_media_b64;
+    std::string result_media_mime_type;
+    int result_frame_count = 0;
+    int result_fps         = 0;
     std::string error_code;
     std::string error_message;
 };
@@ -63,4 +68,11 @@ bool execute_img_gen_job(ServerRuntime& runtime,
                          AsyncGenerationJob& job,
                          std::vector<std::string>& output_images,
                          std::string& error_message);
+bool execute_vid_gen_job(ServerRuntime& runtime,
+                         AsyncGenerationJob& job,
+                         std::string& output_media_b64,
+                         std::string& output_media_mime_type,
+                         int& output_frame_count,
+                         int& output_fps,
+                         std::string& error_message);
 void async_job_worker(ServerRuntime& runtime);

examples/server/routes_openai.cpp

@@ -253,6 +253,12 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
 
     svr.Post("/v1/images/generations", [runtime](const httplib::Request& req, httplib::Response& res) {
         try {
+            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
+                res.status = 400;
+                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
+                return;
+            }
+
             ImgGenJobRequest request;
             std::string error_message;
             if (!build_openai_generation_request(req, *runtime, request, error_message)) {
@@ -319,6 +325,12 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
 
     svr.Post("/v1/images/edits", [runtime](const httplib::Request& req, httplib::Response& res) {
         try {
+            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
+                res.status = 400;
+                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
+                return;
+            }
+
             ImgGenJobRequest request;
             std::string error_message;
             if (!build_openai_edit_request(req, *runtime, request, error_message)) {

examples/server/routes_sdapi.cpp

@@ -246,6 +246,11 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                 res.set_content(R"({"error":"empty body"})", "application/json");
                 return;
             }
+            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
+                res.status = 400;
+                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
+                return;
+            }
 
             json j = json::parse(req.body);
             ImgGenJobRequest request;

examples/server/routes_sdcpp.cpp

@@ -75,48 +75,33 @@ static fs::path resolve_display_model_path(const ServerRuntime& runtime) {
     return {};
 }
 
-static json make_capabilities_json(ServerRuntime& runtime) {
-    refresh_lora_cache(runtime);
-
-    AsyncJobManager& manager  = *runtime.async_job_manager;
-    const auto& defaults      = *runtime.default_gen_params;
-    const auto& sample_params = defaults.sample_params;
-    const auto& guidance      = sample_params.guidance;
-    const fs::path model_path = resolve_display_model_path(runtime);
-    json samplers             = json::array();
-    json schedulers           = json::array();
-    json output_formats       = json::array({"png", "jpeg"});
-    json available_loras      = json::array();
-
-    for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) {
-        samplers.push_back(sd_sample_method_name((sample_method_t)i));
-    }
-
-    for (int i = 0; i < SCHEDULER_COUNT; ++i) {
-        schedulers.push_back(sd_scheduler_name((scheduler_t)i));
-    }
-
-#ifdef SD_USE_WEBP
-    output_formats.push_back("webp");
-#endif
-
-    {
-        std::lock_guard<std::mutex> lock(*runtime.lora_mutex);
-        for (const auto& entry : *runtime.lora_cache) {
-            available_loras.push_back({
-                {"name", entry.name},
-                {"path", entry.path},
-            });
-        }
-    }
-
-    json result;
-    result["model"] = {
-        {"name", model_path.filename().u8string()},
-        {"stem", model_path.stem().u8string()},
-        {"path", model_path.u8string()},
+static json make_sample_params_json(const sd_sample_params_t& sample_params, const std::vector<int>& skip_layers) {
+    const auto& guidance = sample_params.guidance;
+    return {
+        {"scheduler", capability_scheduler_name(sample_params.scheduler)},
+        {"sample_method", capability_sample_method_name(sample_params.sample_method)},
+        {"sample_steps", sample_params.sample_steps},
+        {"eta", finite_number_or_null(sample_params.eta)},
+        {"shifted_timestep", sample_params.shifted_timestep},
+        {"flow_shift", finite_number_or_null(sample_params.flow_shift)},
+        {"guidance",
+         {
+             {"txt_cfg", guidance.txt_cfg},
+             {"img_cfg", finite_number_or_null(guidance.img_cfg)},
+             {"distilled_guidance", guidance.distilled_guidance},
+             {"slg",
+              {
+                  {"layers", skip_layers},
+                  {"layer_start", guidance.slg.layer_start},
+                  {"layer_end", guidance.slg.layer_end},
+                  {"scale", guidance.slg.scale},
+              }},
+         }},
     };
-    result["defaults"] = {
+}
+
+static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const std::string& output_format) {
+    return {
         {"prompt", defaults.prompt},
         {"negative_prompt", defaults.negative_prompt},
         {"clip_skip", defaults.clip_skip},
@@ -128,59 +113,177 @@ static json make_capabilities_json(ServerRuntime& runtime) {
         {"auto_resize_ref_image", defaults.auto_resize_ref_image},
         {"increase_ref_index", defaults.increase_ref_index},
         {"control_strength", defaults.control_strength},
-        {"sample_params",
-         {
-             {"scheduler", capability_scheduler_name(sample_params.scheduler)},
-             {"sample_method", capability_sample_method_name(sample_params.sample_method)},
-             {"sample_steps", sample_params.sample_steps},
-             {"eta", finite_number_or_null(sample_params.eta)},
-             {"shifted_timestep", sample_params.shifted_timestep},
-             {"flow_shift", finite_number_or_null(sample_params.flow_shift)},
-             {"guidance",
-              {
-                  {"txt_cfg", guidance.txt_cfg},
-                  {"img_cfg", finite_number_or_null(guidance.img_cfg)},
-                  {"distilled_guidance", guidance.distilled_guidance},
-                  {"slg",
-                   {
-                       {"layers", defaults.skip_layers},
-                       {"layer_start", guidance.slg.layer_start},
-                       {"layer_end", guidance.slg.layer_end},
-                       {"scale", guidance.slg.scale},
-                   }},
-              }},
-         }},
+        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)},
         {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)},
         {"cache_mode", defaults.cache_mode},
         {"cache_option", defaults.cache_option},
         {"scm_mask", defaults.scm_mask},
         {"scm_policy_dynamic", defaults.scm_policy_dynamic},
-        {"output_format", "png"},
+        {"output_format", output_format},
         {"output_compression", 100},
     };
-    result["limits"] = {
-        {"min_width", 64},
-        {"max_width", 4096},
-        {"min_height", 64},
-        {"max_height", 4096},
-        {"max_batch_count", 8},
-        {"max_queue_size", manager.max_pending_jobs},
+}
+
+static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const std::string& output_format) {
+    return {
+        {"prompt", defaults.prompt},
+        {"negative_prompt", defaults.negative_prompt},
+        {"clip_skip", defaults.clip_skip},
+        {"width", defaults.width > 0 ? defaults.width : 512},
+        {"height", defaults.height > 0 ? defaults.height : 512},
+        {"strength", defaults.strength},
+        {"seed", defaults.seed},
+        {"video_frames", defaults.video_frames},
+        {"fps", defaults.fps},
+        {"moe_boundary", defaults.moe_boundary},
+        {"vace_strength", defaults.vace_strength},
+        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)},
+        {"high_noise_sample_params", make_sample_params_json(defaults.high_noise_sample_params, defaults.high_noise_skip_layers)},
+        {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)},
+        {"cache_mode", defaults.cache_mode},
+        {"cache_option", defaults.cache_option},
+        {"scm_mask", defaults.scm_mask},
+        {"scm_policy_dynamic", defaults.scm_policy_dynamic},
+        {"output_format", output_format},
+        {"output_compression", 100},
     };
-    result["samplers"]       = samplers;
-    result["schedulers"]     = schedulers;
-    result["output_formats"] = output_formats;
-    result["features"]       = {
-              {"init_image", true},
-              {"mask_image", true},
-              {"control_image", true},
-              {"ref_images", true},
-              {"lora", true},
-              {"vae_tiling", true},
-              {"cache", true},
+}
+
+static json make_img_gen_features_json() {
+    return {
+        {"init_image", true},
+        {"mask_image", true},
+        {"control_image", true},
+        {"ref_images", true},
+        {"lora", true},
+        {"vae_tiling", true},
+        {"cache", true},
+        {"cancel_queued", true},
+        {"cancel_generating", false},
+    };
+}
+
+static json make_vid_gen_features_json() {
+    return {
+        {"init_image", true},
+        {"end_image", true},
+        {"control_frames", true},
+        {"high_noise_sample_params", true},
+        {"lora", true},
+        {"vae_tiling", true},
+        {"cache", true},
+        {"cancel_queued", true},
+        {"cancel_generating", false},
+    };
+}
+
+static json make_capabilities_json(ServerRuntime& runtime) {
+    refresh_lora_cache(runtime);
+
+    AsyncJobManager& manager  = *runtime.async_job_manager;
+    const auto& defaults      = *runtime.default_gen_params;
+    const fs::path model_path = resolve_display_model_path(runtime);
+    const bool supports_img   = runtime_supports_generation_mode(runtime, IMG_GEN);
+    const bool supports_vid   = runtime_supports_generation_mode(runtime, VID_GEN);
+    json samplers             = json::array();
+    json schedulers           = json::array();
+    json image_output_formats = supported_img_output_formats();
+    json video_output_formats = supported_vid_output_formats();
+    json available_loras      = json::array();
+    json supported_modes      = json::array();
+
+    for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) {
+        samplers.push_back(sd_sample_method_name((sample_method_t)i));
+    }
+
+    for (int i = 0; i < SCHEDULER_COUNT; ++i) {
+        schedulers.push_back(sd_scheduler_name((scheduler_t)i));
+    }
+
+    {
+        std::lock_guard<std::mutex> lock(*runtime.lora_mutex);
+        for (const auto& entry : *runtime.lora_cache) {
+            available_loras.push_back({
+                {"name", entry.name},
+                {"path", entry.path},
+            });
+        }
+    }
+
+    if (supports_img) {
+        supported_modes.push_back("img_gen");
+    }
+    if (supports_vid) {
+        supported_modes.push_back("vid_gen");
+    }
+
+    std::string default_img_output_format = "png";
+    std::string default_vid_output_format = "avi";
+    if (!image_output_formats.empty()) {
+        default_img_output_format = image_output_formats[0].get<std::string>();
+    }
+    if (!video_output_formats.empty()) {
+        default_vid_output_format = video_output_formats[0].get<std::string>();
+    }
+
+    json defaults_by_mode       = json::object();
+    json output_formats_by_mode = json::object();
+    json features_by_mode       = json::object();
+    if (supports_img) {
+        defaults_by_mode["img_gen"]       = make_img_gen_defaults_json(defaults, default_img_output_format);
+        output_formats_by_mode["img_gen"] = image_output_formats;
+        features_by_mode["img_gen"]       = make_img_gen_features_json();
+    }
+    if (supports_vid) {
+        defaults_by_mode["vid_gen"]       = make_vid_gen_defaults_json(defaults, default_vid_output_format);
+        output_formats_by_mode["vid_gen"] = video_output_formats;
+        features_by_mode["vid_gen"]       = make_vid_gen_features_json();
+    }
+
+    json top_level_defaults       = json::object();
+    json top_level_output_formats = json::array();
+    json top_level_features       = {
               {"cancel_queued", true},
               {"cancel_generating", false},
     };
-    result["loras"] = available_loras;
+    std::string current_mode = "";
+    if (supports_img) {
+        current_mode             = "img_gen";
+        top_level_defaults       = defaults_by_mode["img_gen"];
+        top_level_output_formats = output_formats_by_mode["img_gen"];
+        top_level_features       = features_by_mode["img_gen"];
+    } else if (supports_vid) {
+        current_mode             = "vid_gen";
+        top_level_defaults       = defaults_by_mode["vid_gen"];
+        top_level_output_formats = output_formats_by_mode["vid_gen"];
+        top_level_features       = features_by_mode["vid_gen"];
+    }
+
+    json result;
+    result["model"] = {
+        {"name", model_path.filename().u8string()},
+        {"stem", model_path.stem().u8string()},
+        {"path", model_path.u8string()},
+    };
+    result["current_mode"]     = current_mode;
+    result["supported_modes"]  = supported_modes;
+    result["defaults"]         = top_level_defaults;
+    result["defaults_by_mode"] = defaults_by_mode;
+    result["limits"]           = {
+                  {"min_width", 64},
+                  {"max_width", 4096},
+                  {"min_height", 64},
+                  {"max_height", 4096},
+                  {"max_batch_count", 8},
+                  {"max_queue_size", manager.max_pending_jobs},
+    };
+    result["samplers"]               = samplers;
+    result["schedulers"]             = schedulers;
+    result["output_formats"]         = top_level_output_formats;
+    result["output_formats_by_mode"] = output_formats_by_mode;
+    result["features"]               = top_level_features;
+    result["features_by_mode"]       = features_by_mode;
+    result["loras"]                  = available_loras;
     return result;
 }
 
@@ -211,6 +314,33 @@ static bool parse_img_gen_request(const json& body,
     return true;
 }
 
+static bool parse_vid_gen_request(const json& body,
+                                  ServerRuntime& runtime,
+                                  VidGenJobRequest& request,
+                                  std::string& error_message) {
+    request.gen_params = *runtime.default_gen_params;
+
+    refresh_lora_cache(runtime);
+    if (!request.gen_params.from_json_str(body.dump(), [&](const std::string& path) {
+            return get_lora_full_path(runtime, path);
+        })) {
+        error_message = "invalid generation parameters";
+        return false;
+    }
+
+    std::string output_format = body.value("output_format", "webm");
+    int output_compression    = body.value("output_compression", 100);
+    if (!assign_output_options(request, output_format, output_compression, error_message)) {
+        return false;
+    }
+    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
+    if (!request.gen_params.resolve_and_validate(VID_GEN, "", true)) {
+        error_message = "invalid generation parameters";
+        return false;
+    }
+    return true;
+}
+
 void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
     ServerRuntime* runtime = &rt;
 
@@ -226,6 +356,11 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                 res.set_content(R"({"error":"empty body"})", "application/json");
                 return;
             }
+            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
+                res.status = 400;
+                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
+                return;
+            }
 
             json body = json::parse(req.body);
             ImgGenJobRequest request;
@@ -276,9 +411,66 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         }
     });
 
-    svr.Post("/sdcpp/v1/vid_gen", [](const httplib::Request&, httplib::Response& res) {
-        res.status = 501;
-        res.set_content(R"({"error":"vid_gen is reserved and not implemented yet"})", "application/json");
+    svr.Post("/sdcpp/v1/vid_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
+        try {
+            if (req.body.empty()) {
+                res.status = 400;
+                res.set_content(R"({"error":"empty body"})", "application/json");
+                return;
+            }
+            if (!runtime_supports_generation_mode(*runtime, VID_GEN)) {
+                res.status = 400;
+                res.set_content(json({{"error", unsupported_generation_mode_error(VID_GEN)}}).dump(), "application/json");
+                return;
+            }
+
+            json body = json::parse(req.body);
+            VidGenJobRequest request;
+            std::string error_message;
+            if (!parse_vid_gen_request(body, *runtime, request, error_message)) {
+                res.status = 400;
+                res.set_content(json({{"error", error_message}}).dump(), "application/json");
+                return;
+            }
+
+            AsyncJobManager& manager                = *runtime->async_job_manager;
+            std::shared_ptr<AsyncGenerationJob> job = std::make_shared<AsyncGenerationJob>();
+            job->kind                               = AsyncJobKind::VidGen;
+            job->status                             = AsyncJobStatus::Queued;
+            job->created_at                         = unix_timestamp_now();
+            job->vid_gen                            = std::move(request);
+
+            {
+                std::lock_guard<std::mutex> lock(manager.mutex);
+                purge_expired_jobs(manager);
+                if (count_pending_jobs(manager) >= manager.max_pending_jobs) {
+                    res.status = 429;
+                    res.set_content(R"({"error":"job queue is full"})", "application/json");
+                    return;
+                }
+                job->id               = make_async_job_id(manager);
+                manager.jobs[job->id] = job;
+                manager.queue.push_back(job->id);
+            }
+
+            manager.cv.notify_one();
+
+            json out;
+            out["id"]       = job->id;
+            out["kind"]     = async_job_kind_name(job->kind);
+            out["status"]   = async_job_status_name(job->status);
+            out["created"]  = job->created_at;
+            out["poll_url"] = "/sdcpp/v1/jobs/" + job->id;
+
+            res.status = 202;
+            res.set_content(out.dump(), "application/json");
+        } catch (const json::parse_error& e) {
+            res.status = 400;
+            res.set_content(json({{"error", "invalid json"}, {"message", e.what()}}).dump(), "application/json");
+        } catch (const std::exception& e) {
+            res.status = 500;
+            res.set_content(json({{"error", "server_error"}, {"message", e.what()}}).dump(), "application/json");
+        }
     });
 
     svr.Get(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+))", [runtime](const httplib::Request& req, httplib::Response& res) {

examples/server/runtime.cpp

@@ -45,6 +45,44 @@ std::string normalize_output_format(std::string output_format) {
     return output_format;
 }
 
+std::vector<std::string> supported_img_output_formats(bool allow_webp) {
+    std::vector<std::string> formats = {"png", "jpeg"};
+#ifdef SD_USE_WEBP
+    if (allow_webp) {
+        formats.push_back("webp");
+    }
+#else
+    (void)allow_webp;
+#endif
+    return formats;
+}
+
+std::vector<std::string> supported_vid_output_formats() {
+    std::vector<std::string> formats;
+#ifdef SD_USE_WEBM
+    formats.push_back("webm");
+#endif
+#ifdef SD_USE_WEBP
+    formats.push_back("webp");
+#endif
+    formats.push_back("avi");
+    return formats;
+}
+
+static std::string valid_vid_output_formats_message() {
+    const std::vector<std::string> formats = supported_vid_output_formats();
+
+    std::string message = "invalid output_format, must be one of [";
+    for (size_t i = 0; i < formats.size(); ++i) {
+        if (i > 0) {
+            message += ", ";
+        }
+        message += formats[i];
+    }
+    message += "]";
+    return message;
+}
+
 bool assign_output_options(ImgGenJobRequest& request,
                            std::string output_format,
                            int output_compression,
@@ -53,19 +91,88 @@ bool assign_output_options(ImgGenJobRequest& request,
     request.output_format      = normalize_output_format(std::move(output_format));
     request.output_compression = std::clamp(output_compression, 0, 100);
 
-    const bool valid_format = request.output_format == "png" ||
-                              request.output_format == "jpeg" ||
-                              (allow_webp && request.output_format == "webp");
+    const std::vector<std::string> valid_formats = supported_img_output_formats(allow_webp);
+    const bool valid_format                      = std::find(valid_formats.begin(),
+                                                             valid_formats.end(),
+                                                             request.output_format) != valid_formats.end();
     if (!valid_format) {
-        error_message = allow_webp
-                            ? "invalid output_format, must be one of [png, jpeg, webp]"
-                            : "invalid output_format, must be one of [png, jpeg]";
+        error_message = "invalid output_format, must be one of [";
+        for (size_t i = 0; i < valid_formats.size(); ++i) {
+            if (i > 0) {
+                error_message += ", ";
+            }
+            error_message += valid_formats[i];
+        }
+        error_message += "]";
         return false;
     }
 
     return true;
 }
 
+bool assign_output_options(VidGenJobRequest& request,
+                           std::string output_format,
+                           int output_compression,
+                           std::string& error_message) {
+    request.output_format      = normalize_output_format(std::move(output_format));
+    request.output_compression = std::clamp(output_compression, 0, 100);
+
+    if (request.output_format == "avi") {
+        return true;
+    }
+
+    if (request.output_format == "webm") {
+#ifdef SD_USE_WEBM
+        return true;
+#else
+        error_message = valid_vid_output_formats_message();
+        return false;
+#endif
+    }
+
+    if (request.output_format == "webp") {
+#ifdef SD_USE_WEBP
+        return true;
+#else
+        error_message = valid_vid_output_formats_message();
+        return false;
+#endif
+    }
+
+    error_message = valid_vid_output_formats_message();
+    return false;
+}
+
+std::string video_mime_type(const std::string& output_format) {
+    if (output_format == "webm") {
+        return "video/webm";
+    }
+    if (output_format == "webp") {
+        return "image/webp";
+    }
+    return "video/x-msvideo";
+}
+
+bool runtime_supports_generation_mode(const ServerRuntime& runtime, SDMode mode) {
+    if (mode == VID_GEN) {
+        return sd_ctx_supports_video_generation(runtime.sd_ctx);
+    }
+    if (mode == IMG_GEN) {
+        return sd_ctx_supports_image_generation(runtime.sd_ctx);
+    }
+    return true;
+}
+
+std::string unsupported_generation_mode_error(SDMode mode) {
+    if (mode == VID_GEN) {
+        return "loaded model does not support vid_gen";
+    }
+    if (mode == IMG_GEN) {
+        return "loaded model does not support img_gen";
+    }
+    return "loaded model does not support requested mode";
+}
+
 ArgOptions SDSvrParams::get_options() {
     ArgOptions options;
 

examples/server/runtime.h

@@ -58,13 +58,32 @@ struct ImgGenJobRequest {
     }
 };
 
+struct VidGenJobRequest {
+    SDGenerationParams gen_params;
+    std::string output_format = "webm";
+    int output_compression    = 100;
+
+    sd_vid_gen_params_t to_sd_vid_gen_params_t() {
+        return gen_params.to_sd_vid_gen_params_t();
+    }
+};
+
 std::string base64_encode(const std::vector<uint8_t>& bytes);
 std::string normalize_output_format(std::string output_format);
+std::vector<std::string> supported_img_output_formats(bool allow_webp = true);
+std::vector<std::string> supported_vid_output_formats();
 bool assign_output_options(ImgGenJobRequest& request,
                            std::string output_format,
                            int output_compression,
                            bool allow_webp,
                            std::string& error_message);
+bool assign_output_options(VidGenJobRequest& request,
+                           std::string output_format,
+                           int output_compression,
+                           std::string& error_message);
+std::string video_mime_type(const std::string& output_format);
+bool runtime_supports_generation_mode(const ServerRuntime& runtime, SDMode mode);
+std::string unsupported_generation_mode_error(SDMode mode);
 void refresh_lora_cache(ServerRuntime& rt);
 std::string get_lora_full_path(ServerRuntime& rt, const std::string& path);
 int64_t unix_timestamp_now();

include/stable-diffusion.h

@@ -50,6 +50,7 @@ enum sample_method_t {
     TCD_SAMPLE_METHOD,
     RES_MULTISTEP_SAMPLE_METHOD,
     RES_2S_SAMPLE_METHOD,
+    ER_SDE_SAMPLE_METHOD,
     SAMPLE_METHOD_COUNT
 };
 
@@ -347,6 +348,8 @@ SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
 SD_API void sd_set_preview_callback(sd_preview_cb_t cb, enum preview_t mode, int interval, bool denoised, bool noisy, void* data);
 SD_API int32_t sd_get_num_physical_cores();
 SD_API const char* sd_get_system_info();
+SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx);
+SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx);
 
 SD_API const char* sd_type_name(enum sd_type_t type);
 SD_API enum sd_type_t str_to_sd_type(const char* str);

src/common_block.hpp

@@ -277,6 +277,7 @@ protected:
     int64_t context_dim;
     int64_t n_head;
     int64_t d_head;
+    bool xtra_dim = false;
 
 public:
     CrossAttention(int64_t query_dim,
@@ -288,7 +289,11 @@ public:
           query_dim(query_dim),
           context_dim(context_dim) {
         int64_t inner_dim = d_head * n_head;
-
+        if (context_dim == 320 && d_head == 320) {
+            // LOG_DEBUG("CrossAttention: temp set dim to 1024 for sdxs_09");
+            xtra_dim    = true;
+            context_dim = 1024;
+        }
         blocks["to_q"] = std::shared_ptr<GGMLBlock>(new Linear(query_dim, inner_dim, false));
         blocks["to_k"] = std::shared_ptr<GGMLBlock>(new Linear(context_dim, inner_dim, false));
         blocks["to_v"] = std::shared_ptr<GGMLBlock>(new Linear(context_dim, inner_dim, false));
@@ -313,10 +318,16 @@ public:
         int64_t n_context = context->ne[1];
         int64_t inner_dim = d_head * n_head;
 
-        auto q = to_q->forward(ctx, x);        // [N, n_token, inner_dim]
+        auto q = to_q->forward(ctx, x);  // [N, n_token, inner_dim]
+        if (xtra_dim) {
+            // LOG_DEBUG("CrossAttention: temp set dim to 1024 for sdxs_09");
+            context->ne[0] = 1024;  // patch dim
+        }
         auto k = to_k->forward(ctx, context);  // [N, n_context, inner_dim]
         auto v = to_v->forward(ctx, context);  // [N, n_context, inner_dim]
-
+        if (xtra_dim) {
+            context->ne[0] = 320;  // reset dim to orig
+        }
         x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, nullptr, false, ctx->flash_attn_enabled);  // [N, n_token, inner_dim]
 
         x = to_out_0->forward(ctx, x);  // [N, n_token, query_dim]

src/conditioner.hpp

@@ -1673,14 +1673,18 @@ struct LLMEmbedder : public Conditioner {
                                                                                   size_t max_length = 100000000) {
         std::vector<std::pair<std::string, float>> parsed_attention;
         if (attn_range.first >= 0 && attn_range.second > 0) {
-            parsed_attention.emplace_back(text.substr(0, attn_range.first), 1.f);
+            if (attn_range.first > 0) {
+                parsed_attention.emplace_back(text.substr(0, attn_range.first), 1.f);
+            }
             if (attn_range.second - attn_range.first > 0) {
                 auto new_parsed_attention = parse_prompt_attention(text.substr(attn_range.first, attn_range.second - attn_range.first));
                 parsed_attention.insert(parsed_attention.end(),
                                         new_parsed_attention.begin(),
                                         new_parsed_attention.end());
             }
-            parsed_attention.emplace_back(text.substr(attn_range.second), 1.f);
+            if (attn_range.second < text.size()) {
+                parsed_attention.emplace_back(text.substr(attn_range.second), 1.f);
+            }
         } else {
             parsed_attention.emplace_back(text, 1.f);
         }

src/denoiser.hpp

@@ -953,8 +953,9 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral(denoise_cb_t model,
             float t_next         = t_fn(sigma_down);
             float h              = t_next - t;
             float s              = t + 0.5f * h;
-            sd::Tensor<float> x2 = (sigma_fn(s) / sigma_fn(t)) * x - (exp(-h * 0.5f) - 1) * denoised;
-            auto denoised2_opt   = model(x2, sigmas[i + 1], i + 1);
+            float sigma_s        = sigma_fn(s);
+            sd::Tensor<float> x2 = (sigma_s / sigma_fn(t)) * x - (exp(-h * 0.5f) - 1) * denoised;
+            auto denoised2_opt   = model(x2, sigma_s, i + 1);
             if (denoised2_opt.empty()) {
                 return {};
             }
@@ -969,6 +970,102 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral(denoise_cb_t model,
     return x;
 }
 
+static sd::Tensor<float> sample_dpmpp_2s_ancestral_flow(denoise_cb_t model,
+                                                        sd::Tensor<float> x,
+                                                        const std::vector<float>& sigmas,
+                                                        std::shared_ptr<RNG> rng,
+                                                        float eta = 1.0f) {
+    int steps = static_cast<int>(sigmas.size()) - 1;
+    for (int i = 0; i < steps; i++) {
+        float sigma = sigmas[i];
+        float sigma_to = sigmas[i + 1];
+
+        bool opt_first_step = (1.0 - sigma < 1e-6);
+
+        auto denoised_opt = model(x, sigma, (opt_first_step ? 1 : -1) * (i + 1));
+        if (denoised_opt.empty()) {
+            return {};
+        }
+        sd::Tensor<float> denoised = std::move(denoised_opt);
+
+        if (sigma_to == 0.0f) {
+            // Euler method (final step, no noise)
+            // sigma_to == 0 --> sigma_down = 0, so:
+            //   x +                       d  * (sigma_down - sigma)
+            // = x + ((x - denoised) / sigma) * (sigma_down - sigma)
+            // = x + ((x - denoised) / sigma) * (         0 - sigma)
+            // = x + ((x - denoised)        ) * -1
+            // = x    -x + denoised
+            x = denoised;
+
+        } else {
+            auto [sigma_down, sigma_up, alpha_scale] = get_ancestral_step_flow(sigma, sigma_to, eta);
+            sd::Tensor<float> D_i;
+
+            if (opt_first_step) {
+                // the reformulated exp_s calc already accounts for this, but we can avoid
+                // a redundant model call for the typical sigma 1 at the first step:
+                // exp_s = sqrt((1-sigma)/sigma * (1-sigma_down)/sigma_down)
+                //       = sqrt((1-    1)/    1 * (1-sigma_down)/sigma_down)
+                //       = 0
+                // so sigma_s = 1 = sigma, and sigma_s_i_ratio = sigma_s / sigma = 1
+                // u = (x*sigma_s_i_ratio)+(denoised*(1.0f-sigma_s_i_ratio))
+                //   = (x*1)+(denoised*0) = x
+                // so D_i = model(u, sigma_s, i + 1)
+                //        = model(x, sigma,   i + 1)
+                //        = denoised
+                D_i = denoised;
+
+            } else {
+                float sigma_s;
+
+                // ref implementation would be:
+                // auto lambda_fn = [](float sigma) -> float {
+                //     return std::log((1.0f - sigma) / sigma); };
+                // auto sigma_fn = [](float lbda) -> float {
+                //     return 1.0f / (std::exp(lbda) + 1.0f); };
+                // t_i     = lambda_fn(sigma);
+                // t_down  = lambda_fn(sigma_down);
+                // float r = 0.5f;
+                // h       = t_down - t_i;
+                // s       = t_i + r * h;
+                // sigma_s = sigma_fn(s);
+
+                // assuming r is constant, we sidestep the singularity at sigma -> 1 by:
+                // s   = 0.5 * (lambda_fn(sigma)     + lambda_fn(sigma_down))
+                //     = 0.5 * (log((1-sigma)/sigma) + log((1-sigma_down)/sigma_down))
+                //     = 0.5 * log(((1-sigma)/sigma) *    ((1-sigma_down)/sigma_down))
+                //     = log(sqrt (((1-sigma)/sigma) *    ((1-sigma_down)/sigma_down)))
+                // so exp(s) = sqrt((1-sigma)/sigma  *     (1-sigma_down)/sigma_down)
+                // and sigma_s = sigma_fn(s) = 1.0f / (exp(s) + 1.0f)
+
+                float exp_s = std::sqrt(((1 - sigma) / sigma) * ((1 - sigma_down) / sigma_down));
+                sigma_s = 1.0f / (exp_s + 1.0f);
+
+                float sigma_s_i_ratio = sigma_s / sigma;
+                sd::Tensor<float> u = (x * sigma_s_i_ratio) + (denoised * (1.0f - sigma_s_i_ratio));
+
+                auto denoised2_opt = model(u, sigma_s, i + 1);
+                if (denoised2_opt.empty()) {
+                    return {};
+                }
+                D_i = std::move(denoised2_opt);
+            }
+
+            float sigma_down_i_ratio = sigma_down / sigma;
+            x = (x * sigma_down_i_ratio) + (D_i * (1.0f - sigma_down_i_ratio));
+
+            if (sigma_to > 0.0f && eta > 0.0f) {
+                x = alpha_scale * x + sd::Tensor<float>::randn_like(x, rng) * sigma_up;
+            }
+        }
+    }
+
+    return x;
+}
+
+
+
 static sd::Tensor<float> sample_dpmpp_2m(denoise_cb_t model,
                                          sd::Tensor<float> x,
                                          const std::vector<float>& sigmas) {
@@ -1040,7 +1137,8 @@ static sd::Tensor<float> sample_dpmpp_2m_v2(denoise_cb_t model,
 static sd::Tensor<float> sample_lcm(denoise_cb_t model,
                                     sd::Tensor<float> x,
                                     const std::vector<float>& sigmas,
-                                    std::shared_ptr<RNG> rng) {
+                                    std::shared_ptr<RNG> rng,
+                                    bool is_flow_denoiser) {
     int steps = static_cast<int>(sigmas.size()) - 1;
     for (int i = 0; i < steps; i++) {
         auto denoised_opt = model(x, sigmas[i], i + 1);
@@ -1049,6 +1147,9 @@ static sd::Tensor<float> sample_lcm(denoise_cb_t model,
         }
         x = std::move(denoised_opt);
         if (sigmas[i + 1] > 0) {
+            if (is_flow_denoiser) {
+                x *= (1 - sigmas[i + 1]);
+            }
             x += sd::Tensor<float>::randn_like(x, rng) * sigmas[i + 1];
         }
     }
@@ -1285,6 +1386,140 @@ static sd::Tensor<float> sample_res_2s(denoise_cb_t model,
     return x;
 }
 
+static sd::Tensor<float> sample_er_sde(denoise_cb_t model,
+                                       sd::Tensor<float> x,
+                                       std::vector<float> sigmas,
+                                       std::shared_ptr<RNG> rng,
+                                       bool is_flow_denoiser,
+                                       float eta) {
+    constexpr int max_stage                  = 3;
+    constexpr int num_integration_points     = 200;
+    constexpr float num_integration_points_f = 200.0f;
+    float s_noise                            = eta;
+
+    auto er_sde_flow_sigma = [](float sigma) -> float {
+        sigma = std::max(sigma, 1e-6f);
+        sigma = std::min(sigma, 1.0f - 1e-4f);
+        return sigma;
+    };
+
+    auto sigma_to_er_sde_lambda = [&](float sigma, bool is_flow_denoiser) -> float {
+        if (is_flow_denoiser) {
+            sigma = er_sde_flow_sigma(sigma);
+            return sigma / std::max(1.0f - sigma, 1e-6f);
+        }
+        return std::max(sigma, 1e-6f);
+    };
+
+    auto sigma_to_er_sde_alpha = [&](float sigma, bool is_flow_denoiser) -> float {
+        if (is_flow_denoiser) {
+            sigma = er_sde_flow_sigma(sigma);
+            return 1.0f - sigma;
+        }
+        return 1.0f;
+    };
+
+    auto er_sde_noise_scaler = [](float x) -> float {
+        x = std::max(x, 0.0f);
+        return x * (std::exp(std::pow(x, 0.3f)) + 10.0f);
+    };
+
+    if (is_flow_denoiser) {
+        for (size_t i = 0; i + 1 < sigmas.size(); ++i) {
+            if (sigmas[i] > 1.0f) {
+                sigmas[i] = er_sde_flow_sigma(sigmas[i]);
+            }
+        }
+    }
+
+    std::vector<float> er_lambdas(sigmas.size(), 0.0f);
+    for (size_t i = 0; i < sigmas.size(); ++i) {
+        er_lambdas[i] = sigma_to_er_sde_lambda(sigmas[i], is_flow_denoiser);
+    }
+
+    sd::Tensor<float> old_denoised   = x;
+    sd::Tensor<float> old_denoised_d = x;
+    bool have_old_denoised           = false;
+    bool have_old_denoised_d         = false;
+
+    int steps = static_cast<int>(sigmas.size()) - 1;
+    for (int i = 0; i < steps; i++) {
+        sd::Tensor<float> denoised = model(x, sigmas[i], i + 1);
+        if (denoised.empty()) {
+            return {};
+        }
+
+        int stage_used = std::min(max_stage, i + 1);
+
+        if (sigmas[i + 1] == 0.0f) {
+            x = denoised;
+        } else {
+            float er_lambda_s = er_lambdas[i];
+            float er_lambda_t = er_lambdas[i + 1];
+            float alpha_s     = sigma_to_er_sde_alpha(sigmas[i], is_flow_denoiser);
+            float alpha_t     = sigma_to_er_sde_alpha(sigmas[i + 1], is_flow_denoiser);
+            float scaled_s    = er_sde_noise_scaler(er_lambda_s);
+            float scaled_t    = er_sde_noise_scaler(er_lambda_t);
+            float r_alpha     = alpha_s > 0.0f ? alpha_t / alpha_s : 0.0f;
+            float r           = scaled_s > 0.0f ? scaled_t / scaled_s : 0.0f;
+
+            x = r_alpha * r * x + alpha_t * (1.0f - r) * denoised;
+
+            if (stage_used >= 2 && have_old_denoised) {
+                float dt               = er_lambda_t - er_lambda_s;
+                float lambda_step_size = -dt / num_integration_points_f;
+                float s                = 0.0f;
+                float s_u              = 0.0f;
+
+                for (int p = 0; p < num_integration_points; ++p) {
+                    float lambda_pos = er_lambda_t + p * lambda_step_size;
+                    float scaled_pos = er_sde_noise_scaler(lambda_pos);
+                    if (scaled_pos <= 0.0f) {
+                        continue;
+                    }
+
+                    s += 1.0f / scaled_pos;
+                    if (stage_used >= 3 && have_old_denoised_d) {
+                        s_u += (lambda_pos - er_lambda_s) / scaled_pos;
+                    }
+                }
+
+                s *= lambda_step_size;
+
+                float denom_d = er_lambda_s - er_lambdas[i - 1];
+                if (std::fabs(denom_d) > 1e-12f) {
+                    float coeff_d                = alpha_t * (dt + s * scaled_t);
+                    sd::Tensor<float> denoised_d = (denoised - old_denoised) / denom_d;
+                    x += coeff_d * denoised_d;
+
+                    if (stage_used >= 3 && have_old_denoised_d) {
+                        float denom_u = (er_lambda_s - er_lambdas[i - 2]) * 0.5f;
+                        if (std::fabs(denom_u) > 1e-12f) {
+                            s_u *= lambda_step_size;
+                            float coeff_u                = alpha_t * (0.5f * dt * dt + s_u * scaled_t);
+                            sd::Tensor<float> denoised_u = (denoised_d - old_denoised_d) / denom_u;
+                            x += coeff_u * denoised_u;
+                        }
+                    }
+
+                    old_denoised_d      = denoised_d;
+                    have_old_denoised_d = true;
+                }
+            }
+
+            float noise_scale_sq = er_lambda_t * er_lambda_t - er_lambda_s * er_lambda_s * r * r;
+            if (s_noise > 0.0f && noise_scale_sq > 0.0f) {
+                float noise_scale = alpha_t * std::sqrt(std::max(noise_scale_sq, 0.0f));
+                x += sd::Tensor<float>::randn_like(x, rng) * noise_scale;
+            }
+        }
+
+        old_denoised      = denoised;
+        have_old_denoised = true;
+    }
+    return x;
+}
+
 static sd::Tensor<float> sample_ddim_trailing(denoise_cb_t model,
                                               sd::Tensor<float> x,
                                               const std::vector<float>& sigmas,
@@ -1431,13 +1666,16 @@ static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
         case DPM2_SAMPLE_METHOD:
             return sample_dpm2(model, std::move(x), sigmas);
         case DPMPP2S_A_SAMPLE_METHOD:
-            return sample_dpmpp_2s_ancestral(model, std::move(x), sigmas, rng, eta);
+            if (is_flow_denoiser)
+                return sample_dpmpp_2s_ancestral_flow(model, std::move(x), sigmas, rng, eta);
+            else
+                return sample_dpmpp_2s_ancestral(model, std::move(x), sigmas, rng, eta);
         case DPMPP2M_SAMPLE_METHOD:
             return sample_dpmpp_2m(model, std::move(x), sigmas);
         case DPMPP2Mv2_SAMPLE_METHOD:
             return sample_dpmpp_2m_v2(model, std::move(x), sigmas);
         case LCM_SAMPLE_METHOD:
-            return sample_lcm(model, std::move(x), sigmas, rng);
+            return sample_lcm(model, std::move(x), sigmas, rng, is_flow_denoiser);
         case IPNDM_SAMPLE_METHOD:
             return sample_ipndm(model, std::move(x), sigmas);
         case IPNDM_V_SAMPLE_METHOD:
@@ -1446,6 +1684,8 @@ static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
             return sample_res_multistep(model, std::move(x), sigmas, rng, eta);
         case RES_2S_SAMPLE_METHOD:
             return sample_res_2s(model, std::move(x), sigmas, rng, eta);
+        case ER_SDE_SAMPLE_METHOD:
+            return sample_er_sde(model, std::move(x), sigmas, rng, is_flow_denoiser, eta);
         case DDIM_TRAILING_SAMPLE_METHOD:
             return sample_ddim_trailing(model, std::move(x), sigmas, rng, eta);
         case TCD_SAMPLE_METHOD:

src/model.cpp

@@ -1019,67 +1019,66 @@ SDVersion ModelLoader::get_sd_version() {
     bool has_middle_block_1          = false;
     bool has_output_block_311        = false;
     bool has_output_block_71         = false;
+    bool has_attn_1024               = false;
 
     for (auto& [name, tensor_storage] : tensor_storage_map) {
-        if (!(is_xl)) {
-            if (tensor_storage.name.find("model.diffusion_model.double_blocks.") != std::string::npos) {
-                is_flux = true;
+        if (tensor_storage.name.find("model.diffusion_model.double_blocks.") != std::string::npos) {
+            is_flux = true;
+        }
+        if (tensor_storage.name.find("model.diffusion_model.nerf_final_layer_conv.") != std::string::npos) {
+            return VERSION_CHROMA_RADIANCE;
+        }
+        if (tensor_storage.name.find("model.diffusion_model.joint_blocks.") != std::string::npos) {
+            return VERSION_SD3;
+        }
+        if (tensor_storage.name.find("model.diffusion_model.transformer_blocks.0.img_mod.1.weight") != std::string::npos) {
+            return VERSION_QWEN_IMAGE;
+        }
+        if (tensor_storage.name.find("llm_adapter.blocks.0.cross_attn.q_proj.weight") != std::string::npos) {
+            return VERSION_ANIMA;
+        }
+        if (tensor_storage.name.find("model.diffusion_model.double_stream_modulation_img.lin.weight") != std::string::npos) {
+            is_flux2 = true;
+        }
+        if (tensor_storage.name.find("single_blocks.47.linear1.weight") != std::string::npos) {
+            has_single_block_47 = true;
+        }
+        if (tensor_storage.name.find("model.diffusion_model.double_blocks.0.img_mlp.gate_proj.weight") != std::string::npos) {
+            return VERSION_OVIS_IMAGE;
+        }
+        if (tensor_storage.name.find("model.diffusion_model.cap_embedder.0.weight") != std::string::npos) {
+            return VERSION_Z_IMAGE;
+        }
+        if (tensor_storage.name.find("model.diffusion_model.layers.0.adaLN_sa_ln.weight") != std::string::npos) {
+            return VERSION_ERNIE_IMAGE;
+        }
+        if (tensor_storage.name.find("model.diffusion_model.blocks.0.cross_attn.norm_k.weight") != std::string::npos) {
+            is_wan = true;
+        }
+        if (tensor_storage.name.find("model.diffusion_model.patch_embedding.weight") != std::string::npos) {
+            patch_embedding_channels = tensor_storage.ne[3];
+        }
+        if (tensor_storage.name.find("model.diffusion_model.img_emb") != std::string::npos) {
+            has_img_emb = true;
+        }
+        if (tensor_storage.name.find("model.diffusion_model.input_blocks.") != std::string::npos ||
+            tensor_storage.name.find("unet.down_blocks.") != std::string::npos) {
+            is_unet = true;
+            if (has_multiple_encoders) {
+                is_xl = true;
             }
-            if (tensor_storage.name.find("model.diffusion_model.nerf_final_layer_conv.") != std::string::npos) {
-                return VERSION_CHROMA_RADIANCE;
-            }
-            if (tensor_storage.name.find("model.diffusion_model.joint_blocks.") != std::string::npos) {
-                return VERSION_SD3;
-            }
-            if (tensor_storage.name.find("model.diffusion_model.transformer_blocks.0.img_mod.1.weight") != std::string::npos) {
-                return VERSION_QWEN_IMAGE;
-            }
-            if (tensor_storage.name.find("llm_adapter.blocks.0.cross_attn.q_proj.weight") != std::string::npos) {
-                return VERSION_ANIMA;
-            }
-            if (tensor_storage.name.find("model.diffusion_model.double_stream_modulation_img.lin.weight") != std::string::npos) {
-                is_flux2 = true;
-            }
-            if (tensor_storage.name.find("single_blocks.47.linear1.weight") != std::string::npos) {
-                has_single_block_47 = true;
-            }
-            if (tensor_storage.name.find("model.diffusion_model.double_blocks.0.img_mlp.gate_proj.weight") != std::string::npos) {
-                return VERSION_OVIS_IMAGE;
-            }
-            if (tensor_storage.name.find("model.diffusion_model.cap_embedder.0.weight") != std::string::npos) {
-                return VERSION_Z_IMAGE;
-            }
-            if (tensor_storage.name.find("model.diffusion_model.layers.0.adaLN_sa_ln.weight") != std::string::npos) {
-                return VERSION_ERNIE_IMAGE;
-            }
-            if (tensor_storage.name.find("model.diffusion_model.blocks.0.cross_attn.norm_k.weight") != std::string::npos) {
-                is_wan = true;
-            }
-            if (tensor_storage.name.find("model.diffusion_model.patch_embedding.weight") != std::string::npos) {
-                patch_embedding_channels = tensor_storage.ne[3];
-            }
-            if (tensor_storage.name.find("model.diffusion_model.img_emb") != std::string::npos) {
-                has_img_emb = true;
-            }
-            if (tensor_storage.name.find("model.diffusion_model.input_blocks.") != std::string::npos ||
-                tensor_storage.name.find("unet.down_blocks.") != std::string::npos) {
-                is_unet = true;
-                if (has_multiple_encoders) {
-                    is_xl = true;
-                }
-            }
-            if (tensor_storage.name.find("conditioner.embedders.1") != std::string::npos ||
-                tensor_storage.name.find("cond_stage_model.1") != std::string::npos ||
-                tensor_storage.name.find("te.1") != std::string::npos) {
-                has_multiple_encoders = true;
-                if (is_unet) {
-                    is_xl = true;
-                }
-            }
-            if (tensor_storage.name.find("model.diffusion_model.input_blocks.8.0.time_mixer.mix_factor") != std::string::npos) {
-                return VERSION_SVD;
+        }
+        if (tensor_storage.name.find("conditioner.embedders.1") != std::string::npos ||
+            tensor_storage.name.find("cond_stage_model.1") != std::string::npos ||
+            tensor_storage.name.find("te.1") != std::string::npos) {
+            has_multiple_encoders = true;
+            if (is_unet) {
+                is_xl = true;
             }
         }
+        if (tensor_storage.name.find("model.diffusion_model.input_blocks.8.0.time_mixer.mix_factor") != std::string::npos) {
+            return VERSION_SVD;
+        }
         if (tensor_storage.name.find("model.diffusion_model.middle_block.1.") != std::string::npos ||
             tensor_storage.name.find("unet.mid_block.resnets.1.") != std::string::npos) {
             has_middle_block_1 = true;
@@ -1091,6 +1090,10 @@ SDVersion ModelLoader::get_sd_version() {
         if (tensor_storage.name.find("model.diffusion_model.output_blocks.7.1") != std::string::npos ||
             tensor_storage.name.find("unet.up_blocks.2.attentions.1") != std::string::npos) {
             has_output_block_71 = true;
+            if (tensor_storage.name.find("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_k.weight") != std::string::npos) {
+                if (tensor_storage.ne[0] == 1024)
+                    has_attn_1024 = true;
+            }
         }
         if (tensor_storage.name == "cond_stage_model.transformer.text_model.embeddings.token_embedding.weight" ||
             tensor_storage.name == "cond_stage_model.model.token_embedding.weight" ||
@@ -1164,7 +1167,7 @@ SDVersion ModelLoader::get_sd_version() {
         }
         if (!has_middle_block_1) {
             if (!has_output_block_71) {
-                return VERSION_SDXS;
+                return VERSION_SDXS_512_DS;
             }
             return VERSION_SD1_TINY_UNET;
         }
@@ -1174,7 +1177,7 @@ SDVersion ModelLoader::get_sd_version() {
             return VERSION_SD2_INPAINT;
         }
         if (!has_middle_block_1) {
-            return VERSION_SD2_TINY_UNET;
+            return has_attn_1024 ? VERSION_SDXS_09 : VERSION_SD2_TINY_UNET;
         }
         return VERSION_SD2;
     }

src/model.h

@@ -28,7 +28,8 @@ enum SDVersion {
     VERSION_SD2,
     VERSION_SD2_INPAINT,
     VERSION_SD2_TINY_UNET,
-    VERSION_SDXS,
+    VERSION_SDXS_512_DS,
+    VERSION_SDXS_09,
     VERSION_SDXL,
     VERSION_SDXL_INPAINT,
     VERSION_SDXL_PIX2PIX,
@@ -55,14 +56,14 @@ enum SDVersion {
 };
 
 static inline bool sd_version_is_sd1(SDVersion version) {
-    if (version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX || version == VERSION_SD1_TINY_UNET || version == VERSION_SDXS) {
+    if (version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX || version == VERSION_SD1_TINY_UNET || version == VERSION_SDXS_512_DS) {
         return true;
     }
     return false;
 }
 
 static inline bool sd_version_is_sd2(SDVersion version) {
-    if (version == VERSION_SD2 || version == VERSION_SD2_INPAINT || version == VERSION_SD2_TINY_UNET) {
+    if (version == VERSION_SD2 || version == VERSION_SD2_INPAINT || version == VERSION_SD2_TINY_UNET || version == VERSION_SDXS_09) {
         return true;
     }
     return false;

src/name_conversion.cpp

@@ -1120,7 +1120,7 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
         for (const auto& prefix : first_stage_model_prefix_vec) {
             if (starts_with(name, prefix)) {
                 name = convert_first_stage_model_name(name.substr(prefix.size()), prefix);
-                if (version == VERSION_SDXS) {
+                if (version == VERSION_SDXS_512_DS || version == VERSION_SDXS_09) {
                     name = "tae." + name;
                 } else {
                     name = prefix + name;

src/stable-diffusion.cpp

@@ -30,7 +30,8 @@ const char* model_version_to_str[] = {
     "SD 2.x",
     "SD 2.x Inpaint",
     "SD 2.x Tiny UNet",
-    "SDXS",
+    "SDXS (512-DS)",
+    "SDXS (09)",
     "SDXL",
     "SDXL Inpaint",
     "SDXL Instruct-Pix2Pix",
@@ -70,6 +71,7 @@ const char* sampling_methods_str[] = {
     "TCD",
     "Res Multistep",
     "Res 2s",
+    "ER-SDE",
 };
 
 /*================================================== Helper Functions ================================================*/
@@ -414,7 +416,7 @@ public:
         }
 
         bool tae_preview_only = sd_ctx_params->tae_preview_only;
-        if (version == VERSION_SDXS) {
+        if (version == VERSION_SDXS_512_DS || version == VERSION_SDXS_09) {
             tae_preview_only = false;
             use_tae          = true;
         }
@@ -941,6 +943,8 @@ public:
                     pred_type = FLOW_PRED;
                     if (sd_version_is_wan(version)) {
                         default_flow_shift = 5.f;
+                    } else if (sd_version_is_ernie_image(version)) {
+                        default_flow_shift = 4.f;
                     } else {
                         default_flow_shift = 3.f;
                     }
@@ -1990,6 +1994,7 @@ const char* sample_method_to_str[] = {
     "tcd",
     "res_multistep",
     "res_2s",
+    "er_sde",
 };
 
 const char* sd_sample_method_name(enum sample_method_t sample_method) {
@@ -2385,6 +2390,14 @@ struct sd_ctx_t {
     StableDiffusionGGML* sd = nullptr;
 };
 
+static bool sd_version_supports_video_generation(SDVersion version) {
+    return version == VERSION_SVD || sd_version_is_wan(version);
+}
+
+static bool sd_version_supports_image_generation(SDVersion version) {
+    return !sd_version_supports_video_generation(version);
+}
+
 sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params) {
     sd_ctx_t* sd_ctx = (sd_ctx_t*)malloc(sizeof(sd_ctx_t));
     if (sd_ctx == nullptr) {
@@ -2414,6 +2427,20 @@ void free_sd_ctx(sd_ctx_t* sd_ctx) {
     free(sd_ctx);
 }
 
+SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx) {
+    if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
+        return false;
+    }
+    return sd_version_supports_image_generation(sd_ctx->sd->version);
+}
+
+SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) {
+    if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
+        return false;
+    }
+    return sd_version_supports_video_generation(sd_ctx->sd->version);
+}
+
 enum sample_method_t sd_get_default_sample_method(const sd_ctx_t* sd_ctx) {
     if (sd_ctx != nullptr && sd_ctx->sd != nullptr) {
         if (sd_version_is_dit(sd_ctx->sd->version)) {
@@ -2472,6 +2499,7 @@ static float resolve_eta(sd_ctx_t* sd_ctx,
                 return 0.0f;
             case EULER_A_SAMPLE_METHOD:
             case DPMPP2S_A_SAMPLE_METHOD:
+            case ER_SDE_SAMPLE_METHOD:
                 return 1.0f;
             default:;
         }

src/unet.hpp

@@ -217,11 +217,11 @@ public:
         } else if (sd_version_is_unet_edit(version)) {
             in_channels = 8;
         }
-        if (version == VERSION_SD1_TINY_UNET || version == VERSION_SD2_TINY_UNET || version == VERSION_SDXS) {
+        if (version == VERSION_SD1_TINY_UNET || version == VERSION_SD2_TINY_UNET || version == VERSION_SDXS_512_DS || version == VERSION_SDXS_09) {
             num_res_blocks = 1;
             channel_mult   = {1, 2, 4};
             tiny_unet      = true;
-            if (version == VERSION_SDXS) {
+            if (version == VERSION_SDXS_512_DS) {
                 attention_resolutions = {4, 2};  // here just like SDXL
             }
         }
@@ -264,6 +264,10 @@ public:
             if (version == VERSION_SVD) {
                 return new SpatialVideoTransformer(in_channels, n_head, d_head, depth, context_dim, use_linear_projection);
             } else {
+                if (version == VERSION_SDXS_09 && n_head == 5) {
+                    n_head = 1;    // to carry a special case of sdxs_09 into CrossAttentionLayer,
+                    d_head = 320;  // works as long the product remains equal (5*64 == 1*320)
+                }
                 return new SpatialTransformer(in_channels, n_head, d_head, depth, context_dim, use_linear_projection);
             }
         };