feat(server): accept ref_videos, ref_audios, ref_video_audios in sdcpp vid_gen (#2194)

examples/common/common.cpp

@@ -1959,6 +1959,7 @@ static bool parse_image_json_field(const json& parent,
     return decode_base64_image(parent.at(key).get<std::string>(), channels, 0, 0, out_image);
 }
 
+/** Parses a JSON array of base64 image strings into owned images. */
 static bool parse_image_array_json_field(const json& parent,
                                          const char* key,
                                          int channels,
@@ -1988,6 +1989,84 @@ static bool parse_image_array_json_field(const json& parent,
     return true;
 }
 
+/** Parses a JSON array of base64 WAV strings into owned audio clips. */
+static bool parse_audio_array_json_field(const json& parent,
+                                         const char* key,
+                                         std::vector<SDAudioOwner>& out_audios) {
+    if (!parent.contains(key)) {
+        return true;
+    }
+    if (parent.at(key).is_null()) {
+        out_audios.clear();
+        return true;
+    }
+    if (!parent.at(key).is_array()) {
+        return false;
+    }
+
+    out_audios.clear();
+    for (const auto& item : parent.at(key)) {
+        if (!item.is_string()) {
+            return false;
+        }
+        std::string encoded = item.get<std::string>();
+        auto comma_pos      = encoded.find(',');
+        if (comma_pos != std::string::npos) {
+            encoded = encoded.substr(comma_pos + 1);
+        }
+        std::vector<uint8_t> audio_bytes = decode_base64_bytes(encoded);
+        if (audio_bytes.empty()) {
+            return false;
+        }
+        std::vector<float> samples;
+        uint32_t sample_rate = 0;
+        uint32_t channels    = 0;
+        if (!load_wav_from_memory(audio_bytes.data(), audio_bytes.size(), samples, sample_rate, channels)) {
+            return false;
+        }
+        SDAudioOwner audio;
+        audio.reset(std::move(samples), sample_rate, channels);
+        out_audios.push_back(std::move(audio));
+    }
+    return true;
+}
+
+/** Parses a JSON array of per-video base64 frame arrays into owned videos. */
+static bool parse_video_array_json_field(const json& parent,
+                                         const char* key,
+                                         std::vector<std::vector<SDImageOwner>>& out_videos) {
+    if (!parent.contains(key)) {
+        return true;
+    }
+    if (parent.at(key).is_null()) {
+        out_videos.clear();
+        return true;
+    }
+    if (!parent.at(key).is_array()) {
+        return false;
+    }
+
+    out_videos.clear();
+    for (const auto& video : parent.at(key)) {
+        if (!video.is_array() || video.empty()) {
+            return false;
+        }
+        std::vector<SDImageOwner> frames;
+        for (const auto& frame : video) {
+            if (!frame.is_string()) {
+                return false;
+            }
+            SDImageOwner image;
+            if (!decode_base64_image(frame.get<std::string>(), 0, 0, 0, image)) {
+                return false;
+            }
+            frames.push_back(std::move(image));
+        }
+        out_videos.push_back(std::move(frames));
+    }
+    return true;
+}
+
 static bool parse_lora_json_field(const json& parent,
                                   const std::function<std::string(const std::string&)>& lora_path_resolver,
                                   std::map<std::string, float>& lora_map,
@@ -2102,6 +2181,7 @@ bool SDGenerationParams::parse_image_preprocess_json(const std::string& json_str
     return true;
 }
 
+/** Populates generation params from a JSON payload (server APIs); returns false on any malformed field. */
 bool SDGenerationParams::from_json_str(
     const std::string& json_str,
     const std::function<std::string(const std::string&)>& lora_path_resolver) {
@@ -2330,8 +2410,11 @@ bool SDGenerationParams::from_json_str(
         return false;
     }
     if (!parse_image_array_json_field(j, "ref_images", 0, ref_images) ||
-        !parse_image_array_json_field(j, "control_frames", 3, control_frames)) {
-        LOG_ERROR("invalid input image array");
+        !parse_image_array_json_field(j, "control_frames", 3, control_frames) ||
+        !parse_audio_array_json_field(j, "ref_audios", ref_audios) ||
+        !parse_audio_array_json_field(j, "ref_video_audios", ref_video_audios) ||
+        !parse_video_array_json_field(j, "ref_videos", ref_videos)) {
+        LOG_ERROR("invalid input image/audio/video array");
         return false;
     }
 
@@ -2583,6 +2666,7 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s
     return true;
 }
 
+/** Validates params for a generation mode; enforces mode-specific input rules. */
 bool SDGenerationParams::validate(SDMode mode) {
     if (!image_preprocess.empty() && mode != IMG_GEN && mode != VID_GEN) {
         LOG_ERROR("--image-preprocess requires img_gen or vid_gen mode");
@@ -2636,12 +2720,13 @@ bool SDGenerationParams::validate(SDMode mode) {
         return false;
     }
 
-    if (ref_video_audio_paths.size() > ref_video_paths.size()) {
+    if (ref_video_audio_paths.size() + ref_video_audios.size() > ref_video_paths.size() + ref_videos.size()) {
         LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video");
         return false;
     }
 
-    if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty())) {
+    if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty() ||
+                            !ref_videos.empty() || !ref_video_audios.empty() || !ref_audios.empty())) {
         LOG_ERROR("error: reference video and audio inputs require vid_gen mode");
         return false;
     }

examples/common/media_io.cpp

@@ -1437,70 +1437,94 @@ static uint32_t read_le32(const uint8_t* data) {
            (static_cast<uint32_t>(data[3]) << 24);
 }
 
+/** Loads a WAV file from disk and delegates decoding to load_wav_from_memory(). */
 bool load_wav_from_file(const std::string& path,
                         std::vector<float>& interleaved_samples,
                         uint32_t& sample_rate,
                         uint32_t& channels) {
+    std::ifstream file(fs::u8path(path), std::ios::binary);
+    if (!file) {
+        interleaved_samples.clear();
+        sample_rate = 0;
+        channels    = 0;
+        return false;
+    }
+    file.seekg(0, std::ios::end);
+    const std::streamoff file_size = file.tellg();
+    file.seekg(0, std::ios::beg);
+    if (file_size < 0) {
+        interleaved_samples.clear();
+        sample_rate = 0;
+        channels    = 0;
+        return false;
+    }
+    std::vector<uint8_t> buffer(static_cast<size_t>(file_size));
+    if (file_size > 0 && !file.read(reinterpret_cast<char*>(buffer.data()), file_size)) {
+        interleaved_samples.clear();
+        sample_rate = 0;
+        channels    = 0;
+        return false;
+    }
+    return load_wav_from_memory(buffer.data(), buffer.size(), interleaved_samples, sample_rate, channels);
+}
+
+/** Parses a RIFF/WAVE buffer in place; compares chunk sizes against the remaining
+ * bytes so payload-controlled sizes cannot wrap size_t on 32-bit builds. */
+bool load_wav_from_memory(const uint8_t* data,
+                          size_t size,
+                          std::vector<float>& interleaved_samples,
+                          uint32_t& sample_rate,
+                          uint32_t& channels) {
     interleaved_samples.clear();
     sample_rate = 0;
     channels    = 0;
 
-    std::ifstream file(fs::u8path(path), std::ios::binary);
-    uint8_t riff_header[12];
-    if (!file.read(reinterpret_cast<char*>(riff_header), sizeof(riff_header)) ||
-        std::memcmp(riff_header, "RIFF", 4) != 0 ||
-        std::memcmp(riff_header + 8, "WAVE", 4) != 0) {
+    if (data == nullptr ||
+        size < 12 ||
+        std::memcmp(data, "RIFF", 4) != 0 ||
+        std::memcmp(data + 8, "WAVE", 4) != 0) {
         return false;
     }
 
     uint16_t audio_format    = 0;
     uint16_t bits_per_sample = 0;
     uint16_t block_align     = 0;
-    std::streampos data_pos  = std::streampos(-1);
+    size_t data_pos          = SIZE_MAX;
     uint32_t data_size       = 0;
 
-    while (file.good()) {
-        uint8_t chunk_header[8];
-        if (!file.read(reinterpret_cast<char*>(chunk_header), sizeof(chunk_header))) {
-            break;
-        }
-        uint32_t chunk_size           = read_le32(chunk_header + 4);
-        std::streampos chunk_data_pos = file.tellg();
+    size_t pos = 12;
+    while (pos + 8 <= size) {
+        uint32_t chunk_size    = read_le32(data + pos + 4);
+        size_t chunk_data_pos  = pos + 8;
+        const size_t remaining = size - chunk_data_pos;  // safe: chunk_data_pos <= size
+        // Chunk sizes come from the payload, so compare against the remaining bytes
+        // instead of adding them to offsets (avoids size_t wrap on 32-bit builds).
+        const bool data_fits   = static_cast<size_t>(chunk_size) <= remaining;
 
-        if (std::memcmp(chunk_header, "fmt ", 4) == 0) {
-            if (chunk_size < 16) {
+        if (std::memcmp(data + pos, "fmt ", 4) == 0) {
+            if (chunk_size < 16 || !data_fits) {
                 return false;
             }
-            std::vector<uint8_t> fmt(chunk_size);
-            if (!file.read(reinterpret_cast<char*>(fmt.data()), chunk_size)) {
-                return false;
-            }
-            audio_format    = read_le16(fmt.data());
-            channels        = read_le16(fmt.data() + 2);
-            sample_rate     = read_le32(fmt.data() + 4);
-            block_align     = read_le16(fmt.data() + 12);
-            bits_per_sample = read_le16(fmt.data() + 14);
+            audio_format    = read_le16(data + chunk_data_pos);
+            channels        = read_le16(data + chunk_data_pos + 2);
+            sample_rate     = read_le32(data + chunk_data_pos + 4);
+            block_align     = read_le16(data + chunk_data_pos + 12);
+            bits_per_sample = read_le16(data + chunk_data_pos + 14);
             if (audio_format == 0xfffe && chunk_size >= 40) {
-                audio_format = read_le16(fmt.data() + 24);
+                audio_format = read_le16(data + chunk_data_pos + 24);
             }
-        } else if (std::memcmp(chunk_header, "data", 4) == 0) {
+        } else if (std::memcmp(data + pos, "data", 4) == 0) {
             data_pos  = chunk_data_pos;
             data_size = chunk_size;
-            file.seekg(chunk_size, std::ios::cur);
-        } else {
-            file.seekg(chunk_size, std::ios::cur);
-        }
-
-        if (!file.good()) {
-            break;
-        }
-        if ((chunk_size & 1) != 0) {
-            file.seekg(1, std::ios::cur);
         }
+        // Only advance past a chunk whose size and pad byte fully fit (keeps the sum
+        // overflow-free); a chunk truncated at the buffer end terminates the walk.
+        const bool chunk_fits = data_fits && ((chunk_size & 1) == 0 || static_cast<size_t>(chunk_size) < remaining);
+        pos                   = chunk_fits ? chunk_data_pos + static_cast<size_t>(chunk_size) + (chunk_size & 1) : size;
     }
 
     const uint32_t bytes_per_sample = (bits_per_sample + 7) / 8;
-    if (data_pos == std::streampos(-1) || data_size == 0 || channels == 0 || sample_rate == 0 ||
+    if (data_pos == SIZE_MAX || data_size == 0 || channels == 0 || sample_rate == 0 ||
         block_align == 0 || bytes_per_sample == 0 || block_align < channels * bytes_per_sample ||
         (audio_format != 1 && audio_format != 3)) {
         return false;
@@ -1510,16 +1534,14 @@ bool load_wav_from_file(const std::string& path,
     if (frame_count == 0 || frame_count > SIZE_MAX / channels) {
         return false;
     }
-    std::vector<uint8_t> pcm(data_size);
-    file.clear();
-    file.seekg(data_pos);
-    if (!file.read(reinterpret_cast<char*>(pcm.data()), data_size)) {
+    if (static_cast<size_t>(data_size) > size - data_pos) {
         return false;
     }
+    const uint8_t* pcm = data + data_pos;
 
     interleaved_samples.resize(static_cast<size_t>(frame_count * channels));
     for (uint64_t frame = 0; frame < frame_count; ++frame) {
-        const uint8_t* frame_data = pcm.data() + frame * block_align;
+        const uint8_t* frame_data = pcm + frame * block_align;
         for (uint32_t channel = 0; channel < channels; ++channel) {
             const uint8_t* sample_data = frame_data + channel * bytes_per_sample;
             float sample               = 0.0f;

examples/common/media_io.h

@@ -126,4 +126,11 @@ bool load_wav_from_file(const std::string& path,
                         uint32_t& sample_rate,
                         uint32_t& channels);
 
+/** Decodes a WAV buffer (PCM 8/16/24/32 or IEEE float 32/64) into interleaved samples. */
+bool load_wav_from_memory(const uint8_t* data,
+                          size_t size,
+                          std::vector<float>& interleaved_samples,
+                          uint32_t& sample_rate,
+                          uint32_t& channels);
+
 #endif  // __MEDIA_IO_H__

examples/server/api.md

@@ -183,6 +183,9 @@ Default-enabled features are `img2img`, `mask_image`, `lora`, `vae_tiling`,
 | `clip_skip` | CLIP Skip |
 | `img2img`, `mask_image` | Image initialization/strength and masks |
 | `ref_images` | Reference image inputs |
+| `ref_videos` | Reference videos as frame arrays |
+| `ref_audios` | Standalone reference audio; runtime requires an Audio VAE or Audio Encoder |
+| `ref_video_audios` | Soundtracks paired with reference videos; runtime requires an Audio VAE |
 | `controlnet` | Control image and ControlNet strength |
 | `control_image` | Control image without a separate ControlNet (FLUX Canny/Depth) |
 | `ip_adapter` | IP-Adapter image and strength |

examples/server/model_catalog.cpp

@@ -291,6 +291,7 @@ static void apply_recipe_context(const json& settings, SDContextParams& params)
     }
 }
 
+// Known recipe feature names with their defaults; recipes may flip any of these.
 static const std::map<std::string, bool> k_recipe_features = {
     {"img2img", true},
     {"mask_image", true},
@@ -312,6 +313,9 @@ static const std::map<std::string, bool> k_recipe_features = {
     {"init_image", false},
     {"end_image", false},
     {"control_frames", false},
+    {"ref_videos", false},
+    {"ref_audios", false},
+    {"ref_video_audios", false},
     {"vace", false},
     {"high_noise", false},
     {"cache_unet", false},
@@ -369,6 +373,9 @@ json model_generation_features(const ServerRuntime& runtime) {
     features["controlnet"]  = features.value("controlnet", false) && !runtime.ctx_params->control_net_path.empty();
     features["high_noise"]  = features.value("high_noise", false) && !runtime.ctx_params->high_noise_diffusion_model_path.empty();
     features["preview_tae"] = features.value("preview_tae", false) && !runtime.ctx_params->taesd_path.empty();
+    features["ref_audios"]  = features.value("ref_audios", false) &&
+                             (!runtime.ctx_params->audio_vae_path.empty() || !runtime.ctx_params->audio_encoder_path.empty());
+    features["ref_video_audios"] = features.value("ref_video_audios", false) && !runtime.ctx_params->audio_vae_path.empty();
     if (!runtime.ctx_params->pulid_weights_path.empty())
         features["slg"] = false;
     return features;

examples/server/recipes/minimax-h3-ref2va.json

@@ -45,7 +45,10 @@
   },
   "features": [
     "flow_shift",
-    "ref_images"
+    "ref_images",
+    "ref_videos",
+    "ref_audios",
+    "ref_video_audios"
   ],
   "disabled_features": [
     "img2img",

examples/server/recipes/wan22-s2v.json

@@ -44,7 +44,8 @@
   },
   "features": [
     "flow_shift",
-    "init_image"
+    "init_image",
+    "ref_audios"
   ],
   "disabled_features": [
     "img2img",

examples/server/routes_sdcpp.cpp

@@ -212,11 +212,15 @@ static json make_img_gen_features_json() {
     };
 }
 
+/** Builds the default feature set advertised for vid_gen requests. */
 static json make_vid_gen_features_json() {
     return {
         {"init_image", true},
         {"end_image", true},
         {"control_frames", true},
+        {"ref_videos", true},
+        {"ref_audios", true},
+        {"ref_video_audios", true},
         {"high_noise_sample_params", true},
         {"lora", true},
         {"vae_tiling", true},
@@ -235,6 +239,7 @@ static json make_preview_modes_json() {
     return modes;
 }
 
+/** Builds the /capabilities payload: model, defaults, limits and feature maps. */
 static json make_capabilities_json(ServerRuntime& runtime) {
     refresh_lora_cache(runtime);
     refresh_upscaler_cache(runtime);
@@ -351,8 +356,10 @@ static json make_capabilities_json(ServerRuntime& runtime) {
     const json generation_features = model_generation_features(runtime);
     auto recipe_features           = [&](json features, bool video) {
         features["init_image"] = generation_features.at(video ? "init_image" : "img2img");
-        for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "lora", "vae_tiling", "cache", "preview"}) {
-            if (features.contains(key) || std::string(key) == "ref_images")
+        for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "ref_videos", "ref_audios", "ref_video_audios", "lora", "vae_tiling", "cache", "preview"}) {
+            if (features.contains(key) || std::string(key) == "ref_images" ||
+                (video && (std::string(key) == "ref_videos" || std::string(key) == "ref_audios" ||
+                           std::string(key) == "ref_video_audios")))
                 features[key] = generation_features.at(key);
         }
         if (video)