diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 00301dc9..637bbb65 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -1959,6 +1959,7 @@ static bool parse_image_json_field(const json& parent, return decode_base64_image(parent.at(key).get(), channels, 0, 0, out_image); } +/** Parses a JSON array of base64 image strings into owned images. */ static bool parse_image_array_json_field(const json& parent, const char* key, int channels, @@ -1988,6 +1989,84 @@ static bool parse_image_array_json_field(const json& parent, return true; } +/** Parses a JSON array of base64 WAV strings into owned audio clips. */ +static bool parse_audio_array_json_field(const json& parent, + const char* key, + std::vector& out_audios) { + if (!parent.contains(key)) { + return true; + } + if (parent.at(key).is_null()) { + out_audios.clear(); + return true; + } + if (!parent.at(key).is_array()) { + return false; + } + + out_audios.clear(); + for (const auto& item : parent.at(key)) { + if (!item.is_string()) { + return false; + } + std::string encoded = item.get(); + auto comma_pos = encoded.find(','); + if (comma_pos != std::string::npos) { + encoded = encoded.substr(comma_pos + 1); + } + std::vector audio_bytes = decode_base64_bytes(encoded); + if (audio_bytes.empty()) { + return false; + } + std::vector samples; + uint32_t sample_rate = 0; + uint32_t channels = 0; + if (!load_wav_from_memory(audio_bytes.data(), audio_bytes.size(), samples, sample_rate, channels)) { + return false; + } + SDAudioOwner audio; + audio.reset(std::move(samples), sample_rate, channels); + out_audios.push_back(std::move(audio)); + } + return true; +} + +/** Parses a JSON array of per-video base64 frame arrays into owned videos. */ +static bool parse_video_array_json_field(const json& parent, + const char* key, + std::vector>& out_videos) { + if (!parent.contains(key)) { + return true; + } + if (parent.at(key).is_null()) { + out_videos.clear(); + return true; + } + if (!parent.at(key).is_array()) { + return false; + } + + out_videos.clear(); + for (const auto& video : parent.at(key)) { + if (!video.is_array() || video.empty()) { + return false; + } + std::vector frames; + for (const auto& frame : video) { + if (!frame.is_string()) { + return false; + } + SDImageOwner image; + if (!decode_base64_image(frame.get(), 0, 0, 0, image)) { + return false; + } + frames.push_back(std::move(image)); + } + out_videos.push_back(std::move(frames)); + } + return true; +} + static bool parse_lora_json_field(const json& parent, const std::function& lora_path_resolver, std::map& lora_map, @@ -2102,6 +2181,7 @@ bool SDGenerationParams::parse_image_preprocess_json(const std::string& json_str return true; } +/** Populates generation params from a JSON payload (server APIs); returns false on any malformed field. */ bool SDGenerationParams::from_json_str( const std::string& json_str, const std::function& lora_path_resolver) { @@ -2330,8 +2410,11 @@ bool SDGenerationParams::from_json_str( return false; } if (!parse_image_array_json_field(j, "ref_images", 0, ref_images) || - !parse_image_array_json_field(j, "control_frames", 3, control_frames)) { - LOG_ERROR("invalid input image array"); + !parse_image_array_json_field(j, "control_frames", 3, control_frames) || + !parse_audio_array_json_field(j, "ref_audios", ref_audios) || + !parse_audio_array_json_field(j, "ref_video_audios", ref_video_audios) || + !parse_video_array_json_field(j, "ref_videos", ref_videos)) { + LOG_ERROR("invalid input image/audio/video array"); return false; } @@ -2583,6 +2666,7 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s return true; } +/** Validates params for a generation mode; enforces mode-specific input rules. */ bool SDGenerationParams::validate(SDMode mode) { if (!image_preprocess.empty() && mode != IMG_GEN && mode != VID_GEN) { LOG_ERROR("--image-preprocess requires img_gen or vid_gen mode"); @@ -2636,12 +2720,13 @@ bool SDGenerationParams::validate(SDMode mode) { return false; } - if (ref_video_audio_paths.size() > ref_video_paths.size()) { + if (ref_video_audio_paths.size() + ref_video_audios.size() > ref_video_paths.size() + ref_videos.size()) { LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video"); return false; } - if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty())) { + if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty() || + !ref_videos.empty() || !ref_video_audios.empty() || !ref_audios.empty())) { LOG_ERROR("error: reference video and audio inputs require vid_gen mode"); return false; } diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp index f09c2166..6c03a2a1 100644 --- a/examples/common/media_io.cpp +++ b/examples/common/media_io.cpp @@ -1437,70 +1437,94 @@ static uint32_t read_le32(const uint8_t* data) { (static_cast(data[3]) << 24); } +/** Loads a WAV file from disk and delegates decoding to load_wav_from_memory(). */ bool load_wav_from_file(const std::string& path, std::vector& interleaved_samples, uint32_t& sample_rate, uint32_t& channels) { + std::ifstream file(fs::u8path(path), std::ios::binary); + if (!file) { + interleaved_samples.clear(); + sample_rate = 0; + channels = 0; + return false; + } + file.seekg(0, std::ios::end); + const std::streamoff file_size = file.tellg(); + file.seekg(0, std::ios::beg); + if (file_size < 0) { + interleaved_samples.clear(); + sample_rate = 0; + channels = 0; + return false; + } + std::vector buffer(static_cast(file_size)); + if (file_size > 0 && !file.read(reinterpret_cast(buffer.data()), file_size)) { + interleaved_samples.clear(); + sample_rate = 0; + channels = 0; + return false; + } + return load_wav_from_memory(buffer.data(), buffer.size(), interleaved_samples, sample_rate, channels); +} + +/** Parses a RIFF/WAVE buffer in place; compares chunk sizes against the remaining + * bytes so payload-controlled sizes cannot wrap size_t on 32-bit builds. */ +bool load_wav_from_memory(const uint8_t* data, + size_t size, + std::vector& interleaved_samples, + uint32_t& sample_rate, + uint32_t& channels) { interleaved_samples.clear(); sample_rate = 0; channels = 0; - std::ifstream file(fs::u8path(path), std::ios::binary); - uint8_t riff_header[12]; - if (!file.read(reinterpret_cast(riff_header), sizeof(riff_header)) || - std::memcmp(riff_header, "RIFF", 4) != 0 || - std::memcmp(riff_header + 8, "WAVE", 4) != 0) { + if (data == nullptr || + size < 12 || + std::memcmp(data, "RIFF", 4) != 0 || + std::memcmp(data + 8, "WAVE", 4) != 0) { return false; } uint16_t audio_format = 0; uint16_t bits_per_sample = 0; uint16_t block_align = 0; - std::streampos data_pos = std::streampos(-1); + size_t data_pos = SIZE_MAX; uint32_t data_size = 0; - while (file.good()) { - uint8_t chunk_header[8]; - if (!file.read(reinterpret_cast(chunk_header), sizeof(chunk_header))) { - break; - } - uint32_t chunk_size = read_le32(chunk_header + 4); - std::streampos chunk_data_pos = file.tellg(); + size_t pos = 12; + while (pos + 8 <= size) { + uint32_t chunk_size = read_le32(data + pos + 4); + size_t chunk_data_pos = pos + 8; + const size_t remaining = size - chunk_data_pos; // safe: chunk_data_pos <= size + // Chunk sizes come from the payload, so compare against the remaining bytes + // instead of adding them to offsets (avoids size_t wrap on 32-bit builds). + const bool data_fits = static_cast(chunk_size) <= remaining; - if (std::memcmp(chunk_header, "fmt ", 4) == 0) { - if (chunk_size < 16) { + if (std::memcmp(data + pos, "fmt ", 4) == 0) { + if (chunk_size < 16 || !data_fits) { return false; } - std::vector fmt(chunk_size); - if (!file.read(reinterpret_cast(fmt.data()), chunk_size)) { - return false; - } - audio_format = read_le16(fmt.data()); - channels = read_le16(fmt.data() + 2); - sample_rate = read_le32(fmt.data() + 4); - block_align = read_le16(fmt.data() + 12); - bits_per_sample = read_le16(fmt.data() + 14); + audio_format = read_le16(data + chunk_data_pos); + channels = read_le16(data + chunk_data_pos + 2); + sample_rate = read_le32(data + chunk_data_pos + 4); + block_align = read_le16(data + chunk_data_pos + 12); + bits_per_sample = read_le16(data + chunk_data_pos + 14); if (audio_format == 0xfffe && chunk_size >= 40) { - audio_format = read_le16(fmt.data() + 24); + audio_format = read_le16(data + chunk_data_pos + 24); } - } else if (std::memcmp(chunk_header, "data", 4) == 0) { + } else if (std::memcmp(data + pos, "data", 4) == 0) { data_pos = chunk_data_pos; data_size = chunk_size; - file.seekg(chunk_size, std::ios::cur); - } else { - file.seekg(chunk_size, std::ios::cur); - } - - if (!file.good()) { - break; - } - if ((chunk_size & 1) != 0) { - file.seekg(1, std::ios::cur); } + // Only advance past a chunk whose size and pad byte fully fit (keeps the sum + // overflow-free); a chunk truncated at the buffer end terminates the walk. + const bool chunk_fits = data_fits && ((chunk_size & 1) == 0 || static_cast(chunk_size) < remaining); + pos = chunk_fits ? chunk_data_pos + static_cast(chunk_size) + (chunk_size & 1) : size; } const uint32_t bytes_per_sample = (bits_per_sample + 7) / 8; - if (data_pos == std::streampos(-1) || data_size == 0 || channels == 0 || sample_rate == 0 || + if (data_pos == SIZE_MAX || data_size == 0 || channels == 0 || sample_rate == 0 || block_align == 0 || bytes_per_sample == 0 || block_align < channels * bytes_per_sample || (audio_format != 1 && audio_format != 3)) { return false; @@ -1510,16 +1534,14 @@ bool load_wav_from_file(const std::string& path, if (frame_count == 0 || frame_count > SIZE_MAX / channels) { return false; } - std::vector pcm(data_size); - file.clear(); - file.seekg(data_pos); - if (!file.read(reinterpret_cast(pcm.data()), data_size)) { + if (static_cast(data_size) > size - data_pos) { return false; } + const uint8_t* pcm = data + data_pos; interleaved_samples.resize(static_cast(frame_count * channels)); for (uint64_t frame = 0; frame < frame_count; ++frame) { - const uint8_t* frame_data = pcm.data() + frame * block_align; + const uint8_t* frame_data = pcm + frame * block_align; for (uint32_t channel = 0; channel < channels; ++channel) { const uint8_t* sample_data = frame_data + channel * bytes_per_sample; float sample = 0.0f; diff --git a/examples/common/media_io.h b/examples/common/media_io.h index 18042a8b..7feb0ada 100644 --- a/examples/common/media_io.h +++ b/examples/common/media_io.h @@ -126,4 +126,11 @@ bool load_wav_from_file(const std::string& path, uint32_t& sample_rate, uint32_t& channels); +/** Decodes a WAV buffer (PCM 8/16/24/32 or IEEE float 32/64) into interleaved samples. */ +bool load_wav_from_memory(const uint8_t* data, + size_t size, + std::vector& interleaved_samples, + uint32_t& sample_rate, + uint32_t& channels); + #endif // __MEDIA_IO_H__ diff --git a/examples/server/api.md b/examples/server/api.md index 1258b1af..b5e0250c 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -183,6 +183,9 @@ Default-enabled features are `img2img`, `mask_image`, `lora`, `vae_tiling`, | `clip_skip` | CLIP Skip | | `img2img`, `mask_image` | Image initialization/strength and masks | | `ref_images` | Reference image inputs | +| `ref_videos` | Reference videos as frame arrays | +| `ref_audios` | Standalone reference audio; runtime requires an Audio VAE or Audio Encoder | +| `ref_video_audios` | Soundtracks paired with reference videos; runtime requires an Audio VAE | | `controlnet` | Control image and ControlNet strength | | `control_image` | Control image without a separate ControlNet (FLUX Canny/Depth) | | `ip_adapter` | IP-Adapter image and strength | diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp index eef68561..1041dea6 100644 --- a/examples/server/model_catalog.cpp +++ b/examples/server/model_catalog.cpp @@ -291,6 +291,7 @@ static void apply_recipe_context(const json& settings, SDContextParams& params) } } +// Known recipe feature names with their defaults; recipes may flip any of these. static const std::map k_recipe_features = { {"img2img", true}, {"mask_image", true}, @@ -312,6 +313,9 @@ static const std::map k_recipe_features = { {"init_image", false}, {"end_image", false}, {"control_frames", false}, + {"ref_videos", false}, + {"ref_audios", false}, + {"ref_video_audios", false}, {"vace", false}, {"high_noise", false}, {"cache_unet", false}, @@ -369,6 +373,9 @@ json model_generation_features(const ServerRuntime& runtime) { features["controlnet"] = features.value("controlnet", false) && !runtime.ctx_params->control_net_path.empty(); features["high_noise"] = features.value("high_noise", false) && !runtime.ctx_params->high_noise_diffusion_model_path.empty(); features["preview_tae"] = features.value("preview_tae", false) && !runtime.ctx_params->taesd_path.empty(); + features["ref_audios"] = features.value("ref_audios", false) && + (!runtime.ctx_params->audio_vae_path.empty() || !runtime.ctx_params->audio_encoder_path.empty()); + features["ref_video_audios"] = features.value("ref_video_audios", false) && !runtime.ctx_params->audio_vae_path.empty(); if (!runtime.ctx_params->pulid_weights_path.empty()) features["slg"] = false; return features; diff --git a/examples/server/recipes/minimax-h3-ref2va.json b/examples/server/recipes/minimax-h3-ref2va.json index 085f5d5f..2fb04178 100644 --- a/examples/server/recipes/minimax-h3-ref2va.json +++ b/examples/server/recipes/minimax-h3-ref2va.json @@ -45,7 +45,10 @@ }, "features": [ "flow_shift", - "ref_images" + "ref_images", + "ref_videos", + "ref_audios", + "ref_video_audios" ], "disabled_features": [ "img2img", diff --git a/examples/server/recipes/wan22-s2v.json b/examples/server/recipes/wan22-s2v.json index ba1ee4f7..55440ad3 100644 --- a/examples/server/recipes/wan22-s2v.json +++ b/examples/server/recipes/wan22-s2v.json @@ -44,7 +44,8 @@ }, "features": [ "flow_shift", - "init_image" + "init_image", + "ref_audios" ], "disabled_features": [ "img2img", diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp index b972768b..0dda9a95 100644 --- a/examples/server/routes_sdcpp.cpp +++ b/examples/server/routes_sdcpp.cpp @@ -212,11 +212,15 @@ static json make_img_gen_features_json() { }; } +/** Builds the default feature set advertised for vid_gen requests. */ static json make_vid_gen_features_json() { return { {"init_image", true}, {"end_image", true}, {"control_frames", true}, + {"ref_videos", true}, + {"ref_audios", true}, + {"ref_video_audios", true}, {"high_noise_sample_params", true}, {"lora", true}, {"vae_tiling", true}, @@ -235,6 +239,7 @@ static json make_preview_modes_json() { return modes; } +/** Builds the /capabilities payload: model, defaults, limits and feature maps. */ static json make_capabilities_json(ServerRuntime& runtime) { refresh_lora_cache(runtime); refresh_upscaler_cache(runtime); @@ -351,8 +356,10 @@ static json make_capabilities_json(ServerRuntime& runtime) { const json generation_features = model_generation_features(runtime); auto recipe_features = [&](json features, bool video) { features["init_image"] = generation_features.at(video ? "init_image" : "img2img"); - for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "lora", "vae_tiling", "cache", "preview"}) { - if (features.contains(key) || std::string(key) == "ref_images") + for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "ref_videos", "ref_audios", "ref_video_audios", "lora", "vae_tiling", "cache", "preview"}) { + if (features.contains(key) || std::string(key) == "ref_images" || + (video && (std::string(key) == "ref_videos" || std::string(key) == "ref_audios" || + std::string(key) == "ref_video_audios"))) features[key] = generation_features.at(key); } if (video)