diff --git a/docs/generation_metadata.md b/docs/generation_metadata.md new file mode 100644 index 00000000..1f7fc333 --- /dev/null +++ b/docs/generation_metadata.md @@ -0,0 +1,165 @@ +# Generation results and metadata + +`generate_image()`, `generate_video()`, `adetail_image()` and `upscale()` return +owned results containing the media and its generation record. This is a breaking +C API change: applications and bindings must update their calls and rebuild. + +```c +sd_img_gen_params_t params; +sd_img_gen_params_init(¶ms); +params.prompt = "a small house"; + +sd_image_generation_result_t* result = NULL; +if (generate_image(ctx, ¶ms, &result)) { + for (int i = 0; i < result->count; ++i) { + const sd_image_result_t* item = &result->items[i]; + /* Consume item->image and item->generation_info together. */ + } + free_sd_image_generation_result(result); +} +``` + +`result_out` is required and is set to `NULL` on failure. A successful result +owns its pixels and UTF-8 JSON strings. The video result additionally owns its +audio and contains the effective `fps` and `frame_count`. Results remain valid +after freeing the generation context. Use the matching library result destructor; +do not separately free fields unless transferring ownership and clearing them. +Both result destructors accept `NULL`. + +## Reproduction parameters + +The `sdcpp.generation/v1` JSON is a snapshot of generation parameters for the +same loaded context and the same input images, masks, reference media and other +assets. It is created after `GenerationRequest` resolves the request, before +preprocessing or sampling. A failed call returns no result or metadata. + +Fields follow the C generation parameter structs: `prompt`, `negative_prompt`, +`width`, `height`, `seed`, `sample_params`, `hires`, `cache`, +`vae_tiling_params`, `image_preprocess`, and the image/video-specific options. +`kind` distinguishes image and video parameters. There is no inferred `task`. +Enums use their API string names. Seeds are decimal strings to preserve their +full integer range. Omitted optional fields retain the parameter initializer's +defaults, including automatic values represented by infinity in the C API. + +Disabled optional features are omitted entirely: `cache`, `hires`, VAE tiling, +zero/nonpositive `shifted_timestep`, inactive SLG, false circular axes, and empty +preprocessing/reference arguments. SLG layers remain present when `slg_uncond` +is enabled even with zero SLG scale. Enabled caches store their mode's parameters; +TaylorSeer retains the shared DBCache thresholds and limits used by the runtime. +Image CFG at its neutral value of 1 is omitted and resolves back to 1 by default. + +Image strength, ControlNet/IP-Adapter strength and PhotoMaker settings are stored +when their corresponding input images are supplied. PuLID requires an embedding +path. Layer count is stored for layered generation; video VACE and high-noise +settings require those models. High-noise settings omit scheduler, custom sigmas, +flow shift and timestep shifting because that pass shares the base configuration. +Explicit nonempty argument strings are retained without rewriting their contents. +Core sampling values such as seed, sampler, scheduler, steps and guidance remain +explicit even when equal to their defaults. ADetailer always retains its inpaint +strength because its input crops are created later. +Model-required VAE tiling is resolved before serialization, so forced tiling +(such as MiniMax-H3) remains present even if the caller did not enable it. + +The request fixes the seed, dimensions, clip skip, guidance, sampler, scheduler +and eta before taking the snapshot. `custom_sigmas` is stored only when supplied +as an input. Automatically computed schedules, preprocessing transforms, +conditioning/cache decisions, detector regions and output statistics are not +stored: they are recomputed from the same parameters and inputs. + +Each image records the seed used to sample its latent. `batch_count` is omitted; +replay image parameters with `batch_count = 1`. Layers or frames decoded from +the same latent share its seed, so layered generation reproduces that group of +layers. Video frames share the video request. Partial batch cancellation returns +metadata only for completed outputs. Full cancellation returns no result. + +The current image pipeline prepares input latents once using the starting seed +and shares them across a batch. With a stochastic VAE encoder, replaying a later +image alone using its sampling seed can change those input latents. Per-image +seeds therefore do not guarantee independent reproduction for these paths. + +To replay a record, initialize the matching C parameter struct, populate its +fields from the JSON, restore the same input buffers, and call generation with +the same context. Keep custom-sigma/SLG arrays and string storage alive for the +call. Paths supplied as generation parameters (LoRAs, hires models and identity +embeddings) are retained. `models` identifies successfully loaded context model +files by component (`model`, `diffusion_model`, `clip_l`, `vae`, etc.), using +filenames with extensions and omitting directories. Optional components disabled +for the request and preview-only TAE are omitted. WebUI text includes `Model` +(the diffusion model when supplied separately, otherwise the checkpoint, without +the extension) and `VAE` for a separate active VAE. These names are identifiers, +not hashes or sufficient configuration to rebuild a context; replay still requires +the same model files and context settings. The JSON does not embed media buffers +or model weights. + +When `--lora-model-dir` is configured, CLI/server output records express LoRA +paths relative to that directory, preserving subdirectories and file extensions +and using `/` separators. This applies to file metadata, API responses and +ADetailer records. Resolve these paths against the same LoRA directory before +passing them back to the C API. Core results retain the supplied loading paths. +Files outside the directory retain `..` components; paths on another volume, +or records without a configured LoRA directory, retain their original paths. + +## Postprocessing + +Upscale records `upscale.upscale_factor`. ADetailer records `adetailer` and +`inpaint_params`, with a concrete starting seed chosen before detection. These +operations require the same upscaler/detector context and source image on replay. +They do not store detection results or per-region generation traces. + +The CLI/server append operation parameter snapshots to `postprocess` in call +order. The source generation parameters remain unchanged. Frame postprocessing +uses the same operation parameters for each video frame. Standalone +postprocessing does not invent source diffusion parameters for an input image. + +## File metadata + +Image encoders accept the native record and derive WebUI infotext from it. +Images use the existing metadata locations and combined text layout: +`, SDCPP: `. Standalone postprocessing without +source diffusion parameters stores only native JSON. Video metadata stores +only native JSON. + +| Container | Metadata location | +| --- | --- | +| PNG | `tEXt`, keyword `parameters` | +| JPEG | COM segment, `parameters` label followed by a NUL byte | +| WebP image / animated WebP | XMP property `sdcpp:parameters` | +| AVI | RIFF INFO `ICMT` | +| WebM | `COMMENT` tag | + +Inspect images or videos with `sd-cli -M metadata --image `; the same +command supports all containers above, with `--metadata-format json` for a +structured report. Video metadata inspection does not require a model or video +decoder. +Matroska containers are also supported, including `.webm` outputs whose audio +codec causes the muxer to select the Matroska document type. + +The XMP namespace is `https://github.com/leejet/stable-diffusion.cpp/ns/1.0/`. +PNG and JPEG retain the existing UTF-8 text payload convention. Memory encoding +and file saving use the same implementation. The core result and API +`generation_info` fields remain pure JSON; the combined layout applies only +to embedded image metadata. + +WebUI conversion is lossy: known samplers and schedules have explicit name +mappings; unsupported ones use `SDCPP Sampler`/`SDCPP Schedule` fields. Hires +keeps the base request size and separate configured hires size/steps. The native +record retains the full generation parameters. Standalone postprocessing has no WebUI +infotext because its original diffusion settings are unknown. + +WebUI prompt text appends ordinary LoRAs as ``. +Directory components are omitted to match WebUI's filename lookup; the native +`loras` array retains relative paths and weights. High-noise-only LoRAs and names +that cannot be represented by this tag syntax remain in the native record only. +The native prompt is not modified, and postprocessing LoRAs are not added to the +source generation prompt. + +JPEG requires the COM payload, including the label and terminating NUL bytes, +to fit within 65,533 bytes. Larger records fail encoding with an error rather +than being truncated. Use PNG or WebP for larger records. + +Disabling file metadata omits both image representations, but does not disable +the core generation record or its API response. Native async image results and +OpenAI image response items contain `generation_info`; video results contain a +sequence-level `generation_info`. The SD API `info` object contains per-output +`generation_info` alongside derived `infotexts` and each image's seed. Its outer +`parameters` field continues to echo the request. diff --git a/docs/image_preprocessing.md b/docs/image_preprocessing.md index 1ca6460e..8ec236e7 100644 --- a/docs/image_preprocessing.md +++ b/docs/image_preprocessing.md @@ -149,12 +149,12 @@ an array of rule strings: OpenAI-compatible requests accept it through `{...}` in the prompt. Request rules replace server-default rules. Generation metadata records the -user rules; image encodings and channel conventions are unchanged. +user rules for replay with the same input images; image encodings and channel conventions are unchanged. ## C API Set `image_preprocess` on the existing image/video generation parameters. -The `generate_image()` and `generate_video()` signatures are unchanged: +Generation returns media and actual parameters together; see [generation metadata](generation_metadata.md): ```c sd_img_gen_params_t params; @@ -162,7 +162,10 @@ sd_img_gen_params_init(¶ms); /* Set prompt, original-resolution input images, and generation options. */ params.image_preprocess.rules = "target=init,mode=crop-resize,filter=lanczos;" "target=mask,filter=nearest-exact"; -bool ok = generate_image(ctx, ¶ms, &images, &count); +sd_image_generation_result_t* result = NULL; +bool ok = generate_image(ctx, ¶ms, &result); +/* Consume result->items and their generation_info on success. */ +free_sd_image_generation_result(result); ``` Both generation parameter initializers set `image_preprocess.rules` to `NULL`, diff --git a/examples/cli/CMakeLists.txt b/examples/cli/CMakeLists.txt index 6b1266cf..90164cfc 100644 --- a/examples/cli/CMakeLists.txt +++ b/examples/cli/CMakeLists.txt @@ -2,8 +2,10 @@ set(TARGET sd-cli) add_executable(${TARGET} ../common/common.cpp + ../common/generation_metadata.cpp ../common/log.cpp ../common/media_io.cpp + ../common/media_metadata.cpp image_metadata.cpp main.cpp ) diff --git a/examples/cli/README.md b/examples/cli/README.md index bf9de186..57894554 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -31,14 +31,21 @@ rule for edge detection. See [Image preprocessing](../../docs/image_preprocessing.md) for input selectors, input defaults, downstream model processing, mask alignment, and examples. -Metadata mode inspects PNG/JPEG container metadata without loading any model: +Metadata mode inspects PNG, JPEG, WebP (including animated WebP), AVI, WebM, and Matroska +container metadata without loading any model. Use `--image` for both images and videos: ```bash ./bin/sd-cli -M metadata --image ./output.png ./bin/sd-cli -M metadata --image ./output.jpg --metadata-format json +./bin/sd-cli -M metadata --image ./output.webm +./bin/sd-cli -M metadata --image ./output.avi --metadata-format json ./bin/sd-cli -M metadata --image ./output.png --metadata-raw ./bin/sd-cli -M metadata --image ./output.png --metadata-all ``` For completely black or white images or videos, NaNs, and the `--linear-scale` / `--attn-scale` workaround, see [Troubleshooting](../../docs/troubleshooting.md). + +Generation outputs embed the core [generation record](../../docs/generation_metadata.md). +Images retain the native JSON and an additional WebUI-compatible infotext. +`--disable-image-metadata` omits both from files. diff --git a/examples/cli/image_metadata.cpp b/examples/cli/image_metadata.cpp index ba0d6da2..bb5ab7cf 100644 --- a/examples/cli/image_metadata.cpp +++ b/examples/cli/image_metadata.cpp @@ -2,12 +2,14 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -391,8 +393,39 @@ namespace { json& report, std::string& error); + json parse_exif_user_comment(const uint8_t* data, size_t size) { + if (size >= 8 && memcmp(data, "ASCII\0\0\0", 8) == 0) { + return trim_trailing_nuls(bytes_to_string(data + 8, data + size)); + } + if (size < 8 || memcmp(data, "UNICODE\0", 8) != 0) { + return json{{"raw_hex_preview", hex_preview(data, size)}}; + } + size_t offset = 8; + // WebUI's piexif writer uses big-endian UTF-16 without a BOM. + bool little_endian = false; + if (size >= 10 && ((data[8] == 0xfe && data[9] == 0xff) || (data[8] == 0xff && data[9] == 0xfe))) { + little_endian = data[8] == 0xff; + offset = 10; + } + if ((size - offset) % 2 != 0) { + return json{{"error", "truncated UTF-16 UserComment"}, {"raw_hex_preview", hex_preview(data, size)}}; + } + std::u16string text; + for (; offset < size; offset += 2) { + text.push_back(static_cast(little_endian ? data[offset] | (uint16_t(data[offset + 1]) << 8) + : (uint16_t(data[offset]) << 8) | data[offset + 1])); + } + try { + std::wstring_convert, char16_t> converter; + return trim_trailing_nuls(converter.to_bytes(text)); + } catch (const std::range_error&) { + return json{{"error", "invalid UTF-16 UserComment"}, {"raw_hex_preview", hex_preview(data, size)}}; + } + } + json parse_exif_value(const std::vector& data, size_t value_offset, + uint16_t tag, uint16_t type, uint32_t count, bool little_endian, @@ -466,8 +499,9 @@ namespace { break; } case 7: - value = bytes_to_string(data.data() + value_offset, - data.data() + value_offset + count); + value = tag == 0x9286 ? parse_exif_user_comment(data.data() + value_offset, count) + : json(bytes_to_string(data.data() + value_offset, + data.data() + value_offset + count)); break; case 9: { if (count == 1) { @@ -591,6 +625,7 @@ namespace { tag_json["value"] = parse_exif_value(data, value_offset, + tag, type, value_count, little_endian, @@ -1098,6 +1133,178 @@ namespace { return true; } + static bool parse_avi_chunks(const std::vector& data, size_t offset, size_t end, bool info, bool include_raw, json& result, std::string& error) { + while (offset < end) { + if (end - offset < 8) { + error = "truncated AVI chunk header"; + return false; + } + const auto name = bytes_to_string(data.data() + offset, data.data() + offset + 4); + const size_t size = read_u32_le(data, offset + 4); + offset += 8; + if (size > end - offset || (size & 1u) > end - offset - size) { + error = "AVI chunk exceeds its container"; + return false; + } + json entry = {{"entry_type", "chunk"}, {"name", name}, {"length", size}, {"metadata_like", info}}; + if (name == "LIST") { + if (size < 4) { + error = "truncated AVI LIST type"; + return false; + } + const auto type = bytes_to_string(data.data() + offset, data.data() + offset + 4); + entry["data"]["list_type"] = type; + if (type == "INFO" && !info && + !parse_avi_chunks(data, offset + 4, offset + size, true, include_raw, result, error)) + return false; + } else if (info) { + entry["data"]["text"] = trim_trailing_nuls(bytes_to_string(data.data() + offset, data.data() + offset + size)); + } else { + append_raw_preview(entry, data.data() + offset, size, include_raw); + } + result["entries"].push_back(std::move(entry)); + offset += size + (size & 1u); + } + return true; + } + + static bool parse_avi(const std::vector& data, bool include_raw, json& result, std::string& error) { + result = {{"format", "AVI"}, {"entries", json::array()}}; + size_t offset = 0; + while (offset < data.size()) { + if (data.size() - offset < 12 || memcmp(data.data() + offset, "RIFF", 4) != 0 || + memcmp(data.data() + offset + 8, offset == 0 ? "AVI " : "AVIX", 4) != 0) { + error = "invalid AVI RIFF header"; + return false; + } + const size_t size = read_u32_le(data, offset + 4); + if (size < 4 || size > data.size() - offset - 8 || + (size & 1u) > data.size() - offset - 8 - size) { + error = "AVI RIFF size exceeds file size"; + return false; + } + if (!parse_avi_chunks(data, offset + 12, offset + 8 + size, false, include_raw, result, error)) + return false; + offset += 8 + size + (size & 1u); + } + return true; + } + + static bool read_ebml_uint(const std::vector& data, size_t& offset, size_t end, bool id, uint64_t& value, bool& unknown) { + if (offset == end || data[offset] == 0) + return false; + uint8_t mask = 0x80; + size_t length = 1; + while (!(data[offset] & mask)) { + mask >>= 1; + ++length; + } + if (length > (id ? 4u : 8u) || length > end - offset) + return false; + value = data[offset++] & (id ? 0xff : mask - 1); + for (size_t i = 1; i < length; ++i) + value = (value << 8) | data[offset++]; + unknown = !id && value == (uint64_t(1) << (7 * length)) - 1; + return true; + } + + static bool is_webm_segment_child(uint64_t id) { + return id == 0x114D9B74 || id == 0x1549A966 || id == 0x1654AE6B || + id == 0x1F43B675 || id == 0x1C53BB6B || id == 0x1941A469 || + id == 0x1043A770 || id == 0x1254C367; + } + + static bool parse_webm_elements(const std::vector& data, size_t& offset, size_t end, uint64_t parent, bool unknown_cluster, int depth, bool include_raw, json& result, std::string& error) { + if (depth > 16) { + error = "WebM tag nesting exceeds limit"; + return false; + } + std::string tag_name; + std::string tag_text; + bool has_text = false; + while (offset < end) { + const auto start = offset; + uint64_t id, size; + bool unknown; + if (!read_ebml_uint(data, offset, end, true, id, unknown)) { + error = "invalid or truncated WebM element ID"; + return false; + } + // An unknown-size Cluster ends at the next Segment-level element. + if (unknown_cluster && is_webm_segment_child(id)) { + offset = start; + return true; + } + if (!read_ebml_uint(data, offset, end, false, size, unknown) || + (!unknown && size > end - offset)) { + error = "invalid or truncated WebM element size"; + return false; + } + if (unknown && id != 0x18538067 && id != 0x1F43B675) { + error = "unsupported unknown-size WebM element"; + return false; + } + const size_t stop = unknown ? end : offset + static_cast(size); + const bool container = (parent == 0 && (id == 0x1A45DFA3 || id == 0x18538067)) || + (parent == 0x18538067 && (id == 0x1254C367 || (id == 0x1F43B675 && unknown))) || + (parent == 0x1254C367 && id == 0x7373) || + ((parent == 0x7373 || parent == 0x67C8) && id == 0x67C8); + if (parent == 0 && id == 0x18538067) + result["has_segment"] = true; + if (parent == 0x1A45DFA3 && id == 0x4282) { + const auto type = bytes_to_string(data.data() + offset, data.data() + stop); + if (type != "webm" && type != "matroska") { + error = "EBML document is not WebM or Matroska"; + return false; + } + result["doc_type"] = type; + result["format"] = type == "webm" ? "WEBM" : "MATROSKA"; + } + if (parent == 0x67C8 && (id == 0x45A3 || id == 0x4487)) { + auto value = bytes_to_string(data.data() + offset, data.data() + stop); + if (id == 0x45A3) + tag_name = std::move(value); + else { + tag_text = std::move(value); + has_text = true; + } + } + if (parent == 0 || parent == 0x18538067) { + std::ostringstream name; + name << "0x" << std::hex << std::uppercase << id; + json entry = {{"entry_type", "element"}, {"name", name.str()}, {"length", stop - offset}, {"metadata_like", false}}; + if (unknown) + entry["unknown_size"] = true; + append_raw_preview(entry, data.data() + offset, stop - offset, include_raw); + result["entries"].push_back(std::move(entry)); + } + if (container) { + if (!parse_webm_elements(data, offset, stop, id, id == 0x1F43B675 && unknown, + depth + 1, include_raw, result, error)) + return false; + } else { + offset = stop; + } + } + if (parent == 0x67C8 && has_text) { + result["entries"].push_back({{"entry_type", "tag"}, {"name", tag_name}, {"metadata_like", true}, {"data", {{"text", tag_text}}}}); + } + return true; + } + + static bool parse_webm(const std::vector& data, bool include_raw, json& result, std::string& error) { + result = {{"format", "WEBM"}, {"entries", json::array()}}; + size_t offset = 0; + if (!parse_webm_elements(data, offset, data.size(), 0, false, 0, include_raw, result, error)) + return false; + if (!result.contains("doc_type") || !result.value("has_segment", false)) { + error = "missing WebM document type or Segment"; + return false; + } + result.erase("has_segment"); + return true; + } + std::string abbreviate(const std::string& value, bool brief) { if (!brief || value.size() <= 240) { return value; @@ -1116,7 +1323,7 @@ namespace { return; } if (value.is_primitive()) { - out << prefix << key << ": " << value.dump() << "\n"; + out << prefix << key << ": " << value.dump(-1, ' ', false, json::error_handler_t::replace) << "\n"; return; } if (value.is_array()) { @@ -1125,7 +1332,7 @@ namespace { if (item.is_string()) { out << prefix << " - " << abbreviate(item.get(), brief) << "\n"; } else if (item.is_primitive()) { - out << prefix << " - " << item.dump() << "\n"; + out << prefix << " - " << item.dump(-1, ' ', false, json::error_handler_t::replace) << "\n"; } else { out << prefix << " -\n"; for (auto it = item.begin(); it != item.end(); ++it) { @@ -1158,10 +1365,12 @@ namespace { } for (const auto& entry : entries) { - const bool is_chunk = entry.value("entry_type", "") == "chunk"; - const std::string name = is_chunk ? entry.value("name", "unknown") - : entry.value("marker", "unknown"); - out << (is_chunk ? "Chunk: " : "Segment: ") << name << "\n"; + const auto type = entry.value("entry_type", ""); + const auto name = entry.value("name", entry.value("marker", "unknown")); + const char* label = type == "chunk" ? "Chunk" : type == "tag" ? "Tag" + : type == "element" ? "Element" + : "Segment"; + out << label << ": " << name << "\n"; for (auto it = entry.begin(); it != entry.end(); ++it) { if (it.key() == "entry_type" || it.key() == "name" || it.key() == "marker" || it.key() == "metadata_like") { @@ -1211,7 +1420,13 @@ namespace { return parse_webp(data, include_raw, report, error); } - error = "unsupported image format; only PNG, JPEG, and WebP are supported"; + if (data.size() >= 12 && memcmp(data.data(), "RIFF", 4) == 0 && + memcmp(data.data() + 8, "AVI ", 4) == 0) + return parse_avi(data, include_raw, report, error); + if (data.size() >= 4 && read_u32_be(data, 0) == 0x1A45DFA3) + return parse_webm(data, include_raw, report, error); + + error = "unsupported media format; only PNG, JPEG, WebP, AVI, WebM, and Matroska are supported"; return false; } @@ -1230,7 +1445,7 @@ bool print_image_metadata(const std::string& image_path, if (options.output_format == MetadataOutputFormat::JSON) { visible_report["file"] = image_path; - out << visible_report.dump(2) << "\n"; + out << visible_report.dump(2, ' ', false, json::error_handler_t::replace) << "\n"; } else { print_text_report(image_path, visible_report, options.include_structural, options.brief, out); } diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index ee1e56f2..591a1511 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -69,7 +69,7 @@ struct SDCliParams { &output_path}, {"", "--image", - "path to the image to inspect (for metadata mode)", + "path to the image or video to inspect (for metadata mode)", 0, &image_path}, {"", @@ -227,7 +227,7 @@ struct SDCliParams { } } else { if (image_path.empty()) { - LOG_ERROR("error: metadata mode needs an image path (--image)"); + LOG_ERROR("error: metadata mode needs an image or video path (--image)"); return false; } if (metadata_format != "text" && metadata_format != "json") { @@ -441,10 +441,11 @@ static fs::path get_video_audio_sidecar_path(const SDCliParams& cli_params) { bool save_results(const SDCliParams& cli_params, const SDContextParams& ctx_params, const SDGenerationParams& gen_params, - sd_image_t* results, + SDImageVec& results, int num_results, - const sd_audio_t* generated_audio = nullptr) { - if (results == nullptr || num_results <= 0) { + const sd_audio_t* generated_audio, + const std::string& video_info) { + if (results.empty() || num_results <= 0) { return false; } @@ -487,12 +488,10 @@ bool save_results(const SDCliParams& cli_params, if (!img.data) return false; - int images_per_batch = gen_params.batch_count > 0 ? std::max(1, num_results / gen_params.batch_count) : 1; - const int64_t metadata_seed = cli_params.mode == VID_GEN ? gen_params.seed : gen_params.seed + idx / images_per_batch; - std::string params = gen_params.embed_image_metadata - ? get_image_params(ctx_params, gen_params, metadata_seed, cli_params.mode) - : ""; - const bool ok = write_image_to_file(path.u8string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality); + std::string params = gen_params.embed_image_metadata + ? results.generation_info(idx) + : ""; + const bool ok = write_image_to_file(path.u8string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality); LOG_INFO("save result image %d to '%s' (%s)", idx, path.u8string().c_str(), ok ? "success" : "failure"); return ok; }; @@ -534,7 +533,7 @@ bool save_results(const SDCliParams& cli_params, if (ext_lower != ".avi" && ext_lower != ".webp" && ext_lower != ".webm") ext = ".avi"; std::string params = gen_params.embed_image_metadata - ? get_image_params(ctx_params, gen_params, gen_params.seed, cli_params.mode) + ? video_info : ""; fs::path video_path = base_path; @@ -542,7 +541,7 @@ bool save_results(const SDCliParams& cli_params, std::string final_ext_lower = ext.u8string(); std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower); const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm"); - if (create_video_from_sd_images(video_path.u8string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) { + if (create_video_from_sd_images(video_path.u8string().c_str(), results.data(), num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) { LOG_INFO("save result video to '%s'", video_path.u8string().c_str()); if (generated_audio != nullptr && !mux_audio) { fs::path wav_path = video_path; @@ -607,30 +606,29 @@ static bool apply_adetailer(sd_ctx_t* sd_ctx, continue; } sd_img_gen_params_t ad_generation_params = img_gen_params; - ad_generation_params.seed = img_gen_params.seed + i; + ad_generation_params.seed = generation_seed(results.generation_info(i), img_gen_params.seed); if (mode == IMG_GEN) { ad_generation_params.width = 512; ad_generation_params.height = 512; ad_generation_params.strength = 0.4f; } - sd_image_t* detailed_images = nullptr; - int detailed_count = 0; + sd_image_generation_result_t* detailed = nullptr; if (!adetail_image(ad_ctx.get(), sd_ctx, results[i], &ad_params, &ad_generation_params, - &detailed_images, - &detailed_count) || - detailed_count <= 0 || detailed_images == nullptr || detailed_images[0].data == nullptr) { - free_sd_images(detailed_images, detailed_count); + &detailed) || + detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) { + free_sd_image_generation_result(detailed); LOG_ERROR("ADetailer failed for image %d", i + 1); return false; } + results.append_generation_info(i, detailed->items[0].generation_info, ctx_params.lora_model_dir); free(results[i].data); - results[i] = detailed_images[0]; - detailed_images[0] = {0, 0, 0, nullptr}; - free_sd_images(detailed_images, detailed_count); + results[i] = detailed->items[0].image; + detailed->items[0].image = {}; + free_sd_image_generation_result(detailed); } return true; } @@ -884,6 +882,7 @@ int main(int argc, const char* argv[]) { SDImageVec results; int num_results = 0; sd_audio_t* generated_audio = nullptr; + std::string video_info; if (cli_params.mode == UPSCALE) { num_results = 1; @@ -896,18 +895,6 @@ int main(int argc, const char* argv[]) { return 1; } - if (gen_params.sample_params.sample_method == SAMPLE_METHOD_COUNT) { - gen_params.sample_params.sample_method = sd_get_default_sample_method(sd_ctx.get()); - } - - if (gen_params.high_noise_sample_params.sample_method == SAMPLE_METHOD_COUNT) { - gen_params.high_noise_sample_params.sample_method = sd_get_default_sample_method(sd_ctx.get()); - } - - if (gen_params.sample_params.scheduler == SCHEDULER_COUNT) { - gen_params.sample_params.scheduler = sd_get_default_scheduler(sd_ctx.get(), gen_params.sample_params.sample_method); - } - sd_img_gen_params_t img_gen_params{}; const bool use_img_gen_params = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER; if (use_img_gen_params) { @@ -915,23 +902,24 @@ int main(int argc, const char* argv[]) { } if (cli_params.mode == IMG_GEN) { - sd_image_t* generated_images = nullptr; - if (!generate_image(sd_ctx.get(), &img_gen_params, &generated_images, &num_results)) { - generated_images = nullptr; - num_results = 0; - } - results.adopt(generated_images, num_results); + sd_image_generation_result_t* generated = nullptr; + generate_image(sd_ctx.get(), &img_gen_params, &generated); + results.adopt(generated, ctx_params.lora_model_dir); + num_results = results.count(); } else if (cli_params.mode == ADETAILER) { num_results = 1; results.push_back(gen_params.init_image.release()); } else if (cli_params.mode == VID_GEN) { - sd_vid_gen_params_t vid_gen_params = gen_params.to_sd_vid_gen_params_t(); - sd_image_t* generated_video = nullptr; - if (!generate_video(sd_ctx.get(), &vid_gen_params, &generated_video, &num_results, &generated_audio, &cli_params.preview_fps)) { - generated_video = nullptr; + sd_vid_gen_params_t vid_gen_params = gen_params.to_sd_vid_gen_params_t(); + sd_video_generation_result_t* generated = nullptr; + if (generate_video(sd_ctx.get(), &vid_gen_params, &generated)) { + gen_params.fps = cli_params.preview_fps = generated->fps; + generated_audio = generated->audio; + generated->audio = nullptr; + num_results = generated->frame_count; + results.adopt_video(generated, ctx_params.lora_model_dir); + video_info = results.generation_info(0); } - gen_params.fps = cli_params.preview_fps; - results.adopt(generated_video, num_results); } if (!results) { @@ -971,21 +959,20 @@ int main(int argc, const char* argv[]) { SDImageOwner current_image(results[i]); results[i] = {0, 0, 0, nullptr}; for (int u = 0; u < gen_params.upscale_repeats; ++u) { - sd_image_t* upscaled_images = nullptr; - int upscaled_count = 0; - bool upscale_ok = upscale(upscaler_ctx.get(), - current_image.get(), - upscale_factor, - &upscaled_images, - &upscaled_count); - if (!upscale_ok || upscaled_count <= 0 || upscaled_images[0].data == nullptr) { - free_sd_images(upscaled_images, upscaled_count); + sd_image_generation_result_t* upscaled = nullptr; + bool upscale_ok = upscale(upscaler_ctx.get(), + current_image.get(), + upscale_factor, + &upscaled); + if (!upscale_ok || !upscaled || upscaled->count <= 0 || upscaled->items[0].image.data == nullptr) { + free_sd_image_generation_result(upscaled); LOG_ERROR("upscale failed"); break; } - sd_image_t upscaled_image = upscaled_images[0]; - upscaled_images[0] = {0, 0, 0, nullptr}; - free_sd_images(upscaled_images, upscaled_count); + results.append_generation_info(i, upscaled->items[0].generation_info); + sd_image_t upscaled_image = upscaled->items[0].image; + upscaled->items[0].image = {}; + free_sd_image_generation_result(upscaled); current_image.reset(upscaled_image); } results[i] = current_image.release(); // Set the final upscaled image as the result @@ -993,7 +980,9 @@ int main(int argc, const char* argv[]) { } } - if (!save_results(cli_params, ctx_params, gen_params, results.data(), num_results, generated_audio)) { + if (!video_info.empty() && results.count() > 0) + video_info = results.generation_info(0); + if (!save_results(cli_params, ctx_params, gen_params, results, num_results, generated_audio, video_info)) { free_sd_audio(generated_audio); return 1; } diff --git a/examples/common/common.cpp b/examples/common/common.cpp index bd25f905..00301dc9 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -2548,11 +2548,6 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s return false; } - if (seed < 0) { - srand((int)time(nullptr)); - seed = rand(); - } - if (strict) { batch_count = std::clamp(batch_count, 1, 8); sample_params.sample_steps = std::clamp(sample_params.sample_steps, 1, 100); @@ -3029,301 +3024,3 @@ std::string SDGenerationParams::to_string() const { std::string version_string() { return std::string("stable-diffusion.cpp version ") + sd_version() + ", commit " + sd_commit(); } - -static std::string safe_json_string(const char* value) { - return value ? value : ""; -} - -static void set_json_basename_if_not_empty(json& target, const char* key, const std::string& path) { - if (!path.empty()) { - target[key] = sd_basename(path); - } -} - -static json build_sampling_metadata_json(const sd_sample_params_t& sample_params, - const std::vector& skip_layers, - const std::vector* custom_sigmas = nullptr) { - json sampling = { - {"steps", sample_params.sample_steps}, - {"eta", sample_params.eta}, - {"shifted_timestep", sample_params.shifted_timestep}, - {"flow_shift", sample_params.flow_shift}, - {"extra_sample_args", safe_json_string(sample_params.extra_sample_args)}, - {"guidance", - { - {"txt_cfg", sample_params.guidance.txt_cfg}, - {"img_cfg", sample_params.guidance.img_cfg}, - {"distilled_guidance", sample_params.guidance.distilled_guidance}, - {"slg", - { - {"scale", sample_params.guidance.slg.scale}, - {"layers", skip_layers}, - {"start", sample_params.guidance.slg.layer_start}, - {"end", sample_params.guidance.slg.layer_end}, - }}, - }}, - }; - if (sample_params.sample_method != SAMPLE_METHOD_COUNT) { - sampling["method"] = safe_json_string(sd_sample_method_name(sample_params.sample_method)); - } - if (sample_params.scheduler != SCHEDULER_COUNT) { - sampling["scheduler"] = safe_json_string(sd_scheduler_name(sample_params.scheduler)); - } - if (custom_sigmas != nullptr) { - sampling["custom_sigmas"] = *custom_sigmas; - } - return sampling; -} - -std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params, - const SDGenerationParams& gen_params, - int64_t seed, - SDMode mode) { - json root; - root["schema"] = "sdcpp.image.params/v1"; - root["mode"] = "img_gen"; - if (mode == VID_GEN) { - root["mode"] = "vid_gen"; - } else if (mode == ADETAILER) { - root["mode"] = "adetailer"; - } - root["generator"] = { - {"name", "stable-diffusion.cpp"}, - {"version", safe_json_string(sd_version())}, - {"commit", safe_json_string(sd_commit())}, - }; - root["seed"] = seed; - root["width"] = gen_params.get_resolved_width(); - root["height"] = gen_params.get_resolved_height(); - - root["prompt"] = { - {"positive", gen_params.prompt}, - {"negative", gen_params.negative_prompt}, - }; - if (!gen_params.ad_model_path.empty()) { - root["adetailer"] = { - {"model", sd_basename(gen_params.ad_model_path)}, - {"prompt", gen_params.ad_prompt}, - {"negative_prompt", gen_params.ad_negative_prompt}, - {"extra_args", gen_params.extra_ad_args}, - }; - } - root["sampling"] = build_sampling_metadata_json(gen_params.sample_params, - gen_params.skip_layers, - &gen_params.custom_sigmas); - - json models; - set_json_basename_if_not_empty(models, "model", ctx_params.model_path); - set_json_basename_if_not_empty(models, "clip_l", ctx_params.clip_l_path); - set_json_basename_if_not_empty(models, "clip_g", ctx_params.clip_g_path); - set_json_basename_if_not_empty(models, "clip_vision", ctx_params.clip_vision_path); - set_json_basename_if_not_empty(models, "t5xxl", ctx_params.t5xxl_path); - set_json_basename_if_not_empty(models, "llm", ctx_params.llm_path); - set_json_basename_if_not_empty(models, "llm_vision", ctx_params.llm_vision_path); - set_json_basename_if_not_empty(models, "diffusion_model", ctx_params.diffusion_model_path); - set_json_basename_if_not_empty(models, "high_noise_diffusion_model", ctx_params.high_noise_diffusion_model_path); - set_json_basename_if_not_empty(models, "uncond_diffusion_model", ctx_params.uncond_diffusion_model_path); - set_json_basename_if_not_empty(models, "vae", ctx_params.vae_path); - set_json_basename_if_not_empty(models, "taesd", ctx_params.taesd_path); - set_json_basename_if_not_empty(models, "control_net", ctx_params.control_net_path); - root["models"] = std::move(models); - - root["clip_skip"] = gen_params.clip_skip; - root["strength"] = gen_params.strength; - root["control_strength"] = gen_params.control_strength; - root["ip_adapter_strength"] = gen_params.ip_adapter_strength; - root["ref_image_args"] = gen_params.ref_image_args; - root["image_preprocess"] = gen_params.image_preprocess; - root["increase_ref_index"] = gen_params.increase_ref_index; - if (mode == VID_GEN) { - root["video"] = { - {"frame_count", gen_params.video_frames}, - {"fps", gen_params.fps}, - }; - root["moe_boundary"] = gen_params.moe_boundary; - root["vace_strength"] = gen_params.vace_strength; - root["high_noise_sampling"] = build_sampling_metadata_json(gen_params.high_noise_sample_params, - gen_params.high_noise_skip_layers); - } - - root["rng"] = safe_json_string(sd_rng_type_name(ctx_params.rng_type)); - if (ctx_params.sampler_rng_type != RNG_TYPE_COUNT) { - root["sampler_rng"] = safe_json_string(sd_rng_type_name(ctx_params.sampler_rng_type)); - } - - json loras = json::array(); - for (const auto& entry : gen_params.lora_map) { - loras.push_back({ - {"name", sd_basename(entry.first)}, - {"multiplier", entry.second}, - {"is_high_noise", false}, - }); - } - for (const auto& entry : gen_params.high_noise_lora_map) { - loras.push_back({ - {"name", sd_basename(entry.first)}, - {"multiplier", entry.second}, - {"is_high_noise", true}, - }); - } - if (!loras.empty()) { - root["loras"] = std::move(loras); - } - - if (gen_params.hires_enabled) { - root["hires"] = { - {"enabled", gen_params.hires_enabled}, - {"upscaler", gen_params.hires_upscaler}, - {"model", gen_params.hires_upscaler_model_path.empty() ? "" : sd_basename(gen_params.hires_upscaler_model_path)}, - {"scale", gen_params.hires_scale}, - {"target_width", gen_params.hires_width}, - {"target_height", gen_params.hires_height}, - {"steps", gen_params.hires_steps}, - {"denoising_strength", gen_params.hires_denoising_strength}, - {"custom_sigmas", gen_params.hires_custom_sigmas}, - {"upscale_tile_size", gen_params.hires_upscale_tile_size}, - }; - } - - if (gen_params.cache_params.mode != SD_CACHE_DISABLED) { - root["cache"] = { - {"requested_mode", gen_params.cache_mode}, - {"requested_option", gen_params.cache_option}, - {"mode", gen_params.cache_params.mode}, - {"scm_mask", gen_params.scm_mask}, - {"scm_policy_dynamic", gen_params.scm_policy_dynamic}, - {"reuse_threshold", gen_params.cache_params.reuse_threshold}, - {"start_percent", gen_params.cache_params.start_percent}, - {"end_percent", gen_params.cache_params.end_percent}, - {"error_decay_rate", gen_params.cache_params.error_decay_rate}, - {"use_relative_threshold", gen_params.cache_params.use_relative_threshold}, - {"reset_error_on_compute", gen_params.cache_params.reset_error_on_compute}, - {"Fn_compute_blocks", gen_params.cache_params.Fn_compute_blocks}, - {"Bn_compute_blocks", gen_params.cache_params.Bn_compute_blocks}, - {"residual_diff_threshold", gen_params.cache_params.residual_diff_threshold}, - {"max_warmup_steps", gen_params.cache_params.max_warmup_steps}, - {"max_cached_steps", gen_params.cache_params.max_cached_steps}, - {"max_continuous_cached_steps", gen_params.cache_params.max_continuous_cached_steps}, - {"taylorseer_n_derivatives", gen_params.cache_params.taylorseer_n_derivatives}, - {"taylorseer_skip_interval", gen_params.cache_params.taylorseer_skip_interval}, - {"spectrum_w", gen_params.cache_params.spectrum_w}, - {"spectrum_m", gen_params.cache_params.spectrum_m}, - {"spectrum_lam", gen_params.cache_params.spectrum_lam}, - {"spectrum_window_size", gen_params.cache_params.spectrum_window_size}, - {"spectrum_flex_window", gen_params.cache_params.spectrum_flex_window}, - {"spectrum_warmup_steps", gen_params.cache_params.spectrum_warmup_steps}, - {"spectrum_stop_percent", gen_params.cache_params.spectrum_stop_percent}, - }; - } - - if (gen_params.vae_tiling_params.enabled || - gen_params.vae_tiling_params.temporal_tiling || - !gen_params.extra_tiling_args.empty()) { - root["vae_tiling"] = { - {"enabled", gen_params.vae_tiling_params.enabled}, - {"temporal_tiling", gen_params.vae_tiling_params.temporal_tiling}, - {"tile_size_w", gen_params.vae_tiling_params.tile_size_w}, - {"tile_size_h", gen_params.vae_tiling_params.tile_size_h}, - {"target_overlap", gen_params.vae_tiling_params.target_overlap}, - {"rel_size_w", gen_params.vae_tiling_params.rel_size_w}, - {"rel_size_h", gen_params.vae_tiling_params.rel_size_h}, - {"extra_tiling_args", gen_params.extra_tiling_args}, - }; - } - - return root.dump(); -} - -std::string get_image_params(const SDContextParams& ctx_params, - const SDGenerationParams& gen_params, - int64_t seed, - SDMode mode) { - std::string parameter_string; - if (gen_params.prompt_with_lora.size() != 0) { - parameter_string += gen_params.prompt_with_lora + "\n"; - } else { - parameter_string += gen_params.prompt + "\n"; - } - if (gen_params.negative_prompt.size() != 0) { - parameter_string += "Negative prompt: " + gen_params.negative_prompt + "\n"; - } - parameter_string += "Steps: " + std::to_string(gen_params.sample_params.sample_steps) + ", "; - parameter_string += "CFG scale: " + std::to_string(gen_params.sample_params.guidance.txt_cfg) + ", "; - if (gen_params.sample_params.guidance.slg.scale != 0 && gen_params.skip_layers.size() != 0) { - parameter_string += "SLG scale: " + std::to_string(gen_params.sample_params.guidance.slg.scale) + ", "; - parameter_string += "Skip layers: ["; - for (const auto& layer : gen_params.skip_layers) { - parameter_string += std::to_string(layer) + ", "; - } - parameter_string += "], "; - parameter_string += "Skip layer start: " + std::to_string(gen_params.sample_params.guidance.slg.layer_start) + ", "; - parameter_string += "Skip layer end: " + std::to_string(gen_params.sample_params.guidance.slg.layer_end) + ", "; - } - parameter_string += "Guidance: " + std::to_string(gen_params.sample_params.guidance.distilled_guidance) + ", "; - parameter_string += "Eta: " + std::to_string(gen_params.sample_params.eta) + ", "; - if (!gen_params.extra_sample_args.empty()) { - parameter_string += "Extra sample args: " + gen_params.extra_sample_args + ", "; - } - if (!gen_params.ad_model_path.empty()) { - parameter_string += "ADetailer model: " + sd_basename(gen_params.ad_model_path) + ", "; - if (!gen_params.ad_prompt.empty()) { - parameter_string += "ADetailer prompt: " + gen_params.ad_prompt + ", "; - } - if (!gen_params.ad_negative_prompt.empty()) { - parameter_string += "ADetailer negative prompt: " + gen_params.ad_negative_prompt + ", "; - } - if (!gen_params.extra_ad_args.empty()) { - parameter_string += "ADetailer args: " + gen_params.extra_ad_args + ", "; - } - } - parameter_string += "Seed: " + std::to_string(seed) + ", "; - parameter_string += "Size: " + std::to_string(gen_params.get_resolved_width()) + "x" + std::to_string(gen_params.get_resolved_height()) + ", "; - parameter_string += "Model: " + sd_basename(ctx_params.model_path) + ", "; - parameter_string += "RNG: " + std::string(sd_rng_type_name(ctx_params.rng_type)) + ", "; - if (ctx_params.sampler_rng_type != RNG_TYPE_COUNT) { - parameter_string += "Sampler RNG: " + std::string(sd_rng_type_name(ctx_params.sampler_rng_type)) + ", "; - } - parameter_string += "Sampler: " + std::string(sd_sample_method_name(gen_params.sample_params.sample_method)); - if (!gen_params.custom_sigmas.empty()) { - parameter_string += ", Custom Sigmas: ["; - for (size_t i = 0; i < gen_params.custom_sigmas.size(); ++i) { - std::ostringstream oss; - oss << std::fixed << std::setprecision(4) << gen_params.custom_sigmas[i]; - parameter_string += oss.str() + (i == gen_params.custom_sigmas.size() - 1 ? "" : ", "); - } - parameter_string += "]"; - } else if (gen_params.sample_params.scheduler != SCHEDULER_COUNT) { // Only show schedule if not using custom sigmas - parameter_string += " " + std::string(sd_scheduler_name(gen_params.sample_params.scheduler)); - } - parameter_string += ", "; - for (const auto& te : {ctx_params.clip_l_path, ctx_params.clip_g_path, ctx_params.t5xxl_path, ctx_params.llm_path, ctx_params.llm_vision_path}) { - if (!te.empty()) { - parameter_string += "TE: " + sd_basename(te) + ", "; - } - } - if (!ctx_params.diffusion_model_path.empty()) { - parameter_string += "Unet: " + sd_basename(ctx_params.diffusion_model_path) + ", "; - } - if (!ctx_params.uncond_diffusion_model_path.empty()) { - parameter_string += "Uncond Unet: " + sd_basename(ctx_params.uncond_diffusion_model_path) + ", "; - } - if (!ctx_params.vae_path.empty()) { - parameter_string += "VAE: " + sd_basename(ctx_params.vae_path) + ", "; - } - if (gen_params.clip_skip != -1) { - parameter_string += "Clip skip: " + std::to_string(gen_params.clip_skip) + ", "; - } - if (gen_params.hires_enabled) { - parameter_string += "Hires upscale: " + gen_params.hires_upscaler + ", "; - parameter_string += "Hires scale: " + std::to_string(gen_params.hires_scale) + ", "; - parameter_string += "Hires resize: " + std::to_string(gen_params.hires_width) + "x" + std::to_string(gen_params.hires_height) + ", "; - parameter_string += "Hires steps: " + std::to_string(gen_params.hires_steps) + ", "; - parameter_string += "Denoising strength: " + std::to_string(gen_params.hires_denoising_strength) + ", "; - if (!gen_params.hires_custom_sigmas.empty()) { - parameter_string += "Hires custom sigmas: " + vec_to_string(gen_params.hires_custom_sigmas) + ", "; - } - } - parameter_string += "Version: stable-diffusion.cpp"; - parameter_string += ", SDCPP: " + build_sdcpp_image_metadata_json(ctx_params, gen_params, seed, mode); - return parameter_string; -} diff --git a/examples/common/common.h b/examples/common/common.h index 7b9f691a..39d6011a 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -1,6 +1,8 @@ #ifndef __EXAMPLES_COMMON_COMMON_H__ #define __EXAMPLES_COMMON_COMMON_H__ +#include "generation_metadata.h" + #include #include #include @@ -338,13 +340,4 @@ struct SDGenerationParams { }; std::string version_string(); -std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params, - const SDGenerationParams& gen_params, - int64_t seed, - SDMode mode = IMG_GEN); -std::string get_image_params(const SDContextParams& ctx_params, - const SDGenerationParams& gen_params, - int64_t seed, - SDMode mode = IMG_GEN); - #endif // __EXAMPLES_COMMON_COMMON_H__ diff --git a/examples/common/generation_metadata.cpp b/examples/common/generation_metadata.cpp new file mode 100644 index 00000000..989e9144 --- /dev/null +++ b/examples/common/generation_metadata.cpp @@ -0,0 +1,155 @@ +#include "generation_metadata.h" + +#include +#include + +#include "json.hpp" + +static void relativize_lora_paths(nlohmann::json& info, const std::filesystem::path& root) { + if (info.contains("loras")) { + for (auto& lora : info["loras"]) { + const auto path = lora.value("path", ""); + if (path.empty()) + continue; + std::error_code error; + const auto absolute = std::filesystem::absolute(std::filesystem::u8path(path), error); + if (error) + continue; + const auto relative = absolute.lexically_normal().lexically_relative(root); + if (!relative.empty()) + lora["path"] = relative.generic_u8string(); + } + } + if (info.contains("inpaint_params")) + relativize_lora_paths(info["inpaint_params"], root); + if (info.contains("postprocess")) { + for (auto& operation : info["postprocess"]) + relativize_lora_paths(operation, root); + } +} + +std::string generation_info_with_relative_lora_paths(const std::string& info, const std::string& lora_model_dir) { + if (info.empty() || lora_model_dir.empty()) + return info; + std::error_code error; + const auto root = std::filesystem::absolute(std::filesystem::u8path(lora_model_dir), error); + if (error) + return info; + auto result = nlohmann::json::parse(info); + relativize_lora_paths(result, root.lexically_normal()); + return result.dump(); +} + +std::string merge_generation_info(const std::string& source, const std::string& operation) { + if (source.empty()) + return operation; + auto result = nlohmann::json::parse(source); + const auto next = nlohmann::json::parse(operation); + result["postprocess"].push_back(next); + return result.dump(); +} + +int64_t generation_seed(const std::string& info, int64_t fallback) { + if (info.empty()) + return fallback; + const auto value = nlohmann::json::parse(info); + const auto& params = value.contains("inpaint_params") ? value.at("inpaint_params") : value; + return params.contains("seed") ? std::stoll(params.at("seed").get()) : fallback; +} + +std::string generation_info_to_webui(const std::string& text) { + using nlohmann::json; + if (text.empty()) + return ""; + const auto info = json::parse(text); + if (info.value("schema", "") != "sdcpp.generation/v1") + return ""; + // Standalone postprocessing has no source diffusion settings to import. + if (!info.contains("sample_params")) + return ""; + const auto& sampling = info.at("sample_params"); + std::string result = info.value("prompt", ""); + if (info.contains("loras")) { + for (const auto& lora : info.at("loras")) { + // WebUI tags cannot restrict a LoRA to the high-noise model. + if (lora.value("is_high_noise", false)) + continue; + auto name = lora.at("path").get(); + const auto separator = name.find_last_of("/\\"); + if (separator != std::string::npos) + name.erase(0, separator + 1); + const auto extension = name.find_last_of('.'); + if (extension != std::string::npos && extension > 0) + name.erase(extension); + if (name.empty() || name.find_first_of(":<>\r\n") != std::string::npos) + continue; + if (!result.empty()) + result += " "; + result += ""; + } + } + const auto negative = info.value("negative_prompt", ""); + if (!negative.empty()) + result += "\nNegative prompt: " + negative; + result += "\n"; + bool first = true; + const auto field = [&](const char* key, const json& value) { + std::string v = value.is_string() ? value.get() : value.dump(); + if (v.find_first_of(",\n\r:\"") != std::string::npos) + v = json(v).dump(-1, ' ', false); + if (!first) + result += ", "; + first = false; + result += std::string(key) + ": " + v; + }; + field("Steps", sampling.contains("custom_sigmas") ? json(sampling.at("custom_sigmas").size() - 1) : sampling.at("sample_steps")); + static const std::map samplers = { + {"euler", "Euler"}, {"euler_a", "Euler a"}, {"heun", "Heun"}, {"dpm2", "DPM2"}, {"dpm++2s_a", "DPM++ 2S a"}, {"dpm++2m", "DPM++ 2M"}, {"dpm++2m_sde", "DPM++ 2M SDE"}, {"lcm", "LCM"}}; + const auto method = sampling.at("sample_method").get(); + auto mapped = samplers.find(method); + field(mapped != samplers.end() ? "Sampler" : "SDCPP Sampler", mapped != samplers.end() ? mapped->second : method); + static const std::map schedules = { + {"discrete", "Normal"}, {"karras", "Karras"}, {"exponential", "Exponential"}, {"sgm_uniform", "SGM Uniform"}, {"simple", "Simple"}, {"ddim_uniform", "DDIM"}, {"beta", "Beta"}}; + if (sampling.contains("scheduler")) { + const auto scheduler = sampling.at("scheduler").get(); + auto mapped_schedule = schedules.find(scheduler); + field(mapped_schedule != schedules.end() ? "Schedule type" : "SDCPP Schedule", mapped_schedule != schedules.end() ? mapped_schedule->second : scheduler); + } + field("CFG scale", sampling.at("guidance").at("txt_cfg")); + field("Seed", info.at("seed")); + field("Size", std::to_string(info.at("width").get()) + "x" + std::to_string(info.at("height").get())); + if (info.contains("models")) { + const auto& models = info.at("models"); + auto model = models.value("diffusion_model", models.value("model", "")); + const auto extension = model.find_last_of('.'); + if (extension != std::string::npos && extension > 0) + model.erase(extension); + if (!model.empty()) + field("Model", model); + if (models.contains("vae")) + field("VAE", models.at("vae")); + } + if (sampling.contains("eta")) + field("Eta", sampling.at("eta")); + if (info.value("clip_skip", 0) > 0) + field("Clip skip", info.at("clip_skip")); + if (info.contains("strength")) + field("Denoising strength", info.at("strength")); + if (info.contains("hires") && info.at("hires").value("enabled", false)) { + const auto& hires = info.at("hires"); + field("Hires resize", std::to_string(hires.at("target_width").get()) + "x" + std::to_string(hires.at("target_height").get())); + if (hires.value("steps", 0) > 0) + field("Hires steps", hires.at("steps")); + field("Denoising strength", hires.at("denoising_strength")); + field("Hires upscaler", hires.at("upscaler")); + } + field("Version", "stable-diffusion.cpp"); + return result; +} + +std::string generation_info_to_image_metadata(const std::string& info) { + if (info.empty()) + return ""; + const auto parameters = generation_info_to_webui(info); + return parameters.empty() ? info : parameters + ", SDCPP: " + info; +} diff --git a/examples/common/generation_metadata.h b/examples/common/generation_metadata.h new file mode 100644 index 00000000..7c9e9ffd --- /dev/null +++ b/examples/common/generation_metadata.h @@ -0,0 +1,13 @@ +#ifndef __EXAMPLES_COMMON_GENERATION_METADATA_H__ +#define __EXAMPLES_COMMON_GENERATION_METADATA_H__ + +#include +#include + +std::string merge_generation_info(const std::string& source, const std::string& operation); +std::string generation_info_with_relative_lora_paths(const std::string& info, const std::string& lora_model_dir); +int64_t generation_seed(const std::string& info, int64_t fallback = -1); +std::string generation_info_to_webui(const std::string& text); +std::string generation_info_to_image_metadata(const std::string& info); + +#endif // __EXAMPLES_COMMON_GENERATION_METADATA_H__ diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp index baac6d20..f09c2166 100644 --- a/examples/common/media_io.cpp +++ b/examples/common/media_io.cpp @@ -1,5 +1,7 @@ #include "media_io.h" +#include "generation_metadata.h" #include "log.h" +#include "media_metadata.h" #include "resource_owners.hpp" #include @@ -316,23 +318,22 @@ uint8_t* decode_webp_image_to_buffer(const uint8_t* data, return output.release(); } -std::string build_webp_xmp_packet(const std::string& parameters) { - if (parameters.empty()) { - return ""; - } - - const std::string escaped_parameters = xml_escape(parameters); - return "\n" - "\n" - " \n" - " \n" - " " + - escaped_parameters + - "\n" - " \n" - " \n" - "\n" - ""; +static bool embed_webp_metadata(std::vector& image, const std::string& parameters) { + if (parameters.empty()) + return true; + WebPData data{image.data(), image.size()}; + WebPMuxPtr mux(WebPMuxCreate(&data, 1)); + if (!mux) + return false; + const auto xmp = generation_xmp(parameters); + WebPData xmp_data{reinterpret_cast(xmp.data()), xmp.size()}; + if (WebPMuxSetChunk(mux.get(), "XMP ", &xmp_data, 1) != WEBP_MUX_OK) + return false; + WebPDataGuard assembled; + if (WebPMuxAssemble(mux.get(), &assembled.data) != WEBP_MUX_OK) + return false; + image.assign(assembled.data.bytes, assembled.data.bytes + assembled.data.size); + return true; } bool encode_webp_image_to_vector(const uint8_t* image, @@ -376,37 +377,7 @@ bool encode_webp_image_to_vector(const uint8_t* image, out.assign(encoded.get(), encoded.get() + encoded_size); - if (parameters.empty()) { - return true; - } - - WebPData image_data; - WebPDataInit(&image_data); - WebPDataGuard assembled_data; - - image_data.bytes = out.data(); - image_data.size = out.size(); - - WebPMuxPtr mux(WebPMuxNew()); - if (mux == nullptr) { - return false; - } - - const std::string xmp_packet = build_webp_xmp_packet(parameters); - WebPData xmp_data; - WebPDataInit(&xmp_data); - xmp_data.bytes = reinterpret_cast(xmp_packet.data()); - xmp_data.size = xmp_packet.size(); - - const bool ok = WebPMuxSetImage(mux.get(), &image_data, 1) == WEBP_MUX_OK && - WebPMuxSetChunk(mux.get(), "XMP ", &xmp_data, 1) == WEBP_MUX_OK && - WebPMuxAssemble(mux.get(), &assembled_data.data) == WEBP_MUX_OK; - - if (ok) { - out.assign(assembled_data.data.bytes, assembled_data.data.bytes + assembled_data.data.size); - } - - return ok; + return embed_webp_metadata(out, generation_info_to_image_metadata(parameters)); } #ifdef SD_USE_WEBM @@ -739,6 +710,7 @@ std::vector encode_image_to_vector(EncodedImageFormat format, const std::string& parameters, int quality) { std::vector buffer; + const auto metadata = format == EncodedImageFormat::WEBP ? std::string() : generation_info_to_image_metadata(parameters); auto write_func = [&buffer](void* context, void* data, int size) { (void)context; @@ -761,7 +733,7 @@ std::vector encode_image_to_vector(EncodedImageFormat format, result = stbi_write_jpg_to_func(c_func, &ctx, width, height, channels, image, quality); break; case EncodedImageFormat::PNG: - result = stbi_ext_write_png_to_func(c_func, &ctx, width, height, channels, image, width * channels, parameters.empty() ? nullptr : parameters.c_str()); + result = stbi_ext_write_png_to_func(c_func, &ctx, width, height, channels, image, width * channels, metadata.empty() ? nullptr : metadata.c_str()); break; case EncodedImageFormat::WEBP: #ifdef SD_USE_WEBP @@ -779,9 +751,12 @@ std::vector encode_image_to_vector(EncodedImageFormat format, break; } - if (!result) { - buffer.clear(); + if (result && format == EncodedImageFormat::JPEG && !embed_jpeg_metadata(buffer, metadata)) { + LOG_ERROR("generation metadata exceeds the container capacity or the encoded image is invalid"); + result = 0; } + if (!result) + buffer.clear(); return buffer; } @@ -794,18 +769,8 @@ bool write_image_to_file(const std::string& path, int quality) { const EncodedImageFormat format = encoded_image_format_from_path(path); - switch (format) { - case EncodedImageFormat::JPEG: - return stbi_write_jpg(path.c_str(), width, height, channels, image, quality, parameters.empty() ? nullptr : parameters.c_str()) != 0; - case EncodedImageFormat::PNG: - return stbi_write_png(path.c_str(), width, height, channels, image, 0, parameters.empty() ? nullptr : parameters.c_str()) != 0; - case EncodedImageFormat::WEBP: { - const std::vector encoded = encode_image_to_vector(format, image, width, height, channels, parameters, quality); - return !encoded.empty() && write_binary_file_bytes(path, encoded); - } - default: - return false; - } + const auto encoded = encode_image_to_vector(format, image, width, height, channels, parameters, quality); + return !encoded.empty() && write_binary_file_bytes(path, encoded); } uint8_t* load_image_from_file(const char* image_path, @@ -1384,7 +1349,10 @@ std::vector create_video_from_sd_images_to_vector(const std::string& ou #ifdef SD_USE_WEBP if (format == "webp") { - return create_animated_webp_from_sd_images_to_vector(images, num_images, fps, quality); + auto data = create_animated_webp_from_sd_images_to_vector(images, num_images, fps, quality); + if (!data.empty() && !embed_webp_metadata(data, parameters)) + return {}; + return data; } #endif diff --git a/examples/common/media_io.h b/examples/common/media_io.h index b2b56ecc..18042a8b 100644 --- a/examples/common/media_io.h +++ b/examples/common/media_io.h @@ -16,6 +16,7 @@ enum class EncodedImageFormat { EncodedImageFormat encoded_image_format_from_path(const std::string& path); +// parameters is the core sdcpp.generation/v1 JSON; empty disables both metadata formats. std::vector encode_image_to_vector(EncodedImageFormat format, const uint8_t* image, int width, diff --git a/examples/common/media_metadata.cpp b/examples/common/media_metadata.cpp new file mode 100644 index 00000000..d5e9af3f --- /dev/null +++ b/examples/common/media_metadata.cpp @@ -0,0 +1,47 @@ +#include "media_metadata.h" + +std::string generation_xmp(const std::string& info) { + std::string escaped; + for (char c : info) { + switch (c) { + case '&': + escaped += "&"; + break; + case '<': + escaped += "<"; + break; + case '>': + escaped += ">"; + break; + case '"': + escaped += """; + break; + case '\'': + escaped += "'"; + break; + default: + escaped += c; + break; + } + } + return "" + "" + "" + + escaped + ""; +} + +bool embed_jpeg_metadata(std::vector& image, const std::string& parameters) { + if (parameters.empty()) + return true; + const char label[] = "parameters"; + const size_t overhead = sizeof(label) + 1; + if (image.size() < 2 || parameters.size() > 65533 - overhead) + return false; + const size_t length = overhead + parameters.size() + 2; + std::vector segment = {0xff, 0xfe, static_cast(length >> 8), static_cast(length)}; + segment.insert(segment.end(), label, label + sizeof(label)); + segment.insert(segment.end(), parameters.begin(), parameters.end()); + segment.push_back(0); + image.insert(image.begin() + 2, segment.begin(), segment.end()); + return true; +} diff --git a/examples/common/media_metadata.h b/examples/common/media_metadata.h new file mode 100644 index 00000000..1843dee1 --- /dev/null +++ b/examples/common/media_metadata.h @@ -0,0 +1,11 @@ +#ifndef __EXAMPLES_COMMON_MEDIA_METADATA_H__ +#define __EXAMPLES_COMMON_MEDIA_METADATA_H__ + +#include +#include +#include + +std::string generation_xmp(const std::string& parameters); +bool embed_jpeg_metadata(std::vector& image, const std::string& parameters); + +#endif // __EXAMPLES_COMMON_MEDIA_METADATA_H__ diff --git a/examples/common/resource_owners.hpp b/examples/common/resource_owners.hpp index 73f8bdd8..8a8f8431 100644 --- a/examples/common/resource_owners.hpp +++ b/examples/common/resource_owners.hpp @@ -8,6 +8,7 @@ #include #include +#include "generation_metadata.h" #include "stable-diffusion.h" struct FreeDeleter { @@ -175,6 +176,7 @@ public: class SDImageVec { private: std::vector images_; + std::vector generation_infos_; public: SDImageVec() = default; @@ -183,13 +185,14 @@ public: SDImageVec& operator=(const SDImageVec&) = delete; SDImageVec(SDImageVec&& other) noexcept - : images_(std::move(other.images_)) { + : images_(std::move(other.images_)), generation_infos_(std::move(other.generation_infos_)) { } SDImageVec& operator=(SDImageVec&& other) noexcept { if (this != &other) { clear(); - images_ = std::move(other.images_); + images_ = std::move(other.images_); + generation_infos_ = std::move(other.generation_infos_); } return *this; } @@ -210,6 +213,41 @@ public: images_.reserve(count); } + const std::string& generation_info(size_t index) const { + static const std::string empty; + return index < generation_infos_.size() ? generation_infos_[index] : empty; + } + + void append_generation_info(size_t index, const std::string& info, const std::string& lora_model_dir = {}) { + generation_infos_.resize(images_.size()); + generation_infos_.at(index) = merge_generation_info(generation_infos_.at(index), generation_info_with_relative_lora_paths(info, lora_model_dir)); + } + + void adopt(sd_image_generation_result_t* result, const std::string& lora_model_dir = {}) { + clear(); + if (!result) + return; + for (int i = 0; i < result->count; ++i) { + images_.push_back(result->items[i].image); + result->items[i].image = {}; + generation_infos_.push_back(generation_info_with_relative_lora_paths(result->items[i].generation_info, lora_model_dir)); + } + free_sd_image_generation_result(result); + } + + void adopt_video(sd_video_generation_result_t* result, const std::string& lora_model_dir = {}) { + clear(); + if (!result) + return; + const auto info = generation_info_with_relative_lora_paths(result->generation_info, lora_model_dir); + for (int i = 0; i < result->frame_count; ++i) { + images_.push_back(result->frames[i]); + result->frames[i] = {}; + generation_infos_.push_back(info); + } + free_sd_video_generation_result(result); + } + void adopt(sd_image_t* images, int count) { clear(); if (images == nullptr || count <= 0) { @@ -270,6 +308,7 @@ public: image.data = nullptr; } images_.clear(); + generation_infos_.clear(); } }; diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt index 58260be0..d01f7a7e 100644 --- a/examples/server/CMakeLists.txt +++ b/examples/server/CMakeLists.txt @@ -64,8 +64,10 @@ endif() add_executable(${TARGET} ../common/common.cpp + ../common/generation_metadata.cpp ../common/log.cpp ../common/media_io.cpp + ../common/media_metadata.cpp main.cpp runtime.cpp model_catalog.cpp diff --git a/examples/server/api.md b/examples/server/api.md index 0d83214e..1258b1af 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -1776,3 +1776,14 @@ Example cancelled job: - `400 Bad Request` for an empty body, unsupported model mode, invalid JSON, invalid generation parameters, or an unsupported output format - `429 Too Many Requests` when the job queue is full - `500 Internal Server Error` for unexpected server exceptions during submission + +## Actual generation parameters + +Generation responses expose the core `sdcpp.generation/v1` record as +`generation_info`, independently of `embed_image_metadata`. Native async image +result items and OpenAI image items contain the parameter snapshot with each +image's actual seed and without `batch_count`. Native video +results contain the video request snapshot. SD API `info.generation_info` is an array +aligned with `info.infotexts` and `info.all_seeds`; the outer `parameters` field +remains the request echo. See [generation metadata](../../docs/generation_metadata.md) +for field semantics, postprocessing and container storage. diff --git a/examples/server/async_jobs.cpp b/examples/server/async_jobs.cpp index 943316e9..7a801078 100644 --- a/examples/server/async_jobs.cpp +++ b/examples/server/async_jobs.cpp @@ -179,6 +179,7 @@ bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job) { job.status = AsyncJobStatus::Cancelled; job.completed_at = unix_timestamp_now_ms(); job.result_images_b64.clear(); + job.result_generation_info = nullptr; job.result_media_b64.clear(); job.result_media_mime_type.clear(); job.result_frame_count = 0; @@ -218,11 +219,12 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo {"fps", job.result_fps}, {"frame_count", job.result_frame_count}, {"b64_json", job.result_media_b64}, + {"generation_info", job.result_generation_info}, }; } else { json images = json::array(); for (size_t i = 0; i < job.result_images_b64.size(); ++i) { - images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}}); + images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}, {"generation_info", job.result_generation_info.at(i)}}); } result["result"] = { {"output_format", job.img_gen.output_format}, @@ -263,7 +265,9 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo bool execute_img_gen_job(ServerRuntime& runtime, AsyncGenerationJob& job, std::vector& output_images, + json& output_info, std::string& error_message) { + output_info = json::array(); sd_img_gen_params_t params = job.img_gen.to_sd_img_gen_params_t(); SDImageVec results; @@ -275,14 +279,10 @@ bool execute_img_gen_job(ServerRuntime& runtime, } JobCancellationMonitor cancellation(runtime.sd_ctx, job); set_preview_callback(job); - sd_image_t* raw_results = nullptr; - int num_results = 0; - if (!generate_image(runtime.sd_ctx, ¶ms, &raw_results, &num_results)) { - raw_results = nullptr; - num_results = 0; - } + sd_image_generation_result_t* generated = nullptr; + generate_image(runtime.sd_ctx, ¶ms, &generated); + results.adopt(generated, runtime.ctx_params->lora_model_dir); clear_preview_callback(); - results.adopt(raw_results, num_results); if (job.cancel_requested.load()) { return false; } @@ -305,17 +305,13 @@ bool execute_img_gen_job(ServerRuntime& runtime, encoded_format = EncodedImageFormat::WEBP; } - int batch_count = job.img_gen.gen_params.batch_count; - int images_per_batch = batch_count > 0 ? std::max(1, num_results / batch_count) : 1; for (int i = 0; i < num_results; ++i) { if (results[i].data == nullptr) { continue; } const std::string metadata = job.img_gen.gen_params.embed_image_metadata - ? get_image_params(*runtime.ctx_params, - job.img_gen.gen_params, - job.img_gen.gen_params.seed + i / images_per_batch) + ? results.generation_info(i) : ""; auto image_bytes = encode_image_to_vector(encoded_format, results[i].data, @@ -328,6 +324,7 @@ bool execute_img_gen_job(ServerRuntime& runtime, continue; } output_images.push_back(base64_encode(image_bytes)); + output_info.push_back(json::parse(results.generation_info(i))); } if (output_images.empty()) { @@ -344,11 +341,10 @@ bool execute_vid_gen_job(ServerRuntime& runtime, std::string& output_media_mime_type, int& output_frame_count, int& output_fps, + json& output_info, std::string& error_message) { sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t(); - std::string str_params = job.vid_gen.gen_params.embed_image_metadata - ? get_image_params(*runtime.ctx_params, job.vid_gen.gen_params, job.vid_gen.gen_params.seed, VID_GEN) - : ""; + std::string str_params; SDImageVec results; int num_results = 0; @@ -361,12 +357,17 @@ bool execute_vid_gen_job(ServerRuntime& runtime, } JobCancellationMonitor cancellation(runtime.sd_ctx, job); set_preview_callback(job); - sd_image_t* raw_results = nullptr; - if (!generate_video(runtime.sd_ctx, ¶ms, &raw_results, &num_results, &generated_audio, &output_fps)) { - raw_results = nullptr; + sd_video_generation_result_t* generated = nullptr; + if (generate_video(runtime.sd_ctx, ¶ms, &generated)) { + output_fps = generated->fps; + generated_audio = generated->audio; + generated->audio = nullptr; + results.adopt_video(generated, runtime.ctx_params->lora_model_dir); + output_info = json::parse(results.generation_info(0)); + if (job.vid_gen.gen_params.embed_image_metadata) + str_params = results.generation_info(0); } clear_preview_callback(); - results.adopt(raw_results, num_results); } num_results = results.count(); @@ -427,6 +428,7 @@ void async_job_worker(ServerRuntime& runtime) { } std::vector output_images; + json output_info; std::string output_media_b64; std::string output_media_mime_type; int output_frame_count = 0; @@ -435,7 +437,7 @@ void async_job_worker(ServerRuntime& runtime) { bool ok = false; if (job->kind == AsyncJobKind::ImgGen) { - ok = execute_img_gen_job(runtime, *job, output_images, error_message); + ok = execute_img_gen_job(runtime, *job, output_images, output_info, error_message); } else if (job->kind == AsyncJobKind::VidGen) { ok = execute_vid_gen_job(runtime, *job, @@ -443,6 +445,7 @@ void async_job_worker(ServerRuntime& runtime) { output_media_mime_type, output_frame_count, output_fps, + output_info, error_message); } else { error_message = "unsupported job kind"; @@ -463,6 +466,7 @@ void async_job_worker(ServerRuntime& runtime) { } else if (ok) { job->status = AsyncJobStatus::Completed; job->result_images_b64 = std::move(output_images); + job->result_generation_info = std::move(output_info); job->result_media_b64 = std::move(output_media_b64); job->result_media_mime_type = std::move(output_media_mime_type); job->result_frame_count = output_frame_count; @@ -474,6 +478,7 @@ void async_job_worker(ServerRuntime& runtime) { job->error_code = "generation_failed"; job->error_message = error_message.empty() ? "unknown generation error" : error_message; job->result_images_b64.clear(); + job->result_generation_info = nullptr; job->result_media_b64.clear(); job->result_media_mime_type.clear(); job->result_frame_count = 0; diff --git a/examples/server/async_jobs.h b/examples/server/async_jobs.h index aa54370b..6fa3cbd1 100644 --- a/examples/server/async_jobs.h +++ b/examples/server/async_jobs.h @@ -40,6 +40,7 @@ struct AsyncGenerationJob { ImgGenJobRequest img_gen; VidGenJobRequest vid_gen; std::vector result_images_b64; + json result_generation_info; std::string result_media_b64; std::string result_media_mime_type; int result_frame_count = 0; @@ -74,6 +75,7 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo bool execute_img_gen_job(ServerRuntime& runtime, AsyncGenerationJob& job, std::vector& output_images, + json& output_info, std::string& error_message); bool execute_vid_gen_job(ServerRuntime& runtime, AsyncGenerationJob& job, @@ -81,5 +83,6 @@ bool execute_vid_gen_job(ServerRuntime& runtime, std::string& output_media_mime_type, int& output_frame_count, int& output_fps, + json& output_info, std::string& error_message); void async_job_worker(ServerRuntime& runtime); diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp index 409c88de..f721e149 100644 --- a/examples/server/routes_openai.cpp +++ b/examples/server/routes_openai.cpp @@ -223,12 +223,10 @@ static bool execute_sync_img_gen_request(ServerRuntime& runtime, { std::lock_guard lock(*runtime.sd_ctx_mutex); - sd_image_t* raw_results = nullptr; - if (!generate_image(runtime.sd_ctx, &img_gen_params, &raw_results, &num_results)) { - raw_results = nullptr; - num_results = 0; - } - results.adopt(raw_results, num_results); + sd_image_generation_result_t* generated = nullptr; + generate_image(runtime.sd_ctx, &img_gen_params, &generated); + results.adopt(generated, runtime.ctx_params->lora_model_dir); + num_results = results.count(); if (!results.empty() && !apply_server_adetailer(runtime, request, img_gen_params, results)) { error_message = "ADetailer failed"; return false; @@ -285,16 +283,14 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { out["data"] = json::array(); out["output_format"] = request.output_format; - int result_count = results.count(); - int images_per_batch = request.gen_params.batch_count > 0 ? std::max(1, result_count / request.gen_params.batch_count) : 1; + int result_count = results.count(); + for (int i = 0; i < result_count; ++i) { if (results[i].data == nullptr) { continue; } std::string params = request.gen_params.embed_image_metadata - ? get_image_params(*runtime->ctx_params, - request.gen_params, - request.gen_params.seed + i / images_per_batch) + ? results.generation_info(i) : ""; auto image_bytes = encode_image_to_vector(request.output_format == "jpeg" ? EncodedImageFormat::JPEG @@ -313,7 +309,8 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { } json item; - item["b64_json"] = base64_encode(image_bytes); + item["b64_json"] = base64_encode(image_bytes); + item["generation_info"] = json::parse(results.generation_info(i)); out["data"].push_back(item); } @@ -359,16 +356,14 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { out["data"] = json::array(); out["output_format"] = request.output_format; - int result_count = results.count(); - int images_per_batch = request.gen_params.batch_count > 0 ? std::max(1, result_count / request.gen_params.batch_count) : 1; + int result_count = results.count(); + for (int i = 0; i < result_count; ++i) { if (results[i].data == nullptr) { continue; } std::string params = request.gen_params.embed_image_metadata - ? get_image_params(*runtime->ctx_params, - request.gen_params, - request.gen_params.seed + i / images_per_batch) + ? results.generation_info(i) : ""; auto image_bytes = encode_image_to_vector(request.output_format == "jpeg" ? EncodedImageFormat::JPEG : EncodedImageFormat::PNG, results[i].data, @@ -378,7 +373,8 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { params, request.output_compression); json item; - item["b64_json"] = base64_encode(image_bytes); + item["b64_json"] = base64_encode(image_bytes); + item["generation_info"] = json::parse(results.generation_info(i)); out["data"].push_back(item); } diff --git a/examples/server/routes_sdapi.cpp b/examples/server/routes_sdapi.cpp index ef9a6feb..0d37898a 100644 --- a/examples/server/routes_sdapi.cpp +++ b/examples/server/routes_sdapi.cpp @@ -462,46 +462,27 @@ static bool build_sdapi_img_gen_request(const json& j, validate_server_img_gen_request(runtime, request, error_message); } -static nlohmann::json prepare_info_field(const SDContextParams& ctx_params, - const SDGenerationParams& gen_params, - bool img2img) { - nlohmann::json jsoninfo = nlohmann::json::object(); - jsoninfo["prompt"] = gen_params.prompt; - if (!gen_params.negative_prompt.empty()) { - jsoninfo["negative_prompt"] = gen_params.negative_prompt; - } - jsoninfo["seed"] = gen_params.seed; - jsoninfo["cfg_scale"] = gen_params.sample_params.guidance.txt_cfg; - jsoninfo["width"] = gen_params.get_resolved_width(); - jsoninfo["height"] = gen_params.get_resolved_height(); - jsoninfo["steps"] = gen_params.sample_params.sample_steps; - jsoninfo["sampler_name"] = sd_sample_method_name(gen_params.sample_params.sample_method); - if (gen_params.clip_skip != -1) { - jsoninfo["clip_skip"] = gen_params.clip_skip; - } - if (gen_params.sample_params.scheduler != scheduler_t::SCHEDULER_COUNT) { - jsoninfo["extra_generation_params"] = nlohmann::json::object(); - jsoninfo["extra_generation_params"]["Schedule type"] = sd_scheduler_name(gen_params.sample_params.scheduler); - } - if (img2img) { - jsoninfo["denoising_strength"] = gen_params.strength; - } - // not clear what should happen if we have both model and diffusion_model - if (!ctx_params.diffusion_model_path.empty()) { - jsoninfo["sd_model_name"] = sd_basename(ctx_params.diffusion_model_path); - } else if (!ctx_params.model_path.empty()) { - jsoninfo["sd_model_name"] = sd_basename(ctx_params.model_path); - } - if (!ctx_params.vae_path.empty()) { - jsoninfo["sd_vae_name"] = sd_basename(ctx_params.vae_path); - } - jsoninfo["version"] = "stable-diffusion.cpp"; - - jsoninfo["infotexts"] = nlohmann::json::array(); - jsoninfo["all_prompts"] = nlohmann::json::array(); - jsoninfo["all_negative_prompts"] = nlohmann::json::array(); - jsoninfo["all_seeds"] = nlohmann::json::array(); - return jsoninfo; +static nlohmann::json prepare_info_field(const std::string& generation_info) { + const auto info = nlohmann::json::parse(generation_info); + const auto& sampling = info.at("sample_params"); + nlohmann::json result = {{"prompt", info.at("prompt")}, + {"negative_prompt", info.at("negative_prompt")}, + {"seed", generation_seed(generation_info)}, + {"cfg_scale", sampling.at("guidance").at("txt_cfg")}, + {"width", info.at("width")}, + {"height", info.at("height")}, + {"steps", sampling.contains("custom_sigmas") ? json(sampling.at("custom_sigmas").size() - 1) : sampling.at("sample_steps")}, + {"sampler_name", sampling.at("sample_method")}, + {"version", "stable-diffusion.cpp"}}; + if (info.contains("clip_skip")) + result["clip_skip"] = info.at("clip_skip"); + if (info.contains("strength")) + result["denoising_strength"] = info.at("strength"); + if (sampling.contains("scheduler")) + result["extra_generation_params"]["Schedule type"] = sampling.at("scheduler"); + for (const auto* key : {"infotexts", "all_prompts", "all_negative_prompts", "all_seeds", "generation_info"}) + result[key] = nlohmann::json::array(); + return result; } struct SDAPIPreviewState { @@ -621,12 +602,10 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { std::lock_guard lock(*runtime->sd_ctx_mutex); const bool detailing = request.adetailer_enabled && !runtime->default_gen_params->ad_model_path.empty(); SDAPIPreviewGuard preview_guard(id_task, img_gen_params, img2img ? "img2img" : "txt2img", detailing); - sd_image_t* raw_results = nullptr; - if (!generate_image(runtime->sd_ctx, &img_gen_params, &raw_results, &num_results)) { - raw_results = nullptr; - num_results = 0; - } - results.adopt(raw_results, num_results); + sd_image_generation_result_t* generated = nullptr; + generate_image(runtime->sd_ctx, &img_gen_params, &generated); + results.adopt(generated, runtime->ctx_params->lora_model_dir); + num_results = results.count(); if (detailing && !results.empty()) preview_guard.begin_detailing(); if (!results.empty() && !apply_server_adetailer(*runtime, request, img_gen_params, results)) { @@ -645,9 +624,8 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { json out; out["images"] = json::array(); out["parameters"] = j; - json jsoninfo = prepare_info_field(*runtime->ctx_params, request.gen_params, img2img); + json jsoninfo = prepare_info_field(results.generation_info(0)); - int images_per_batch = request.gen_params.batch_count > 0 ? std::max(1, num_results / request.gen_params.batch_count) : 1; for (int i = 0; i < num_results; ++i) { if (results[i].data == nullptr) { continue; @@ -655,16 +633,14 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { bool embed_meta = request.gen_params.embed_image_metadata; - std::string params = get_image_params(*runtime->ctx_params, - request.gen_params, - request.gen_params.seed + i / images_per_batch); + std::string params = generation_info_to_webui(results.generation_info(i)); auto image_bytes = encode_image_to_vector(EncodedImageFormat::PNG, results[i].data, results[i].width, results[i].height, results[i].channel, - embed_meta ? params : ""); + embed_meta ? results.generation_info(i) : ""); if (image_bytes.empty()) { LOG_ERROR("write image to mem failed"); @@ -673,10 +649,12 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { out["images"].push_back(base64_encode(image_bytes)); - jsoninfo["infotexts"][i] = params; - jsoninfo["all_seeds"][i] = request.gen_params.seed + i; - jsoninfo["all_prompts"][i] = request.gen_params.prompt; - jsoninfo["all_negative_prompts"][i] = request.gen_params.negative_prompt; + const auto actual = json::parse(results.generation_info(i)); + jsoninfo["infotexts"].push_back(params); + jsoninfo["all_seeds"].push_back(generation_seed(results.generation_info(i))); + jsoninfo["generation_info"].push_back(actual); + jsoninfo["all_prompts"].push_back(actual["prompt"]); + jsoninfo["all_negative_prompts"].push_back(actual["negative_prompt"]); } // not a mistake: it is supposed to be a string in json format diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp index 29eed7e1..b972768b 100644 --- a/examples/server/routes_sdcpp.cpp +++ b/examples/server/routes_sdcpp.cpp @@ -609,19 +609,20 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { } SDImageOwner current(input.release()); + std::string generation_info; for (int i = 0; i < repeats; ++i) { - sd_image_t* out_images = nullptr; - int out_count = 0; - if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &out_images, &out_count) || - out_count <= 0 || out_images[0].data == nullptr) { - free_sd_images(out_images, out_count); + sd_image_generation_result_t* generated = nullptr; + if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &generated) || + !generated || generated->count <= 0 || generated->items[0].image.data == nullptr) { + free_sd_image_generation_result(generated); res.status = 500; res.set_content(R"({"error":"upscale failed"})", "application/json"); return; } - sd_image_t produced = out_images[0]; - out_images[0] = {0, 0, 0, nullptr}; - free_sd_images(out_images, out_count); + generation_info = merge_generation_info(generation_info, generated->items[0].generation_info); + sd_image_t produced = generated->items[0].image; + generated->items[0].image = {}; + free_sd_image_generation_result(generated); current.reset(produced); } @@ -635,7 +636,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { result.width, result.height, result.channel, - "", + generation_info, compression); if (image_bytes.empty()) { res.status = 500; @@ -644,13 +645,14 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { } json out; - out["upscaler"] = used_name; - out["scale"] = factor; - out["repeats"] = repeats; - out["width"] = result.width; - out["height"] = result.height; - out["output_format"] = format; - json images = json::array(); + out["generation_info"] = json::parse(generation_info); + out["upscaler"] = used_name; + out["scale"] = factor; + out["repeats"] = repeats; + out["width"] = result.width; + out["height"] = result.height; + out["output_format"] = format; + json images = json::array(); images.push_back({{"index", 0}, {"b64_json", base64_encode(image_bytes)}}); out["images"] = std::move(images); res.set_content(out.dump(), "application/json"); diff --git a/examples/server/runtime.cpp b/examples/server/runtime.cpp index 5b33b338..b187ff85 100644 --- a/examples/server/runtime.cpp +++ b/examples/server/runtime.cpp @@ -136,7 +136,7 @@ bool apply_server_adetailer(ServerRuntime& runtime, continue; } sd_img_gen_params_t ad_generation_params = params; - ad_generation_params.seed = params.seed + i; + ad_generation_params.seed = generation_seed(results.generation_info(i)); ad_generation_params.width = pass.width; ad_generation_params.height = pass.height; ad_generation_params.strength = pass.strength; @@ -147,24 +147,23 @@ bool apply_server_adetailer(ServerRuntime& runtime, ad_generation_params.sample_params.scheduler = SCHEDULER_COUNT; } - sd_image_t* detailed_images = nullptr; - int detailed_count = 0; + sd_image_generation_result_t* detailed = nullptr; if (!adetail_image(ad_ctx.get(), runtime.sd_ctx, results[i], &ad_params, &ad_generation_params, - &detailed_images, - &detailed_count) || - detailed_count <= 0 || detailed_images == nullptr || detailed_images[0].data == nullptr) { - free_sd_images(detailed_images, detailed_count); + &detailed) || + detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) { + free_sd_image_generation_result(detailed); LOG_ERROR("ADetailer failed for image %d", i + 1); return false; } + results.append_generation_info(i, detailed->items[0].generation_info, runtime.ctx_params->lora_model_dir); free(results[i].data); - results[i] = detailed_images[0]; - detailed_images[0] = {0, 0, 0, nullptr}; - free_sd_images(detailed_images, detailed_count); + results[i] = detailed->items[0].image; + detailed->items[0].image = {}; + free_sd_image_generation_result(detailed); } } return true; diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index d0292619..f4ea0eaf 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -272,6 +272,25 @@ typedef struct { uint8_t* data; } sd_image_t; +typedef struct { + sd_image_t image; + // Owned UTF-8 JSON using the sdcpp.generation/v1 schema. + char* generation_info; +} sd_image_result_t; + +typedef struct { + sd_image_result_t* items; + int count; +} sd_image_generation_result_t; + +typedef struct { + sd_image_t* frames; + int frame_count; + sd_audio_t* audio; + int fps; + char* generation_info; +} sd_video_generation_result_t; + typedef struct { // Semicolon-separated target=...,key=value rules. NULL preserves defaults. const char* rules; @@ -536,8 +555,12 @@ SD_API void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params); SD_API char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params); SD_API bool generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* sd_img_gen_params, - sd_image_t** images_out, - int* num_images_out); + sd_image_generation_result_t** result_out); + +// Results own their pixels, audio and generation_info, independently of the context. +// Generation calls require result_out and set it to NULL on failure. +SD_API void free_sd_image_generation_result(sd_image_generation_result_t* result); +SD_API void free_sd_video_generation_result(sd_video_generation_result_t* result); enum sd_cancel_mode_t { // Stop the current generation as soon as possible. @@ -551,13 +574,9 @@ enum sd_cancel_mode_t { SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode); SD_API void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params); -// If non-NULL, fps_out receives the effective encoding frame rate before preview callbacks. SD_API bool generate_video(sd_ctx_t* sd_ctx, const sd_vid_gen_params_t* sd_vid_gen_params, - sd_image_t** frames_out, - int* num_frames_out, - sd_audio_t** audio_out, - int* fps_out); + sd_video_generation_result_t** result_out); typedef struct upscaler_ctx_t upscaler_ctx_t; @@ -572,8 +591,7 @@ SD_API void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx); SD_API bool upscale(upscaler_ctx_t* upscaler_ctx, sd_image_t input_image, uint32_t upscale_factor, - sd_image_t** images_out, - int* num_images_out); + sd_image_generation_result_t** result_out); SD_API int get_upscale_factor(upscaler_ctx_t* upscaler_ctx); // Reads model metadata only; returns 0 if the file is not a recognized RGB ESRGAN model. @@ -597,8 +615,7 @@ SD_API bool adetail_image(adetailer_ctx_t* adetailer_ctx, sd_image_t input_image, const sd_adetailer_params_t* adetailer_params, const sd_img_gen_params_t* inpaint_params, - sd_image_t** images_out, - int* num_images_out); + sd_image_generation_result_t** result_out); SD_API bool convert(const char* input_path, const char* vae_path, diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index c7ef70f5..e3832a1d 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -141,6 +141,7 @@ struct ConditionerParams { }; struct Conditioner { + virtual int resolve_clip_skip(int) const { return 0; } virtual ~Conditioner() = default; public: @@ -173,6 +174,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { std::vector token_embed_custom2; std::map> embedding_pos_map; + int resolve_clip_skip(int value) const override { + return value > 0 ? value : ((sd_version_is_sd2(version) || sd_version_is_sdxl(version)) ? 2 : 1); + } + FrozenCLIPEmbedderWithCustomWords(ggml_backend_t backend, const String2TensorStorage& tensor_storage_map, const std::map& orig_embedding_map, @@ -456,10 +461,6 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { sd::Tensor hidden_states; // [n_token, hidden_size] or [n_token, hidden_size + hidden_size2] sd::Tensor pooled; - if (clip_skip <= 0) { - clip_skip = (sd_version_is_sd2(version) || sd_version_is_sdxl(version)) ? 2 : 1; - } - size_t chunk_len = 77; size_t chunk_count = tokens.size() / chunk_len; for (int chunk_idx = 0; chunk_idx < chunk_count; chunk_idx++) { @@ -651,6 +652,8 @@ struct SD3CLIPEmbedder : public Conditioner { std::shared_ptr clip_g; std::shared_ptr t5; + int resolve_clip_skip(int value) const override { return value > 0 ? value : 2; } + SD3CLIPEmbedder(ggml_backend_t backend, const String2TensorStorage& tensor_storage_map = {}, std::shared_ptr weight_manager = nullptr, @@ -898,10 +901,6 @@ struct SD3CLIPEmbedder : public Conditioner { auto& t5_tokens = token_and_weights[2].first; auto& t5_weights = token_and_weights[2].second; - if (clip_skip <= 0) { - clip_skip = 2; - } - size_t chunk_len = 77; int64_t t0 = ggml_time_ms(); sd::Tensor hidden_states; @@ -1097,6 +1096,8 @@ struct FluxCLIPEmbedder : public Conditioner { std::shared_ptr t5; size_t chunk_len = 256; + int resolve_clip_skip(int value) const override { return value > 0 ? value : 2; } + FluxCLIPEmbedder(ggml_backend_t backend, const String2TensorStorage& tensor_storage_map = {}, std::shared_ptr weight_manager = nullptr, @@ -1293,10 +1294,6 @@ struct FluxCLIPEmbedder : public Conditioner { auto& t5_tokens = token_and_weights[1].first; auto& t5_weights = token_and_weights[1].second; - if (clip_skip <= 0) { - clip_skip = 2; - } - int64_t t0 = ggml_time_ms(); sd::Tensor hidden_states; // [N, n_token, 4096] sd::Tensor pooled; // [768,] diff --git a/src/detailer.cpp b/src/detailer.cpp index ec4d6065..9c48ca47 100644 --- a/src/detailer.cpp +++ b/src/detailer.cpp @@ -1,4 +1,6 @@ #include "detailer.h" +#include "generation_info.h" +#include "pipeline/request.h" #include #include @@ -915,14 +917,11 @@ bool adetail_image(adetailer_ctx_t* context, sd_image_t input_image, const sd_adetailer_params_t* public_params, const sd_img_gen_params_t* inpaint_params, - sd_image_t** images_out, - int* num_images_out) { - if (images_out != nullptr) { - *images_out = nullptr; - } - if (num_images_out != nullptr) { - *num_images_out = 0; - } + sd_image_generation_result_t** result_out) try { + if (result_out) + *result_out = nullptr; + if (!result_out) + return false; if (context == nullptr || context->detailer == nullptr || sd_ctx == nullptr || public_params == nullptr || inpaint_params == nullptr || input_image.data == nullptr) { return false; @@ -933,7 +932,10 @@ bool adetail_image(adetailer_ctx_t* context, return false; } - auto detections = context->detailer->predict(input_image, params); + auto resolved_inpaint = *inpaint_params; + resolved_inpaint.seed = sd::pipeline::resolve_seed(resolved_inpaint.seed); + const auto info = sd::adetailer_generation_info(*public_params, resolved_inpaint); + auto detections = context->detailer->predict(input_image, params); filter_and_sort_detections(&detections, input_image.width, input_image.height, params); auto masks = make_masks(detections, input_image.width, input_image.height, params); @@ -942,6 +944,7 @@ bool adetail_image(adetailer_ctx_t* context, return false; } + std::unique_ptr current_owner(current.data, &free); int applied = 0; for (size_t i = 0; i < masks.size(); ++i) { std::string prompt = resolve_prompt(params.prompt, inpaint_params->prompt, static_cast(i)); @@ -968,11 +971,10 @@ bool adetail_image(adetailer_ctx_t* context, if (local_image.data == nullptr || local_mask.data == nullptr) { free(local_image.data); free(local_mask.data); - free(current.data); return false; } - sd_img_gen_params_t generation = *inpaint_params; + sd_img_gen_params_t generation = resolved_inpaint; generation.prompt = prompt.c_str(); generation.negative_prompt = negative_prompt.c_str(); generation.init_image = local_image; @@ -980,7 +982,7 @@ bool adetail_image(adetailer_ctx_t* context, generation.width = params.inpaint_width; generation.height = params.inpaint_height; generation.strength = params.denoising_strength; - generation.seed = inpaint_params->seed + static_cast(i); + generation.seed = static_cast((static_cast(resolved_inpaint.seed) + i) & INT64_MAX); generation.batch_count = 1; generation.ref_images = nullptr; generation.ref_images_count = 0; @@ -1004,37 +1006,32 @@ bool adetail_image(adetailer_ctx_t* context, generation.sample_params.scheduler = params.scheduler; } - sd_image_t* generated = nullptr; - int generated_count = 0; - bool success = generate_image(sd_ctx, &generation, &generated, &generated_count); + sd_image_generation_result_t* generated = nullptr; + bool success = generate_image(sd_ctx, &generation, &generated); free(local_image.data); free(local_mask.data); - if (!success || generated_count <= 0 || generated == nullptr || generated[0].data == nullptr) { - free_sd_images(generated, generated_count); - free(current.data); + if (!success || generated == nullptr || generated->count <= 0 || generated->items[0].image.data == nullptr) { + free_sd_image_generation_result(generated); return false; } + std::unique_ptr generated_owner(generated, &free_sd_image_generation_result); Mask feather = gaussian_blur_mask(masks[i], params.mask_blur); - composite_crop(¤t, generated[0], feather, crop); - free_sd_images(generated, generated_count); + composite_crop(¤t, generated->items[0].image, feather, crop); ++applied; } + const std::vector infos{info}; auto* outputs = static_cast(calloc(1, sizeof(sd_image_t))); - if (outputs == nullptr) { - free(current.data); + if (!outputs) return false; - } outputs[0] = current; - if (images_out != nullptr) { - *images_out = outputs; - } else { - free_sd_images(outputs, 1); - } - if (num_images_out != nullptr) { - *num_images_out = 1; - } + current_owner.release(); + if (!sd::make_image_generation_result(outputs, 1, infos, result_out)) + return false; LOG_INFO("ADetailer applied %d mask(s)", applied); return true; +} catch (const std::exception& e) { + LOG_ERROR("ADetailer failed: %s", e.what()); + return false; } diff --git a/src/generation_info.cpp b/src/generation_info.cpp new file mode 100644 index 00000000..2f49054e --- /dev/null +++ b/src/generation_info.cpp @@ -0,0 +1,276 @@ +#include "generation_info.h" + +#include +#include +#include + +#include "pipeline/request.h" +#include "runtime/guidance.h" + +namespace sd { + using nlohmann::json; + + static const char* text(const char* value) { + return value ? value : ""; + } + + static json sampling_info(const sd_sample_params_t& p, bool high_noise = false) { + const auto& g = p.guidance; + json info = {{"sample_steps", p.sample_steps}, {"guidance", {{"txt_cfg", g.txt_cfg}, {"distilled_guidance", g.distilled_guidance}}}}; + if (!high_noise && p.shifted_timestep > 0) + info["shifted_timestep"] = p.shifted_timestep; + if (std::isfinite(g.img_cfg) && g.img_cfg != 1.f) + info["guidance"]["img_cfg"] = g.img_cfg; + if (p.sample_method != SAMPLE_METHOD_COUNT) + info["sample_method"] = sd_sample_method_name(p.sample_method); + if (!high_noise && p.scheduler != SCHEDULER_COUNT) + info["scheduler"] = sd_scheduler_name(p.scheduler); + if (std::isfinite(p.eta)) + info["eta"] = p.eta; + if (!high_noise && std::isfinite(p.flow_shift)) + info["flow_shift"] = p.flow_shift; + if (!high_noise && p.custom_sigmas && p.custom_sigmas_count > 0) + info["custom_sigmas"] = std::vector(p.custom_sigmas, p.custom_sigmas + p.custom_sigmas_count); + if (p.extra_sample_args && *p.extra_sample_args) + info["extra_sample_args"] = p.extra_sample_args; + if (g.slg.layers && g.slg.layer_count > 0 && + (g.slg.scale != 0.f || guidance::parse_skip_layer_guidance_uncond_arg(p.extra_sample_args))) + info["guidance"]["slg"] = {{"scale", g.slg.scale}, {"layer_start", g.slg.layer_start}, {"layer_end", g.slg.layer_end}, {"layers", std::vector(g.slg.layers, g.slg.layers + g.slg.layer_count)}}; + return info; + } + + static json hires_info(const sd_hires_params_t& p) { + json info = {{"enabled", p.enabled}}; + if (!p.enabled) + return info; + info.update({{"upscaler", sd_hires_upscaler_name(p.upscaler)}, {"scale", p.scale}, {"target_width", p.target_width}, {"target_height", p.target_height}, {"steps", p.steps}, {"denoising_strength", p.denoising_strength}}); + if (p.upscaler == SD_HIRES_UPSCALER_MODEL) { + info["model_path"] = text(p.model_path); + info["upscale_tile_size"] = p.upscale_tile_size; + } + if (p.custom_sigmas && p.custom_sigmas_count > 0) + info["custom_sigmas"] = std::vector(p.custom_sigmas, p.custom_sigmas + p.custom_sigmas_count); + return info; + } + + static json cache_info(const sd_cache_params_t& p) { + static const char* modes[] = {"disabled", "easycache", "ucache", "dbcache", "taylorseer", "cache_dit", "spectrum"}; + if (p.mode < SD_CACHE_DISABLED || p.mode > SD_CACHE_SPECTRUM) + return {{"mode", static_cast(p.mode)}}; + json info = {{"mode", modes[p.mode]}}; + if (p.mode == SD_CACHE_DISABLED) + return info; + if (p.mode == SD_CACHE_EASYCACHE || p.mode == SD_CACHE_UCACHE) { + if (std::isfinite(p.reuse_threshold)) + info["reuse_threshold"] = p.reuse_threshold; + info.update({{"start_percent", p.start_percent}, {"end_percent", p.end_percent}}); + if (p.mode == SD_CACHE_UCACHE) + info.update({{"error_decay_rate", p.error_decay_rate}, {"use_relative_threshold", p.use_relative_threshold}, {"reset_error_on_compute", p.reset_error_on_compute}}); + } else if (p.mode == SD_CACHE_DBCACHE || p.mode == SD_CACHE_TAYLORSEER || p.mode == SD_CACHE_CACHE_DIT) { + // TaylorSeer also uses the shared DBCache thresholds and limits. + info.update({{"Fn_compute_blocks", p.Fn_compute_blocks}, + {"Bn_compute_blocks", p.Bn_compute_blocks}, + {"residual_diff_threshold", p.residual_diff_threshold}, + {"max_warmup_steps", p.max_warmup_steps}, + {"max_cached_steps", p.max_cached_steps}, + {"max_continuous_cached_steps", p.max_continuous_cached_steps}}); + if (p.mode == SD_CACHE_TAYLORSEER || p.mode == SD_CACHE_CACHE_DIT) + info.update({{"taylorseer_n_derivatives", p.taylorseer_n_derivatives}, {"taylorseer_skip_interval", p.taylorseer_skip_interval}}); + if (p.scm_mask && *p.scm_mask) + info.update({{"scm_mask", p.scm_mask}, {"scm_policy_dynamic", p.scm_policy_dynamic}}); + } else if (p.mode == SD_CACHE_SPECTRUM) { + info.update({{"spectrum_w", p.spectrum_w}, + {"spectrum_m", p.spectrum_m}, + {"spectrum_lam", p.spectrum_lam}, + {"spectrum_window_size", p.spectrum_window_size}, + {"spectrum_flex_window", p.spectrum_flex_window}, + {"spectrum_warmup_steps", p.spectrum_warmup_steps}, + {"spectrum_stop_percent", p.spectrum_stop_percent}}); + } + return info; + } + + template + static json common_info(const Params& p, const char* kind) { + json info = {{"schema", "sdcpp.generation/v1"}, {"kind", kind}, {"prompt", text(p.prompt)}, {"negative_prompt", text(p.negative_prompt)}, {"width", p.width}, {"height", p.height}, {"clip_skip", p.clip_skip}, {"seed", std::to_string(p.seed)}, {"sample_params", sampling_info(p.sample_params)}}; + if (p.hires.enabled) + info["hires"] = hires_info(p.hires); + if (p.cache.mode != SD_CACHE_DISABLED) + info["cache"] = cache_info(p.cache); + if (p.circular_x) + info["circular_x"] = true; + if (p.circular_y) + info["circular_y"] = true; + if (p.ref_image_args && *p.ref_image_args) + info["ref_image_args"] = p.ref_image_args; + if (p.image_preprocess.rules && *p.image_preprocess.rules) + info["image_preprocess"] = {{"rules", p.image_preprocess.rules}}; + const auto& t = p.vae_tiling_params; + if (t.enabled || t.temporal_tiling) { + auto& tiling = info["vae_tiling_params"]; + if (t.enabled) + tiling.update({{"enabled", true}, {"tile_size_w", t.tile_size_w}, {"tile_size_h", t.tile_size_h}, {"target_overlap", t.target_overlap}, {"rel_size_w", t.rel_size_w}, {"rel_size_h", t.rel_size_h}}); + if (t.temporal_tiling) + tiling["temporal_tiling"] = true; + if (t.extra_tiling_args && *t.extra_tiling_args) + tiling["extra_tiling_args"] = t.extra_tiling_args; + } + if (p.lora_count > 0) { + info["loras"] = json::array(); + for (uint32_t i = 0; i < p.lora_count; ++i) + info["loras"].push_back({{"path", text(p.loras[i].path)}, {"multiplier", p.loras[i].multiplier}, {"is_high_noise", p.loras[i].is_high_noise}}); + } + return info; + } + + static json image_info(const sd_img_gen_params_t& p) { + auto info = common_info(p, "image"); + if (p.init_image.data) + info["strength"] = p.strength; + if (p.control_image.data) + info["control_strength"] = p.control_strength; + if (p.ip_adapter_image.data) + info["ip_adapter_strength"] = p.ip_adapter_strength; + if (p.pm_params.id_images && p.pm_params.id_images_count > 0) { + info["pm_params"] = {{"style_strength", p.pm_params.style_strength}}; + if (p.pm_params.id_embed_path && *p.pm_params.id_embed_path) + info["pm_params"]["id_embed_path"] = p.pm_params.id_embed_path; + } + if (p.pulid_params.id_embedding_path && *p.pulid_params.id_embedding_path) + info["pulid_params"] = {{"id_embedding_path", p.pulid_params.id_embedding_path}, {"id_weight", p.pulid_params.id_weight}}; + return info; + } + + template + static Params resolved_params(const pipeline::GenerationRequest& request, Params p) { + p.width = request.width; + p.height = request.height; + p.seed = request.seed; + p.clip_skip = request.clip_skip; + p.sample_params = request.sample_params; + p.hires = request.hires; + p.vae_tiling_params = request.vae_tiling_params; + return p; + } + + json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params) { + auto p = resolved_params(request, params); + p.qwen_image_layers = request.qwen_image_layers; + auto info = image_info(p); + if (!request.models.empty()) + info["models"] = request.models; + if (request.is_layered_image) + info["qwen_image_layers"] = p.qwen_image_layers; + return info; + } + + json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params) { + const auto p = resolved_params(request, params); + auto info = common_info(p, "video"); + if (!request.models.empty()) + info["models"] = request.models; + info.update({{"video_frames", request.frames}, {"fps", request.fps}}); + if (p.init_image.data || p.end_image.data) + info["strength"] = p.strength; + if (request.use_vace) + info["vace_strength"] = p.vace_strength; + if (request.use_high_noise) + info.update({{"moe_boundary", p.moe_boundary}, {"high_noise_sample_params", sampling_info(request.high_noise_sample_params, true)}}); + if (p.ref_videos_count > 0) { + info["ref_videos"] = json::array(); + for (int i = 0; i < p.ref_videos_count; ++i) + info["ref_videos"].push_back({{"fps", p.ref_videos[i].fps}}); + } + return info; + } + + json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params) { + auto inpaint = image_info(inpaint_params); + inpaint["strength"] = inpaint_params.strength; + // ADetailer clears these inputs before generating each crop. + for (const auto* key : {"control_strength", "pm_params", "pulid_params", "hires", "image_preprocess"}) + inpaint.erase(key); + sd_img_gen_params_t defaults; + sd_img_gen_params_init(&defaults); + if (inpaint_params.qwen_image_layers != defaults.qwen_image_layers) + inpaint["qwen_image_layers"] = inpaint_params.qwen_image_layers; + json adetailer = json::object(); + if (params.prompt && *params.prompt) + adetailer["prompt"] = params.prompt; + if (params.negative_prompt && *params.negative_prompt) + adetailer["negative_prompt"] = params.negative_prompt; + if (params.extra_ad_args && *params.extra_ad_args) + adetailer["extra_ad_args"] = params.extra_ad_args; + return {{"schema", "sdcpp.generation/v1"}, {"kind", "image"}, {"adetailer", adetailer}, {"inpaint_params", inpaint}}; + } + + json upscale_generation_info(uint32_t upscale_factor) { + return {{"schema", "sdcpp.generation/v1"}, {"kind", "image"}, {"upscale", {{"upscale_factor", upscale_factor}}}}; + } + + char* copy_generation_info(const json& info) { + const std::string text = info.dump(-1, ' ', false, json::error_handler_t::replace); + char* result = static_cast(malloc(text.size() + 1)); + if (result) + memcpy(result, text.c_str(), text.size() + 1); + return result; + } + + bool make_image_generation_result(sd_image_t* images, int count, const std::vector& infos, sd_image_generation_result_t** result_out) { + if (!result_out || !images || count <= 0 || infos.size() != static_cast(count)) { + free_sd_images(images, count); + return false; + } + auto* result = static_cast(calloc(1, sizeof(sd_image_generation_result_t))); + if (!result) { + free_sd_images(images, count); + return false; + } + result->items = static_cast(calloc(count, sizeof(sd_image_result_t))); + if (!result->items) { + free(result); + free_sd_images(images, count); + return false; + } + result->count = count; + try { + for (int i = 0; i < count; ++i) { + result->items[i].image = images[i]; + images[i] = {}; + result->items[i].generation_info = copy_generation_info(infos[i]); + if (!result->items[i].generation_info) { + free_sd_images(images, count); + free_sd_image_generation_result(result); + return false; + } + } + } catch (...) { + free_sd_images(images, count); + free_sd_image_generation_result(result); + return false; + } + free(images); + *result_out = result; + return true; + } +} + +void free_sd_image_generation_result(sd_image_generation_result_t* result) { + if (!result) + return; + for (int i = 0; i < result->count; ++i) { + free(result->items[i].image.data); + free(result->items[i].generation_info); + } + free(result->items); + free(result); +} + +void free_sd_video_generation_result(sd_video_generation_result_t* result) { + if (!result) + return; + free_sd_images(result->frames, result->frame_count); + free_sd_audio(result->audio); + free(result->generation_info); + free(result); +} diff --git a/src/generation_info.h b/src/generation_info.h new file mode 100644 index 00000000..56688ee8 --- /dev/null +++ b/src/generation_info.h @@ -0,0 +1,23 @@ +#ifndef __SD_GENERATION_INFO_H__ +#define __SD_GENERATION_INFO_H__ + +#include + +#include "json.hpp" +#include "stable-diffusion.h" + +namespace sd::pipeline { + struct GenerationRequest; +} + +namespace sd { + nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params); + nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params); + nlohmann::json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params); + nlohmann::json upscale_generation_info(uint32_t upscale_factor); + char* copy_generation_info(const nlohmann::json& info); + // Consumes images even on failure. + bool make_image_generation_result(sd_image_t* images, int count, const std::vector& infos, sd_image_generation_result_t** result_out); +} + +#endif // __SD_GENERATION_INFO_H__ diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 6a1ff359..f44580cf 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -675,87 +675,93 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf const auto* sd_ctx_params = &configuration.params; auto& use_tae = configuration.use_tae; auto& use_audio_vae = configuration.use_audio_vae; + const auto load_model = [&](const char* name, const char* path, const std::string& prefix = "") { + if (!model_loader.init_from_file(path, prefix)) + return false; + configuration.record_model(name, path); + return true; + }; if (strlen(SAFE_STR(sd_ctx_params->model_path)) > 0) { LOG_INFO("loading model from '%s'", sd_ctx_params->model_path); - if (!model_loader.init_from_file(sd_ctx_params->model_path)) { + if (!load_model("model", sd_ctx_params->model_path)) { LOG_ERROR("init model loader from file failed: '%s'", sd_ctx_params->model_path); } } if (strlen(SAFE_STR(sd_ctx_params->diffusion_model_path)) > 0) { LOG_INFO("loading diffusion model from '%s'", sd_ctx_params->diffusion_model_path); - if (!model_loader.init_from_file(sd_ctx_params->diffusion_model_path, "model.diffusion_model.")) { + if (!load_model("diffusion_model", sd_ctx_params->diffusion_model_path, "model.diffusion_model.")) { LOG_WARN("loading diffusion model from '%s' failed", sd_ctx_params->diffusion_model_path); } } if (strlen(SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path)) > 0) { LOG_INFO("loading high noise diffusion model from '%s'", sd_ctx_params->high_noise_diffusion_model_path); - if (!model_loader.init_from_file(sd_ctx_params->high_noise_diffusion_model_path, "model.high_noise_diffusion_model.")) { + if (!load_model("high_noise_diffusion_model", sd_ctx_params->high_noise_diffusion_model_path, "model.high_noise_diffusion_model.")) { LOG_WARN("loading diffusion model from '%s' failed", sd_ctx_params->high_noise_diffusion_model_path); } } if (strlen(SAFE_STR(sd_ctx_params->uncond_diffusion_model_path)) > 0) { LOG_INFO("loading unconditional diffusion model from '%s'", sd_ctx_params->uncond_diffusion_model_path); - if (!model_loader.init_from_file(sd_ctx_params->uncond_diffusion_model_path, "model.diffusion_model.uncond.")) { + if (!load_model("uncond_diffusion_model", sd_ctx_params->uncond_diffusion_model_path, "model.diffusion_model.uncond.")) { LOG_WARN("loading unconditional diffusion model from '%s' failed", sd_ctx_params->uncond_diffusion_model_path); } } if (strlen(SAFE_STR(sd_ctx_params->clip_l_path)) > 0) { LOG_INFO("loading clip_l from '%s'", sd_ctx_params->clip_l_path); - if (!model_loader.init_from_file(sd_ctx_params->clip_l_path, "clip_l.")) { + if (!load_model("clip_l", sd_ctx_params->clip_l_path, "clip_l.")) { LOG_WARN("loading clip_l from '%s' failed", sd_ctx_params->clip_l_path); } } if (strlen(SAFE_STR(sd_ctx_params->clip_g_path)) > 0) { LOG_INFO("loading clip_g from '%s'", sd_ctx_params->clip_g_path); - if (!model_loader.init_from_file(sd_ctx_params->clip_g_path, "clip_g.")) { + if (!load_model("clip_g", sd_ctx_params->clip_g_path, "clip_g.")) { LOG_WARN("loading clip_g from '%s' failed", sd_ctx_params->clip_g_path); } } if (strlen(SAFE_STR(sd_ctx_params->clip_vision_path)) > 0) { LOG_INFO("loading clip_vision from '%s'", sd_ctx_params->clip_vision_path); - if (!model_loader.init_from_file(sd_ctx_params->clip_vision_path, "clip_vision.")) { + if (!load_model("clip_vision", sd_ctx_params->clip_vision_path, "clip_vision.")) { LOG_WARN("loading clip_vision from '%s' failed", sd_ctx_params->clip_vision_path); } } if (strlen(SAFE_STR(sd_ctx_params->t5xxl_path)) > 0) { LOG_INFO("loading t5xxl from '%s'", sd_ctx_params->t5xxl_path); - if (!model_loader.init_from_file(sd_ctx_params->t5xxl_path, "text_encoders.t5xxl.transformer.")) { + if (!load_model("t5xxl", sd_ctx_params->t5xxl_path, "text_encoders.t5xxl.transformer.")) { LOG_WARN("loading t5xxl from '%s' failed", sd_ctx_params->t5xxl_path); } } if (strlen(SAFE_STR(sd_ctx_params->pulid_weights_path)) > 0) { LOG_INFO("loading PuLID weights from '%s'", sd_ctx_params->pulid_weights_path); - if (!model_loader.init_from_file(sd_ctx_params->pulid_weights_path, - "model.diffusion_model.")) { + if (!load_model("pulid_weights", sd_ctx_params->pulid_weights_path, + "model.diffusion_model.")) { LOG_WARN("loading PuLID weights from '%s' failed", sd_ctx_params->pulid_weights_path); } } if (strlen(SAFE_STR(sd_ctx_params->llm_path)) > 0) { LOG_INFO("loading llm from '%s'", sd_ctx_params->llm_path); - if (!model_loader.init_from_file(sd_ctx_params->llm_path, "text_encoders.llm.")) { + if (!load_model("llm", sd_ctx_params->llm_path, "text_encoders.llm.")) { LOG_WARN("loading llm from '%s' failed", sd_ctx_params->llm_path); } } if (strlen(SAFE_STR(sd_ctx_params->llm_vision_path)) > 0) { LOG_INFO("loading llm vision from '%s'", sd_ctx_params->llm_vision_path); - if (!model_loader.init_from_file(sd_ctx_params->llm_vision_path, "text_encoders.llm.visual.")) { + if (!load_model("llm_vision", sd_ctx_params->llm_vision_path, "text_encoders.llm.visual.")) { LOG_WARN("loading llm vision from '%s' failed", sd_ctx_params->llm_vision_path); } } if (strlen(SAFE_STR(sd_ctx_params->vae_path)) > 0) { LOG_INFO("loading vae from '%s'", sd_ctx_params->vae_path); - if (!model_loader.init_from_file(sd_ctx_params->vae_path, "vae.")) { + if (!load_model("vae", sd_ctx_params->vae_path, "vae.")) { LOG_WARN("loading vae from '%s' failed", sd_ctx_params->vae_path); external_vae_is_invalid = true; } @@ -763,7 +769,7 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf if (strlen(SAFE_STR(sd_ctx_params->taesd_path)) > 0) { LOG_INFO("loading tae from '%s'", sd_ctx_params->taesd_path); - if (!model_loader.init_from_file(sd_ctx_params->taesd_path, "tae.")) { + if (!load_model("taesd", sd_ctx_params->taesd_path, "tae.")) { LOG_WARN("loading tae from '%s' failed", sd_ctx_params->taesd_path); } else { use_tae = true; @@ -772,14 +778,14 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf if (strlen(SAFE_STR(sd_ctx_params->embeddings_connectors_path)) > 0) { LOG_INFO("loading embeddings connectors from '%s'", sd_ctx_params->embeddings_connectors_path); - if (!model_loader.init_from_file(sd_ctx_params->embeddings_connectors_path)) { + if (!load_model("embeddings_connectors", sd_ctx_params->embeddings_connectors_path)) { LOG_WARN("loading embeddings connectors from '%s' failed", sd_ctx_params->embeddings_connectors_path); } } if (strlen(SAFE_STR(sd_ctx_params->audio_vae_path)) > 0) { LOG_INFO("loading audio VAE from '%s'", sd_ctx_params->audio_vae_path); - if (!model_loader.init_from_file(sd_ctx_params->audio_vae_path)) { + if (!load_model("audio_vae", sd_ctx_params->audio_vae_path)) { LOG_WARN("loading audio VAE weights from '%s' failed", sd_ctx_params->audio_vae_path); } else { use_audio_vae = true; @@ -788,15 +794,15 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf if (strlen(SAFE_STR(sd_ctx_params->audio_encoder_path)) > 0) { LOG_INFO("loading audio encoder (wav2vec2) from '%s'", sd_ctx_params->audio_encoder_path); - if (!model_loader.init_from_file(sd_ctx_params->audio_encoder_path, "wav2vec2.")) { + if (!load_model("audio_encoder", sd_ctx_params->audio_encoder_path, "wav2vec2.")) { LOG_WARN("loading audio encoder weights from '%s' failed", sd_ctx_params->audio_encoder_path); } } if (strlen(SAFE_STR(sd_ctx_params->motion_module_path)) > 0) { LOG_INFO("loading motion module (AnimateDiff) from '%s'", sd_ctx_params->motion_module_path); - if (!model_loader.init_from_file(sd_ctx_params->motion_module_path, - "model.diffusion_model.motion_module.")) { + if (!load_model("motion_module", sd_ctx_params->motion_module_path, + "model.diffusion_model.motion_module.")) { LOG_WARN("loading motion module from '%s' failed", sd_ctx_params->motion_module_path); } else { configuration.animatediff_loaded = true; @@ -810,8 +816,11 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf } } + if (configuration.control_net_file) + configuration.record_model("control_net", sd_ctx_params->control_net_path); + if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0) { - if (!model_loader.init_from_file(sd_ctx_params->ip_adapter_path)) { + if (!load_model("ip_adapter", sd_ctx_params->ip_adapter_path)) { LOG_ERROR("init ip-adapter model loader from file failed: '%s'", sd_ctx_params->ip_adapter_path); return false; } @@ -819,6 +828,8 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf if (strlen(SAFE_STR(sd_ctx_params->photo_maker_path)) > 0) { configuration.photomaker_source_available = model_loader.add_file(sd_ctx_params->photo_maker_path, "pmid."); + if (configuration.photomaker_source_available) + configuration.record_model("photo_maker", sd_ctx_params->photo_maker_path); if (!configuration.photomaker_source_available) { LOG_WARN("loading stacked ID embedding from '%s' failed", sd_ctx_params->photo_maker_path); } diff --git a/src/pipeline/diffusion_engine.h b/src/pipeline/diffusion_engine.h index 1e41662d..c7295d8f 100644 --- a/src/pipeline/diffusion_engine.h +++ b/src/pipeline/diffusion_engine.h @@ -123,6 +123,7 @@ public: sd_ctx_params_t params{}; std::list strings; std::vector embeddings; + std::map models; ModelLoader::FileId control_net_file = 0; bool use_tae = false; bool use_audio_vae = false; @@ -164,13 +165,24 @@ public: use_audio_vae = other.use_audio_vae; photomaker_source_available = other.photomaker_source_available; animatediff_loaded = other.animatediff_loaded; + models = other.models; } ModelConfig& operator=(const ModelConfig&) = delete; + void record_model(const std::string& name, const std::string& path) { + if (path.empty()) { + models.erase(name); + return; + } + const auto separator = path.find_last_of("/\\"); + models[name] = separator == std::string::npos ? path : path.substr(separator + 1); + } + void set_control_net(ModelLoader::FileId id, const std::string& path) { control_net_file = id; strings.push_back(path); params.control_net_path = strings.back().c_str(); + record_model("control_net", path); } }; diff --git a/src/pipeline/generation.h b/src/pipeline/generation.h index 34aa000a..0bc39131 100644 --- a/src/pipeline/generation.h +++ b/src/pipeline/generation.h @@ -2,6 +2,7 @@ #define __SD_PIPELINE_GENERATION_H__ #include "conditioning/conditioner.hpp" +#include "json.hpp" #include "stable-diffusion.h" class StableDiffusionGGML; @@ -54,14 +55,16 @@ namespace sd::pipeline { bool generate_image(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params, sd_image_t** images_out, - int* num_images_out); + int* num_images_out, + std::vector* infos_out = nullptr); bool generate_video(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params, sd_image_t** frames_out, int* num_frames_out, sd_audio_t** audio_out, - int* fps_out); + int* fps_out, + nlohmann::json* info_out = nullptr); sd::Tensor upscale_ltx_spatial_video_latent(StableDiffusionGGML* sd, const char* model_path, diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp index 046e9cfc..c549280a 100644 --- a/src/pipeline/image.cpp +++ b/src/pipeline/image.cpp @@ -1,4 +1,5 @@ #include "generation.h" +#include "generation_info.h" #include #include @@ -668,7 +669,7 @@ namespace sd::pipeline { int64_t t4 = ggml_time_ms(); LOG_INFO("decode_first_stage completed, taking %.2fs", (t4 - t0) * 1.0f / 1000); - if (decoded_images.empty()) { + if (decoded_images.empty() || cancelled) { LOG_ERROR(cancelled ? "cancelled before any latent images were decoded" : "no decoded images"); return nullptr; } @@ -829,7 +830,10 @@ namespace sd::pipeline { bool generate_image(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params, sd_image_t** images_out, - int* num_images_out) { + int* num_images_out, + std::vector* infos_out) { + if (infos_out) + infos_out->clear(); if (images_out != nullptr) { *images_out = nullptr; } @@ -851,6 +855,7 @@ namespace sd::pipeline { int64_t t0 = ggml_time_ms(); sd->vae_tiling_params = sd_img_gen_params->vae_tiling_params; GenerationRequest request(sd, sd_img_gen_params); + const auto info = sd::generation_info(request, *sd_img_gen_params); sd::ImagePreprocessor preprocessing(sd_img_gen_params->image_preprocess.rules); sd_img_gen_params_t processed_params = *sd_img_gen_params; if (!preprocessing.prepare_inputs(processed_params, request.width, request.height)) @@ -891,7 +896,6 @@ namespace sd::pipeline { return false; } ImageGenerationEmbeds embeds = std::move(*embeds_opt); - std::vector> final_latents; int64_t denoise_start = ggml_time_ms(); for (int b = 0; b < request.batch_count; b++) { @@ -908,7 +912,7 @@ namespace sd::pipeline { } int64_t sampling_start = ggml_time_ms(); - int64_t cur_seed = request.seed + b; + int64_t cur_seed = request.seed_for_batch(b); LOG_INFO("generating image: %i/%i - seed %" PRId64, b + 1, request.batch_count, cur_seed); sd->rng->manual_seed(cur_seed); @@ -994,7 +998,7 @@ namespace sd::pipeline { std::vector hires_sigma_sched = make_hires_sigma_schedule(sd, request.hires, - sd_img_gen_params->sample_params, + request.sample_params, plan.sample_method, plan.sample_steps, sd->get_image_seq_len(request.hires.target_height, request.hires.target_width), @@ -1035,7 +1039,7 @@ namespace sd::pipeline { LOG_ERROR("cancelling generation during hires fix"); return false; } - int64_t cur_seed = request.seed + b; + int64_t cur_seed = request.seed_for_batch(b); sd->rng->manual_seed(cur_seed); sd->sampler_rng->manual_seed(cur_seed); @@ -1113,6 +1117,19 @@ namespace sd::pipeline { return false; } + if (infos_out) { + try { + infos_out->assign(num_images, info); + // Layers and frames decoded from the same latent share its seed. + const int images_per_latent = num_images / static_cast(final_latents.size()); + for (int i = 0; i < num_images; ++i) { + (*infos_out)[i]["seed"] = std::to_string(request.seed_for_batch(i / images_per_latent)); + } + } catch (...) { + free_sd_images(result, num_images); + throw; + } + } sd->lora_stat(); int64_t t1 = ggml_time_ms(); diff --git a/src/pipeline/request.cpp b/src/pipeline/request.cpp index 0693532f..2647efb8 100644 --- a/src/pipeline/request.cpp +++ b/src/pipeline/request.cpp @@ -1,4 +1,5 @@ #include "request.h" +#include "conditioning/conditioner.hpp" #include #include @@ -6,6 +7,8 @@ #include #include "diffusion_engine.h" +#include "generation.h" +#include "model/vae/vae.hpp" #include "model_definition.h" #include "runtime/denoiser.hpp" @@ -66,7 +69,7 @@ namespace sd::pipeline { return DISCRETE_SCHEDULER; } - static int64_t resolve_seed(int64_t seed) { + int64_t resolve_seed(int64_t seed) { if (seed >= 0) { return seed; } @@ -127,6 +130,7 @@ namespace sd::pipeline { seed = sd_img_gen_params->seed; batch_count = sd_img_gen_params->batch_count; qwen_image_layers = std::max(0, sd_img_gen_params->qwen_image_layers); + is_layered_image = sd->version == VERSION_QWEN_IMAGE_LAYERED; clip_skip = sd_img_gen_params->clip_skip; shifted_timestep = sd_img_gen_params->sample_params.shifted_timestep; strength = sd_img_gen_params->strength; @@ -137,8 +141,18 @@ namespace sd::pipeline { pm_params = sd_img_gen_params->pm_params; pulid_params = sd_img_gen_params->pulid_params; hires = sd_img_gen_params->hires; + vae_tiling_params = sd_img_gen_params->vae_tiling_params; cache_params = &sd_img_gen_params->cache; + sample_params = sd_img_gen_params->sample_params; resolve(sd); + if (!sd_img_gen_params->control_image.data) + models.erase("control_net"); + if (!sd_img_gen_params->ip_adapter_image.data) + models.erase("ip_adapter"); + if (!pm_params.id_images || pm_params.id_images_count <= 0) + models.erase("photo_maker"); + if (!pulid_params.id_embedding_path || !*pulid_params.id_embedding_path) + models.erase("pulid_weights"); } GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params) { @@ -166,7 +180,20 @@ namespace sd::pipeline { guidance = sd_vid_gen_params->sample_params.guidance; high_noise_guidance = sd_vid_gen_params->high_noise_sample_params.guidance; hires = sd_vid_gen_params->hires; + vae_tiling_params = sd_vid_gen_params->vae_tiling_params; + sample_params = sd_vid_gen_params->sample_params; + high_noise_sample_params = sd_vid_gen_params->high_noise_sample_params; + use_high_noise = sd->high_noise_diffusion_model != nullptr; + use_vace = sd->diffusion_model && (sd->diffusion_model->get_desc() == "Wan2.1-VACE-1.3B" || + sd->diffusion_model->get_desc() == "Wan2.x-VACE-14B"); + shifted_timestep = sample_params.shifted_timestep; + if (sd->config_ && sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd->version)) + frames = std::min(requested_frames, 32); resolve(sd); + models.erase("control_net"); + models.erase("ip_adapter"); + models.erase("photo_maker"); + models.erase("pulid_weights"); if (frames != requested_frames) { LOG_WARN("align video frames from %d to %d for %s", requested_frames, @@ -201,6 +228,10 @@ namespace sd::pipeline { spatial_multiple); } + int64_t GenerationRequest::seed_for_batch(int index) const { + return static_cast((static_cast(seed) + index) & INT64_MAX); + } + void GenerationRequest::resolve_hires() { if (!hires.enabled) { return; @@ -307,6 +338,23 @@ namespace sd::pipeline { } void GenerationRequest::resolve(StableDiffusionGGML* sd) { + if (sd->config_) { + models = sd->config_->models; + if (!use_high_noise) + models.erase("high_noise_diffusion_model"); + if (!sd->clip_vision) + models.erase("clip_vision"); + if (!sd->audio_vae_model || frames <= 0) + models.erase("audio_vae"); + if (!sd->audio_encoder || frames <= 0) + models.erase("audio_encoder"); + if (!sd->config_->animatediff_loaded || !sd_version_supports_animatediff(sd->version) || frames <= 0) + models.erase("motion_module"); + if (sd->uses_tae() && !sd->tae_preview_only()) + models.erase("vae"); + else + models.erase("taesd"); + } if (sd_version_is_minit2i(sd->version) && (width != 512 || height != 512)) { width = 512; height = 512; @@ -314,7 +362,11 @@ namespace sd::pipeline { } align_generation_request_size(); resolve_hires(); + if (sd->first_stage_model) + vae_tiling_params = sd->first_stage_model->resolve_tiling_params(vae_tiling_params); seed = resolve_seed(seed); + if (sd->cond_stage_model) + clip_skip = sd->cond_stage_model->resolve_clip_skip(clip_skip); resolve_guidance(sd, &guidance, &use_uncond, &use_img_uncond, has_ref_images); if (sd->high_noise_diffusion_model) { @@ -330,33 +382,44 @@ namespace sd::pipeline { LOG_WARN("timestep shifting is only supported for SDXL models!"); shifted_timestep = 0; } + sample_params.guidance = guidance; + sample_params.shifted_timestep = shifted_timestep; + sample_params.sample_method = resolve_sample_method(sd, sample_params.sample_method); + sample_params.scheduler = resolve_scheduler(sd, sample_params.scheduler, sample_params.sample_method); + sample_params.eta = resolve_eta(sd, sample_params.eta, sample_params.sample_method); + if (frames > 0) { + high_noise_sample_params.guidance = high_noise_guidance; + high_noise_sample_params.sample_method = resolve_sample_method(sd, high_noise_sample_params.sample_method); + high_noise_sample_params.scheduler = resolve_scheduler(sd, high_noise_sample_params.scheduler, high_noise_sample_params.sample_method); + high_noise_sample_params.eta = resolve_eta(sd, high_noise_sample_params.eta, high_noise_sample_params.sample_method); + } } SamplePlan::SamplePlan(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params, const GenerationRequest& request) { - sample_method = sd_img_gen_params->sample_params.sample_method; - extra_sample_args = sd_img_gen_params->sample_params.extra_sample_args; - eta = sd_img_gen_params->sample_params.eta; - sample_steps = sd_img_gen_params->sample_params.sample_steps; - resolve(sd, &request, &sd_img_gen_params->sample_params); + sample_method = request.sample_params.sample_method; + extra_sample_args = request.sample_params.extra_sample_args; + eta = request.sample_params.eta; + sample_steps = request.sample_params.sample_steps; + resolve(sd, &request, &request.sample_params); } SamplePlan::SamplePlan(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params, const GenerationRequest& request) { - sample_method = sd_vid_gen_params->sample_params.sample_method; - extra_sample_args = sd_vid_gen_params->sample_params.extra_sample_args; - eta = sd_vid_gen_params->sample_params.eta; - sample_steps = sd_vid_gen_params->sample_params.sample_steps; + sample_method = request.sample_params.sample_method; + extra_sample_args = request.sample_params.extra_sample_args; + eta = request.sample_params.eta; + sample_steps = request.sample_params.sample_steps; if (sd->high_noise_diffusion_model) { - high_noise_sample_steps = sd_vid_gen_params->high_noise_sample_params.sample_steps; - high_noise_sample_method = sd_vid_gen_params->high_noise_sample_params.sample_method; - high_noise_extra_sample_args = sd_vid_gen_params->high_noise_sample_params.extra_sample_args; - high_noise_eta = sd_vid_gen_params->high_noise_sample_params.eta; + high_noise_sample_steps = request.high_noise_sample_params.sample_steps; + high_noise_sample_method = request.high_noise_sample_params.sample_method; + high_noise_extra_sample_args = request.high_noise_sample_params.extra_sample_args; + high_noise_eta = request.high_noise_sample_params.eta; } moe_boundary = sd_vid_gen_params->moe_boundary; - resolve(sd, &request, &sd_vid_gen_params->sample_params); + resolve(sd, &request, &request.sample_params); } void SamplePlan::resolve(StableDiffusionGGML* sd, @@ -382,10 +445,10 @@ namespace sd::pipeline { } } } else { - scheduler_t scheduler = resolve_scheduler(sd, - sample_params->scheduler, - sample_method); - int sample_seq_len = sd->get_image_seq_len(request->height, request->width); + scheduler = resolve_scheduler(sd, + sample_params->scheduler, + sample_method); + int sample_seq_len = sd->get_image_seq_len(request->height, request->width); if (sd_version_is_ltxav(sd->version) && request->frames > 0) { int latent_frames = ((request->frames - 1) / 8) + 1; sample_seq_len *= latent_frames; diff --git a/src/pipeline/request.h b/src/pipeline/request.h index fac4aba5..1cf1cee6 100644 --- a/src/pipeline/request.h +++ b/src/pipeline/request.h @@ -1,6 +1,7 @@ #ifndef __SD_PIPELINE_REQUEST_H__ #define __SD_PIPELINE_REQUEST_H__ +#include #include #include @@ -16,6 +17,8 @@ namespace sd::pipeline { enum scheduler_t default_scheduler(const StableDiffusionGGML* sd, enum sample_method_t sample_method); + int64_t resolve_seed(int64_t seed); + float resolve_eta(StableDiffusionGGML* sd, float eta, enum sample_method_t sample_method); @@ -23,38 +26,47 @@ namespace sd::pipeline { struct GenerationRequest { std::string prompt; std::string negative_prompt; - int width = -1; - int height = -1; - int clip_skip = -1; - int vae_scale_factor = -1; - int diffusion_model_down_factor = -1; - int64_t seed = -1; - bool use_uncond = false; - bool use_img_uncond = false; - bool use_high_noise_uncond = false; - bool use_high_noise_img_uncond = false; - bool has_ref_images = false; - const sd_cache_params_t* cache_params = nullptr; - int batch_count = 1; - int qwen_image_layers = 3; - int shifted_timestep = 0; - float strength = 1.f; - float control_strength = 0.f; - float eta = 0.f; - sd_guidance_params_t guidance = {}; - sd_guidance_params_t high_noise_guidance = {}; - sd_pm_params_t pm_params = {}; - sd_pulid_params_t pulid_params = {}; - sd_hires_params_t hires = {}; - int frames = -1; - int requested_frames = -1; - int fps = 16; - float vace_strength = 1.f; + std::map models; + int width = -1; + int height = -1; + int clip_skip = -1; + int vae_scale_factor = -1; + int diffusion_model_down_factor = -1; + int64_t seed = -1; + bool use_uncond = false; + bool use_img_uncond = false; + bool use_high_noise_uncond = false; + bool use_high_noise_img_uncond = false; + bool has_ref_images = false; + bool is_layered_image = false; + bool use_high_noise = false; + bool use_vace = false; + const sd_cache_params_t* cache_params = nullptr; + int batch_count = 1; + int qwen_image_layers = 3; + int shifted_timestep = 0; + float strength = 1.f; + float control_strength = 0.f; + float eta = 0.f; + sd_sample_params_t sample_params = {}; + sd_sample_params_t high_noise_sample_params = {}; + sd_guidance_params_t guidance = {}; + sd_guidance_params_t high_noise_guidance = {}; + sd_pm_params_t pm_params = {}; + sd_pulid_params_t pulid_params = {}; + sd_hires_params_t hires = {}; + sd_tiling_params_t vae_tiling_params = {}; + int frames = -1; + int requested_frames = -1; + int fps = 16; + float vace_strength = 1.f; GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params); GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params); + int64_t seed_for_batch(int index) const; + void align_generation_request_size(); void align_image_size(int* target_width, int* target_height, const char* label); @@ -81,6 +93,7 @@ namespace sd::pipeline { int sample_steps = 0; int high_noise_sample_steps = 0; int total_steps = 0; + scheduler_t scheduler = SCHEDULER_COUNT; float moe_boundary = 0.f; std::vector sigmas; diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index 9aef07b1..a07b9317 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -1,4 +1,5 @@ #include "generation.h" +#include "generation_info.h" #include #include @@ -1575,7 +1576,13 @@ namespace sd::pipeline { img_gen_params.ref_image_args = sd_vid_gen_params->ref_image_args; sd->animatediff_num_frames = n_frames; - bool ok = generate_image(sd, &img_gen_params, frames_out, num_frames_out); + bool ok = false; + try { + ok = generate_image(sd, &img_gen_params, frames_out, num_frames_out); + } catch (...) { + sd->animatediff_num_frames = 0; + throw; + } sd->animatediff_num_frames = 0; return ok; } @@ -1585,14 +1592,24 @@ namespace sd::pipeline { sd_image_t** frames_out, int* num_frames_out, sd_audio_t** audio_out, - int* fps_out) { + int* fps_out, + nlohmann::json* info_out) { + if (info_out) + *info_out = nlohmann::json(); + GenerationRequest request(sd, sd_vid_gen_params); + const auto info = sd::generation_info(request, *sd_vid_gen_params); if (sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd->version)) { LOG_INFO("AnimateDiff dispatch: %d frames, %dx%d", sd_vid_gen_params->video_frames, sd_vid_gen_params->width, sd_vid_gen_params->height); if (fps_out != nullptr) { *fps_out = std::max(1, sd_vid_gen_params->fps); } - return generate_animatediff_video(sd, sd_vid_gen_params, frames_out, num_frames_out); + auto params = *sd_vid_gen_params; + params.seed = request.seed; + const bool ok = generate_animatediff_video(sd, ¶ms, frames_out, num_frames_out); + if (ok && info_out) + *info_out = info; + return ok; } sd->reset_cancel_flag(); @@ -1602,7 +1619,6 @@ namespace sd::pipeline { int64_t t0 = ggml_time_ms(); sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params; sd->apply_circular_axes(sd_vid_gen_params->circular_x, sd_vid_gen_params->circular_y); - GenerationRequest request(sd, sd_vid_gen_params); sd::ImagePreprocessor preprocessing(sd_vid_gen_params->image_preprocess.rules); sd_vid_gen_params_t processed_params = *sd_vid_gen_params; if (!preprocessing.prepare_inputs(processed_params, request.width, request.height)) @@ -1734,9 +1750,9 @@ namespace sd::pipeline { embeds.img_uncond, sd::Tensor(), 0.f, - sd_vid_gen_params->sample_params.guidance, + request.guidance, plan.eta, - sd_vid_gen_params->sample_params.shifted_timestep, + request.shifted_timestep, plan.sample_method, sd->is_flow_denoiser(), plan.extra_sample_args, @@ -1846,7 +1862,7 @@ namespace sd::pipeline { std::vector hires_sigma_sched = make_hires_sigma_schedule(sd, request.hires, - sd_vid_gen_params->sample_params, + request.sample_params, hires_sample_method, plan.sample_steps, sd->get_image_seq_len(hires_request.height, hires_request.width) * T, @@ -1873,9 +1889,9 @@ namespace sd::pipeline { embeds.img_uncond, sd::Tensor(), 0.f, - sd_vid_gen_params->sample_params.guidance, + request.guidance, hires_eta, - sd_vid_gen_params->sample_params.shifted_timestep, + request.shifted_timestep, hires_sample_method, sd->is_flow_denoiser(), plan.extra_sample_args, @@ -2007,6 +2023,14 @@ namespace sd::pipeline { generated_audio->sample_count = want_samples; } } + if (info_out) { + try { + *info_out = info; + } catch (...) { + free_sd_audio(generated_audio); + throw; + } + } if (audio_out != nullptr) { *audio_out = generated_audio; } else { diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 00c5e8db..bca4b79b 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -1,4 +1,5 @@ #include "stable-diffusion.h" +#include "generation_info.h" #include #include @@ -752,43 +753,62 @@ enum scheduler_t sd_get_default_scheduler(const sd_ctx_t* sd_ctx, enum sample_me SD_API bool generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* params, - sd_image_t** images_out, - int* num_images_out) { - if (images_out != nullptr) - *images_out = nullptr; - if (num_images_out != nullptr) - *num_images_out = 0; - if (sd_ctx == nullptr || sd_ctx->sd == nullptr || params == nullptr) { + sd_image_generation_result_t** result_out) { + if (result_out) + *result_out = nullptr; + if (!result_out || !sd_ctx || !sd_ctx->sd || !params) + return false; + StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd); + if (!execution.ready) + return false; + sd_image_t* images = nullptr; + int count = 0; + std::vector infos; + try { + if (!sd::pipeline::generate_image(sd_ctx->sd, params, &images, &count, &infos)) { + free_sd_images(images, count); + return false; + } + } catch (const std::exception& e) { + LOG_ERROR("image generation failed: %s", e.what()); + free_sd_images(images, count); return false; } - StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd); - return execution.ready && sd::pipeline::generate_image(sd_ctx->sd, params, images_out, num_images_out); + return sd::make_image_generation_result(images, count, infos, result_out); } SD_API bool generate_video(sd_ctx_t* sd_ctx, - const sd_vid_gen_params_t* sd_vid_gen_params, - sd_image_t** frames_out, - int* num_frames_out, - sd_audio_t** audio_out, - int* fps_out) { - if (frames_out != nullptr) - *frames_out = nullptr; - if (audio_out != nullptr) - *audio_out = nullptr; - if (num_frames_out != nullptr) - *num_frames_out = 0; - if (fps_out != nullptr) - *fps_out = 0; - if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_vid_gen_params == nullptr) { + const sd_vid_gen_params_t* params, + sd_video_generation_result_t** result_out) { + if (result_out) + *result_out = nullptr; + if (!result_out || !sd_ctx || !sd_ctx->sd || !params) return false; - } - StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd); - if (!execution.ready) { + if (!execution.ready) + return false; + auto* result = static_cast(calloc(1, sizeof(sd_video_generation_result_t))); + if (!result) + return false; + try { + nlohmann::json info; + if (!sd::pipeline::generate_video(sd_ctx->sd, params, &result->frames, &result->frame_count, + &result->audio, &result->fps, &info)) { + free_sd_video_generation_result(result); + return false; + } + result->generation_info = sd::copy_generation_info(info); + if (!result->generation_info) { + free_sd_video_generation_result(result); + return false; + } + } catch (const std::exception& e) { + LOG_ERROR("video generation failed: %s", e.what()); + free_sd_video_generation_result(result); return false; } - - return sd::pipeline::generate_video(sd_ctx->sd, sd_vid_gen_params, frames_out, num_frames_out, audio_out, fps_out); + *result_out = result; + return true; } SD_API void free_sd_images(sd_image_t* result_images, int num_images) { diff --git a/src/upscaler.cpp b/src/upscaler.cpp index dfe5e4b2..c9bbbcb6 100644 --- a/src/upscaler.cpp +++ b/src/upscaler.cpp @@ -1,6 +1,7 @@ #include "upscaler.h" #include "core/ggml_extend_backend.h" #include "core/util.h" +#include "generation_info.h" #include "model_loader.h" #include "runtime/tiling.h" #include "stable-diffusion.h" @@ -259,14 +260,11 @@ upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path_c_str, bool upscale(upscaler_ctx_t* upscaler_ctx, sd_image_t input_image, uint32_t upscale_factor, - sd_image_t** images_out, - int* num_images_out) { - if (images_out != nullptr) { - *images_out = nullptr; - } - if (num_images_out != nullptr) { - *num_images_out = 0; - } + sd_image_generation_result_t** result_out) try { + if (result_out) + *result_out = nullptr; + if (!result_out) + return false; if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr) { return false; } @@ -276,21 +274,17 @@ bool upscale(upscaler_ctx_t* upscaler_ctx, return false; } + auto release_images = [](sd_image_t* images) { free_sd_images(images, 1); }; + std::unique_ptr image_owner(result_images, release_images); + const std::vector infos{sd::upscale_generation_info(upscale_factor)}; result_images[0] = upscaler_ctx->upscaler->upscale(input_image, upscale_factor); - if (result_images[0].data == nullptr) { - free(result_images); + if (!result_images[0].data) return false; - } - if (num_images_out != nullptr) { - *num_images_out = 1; - } - if (images_out != nullptr) { - *images_out = result_images; - } else { - free_sd_images(result_images, 1); - } - return true; + return sd::make_image_generation_result(image_owner.release(), 1, infos, result_out); +} catch (const std::exception& e) { + LOG_ERROR("upscale failed: %s", e.what()); + return false; } int get_upscale_factor(upscaler_ctx_t* upscaler_ctx) { @@ -329,6 +323,8 @@ int get_upscaler_model_scale(const char* model_path) { } void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx) { + if (!upscaler_ctx) + return; if (upscaler_ctx->upscaler != nullptr) { delete upscaler_ctx->upscaler; upscaler_ctx->upscaler = nullptr;