trending/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-10-21 06:19:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: support for merging LoRA weights on model conversion (#2079)

Browse Source 
Co-authored-by: leejet <leejet714@gmail.com>
This commit is contained in:
Wagner Bruna
2026-10-06 17:50:16 +08:00
committed by GitHub
co-authored by leejet
parent bbdcad48dd
commit f4b5326da7
7 changed files with 279 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/cli/main.cpp
+12 -1
View File
@@ -716,6 +716,15 @@ int main(int argc, const char* argv[]) {
}
if (cli_params.mode == CONVERT) {
std::vector<sd_lora_t> loras;
loras.reserve(gen_params.lora_map.size() + gen_params.high_noise_lora_map.size());
for (const auto& [path, multiplier] : gen_params.lora_map) {
loras.push_back({false, multiplier, path.c_str()});
}
for (const auto& [path, multiplier] : gen_params.high_noise_lora_map) {
loras.push_back({true, multiplier, path.c_str()});
}
bool success = convert_with_components(ctx_params.model_path.c_str(),
ctx_params.clip_l_path.c_str(),
ctx_params.clip_g_path.c_str(),
@@ -726,7 +735,9 @@ int main(int argc, const char* argv[]) {
ctx_params.wtype,
ctx_params.tensor_type_rules.c_str(),
cli_params.convert_name,
ctx_params.n_threads);
ctx_params.n_threads,
loras.empty() ? nullptr : loras.data(),
(int)loras.size());
if (!success) {
LOG_ERROR("convert '%s'/'%s' to '%s' failed",
ctx_params.model_path.c_str(),
examples/common/common.cpp
+13 -7
View File
@@ -2271,14 +2271,15 @@ bool SDGenerationParams::from_json_str(
return true;
}
void SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_dir) {
bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_dir) {
if (lora_model_dir.empty()) {
return;
return true;
}
static const std::regex re(R"(<lora:([^:>]+):([^>]+)>)");
static const std::vector<std::string> valid_ext = {".gguf", ".safetensors", ".pt", ".ckpt"};
std::smatch m;
bool missing = false;
std::string tmp = prompt;
while (std::regex_search(tmp, m, re)) {
@@ -2320,8 +2321,9 @@ void SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d
}
if (!found) {
LOG_WARN("can not found lora %s", final_path.lexically_normal().string().c_str());
tmp = m.suffix().str();
prompt = std::regex_replace(prompt, re, "", std::regex_constants::format_first_only);
tmp = m.suffix().str();
prompt = std::regex_replace(prompt, re, "", std::regex_constants::format_first_only);
missing = true;
continue;
}
}
@@ -2337,6 +2339,8 @@ void SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d
tmp = m.suffix().str();
}
return !missing;
}
bool SDGenerationParams::width_and_height_are_set() const {
@@ -2458,7 +2462,7 @@ bool SDGenerationParams::initialize_cache_params() {
return true;
}
bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict) {
bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict, bool validate_missing_loras) {
vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str();
if (high_noise_sample_params.sample_steps <= 0) {
@@ -2502,7 +2506,9 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s
prompt_with_lora = prompt;
if (!lora_model_dir.empty()) {
extract_and_remove_lora(lora_model_dir);
if (!extract_and_remove_lora(lora_model_dir) && validate_missing_loras) {
return false;
}
}
return true;
}
@@ -2630,7 +2636,7 @@ bool SDGenerationParams::resolve_and_validate(SDMode mode,
const std::string& lora_model_dir,
const std::string& hires_upscalers_dir,
bool strict) {
if (!resolve(lora_model_dir, hires_upscalers_dir, strict)) {
if (!resolve(lora_model_dir, hires_upscalers_dir, strict, mode == CONVERT)) {
return false;
}
if (!validate(mode)) {
examples/common/common.h
+2 -2
View File
@@ -313,12 +313,12 @@ struct SDGenerationParams {
const std::function<std::string(const std::string&)>& lora_path_resolver = {});
bool parse_image_preprocess_json(const std::string& json_str);
bool initialize_cache_params();
void extract_and_remove_lora(const std::string& lora_model_dir);
bool extract_and_remove_lora(const std::string& lora_model_dir);
bool width_and_height_are_set() const;
void set_width_and_height_if_unset(int w, int h);
int get_resolved_width() const;
int get_resolved_height() const;
bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false);
bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false, bool validate_missing_loras = false);
bool validate(SDMode mode);
bool resolve_and_validate(SDMode mode,
const std::string& lora_model_dir,
include/stable-diffusion.h
+3 -1
View File
@@ -601,7 +601,9 @@ SD_API bool convert_with_components(const char* model_path,
enum sd_type_t output_type,
const char* tensor_type_rules,
bool convert_name,
int n_threads);
int n_threads,
const sd_lora_t* loras,
int lora_count);
SD_API bool preprocess_canny(sd_image_t image,
float high_threshold,
src/convert.cpp
+240 -11
View File
@@ -4,6 +4,7 @@
#include <cstring>
#include <exception>
#include <fstream>
#include <map>
#include <memory>
#include <mutex>
#include <regex>
@@ -11,11 +12,15 @@
#include <thread>
#include <vector>
#include "core/ggml_extend_backend.h"
#include "core/util.h"
#include "model_io/gguf_io.h"
#include "model_io/safetensors_io.h"
#include "model_io/streaming_writer.h"
#include "model_loader.h"
#include "model_manager.h"
#include "name_conversion.h"
#include "runtime/imatrix.h"
struct TensorExportInfo {
TensorStorage storage;
@@ -75,9 +80,9 @@ static size_t export_tensor_nbytes(const TensorExportInfo& info) {
return static_cast<size_t>(output_storage.nbytes());
}
static TensorWritePlan tensor_write_plan_from_export_info(const TensorExportInfo& info) {
static TensorWritePlan tensor_write_plan_from_export_info(const TensorExportInfo& info, const std::string& name) {
TensorWritePlan plan;
plan.name = info.storage.name;
plan.name = name;
plan.type = info.type;
plan.n_dims = info.storage.n_dims;
for (int i = 0; i < SD_MAX_DIMS; i++) {
@@ -86,11 +91,14 @@ static TensorWritePlan tensor_write_plan_from_export_info(const TensorExportInfo
return plan;
}
static std::vector<TensorWritePlan> tensor_write_plans_from_export_infos(const std::vector<TensorExportInfo>& tensors) {
static std::vector<TensorWritePlan> tensor_write_plans_from_export_infos(const std::vector<TensorExportInfo>& tensors,
bool convert_name,
SDVersion version) {
std::vector<TensorWritePlan> plans;
plans.reserve(tensors.size());
for (const TensorExportInfo& info : tensors) {
plans.push_back(tensor_write_plan_from_export_info(info));
const std::string name = convert_name ? convert_tensor_name(info.storage.name, version) : info.storage.name;
plans.push_back(tensor_write_plan_from_export_info(info, name));
}
return plans;
}
@@ -158,16 +166,42 @@ static bool load_tensor_for_export(ModelLoader& model_loader, TensorExportJob& j
return true;
}
static bool export_tensor_from_memory(const TensorExportInfo& info,
const std::map<std::string, ggml_tensor*>& mem,
SDVersion version,
std::vector<uint8_t>& out) {
auto found = mem.find(info.storage.name);
if (found == mem.end() || found->second == nullptr || found->second->data == nullptr) {
return false;
}
const TensorStorage& storage = info.storage;
out.resize(export_tensor_nbytes(info));
if (out.size() > 0) {
std::vector<float> imatrix = get_imatrix_collector().get_values(convert_tensor_name(info.storage.name, version));
convert_tensor(found->second->data,
found->second->type,
out.data(),
info.type,
(int)(storage.nelements() / storage.ne[0]),
(int)storage.ne[0],
std::move(imatrix));
}
return true;
}
static bool stream_tensor_data(ModelLoader& model_loader,
const std::string& output_path,
const std::vector<TensorExportInfo>& tensors,
const StreamingModelWriter& writer,
int n_threads,
const std::map<std::string, ggml_tensor*>* mem,
std::string* error) {
n_threads = n_threads > 0 ? n_threads : sd_get_num_physical_cores();
n_threads = std::max(1, n_threads);
LOG_INFO("streaming convert with %d threads", n_threads);
const SDVersion version = model_loader.get_sd_version();
int64_t start_time = ggml_time_ms();
uint64_t bytes_written = 0;
size_t tensors_written = 0;
@@ -241,7 +275,10 @@ static bool stream_tensor_data(ModelLoader& model_loader,
TensorExportJob job;
job.info = tensors[tensor_index];
try {
job.success = load_tensor_for_export(model_loader, job);
job.success = (mem != nullptr) && export_tensor_from_memory(job.info, *mem, version, job.data);
if (!job.success) {
job.success = load_tensor_for_export(model_loader, job);
}
} catch (const std::exception& e) {
job.error = e.what();
job.success = false;
@@ -298,8 +335,10 @@ static bool write_model_file_streaming(ModelLoader& model_loader,
const std::vector<TensorExportInfo>& tensors,
StreamingModelWriter& writer,
int n_threads,
bool convert_name,
const std::map<std::string, ggml_tensor*>* mem,
std::string* error) {
std::vector<TensorWritePlan> plans = tensor_write_plans_from_export_infos(tensors);
std::vector<TensorWritePlan> plans = tensor_write_plans_from_export_infos(tensors, convert_name, model_loader.get_sd_version());
if (!writer.write_metadata(output_path, plans, error)) {
return false;
}
@@ -307,7 +346,7 @@ static bool write_model_file_streaming(ModelLoader& model_loader,
return false;
}
model_loader.process_model_files(false, false);
return stream_tensor_data(model_loader, output_path, tensors, writer, n_threads, error);
return stream_tensor_data(model_loader, output_path, tensors, writer, n_threads, mem, error);
}
static bool init_convert_path(ModelLoader& model_loader, const char* path, const char* prefix, bool& loaded_any) {
@@ -326,7 +365,9 @@ static bool export_loaded_model(ModelLoader& model_loader,
const char* output_path,
sd_type_t output_type,
const char* tensor_type_rules,
int n_threads) {
int n_threads,
bool convert_name,
const std::map<std::string, ggml_tensor*>* mem = nullptr) {
ggml_type type = sd_type_to_ggml_type(output_type);
bool output_is_safetensors = ends_with(output_path, ".safetensors");
TensorTypeRules type_rules = parse_tensor_type_rules(tensor_type_rules);
@@ -341,7 +382,7 @@ static bool export_loaded_model(ModelLoader& model_loader,
} else {
writer = std::make_unique<GGUFStreamingWriter>();
}
success = write_model_file_streaming(model_loader, output_path, tensors, *writer, n_threads, &error);
success = write_model_file_streaming(model_loader, output_path, tensors, *writer, n_threads, convert_name, mem, &error);
}
if (!success && !error.empty()) {
@@ -351,6 +392,174 @@ static bool export_loaded_model(ModelLoader& model_loader,
return success;
}
static bool has_active_loras(const sd_lora_t* loras, int lora_count) {
if (loras == nullptr) {
return false;
}
for (int i = 0; i < lora_count; i++) {
if (loras[i].multiplier != 0.0f) {
return true;
}
}
return false;
}
static std::vector<ModelManager::LoraSpec> build_lora_specs(const sd_lora_t* loras, int lora_count) {
std::vector<ModelManager::LoraSpec> specs;
specs.reserve(lora_count);
for (int i = 0; i < lora_count; i++) {
ModelManager::LoraSpec spec;
spec.path = loras[i].path != nullptr ? loras[i].path : "";
spec.multiplier = loras[i].multiplier;
spec.is_high_noise = loras[i].is_high_noise;
spec.required = true;
specs.push_back(std::move(spec));
}
return specs;
}
static ModelComponent component_for_tensor_name(const std::string& name) {
if (starts_with(name, "text_encoders.") || starts_with(name, "cond_stage_model.")) {
return ModelComponent::Conditioner;
}
if (starts_with(name, "model.high_noise_diffusion_model.")) {
return ModelComponent::HighNoiseDiffusion;
}
if (starts_with(name, "model.diffusion_model.")) {
return ModelComponent::Diffusion;
}
if (starts_with(name, "vae.") || starts_with(name, "first_stage_model.")) {
return ModelComponent::VAE;
}
return ModelComponent::Diffusion;
}
static bool load_model_into_memory(ModelLoader& model_loader,
const sd_lora_t* loras,
int lora_count,
int n_threads,
ModelManager& manager,
ggml_backend_t cpu,
ggml_context* ctx,
std::map<std::string, ggml_tensor*>& mem,
std::vector<ggml_tensor*>& all_tensors,
ggml_type output_type,
const TensorTypeRules& type_rules) {
const String2TensorStorage& storage_map = model_loader.get_tensor_storage_map();
if (storage_map.empty()) {
LOG_ERROR("no tensors to load into memory for convert");
return false;
}
const SDVersion version = model_loader.get_sd_version();
ModelLoader merge_loader = model_loader;
merge_loader.convert_tensors_name();
std::map<ModelComponent, std::map<std::string, ggml_tensor*>> groups;
size_t total_bytes = 0;
for (const auto& [name, storage] : storage_map) {
ggml_type load_type = storage.type;
const ggml_type export_type = get_export_tensor_type(model_loader, storage, output_type, type_rules);
const bool quant_src = ggml_is_quantized(storage.type);
const bool quant_dst = ggml_is_quantized(export_type);
if (quant_src && quant_dst) {
load_type = GGML_TYPE_F16;
} else if (quant_src || quant_dst) {
load_type = quant_src ? export_type : storage.type;
} else if (storage.type == GGML_TYPE_F32 || export_type == GGML_TYPE_F32) {
load_type = GGML_TYPE_F32;
}
ggml_tensor* tensor = ggml_new_tensor(ctx, load_type, storage.n_dims, storage.ne);
if (tensor == nullptr) {
LOG_ERROR("failed to create tensor '%s' for in-memory convert", name.c_str());
return false;
}
const std::string merge_name = convert_tensor_name(name, version);
auto& group = groups[component_for_tensor_name(merge_name)];
if (!group.emplace(merge_name, tensor).second) {
LOG_ERROR("duplicate tensor name '%s' after name conversion", merge_name.c_str());
return false;
}
mem[name] = tensor;
all_tensors.push_back(tensor);
total_bytes += ggml_nbytes(tensor);
}
LOG_INFO("loading %zu tensors (%.2f GB) into memory for convert",
all_tensors.size(), total_bytes / (1024.0 * 1024.0 * 1024.0));
manager.set_n_threads(n_threads);
manager.set_enable_mmap(false);
if (!manager.set_loader(std::move(merge_loader))) {
LOG_ERROR("failed to set model loader for in-memory convert");
return false;
}
for (const auto& [component, tensors] : groups) {
if (!manager.register_param_tensors(component, tensors, ModelManager::ResidencyMode::ParamBackend, cpu, cpu)) {
LOG_ERROR("failed to register %s tensors for in-memory convert", model_component_name(component));
return false;
}
}
std::vector<ModelManager::LoraSpec> lora_specs = build_lora_specs(loras, lora_count);
if (!manager.prepare_lora_sources(lora_specs)) {
LOG_ERROR("failed to prepare LoRA sources for convert");
return false;
}
if (!manager.set_loras(lora_specs, version)) {
LOG_ERROR("failed to set LoRAs for convert");
return false;
}
if (!manager.prepare_params(all_tensors)) {
LOG_ERROR("failed to load model into memory for convert");
return false;
}
return true;
}
static bool convert_model_in_memory(ModelLoader& model_loader,
const char* output_path,
sd_type_t output_type,
const char* tensor_type_rules,
bool convert_name,
int n_threads,
const sd_lora_t* loras,
int lora_count) {
auto cpu = std::unique_ptr<ggml_backend, decltype(&ggml_backend_free)>(sd_backend_cpu_init(), ggml_backend_free);
if (cpu == nullptr) {
LOG_ERROR("failed to init CPU backend for in-memory convert");
return false;
}
ggml_init_params ctx_params;
ctx_params.mem_size = model_loader.get_tensor_storage_map().size() * ggml_tensor_overhead();
ctx_params.mem_buffer = nullptr;
ctx_params.no_alloc = true;
ggml_context* ctx = ggml_init(ctx_params);
if (ctx == nullptr) {
LOG_ERROR("ggml_init failed for in-memory convert");
return false;
}
ggml_type type = sd_type_to_ggml_type(output_type);
TensorTypeRules type_rules = parse_tensor_type_rules(tensor_type_rules);
std::vector<ggml_tensor*> all_tensors;
std::map<std::string, ggml_tensor*> mem;
bool success = false;
{
// The manager owns the buffers backing `mem` and must outlive the export; it is
// destroyed (freeing those buffers) before the tensor context is freed below.
ModelManager manager;
if (load_model_into_memory(model_loader, loras, lora_count, n_threads, manager, cpu.get(), ctx, mem, all_tensors, type, type_rules)) {
success = export_loaded_model(model_loader, output_path, output_type, tensor_type_rules, n_threads, convert_name, &mem);
}
}
ggml_free(ctx);
return success;
}
bool convert_with_components(const char* model_path,
const char* clip_l_path,
const char* clip_g_path,
@@ -361,10 +570,22 @@ bool convert_with_components(const char* model_path,
sd_type_t output_type,
const char* tensor_type_rules,
bool convert_name,
int n_threads) {
int n_threads,
const sd_lora_t* loras,
int lora_count) {
if (!validate_tensor_types(output_type, tensor_type_rules)) {
return false;
}
if (loras != nullptr) {
for (int i = 0; i < lora_count; i++) {
LOG_INFO("lora %d: '%s' (multiplier %.2f%s)",
i + 1, loras[i].path != nullptr ? loras[i].path : "",
loras[i].multiplier,
loras[i].is_high_noise ? ", high noise" : "");
}
}
ModelLoader model_loader;
bool loaded_any = false;
@@ -382,11 +603,17 @@ bool convert_with_components(const char* model_path,
return false;
}
if (has_active_loras(loras, lora_count)) {
LOG_INFO("loading the full model into memory to apply LoRAs");
return convert_model_in_memory(model_loader, output_path, output_type, tensor_type_rules, convert_name, n_threads,
loras, lora_count);
}
if (convert_name) {
model_loader.convert_tensors_name();
}
return export_loaded_model(model_loader, output_path, output_type, tensor_type_rules, n_threads);
return export_loaded_model(model_loader, output_path, output_type, tensor_type_rules, n_threads, false, nullptr);
}
bool convert(const char* input_path,
@@ -405,5 +632,7 @@ bool convert(const char* input_path,
output_type,
tensor_type_rules,
convert_name,
0,
nullptr,
0);
}
src/model_loader.cpp
+1 -1
View File
@@ -115,7 +115,7 @@ void convert_tensor(void* src,
ggml_type dst_type,
int nrows,
int n_per_row,
std::vector<float> imatrix = {}) {
std::vector<float> imatrix) {
int n = nrows * n_per_row;
if (src_type == dst_type) {
size_t nbytes = n * ggml_type_size(src_type) / ggml_blck_size(src_type);
src/model_loader.h
+8
View File
@@ -13,6 +13,14 @@
TensorTypeRules parse_tensor_type_rules(const std::string& tensor_type_rules);
void convert_tensor(void* src,
ggml_type src_type,
void* dst,
ggml_type dst_type,
int nrows,
int n_per_row,
std::vector<float> imatrix = {});
class MmapWrapper;
struct ModelFileData {