perf: improved flux attention qkv unpacking (#1306)

examples/server/main.cpp

@@ -266,6 +266,7 @@ void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
 struct LoraEntry {
     std::string name;
     std::string path;
+    std::string fullpath;
 };
 
 int main(int argc, const char** argv) {
@@ -321,7 +322,8 @@ int main(int argc, const char** argv) {
 
                 LoraEntry e;
                 e.name          = p.stem().u8string();
-                std::string rel = fs::relative(p, lora_dir).u8string();
+                e.fullpath      = p.u8string();
+                std::string rel = p.lexically_relative(lora_dir).u8string();
                 std::replace(rel.begin(), rel.end(), '\\', '/');
                 e.path = rel;
 
@@ -340,10 +342,11 @@ int main(int argc, const char** argv) {
         }
     };
 
-    auto is_valid_lora_path = [&](const std::string& path) -> bool {
+    auto get_lora_full_path = [&](const std::string& path) -> std::string {
         std::lock_guard<std::mutex> lock(lora_mutex);
-        return std::any_of(lora_cache.begin(), lora_cache.end(),
+        auto it = std::find_if(lora_cache.begin(), lora_cache.end(),
                            [&](const LoraEntry& e) { return e.path == path; });
+        return (it != lora_cache.end()) ? it->fullpath : "";
     };
 
     httplib::Server svr;
@@ -565,7 +568,8 @@ int main(int argc, const char** argv) {
             std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(prompt);
 
             size_t image_count = req.form.get_file_count("image[]");
-            if (image_count == 0) {
+            bool has_legacy_image = req.form.has_file("image");
+            if (image_count == 0 && !has_legacy_image) {
                 res.status = 400;
                 res.set_content(R"({"error":"at least one image[] required"})", "application/json");
                 return;
@@ -576,6 +580,10 @@ int main(int argc, const char** argv) {
                 auto file = req.form.get_file("image[]", i);
                 images_bytes.emplace_back(file.content.begin(), file.content.end());
             }
+            if (image_count == 0 && has_legacy_image) {
+                auto file = req.form.get_file("image");
+                images_bytes.emplace_back(file.content.begin(), file.content.end());
+            }
 
             std::vector<uint8_t> mask_bytes;
             if (req.form.has_file("mask")) {
@@ -862,11 +870,12 @@ int main(int argc, const char** argv) {
                         return bad("lora.path required");
                     }
 
-                    if (!is_valid_lora_path(path)) {
+                    std::string fullpath = get_lora_full_path(path);
+                    if (fullpath.empty()) {
                         return bad("invalid lora path: " + path);
                     }
 
-                    lora_path_storage.push_back(path);
+                    lora_path_storage.push_back(fullpath);
                     sd_lora_t l;
                     l.is_high_noise = is_high_noise;
                     l.multiplier    = multiplier;

src/flux.hpp

@@ -103,11 +103,13 @@ namespace Flux {
             auto norm     = std::dynamic_pointer_cast<QKNorm>(blocks["norm"]);
 
             auto qkv         = qkv_proj->forward(ctx, x);
-            auto qkv_vec     = ggml_ext_chunk(ctx->ggml_ctx, qkv, 3, 0, true);
-            int64_t head_dim = qkv_vec[0]->ne[0] / num_heads;
-            auto q           = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[0], head_dim, num_heads, qkv_vec[0]->ne[1], qkv_vec[0]->ne[2]);
-            auto k           = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[1], head_dim, num_heads, qkv_vec[1]->ne[1], qkv_vec[1]->ne[2]);
-            auto v           = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[2], head_dim, num_heads, qkv_vec[2]->ne[1], qkv_vec[2]->ne[2]);
+            int64_t head_dim = qkv->ne[0] / 3 / num_heads;
+            auto q           = ggml_view_4d(ctx->ggml_ctx, qkv, head_dim, num_heads, qkv->ne[1], qkv->ne[2],
+                                            qkv->nb[0] * head_dim, qkv->nb[1], qkv->nb[2], 0);
+            auto k           = ggml_view_4d(ctx->ggml_ctx, qkv, head_dim, num_heads, qkv->ne[1], qkv->ne[2],
+                                            qkv->nb[0] * head_dim, qkv->nb[1], qkv->nb[2], (qkv->nb[0]) * qkv->ne[0] / 3);
+            auto v           = ggml_view_4d(ctx->ggml_ctx, qkv, head_dim, num_heads, qkv->ne[1], qkv->ne[2],
+                                            qkv->nb[0] * head_dim, qkv->nb[1], qkv->nb[2], (qkv->nb[0]) * 2 * qkv->ne[0] / 3);
             q                = norm->query_norm(ctx, q);
             k                = norm->key_norm(ctx, k);
             return {q, k, v};
@@ -491,15 +493,14 @@ namespace Flux {
             auto x_mod   = Flux::modulate(ctx->ggml_ctx, pre_norm->forward(ctx, x), mod.shift, mod.scale);
             auto qkv_mlp = linear1->forward(ctx, x_mod);  // [N, n_token, hidden_size * 3 + mlp_hidden_dim*mlp_mult_factor]
 
-            auto q = ggml_view_3d(ctx->ggml_ctx, qkv_mlp, hidden_size, qkv_mlp->ne[1], qkv_mlp->ne[2], qkv_mlp->nb[1], qkv_mlp->nb[2], 0);
-            auto k = ggml_view_3d(ctx->ggml_ctx, qkv_mlp, hidden_size, qkv_mlp->ne[1], qkv_mlp->ne[2], qkv_mlp->nb[1], qkv_mlp->nb[2], hidden_size * qkv_mlp->nb[0]);
-            auto v = ggml_view_3d(ctx->ggml_ctx, qkv_mlp, hidden_size, qkv_mlp->ne[1], qkv_mlp->ne[2], qkv_mlp->nb[1], qkv_mlp->nb[2], hidden_size * 2 * qkv_mlp->nb[0]);
-
             int64_t head_dim = hidden_size / num_heads;
 
-            q = ggml_reshape_4d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, q), head_dim, num_heads, q->ne[1], q->ne[2]);  // [N, n_token, n_head, d_head]
-            k = ggml_reshape_4d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, k), head_dim, num_heads, k->ne[1], k->ne[2]);  // [N, n_token, n_head, d_head]
-            v = ggml_reshape_4d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, v), head_dim, num_heads, v->ne[1], v->ne[2]);  // [N, n_token, n_head, d_head]
+            auto q = ggml_view_4d(ctx->ggml_ctx, qkv_mlp, head_dim, num_heads, qkv_mlp->ne[1], qkv_mlp->ne[2],
+                                  qkv_mlp->nb[0] * head_dim, qkv_mlp->nb[1], qkv_mlp->nb[2], 0);
+            auto k = ggml_view_4d(ctx->ggml_ctx, qkv_mlp, head_dim, num_heads, qkv_mlp->ne[1], qkv_mlp->ne[2],
+                                  qkv_mlp->nb[0] * head_dim, qkv_mlp->nb[1], qkv_mlp->nb[2], (qkv_mlp->nb[0]) * hidden_size);
+            auto v = ggml_view_4d(ctx->ggml_ctx, qkv_mlp, head_dim, num_heads, qkv_mlp->ne[1], qkv_mlp->ne[2],
+                                  qkv_mlp->nb[0] * head_dim, qkv_mlp->nb[1], qkv_mlp->nb[2], (qkv_mlp->nb[0]) * 2 * hidden_size);
 
             q         = norm->query_norm(ctx, q);
             k         = norm->key_norm(ctx, k);

src/model.cpp

@@ -1057,7 +1057,7 @@ SDVersion ModelLoader::get_sd_version() {
             if (tensor_storage.name.find("model.diffusion_model.transformer_blocks.0.img_mod.1.weight") != std::string::npos) {
                 return VERSION_QWEN_IMAGE;
             }
-            if (tensor_storage.name.find("model.diffusion_model.net.llm_adapter.blocks.0.cross_attn.q_proj.weight") != std::string::npos) {
+            if (tensor_storage.name.find("llm_adapter.blocks.0.cross_attn.q_proj.weight") != std::string::npos) {
                 return VERSION_ANIMA;
             }
             if (tensor_storage.name.find("model.diffusion_model.double_stream_modulation_img.lin.weight") != std::string::npos) {

src/name_conversion.cpp

@@ -653,6 +653,14 @@ std::string convert_diffusers_dit_to_original_lumina2(std::string name) {
     return name;
 }
 
+std::string convert_other_dit_to_original_anima(std::string name) {
+    static const std::string anima_net_prefix = "net.";
+    if (!starts_with(name, anima_net_prefix)) {
+        name = anima_net_prefix + name;
+    }
+    return name;
+}
+
 std::string convert_diffusion_model_name(std::string name, std::string prefix, SDVersion version) {
     if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
         name = convert_diffusers_unet_to_original_sd1(name);
@@ -664,6 +672,8 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S
         name = convert_diffusers_dit_to_original_flux(name);
     } else if (sd_version_is_z_image(version)) {
         name = convert_diffusers_dit_to_original_lumina2(name);
+    } else if (sd_version_is_anima(version)) {
+        name = convert_other_dit_to_original_anima(name);
     }
     return name;
 }
@@ -1094,14 +1104,6 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
         }
     }
 
-    if (is_lora && sd_version_is_anima(version)) {
-        static const std::string anima_diffusion_prefix = "model.diffusion_model.";
-        static const std::string anima_net_prefix       = "model.diffusion_model.net.";
-        if (starts_with(name, anima_diffusion_prefix) && !starts_with(name, anima_net_prefix)) {
-            name = anima_net_prefix + name.substr(anima_diffusion_prefix.size());
-        }
-    }
-
     // cond_stage_model
     {
         for (const auto& prefix : cond_stage_model_prefix_vec) {

src/stable-diffusion.cpp

@@ -1098,6 +1098,18 @@ public:
         cond_stage_lora_models.clear();
         diffusion_lora_models.clear();
         first_stage_lora_models.clear();
+        if (cond_stage_model) {
+            cond_stage_model->set_weight_adapter(nullptr);
+        }
+        if (diffusion_model) {
+            diffusion_model->set_weight_adapter(nullptr);
+        }
+        if (high_noise_diffusion_model) {
+            high_noise_diffusion_model->set_weight_adapter(nullptr);
+        }
+        if (first_stage_model) {
+            first_stage_model->set_weight_adapter(nullptr);
+        }
         if (lora_state.empty()) {
             return;
         }