mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-10-16 05:29:43 +08:00
Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 leejet
|
ee89afc878 | ||
|
akleine
|
d2d3944f50 | ||
|
akleine
|
0fa3e1a383 | ||
|
leejet
|
c2d8ffc22c | ||
|
stduhpf
|
fb748bb8a4 | ||
|
leejet
|
8f6c5c217b | ||
|
leejet
|
6103d86e2c | ||
|
stduhpf
|
c42826b77c | ||
|
Wagner Bruna
|
945d9a9ee3 | ||
|
Wagner Bruna
|
353e708844 | ||
|
leejet
|
dd75fc081c | ||
|
stduhpf
|
77eb95f8e4 | ||
|
Wagner Bruna
|
8a45d0ff7f | ||
|
leejet
|
9e28be6479 | ||
|
akleineandleejet
|
062490aa7c | ||
|
stduhpf
|
faabc5ad3c | ||
|
leejet
|
69b9511ce9 | ||
|
stduhpf
|
917f7bfe99 | ||
|
leejet
|
48e0a28ddf |
@@ -29,15 +29,17 @@ API and command-line option may change frequently.***
|
||||
|
||||
## Features
|
||||
|
||||
- Plain C/C++ implementation based on [ggml](https://github.com/ggerganov/ggml), working in the same way as [llama.cpp](https://github.com/ggerganov/llama.cpp)
|
||||
- Plain C/C++ implementation based on [ggml](https://github.com/ggml-org/ggml), working in the same way as [llama.cpp](https://github.com/ggml-org/llama.cpp)
|
||||
- Super lightweight and without external dependencies
|
||||
- Supported models
|
||||
- Image Models
|
||||
- SD1.x, SD2.x, [SD-Turbo](https://huggingface.co/stabilityai/sd-turbo)
|
||||
- SDXL, [SDXL-Turbo](https://huggingface.co/stabilityai/sdxl-turbo)
|
||||
- [Some SD1.x and SDXL distilled models](./docs/distilled_sd.md)
|
||||
- [SD3/SD3.5](./docs/sd3.md)
|
||||
- [Flux-dev/Flux-schnell](./docs/flux.md)
|
||||
- [Chroma](./docs/chroma.md)
|
||||
- [Chroma1-Radiance](./docs/chroma_radiance.md)
|
||||
- [Qwen Image](./docs/qwen_image.md)
|
||||
- Image Edit Models
|
||||
- [FLUX.1-Kontext-dev](./docs/kontext.md)
|
||||
@@ -150,6 +152,7 @@ These projects use `stable-diffusion.cpp` as a backend for their image generatio
|
||||
- [sd.cpp-webui](https://github.com/daniandtheweb/sd.cpp-webui)
|
||||
- [LocalAI](https://github.com/mudler/LocalAI)
|
||||
- [Neural-Pixel](https://github.com/Luiz-Alcantara/Neural-Pixel)
|
||||
- [KoboldCpp](https://github.com/LostRuins/koboldcpp)
|
||||
|
||||
## Contributors
|
||||
|
||||
@@ -163,7 +166,7 @@ Thank you to all the people who have already contributed to stable-diffusion.cpp
|
||||
|
||||
## References
|
||||
|
||||
- [ggml](https://github.com/ggerganov/ggml)
|
||||
- [ggml](https://github.com/ggml-org/ggml)
|
||||
- [diffusers](https://github.com/huggingface/diffusers)
|
||||
- [stable-diffusion](https://github.com/CompVis/stable-diffusion)
|
||||
- [sd3-ref](https://github.com/Stability-AI/sd3-ref)
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 477 KiB |
@@ -451,16 +451,16 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [N, n_token, d_model]
|
||||
auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
|
||||
auto fc2 = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
|
||||
|
||||
x = fc1->forward(ctx, x);
|
||||
if (use_gelu) {
|
||||
x = ggml_gelu_inplace(ctx, x);
|
||||
x = ggml_gelu_inplace(ctx->ggml_ctx, x);
|
||||
} else {
|
||||
x = ggml_gelu_quick_inplace(ctx, x);
|
||||
x = ggml_gelu_quick_inplace(ctx->ggml_ctx, x);
|
||||
}
|
||||
x = fc2->forward(ctx, x);
|
||||
return x;
|
||||
@@ -476,11 +476,12 @@ protected:
|
||||
public:
|
||||
CLIPLayer(int64_t d_model,
|
||||
int64_t n_head,
|
||||
int64_t intermediate_size)
|
||||
int64_t intermediate_size,
|
||||
bool proj_in = false)
|
||||
: d_model(d_model),
|
||||
n_head(n_head),
|
||||
intermediate_size(intermediate_size) {
|
||||
blocks["self_attn"] = std::shared_ptr<GGMLBlock>(new MultiheadAttention(d_model, n_head, true, true));
|
||||
blocks["self_attn"] = std::shared_ptr<GGMLBlock>(new MultiheadAttention(d_model, n_head, true, true, proj_in));
|
||||
|
||||
blocks["layer_norm1"] = std::shared_ptr<GGMLBlock>(new LayerNorm(d_model));
|
||||
blocks["layer_norm2"] = std::shared_ptr<GGMLBlock>(new LayerNorm(d_model));
|
||||
@@ -488,15 +489,15 @@ public:
|
||||
blocks["mlp"] = std::shared_ptr<GGMLBlock>(new CLIPMLP(d_model, intermediate_size));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, ggml_backend_t backend, struct ggml_tensor* x, bool mask = true) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x, bool mask = true) {
|
||||
// x: [N, n_token, d_model]
|
||||
auto self_attn = std::dynamic_pointer_cast<MultiheadAttention>(blocks["self_attn"]);
|
||||
auto layer_norm1 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm1"]);
|
||||
auto layer_norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm2"]);
|
||||
auto mlp = std::dynamic_pointer_cast<CLIPMLP>(blocks["mlp"]);
|
||||
|
||||
x = ggml_add(ctx, x, self_attn->forward(ctx, backend, layer_norm1->forward(ctx, x), mask));
|
||||
x = ggml_add(ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x)));
|
||||
x = ggml_add(ctx->ggml_ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask));
|
||||
x = ggml_add(ctx->ggml_ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x)));
|
||||
return x;
|
||||
}
|
||||
};
|
||||
@@ -509,16 +510,16 @@ public:
|
||||
CLIPEncoder(int64_t n_layer,
|
||||
int64_t d_model,
|
||||
int64_t n_head,
|
||||
int64_t intermediate_size)
|
||||
int64_t intermediate_size,
|
||||
bool proj_in = false)
|
||||
: n_layer(n_layer) {
|
||||
for (int i = 0; i < n_layer; i++) {
|
||||
std::string name = "layers." + std::to_string(i);
|
||||
blocks[name] = std::shared_ptr<GGMLBlock>(new CLIPLayer(d_model, n_head, intermediate_size));
|
||||
blocks[name] = std::shared_ptr<GGMLBlock>(new CLIPLayer(d_model, n_head, intermediate_size, proj_in));
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
int clip_skip = -1,
|
||||
bool mask = true) {
|
||||
@@ -536,7 +537,7 @@ public:
|
||||
}
|
||||
std::string name = "layers." + std::to_string(i);
|
||||
auto layer = std::dynamic_pointer_cast<CLIPLayer>(blocks[name]);
|
||||
x = layer->forward(ctx, backend, x, mask); // [N, n_token, d_model]
|
||||
x = layer->forward(ctx, x, mask); // [N, n_token, d_model]
|
||||
// LOG_DEBUG("layer %d", i);
|
||||
}
|
||||
return x;
|
||||
@@ -550,10 +551,10 @@ protected:
|
||||
int64_t num_positions;
|
||||
bool force_clip_f32;
|
||||
|
||||
void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, const std::string prefix = "") override {
|
||||
void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
||||
enum ggml_type token_wtype = GGML_TYPE_F32;
|
||||
if (!force_clip_f32) {
|
||||
token_wtype = get_type(prefix + "token_embedding.weight", tensor_types, GGML_TYPE_F32);
|
||||
token_wtype = get_type(prefix + "token_embedding.weight", tensor_storage_map, GGML_TYPE_F32);
|
||||
if (!support_get_rows(token_wtype)) {
|
||||
token_wtype = GGML_TYPE_F32;
|
||||
}
|
||||
@@ -578,7 +579,7 @@ public:
|
||||
return params["token_embedding.weight"];
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* input_ids,
|
||||
struct ggml_tensor* custom_embed_weight) {
|
||||
// input_ids: [N, n_token]
|
||||
@@ -586,12 +587,12 @@ public:
|
||||
auto position_embed_weight = params["position_embedding.weight"];
|
||||
|
||||
GGML_ASSERT(input_ids->ne[0] == position_embed_weight->ne[1]);
|
||||
input_ids = ggml_reshape_3d(ctx, input_ids, input_ids->ne[0], 1, input_ids->ne[1]);
|
||||
auto token_embedding = ggml_get_rows(ctx, custom_embed_weight != nullptr ? custom_embed_weight : token_embed_weight, input_ids);
|
||||
token_embedding = ggml_reshape_3d(ctx, token_embedding, token_embedding->ne[0], token_embedding->ne[1], token_embedding->ne[3]);
|
||||
input_ids = ggml_reshape_3d(ctx->ggml_ctx, input_ids, input_ids->ne[0], 1, input_ids->ne[1]);
|
||||
auto token_embedding = ggml_get_rows(ctx->ggml_ctx, custom_embed_weight != nullptr ? custom_embed_weight : token_embed_weight, input_ids);
|
||||
token_embedding = ggml_reshape_3d(ctx->ggml_ctx, token_embedding, token_embedding->ne[0], token_embedding->ne[1], token_embedding->ne[3]);
|
||||
|
||||
// token_embedding + position_embedding
|
||||
auto x = ggml_add(ctx,
|
||||
auto x = ggml_add(ctx->ggml_ctx,
|
||||
token_embedding,
|
||||
position_embed_weight); // [N, n_token, embed_dim]
|
||||
return x;
|
||||
@@ -606,7 +607,8 @@ protected:
|
||||
int64_t image_size;
|
||||
int64_t num_patches;
|
||||
int64_t num_positions;
|
||||
void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, const std::string prefix = "") override {
|
||||
|
||||
void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
||||
enum ggml_type patch_wtype = GGML_TYPE_F16;
|
||||
enum ggml_type class_wtype = GGML_TYPE_F32;
|
||||
enum ggml_type position_wtype = GGML_TYPE_F32;
|
||||
@@ -629,7 +631,7 @@ public:
|
||||
num_positions = num_patches + 1;
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* pixel_values) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* pixel_values) {
|
||||
// pixel_values: [N, num_channels, image_size, image_size]
|
||||
// return: [N, num_positions, embed_dim]
|
||||
GGML_ASSERT(pixel_values->ne[0] == image_size && pixel_values->ne[1] == image_size && pixel_values->ne[2] == num_channels);
|
||||
@@ -641,18 +643,18 @@ public:
|
||||
// concat(patch_embedding, class_embedding) + position_embedding
|
||||
struct ggml_tensor* patch_embedding;
|
||||
int64_t N = pixel_values->ne[3];
|
||||
patch_embedding = ggml_nn_conv_2d(ctx, pixel_values, patch_embed_weight, nullptr, patch_size, patch_size); // [N, embed_dim, image_size // pacht_size, image_size // pacht_size]
|
||||
patch_embedding = ggml_reshape_3d(ctx, patch_embedding, num_patches, embed_dim, N); // [N, embed_dim, num_patches]
|
||||
patch_embedding = ggml_cont(ctx, ggml_permute(ctx, patch_embedding, 1, 0, 2, 3)); // [N, num_patches, embed_dim]
|
||||
patch_embedding = ggml_reshape_4d(ctx, patch_embedding, 1, embed_dim, num_patches, N); // [N, num_patches, embed_dim, 1]
|
||||
patch_embedding = ggml_ext_conv_2d(ctx->ggml_ctx, pixel_values, patch_embed_weight, nullptr, patch_size, patch_size); // [N, embed_dim, image_size // pacht_size, image_size // pacht_size]
|
||||
patch_embedding = ggml_reshape_3d(ctx->ggml_ctx, patch_embedding, num_patches, embed_dim, N); // [N, embed_dim, num_patches]
|
||||
patch_embedding = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, patch_embedding, 1, 0, 2, 3)); // [N, num_patches, embed_dim]
|
||||
patch_embedding = ggml_reshape_4d(ctx->ggml_ctx, patch_embedding, 1, embed_dim, num_patches, N); // [N, num_patches, embed_dim, 1]
|
||||
|
||||
struct ggml_tensor* class_embedding = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, embed_dim, N);
|
||||
class_embedding = ggml_repeat(ctx, class_embed_weight, class_embedding); // [N, embed_dim]
|
||||
class_embedding = ggml_reshape_4d(ctx, class_embedding, 1, embed_dim, 1, N); // [N, 1, embed_dim, 1]
|
||||
struct ggml_tensor* class_embedding = ggml_new_tensor_2d(ctx->ggml_ctx, GGML_TYPE_F32, embed_dim, N);
|
||||
class_embedding = ggml_repeat(ctx->ggml_ctx, class_embed_weight, class_embedding); // [N, embed_dim]
|
||||
class_embedding = ggml_reshape_4d(ctx->ggml_ctx, class_embedding, 1, embed_dim, 1, N); // [N, 1, embed_dim, 1]
|
||||
|
||||
struct ggml_tensor* x = ggml_concat(ctx, class_embedding, patch_embedding, 2); // [N, num_positions, embed_dim, 1]
|
||||
x = ggml_reshape_3d(ctx, x, embed_dim, num_positions, N); // [N, num_positions, embed_dim]
|
||||
x = ggml_add(ctx, x, position_embed_weight);
|
||||
struct ggml_tensor* x = ggml_concat(ctx->ggml_ctx, class_embedding, patch_embedding, 2); // [N, num_positions, embed_dim, 1]
|
||||
x = ggml_reshape_3d(ctx->ggml_ctx, x, embed_dim, num_positions, N); // [N, num_positions, embed_dim]
|
||||
x = ggml_add(ctx->ggml_ctx, x, position_embed_weight);
|
||||
return x; // [N, num_positions, embed_dim]
|
||||
}
|
||||
};
|
||||
@@ -669,7 +671,7 @@ enum CLIPVersion {
|
||||
|
||||
class CLIPTextModel : public GGMLBlock {
|
||||
protected:
|
||||
void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, const std::string prefix = "") override {
|
||||
void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
||||
if (version == OPEN_CLIP_VIT_BIGG_14) {
|
||||
enum ggml_type wtype = GGML_TYPE_F32;
|
||||
params["text_projection"] = ggml_new_tensor_2d(ctx, wtype, projection_dim, hidden_size);
|
||||
@@ -690,7 +692,8 @@ public:
|
||||
|
||||
CLIPTextModel(CLIPVersion version = OPENAI_CLIP_VIT_L_14,
|
||||
bool with_final_ln = true,
|
||||
bool force_clip_f32 = false)
|
||||
bool force_clip_f32 = false,
|
||||
bool proj_in = false)
|
||||
: version(version), with_final_ln(with_final_ln) {
|
||||
if (version == OPEN_CLIP_VIT_H_14) {
|
||||
hidden_size = 1024;
|
||||
@@ -705,7 +708,7 @@ public:
|
||||
}
|
||||
|
||||
blocks["embeddings"] = std::shared_ptr<GGMLBlock>(new CLIPEmbeddings(hidden_size, vocab_size, n_token, force_clip_f32));
|
||||
blocks["encoder"] = std::shared_ptr<GGMLBlock>(new CLIPEncoder(n_layer, hidden_size, n_head, intermediate_size));
|
||||
blocks["encoder"] = std::shared_ptr<GGMLBlock>(new CLIPEncoder(n_layer, hidden_size, n_head, intermediate_size, proj_in));
|
||||
blocks["final_layer_norm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size));
|
||||
}
|
||||
|
||||
@@ -714,8 +717,7 @@ public:
|
||||
return embeddings->get_token_embed_weight();
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* input_ids,
|
||||
struct ggml_tensor* tkn_embeddings,
|
||||
size_t max_token_idx = 0,
|
||||
@@ -727,16 +729,16 @@ public:
|
||||
auto final_layer_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["final_layer_norm"]);
|
||||
|
||||
auto x = embeddings->forward(ctx, input_ids, tkn_embeddings); // [N, n_token, hidden_size]
|
||||
x = encoder->forward(ctx, backend, x, return_pooled ? -1 : clip_skip, true);
|
||||
x = encoder->forward(ctx, x, return_pooled ? -1 : clip_skip, true);
|
||||
if (return_pooled || with_final_ln) {
|
||||
x = final_layer_norm->forward(ctx, x);
|
||||
}
|
||||
|
||||
if (return_pooled) {
|
||||
auto text_projection = params["text_projection"];
|
||||
ggml_tensor* pooled = ggml_view_1d(ctx, x, hidden_size, x->nb[1] * max_token_idx);
|
||||
ggml_tensor* pooled = ggml_view_1d(ctx->ggml_ctx, x, hidden_size, x->nb[1] * max_token_idx);
|
||||
if (text_projection != nullptr) {
|
||||
pooled = ggml_nn_linear(ctx, pooled, text_projection, nullptr);
|
||||
pooled = ggml_ext_linear(ctx->ggml_ctx, pooled, text_projection, nullptr);
|
||||
} else {
|
||||
LOG_DEBUG("identity projection");
|
||||
}
|
||||
@@ -760,7 +762,7 @@ public:
|
||||
int32_t n_layer = 24;
|
||||
|
||||
public:
|
||||
CLIPVisionModel(CLIPVersion version = OPENAI_CLIP_VIT_L_14) {
|
||||
CLIPVisionModel(CLIPVersion version = OPENAI_CLIP_VIT_L_14, bool proj_in = false) {
|
||||
if (version == OPEN_CLIP_VIT_H_14) {
|
||||
hidden_size = 1280;
|
||||
intermediate_size = 5120;
|
||||
@@ -775,12 +777,11 @@ public:
|
||||
|
||||
blocks["embeddings"] = std::shared_ptr<GGMLBlock>(new CLIPVisionEmbeddings(hidden_size, num_channels, patch_size, image_size));
|
||||
blocks["pre_layernorm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size));
|
||||
blocks["encoder"] = std::shared_ptr<GGMLBlock>(new CLIPEncoder(n_layer, hidden_size, n_head, intermediate_size));
|
||||
blocks["encoder"] = std::shared_ptr<GGMLBlock>(new CLIPEncoder(n_layer, hidden_size, n_head, intermediate_size, proj_in));
|
||||
blocks["post_layernorm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* pixel_values,
|
||||
bool return_pooled = true,
|
||||
int clip_skip = -1) {
|
||||
@@ -792,14 +793,14 @@ public:
|
||||
|
||||
auto x = embeddings->forward(ctx, pixel_values); // [N, num_positions, embed_dim]
|
||||
x = pre_layernorm->forward(ctx, x);
|
||||
x = encoder->forward(ctx, backend, x, clip_skip, false);
|
||||
x = encoder->forward(ctx, x, clip_skip, false);
|
||||
// print_ggml_tensor(x, true, "ClipVisionModel x: ");
|
||||
auto last_hidden_state = x;
|
||||
x = post_layernorm->forward(ctx, x); // [N, n_token, hidden_size]
|
||||
|
||||
GGML_ASSERT(x->ne[3] == 1);
|
||||
if (return_pooled) {
|
||||
ggml_tensor* pooled = ggml_cont(ctx, ggml_view_2d(ctx, x, x->ne[0], x->ne[2], x->nb[2], 0));
|
||||
ggml_tensor* pooled = ggml_cont(ctx->ggml_ctx, ggml_view_2d(ctx->ggml_ctx, x, x->ne[0], x->ne[2], x->nb[2], 0));
|
||||
return pooled; // [N, hidden_size]
|
||||
} else {
|
||||
// return x; // [N, n_token, hidden_size]
|
||||
@@ -814,8 +815,8 @@ protected:
|
||||
int64_t out_features;
|
||||
bool transpose_weight;
|
||||
|
||||
void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, const std::string prefix = "") override {
|
||||
enum ggml_type wtype = get_type(prefix + "weight", tensor_types, GGML_TYPE_F32);
|
||||
void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
||||
enum ggml_type wtype = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
|
||||
if (transpose_weight) {
|
||||
params["weight"] = ggml_new_tensor_2d(ctx, wtype, out_features, in_features);
|
||||
} else {
|
||||
@@ -831,12 +832,12 @@ public:
|
||||
out_features(out_features),
|
||||
transpose_weight(transpose_weight) {}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* w = params["weight"];
|
||||
if (transpose_weight) {
|
||||
w = ggml_cont(ctx, ggml_transpose(ctx, w));
|
||||
w = ggml_cont(ctx->ggml_ctx, ggml_transpose(ctx->ggml_ctx, w));
|
||||
}
|
||||
return ggml_nn_linear(ctx, x, w, nullptr);
|
||||
return ggml_ext_linear(ctx->ggml_ctx, x, w, nullptr);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -848,7 +849,8 @@ public:
|
||||
|
||||
public:
|
||||
CLIPVisionModelProjection(CLIPVersion version = OPENAI_CLIP_VIT_L_14,
|
||||
bool transpose_proj_w = false) {
|
||||
bool transpose_proj_w = false,
|
||||
bool proj_in = false) {
|
||||
if (version == OPEN_CLIP_VIT_H_14) {
|
||||
hidden_size = 1280;
|
||||
projection_dim = 1024;
|
||||
@@ -856,12 +858,11 @@ public:
|
||||
hidden_size = 1664;
|
||||
}
|
||||
|
||||
blocks["vision_model"] = std::shared_ptr<GGMLBlock>(new CLIPVisionModel(version));
|
||||
blocks["vision_model"] = std::shared_ptr<GGMLBlock>(new CLIPVisionModel(version, proj_in));
|
||||
blocks["visual_projection"] = std::shared_ptr<GGMLBlock>(new CLIPProjection(hidden_size, projection_dim, transpose_proj_w));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* pixel_values,
|
||||
bool return_pooled = true,
|
||||
int clip_skip = -1) {
|
||||
@@ -870,7 +871,7 @@ public:
|
||||
auto vision_model = std::dynamic_pointer_cast<CLIPVisionModel>(blocks["vision_model"]);
|
||||
auto visual_projection = std::dynamic_pointer_cast<CLIPProjection>(blocks["visual_projection"]);
|
||||
|
||||
auto x = vision_model->forward(ctx, backend, pixel_values, return_pooled, clip_skip); // [N, hidden_size] or [N, n_token, hidden_size]
|
||||
auto x = vision_model->forward(ctx, pixel_values, return_pooled, clip_skip); // [N, hidden_size] or [N, n_token, hidden_size]
|
||||
|
||||
if (return_pooled) {
|
||||
x = visual_projection->forward(ctx, x); // [N, projection_dim]
|
||||
@@ -885,13 +886,24 @@ struct CLIPTextModelRunner : public GGMLRunner {
|
||||
|
||||
CLIPTextModelRunner(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types,
|
||||
const String2TensorStorage& tensor_storage_map,
|
||||
const std::string prefix,
|
||||
CLIPVersion version = OPENAI_CLIP_VIT_L_14,
|
||||
bool with_final_ln = true,
|
||||
bool force_clip_f32 = false)
|
||||
: GGMLRunner(backend, offload_params_to_cpu), model(version, with_final_ln, force_clip_f32) {
|
||||
model.init(params_ctx, tensor_types, prefix);
|
||||
: GGMLRunner(backend, offload_params_to_cpu) {
|
||||
bool proj_in = false;
|
||||
for (const auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (!starts_with(name, prefix)) {
|
||||
continue;
|
||||
}
|
||||
if (contains(name, "self_attn.in_proj")) {
|
||||
proj_in = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
model = CLIPTextModel(version, with_final_ln, force_clip_f32, proj_in);
|
||||
model.init(params_ctx, tensor_storage_map, prefix);
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -902,8 +914,7 @@ struct CLIPTextModelRunner : public GGMLRunner {
|
||||
model.get_param_tensors(tensors, prefix);
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* input_ids,
|
||||
struct ggml_tensor* embeddings,
|
||||
size_t max_token_idx = 0,
|
||||
@@ -913,10 +924,10 @@ struct CLIPTextModelRunner : public GGMLRunner {
|
||||
size_t n_token = input_ids->ne[0];
|
||||
if (input_ids->ne[0] > model.n_token) {
|
||||
GGML_ASSERT(input_ids->ne[0] % model.n_token == 0);
|
||||
input_ids = ggml_reshape_2d(ctx, input_ids, model.n_token, input_ids->ne[0] / model.n_token);
|
||||
input_ids = ggml_reshape_2d(ctx->ggml_ctx, input_ids, model.n_token, input_ids->ne[0] / model.n_token);
|
||||
}
|
||||
|
||||
return model.forward(ctx, backend, input_ids, embeddings, max_token_idx, return_pooled, clip_skip);
|
||||
return model.forward(ctx, input_ids, embeddings, max_token_idx, return_pooled, clip_skip);
|
||||
}
|
||||
|
||||
struct ggml_cgraph* build_graph(struct ggml_tensor* input_ids,
|
||||
@@ -943,7 +954,9 @@ struct CLIPTextModelRunner : public GGMLRunner {
|
||||
embeddings = ggml_concat(compute_ctx, token_embed_weight, custom_embeddings, 1);
|
||||
}
|
||||
|
||||
struct ggml_tensor* hidden_states = forward(compute_ctx, runtime_backend, input_ids, embeddings, max_token_idx, return_pooled, clip_skip);
|
||||
auto runner_ctx = get_context();
|
||||
|
||||
struct ggml_tensor* hidden_states = forward(&runner_ctx, input_ids, embeddings, max_token_idx, return_pooled, clip_skip);
|
||||
|
||||
ggml_build_forward_expand(gf, hidden_states);
|
||||
|
||||
|
||||
+111
-80
@@ -23,12 +23,12 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [N, channels, h, w]
|
||||
if (vae_downsample) {
|
||||
auto conv = std::dynamic_pointer_cast<Conv2d>(blocks["conv"]);
|
||||
|
||||
x = ggml_pad(ctx, x, 1, 1, 0, 0);
|
||||
x = ggml_pad(ctx->ggml_ctx, x, 1, 1, 0, 0);
|
||||
x = conv->forward(ctx, x);
|
||||
} else {
|
||||
auto conv = std::dynamic_pointer_cast<Conv2d>(blocks["op"]);
|
||||
@@ -52,12 +52,12 @@ public:
|
||||
blocks["conv"] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, out_channels, {3, 3}, {1, 1}, {1, 1}));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [N, channels, h, w]
|
||||
auto conv = std::dynamic_pointer_cast<Conv2d>(blocks["conv"]);
|
||||
|
||||
x = ggml_upscale(ctx, x, 2, GGML_SCALE_MODE_NEAREST); // [N, channels, h*2, w*2]
|
||||
x = conv->forward(ctx, x); // [N, out_channels, h*2, w*2]
|
||||
x = ggml_upscale(ctx->ggml_ctx, x, 2, GGML_SCALE_MODE_NEAREST); // [N, channels, h*2, w*2]
|
||||
x = conv->forward(ctx, x); // [N, out_channels, h*2, w*2]
|
||||
return x;
|
||||
}
|
||||
};
|
||||
@@ -121,7 +121,7 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
virtual struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x, struct ggml_tensor* emb = nullptr) {
|
||||
virtual struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x, struct ggml_tensor* emb = nullptr) {
|
||||
// For dims==3, we reduce dimension from 5d to 4d by merging h and w, in order not to change ggml
|
||||
// [N, c, t, h, w] => [N, c, t, h * w]
|
||||
// x: [N, channels, h, w] if dims == 2 else [N, channels, t, h, w]
|
||||
@@ -137,32 +137,32 @@ public:
|
||||
|
||||
// in_layers
|
||||
auto h = in_layers_0->forward(ctx, x);
|
||||
h = ggml_silu_inplace(ctx, h);
|
||||
h = ggml_silu_inplace(ctx->ggml_ctx, h);
|
||||
h = in_layers_2->forward(ctx, h); // [N, out_channels, h, w] if dims == 2 else [N, out_channels, t, h, w]
|
||||
|
||||
// emb_layers
|
||||
if (!skip_t_emb) {
|
||||
auto emb_layer_1 = std::dynamic_pointer_cast<Linear>(blocks["emb_layers.1"]);
|
||||
|
||||
auto emb_out = ggml_silu(ctx, emb);
|
||||
auto emb_out = ggml_silu(ctx->ggml_ctx, emb);
|
||||
emb_out = emb_layer_1->forward(ctx, emb_out); // [N, out_channels] if dims == 2 else [N, t, out_channels]
|
||||
|
||||
if (dims == 2) {
|
||||
emb_out = ggml_reshape_4d(ctx, emb_out, 1, 1, emb_out->ne[0], emb_out->ne[1]); // [N, out_channels, 1, 1]
|
||||
emb_out = ggml_reshape_4d(ctx->ggml_ctx, emb_out, 1, 1, emb_out->ne[0], emb_out->ne[1]); // [N, out_channels, 1, 1]
|
||||
} else {
|
||||
emb_out = ggml_reshape_4d(ctx, emb_out, 1, emb_out->ne[0], emb_out->ne[1], emb_out->ne[2]); // [N, t, out_channels, 1]
|
||||
emb_out = ggml_reshape_4d(ctx->ggml_ctx, emb_out, 1, emb_out->ne[0], emb_out->ne[1], emb_out->ne[2]); // [N, t, out_channels, 1]
|
||||
if (exchange_temb_dims) {
|
||||
// emb_out = rearrange(emb_out, "b t c ... -> b c t ...")
|
||||
emb_out = ggml_cont(ctx, ggml_permute(ctx, emb_out, 0, 2, 1, 3)); // [N, out_channels, t, 1]
|
||||
emb_out = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, emb_out, 0, 2, 1, 3)); // [N, out_channels, t, 1]
|
||||
}
|
||||
}
|
||||
|
||||
h = ggml_add(ctx, h, emb_out); // [N, out_channels, h, w] if dims == 2 else [N, out_channels, t, h, w]
|
||||
h = ggml_add(ctx->ggml_ctx, h, emb_out); // [N, out_channels, h, w] if dims == 2 else [N, out_channels, t, h, w]
|
||||
}
|
||||
|
||||
// out_layers
|
||||
h = out_layers_0->forward(ctx, h);
|
||||
h = ggml_silu_inplace(ctx, h);
|
||||
h = ggml_silu_inplace(ctx->ggml_ctx, h);
|
||||
// dropout, skip for inference
|
||||
h = out_layers_3->forward(ctx, h);
|
||||
|
||||
@@ -172,7 +172,7 @@ public:
|
||||
x = skip_connection->forward(ctx, x); // [N, out_channels, h, w] if dims == 2 else [N, out_channels, t, h, w]
|
||||
}
|
||||
|
||||
h = ggml_add(ctx, h, x);
|
||||
h = ggml_add(ctx->ggml_ctx, h, x);
|
||||
return h; // [N, out_channels, h, w] if dims == 2 else [N, out_channels, t, h, w]
|
||||
}
|
||||
};
|
||||
@@ -182,8 +182,8 @@ protected:
|
||||
int64_t dim_in;
|
||||
int64_t dim_out;
|
||||
|
||||
void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, std::string prefix = "") override {
|
||||
enum ggml_type wtype = get_type(prefix + "proj.weight", tensor_types, GGML_TYPE_F32);
|
||||
void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") override {
|
||||
enum ggml_type wtype = get_type(prefix + "proj.weight", tensor_storage_map, GGML_TYPE_F32);
|
||||
enum ggml_type bias_wtype = GGML_TYPE_F32;
|
||||
params["proj.weight"] = ggml_new_tensor_2d(ctx, wtype, dim_in, dim_out * 2);
|
||||
params["proj.bias"] = ggml_new_tensor_1d(ctx, bias_wtype, dim_out * 2);
|
||||
@@ -193,24 +193,24 @@ public:
|
||||
GEGLU(int64_t dim_in, int64_t dim_out)
|
||||
: dim_in(dim_in), dim_out(dim_out) {}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
// x: [ne3, ne2, ne1, dim_in]
|
||||
// return: [ne3, ne2, ne1, dim_out]
|
||||
struct ggml_tensor* w = params["proj.weight"];
|
||||
struct ggml_tensor* b = params["proj.bias"];
|
||||
|
||||
auto x_w = ggml_view_2d(ctx, w, w->ne[0], w->ne[1] / 2, w->nb[1], 0); // [dim_out, dim_in]
|
||||
auto x_b = ggml_view_1d(ctx, b, b->ne[0] / 2, 0); // [dim_out, dim_in]
|
||||
auto gate_w = ggml_view_2d(ctx, w, w->ne[0], w->ne[1] / 2, w->nb[1], w->nb[1] * w->ne[1] / 2); // [dim_out, ]
|
||||
auto gate_b = ggml_view_1d(ctx, b, b->ne[0] / 2, b->nb[0] * b->ne[0] / 2); // [dim_out, ]
|
||||
auto x_w = ggml_view_2d(ctx->ggml_ctx, w, w->ne[0], w->ne[1] / 2, w->nb[1], 0); // [dim_out, dim_in]
|
||||
auto x_b = ggml_view_1d(ctx->ggml_ctx, b, b->ne[0] / 2, 0); // [dim_out, dim_in]
|
||||
auto gate_w = ggml_view_2d(ctx->ggml_ctx, w, w->ne[0], w->ne[1] / 2, w->nb[1], w->nb[1] * w->ne[1] / 2); // [dim_out, ]
|
||||
auto gate_b = ggml_view_1d(ctx->ggml_ctx, b, b->ne[0] / 2, b->nb[0] * b->ne[0] / 2); // [dim_out, ]
|
||||
|
||||
auto x_in = x;
|
||||
x = ggml_nn_linear(ctx, x_in, x_w, x_b); // [ne3, ne2, ne1, dim_out]
|
||||
auto gate = ggml_nn_linear(ctx, x_in, gate_w, gate_b); // [ne3, ne2, ne1, dim_out]
|
||||
x = ggml_ext_linear(ctx->ggml_ctx, x_in, x_w, x_b); // [ne3, ne2, ne1, dim_out]
|
||||
auto gate = ggml_ext_linear(ctx->ggml_ctx, x_in, gate_w, gate_b); // [ne3, ne2, ne1, dim_out]
|
||||
|
||||
gate = ggml_gelu_inplace(ctx, gate);
|
||||
gate = ggml_gelu_inplace(ctx->ggml_ctx, gate);
|
||||
|
||||
x = ggml_mul(ctx, x, gate); // [ne3, ne2, ne1, dim_out]
|
||||
x = ggml_mul(ctx->ggml_ctx, x, gate); // [ne3, ne2, ne1, dim_out]
|
||||
|
||||
return x;
|
||||
}
|
||||
@@ -222,13 +222,13 @@ public:
|
||||
blocks["proj"] = std::shared_ptr<GGMLBlock>(new Linear(dim_in, dim_out, bias));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
// x: [ne3, ne2, ne1, dim_in]
|
||||
// return: [ne3, ne2, ne1, dim_out]
|
||||
auto proj = std::dynamic_pointer_cast<Linear>(blocks["proj"]);
|
||||
|
||||
x = proj->forward(ctx, x);
|
||||
x = ggml_gelu_inplace(ctx, x);
|
||||
x = ggml_gelu_inplace(ctx->ggml_ctx, x);
|
||||
return x;
|
||||
}
|
||||
};
|
||||
@@ -262,7 +262,7 @@ public:
|
||||
blocks["net.2"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, dim_out, true, false, false, scale));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [ne3, ne2, ne1, dim]
|
||||
// return: [ne3, ne2, ne1, dim_out]
|
||||
|
||||
@@ -281,19 +281,16 @@ protected:
|
||||
int64_t context_dim;
|
||||
int64_t n_head;
|
||||
int64_t d_head;
|
||||
bool flash_attn;
|
||||
|
||||
public:
|
||||
CrossAttention(int64_t query_dim,
|
||||
int64_t context_dim,
|
||||
int64_t n_head,
|
||||
int64_t d_head,
|
||||
bool flash_attn = false)
|
||||
int64_t d_head)
|
||||
: n_head(n_head),
|
||||
d_head(d_head),
|
||||
query_dim(query_dim),
|
||||
context_dim(context_dim),
|
||||
flash_attn(flash_attn) {
|
||||
context_dim(context_dim) {
|
||||
int64_t inner_dim = d_head * n_head;
|
||||
|
||||
blocks["to_q"] = std::shared_ptr<GGMLBlock>(new Linear(query_dim, inner_dim, false));
|
||||
@@ -304,8 +301,7 @@ public:
|
||||
// to_out_1 is nn.Dropout(), skip for inference
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* context) {
|
||||
// x: [N, n_token, query_dim]
|
||||
@@ -325,7 +321,7 @@ public:
|
||||
auto k = to_k->forward(ctx, context); // [N, n_context, inner_dim]
|
||||
auto v = to_v->forward(ctx, context); // [N, n_context, inner_dim]
|
||||
|
||||
x = ggml_nn_attention_ext(ctx, backend, q, k, v, n_head, nullptr, false, false, flash_attn); // [N, n_token, inner_dim]
|
||||
x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, nullptr, false, false, ctx->flash_attn_enabled); // [N, n_token, inner_dim]
|
||||
|
||||
x = to_out_0->forward(ctx, x); // [N, n_token, query_dim]
|
||||
return x;
|
||||
@@ -343,16 +339,15 @@ public:
|
||||
int64_t n_head,
|
||||
int64_t d_head,
|
||||
int64_t context_dim,
|
||||
bool ff_in = false,
|
||||
bool flash_attn = false)
|
||||
bool ff_in = false)
|
||||
: n_head(n_head), d_head(d_head), ff_in(ff_in) {
|
||||
// disable_self_attn is always False
|
||||
// disable_temporal_crossattention is always False
|
||||
// switch_temporal_ca_to_sa is always False
|
||||
// inner_dim is always None or equal to dim
|
||||
// gated_ff is always True
|
||||
blocks["attn1"] = std::shared_ptr<GGMLBlock>(new CrossAttention(dim, dim, n_head, d_head, flash_attn));
|
||||
blocks["attn2"] = std::shared_ptr<GGMLBlock>(new CrossAttention(dim, context_dim, n_head, d_head, flash_attn));
|
||||
blocks["attn1"] = std::shared_ptr<GGMLBlock>(new CrossAttention(dim, dim, n_head, d_head));
|
||||
blocks["attn2"] = std::shared_ptr<GGMLBlock>(new CrossAttention(dim, context_dim, n_head, d_head));
|
||||
blocks["ff"] = std::shared_ptr<GGMLBlock>(new FeedForward(dim, dim));
|
||||
blocks["norm1"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
|
||||
blocks["norm2"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
|
||||
@@ -364,8 +359,7 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* context) {
|
||||
// x: [N, n_token, query_dim]
|
||||
@@ -387,21 +381,21 @@ public:
|
||||
x = norm_in->forward(ctx, x);
|
||||
x = ff_in->forward(ctx, x);
|
||||
// self.is_res is always True
|
||||
x = ggml_add(ctx, x, x_skip);
|
||||
x = ggml_add(ctx->ggml_ctx, x, x_skip);
|
||||
}
|
||||
|
||||
auto r = x;
|
||||
x = norm1->forward(ctx, x);
|
||||
x = attn1->forward(ctx, backend, x, x); // self-attention
|
||||
x = ggml_add(ctx, x, r);
|
||||
x = attn1->forward(ctx, x, x); // self-attention
|
||||
x = ggml_add(ctx->ggml_ctx, x, r);
|
||||
r = x;
|
||||
x = norm2->forward(ctx, x);
|
||||
x = attn2->forward(ctx, backend, x, context); // cross-attention
|
||||
x = ggml_add(ctx, x, r);
|
||||
x = attn2->forward(ctx, x, context); // cross-attention
|
||||
x = ggml_add(ctx->ggml_ctx, x, r);
|
||||
r = x;
|
||||
x = norm3->forward(ctx, x);
|
||||
x = ff->forward(ctx, x);
|
||||
x = ggml_add(ctx, x, r);
|
||||
x = ggml_add(ctx->ggml_ctx, x, r);
|
||||
|
||||
return x;
|
||||
}
|
||||
@@ -414,6 +408,23 @@ protected:
|
||||
int64_t d_head;
|
||||
int64_t depth = 1; // 1
|
||||
int64_t context_dim = 768; // hidden_size, 1024 for VERSION_SD2
|
||||
bool use_linear = false;
|
||||
|
||||
void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") {
|
||||
auto iter = tensor_storage_map.find(prefix + "proj_out.weight");
|
||||
if (iter != tensor_storage_map.end()) {
|
||||
int64_t inner_dim = n_head * d_head;
|
||||
if (iter->second.n_dims == 4 && use_linear) {
|
||||
use_linear = false;
|
||||
blocks["proj_in"] = std::make_shared<Conv2d>(in_channels, inner_dim, std::pair{1, 1});
|
||||
blocks["proj_out"] = std::make_shared<Conv2d>(inner_dim, in_channels, std::pair{1, 1});
|
||||
} else if (iter->second.n_dims == 2 && !use_linear) {
|
||||
use_linear = true;
|
||||
blocks["proj_in"] = std::make_shared<Linear>(in_channels, inner_dim);
|
||||
blocks["proj_out"] = std::make_shared<Linear>(inner_dim, in_channels);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
SpatialTransformer(int64_t in_channels,
|
||||
@@ -421,35 +432,42 @@ public:
|
||||
int64_t d_head,
|
||||
int64_t depth,
|
||||
int64_t context_dim,
|
||||
bool flash_attn = false)
|
||||
bool use_linear)
|
||||
: in_channels(in_channels),
|
||||
n_head(n_head),
|
||||
d_head(d_head),
|
||||
depth(depth),
|
||||
context_dim(context_dim) {
|
||||
// We will convert unet transformer linear to conv2d 1x1 when loading the weights, so use_linear is always False
|
||||
context_dim(context_dim),
|
||||
use_linear(use_linear) {
|
||||
// disable_self_attn is always False
|
||||
int64_t inner_dim = n_head * d_head; // in_channels
|
||||
blocks["norm"] = std::shared_ptr<GGMLBlock>(new GroupNorm32(in_channels));
|
||||
blocks["proj_in"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_channels, inner_dim, {1, 1}));
|
||||
if (use_linear) {
|
||||
blocks["proj_in"] = std::shared_ptr<GGMLBlock>(new Linear(in_channels, inner_dim));
|
||||
} else {
|
||||
blocks["proj_in"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_channels, inner_dim, {1, 1}));
|
||||
}
|
||||
|
||||
for (int i = 0; i < depth; i++) {
|
||||
std::string name = "transformer_blocks." + std::to_string(i);
|
||||
blocks[name] = std::shared_ptr<GGMLBlock>(new BasicTransformerBlock(inner_dim, n_head, d_head, context_dim, false, flash_attn));
|
||||
blocks[name] = std::shared_ptr<GGMLBlock>(new BasicTransformerBlock(inner_dim, n_head, d_head, context_dim, false));
|
||||
}
|
||||
|
||||
blocks["proj_out"] = std::shared_ptr<GGMLBlock>(new Conv2d(inner_dim, in_channels, {1, 1}));
|
||||
if (use_linear) {
|
||||
blocks["proj_out"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, in_channels));
|
||||
} else {
|
||||
blocks["proj_out"] = std::shared_ptr<GGMLBlock>(new Conv2d(inner_dim, in_channels, {1, 1}));
|
||||
}
|
||||
}
|
||||
|
||||
virtual struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
virtual struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* context) {
|
||||
// x: [N, in_channels, h, w]
|
||||
// context: [N, max_position(aka n_token), hidden_size(aka context_dim)]
|
||||
auto norm = std::dynamic_pointer_cast<GroupNorm32>(blocks["norm"]);
|
||||
auto proj_in = std::dynamic_pointer_cast<Conv2d>(blocks["proj_in"]);
|
||||
auto proj_out = std::dynamic_pointer_cast<Conv2d>(blocks["proj_out"]);
|
||||
auto proj_in = std::dynamic_pointer_cast<UnaryBlock>(blocks["proj_in"]);
|
||||
auto proj_out = std::dynamic_pointer_cast<UnaryBlock>(blocks["proj_out"]);
|
||||
|
||||
auto x_in = x;
|
||||
int64_t n = x->ne[3];
|
||||
@@ -458,32 +476,45 @@ public:
|
||||
int64_t inner_dim = n_head * d_head;
|
||||
|
||||
x = norm->forward(ctx, x);
|
||||
x = proj_in->forward(ctx, x); // [N, inner_dim, h, w]
|
||||
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 1, 2, 0, 3)); // [N, h, w, inner_dim]
|
||||
x = ggml_reshape_3d(ctx, x, inner_dim, w * h, n); // [N, h * w, inner_dim]
|
||||
if (use_linear) {
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 2, 0, 3)); // [N, h, w, inner_dim]
|
||||
x = ggml_reshape_3d(ctx->ggml_ctx, x, inner_dim, w * h, n); // [N, h * w, inner_dim]
|
||||
x = proj_in->forward(ctx, x); // [N, inner_dim, h, w]
|
||||
} else {
|
||||
x = proj_in->forward(ctx, x); // [N, inner_dim, h, w]
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 2, 0, 3)); // [N, h, w, inner_dim]
|
||||
x = ggml_reshape_3d(ctx->ggml_ctx, x, inner_dim, w * h, n); // [N, h * w, inner_dim]
|
||||
}
|
||||
|
||||
for (int i = 0; i < depth; i++) {
|
||||
std::string name = "transformer_blocks." + std::to_string(i);
|
||||
auto transformer_block = std::dynamic_pointer_cast<BasicTransformerBlock>(blocks[name]);
|
||||
|
||||
x = transformer_block->forward(ctx, backend, x, context);
|
||||
x = transformer_block->forward(ctx, x, context);
|
||||
}
|
||||
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 1, 0, 2, 3)); // [N, inner_dim, h * w]
|
||||
x = ggml_reshape_4d(ctx, x, w, h, inner_dim, n); // [N, inner_dim, h, w]
|
||||
if (use_linear) {
|
||||
// proj_out
|
||||
x = proj_out->forward(ctx, x); // [N, in_channels, h, w]
|
||||
|
||||
// proj_out
|
||||
x = proj_out->forward(ctx, x); // [N, in_channels, h, w]
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); // [N, inner_dim, h * w]
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx, x, w, h, inner_dim, n); // [N, inner_dim, h, w]
|
||||
} else {
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); // [N, inner_dim, h * w]
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx, x, w, h, inner_dim, n); // [N, inner_dim, h, w]
|
||||
|
||||
x = ggml_add(ctx, x, x_in);
|
||||
// proj_out
|
||||
x = proj_out->forward(ctx, x); // [N, in_channels, h, w]
|
||||
}
|
||||
|
||||
x = ggml_add(ctx->ggml_ctx, x, x_in);
|
||||
return x;
|
||||
}
|
||||
};
|
||||
|
||||
class AlphaBlender : public GGMLBlock {
|
||||
protected:
|
||||
void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, std::string prefix = "") override {
|
||||
void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") override {
|
||||
// Get the type of the "mix_factor" tensor from the input tensors map with the specified prefix
|
||||
enum ggml_type wtype = GGML_TYPE_F32;
|
||||
params["mix_factor"] = ggml_new_tensor_1d(ctx, wtype, 1);
|
||||
@@ -492,7 +523,7 @@ protected:
|
||||
float get_alpha() {
|
||||
// image_only_indicator is always tensor([0.]) and since mix_factor.shape is [1,]
|
||||
// so learned_with_images is same as learned
|
||||
float alpha = ggml_backend_tensor_get_f32(params["mix_factor"]);
|
||||
float alpha = ggml_ext_backend_tensor_get_f32(params["mix_factor"]);
|
||||
return sigmoid(alpha);
|
||||
}
|
||||
|
||||
@@ -503,14 +534,14 @@ public:
|
||||
// since mix_factor.shape is [1,], we don't need rearrange using rearrange_pattern
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x_spatial,
|
||||
struct ggml_tensor* x_temporal) {
|
||||
// image_only_indicator is always tensor([0.])
|
||||
float alpha = get_alpha();
|
||||
auto x = ggml_add(ctx,
|
||||
ggml_scale(ctx, x_spatial, alpha),
|
||||
ggml_scale(ctx, x_temporal, 1.0f - alpha));
|
||||
auto x = ggml_add(ctx->ggml_ctx,
|
||||
ggml_scale(ctx->ggml_ctx, x_spatial, alpha),
|
||||
ggml_scale(ctx->ggml_ctx, x_temporal, 1.0f - alpha));
|
||||
return x;
|
||||
}
|
||||
};
|
||||
@@ -528,7 +559,7 @@ public:
|
||||
blocks["time_mixer"] = std::shared_ptr<GGMLBlock>(new AlphaBlender());
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* emb,
|
||||
int num_video_frames) {
|
||||
@@ -546,18 +577,18 @@ public:
|
||||
int64_t H = x->ne[1];
|
||||
int64_t W = x->ne[0];
|
||||
|
||||
x = ggml_reshape_4d(ctx, x, W * H, C, T, B); // (b t) c h w -> b t c (h w)
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); // b t c (h w) -> b c t (h w)
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx, x, W * H, C, T, B); // (b t) c h w -> b t c (h w)
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 0, 2, 1, 3)); // b t c (h w) -> b c t (h w)
|
||||
auto x_mix = x;
|
||||
|
||||
emb = ggml_reshape_4d(ctx, emb, emb->ne[0], T, B, emb->ne[3]); // (b t) ... -> b t ...
|
||||
emb = ggml_reshape_4d(ctx->ggml_ctx, emb, emb->ne[0], T, B, emb->ne[3]); // (b t) ... -> b t ...
|
||||
|
||||
x = time_stack->forward(ctx, x, emb); // b t c (h w)
|
||||
|
||||
x = time_mixer->forward(ctx, x_mix, x); // b t c (h w)
|
||||
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); // b c t (h w) -> b t c (h w)
|
||||
x = ggml_reshape_4d(ctx, x, W, H, C, T * B); // b t c (h w) -> (b t) c h w
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 0, 2, 1, 3)); // b c t (h w) -> b t c (h w)
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx, x, W, H, C, T * B); // b t c (h w) -> (b t) c h w
|
||||
|
||||
return x;
|
||||
}
|
||||
|
||||
+347
-165
@@ -63,19 +63,19 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
|
||||
|
||||
FrozenCLIPEmbedderWithCustomWords(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types,
|
||||
const String2TensorStorage& tensor_storage_map,
|
||||
const std::string& embd_dir,
|
||||
SDVersion version = VERSION_SD1,
|
||||
PMVersion pv = PM_VERSION_1)
|
||||
: version(version), pm_version(pv), tokenizer(sd_version_is_sd2(version) ? 0 : 49407), embd_dir(embd_dir) {
|
||||
bool force_clip_f32 = embd_dir.size() > 0;
|
||||
if (sd_version_is_sd1(version)) {
|
||||
text_model = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_types, "cond_stage_model.transformer.text_model", OPENAI_CLIP_VIT_L_14, true, force_clip_f32);
|
||||
text_model = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_storage_map, "cond_stage_model.transformer.text_model", OPENAI_CLIP_VIT_L_14, true, force_clip_f32);
|
||||
} else if (sd_version_is_sd2(version)) {
|
||||
text_model = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_types, "cond_stage_model.transformer.text_model", OPEN_CLIP_VIT_H_14, true, force_clip_f32);
|
||||
text_model = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_storage_map, "cond_stage_model.transformer.text_model", OPEN_CLIP_VIT_H_14, true, force_clip_f32);
|
||||
} else if (sd_version_is_sdxl(version)) {
|
||||
text_model = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_types, "cond_stage_model.transformer.text_model", OPENAI_CLIP_VIT_L_14, false, force_clip_f32);
|
||||
text_model2 = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_types, "cond_stage_model.1.transformer.text_model", OPEN_CLIP_VIT_BIGG_14, false, force_clip_f32);
|
||||
text_model = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_storage_map, "cond_stage_model.transformer.text_model", OPENAI_CLIP_VIT_L_14, false, force_clip_f32);
|
||||
text_model2 = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_storage_map, "cond_stage_model.1.transformer.text_model", OPEN_CLIP_VIT_BIGG_14, false, force_clip_f32);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -462,7 +462,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
|
||||
clip_skip,
|
||||
&chunk_hidden_states2, work_ctx);
|
||||
// concat
|
||||
chunk_hidden_states = ggml_tensor_concat(work_ctx, chunk_hidden_states1, chunk_hidden_states2, 0);
|
||||
chunk_hidden_states = ggml_ext_tensor_concat(work_ctx, chunk_hidden_states1, chunk_hidden_states2, 0);
|
||||
|
||||
if (chunk_idx == 0) {
|
||||
text_model2->compute(n_threads,
|
||||
@@ -484,18 +484,18 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
|
||||
LOG_DEBUG("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
|
||||
ggml_tensor* result = ggml_dup_tensor(work_ctx, chunk_hidden_states);
|
||||
{
|
||||
float original_mean = ggml_tensor_mean(chunk_hidden_states);
|
||||
float original_mean = ggml_ext_tensor_mean(chunk_hidden_states);
|
||||
for (int i2 = 0; i2 < chunk_hidden_states->ne[2]; i2++) {
|
||||
for (int i1 = 0; i1 < chunk_hidden_states->ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < chunk_hidden_states->ne[0]; i0++) {
|
||||
float value = ggml_tensor_get_f32(chunk_hidden_states, i0, i1, i2);
|
||||
float value = ggml_ext_tensor_get_f32(chunk_hidden_states, i0, i1, i2);
|
||||
value *= chunk_weights[i1];
|
||||
ggml_tensor_set_f32(result, value, i0, i1, i2);
|
||||
ggml_ext_tensor_set_f32(result, value, i0, i1, i2);
|
||||
}
|
||||
}
|
||||
}
|
||||
float new_mean = ggml_tensor_mean(result);
|
||||
ggml_tensor_scale(result, (original_mean / new_mean));
|
||||
float new_mean = ggml_ext_tensor_mean(result);
|
||||
ggml_ext_tensor_scale_inplace(result, (original_mean / new_mean));
|
||||
}
|
||||
if (zero_out_masked) {
|
||||
float* vec = (float*)result->data;
|
||||
@@ -623,9 +623,21 @@ struct FrozenCLIPVisionEmbedder : public GGMLRunner {
|
||||
|
||||
FrozenCLIPVisionEmbedder(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {})
|
||||
: vision_model(OPEN_CLIP_VIT_H_14), GGMLRunner(backend, offload_params_to_cpu) {
|
||||
vision_model.init(params_ctx, tensor_types, "cond_stage_model.transformer");
|
||||
const String2TensorStorage& tensor_storage_map = {})
|
||||
: GGMLRunner(backend, offload_params_to_cpu) {
|
||||
std::string prefix = "cond_stage_model.transformer";
|
||||
bool proj_in = false;
|
||||
for (const auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (!starts_with(name, prefix)) {
|
||||
continue;
|
||||
}
|
||||
if (contains(name, "self_attn.in_proj")) {
|
||||
proj_in = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
vision_model = CLIPVisionModelProjection(OPEN_CLIP_VIT_H_14, false, proj_in);
|
||||
vision_model.init(params_ctx, tensor_storage_map, prefix);
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -641,7 +653,9 @@ struct FrozenCLIPVisionEmbedder : public GGMLRunner {
|
||||
|
||||
pixel_values = to_backend(pixel_values);
|
||||
|
||||
struct ggml_tensor* hidden_states = vision_model.forward(compute_ctx, runtime_backend, pixel_values, return_pooled, clip_skip);
|
||||
auto runner_ctx = get_context();
|
||||
|
||||
struct ggml_tensor* hidden_states = vision_model.forward(&runner_ctx, pixel_values, return_pooled, clip_skip);
|
||||
|
||||
ggml_build_forward_expand(gf, hidden_states);
|
||||
|
||||
@@ -671,35 +685,82 @@ struct SD3CLIPEmbedder : public Conditioner {
|
||||
|
||||
SD3CLIPEmbedder(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {})
|
||||
const String2TensorStorage& tensor_storage_map = {})
|
||||
: clip_g_tokenizer(0) {
|
||||
clip_l = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_types, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, false);
|
||||
clip_g = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_types, "text_encoders.clip_g.transformer.text_model", OPEN_CLIP_VIT_BIGG_14, false);
|
||||
t5 = std::make_shared<T5Runner>(backend, offload_params_to_cpu, tensor_types, "text_encoders.t5xxl.transformer");
|
||||
bool use_clip_l = false;
|
||||
bool use_clip_g = false;
|
||||
bool use_t5 = false;
|
||||
for (auto pair : tensor_storage_map) {
|
||||
if (pair.first.find("text_encoders.clip_l") != std::string::npos) {
|
||||
use_clip_l = true;
|
||||
} else if (pair.first.find("text_encoders.clip_g") != std::string::npos) {
|
||||
use_clip_g = true;
|
||||
} else if (pair.first.find("text_encoders.t5xxl") != std::string::npos) {
|
||||
use_t5 = true;
|
||||
}
|
||||
}
|
||||
if (!use_clip_l && !use_clip_g && !use_t5) {
|
||||
LOG_WARN("IMPORTANT NOTICE: No text encoders provided, cannot process prompts!");
|
||||
return;
|
||||
}
|
||||
if (use_clip_l) {
|
||||
clip_l = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_storage_map, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, false);
|
||||
}
|
||||
if (use_clip_g) {
|
||||
clip_g = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_storage_map, "text_encoders.clip_g.transformer.text_model", OPEN_CLIP_VIT_BIGG_14, false);
|
||||
}
|
||||
if (use_t5) {
|
||||
t5 = std::make_shared<T5Runner>(backend, offload_params_to_cpu, tensor_storage_map, "text_encoders.t5xxl.transformer");
|
||||
}
|
||||
}
|
||||
|
||||
void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
|
||||
clip_l->get_param_tensors(tensors, "text_encoders.clip_l.transformer.text_model");
|
||||
clip_g->get_param_tensors(tensors, "text_encoders.clip_g.transformer.text_model");
|
||||
t5->get_param_tensors(tensors, "text_encoders.t5xxl.transformer");
|
||||
if (clip_l) {
|
||||
clip_l->get_param_tensors(tensors, "text_encoders.clip_l.transformer.text_model");
|
||||
}
|
||||
if (clip_g) {
|
||||
clip_g->get_param_tensors(tensors, "text_encoders.clip_g.transformer.text_model");
|
||||
}
|
||||
if (t5) {
|
||||
t5->get_param_tensors(tensors, "text_encoders.t5xxl.transformer");
|
||||
}
|
||||
}
|
||||
|
||||
void alloc_params_buffer() override {
|
||||
clip_l->alloc_params_buffer();
|
||||
clip_g->alloc_params_buffer();
|
||||
t5->alloc_params_buffer();
|
||||
if (clip_l) {
|
||||
clip_l->alloc_params_buffer();
|
||||
}
|
||||
if (clip_g) {
|
||||
clip_g->alloc_params_buffer();
|
||||
}
|
||||
if (t5) {
|
||||
t5->alloc_params_buffer();
|
||||
}
|
||||
}
|
||||
|
||||
void free_params_buffer() override {
|
||||
clip_l->free_params_buffer();
|
||||
clip_g->free_params_buffer();
|
||||
t5->free_params_buffer();
|
||||
if (clip_l) {
|
||||
clip_l->free_params_buffer();
|
||||
}
|
||||
if (clip_g) {
|
||||
clip_g->free_params_buffer();
|
||||
}
|
||||
if (t5) {
|
||||
t5->free_params_buffer();
|
||||
}
|
||||
}
|
||||
|
||||
size_t get_params_buffer_size() override {
|
||||
size_t buffer_size = clip_l->get_params_buffer_size();
|
||||
buffer_size += clip_g->get_params_buffer_size();
|
||||
buffer_size += t5->get_params_buffer_size();
|
||||
size_t buffer_size = 0;
|
||||
if (clip_l) {
|
||||
buffer_size += clip_l->get_params_buffer_size();
|
||||
}
|
||||
if (clip_g) {
|
||||
buffer_size += clip_g->get_params_buffer_size();
|
||||
}
|
||||
if (t5) {
|
||||
buffer_size += t5->get_params_buffer_size();
|
||||
}
|
||||
return buffer_size;
|
||||
}
|
||||
|
||||
@@ -731,23 +792,32 @@ struct SD3CLIPEmbedder : public Conditioner {
|
||||
for (const auto& item : parsed_attention) {
|
||||
const std::string& curr_text = item.first;
|
||||
float curr_weight = item.second;
|
||||
|
||||
std::vector<int> curr_tokens = clip_l_tokenizer.encode(curr_text, on_new_token_cb);
|
||||
clip_l_tokens.insert(clip_l_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
clip_l_weights.insert(clip_l_weights.end(), curr_tokens.size(), curr_weight);
|
||||
|
||||
curr_tokens = clip_g_tokenizer.encode(curr_text, on_new_token_cb);
|
||||
clip_g_tokens.insert(clip_g_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
clip_g_weights.insert(clip_g_weights.end(), curr_tokens.size(), curr_weight);
|
||||
|
||||
curr_tokens = t5_tokenizer.Encode(curr_text, true);
|
||||
t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
|
||||
if (clip_l) {
|
||||
std::vector<int> curr_tokens = clip_l_tokenizer.encode(curr_text, on_new_token_cb);
|
||||
clip_l_tokens.insert(clip_l_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
clip_l_weights.insert(clip_l_weights.end(), curr_tokens.size(), curr_weight);
|
||||
}
|
||||
if (clip_g) {
|
||||
std::vector<int> curr_tokens = clip_g_tokenizer.encode(curr_text, on_new_token_cb);
|
||||
clip_g_tokens.insert(clip_g_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
clip_g_weights.insert(clip_g_weights.end(), curr_tokens.size(), curr_weight);
|
||||
}
|
||||
if (t5) {
|
||||
std::vector<int> curr_tokens = t5_tokenizer.Encode(curr_text, true);
|
||||
t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
|
||||
}
|
||||
}
|
||||
|
||||
clip_l_tokenizer.pad_tokens(clip_l_tokens, clip_l_weights, max_length, padding);
|
||||
clip_g_tokenizer.pad_tokens(clip_g_tokens, clip_g_weights, max_length, padding);
|
||||
t5_tokenizer.pad_tokens(t5_tokens, t5_weights, nullptr, max_length, padding);
|
||||
if (clip_l) {
|
||||
clip_l_tokenizer.pad_tokens(clip_l_tokens, clip_l_weights, max_length, padding);
|
||||
}
|
||||
if (clip_g) {
|
||||
clip_g_tokenizer.pad_tokens(clip_g_tokens, clip_g_weights, max_length, padding);
|
||||
}
|
||||
if (t5) {
|
||||
t5_tokenizer.pad_tokens(t5_tokens, t5_weights, nullptr, max_length, padding);
|
||||
}
|
||||
|
||||
// for (int i = 0; i < clip_l_tokens.size(); i++) {
|
||||
// std::cout << clip_l_tokens[i] << ":" << clip_l_weights[i] << ", ";
|
||||
@@ -795,10 +865,10 @@ struct SD3CLIPEmbedder : public Conditioner {
|
||||
std::vector<float> hidden_states_vec;
|
||||
|
||||
size_t chunk_len = 77;
|
||||
size_t chunk_count = clip_l_tokens.size() / chunk_len;
|
||||
size_t chunk_count = std::max(std::max(clip_l_tokens.size(), clip_g_tokens.size()), t5_tokens.size()) / chunk_len;
|
||||
for (int chunk_idx = 0; chunk_idx < chunk_count; chunk_idx++) {
|
||||
// clip_l
|
||||
{
|
||||
if (clip_l) {
|
||||
std::vector<int> chunk_tokens(clip_l_tokens.begin() + chunk_idx * chunk_len,
|
||||
clip_l_tokens.begin() + (chunk_idx + 1) * chunk_len);
|
||||
std::vector<float> chunk_weights(clip_l_weights.begin() + chunk_idx * chunk_len,
|
||||
@@ -818,18 +888,18 @@ struct SD3CLIPEmbedder : public Conditioner {
|
||||
work_ctx);
|
||||
{
|
||||
auto tensor = chunk_hidden_states_l;
|
||||
float original_mean = ggml_tensor_mean(tensor);
|
||||
float original_mean = ggml_ext_tensor_mean(tensor);
|
||||
for (int i2 = 0; i2 < tensor->ne[2]; i2++) {
|
||||
for (int i1 = 0; i1 < tensor->ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < tensor->ne[0]; i0++) {
|
||||
float value = ggml_tensor_get_f32(tensor, i0, i1, i2);
|
||||
float value = ggml_ext_tensor_get_f32(tensor, i0, i1, i2);
|
||||
value *= chunk_weights[i1];
|
||||
ggml_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
ggml_ext_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
}
|
||||
}
|
||||
}
|
||||
float new_mean = ggml_tensor_mean(tensor);
|
||||
ggml_tensor_scale(tensor, (original_mean / new_mean));
|
||||
float new_mean = ggml_ext_tensor_mean(tensor);
|
||||
ggml_ext_tensor_scale_inplace(tensor, (original_mean / new_mean));
|
||||
}
|
||||
|
||||
if (chunk_idx == 0) {
|
||||
@@ -845,10 +915,17 @@ struct SD3CLIPEmbedder : public Conditioner {
|
||||
&pooled_l,
|
||||
work_ctx);
|
||||
}
|
||||
} else {
|
||||
chunk_hidden_states_l = ggml_new_tensor_2d(work_ctx, GGML_TYPE_F32, 768, chunk_len);
|
||||
ggml_set_f32(chunk_hidden_states_l, 0.f);
|
||||
if (chunk_idx == 0) {
|
||||
pooled_l = ggml_new_tensor_1d(work_ctx, GGML_TYPE_F32, 768);
|
||||
ggml_set_f32(pooled_l, 0.f);
|
||||
}
|
||||
}
|
||||
|
||||
// clip_g
|
||||
{
|
||||
if (clip_g) {
|
||||
std::vector<int> chunk_tokens(clip_g_tokens.begin() + chunk_idx * chunk_len,
|
||||
clip_g_tokens.begin() + (chunk_idx + 1) * chunk_len);
|
||||
std::vector<float> chunk_weights(clip_g_weights.begin() + chunk_idx * chunk_len,
|
||||
@@ -869,18 +946,18 @@ struct SD3CLIPEmbedder : public Conditioner {
|
||||
|
||||
{
|
||||
auto tensor = chunk_hidden_states_g;
|
||||
float original_mean = ggml_tensor_mean(tensor);
|
||||
float original_mean = ggml_ext_tensor_mean(tensor);
|
||||
for (int i2 = 0; i2 < tensor->ne[2]; i2++) {
|
||||
for (int i1 = 0; i1 < tensor->ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < tensor->ne[0]; i0++) {
|
||||
float value = ggml_tensor_get_f32(tensor, i0, i1, i2);
|
||||
float value = ggml_ext_tensor_get_f32(tensor, i0, i1, i2);
|
||||
value *= chunk_weights[i1];
|
||||
ggml_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
ggml_ext_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
}
|
||||
}
|
||||
}
|
||||
float new_mean = ggml_tensor_mean(tensor);
|
||||
ggml_tensor_scale(tensor, (original_mean / new_mean));
|
||||
float new_mean = ggml_ext_tensor_mean(tensor);
|
||||
ggml_ext_tensor_scale_inplace(tensor, (original_mean / new_mean));
|
||||
}
|
||||
|
||||
if (chunk_idx == 0) {
|
||||
@@ -896,10 +973,17 @@ struct SD3CLIPEmbedder : public Conditioner {
|
||||
&pooled_g,
|
||||
work_ctx);
|
||||
}
|
||||
} else {
|
||||
chunk_hidden_states_g = ggml_new_tensor_2d(work_ctx, GGML_TYPE_F32, 1280, chunk_len);
|
||||
ggml_set_f32(chunk_hidden_states_g, 0.f);
|
||||
if (chunk_idx == 0) {
|
||||
pooled_g = ggml_new_tensor_1d(work_ctx, GGML_TYPE_F32, 1280);
|
||||
ggml_set_f32(pooled_g, 0.f);
|
||||
}
|
||||
}
|
||||
|
||||
// t5
|
||||
{
|
||||
if (t5) {
|
||||
std::vector<int> chunk_tokens(t5_tokens.begin() + chunk_idx * chunk_len,
|
||||
t5_tokens.begin() + (chunk_idx + 1) * chunk_len);
|
||||
std::vector<float> chunk_weights(t5_weights.begin() + chunk_idx * chunk_len,
|
||||
@@ -914,19 +998,22 @@ struct SD3CLIPEmbedder : public Conditioner {
|
||||
work_ctx);
|
||||
{
|
||||
auto tensor = chunk_hidden_states_t5;
|
||||
float original_mean = ggml_tensor_mean(tensor);
|
||||
float original_mean = ggml_ext_tensor_mean(tensor);
|
||||
for (int i2 = 0; i2 < tensor->ne[2]; i2++) {
|
||||
for (int i1 = 0; i1 < tensor->ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < tensor->ne[0]; i0++) {
|
||||
float value = ggml_tensor_get_f32(tensor, i0, i1, i2);
|
||||
float value = ggml_ext_tensor_get_f32(tensor, i0, i1, i2);
|
||||
value *= chunk_weights[i1];
|
||||
ggml_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
ggml_ext_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
}
|
||||
}
|
||||
}
|
||||
float new_mean = ggml_tensor_mean(tensor);
|
||||
ggml_tensor_scale(tensor, (original_mean / new_mean));
|
||||
float new_mean = ggml_ext_tensor_mean(tensor);
|
||||
ggml_ext_tensor_scale_inplace(tensor, (original_mean / new_mean));
|
||||
}
|
||||
} else {
|
||||
chunk_hidden_states_t5 = ggml_new_tensor_2d(work_ctx, GGML_TYPE_F32, 4096, chunk_len);
|
||||
ggml_set_f32(chunk_hidden_states_t5, 0.f);
|
||||
}
|
||||
|
||||
auto chunk_hidden_states_lg_pad = ggml_new_tensor_3d(work_ctx,
|
||||
@@ -940,19 +1027,19 @@ struct SD3CLIPEmbedder : public Conditioner {
|
||||
for (int i0 = 0; i0 < chunk_hidden_states_lg_pad->ne[0]; i0++) {
|
||||
float value = 0.f;
|
||||
if (i0 < chunk_hidden_states_l->ne[0]) {
|
||||
value = ggml_tensor_get_f32(chunk_hidden_states_l, i0, i1, i2);
|
||||
value = ggml_ext_tensor_get_f32(chunk_hidden_states_l, i0, i1, i2);
|
||||
} else if (i0 < chunk_hidden_states_l->ne[0] + chunk_hidden_states_g->ne[0]) {
|
||||
value = ggml_tensor_get_f32(chunk_hidden_states_g, i0 - chunk_hidden_states_l->ne[0], i1, i2);
|
||||
value = ggml_ext_tensor_get_f32(chunk_hidden_states_g, i0 - chunk_hidden_states_l->ne[0], i1, i2);
|
||||
}
|
||||
ggml_tensor_set_f32(chunk_hidden_states_lg_pad, value, i0, i1, i2);
|
||||
ggml_ext_tensor_set_f32(chunk_hidden_states_lg_pad, value, i0, i1, i2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
chunk_hidden_states = ggml_tensor_concat(work_ctx, chunk_hidden_states_lg_pad, chunk_hidden_states_t5, 1); // [n_token*2, 4096]
|
||||
chunk_hidden_states = ggml_ext_tensor_concat(work_ctx, chunk_hidden_states_lg_pad, chunk_hidden_states_t5, 1); // [n_token*2, 4096]
|
||||
|
||||
if (chunk_idx == 0) {
|
||||
pooled = ggml_tensor_concat(work_ctx, pooled_l, pooled_g, 0); // [768 + 1280]
|
||||
pooled = ggml_ext_tensor_concat(work_ctx, pooled_l, pooled_g, 0); // [768 + 1280]
|
||||
}
|
||||
|
||||
int64_t t1 = ggml_time_ms();
|
||||
@@ -969,11 +1056,20 @@ struct SD3CLIPEmbedder : public Conditioner {
|
||||
((float*)chunk_hidden_states->data) + ggml_nelements(chunk_hidden_states));
|
||||
}
|
||||
|
||||
hidden_states = vector_to_ggml_tensor(work_ctx, hidden_states_vec);
|
||||
hidden_states = ggml_reshape_2d(work_ctx,
|
||||
hidden_states,
|
||||
chunk_hidden_states->ne[0],
|
||||
ggml_nelements(hidden_states) / chunk_hidden_states->ne[0]);
|
||||
if (hidden_states_vec.size() > 0) {
|
||||
hidden_states = vector_to_ggml_tensor(work_ctx, hidden_states_vec);
|
||||
hidden_states = ggml_reshape_2d(work_ctx,
|
||||
hidden_states,
|
||||
chunk_hidden_states->ne[0],
|
||||
ggml_nelements(hidden_states) / chunk_hidden_states->ne[0]);
|
||||
} else {
|
||||
hidden_states = ggml_new_tensor_2d(work_ctx, GGML_TYPE_F32, 4096, 256);
|
||||
ggml_set_f32(hidden_states, 0.f);
|
||||
}
|
||||
if (pooled == nullptr) {
|
||||
pooled = ggml_new_tensor_1d(work_ctx, GGML_TYPE_F32, 2048);
|
||||
ggml_set_f32(pooled, 0.f);
|
||||
}
|
||||
return {hidden_states, pooled, nullptr};
|
||||
}
|
||||
|
||||
@@ -998,29 +1094,69 @@ struct FluxCLIPEmbedder : public Conditioner {
|
||||
|
||||
FluxCLIPEmbedder(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {}) {
|
||||
clip_l = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_types, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, true);
|
||||
t5 = std::make_shared<T5Runner>(backend, offload_params_to_cpu, tensor_types, "text_encoders.t5xxl.transformer");
|
||||
const String2TensorStorage& tensor_storage_map = {}) {
|
||||
bool use_clip_l = false;
|
||||
bool use_t5 = false;
|
||||
for (auto pair : tensor_storage_map) {
|
||||
if (pair.first.find("text_encoders.clip_l") != std::string::npos) {
|
||||
use_clip_l = true;
|
||||
} else if (pair.first.find("text_encoders.t5xxl") != std::string::npos) {
|
||||
use_t5 = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!use_clip_l && !use_t5) {
|
||||
LOG_WARN("IMPORTANT NOTICE: No text encoders provided, cannot process prompts!");
|
||||
return;
|
||||
}
|
||||
|
||||
if (use_clip_l) {
|
||||
clip_l = std::make_shared<CLIPTextModelRunner>(backend, offload_params_to_cpu, tensor_storage_map, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, true);
|
||||
} else {
|
||||
LOG_WARN("clip_l text encoder not found! Prompt adherence might be degraded.");
|
||||
}
|
||||
if (use_t5) {
|
||||
t5 = std::make_shared<T5Runner>(backend, offload_params_to_cpu, tensor_storage_map, "text_encoders.t5xxl.transformer");
|
||||
} else {
|
||||
LOG_WARN("t5xxl text encoder not found! Prompt adherence might be degraded.");
|
||||
}
|
||||
}
|
||||
|
||||
void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
|
||||
clip_l->get_param_tensors(tensors, "text_encoders.clip_l.transformer.text_model");
|
||||
t5->get_param_tensors(tensors, "text_encoders.t5xxl.transformer");
|
||||
if (clip_l) {
|
||||
clip_l->get_param_tensors(tensors, "text_encoders.clip_l.transformer.text_model");
|
||||
}
|
||||
if (t5) {
|
||||
t5->get_param_tensors(tensors, "text_encoders.t5xxl.transformer");
|
||||
}
|
||||
}
|
||||
|
||||
void alloc_params_buffer() override {
|
||||
clip_l->alloc_params_buffer();
|
||||
t5->alloc_params_buffer();
|
||||
if (clip_l) {
|
||||
clip_l->alloc_params_buffer();
|
||||
}
|
||||
if (t5) {
|
||||
t5->alloc_params_buffer();
|
||||
}
|
||||
}
|
||||
|
||||
void free_params_buffer() override {
|
||||
clip_l->free_params_buffer();
|
||||
t5->free_params_buffer();
|
||||
if (clip_l) {
|
||||
clip_l->free_params_buffer();
|
||||
}
|
||||
if (t5) {
|
||||
t5->free_params_buffer();
|
||||
}
|
||||
}
|
||||
|
||||
size_t get_params_buffer_size() override {
|
||||
size_t buffer_size = clip_l->get_params_buffer_size();
|
||||
buffer_size += t5->get_params_buffer_size();
|
||||
size_t buffer_size = 0;
|
||||
if (clip_l) {
|
||||
buffer_size += clip_l->get_params_buffer_size();
|
||||
}
|
||||
if (t5) {
|
||||
buffer_size += t5->get_params_buffer_size();
|
||||
}
|
||||
return buffer_size;
|
||||
}
|
||||
|
||||
@@ -1050,18 +1186,24 @@ struct FluxCLIPEmbedder : public Conditioner {
|
||||
for (const auto& item : parsed_attention) {
|
||||
const std::string& curr_text = item.first;
|
||||
float curr_weight = item.second;
|
||||
|
||||
std::vector<int> curr_tokens = clip_l_tokenizer.encode(curr_text, on_new_token_cb);
|
||||
clip_l_tokens.insert(clip_l_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
clip_l_weights.insert(clip_l_weights.end(), curr_tokens.size(), curr_weight);
|
||||
|
||||
curr_tokens = t5_tokenizer.Encode(curr_text, true);
|
||||
t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
|
||||
if (clip_l) {
|
||||
std::vector<int> curr_tokens = clip_l_tokenizer.encode(curr_text, on_new_token_cb);
|
||||
clip_l_tokens.insert(clip_l_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
clip_l_weights.insert(clip_l_weights.end(), curr_tokens.size(), curr_weight);
|
||||
}
|
||||
if (t5) {
|
||||
std::vector<int> curr_tokens = t5_tokenizer.Encode(curr_text, true);
|
||||
t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
|
||||
}
|
||||
}
|
||||
|
||||
clip_l_tokenizer.pad_tokens(clip_l_tokens, clip_l_weights, 77, padding);
|
||||
t5_tokenizer.pad_tokens(t5_tokens, t5_weights, nullptr, max_length, padding);
|
||||
if (clip_l) {
|
||||
clip_l_tokenizer.pad_tokens(clip_l_tokens, clip_l_weights, 77, padding);
|
||||
}
|
||||
if (t5) {
|
||||
t5_tokenizer.pad_tokens(t5_tokens, t5_weights, nullptr, max_length, padding);
|
||||
}
|
||||
|
||||
// for (int i = 0; i < clip_l_tokens.size(); i++) {
|
||||
// std::cout << clip_l_tokens[i] << ":" << clip_l_weights[i] << ", ";
|
||||
@@ -1096,35 +1238,37 @@ struct FluxCLIPEmbedder : public Conditioner {
|
||||
struct ggml_tensor* pooled = nullptr; // [768,]
|
||||
std::vector<float> hidden_states_vec;
|
||||
|
||||
size_t chunk_count = t5_tokens.size() / chunk_len;
|
||||
size_t chunk_count = std::max(clip_l_tokens.size() > 0 ? chunk_len : 0, t5_tokens.size()) / chunk_len;
|
||||
for (int chunk_idx = 0; chunk_idx < chunk_count; chunk_idx++) {
|
||||
// clip_l
|
||||
if (chunk_idx == 0) {
|
||||
size_t chunk_len_l = 77;
|
||||
std::vector<int> chunk_tokens(clip_l_tokens.begin(),
|
||||
clip_l_tokens.begin() + chunk_len_l);
|
||||
std::vector<float> chunk_weights(clip_l_weights.begin(),
|
||||
clip_l_weights.begin() + chunk_len_l);
|
||||
if (clip_l) {
|
||||
size_t chunk_len_l = 77;
|
||||
std::vector<int> chunk_tokens(clip_l_tokens.begin(),
|
||||
clip_l_tokens.begin() + chunk_len_l);
|
||||
std::vector<float> chunk_weights(clip_l_weights.begin(),
|
||||
clip_l_weights.begin() + chunk_len_l);
|
||||
|
||||
auto input_ids = vector_to_ggml_tensor_i32(work_ctx, chunk_tokens);
|
||||
size_t max_token_idx = 0;
|
||||
auto input_ids = vector_to_ggml_tensor_i32(work_ctx, chunk_tokens);
|
||||
size_t max_token_idx = 0;
|
||||
|
||||
auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_l_tokenizer.EOS_TOKEN_ID);
|
||||
max_token_idx = std::min<size_t>(std::distance(chunk_tokens.begin(), it), chunk_tokens.size() - 1);
|
||||
auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_l_tokenizer.EOS_TOKEN_ID);
|
||||
max_token_idx = std::min<size_t>(std::distance(chunk_tokens.begin(), it), chunk_tokens.size() - 1);
|
||||
|
||||
clip_l->compute(n_threads,
|
||||
input_ids,
|
||||
0,
|
||||
nullptr,
|
||||
max_token_idx,
|
||||
true,
|
||||
clip_skip,
|
||||
&pooled,
|
||||
work_ctx);
|
||||
clip_l->compute(n_threads,
|
||||
input_ids,
|
||||
0,
|
||||
nullptr,
|
||||
max_token_idx,
|
||||
true,
|
||||
clip_skip,
|
||||
&pooled,
|
||||
work_ctx);
|
||||
}
|
||||
}
|
||||
|
||||
// t5
|
||||
{
|
||||
if (t5) {
|
||||
std::vector<int> chunk_tokens(t5_tokens.begin() + chunk_idx * chunk_len,
|
||||
t5_tokens.begin() + (chunk_idx + 1) * chunk_len);
|
||||
std::vector<float> chunk_weights(t5_weights.begin() + chunk_idx * chunk_len,
|
||||
@@ -1139,19 +1283,22 @@ struct FluxCLIPEmbedder : public Conditioner {
|
||||
work_ctx);
|
||||
{
|
||||
auto tensor = chunk_hidden_states;
|
||||
float original_mean = ggml_tensor_mean(tensor);
|
||||
float original_mean = ggml_ext_tensor_mean(tensor);
|
||||
for (int i2 = 0; i2 < tensor->ne[2]; i2++) {
|
||||
for (int i1 = 0; i1 < tensor->ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < tensor->ne[0]; i0++) {
|
||||
float value = ggml_tensor_get_f32(tensor, i0, i1, i2);
|
||||
float value = ggml_ext_tensor_get_f32(tensor, i0, i1, i2);
|
||||
value *= chunk_weights[i1];
|
||||
ggml_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
ggml_ext_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
}
|
||||
}
|
||||
}
|
||||
float new_mean = ggml_tensor_mean(tensor);
|
||||
ggml_tensor_scale(tensor, (original_mean / new_mean));
|
||||
float new_mean = ggml_ext_tensor_mean(tensor);
|
||||
ggml_ext_tensor_scale_inplace(tensor, (original_mean / new_mean));
|
||||
}
|
||||
} else {
|
||||
chunk_hidden_states = ggml_new_tensor_2d(work_ctx, GGML_TYPE_F32, 4096, chunk_len);
|
||||
ggml_set_f32(chunk_hidden_states, 0.f);
|
||||
}
|
||||
|
||||
int64_t t1 = ggml_time_ms();
|
||||
@@ -1168,11 +1315,20 @@ struct FluxCLIPEmbedder : public Conditioner {
|
||||
((float*)chunk_hidden_states->data) + ggml_nelements(chunk_hidden_states));
|
||||
}
|
||||
|
||||
hidden_states = vector_to_ggml_tensor(work_ctx, hidden_states_vec);
|
||||
hidden_states = ggml_reshape_2d(work_ctx,
|
||||
hidden_states,
|
||||
chunk_hidden_states->ne[0],
|
||||
ggml_nelements(hidden_states) / chunk_hidden_states->ne[0]);
|
||||
if (hidden_states_vec.size() > 0) {
|
||||
hidden_states = vector_to_ggml_tensor(work_ctx, hidden_states_vec);
|
||||
hidden_states = ggml_reshape_2d(work_ctx,
|
||||
hidden_states,
|
||||
chunk_hidden_states->ne[0],
|
||||
ggml_nelements(hidden_states) / chunk_hidden_states->ne[0]);
|
||||
} else {
|
||||
hidden_states = ggml_new_tensor_2d(work_ctx, GGML_TYPE_F32, 4096, 256);
|
||||
ggml_set_f32(hidden_states, 0.f);
|
||||
}
|
||||
if (pooled == nullptr) {
|
||||
pooled = ggml_new_tensor_1d(work_ctx, GGML_TYPE_F32, 768);
|
||||
ggml_set_f32(pooled, 0.f);
|
||||
}
|
||||
return {hidden_states, pooled, nullptr};
|
||||
}
|
||||
|
||||
@@ -1198,31 +1354,49 @@ struct T5CLIPEmbedder : public Conditioner {
|
||||
|
||||
T5CLIPEmbedder(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {},
|
||||
bool use_mask = false,
|
||||
int mask_pad = 1,
|
||||
bool is_umt5 = false)
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
bool use_mask = false,
|
||||
int mask_pad = 1,
|
||||
bool is_umt5 = false)
|
||||
: use_mask(use_mask), mask_pad(mask_pad), t5_tokenizer(is_umt5) {
|
||||
t5 = std::make_shared<T5Runner>(backend, offload_params_to_cpu, tensor_types, "text_encoders.t5xxl.transformer", is_umt5);
|
||||
bool use_t5 = false;
|
||||
for (auto pair : tensor_storage_map) {
|
||||
if (pair.first.find("text_encoders.t5xxl") != std::string::npos) {
|
||||
use_t5 = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!use_t5) {
|
||||
LOG_WARN("IMPORTANT NOTICE: No text encoders provided, cannot process prompts!");
|
||||
return;
|
||||
} else {
|
||||
t5 = std::make_shared<T5Runner>(backend, offload_params_to_cpu, tensor_storage_map, "text_encoders.t5xxl.transformer", is_umt5);
|
||||
}
|
||||
}
|
||||
|
||||
void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) override {
|
||||
t5->get_param_tensors(tensors, "text_encoders.t5xxl.transformer");
|
||||
if (t5) {
|
||||
t5->get_param_tensors(tensors, "text_encoders.t5xxl.transformer");
|
||||
}
|
||||
}
|
||||
|
||||
void alloc_params_buffer() override {
|
||||
t5->alloc_params_buffer();
|
||||
if (t5) {
|
||||
t5->alloc_params_buffer();
|
||||
}
|
||||
}
|
||||
|
||||
void free_params_buffer() override {
|
||||
t5->free_params_buffer();
|
||||
if (t5) {
|
||||
t5->free_params_buffer();
|
||||
}
|
||||
}
|
||||
|
||||
size_t get_params_buffer_size() override {
|
||||
size_t buffer_size = 0;
|
||||
|
||||
buffer_size += t5->get_params_buffer_size();
|
||||
|
||||
if (t5) {
|
||||
buffer_size += t5->get_params_buffer_size();
|
||||
}
|
||||
return buffer_size;
|
||||
}
|
||||
|
||||
@@ -1248,17 +1422,18 @@ struct T5CLIPEmbedder : public Conditioner {
|
||||
std::vector<int> t5_tokens;
|
||||
std::vector<float> t5_weights;
|
||||
std::vector<float> t5_mask;
|
||||
for (const auto& item : parsed_attention) {
|
||||
const std::string& curr_text = item.first;
|
||||
float curr_weight = item.second;
|
||||
if (t5) {
|
||||
for (const auto& item : parsed_attention) {
|
||||
const std::string& curr_text = item.first;
|
||||
float curr_weight = item.second;
|
||||
|
||||
std::vector<int> curr_tokens = t5_tokenizer.Encode(curr_text, true);
|
||||
t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
|
||||
std::vector<int> curr_tokens = t5_tokenizer.Encode(curr_text, true);
|
||||
t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
|
||||
}
|
||||
|
||||
t5_tokenizer.pad_tokens(t5_tokens, t5_weights, &t5_mask, max_length, padding);
|
||||
}
|
||||
|
||||
t5_tokenizer.pad_tokens(t5_tokens, t5_weights, &t5_mask, max_length, padding);
|
||||
|
||||
return {t5_tokens, t5_weights, t5_mask};
|
||||
}
|
||||
|
||||
@@ -1282,6 +1457,13 @@ struct T5CLIPEmbedder : public Conditioner {
|
||||
std::tuple<std::vector<int>, std::vector<float>, std::vector<float>> token_and_weights,
|
||||
int clip_skip,
|
||||
bool zero_out_masked = false) {
|
||||
if (!t5) {
|
||||
auto hidden_states = ggml_new_tensor_2d(work_ctx, GGML_TYPE_F32, 4096, 256);
|
||||
ggml_set_f32(hidden_states, 0.f);
|
||||
auto t5_attn_mask = ggml_new_tensor_1d(work_ctx, GGML_TYPE_F32, 256);
|
||||
ggml_set_f32(t5_attn_mask, -HUGE_VALF);
|
||||
return {hidden_states, t5_attn_mask, nullptr};
|
||||
}
|
||||
auto& t5_tokens = std::get<0>(token_and_weights);
|
||||
auto& t5_weights = std::get<1>(token_and_weights);
|
||||
auto& t5_attn_mask_vec = std::get<2>(token_and_weights);
|
||||
@@ -1315,18 +1497,18 @@ struct T5CLIPEmbedder : public Conditioner {
|
||||
work_ctx);
|
||||
{
|
||||
auto tensor = chunk_hidden_states;
|
||||
float original_mean = ggml_tensor_mean(tensor);
|
||||
float original_mean = ggml_ext_tensor_mean(tensor);
|
||||
for (int i2 = 0; i2 < tensor->ne[2]; i2++) {
|
||||
for (int i1 = 0; i1 < tensor->ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < tensor->ne[0]; i0++) {
|
||||
float value = ggml_tensor_get_f32(tensor, i0, i1, i2);
|
||||
float value = ggml_ext_tensor_get_f32(tensor, i0, i1, i2);
|
||||
value *= chunk_weights[i1];
|
||||
ggml_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
ggml_ext_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
}
|
||||
}
|
||||
}
|
||||
float new_mean = ggml_tensor_mean(tensor);
|
||||
ggml_tensor_scale(tensor, (original_mean / new_mean));
|
||||
float new_mean = ggml_ext_tensor_mean(tensor);
|
||||
ggml_ext_tensor_scale_inplace(tensor, (original_mean / new_mean));
|
||||
}
|
||||
|
||||
int64_t t1 = ggml_time_ms();
|
||||
@@ -1337,7 +1519,7 @@ struct T5CLIPEmbedder : public Conditioner {
|
||||
for (int i1 = 0; i1 < tensor->ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < tensor->ne[0]; i0++) {
|
||||
if (chunk_mask[i1] < 0.f) {
|
||||
ggml_tensor_set_f32(tensor, 0.f, i0, i1, i2);
|
||||
ggml_ext_tensor_set_f32(tensor, 0.f, i0, i1, i2);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1379,12 +1561,12 @@ struct Qwen2_5_VLCLIPEmbedder : public Conditioner {
|
||||
|
||||
Qwen2_5_VLCLIPEmbedder(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {},
|
||||
const std::string prefix = "",
|
||||
bool enable_vision = false) {
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
const std::string prefix = "",
|
||||
bool enable_vision = false) {
|
||||
qwenvl = std::make_shared<Qwen::Qwen2_5_VLRunner>(backend,
|
||||
offload_params_to_cpu,
|
||||
tensor_types,
|
||||
tensor_storage_map,
|
||||
"text_encoders.qwen2vl",
|
||||
enable_vision);
|
||||
}
|
||||
@@ -1496,7 +1678,7 @@ struct Qwen2_5_VLCLIPEmbedder : public Conditioner {
|
||||
image.data = nullptr;
|
||||
|
||||
ggml_tensor* image_tensor = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, resized_image.width, resized_image.height, 3, 1);
|
||||
sd_image_f32_to_tensor(resized_image, image_tensor, false);
|
||||
sd_image_f32_to_ggml_tensor(resized_image, image_tensor, false);
|
||||
free(resized_image.data);
|
||||
resized_image.data = nullptr;
|
||||
|
||||
@@ -1541,18 +1723,18 @@ struct Qwen2_5_VLCLIPEmbedder : public Conditioner {
|
||||
work_ctx);
|
||||
{
|
||||
auto tensor = hidden_states;
|
||||
float original_mean = ggml_tensor_mean(tensor);
|
||||
float original_mean = ggml_ext_tensor_mean(tensor);
|
||||
for (int i2 = 0; i2 < tensor->ne[2]; i2++) {
|
||||
for (int i1 = 0; i1 < tensor->ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < tensor->ne[0]; i0++) {
|
||||
float value = ggml_tensor_get_f32(tensor, i0, i1, i2);
|
||||
float value = ggml_ext_tensor_get_f32(tensor, i0, i1, i2);
|
||||
value *= weights[i1];
|
||||
ggml_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
ggml_ext_tensor_set_f32(tensor, value, i0, i1, i2);
|
||||
}
|
||||
}
|
||||
}
|
||||
float new_mean = ggml_tensor_mean(tensor);
|
||||
ggml_tensor_scale(tensor, (original_mean / new_mean));
|
||||
float new_mean = ggml_ext_tensor_mean(tensor);
|
||||
ggml_ext_tensor_scale_inplace(tensor, (original_mean / new_mean));
|
||||
}
|
||||
|
||||
GGML_ASSERT(hidden_states->ne[1] > prompt_template_encode_start_idx);
|
||||
@@ -1563,9 +1745,9 @@ struct Qwen2_5_VLCLIPEmbedder : public Conditioner {
|
||||
hidden_states->ne[1] - prompt_template_encode_start_idx,
|
||||
hidden_states->ne[2]);
|
||||
|
||||
ggml_tensor_iter(new_hidden_states, [&](ggml_tensor* new_hidden_states, int64_t i0, int64_t i1, int64_t i2, int64_t i3) {
|
||||
float value = ggml_tensor_get_f32(hidden_states, i0, i1 + prompt_template_encode_start_idx, i2, i3);
|
||||
ggml_tensor_set_f32(new_hidden_states, value, i0, i1, i2, i3);
|
||||
ggml_ext_tensor_iter(new_hidden_states, [&](ggml_tensor* new_hidden_states, int64_t i0, int64_t i1, int64_t i2, int64_t i3) {
|
||||
float value = ggml_ext_tensor_get_f32(hidden_states, i0, i1 + prompt_template_encode_start_idx, i2, i3);
|
||||
ggml_ext_tensor_set_f32(new_hidden_states, value, i0, i1, i2, i3);
|
||||
});
|
||||
|
||||
int64_t t1 = ggml_time_ms();
|
||||
|
||||
+25
-36
@@ -27,6 +27,7 @@ protected:
|
||||
int num_heads = 8;
|
||||
int num_head_channels = -1; // channels // num_heads
|
||||
int context_dim = 768; // 1024 for VERSION_SD2, 2048 for VERSION_SDXL
|
||||
bool use_linear_projection = false;
|
||||
|
||||
public:
|
||||
int model_channels = 320;
|
||||
@@ -82,7 +83,7 @@ public:
|
||||
int64_t d_head,
|
||||
int64_t depth,
|
||||
int64_t context_dim) -> SpatialTransformer* {
|
||||
return new SpatialTransformer(in_channels, n_head, d_head, depth, context_dim);
|
||||
return new SpatialTransformer(in_channels, n_head, d_head, depth, context_dim, use_linear_projection);
|
||||
};
|
||||
|
||||
auto make_zero_conv = [&](int64_t channels) {
|
||||
@@ -165,7 +166,7 @@ public:
|
||||
}
|
||||
|
||||
struct ggml_tensor* resblock_forward(std::string name,
|
||||
struct ggml_context* ctx,
|
||||
GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* emb) {
|
||||
auto block = std::dynamic_pointer_cast<ResBlock>(blocks[name]);
|
||||
@@ -173,15 +174,14 @@ public:
|
||||
}
|
||||
|
||||
struct ggml_tensor* attention_layer_forward(std::string name,
|
||||
struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* context) {
|
||||
auto block = std::dynamic_pointer_cast<SpatialTransformer>(blocks[name]);
|
||||
return block->forward(ctx, backend, x, context);
|
||||
return block->forward(ctx, x, context);
|
||||
}
|
||||
|
||||
struct ggml_tensor* input_hint_block_forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* input_hint_block_forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* hint,
|
||||
struct ggml_tensor* emb,
|
||||
struct ggml_tensor* context) {
|
||||
@@ -193,14 +193,13 @@ public:
|
||||
|
||||
h = block->forward(ctx, h);
|
||||
} else {
|
||||
h = ggml_silu_inplace(ctx, h);
|
||||
h = ggml_silu_inplace(ctx->ggml_ctx, h);
|
||||
}
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
std::vector<struct ggml_tensor*> forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
std::vector<struct ggml_tensor*> forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* hint,
|
||||
struct ggml_tensor* guided_hint,
|
||||
@@ -213,13 +212,13 @@ public:
|
||||
// y: [N, adm_in_channels] or [1, adm_in_channels]
|
||||
if (context != nullptr) {
|
||||
if (context->ne[2] != x->ne[3]) {
|
||||
context = ggml_repeat(ctx, context, ggml_new_tensor_3d(ctx, GGML_TYPE_F32, context->ne[0], context->ne[1], x->ne[3]));
|
||||
context = ggml_repeat(ctx->ggml_ctx, context, ggml_new_tensor_3d(ctx->ggml_ctx, GGML_TYPE_F32, context->ne[0], context->ne[1], x->ne[3]));
|
||||
}
|
||||
}
|
||||
|
||||
if (y != nullptr) {
|
||||
if (y->ne[1] != x->ne[3]) {
|
||||
y = ggml_repeat(ctx, y, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, y->ne[0], x->ne[3]));
|
||||
y = ggml_repeat(ctx->ggml_ctx, y, ggml_new_tensor_2d(ctx->ggml_ctx, GGML_TYPE_F32, y->ne[0], x->ne[3]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,10 +229,10 @@ public:
|
||||
|
||||
auto middle_block_out = std::dynamic_pointer_cast<Conv2d>(blocks["middle_block_out.0"]);
|
||||
|
||||
auto t_emb = ggml_nn_timestep_embedding(ctx, timesteps, model_channels); // [N, model_channels]
|
||||
auto t_emb = ggml_ext_timestep_embedding(ctx->ggml_ctx, timesteps, model_channels); // [N, model_channels]
|
||||
|
||||
auto emb = time_embed_0->forward(ctx, t_emb);
|
||||
emb = ggml_silu_inplace(ctx, emb);
|
||||
emb = ggml_silu_inplace(ctx->ggml_ctx, emb);
|
||||
emb = time_embed_2->forward(ctx, emb); // [N, time_embed_dim]
|
||||
|
||||
// SDXL/SVD
|
||||
@@ -242,10 +241,10 @@ public:
|
||||
auto label_embed_2 = std::dynamic_pointer_cast<Linear>(blocks["label_emb.0.2"]);
|
||||
|
||||
auto label_emb = label_embed_0->forward(ctx, y);
|
||||
label_emb = ggml_silu_inplace(ctx, label_emb);
|
||||
label_emb = ggml_silu_inplace(ctx->ggml_ctx, label_emb);
|
||||
label_emb = label_embed_2->forward(ctx, label_emb); // [N, time_embed_dim]
|
||||
|
||||
emb = ggml_add(ctx, emb, label_emb); // [N, time_embed_dim]
|
||||
emb = ggml_add(ctx->ggml_ctx, emb, label_emb); // [N, time_embed_dim]
|
||||
}
|
||||
|
||||
std::vector<struct ggml_tensor*> outs;
|
||||
@@ -259,7 +258,7 @@ public:
|
||||
|
||||
// input block 0
|
||||
auto h = input_blocks_0_0->forward(ctx, x);
|
||||
h = ggml_add(ctx, h, guided_hint);
|
||||
h = ggml_add(ctx->ggml_ctx, h, guided_hint);
|
||||
outs.push_back(zero_convs_0->forward(ctx, h));
|
||||
|
||||
// input block 1-11
|
||||
@@ -274,7 +273,7 @@ public:
|
||||
h = resblock_forward(name, ctx, h, emb); // [N, mult*model_channels, h, w]
|
||||
if (std::find(attention_resolutions.begin(), attention_resolutions.end(), ds) != attention_resolutions.end()) {
|
||||
std::string name = "input_blocks." + std::to_string(input_block_idx) + ".1";
|
||||
h = attention_layer_forward(name, ctx, backend, h, context); // [N, mult*model_channels, h, w]
|
||||
h = attention_layer_forward(name, ctx, h, context); // [N, mult*model_channels, h, w]
|
||||
}
|
||||
|
||||
auto zero_conv = std::dynamic_pointer_cast<Conv2d>(blocks["zero_convs." + std::to_string(input_block_idx) + ".0"]);
|
||||
@@ -298,9 +297,9 @@ public:
|
||||
// [N, 4*model_channels, h/8, w/8]
|
||||
|
||||
// middle_block
|
||||
h = resblock_forward("middle_block.0", ctx, h, emb); // [N, 4*model_channels, h/8, w/8]
|
||||
h = attention_layer_forward("middle_block.1", ctx, backend, h, context); // [N, 4*model_channels, h/8, w/8]
|
||||
h = resblock_forward("middle_block.2", ctx, h, emb); // [N, 4*model_channels, h/8, w/8]
|
||||
h = resblock_forward("middle_block.0", ctx, h, emb); // [N, 4*model_channels, h/8, w/8]
|
||||
h = attention_layer_forward("middle_block.1", ctx, h, context); // [N, 4*model_channels, h/8, w/8]
|
||||
h = resblock_forward("middle_block.2", ctx, h, emb); // [N, 4*model_channels, h/8, w/8]
|
||||
|
||||
// out
|
||||
outs.push_back(middle_block_out->forward(ctx, h));
|
||||
@@ -320,21 +319,10 @@ struct ControlNet : public GGMLRunner {
|
||||
|
||||
ControlNet(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {},
|
||||
SDVersion version = VERSION_SD1)
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
SDVersion version = VERSION_SD1)
|
||||
: GGMLRunner(backend, offload_params_to_cpu), control_net(version) {
|
||||
control_net.init(params_ctx, tensor_types, "");
|
||||
}
|
||||
|
||||
void enable_conv2d_direct() {
|
||||
std::vector<GGMLBlock*> blocks;
|
||||
control_net.get_all_blocks(blocks);
|
||||
for (auto block : blocks) {
|
||||
if (block->get_desc() == "Conv2d") {
|
||||
auto conv_block = (Conv2d*)block;
|
||||
conv_block->enable_direct();
|
||||
}
|
||||
}
|
||||
control_net.init(params_ctx, tensor_storage_map, "");
|
||||
}
|
||||
|
||||
~ControlNet() override {
|
||||
@@ -404,8 +392,9 @@ struct ControlNet : public GGMLRunner {
|
||||
y = to_backend(y);
|
||||
timesteps = to_backend(timesteps);
|
||||
|
||||
auto outs = control_net.forward(compute_ctx,
|
||||
runtime_backend,
|
||||
auto runner_ctx = get_context();
|
||||
|
||||
auto outs = control_net.forward(&runner_ctx,
|
||||
x,
|
||||
hint,
|
||||
guided_hint_cached ? guided_hint : nullptr,
|
||||
|
||||
+15
-15
@@ -401,8 +401,8 @@ struct CompVisDenoiser : public Denoiser {
|
||||
|
||||
// this function will modify noise/latent
|
||||
ggml_tensor* noise_scaling(float sigma, ggml_tensor* noise, ggml_tensor* latent) override {
|
||||
ggml_tensor_scale(noise, sigma);
|
||||
ggml_tensor_add(latent, noise);
|
||||
ggml_ext_tensor_scale_inplace(noise, sigma);
|
||||
ggml_ext_tensor_add_inplace(latent, noise);
|
||||
return latent;
|
||||
}
|
||||
|
||||
@@ -496,14 +496,14 @@ struct DiscreteFlowDenoiser : public Denoiser {
|
||||
|
||||
// this function will modify noise/latent
|
||||
ggml_tensor* noise_scaling(float sigma, ggml_tensor* noise, ggml_tensor* latent) override {
|
||||
ggml_tensor_scale(noise, sigma);
|
||||
ggml_tensor_scale(latent, 1.0f - sigma);
|
||||
ggml_tensor_add(latent, noise);
|
||||
ggml_ext_tensor_scale_inplace(noise, sigma);
|
||||
ggml_ext_tensor_scale_inplace(latent, 1.0f - sigma);
|
||||
ggml_ext_tensor_add_inplace(latent, noise);
|
||||
return latent;
|
||||
}
|
||||
|
||||
ggml_tensor* inverse_noise_scaling(float sigma, ggml_tensor* latent) override {
|
||||
ggml_tensor_scale(latent, 1.0f / (1.0f - sigma));
|
||||
ggml_ext_tensor_scale_inplace(latent, 1.0f / (1.0f - sigma));
|
||||
return latent;
|
||||
}
|
||||
};
|
||||
@@ -555,14 +555,14 @@ struct FluxFlowDenoiser : public Denoiser {
|
||||
|
||||
// this function will modify noise/latent
|
||||
ggml_tensor* noise_scaling(float sigma, ggml_tensor* noise, ggml_tensor* latent) override {
|
||||
ggml_tensor_scale(noise, sigma);
|
||||
ggml_tensor_scale(latent, 1.0f - sigma);
|
||||
ggml_tensor_add(latent, noise);
|
||||
ggml_ext_tensor_scale_inplace(noise, sigma);
|
||||
ggml_ext_tensor_scale_inplace(latent, 1.0f - sigma);
|
||||
ggml_ext_tensor_add_inplace(latent, noise);
|
||||
return latent;
|
||||
}
|
||||
|
||||
ggml_tensor* inverse_noise_scaling(float sigma, ggml_tensor* latent) override {
|
||||
ggml_tensor_scale(latent, 1.0f / (1.0f - sigma));
|
||||
ggml_ext_tensor_scale_inplace(latent, 1.0f / (1.0f - sigma));
|
||||
return latent;
|
||||
}
|
||||
};
|
||||
@@ -620,7 +620,7 @@ static void sample_k_diffusion(sample_method_t method,
|
||||
|
||||
if (sigmas[i + 1] > 0) {
|
||||
// x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * s_noise * sigma_up
|
||||
ggml_tensor_set_f32_randn(noise, rng);
|
||||
ggml_ext_im_set_randn_f32(noise, rng);
|
||||
// noise = load_tensor_from_file(work_ctx, "./rand" + std::to_string(i+1) + ".bin");
|
||||
{
|
||||
float* vec_x = (float*)x->data;
|
||||
@@ -820,7 +820,7 @@ static void sample_k_diffusion(sample_method_t method,
|
||||
|
||||
// Noise addition
|
||||
if (sigmas[i + 1] > 0) {
|
||||
ggml_tensor_set_f32_randn(noise, rng);
|
||||
ggml_ext_im_set_randn_f32(noise, rng);
|
||||
{
|
||||
float* vec_x = (float*)x->data;
|
||||
float* vec_noise = (float*)noise->data;
|
||||
@@ -1085,7 +1085,7 @@ static void sample_k_diffusion(sample_method_t method,
|
||||
|
||||
if (sigmas[i + 1] > 0) {
|
||||
// x += sigmas[i + 1] * noise_sampler(sigmas[i], sigmas[i + 1])
|
||||
ggml_tensor_set_f32_randn(noise, rng);
|
||||
ggml_ext_im_set_randn_f32(noise, rng);
|
||||
// noise = load_tensor_from_file(res_ctx, "./rand" + std::to_string(i+1) + ".bin");
|
||||
{
|
||||
float* vec_x = (float*)x->data;
|
||||
@@ -1276,7 +1276,7 @@ static void sample_k_diffusion(sample_method_t method,
|
||||
}
|
||||
}
|
||||
if (eta > 0) {
|
||||
ggml_tensor_set_f32_randn(variance_noise, rng);
|
||||
ggml_ext_im_set_randn_f32(variance_noise, rng);
|
||||
float* vec_variance_noise =
|
||||
(float*)variance_noise->data;
|
||||
float* vec_x = (float*)x->data;
|
||||
@@ -1444,7 +1444,7 @@ static void sample_k_diffusion(sample_method_t method,
|
||||
if (eta > 0 && i != steps - 1) {
|
||||
// In this case, x is still pred_noised_sample,
|
||||
// continue in-place
|
||||
ggml_tensor_set_f32_randn(noise, rng);
|
||||
ggml_ext_im_set_randn_f32(noise, rng);
|
||||
float* vec_x = (float*)x->data;
|
||||
float* vec_noise = (float*)noise->data;
|
||||
for (int j = 0; j < ggml_nelements(x); j++) {
|
||||
|
||||
+38
-22
@@ -36,6 +36,7 @@ struct DiffusionModel {
|
||||
virtual void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors) = 0;
|
||||
virtual size_t get_params_buffer_size() = 0;
|
||||
virtual int64_t get_adm_in_channels() = 0;
|
||||
virtual void set_flash_attn_enabled(bool enabled) = 0;
|
||||
};
|
||||
|
||||
struct UNetModel : public DiffusionModel {
|
||||
@@ -43,10 +44,9 @@ struct UNetModel : public DiffusionModel {
|
||||
|
||||
UNetModel(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {},
|
||||
SDVersion version = VERSION_SD1,
|
||||
bool flash_attn = false)
|
||||
: unet(backend, offload_params_to_cpu, tensor_types, "model.diffusion_model", version, flash_attn) {
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
SDVersion version = VERSION_SD1)
|
||||
: unet(backend, offload_params_to_cpu, tensor_storage_map, "model.diffusion_model", version) {
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -77,6 +77,10 @@ struct UNetModel : public DiffusionModel {
|
||||
return unet.unet.adm_in_channels;
|
||||
}
|
||||
|
||||
void set_flash_attn_enabled(bool enabled) {
|
||||
unet.set_flash_attention_enabled(enabled);
|
||||
}
|
||||
|
||||
void compute(int n_threads,
|
||||
DiffusionParams diffusion_params,
|
||||
struct ggml_tensor** output = nullptr,
|
||||
@@ -98,9 +102,8 @@ struct MMDiTModel : public DiffusionModel {
|
||||
|
||||
MMDiTModel(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
bool flash_attn = false,
|
||||
const String2GGMLType& tensor_types = {})
|
||||
: mmdit(backend, offload_params_to_cpu, flash_attn, tensor_types, "model.diffusion_model") {
|
||||
const String2TensorStorage& tensor_storage_map = {})
|
||||
: mmdit(backend, offload_params_to_cpu, tensor_storage_map, "model.diffusion_model") {
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -131,6 +134,10 @@ struct MMDiTModel : public DiffusionModel {
|
||||
return 768 + 1280;
|
||||
}
|
||||
|
||||
void set_flash_attn_enabled(bool enabled) {
|
||||
mmdit.set_flash_attention_enabled(enabled);
|
||||
}
|
||||
|
||||
void compute(int n_threads,
|
||||
DiffusionParams diffusion_params,
|
||||
struct ggml_tensor** output = nullptr,
|
||||
@@ -151,11 +158,10 @@ struct FluxModel : public DiffusionModel {
|
||||
|
||||
FluxModel(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {},
|
||||
SDVersion version = VERSION_FLUX,
|
||||
bool flash_attn = false,
|
||||
bool use_mask = false)
|
||||
: flux(backend, offload_params_to_cpu, tensor_types, "model.diffusion_model", version, flash_attn, use_mask) {
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
SDVersion version = VERSION_FLUX,
|
||||
bool use_mask = false)
|
||||
: flux(backend, offload_params_to_cpu, tensor_storage_map, "model.diffusion_model", version, use_mask) {
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -186,6 +192,10 @@ struct FluxModel : public DiffusionModel {
|
||||
return 768;
|
||||
}
|
||||
|
||||
void set_flash_attn_enabled(bool enabled) {
|
||||
flux.set_flash_attention_enabled(enabled);
|
||||
}
|
||||
|
||||
void compute(int n_threads,
|
||||
DiffusionParams diffusion_params,
|
||||
struct ggml_tensor** output = nullptr,
|
||||
@@ -211,11 +221,10 @@ struct WanModel : public DiffusionModel {
|
||||
|
||||
WanModel(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {},
|
||||
const std::string prefix = "model.diffusion_model",
|
||||
SDVersion version = VERSION_WAN2,
|
||||
bool flash_attn = false)
|
||||
: prefix(prefix), wan(backend, offload_params_to_cpu, tensor_types, prefix, version, flash_attn) {
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
const std::string prefix = "model.diffusion_model",
|
||||
SDVersion version = VERSION_WAN2)
|
||||
: prefix(prefix), wan(backend, offload_params_to_cpu, tensor_storage_map, prefix, version) {
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -246,6 +255,10 @@ struct WanModel : public DiffusionModel {
|
||||
return 768;
|
||||
}
|
||||
|
||||
void set_flash_attn_enabled(bool enabled) {
|
||||
wan.set_flash_attention_enabled(enabled);
|
||||
}
|
||||
|
||||
void compute(int n_threads,
|
||||
DiffusionParams diffusion_params,
|
||||
struct ggml_tensor** output = nullptr,
|
||||
@@ -270,11 +283,10 @@ struct QwenImageModel : public DiffusionModel {
|
||||
|
||||
QwenImageModel(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {},
|
||||
const std::string prefix = "model.diffusion_model",
|
||||
SDVersion version = VERSION_QWEN_IMAGE,
|
||||
bool flash_attn = false)
|
||||
: prefix(prefix), qwen_image(backend, offload_params_to_cpu, tensor_types, prefix, version, flash_attn) {
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
const std::string prefix = "model.diffusion_model",
|
||||
SDVersion version = VERSION_QWEN_IMAGE)
|
||||
: prefix(prefix), qwen_image(backend, offload_params_to_cpu, tensor_storage_map, prefix, version) {
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -305,6 +317,10 @@ struct QwenImageModel : public DiffusionModel {
|
||||
return 768;
|
||||
}
|
||||
|
||||
void set_flash_attn_enabled(bool enabled) {
|
||||
qwen_image.set_flash_attention_enabled(enabled);
|
||||
}
|
||||
|
||||
void compute(int n_threads,
|
||||
DiffusionParams diffusion_params,
|
||||
struct ggml_tensor** output = nullptr,
|
||||
|
||||
+1
-1
@@ -157,7 +157,7 @@ ninja
|
||||
|
||||
## Build with SYCL
|
||||
|
||||
Using SYCL makes the computation run on the Intel GPU. Please make sure you have installed the related driver and [Intel® oneAPI Base toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) before start. More details and steps can refer to [llama.cpp SYCL backend](https://github.com/ggerganov/llama.cpp/blob/master/docs/backend/SYCL.md#linux).
|
||||
Using SYCL makes the computation run on the Intel GPU. Please make sure you have installed the related driver and [Intel® oneAPI Base toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) before start. More details and steps can refer to [llama.cpp SYCL backend](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/SYCL.md#linux).
|
||||
|
||||
```shell
|
||||
# Export relevant ENV variables
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# How to Use
|
||||
|
||||
## Download weights
|
||||
|
||||
- Download Chroma1-Radiance
|
||||
- safetensors: https://huggingface.co/lodestones/Chroma1-Radiance/tree/main
|
||||
- gguf: https://huggingface.co/silveroxides/Chroma1-Radiance-GGUF/tree/main
|
||||
|
||||
- Download t5xxl
|
||||
- safetensors: https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors
|
||||
|
||||
## Examples
|
||||
|
||||
```
|
||||
.\bin\Release\sd.exe --diffusion-model ..\..\ComfyUI\models\diffusion_models\Chroma1-Radiance-v0.4-Q8_0.gguf --t5xxl ..\..\ComfyUI\models\clip\t5xxl_fp16.safetensors -p "a lovely cat holding a sign says 'chroma radiance cpp'" --cfg-scale 4.0 --sampling-method euler -v
|
||||
```
|
||||
|
||||
<img alt="Chroma1-Radiance" src="../assets/flux/chroma1-radiance.png" />
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
# Running distilled models: SSD1B and SDx.x with tiny U-Nets
|
||||
|
||||
## Preface
|
||||
|
||||
These models feature a reduced U-Net architecture. Unlike standard SDXL models, the SSD-1B U-Net contains only one middle block and fewer attention layers in its up- and down-blocks, resulting in significantly smaller file sizes. Using these models can reduce inference time by more than 33%. For more details, refer to Segmind's paper: https://arxiv.org/abs/2401.02677v1.
|
||||
Similarly, SD1.x- and SD2.x-style models with a tiny U-Net consist of only 6 U-Net blocks, leading to very small files and time savings of up to 50%. For more information, see the paper: https://arxiv.org/pdf/2305.15798.pdf.
|
||||
|
||||
## SSD1B
|
||||
|
||||
Note that not all of these models follow the standard parameter naming conventions. However, several useful SSD-1B models are available online, such as:
|
||||
|
||||
* https://huggingface.co/segmind/SSD-1B/resolve/main/SSD-1B-A1111.safetensors
|
||||
* https://huggingface.co/hassenhamdi/SSD-1B-fp8_e4m3fn/resolve/main/SSD-1B_fp8_e4m3fn.safetensors
|
||||
|
||||
Useful LoRAs are also available:
|
||||
|
||||
* https://huggingface.co/seungminh/lora-swarovski-SSD-1B/resolve/main/pytorch_lora_weights.safetensors
|
||||
* https://huggingface.co/kylielee505/mylcmlorassd/resolve/main/pytorch_lora_weights.safetensors
|
||||
|
||||
These files can be used out-of-the-box, unlike the models described in the next section.
|
||||
|
||||
|
||||
## SD1.x, SD2.x with tiny U-Nets
|
||||
|
||||
These models require conversion before use. You will need a Python script provided by the diffusers team, available on GitHub:
|
||||
|
||||
* https://raw.githubusercontent.com/huggingface/diffusers/refs/heads/main/scripts/convert_diffusers_to_original_stable_diffusion.py
|
||||
|
||||
### SD2.x
|
||||
|
||||
NotaAI provides the following model online:
|
||||
|
||||
* https://huggingface.co/nota-ai/bk-sdm-v2-tiny
|
||||
|
||||
Creating a .safetensors file involves two steps. First, run this short Python script to download the model from Hugging Face:
|
||||
|
||||
```python
|
||||
from diffusers import StableDiffusionPipeline
|
||||
pipe = StableDiffusionPipeline.from_pretrained("nota-ai/bk-sdm-v2-tiny",cache_dir="./")
|
||||
```
|
||||
|
||||
Second, create the .safetensors file by running:
|
||||
|
||||
```bash
|
||||
python convert_diffusers_to_original_stable_diffusion.py \
|
||||
--model_path models--nota-ai--bk-sdm-v2-tiny/snapshots/68277af553777858cd47e133f92e4db47321bc74 \
|
||||
--checkpoint_path bk-sdm-v2-tiny.safetensors --half --use_safetensors
|
||||
```
|
||||
|
||||
This will generate the **file bk-sdm-v2-tiny.safetensors**, which is now ready for use with sd.cpp.
|
||||
|
||||
### SD1.x
|
||||
|
||||
Several Tiny SD 1.x models are available online, such as:
|
||||
|
||||
* https://huggingface.co/segmind/tiny-sd
|
||||
* https://huggingface.co/segmind/portrait-finetuned
|
||||
* https://huggingface.co/nota-ai/bk-sdm-tiny
|
||||
|
||||
These models also require conversion, partly because some tensors are stored in a non-contiguous manner. To create a usable checkpoint file, follow these simple steps:
|
||||
Download and prepare the model using Python:
|
||||
|
||||
##### Download the model using Python on your computer, for example this way:
|
||||
|
||||
```python
|
||||
import torch
|
||||
from diffusers import StableDiffusionPipeline
|
||||
pipe = StableDiffusionPipeline.from_pretrained("segmind/tiny-sd")
|
||||
unet=pipe.unet
|
||||
for param in unet.parameters():
|
||||
param.data = param.data.contiguous() # <- important here
|
||||
pipe.save_pretrained("segmindtiny-sd", safe_serialization=True)
|
||||
```
|
||||
|
||||
##### Run the conversion script:
|
||||
|
||||
```bash
|
||||
python convert_diffusers_to_original_stable_diffusion.py \
|
||||
--model_path ./segmindtiny-sd \
|
||||
--checkpoint_path ./segmind_tiny-sd.ckpt --half
|
||||
```
|
||||
|
||||
The file segmind_tiny-sd.ckpt will be generated and is now ready for use with sd.cpp. You can follow a similar process for the other models mentioned above.
|
||||
|
||||
|
||||
### Another available .ckpt file:
|
||||
|
||||
* https://huggingface.co/ClashSAN/small-sd/resolve/main/tinySDdistilled.ckpt
|
||||
|
||||
To use this file, you must first adjust its non-contiguous tensors:
|
||||
|
||||
```python
|
||||
import torch
|
||||
ckpt = torch.load("tinySDdistilled.ckpt", map_location=torch.device('cpu'))
|
||||
for key, value in ckpt['state_dict'].items():
|
||||
if isinstance(value, torch.Tensor):
|
||||
ckpt['state_dict'][key] = value.contiguous()
|
||||
torch.save(ckpt, "tinySDdistilled_fixed.ckpt")
|
||||
```
|
||||
+2
-2
@@ -40,7 +40,7 @@ Running PMV2 is now a two-step process:
|
||||
```
|
||||
python face_detect.py input_image_dir
|
||||
```
|
||||
An ```id_embeds.safetensors``` file will be generated in ```input_images_dir```
|
||||
An ```id_embeds.bin``` file will be generated in ```input_images_dir```
|
||||
|
||||
**Note: this step is only needed to run once; the same ```id_embeds``` can be reused**
|
||||
|
||||
@@ -48,6 +48,6 @@ An ```id_embeds.safetensors``` file will be generated in ```input_images_dir```
|
||||
|
||||
You can download ```photomaker-v2.safetensors``` from [here](https://huggingface.co/bssrdf/PhotoMakerV2)
|
||||
|
||||
- All the command line parameters from Version 1 remain the same for Version 2
|
||||
- All the command line parameters from Version 1 remain the same for Version 2 plus one extra pointing to a valid ```id_embeds``` file: --pm-id-embed-path [path_to__id_embeds.bin]
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ You can use TAESD to accelerate the decoding of latent images by following these
|
||||
Or curl
|
||||
|
||||
```bash
|
||||
curl -L -O https://huggingface.co/madebyollin/taesd/blob/main/diffusion_pytorch_model.safetensors
|
||||
curl -L -O https://huggingface.co/madebyollin/taesd/resolve/main/diffusion_pytorch_model.safetensors
|
||||
```
|
||||
|
||||
- Specify the model path using the `--taesd PATH` parameter. example:
|
||||
|
||||
+20
-31
@@ -27,11 +27,11 @@ public:
|
||||
blocks["conv5"] = std::shared_ptr<GGMLBlock>(new Conv2d(num_feat + 4 * num_grow_ch, num_feat, {3, 3}, {1, 1}, {1, 1}));
|
||||
}
|
||||
|
||||
struct ggml_tensor* lrelu(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
return ggml_leaky_relu(ctx, x, 0.2f, true);
|
||||
struct ggml_tensor* lrelu(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
return ggml_leaky_relu(ctx->ggml_ctx, x, 0.2f, true);
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [n, num_feat, h, w]
|
||||
// return: [n, num_feat, h, w]
|
||||
|
||||
@@ -42,16 +42,16 @@ public:
|
||||
auto conv5 = std::dynamic_pointer_cast<Conv2d>(blocks["conv5"]);
|
||||
|
||||
auto x1 = lrelu(ctx, conv1->forward(ctx, x));
|
||||
auto x_cat = ggml_concat(ctx, x, x1, 2);
|
||||
auto x_cat = ggml_concat(ctx->ggml_ctx, x, x1, 2);
|
||||
auto x2 = lrelu(ctx, conv2->forward(ctx, x_cat));
|
||||
x_cat = ggml_concat(ctx, x_cat, x2, 2);
|
||||
x_cat = ggml_concat(ctx->ggml_ctx, x_cat, x2, 2);
|
||||
auto x3 = lrelu(ctx, conv3->forward(ctx, x_cat));
|
||||
x_cat = ggml_concat(ctx, x_cat, x3, 2);
|
||||
x_cat = ggml_concat(ctx->ggml_ctx, x_cat, x3, 2);
|
||||
auto x4 = lrelu(ctx, conv4->forward(ctx, x_cat));
|
||||
x_cat = ggml_concat(ctx, x_cat, x4, 2);
|
||||
x_cat = ggml_concat(ctx->ggml_ctx, x_cat, x4, 2);
|
||||
auto x5 = conv5->forward(ctx, x_cat);
|
||||
|
||||
x5 = ggml_add(ctx, ggml_scale(ctx, x5, 0.2f), x);
|
||||
x5 = ggml_add(ctx->ggml_ctx, ggml_scale(ctx->ggml_ctx, x5, 0.2f), x);
|
||||
return x5;
|
||||
}
|
||||
};
|
||||
@@ -64,7 +64,7 @@ public:
|
||||
blocks["rdb3"] = std::shared_ptr<GGMLBlock>(new ResidualDenseBlock(num_feat, num_grow_ch));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [n, num_feat, h, w]
|
||||
// return: [n, num_feat, h, w]
|
||||
|
||||
@@ -76,7 +76,7 @@ public:
|
||||
out = rdb2->forward(ctx, out);
|
||||
out = rdb3->forward(ctx, out);
|
||||
|
||||
out = ggml_add(ctx, ggml_scale(ctx, out, 0.2f), x);
|
||||
out = ggml_add(ctx->ggml_ctx, ggml_scale(ctx->ggml_ctx, out, 0.2f), x);
|
||||
return out;
|
||||
}
|
||||
};
|
||||
@@ -112,11 +112,11 @@ public:
|
||||
int get_scale() { return scale; }
|
||||
int get_num_block() { return num_block; }
|
||||
|
||||
struct ggml_tensor* lrelu(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
return ggml_leaky_relu(ctx, x, 0.2f, true);
|
||||
struct ggml_tensor* lrelu(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
return ggml_leaky_relu(ctx->ggml_ctx, x, 0.2f, true);
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [n, num_in_ch, h, w]
|
||||
// return: [n, num_out_ch, h*scale, w*scale]
|
||||
auto conv_first = std::dynamic_pointer_cast<Conv2d>(blocks["conv_first"]);
|
||||
@@ -133,14 +133,14 @@ public:
|
||||
body_feat = block->forward(ctx, body_feat);
|
||||
}
|
||||
body_feat = conv_body->forward(ctx, body_feat);
|
||||
feat = ggml_add(ctx, feat, body_feat);
|
||||
feat = ggml_add(ctx->ggml_ctx, feat, body_feat);
|
||||
// upsample
|
||||
if (scale >= 2) {
|
||||
auto conv_up1 = std::dynamic_pointer_cast<Conv2d>(blocks["conv_up1"]);
|
||||
feat = lrelu(ctx, conv_up1->forward(ctx, ggml_upscale(ctx, feat, 2, GGML_SCALE_MODE_NEAREST)));
|
||||
feat = lrelu(ctx, conv_up1->forward(ctx, ggml_upscale(ctx->ggml_ctx, feat, 2, GGML_SCALE_MODE_NEAREST)));
|
||||
if (scale == 4) {
|
||||
auto conv_up2 = std::dynamic_pointer_cast<Conv2d>(blocks["conv_up2"]);
|
||||
feat = lrelu(ctx, conv_up2->forward(ctx, ggml_upscale(ctx, feat, 2, GGML_SCALE_MODE_NEAREST)));
|
||||
feat = lrelu(ctx, conv_up2->forward(ctx, ggml_upscale(ctx->ggml_ctx, feat, 2, GGML_SCALE_MODE_NEAREST)));
|
||||
}
|
||||
}
|
||||
// for all scales
|
||||
@@ -156,24 +156,11 @@ struct ESRGAN : public GGMLRunner {
|
||||
|
||||
ESRGAN(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {})
|
||||
const String2TensorStorage& tensor_storage_map = {})
|
||||
: GGMLRunner(backend, offload_params_to_cpu) {
|
||||
// rrdb_net will be created in load_from_file
|
||||
}
|
||||
|
||||
void enable_conv2d_direct() {
|
||||
if (!rrdb_net)
|
||||
return;
|
||||
std::vector<GGMLBlock*> blocks;
|
||||
rrdb_net->get_all_blocks(blocks);
|
||||
for (auto block : blocks) {
|
||||
if (block->get_desc() == "Conv2d") {
|
||||
auto conv_block = (Conv2d*)block;
|
||||
conv_block->enable_direct();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
return "esrgan";
|
||||
}
|
||||
@@ -359,7 +346,9 @@ struct ESRGAN : public GGMLRunner {
|
||||
constexpr int kGraphNodes = 1 << 16; // 65k
|
||||
struct ggml_cgraph* gf = ggml_new_graph_custom(compute_ctx, kGraphNodes, /*grads*/ false);
|
||||
x = to_backend(x);
|
||||
struct ggml_tensor* out = rrdb_net->forward(compute_ctx, x);
|
||||
|
||||
auto runner_ctx = get_context();
|
||||
struct ggml_tensor* out = rrdb_net->forward(&runner_ctx, x);
|
||||
ggml_build_forward_expand(gf, out);
|
||||
return gf;
|
||||
}
|
||||
|
||||
+1
-1
Submodule ggml updated: c538174d26...2d3876d554
+349
-293
File diff suppressed because it is too large
Load Diff
@@ -372,15 +372,15 @@ struct LoraModel : public GGMLRunner {
|
||||
continue;
|
||||
}
|
||||
|
||||
struct ggml_tensor* updown_1 = ggml_merge_lora(compute_ctx, hada_1_down, hada_1_up, hada_1_mid);
|
||||
struct ggml_tensor* updown_2 = ggml_merge_lora(compute_ctx, hada_2_down, hada_2_up, hada_2_mid);
|
||||
struct ggml_tensor* updown_1 = ggml_ext_merge_lora(compute_ctx, hada_1_down, hada_1_up, hada_1_mid);
|
||||
struct ggml_tensor* updown_2 = ggml_ext_merge_lora(compute_ctx, hada_2_down, hada_2_up, hada_2_mid);
|
||||
updown = ggml_mul_inplace(compute_ctx, updown_1, updown_2);
|
||||
|
||||
// calc_scale
|
||||
// TODO: .dora_scale?
|
||||
int64_t rank = hada_1_down->ne[ggml_n_dims(hada_1_down) - 1];
|
||||
if (lora_tensors.find(alpha_name) != lora_tensors.end()) {
|
||||
float alpha = ggml_backend_tensor_get_f32(lora_tensors[alpha_name]);
|
||||
float alpha = ggml_ext_backend_tensor_get_f32(lora_tensors[alpha_name]);
|
||||
scale_value = alpha / rank;
|
||||
}
|
||||
} else if (lora_tensors.find(full_key + ".lokr_w1") != lora_tensors.end() || lora_tensors.find(full_key + ".lokr_w1_a") != lora_tensors.end()) {
|
||||
@@ -418,7 +418,7 @@ struct LoraModel : public GGMLRunner {
|
||||
|
||||
int64_t rank = down->ne[ggml_n_dims(down) - 1];
|
||||
if (lora_tensors.find(alpha_name) != lora_tensors.end()) {
|
||||
float alpha = ggml_backend_tensor_get_f32(lora_tensors[alpha_name]);
|
||||
float alpha = ggml_ext_backend_tensor_get_f32(lora_tensors[alpha_name]);
|
||||
scale_value = alpha / rank;
|
||||
}
|
||||
}
|
||||
@@ -426,7 +426,7 @@ struct LoraModel : public GGMLRunner {
|
||||
up = to_f32(compute_ctx, lora_tensors[up_name]);
|
||||
applied_lora_tensors.insert(up_name);
|
||||
}
|
||||
lokr_w1 = ggml_merge_lora(compute_ctx, down, up);
|
||||
lokr_w1 = ggml_ext_merge_lora(compute_ctx, down, up);
|
||||
}
|
||||
if (lora_tensors.find(lokr_w2_name) != lora_tensors.end()) {
|
||||
lokr_w2 = to_f32(compute_ctx, lora_tensors[lokr_w2_name]);
|
||||
@@ -442,7 +442,7 @@ struct LoraModel : public GGMLRunner {
|
||||
|
||||
int64_t rank = down->ne[ggml_n_dims(down) - 1];
|
||||
if (lora_tensors.find(alpha_name) != lora_tensors.end()) {
|
||||
float alpha = ggml_backend_tensor_get_f32(lora_tensors[alpha_name]);
|
||||
float alpha = ggml_ext_backend_tensor_get_f32(lora_tensors[alpha_name]);
|
||||
scale_value = alpha / rank;
|
||||
}
|
||||
}
|
||||
@@ -450,13 +450,13 @@ struct LoraModel : public GGMLRunner {
|
||||
up = to_f32(compute_ctx, lora_tensors[up_name]);
|
||||
applied_lora_tensors.insert(up_name);
|
||||
}
|
||||
lokr_w2 = ggml_merge_lora(compute_ctx, down, up);
|
||||
lokr_w2 = ggml_ext_merge_lora(compute_ctx, down, up);
|
||||
}
|
||||
|
||||
// Technically it might be unused, but I believe it's the expected behavior
|
||||
applied_lora_tensors.insert(alpha_name);
|
||||
|
||||
updown = ggml_kronecker(compute_ctx, lokr_w1, lokr_w2);
|
||||
updown = ggml_ext_kronecker(compute_ctx, lokr_w1, lokr_w2);
|
||||
|
||||
} else {
|
||||
// LoRA mode
|
||||
@@ -535,30 +535,30 @@ struct LoraModel : public GGMLRunner {
|
||||
float lora_v_scale = 1;
|
||||
|
||||
if (lora_tensors.find(split_q_scale_name) != lora_tensors.end()) {
|
||||
lora_q_scale = ggml_backend_tensor_get_f32(lora_tensors[split_q_scale_name]);
|
||||
lora_q_scale = ggml_ext_backend_tensor_get_f32(lora_tensors[split_q_scale_name]);
|
||||
applied_lora_tensors.insert(split_q_scale_name);
|
||||
}
|
||||
if (lora_tensors.find(split_k_scale_name) != lora_tensors.end()) {
|
||||
lora_k_scale = ggml_backend_tensor_get_f32(lora_tensors[split_k_scale_name]);
|
||||
lora_k_scale = ggml_ext_backend_tensor_get_f32(lora_tensors[split_k_scale_name]);
|
||||
applied_lora_tensors.insert(split_k_scale_name);
|
||||
}
|
||||
if (lora_tensors.find(split_v_scale_name) != lora_tensors.end()) {
|
||||
lora_v_scale = ggml_backend_tensor_get_f32(lora_tensors[split_v_scale_name]);
|
||||
lora_v_scale = ggml_ext_backend_tensor_get_f32(lora_tensors[split_v_scale_name]);
|
||||
applied_lora_tensors.insert(split_v_scale_name);
|
||||
}
|
||||
|
||||
if (lora_tensors.find(split_q_alpha_name) != lora_tensors.end()) {
|
||||
float lora_q_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_q_alpha_name]);
|
||||
float lora_q_alpha = ggml_ext_backend_tensor_get_f32(lora_tensors[split_q_alpha_name]);
|
||||
applied_lora_tensors.insert(split_q_alpha_name);
|
||||
lora_q_scale = lora_q_alpha / q_rank;
|
||||
}
|
||||
if (lora_tensors.find(split_k_alpha_name) != lora_tensors.end()) {
|
||||
float lora_k_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_k_alpha_name]);
|
||||
float lora_k_alpha = ggml_ext_backend_tensor_get_f32(lora_tensors[split_k_alpha_name]);
|
||||
applied_lora_tensors.insert(split_k_alpha_name);
|
||||
lora_k_scale = lora_k_alpha / k_rank;
|
||||
}
|
||||
if (lora_tensors.find(split_v_alpha_name) != lora_tensors.end()) {
|
||||
float lora_v_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_v_alpha_name]);
|
||||
float lora_v_alpha = ggml_ext_backend_tensor_get_f32(lora_tensors[split_v_alpha_name]);
|
||||
applied_lora_tensors.insert(split_v_alpha_name);
|
||||
lora_v_scale = lora_v_alpha / v_rank;
|
||||
}
|
||||
@@ -688,39 +688,39 @@ struct LoraModel : public GGMLRunner {
|
||||
float lora_m_scale = 1;
|
||||
|
||||
if (lora_tensors.find(split_q_scale_name) != lora_tensors.end()) {
|
||||
lora_q_scale = ggml_backend_tensor_get_f32(lora_tensors[split_q_scale_name]);
|
||||
lora_q_scale = ggml_ext_backend_tensor_get_f32(lora_tensors[split_q_scale_name]);
|
||||
applied_lora_tensors.insert(split_q_scale_name);
|
||||
}
|
||||
if (lora_tensors.find(split_k_scale_name) != lora_tensors.end()) {
|
||||
lora_k_scale = ggml_backend_tensor_get_f32(lora_tensors[split_k_scale_name]);
|
||||
lora_k_scale = ggml_ext_backend_tensor_get_f32(lora_tensors[split_k_scale_name]);
|
||||
applied_lora_tensors.insert(split_k_scale_name);
|
||||
}
|
||||
if (lora_tensors.find(split_v_scale_name) != lora_tensors.end()) {
|
||||
lora_v_scale = ggml_backend_tensor_get_f32(lora_tensors[split_v_scale_name]);
|
||||
lora_v_scale = ggml_ext_backend_tensor_get_f32(lora_tensors[split_v_scale_name]);
|
||||
applied_lora_tensors.insert(split_v_scale_name);
|
||||
}
|
||||
if (lora_tensors.find(split_m_scale_name) != lora_tensors.end()) {
|
||||
lora_m_scale = ggml_backend_tensor_get_f32(lora_tensors[split_m_scale_name]);
|
||||
lora_m_scale = ggml_ext_backend_tensor_get_f32(lora_tensors[split_m_scale_name]);
|
||||
applied_lora_tensors.insert(split_m_scale_name);
|
||||
}
|
||||
|
||||
if (lora_tensors.find(split_q_alpha_name) != lora_tensors.end()) {
|
||||
float lora_q_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_q_alpha_name]);
|
||||
float lora_q_alpha = ggml_ext_backend_tensor_get_f32(lora_tensors[split_q_alpha_name]);
|
||||
applied_lora_tensors.insert(split_q_alpha_name);
|
||||
lora_q_scale = lora_q_alpha / q_rank;
|
||||
}
|
||||
if (lora_tensors.find(split_k_alpha_name) != lora_tensors.end()) {
|
||||
float lora_k_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_k_alpha_name]);
|
||||
float lora_k_alpha = ggml_ext_backend_tensor_get_f32(lora_tensors[split_k_alpha_name]);
|
||||
applied_lora_tensors.insert(split_k_alpha_name);
|
||||
lora_k_scale = lora_k_alpha / k_rank;
|
||||
}
|
||||
if (lora_tensors.find(split_v_alpha_name) != lora_tensors.end()) {
|
||||
float lora_v_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_v_alpha_name]);
|
||||
float lora_v_alpha = ggml_ext_backend_tensor_get_f32(lora_tensors[split_v_alpha_name]);
|
||||
applied_lora_tensors.insert(split_v_alpha_name);
|
||||
lora_v_scale = lora_v_alpha / v_rank;
|
||||
}
|
||||
if (lora_tensors.find(split_m_alpha_name) != lora_tensors.end()) {
|
||||
float lora_m_alpha = ggml_backend_tensor_get_f32(lora_tensors[split_m_alpha_name]);
|
||||
float lora_m_alpha = ggml_ext_backend_tensor_get_f32(lora_tensors[split_m_alpha_name]);
|
||||
applied_lora_tensors.insert(split_m_alpha_name);
|
||||
lora_m_scale = lora_m_alpha / m_rank;
|
||||
}
|
||||
@@ -816,16 +816,16 @@ struct LoraModel : public GGMLRunner {
|
||||
// TODO: .dora_scale?
|
||||
int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1];
|
||||
if (lora_tensors.find(scale_name) != lora_tensors.end()) {
|
||||
scale_value = ggml_backend_tensor_get_f32(lora_tensors[scale_name]);
|
||||
scale_value = ggml_ext_backend_tensor_get_f32(lora_tensors[scale_name]);
|
||||
applied_lora_tensors.insert(scale_name);
|
||||
} else if (lora_tensors.find(alpha_name) != lora_tensors.end()) {
|
||||
float alpha = ggml_backend_tensor_get_f32(lora_tensors[alpha_name]);
|
||||
float alpha = ggml_ext_backend_tensor_get_f32(lora_tensors[alpha_name]);
|
||||
scale_value = alpha / rank;
|
||||
// LOG_DEBUG("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value);
|
||||
applied_lora_tensors.insert(alpha_name);
|
||||
}
|
||||
|
||||
updown = ggml_merge_lora(compute_ctx, lora_down, lora_up, lora_mid);
|
||||
updown = ggml_ext_merge_lora(compute_ctx, lora_down, lora_up, lora_mid);
|
||||
}
|
||||
scale_value *= multiplier;
|
||||
ggml_tensor* original_tensor = model_tensor;
|
||||
|
||||
@@ -27,7 +27,7 @@ namespace LTXV {
|
||||
bias));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
bool causal = true) {
|
||||
// x: [N*IC, ID, IH, IW]
|
||||
|
||||
@@ -27,13 +27,13 @@ public:
|
||||
blocks["fc2"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_features, out_features, bias));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [N, n_token, in_features]
|
||||
auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
|
||||
auto fc2 = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
|
||||
|
||||
x = fc1->forward(ctx, x);
|
||||
x = ggml_gelu_inplace(ctx, x);
|
||||
x = ggml_gelu_inplace(ctx->ggml_ctx, x);
|
||||
x = fc2->forward(ctx, x);
|
||||
return x;
|
||||
}
|
||||
@@ -72,7 +72,7 @@ public:
|
||||
bias));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [N, C, H, W]
|
||||
// return: [N, H*W, embed_dim]
|
||||
auto proj = std::dynamic_pointer_cast<Conv2d>(blocks["proj"]);
|
||||
@@ -82,13 +82,13 @@ public:
|
||||
int64_t H = x->ne[1];
|
||||
int pad_h = (patch_size - H % patch_size) % patch_size;
|
||||
int pad_w = (patch_size - W % patch_size) % patch_size;
|
||||
x = ggml_pad(ctx, x, pad_w, pad_h, 0, 0); // TODO: reflect pad mode
|
||||
x = ggml_pad(ctx->ggml_ctx, x, pad_w, pad_h, 0, 0); // TODO: reflect pad mode
|
||||
}
|
||||
x = proj->forward(ctx, x);
|
||||
|
||||
if (flatten) {
|
||||
x = ggml_reshape_3d(ctx, x, x->ne[0] * x->ne[1], x->ne[2], x->ne[3]);
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 1, 0, 2, 3));
|
||||
x = ggml_reshape_3d(ctx->ggml_ctx, x, x->ne[0] * x->ne[1], x->ne[2], x->ne[3]);
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
|
||||
}
|
||||
return x;
|
||||
}
|
||||
@@ -107,16 +107,16 @@ public:
|
||||
blocks["mlp.2"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, hidden_size, true, true));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* t) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* t) {
|
||||
// t: [N, ]
|
||||
// return: [N, hidden_size]
|
||||
auto mlp_0 = std::dynamic_pointer_cast<Linear>(blocks["mlp.0"]);
|
||||
auto mlp_2 = std::dynamic_pointer_cast<Linear>(blocks["mlp.2"]);
|
||||
|
||||
auto t_freq = ggml_nn_timestep_embedding(ctx, t, frequency_embedding_size); // [N, frequency_embedding_size]
|
||||
auto t_freq = ggml_ext_timestep_embedding(ctx->ggml_ctx, t, frequency_embedding_size); // [N, frequency_embedding_size]
|
||||
|
||||
auto t_emb = mlp_0->forward(ctx, t_freq);
|
||||
t_emb = ggml_silu_inplace(ctx, t_emb);
|
||||
t_emb = ggml_silu_inplace(ctx->ggml_ctx, t_emb);
|
||||
t_emb = mlp_2->forward(ctx, t_emb);
|
||||
return t_emb;
|
||||
}
|
||||
@@ -131,14 +131,14 @@ public:
|
||||
blocks["mlp.2"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, hidden_size, true, true));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [N, input_dim]
|
||||
// return: [N, hidden_size]
|
||||
auto mlp_0 = std::dynamic_pointer_cast<Linear>(blocks["mlp.0"]);
|
||||
auto mlp_2 = std::dynamic_pointer_cast<Linear>(blocks["mlp.2"]);
|
||||
|
||||
x = mlp_0->forward(ctx, x);
|
||||
x = ggml_silu_inplace(ctx, x);
|
||||
x = ggml_silu_inplace(ctx->ggml_ctx, x);
|
||||
x = mlp_2->forward(ctx, x);
|
||||
return x;
|
||||
}
|
||||
@@ -149,16 +149,14 @@ public:
|
||||
int64_t num_heads;
|
||||
bool pre_only;
|
||||
std::string qk_norm;
|
||||
bool flash_attn;
|
||||
|
||||
public:
|
||||
SelfAttention(int64_t dim,
|
||||
int64_t num_heads = 8,
|
||||
std::string qk_norm = "",
|
||||
bool qkv_bias = false,
|
||||
bool pre_only = false,
|
||||
bool flash_attn = false)
|
||||
: num_heads(num_heads), pre_only(pre_only), qk_norm(qk_norm), flash_attn(flash_attn) {
|
||||
bool pre_only = false)
|
||||
: num_heads(num_heads), pre_only(pre_only), qk_norm(qk_norm) {
|
||||
int64_t d_head = dim / num_heads;
|
||||
blocks["qkv"] = std::shared_ptr<GGMLBlock>(new Linear(dim, dim * 3, qkv_bias));
|
||||
if (!pre_only) {
|
||||
@@ -173,15 +171,15 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<struct ggml_tensor*> pre_attention(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
std::vector<struct ggml_tensor*> pre_attention(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
auto qkv_proj = std::dynamic_pointer_cast<Linear>(blocks["qkv"]);
|
||||
|
||||
auto qkv = qkv_proj->forward(ctx, x);
|
||||
auto qkv_vec = split_qkv(ctx, qkv);
|
||||
auto qkv_vec = split_qkv(ctx->ggml_ctx, qkv);
|
||||
int64_t head_dim = qkv_vec[0]->ne[0] / num_heads;
|
||||
auto q = ggml_reshape_4d(ctx, qkv_vec[0], head_dim, num_heads, qkv_vec[0]->ne[1], qkv_vec[0]->ne[2]); // [N, n_token, n_head, d_head]
|
||||
auto k = ggml_reshape_4d(ctx, qkv_vec[1], head_dim, num_heads, qkv_vec[1]->ne[1], qkv_vec[1]->ne[2]); // [N, n_token, n_head, d_head]
|
||||
auto v = qkv_vec[2]; // [N, n_token, n_head*d_head]
|
||||
auto q = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[0], head_dim, num_heads, qkv_vec[0]->ne[1], qkv_vec[0]->ne[2]); // [N, n_token, n_head, d_head]
|
||||
auto k = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[1], head_dim, num_heads, qkv_vec[1]->ne[1], qkv_vec[1]->ne[2]); // [N, n_token, n_head, d_head]
|
||||
auto v = qkv_vec[2]; // [N, n_token, n_head*d_head]
|
||||
|
||||
if (qk_norm == "rms" || qk_norm == "ln") {
|
||||
auto ln_q = std::dynamic_pointer_cast<UnaryBlock>(blocks["ln_q"]);
|
||||
@@ -190,13 +188,13 @@ public:
|
||||
k = ln_k->forward(ctx, k);
|
||||
}
|
||||
|
||||
q = ggml_reshape_3d(ctx, q, q->ne[0] * q->ne[1], q->ne[2], q->ne[3]); // [N, n_token, n_head*d_head]
|
||||
k = ggml_reshape_3d(ctx, k, k->ne[0] * k->ne[1], k->ne[2], k->ne[3]); // [N, n_token, n_head*d_head]
|
||||
q = ggml_reshape_3d(ctx->ggml_ctx, q, q->ne[0] * q->ne[1], q->ne[2], q->ne[3]); // [N, n_token, n_head*d_head]
|
||||
k = ggml_reshape_3d(ctx->ggml_ctx, k, k->ne[0] * k->ne[1], k->ne[2], k->ne[3]); // [N, n_token, n_head*d_head]
|
||||
|
||||
return {q, k, v};
|
||||
}
|
||||
|
||||
struct ggml_tensor* post_attention(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* post_attention(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
GGML_ASSERT(!pre_only);
|
||||
|
||||
auto proj = std::dynamic_pointer_cast<Linear>(blocks["proj"]);
|
||||
@@ -206,12 +204,11 @@ public:
|
||||
}
|
||||
|
||||
// x: [N, n_token, dim]
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x) {
|
||||
auto qkv = pre_attention(ctx, x);
|
||||
x = ggml_nn_attention_ext(ctx, backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, false, true); // [N, n_token, dim]
|
||||
x = post_attention(ctx, x); // [N, n_token, dim]
|
||||
x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, false, ctx->flash_attn_enabled); // [N, n_token, dim]
|
||||
x = post_attention(ctx, x); // [N, n_token, dim]
|
||||
return x;
|
||||
}
|
||||
};
|
||||
@@ -236,7 +233,6 @@ public:
|
||||
int64_t num_heads;
|
||||
bool pre_only;
|
||||
bool self_attn;
|
||||
bool flash_attn;
|
||||
|
||||
public:
|
||||
DismantledBlock(int64_t hidden_size,
|
||||
@@ -245,17 +241,16 @@ public:
|
||||
std::string qk_norm = "",
|
||||
bool qkv_bias = false,
|
||||
bool pre_only = false,
|
||||
bool self_attn = false,
|
||||
bool flash_attn = false)
|
||||
bool self_attn = false)
|
||||
: num_heads(num_heads), pre_only(pre_only), self_attn(self_attn) {
|
||||
// rmsnorm is always Flase
|
||||
// scale_mod_only is always Flase
|
||||
// swiglu is always Flase
|
||||
blocks["norm1"] = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size, 1e-06f, false));
|
||||
blocks["attn"] = std::shared_ptr<GGMLBlock>(new SelfAttention(hidden_size, num_heads, qk_norm, qkv_bias, pre_only, flash_attn));
|
||||
blocks["attn"] = std::shared_ptr<GGMLBlock>(new SelfAttention(hidden_size, num_heads, qk_norm, qkv_bias, pre_only));
|
||||
|
||||
if (self_attn) {
|
||||
blocks["attn2"] = std::shared_ptr<GGMLBlock>(new SelfAttention(hidden_size, num_heads, qk_norm, qkv_bias, false, flash_attn));
|
||||
blocks["attn2"] = std::shared_ptr<GGMLBlock>(new SelfAttention(hidden_size, num_heads, qk_norm, qkv_bias, false));
|
||||
}
|
||||
|
||||
if (!pre_only) {
|
||||
@@ -274,9 +269,9 @@ public:
|
||||
blocks["adaLN_modulation.1"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, n_mods * hidden_size));
|
||||
}
|
||||
|
||||
std::tuple<std::vector<struct ggml_tensor*>, std::vector<struct ggml_tensor*>, std::vector<struct ggml_tensor*>> pre_attention_x(struct ggml_context* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* c) {
|
||||
std::tuple<std::vector<ggml_tensor*>, std::vector<ggml_tensor*>, std::vector<ggml_tensor*>> pre_attention_x(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* c) {
|
||||
GGML_ASSERT(self_attn);
|
||||
// x: [N, n_token, hidden_size]
|
||||
// c: [N, hidden_size]
|
||||
@@ -286,35 +281,35 @@ public:
|
||||
auto adaLN_modulation_1 = std::dynamic_pointer_cast<Linear>(blocks["adaLN_modulation.1"]);
|
||||
|
||||
int64_t n_mods = 9;
|
||||
auto m = adaLN_modulation_1->forward(ctx, ggml_silu(ctx, c)); // [N, n_mods * hidden_size]
|
||||
m = ggml_reshape_3d(ctx, m, c->ne[0], n_mods, c->ne[1]); // [N, n_mods, hidden_size]
|
||||
m = ggml_cont(ctx, ggml_permute(ctx, m, 0, 2, 1, 3)); // [n_mods, N, hidden_size]
|
||||
auto m = adaLN_modulation_1->forward(ctx, ggml_silu(ctx->ggml_ctx, c)); // [N, n_mods * hidden_size]
|
||||
m = ggml_reshape_3d(ctx->ggml_ctx, m, c->ne[0], n_mods, c->ne[1]); // [N, n_mods, hidden_size]
|
||||
m = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, m, 0, 2, 1, 3)); // [n_mods, N, hidden_size]
|
||||
|
||||
int64_t offset = m->nb[1] * m->ne[1];
|
||||
auto shift_msa = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 0); // [N, hidden_size]
|
||||
auto scale_msa = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 1); // [N, hidden_size]
|
||||
auto gate_msa = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 2); // [N, hidden_size]
|
||||
auto shift_msa = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 0); // [N, hidden_size]
|
||||
auto scale_msa = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 1); // [N, hidden_size]
|
||||
auto gate_msa = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 2); // [N, hidden_size]
|
||||
|
||||
auto shift_mlp = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 3); // [N, hidden_size]
|
||||
auto scale_mlp = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 4); // [N, hidden_size]
|
||||
auto gate_mlp = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 5); // [N, hidden_size]
|
||||
auto shift_mlp = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 3); // [N, hidden_size]
|
||||
auto scale_mlp = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 4); // [N, hidden_size]
|
||||
auto gate_mlp = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 5); // [N, hidden_size]
|
||||
|
||||
auto shift_msa2 = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 6); // [N, hidden_size]
|
||||
auto scale_msa2 = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 7); // [N, hidden_size]
|
||||
auto gate_msa2 = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 8); // [N, hidden_size]
|
||||
auto shift_msa2 = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 6); // [N, hidden_size]
|
||||
auto scale_msa2 = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 7); // [N, hidden_size]
|
||||
auto gate_msa2 = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 8); // [N, hidden_size]
|
||||
|
||||
auto x_norm = norm1->forward(ctx, x);
|
||||
|
||||
auto attn_in = modulate(ctx, x_norm, shift_msa, scale_msa);
|
||||
auto attn_in = modulate(ctx->ggml_ctx, x_norm, shift_msa, scale_msa);
|
||||
auto qkv = attn->pre_attention(ctx, attn_in);
|
||||
|
||||
auto attn2_in = modulate(ctx, x_norm, shift_msa2, scale_msa2);
|
||||
auto attn2_in = modulate(ctx->ggml_ctx, x_norm, shift_msa2, scale_msa2);
|
||||
auto qkv2 = attn2->pre_attention(ctx, attn2_in);
|
||||
|
||||
return {qkv, qkv2, {x, gate_msa, shift_mlp, scale_mlp, gate_mlp, gate_msa2}};
|
||||
}
|
||||
|
||||
std::pair<std::vector<struct ggml_tensor*>, std::vector<struct ggml_tensor*>> pre_attention(struct ggml_context* ctx,
|
||||
std::pair<std::vector<struct ggml_tensor*>, std::vector<struct ggml_tensor*>> pre_attention(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* c) {
|
||||
// x: [N, n_token, hidden_size]
|
||||
@@ -327,33 +322,33 @@ public:
|
||||
if (pre_only) {
|
||||
n_mods = 2;
|
||||
}
|
||||
auto m = adaLN_modulation_1->forward(ctx, ggml_silu(ctx, c)); // [N, n_mods * hidden_size]
|
||||
m = ggml_reshape_3d(ctx, m, c->ne[0], n_mods, c->ne[1]); // [N, n_mods, hidden_size]
|
||||
m = ggml_cont(ctx, ggml_permute(ctx, m, 0, 2, 1, 3)); // [n_mods, N, hidden_size]
|
||||
auto m = adaLN_modulation_1->forward(ctx, ggml_silu(ctx->ggml_ctx, c)); // [N, n_mods * hidden_size]
|
||||
m = ggml_reshape_3d(ctx->ggml_ctx, m, c->ne[0], n_mods, c->ne[1]); // [N, n_mods, hidden_size]
|
||||
m = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, m, 0, 2, 1, 3)); // [n_mods, N, hidden_size]
|
||||
|
||||
int64_t offset = m->nb[1] * m->ne[1];
|
||||
auto shift_msa = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 0); // [N, hidden_size]
|
||||
auto scale_msa = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 1); // [N, hidden_size]
|
||||
auto shift_msa = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 0); // [N, hidden_size]
|
||||
auto scale_msa = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 1); // [N, hidden_size]
|
||||
if (!pre_only) {
|
||||
auto gate_msa = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 2); // [N, hidden_size]
|
||||
auto shift_mlp = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 3); // [N, hidden_size]
|
||||
auto scale_mlp = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 4); // [N, hidden_size]
|
||||
auto gate_mlp = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 5); // [N, hidden_size]
|
||||
auto gate_msa = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 2); // [N, hidden_size]
|
||||
auto shift_mlp = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 3); // [N, hidden_size]
|
||||
auto scale_mlp = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 4); // [N, hidden_size]
|
||||
auto gate_mlp = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 5); // [N, hidden_size]
|
||||
|
||||
auto attn_in = modulate(ctx, norm1->forward(ctx, x), shift_msa, scale_msa);
|
||||
auto attn_in = modulate(ctx->ggml_ctx, norm1->forward(ctx, x), shift_msa, scale_msa);
|
||||
|
||||
auto qkv = attn->pre_attention(ctx, attn_in);
|
||||
|
||||
return {qkv, {x, gate_msa, shift_mlp, scale_mlp, gate_mlp}};
|
||||
} else {
|
||||
auto attn_in = modulate(ctx, norm1->forward(ctx, x), shift_msa, scale_msa);
|
||||
auto attn_in = modulate(ctx->ggml_ctx, norm1->forward(ctx, x), shift_msa, scale_msa);
|
||||
auto qkv = attn->pre_attention(ctx, attn_in);
|
||||
|
||||
return {qkv, {nullptr, nullptr, nullptr, nullptr, nullptr}};
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* post_attention_x(struct ggml_context* ctx,
|
||||
struct ggml_tensor* post_attention_x(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* attn_out,
|
||||
struct ggml_tensor* attn2_out,
|
||||
struct ggml_tensor* x,
|
||||
@@ -376,22 +371,22 @@ public:
|
||||
auto norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["norm2"]);
|
||||
auto mlp = std::dynamic_pointer_cast<Mlp>(blocks["mlp"]);
|
||||
|
||||
gate_msa = ggml_reshape_3d(ctx, gate_msa, gate_msa->ne[0], 1, gate_msa->ne[1]); // [N, 1, hidden_size]
|
||||
gate_mlp = ggml_reshape_3d(ctx, gate_mlp, gate_mlp->ne[0], 1, gate_mlp->ne[1]); // [N, 1, hidden_size]
|
||||
gate_msa2 = ggml_reshape_3d(ctx, gate_msa2, gate_msa2->ne[0], 1, gate_msa2->ne[1]); // [N, 1, hidden_size]
|
||||
gate_msa = ggml_reshape_3d(ctx->ggml_ctx, gate_msa, gate_msa->ne[0], 1, gate_msa->ne[1]); // [N, 1, hidden_size]
|
||||
gate_mlp = ggml_reshape_3d(ctx->ggml_ctx, gate_mlp, gate_mlp->ne[0], 1, gate_mlp->ne[1]); // [N, 1, hidden_size]
|
||||
gate_msa2 = ggml_reshape_3d(ctx->ggml_ctx, gate_msa2, gate_msa2->ne[0], 1, gate_msa2->ne[1]); // [N, 1, hidden_size]
|
||||
|
||||
attn_out = attn->post_attention(ctx, attn_out);
|
||||
attn2_out = attn2->post_attention(ctx, attn2_out);
|
||||
|
||||
x = ggml_add(ctx, x, ggml_mul(ctx, attn_out, gate_msa));
|
||||
x = ggml_add(ctx, x, ggml_mul(ctx, attn2_out, gate_msa2));
|
||||
auto mlp_out = mlp->forward(ctx, modulate(ctx, norm2->forward(ctx, x), shift_mlp, scale_mlp));
|
||||
x = ggml_add(ctx, x, ggml_mul(ctx, mlp_out, gate_mlp));
|
||||
x = ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, attn_out, gate_msa));
|
||||
x = ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, attn2_out, gate_msa2));
|
||||
auto mlp_out = mlp->forward(ctx, modulate(ctx->ggml_ctx, norm2->forward(ctx, x), shift_mlp, scale_mlp));
|
||||
x = ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, mlp_out, gate_mlp));
|
||||
|
||||
return x;
|
||||
}
|
||||
|
||||
struct ggml_tensor* post_attention(struct ggml_context* ctx,
|
||||
struct ggml_tensor* post_attention(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* attn_out,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* gate_msa,
|
||||
@@ -411,20 +406,19 @@ public:
|
||||
auto norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["norm2"]);
|
||||
auto mlp = std::dynamic_pointer_cast<Mlp>(blocks["mlp"]);
|
||||
|
||||
gate_msa = ggml_reshape_3d(ctx, gate_msa, gate_msa->ne[0], 1, gate_msa->ne[1]); // [N, 1, hidden_size]
|
||||
gate_mlp = ggml_reshape_3d(ctx, gate_mlp, gate_mlp->ne[0], 1, gate_mlp->ne[1]); // [N, 1, hidden_size]
|
||||
gate_msa = ggml_reshape_3d(ctx->ggml_ctx, gate_msa, gate_msa->ne[0], 1, gate_msa->ne[1]); // [N, 1, hidden_size]
|
||||
gate_mlp = ggml_reshape_3d(ctx->ggml_ctx, gate_mlp, gate_mlp->ne[0], 1, gate_mlp->ne[1]); // [N, 1, hidden_size]
|
||||
|
||||
attn_out = attn->post_attention(ctx, attn_out);
|
||||
|
||||
x = ggml_add(ctx, x, ggml_mul(ctx, attn_out, gate_msa));
|
||||
auto mlp_out = mlp->forward(ctx, modulate(ctx, norm2->forward(ctx, x), shift_mlp, scale_mlp));
|
||||
x = ggml_add(ctx, x, ggml_mul(ctx, mlp_out, gate_mlp));
|
||||
x = ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, attn_out, gate_msa));
|
||||
auto mlp_out = mlp->forward(ctx, modulate(ctx->ggml_ctx, norm2->forward(ctx, x), shift_mlp, scale_mlp));
|
||||
x = ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, mlp_out, gate_mlp));
|
||||
|
||||
return x;
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* c) {
|
||||
// x: [N, n_token, hidden_size]
|
||||
@@ -441,8 +435,8 @@ public:
|
||||
auto qkv2 = std::get<1>(qkv_intermediates);
|
||||
auto intermediates = std::get<2>(qkv_intermediates);
|
||||
|
||||
auto attn_out = ggml_nn_attention_ext(ctx, backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, false, flash_attn); // [N, n_token, dim]
|
||||
auto attn2_out = ggml_nn_attention_ext(ctx, backend, qkv2[0], qkv2[1], qkv2[2], num_heads, nullptr, false, false, flash_attn); // [N, n_token, dim]
|
||||
auto attn_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, false, ctx->flash_attn_enabled); // [N, n_token, dim]
|
||||
auto attn2_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv2[0], qkv2[1], qkv2[2], num_heads, nullptr, false, false, ctx->flash_attn_enabled); // [N, n_token, dim]
|
||||
x = post_attention_x(ctx,
|
||||
attn_out,
|
||||
attn2_out,
|
||||
@@ -458,7 +452,7 @@ public:
|
||||
auto qkv = qkv_intermediates.first;
|
||||
auto intermediates = qkv_intermediates.second;
|
||||
|
||||
auto attn_out = ggml_nn_attention_ext(ctx, backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, false, flash_attn); // [N, n_token, dim]
|
||||
auto attn_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, false, ctx->flash_attn_enabled); // [N, n_token, dim]
|
||||
x = post_attention(ctx,
|
||||
attn_out,
|
||||
intermediates[0],
|
||||
@@ -472,9 +466,7 @@ public:
|
||||
};
|
||||
|
||||
__STATIC_INLINE__ std::pair<struct ggml_tensor*, struct ggml_tensor*>
|
||||
block_mixing(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
bool flash_attn,
|
||||
block_mixing(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* context,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* c,
|
||||
@@ -501,29 +493,29 @@ block_mixing(struct ggml_context* ctx,
|
||||
}
|
||||
std::vector<struct ggml_tensor*> qkv;
|
||||
for (int i = 0; i < 3; i++) {
|
||||
qkv.push_back(ggml_concat(ctx, context_qkv[i], x_qkv[i], 1));
|
||||
qkv.push_back(ggml_concat(ctx->ggml_ctx, context_qkv[i], x_qkv[i], 1));
|
||||
}
|
||||
|
||||
auto attn = ggml_nn_attention_ext(ctx, backend, qkv[0], qkv[1], qkv[2], x_block->num_heads, nullptr, false, false, flash_attn); // [N, n_context + n_token, hidden_size]
|
||||
attn = ggml_cont(ctx, ggml_permute(ctx, attn, 0, 2, 1, 3)); // [n_context + n_token, N, hidden_size]
|
||||
auto context_attn = ggml_view_3d(ctx,
|
||||
auto attn = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], x_block->num_heads, nullptr, false, false, ctx->flash_attn_enabled); // [N, n_context + n_token, hidden_size]
|
||||
attn = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, attn, 0, 2, 1, 3)); // [n_context + n_token, N, hidden_size]
|
||||
auto context_attn = ggml_view_3d(ctx->ggml_ctx,
|
||||
attn,
|
||||
attn->ne[0],
|
||||
attn->ne[1],
|
||||
context->ne[1],
|
||||
attn->nb[1],
|
||||
attn->nb[2],
|
||||
0); // [n_context, N, hidden_size]
|
||||
context_attn = ggml_cont(ctx, ggml_permute(ctx, context_attn, 0, 2, 1, 3)); // [N, n_context, hidden_size]
|
||||
auto x_attn = ggml_view_3d(ctx,
|
||||
0); // [n_context, N, hidden_size]
|
||||
context_attn = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, context_attn, 0, 2, 1, 3)); // [N, n_context, hidden_size]
|
||||
auto x_attn = ggml_view_3d(ctx->ggml_ctx,
|
||||
attn,
|
||||
attn->ne[0],
|
||||
attn->ne[1],
|
||||
x->ne[1],
|
||||
attn->nb[1],
|
||||
attn->nb[2],
|
||||
attn->nb[2] * context->ne[1]); // [n_token, N, hidden_size]
|
||||
x_attn = ggml_cont(ctx, ggml_permute(ctx, x_attn, 0, 2, 1, 3)); // [N, n_token, hidden_size]
|
||||
attn->nb[2] * context->ne[1]); // [n_token, N, hidden_size]
|
||||
x_attn = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x_attn, 0, 2, 1, 3)); // [N, n_token, hidden_size]
|
||||
|
||||
if (!context_block->pre_only) {
|
||||
context = context_block->post_attention(ctx,
|
||||
@@ -538,7 +530,7 @@ block_mixing(struct ggml_context* ctx,
|
||||
}
|
||||
|
||||
if (x_block->self_attn) {
|
||||
auto attn2 = ggml_nn_attention_ext(ctx, backend, x_qkv2[0], x_qkv2[1], x_qkv2[2], x_block->num_heads); // [N, n_token, hidden_size]
|
||||
auto attn2 = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, x_qkv2[0], x_qkv2[1], x_qkv2[2], x_block->num_heads, nullptr, false, false, ctx->flash_attn_enabled); // [N, n_token, hidden_size]
|
||||
|
||||
x = x_block->post_attention_x(ctx,
|
||||
x_attn,
|
||||
@@ -563,8 +555,6 @@ block_mixing(struct ggml_context* ctx,
|
||||
}
|
||||
|
||||
struct JointBlock : public GGMLBlock {
|
||||
bool flash_attn;
|
||||
|
||||
public:
|
||||
JointBlock(int64_t hidden_size,
|
||||
int64_t num_heads,
|
||||
@@ -572,22 +562,19 @@ public:
|
||||
std::string qk_norm = "",
|
||||
bool qkv_bias = false,
|
||||
bool pre_only = false,
|
||||
bool self_attn_x = false,
|
||||
bool flash_attn = false)
|
||||
: flash_attn(flash_attn) {
|
||||
blocks["context_block"] = std::shared_ptr<GGMLBlock>(new DismantledBlock(hidden_size, num_heads, mlp_ratio, qk_norm, qkv_bias, pre_only, false, flash_attn));
|
||||
blocks["x_block"] = std::shared_ptr<GGMLBlock>(new DismantledBlock(hidden_size, num_heads, mlp_ratio, qk_norm, qkv_bias, false, self_attn_x, flash_attn));
|
||||
bool self_attn_x = false) {
|
||||
blocks["context_block"] = std::shared_ptr<GGMLBlock>(new DismantledBlock(hidden_size, num_heads, mlp_ratio, qk_norm, qkv_bias, pre_only, false));
|
||||
blocks["x_block"] = std::shared_ptr<GGMLBlock>(new DismantledBlock(hidden_size, num_heads, mlp_ratio, qk_norm, qkv_bias, false, self_attn_x));
|
||||
}
|
||||
|
||||
std::pair<struct ggml_tensor*, struct ggml_tensor*> forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
std::pair<struct ggml_tensor*, struct ggml_tensor*> forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* context,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* c) {
|
||||
auto context_block = std::dynamic_pointer_cast<DismantledBlock>(blocks["context_block"]);
|
||||
auto x_block = std::dynamic_pointer_cast<DismantledBlock>(blocks["x_block"]);
|
||||
|
||||
return block_mixing(ctx, backend, flash_attn, context, x, c, context_block, x_block);
|
||||
return block_mixing(ctx, context, x, c, context_block, x_block);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -603,7 +590,7 @@ public:
|
||||
blocks["adaLN_modulation.1"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, 2 * hidden_size));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* c) {
|
||||
// x: [N, n_token, hidden_size]
|
||||
@@ -613,15 +600,15 @@ public:
|
||||
auto linear = std::dynamic_pointer_cast<Linear>(blocks["linear"]);
|
||||
auto adaLN_modulation_1 = std::dynamic_pointer_cast<Linear>(blocks["adaLN_modulation.1"]);
|
||||
|
||||
auto m = adaLN_modulation_1->forward(ctx, ggml_silu(ctx, c)); // [N, 2 * hidden_size]
|
||||
m = ggml_reshape_3d(ctx, m, c->ne[0], 2, c->ne[1]); // [N, 2, hidden_size]
|
||||
m = ggml_cont(ctx, ggml_permute(ctx, m, 0, 2, 1, 3)); // [2, N, hidden_size]
|
||||
auto m = adaLN_modulation_1->forward(ctx, ggml_silu(ctx->ggml_ctx, c)); // [N, 2 * hidden_size]
|
||||
m = ggml_reshape_3d(ctx->ggml_ctx, m, c->ne[0], 2, c->ne[1]); // [N, 2, hidden_size]
|
||||
m = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, m, 0, 2, 1, 3)); // [2, N, hidden_size]
|
||||
|
||||
int64_t offset = m->nb[1] * m->ne[1];
|
||||
auto shift = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 0); // [N, hidden_size]
|
||||
auto scale = ggml_view_2d(ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 1); // [N, hidden_size]
|
||||
auto shift = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 0); // [N, hidden_size]
|
||||
auto scale = ggml_view_2d(ctx->ggml_ctx, m, m->ne[0], m->ne[1], m->nb[1], offset * 1); // [N, hidden_size]
|
||||
|
||||
x = modulate(ctx, norm_final->forward(ctx, x), shift, scale);
|
||||
x = modulate(ctx->ggml_ctx, norm_final->forward(ctx, x), shift, scale);
|
||||
x = linear->forward(ctx, x);
|
||||
|
||||
return x;
|
||||
@@ -645,16 +632,14 @@ protected:
|
||||
int64_t context_embedder_out_dim = 1536;
|
||||
int64_t hidden_size;
|
||||
std::string qk_norm;
|
||||
bool flash_attn = false;
|
||||
|
||||
void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, std::string prefix = "") override {
|
||||
void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") override {
|
||||
enum ggml_type wtype = GGML_TYPE_F32;
|
||||
params["pos_embed"] = ggml_new_tensor_3d(ctx, wtype, hidden_size, num_patchs, 1);
|
||||
}
|
||||
|
||||
public:
|
||||
MMDiT(bool flash_attn = false, const String2GGMLType& tensor_types = {})
|
||||
: flash_attn(flash_attn) {
|
||||
MMDiT(const String2TensorStorage& tensor_storage_map = {}) {
|
||||
// input_size is always None
|
||||
// learn_sigma is always False
|
||||
// register_length is alwalys 0
|
||||
@@ -667,8 +652,7 @@ public:
|
||||
// pos_embed_offset is not used
|
||||
// context_embedder_config is always {'target': 'torch.nn.Linear', 'params': {'in_features': 4096, 'out_features': 1536}}
|
||||
|
||||
// read tensors from tensor_types
|
||||
for (auto pair : tensor_types) {
|
||||
for (auto pair : tensor_storage_map) {
|
||||
std::string tensor_name = pair.first;
|
||||
if (tensor_name.find("model.diffusion_model.") == std::string::npos)
|
||||
continue;
|
||||
@@ -722,8 +706,7 @@ public:
|
||||
qk_norm,
|
||||
true,
|
||||
i == depth - 1,
|
||||
i <= d_self,
|
||||
flash_attn));
|
||||
i <= d_self));
|
||||
}
|
||||
|
||||
blocks["final_layer"] = std::shared_ptr<GGMLBlock>(new FinalLayer(hidden_size, patch_size, out_channels));
|
||||
@@ -791,8 +774,7 @@ public:
|
||||
return x;
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward_core_with_concat(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward_core_with_concat(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* c_mod,
|
||||
struct ggml_tensor* context,
|
||||
@@ -811,7 +793,7 @@ public:
|
||||
|
||||
auto block = std::dynamic_pointer_cast<JointBlock>(blocks["joint_blocks." + std::to_string(i)]);
|
||||
|
||||
auto context_x = block->forward(ctx, backend, context, x, c_mod);
|
||||
auto context_x = block->forward(ctx, context, x, c_mod);
|
||||
context = context_x.first;
|
||||
x = context_x.second;
|
||||
}
|
||||
@@ -821,8 +803,7 @@ public:
|
||||
return x;
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* t,
|
||||
struct ggml_tensor* y = nullptr,
|
||||
@@ -840,16 +821,16 @@ public:
|
||||
int64_t w = x->ne[0];
|
||||
int64_t h = x->ne[1];
|
||||
|
||||
auto patch_embed = x_embedder->forward(ctx, x); // [N, H*W, hidden_size]
|
||||
auto pos_embed = cropped_pos_embed(ctx, h, w); // [1, H*W, hidden_size]
|
||||
x = ggml_add(ctx, patch_embed, pos_embed); // [N, H*W, hidden_size]
|
||||
auto patch_embed = x_embedder->forward(ctx, x); // [N, H*W, hidden_size]
|
||||
auto pos_embed = cropped_pos_embed(ctx->ggml_ctx, h, w); // [1, H*W, hidden_size]
|
||||
x = ggml_add(ctx->ggml_ctx, patch_embed, pos_embed); // [N, H*W, hidden_size]
|
||||
|
||||
auto c = t_embedder->forward(ctx, t); // [N, hidden_size]
|
||||
if (y != nullptr && adm_in_channels != -1) {
|
||||
auto y_embedder = std::dynamic_pointer_cast<VectorEmbedder>(blocks["y_embedder"]);
|
||||
|
||||
y = y_embedder->forward(ctx, y); // [N, hidden_size]
|
||||
c = ggml_add(ctx, c, y);
|
||||
c = ggml_add(ctx->ggml_ctx, c, y);
|
||||
}
|
||||
|
||||
if (context != nullptr) {
|
||||
@@ -858,9 +839,9 @@ public:
|
||||
context = context_embedder->forward(ctx, context); // [N, L, D] aka [N, L, 1536]
|
||||
}
|
||||
|
||||
x = forward_core_with_concat(ctx, backend, x, c, context, skip_layers); // (N, H*W, patch_size ** 2 * out_channels)
|
||||
x = forward_core_with_concat(ctx, x, c, context, skip_layers); // (N, H*W, patch_size ** 2 * out_channels)
|
||||
|
||||
x = unpatchify(ctx, x, h, w); // [N, C, H, W]
|
||||
x = unpatchify(ctx->ggml_ctx, x, h, w); // [N, C, H, W]
|
||||
|
||||
return x;
|
||||
}
|
||||
@@ -870,11 +851,10 @@ struct MMDiTRunner : public GGMLRunner {
|
||||
|
||||
MMDiTRunner(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
bool flash_attn,
|
||||
const String2GGMLType& tensor_types = {},
|
||||
const std::string prefix = "")
|
||||
: GGMLRunner(backend, offload_params_to_cpu), mmdit(flash_attn, tensor_types) {
|
||||
mmdit.init(params_ctx, tensor_types, prefix);
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
const std::string prefix = "")
|
||||
: GGMLRunner(backend, offload_params_to_cpu), mmdit(tensor_storage_map) {
|
||||
mmdit.init(params_ctx, tensor_storage_map, prefix);
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -897,8 +877,8 @@ struct MMDiTRunner : public GGMLRunner {
|
||||
y = to_backend(y);
|
||||
timesteps = to_backend(timesteps);
|
||||
|
||||
struct ggml_tensor* out = mmdit.forward(compute_ctx,
|
||||
runtime_backend,
|
||||
auto runner_ctx = get_context();
|
||||
struct ggml_tensor* out = mmdit.forward(&runner_ctx,
|
||||
x,
|
||||
timesteps,
|
||||
y,
|
||||
@@ -972,7 +952,7 @@ struct MMDiTRunner : public GGMLRunner {
|
||||
// ggml_backend_t backend = ggml_backend_cuda_init(0);
|
||||
ggml_backend_t backend = ggml_backend_cpu_init();
|
||||
ggml_type model_data_type = GGML_TYPE_F16;
|
||||
std::shared_ptr<MMDiTRunner> mmdit = std::make_shared<MMDiTRunner>(backend, false, false);
|
||||
std::shared_ptr<MMDiTRunner> mmdit = std::make_shared<MMDiTRunner>(backend, false);
|
||||
{
|
||||
LOG_INFO("loading from '%s'", file_path.c_str());
|
||||
|
||||
|
||||
@@ -140,7 +140,9 @@ std::unordered_map<std::string, std::string> open_clip_to_hf_clip_model = {
|
||||
{"model.visual.proj", "transformer.visual_projection.weight"},
|
||||
};
|
||||
|
||||
std::unordered_map<std::string, std::string> open_clip_to_hk_clip_resblock = {
|
||||
std::unordered_map<std::string, std::string> open_clip_to_hf_clip_resblock = {
|
||||
{"attn.in_proj_bias", "self_attn.in_proj.bias"},
|
||||
{"attn.in_proj_weight", "self_attn.in_proj.weight"},
|
||||
{"attn.out_proj.bias", "self_attn.out_proj.bias"},
|
||||
{"attn.out_proj.weight", "self_attn.out_proj.weight"},
|
||||
{"ln_1.bias", "layer_norm1.bias"},
|
||||
@@ -330,6 +332,10 @@ std::string convert_cond_model_name(const std::string& name) {
|
||||
return new_name;
|
||||
}
|
||||
|
||||
if (new_name == "model.text_projection.weight") {
|
||||
new_name = "transformer.text_model.text_projection";
|
||||
}
|
||||
|
||||
if (open_clip_to_hf_clip_model.find(new_name) != open_clip_to_hf_clip_model.end()) {
|
||||
new_name = open_clip_to_hf_clip_model[new_name];
|
||||
}
|
||||
@@ -347,10 +353,8 @@ std::string convert_cond_model_name(const std::string& name) {
|
||||
std::string idx = remain.substr(0, remain.find("."));
|
||||
std::string suffix = remain.substr(idx.length() + 1);
|
||||
|
||||
if (suffix == "attn.in_proj_weight" || suffix == "attn.in_proj_bias") {
|
||||
new_name = hf_clip_resblock_prefix + idx + "." + suffix;
|
||||
} else if (open_clip_to_hk_clip_resblock.find(suffix) != open_clip_to_hk_clip_resblock.end()) {
|
||||
std::string new_suffix = open_clip_to_hk_clip_resblock[suffix];
|
||||
if (open_clip_to_hf_clip_resblock.find(suffix) != open_clip_to_hf_clip_resblock.end()) {
|
||||
std::string new_suffix = open_clip_to_hf_clip_resblock[suffix];
|
||||
new_name = hf_clip_resblock_prefix + idx + "." + new_suffix;
|
||||
}
|
||||
}
|
||||
@@ -623,6 +627,14 @@ std::string convert_tensor_name(std::string name) {
|
||||
if (starts_with(name, "diffusion_model")) {
|
||||
name = "model." + name;
|
||||
}
|
||||
if (starts_with(name, "model.diffusion_model.up_blocks.0.attentions.0.")) {
|
||||
name.replace(0, sizeof("model.diffusion_model.up_blocks.0.attentions.0.") - 1,
|
||||
"model.diffusion_model.output_blocks.0.1.");
|
||||
}
|
||||
if (starts_with(name, "model.diffusion_model.up_blocks.0.attentions.1.")) {
|
||||
name.replace(0, sizeof("model.diffusion_model.up_blocks.0.attentions.1.") - 1,
|
||||
"model.diffusion_model.output_blocks.1.1.");
|
||||
}
|
||||
// size_t pos = name.find("lora_A");
|
||||
// if (pos != std::string::npos) {
|
||||
// name.replace(pos, strlen("lora_A"), "lora_up");
|
||||
@@ -728,80 +740,6 @@ std::string convert_tensor_name(std::string name) {
|
||||
return new_name;
|
||||
}
|
||||
|
||||
void add_preprocess_tensor_storage_types(String2GGMLType& tensor_storages_types, std::string name, enum ggml_type type) {
|
||||
std::string new_name = convert_tensor_name(name);
|
||||
|
||||
if (new_name.find("cond_stage_model") != std::string::npos && ends_with(new_name, "attn.in_proj_weight")) {
|
||||
size_t prefix_size = new_name.find("attn.in_proj_weight");
|
||||
std::string prefix = new_name.substr(0, prefix_size);
|
||||
tensor_storages_types[prefix + "self_attn.q_proj.weight"] = type;
|
||||
tensor_storages_types[prefix + "self_attn.k_proj.weight"] = type;
|
||||
tensor_storages_types[prefix + "self_attn.v_proj.weight"] = type;
|
||||
} else if (new_name.find("cond_stage_model") != std::string::npos && ends_with(new_name, "attn.in_proj_bias")) {
|
||||
size_t prefix_size = new_name.find("attn.in_proj_bias");
|
||||
std::string prefix = new_name.substr(0, prefix_size);
|
||||
tensor_storages_types[prefix + "self_attn.q_proj.bias"] = type;
|
||||
tensor_storages_types[prefix + "self_attn.k_proj.bias"] = type;
|
||||
tensor_storages_types[prefix + "self_attn.v_proj.bias"] = type;
|
||||
} else {
|
||||
tensor_storages_types[new_name] = type;
|
||||
}
|
||||
}
|
||||
|
||||
void preprocess_tensor(TensorStorage tensor_storage,
|
||||
std::vector<TensorStorage>& processed_tensor_storages) {
|
||||
std::vector<TensorStorage> result;
|
||||
std::string new_name = convert_tensor_name(tensor_storage.name);
|
||||
|
||||
// convert unet transformer linear to conv2d 1x1
|
||||
if (starts_with(new_name, "model.diffusion_model.") &&
|
||||
!starts_with(new_name, "model.diffusion_model.proj_out.") &&
|
||||
(ends_with(new_name, "proj_in.weight") || ends_with(new_name, "proj_out.weight"))) {
|
||||
tensor_storage.unsqueeze();
|
||||
}
|
||||
|
||||
// convert vae attn block linear to conv2d 1x1
|
||||
if (starts_with(new_name, "first_stage_model.") && new_name.find("attn_1") != std::string::npos) {
|
||||
tensor_storage.unsqueeze();
|
||||
}
|
||||
|
||||
// wan vae
|
||||
if (ends_with(new_name, "gamma")) {
|
||||
tensor_storage.reverse_ne();
|
||||
tensor_storage.n_dims = 1;
|
||||
tensor_storage.reverse_ne();
|
||||
}
|
||||
|
||||
tensor_storage.name = new_name;
|
||||
|
||||
if (new_name.find("cond_stage_model") != std::string::npos &&
|
||||
ends_with(new_name, "attn.in_proj_weight")) {
|
||||
size_t prefix_size = new_name.find("attn.in_proj_weight");
|
||||
std::string prefix = new_name.substr(0, prefix_size);
|
||||
|
||||
std::vector<TensorStorage> chunks = tensor_storage.chunk(3);
|
||||
chunks[0].name = prefix + "self_attn.q_proj.weight";
|
||||
chunks[1].name = prefix + "self_attn.k_proj.weight";
|
||||
chunks[2].name = prefix + "self_attn.v_proj.weight";
|
||||
|
||||
processed_tensor_storages.insert(processed_tensor_storages.end(), chunks.begin(), chunks.end());
|
||||
|
||||
} else if (new_name.find("cond_stage_model") != std::string::npos &&
|
||||
ends_with(new_name, "attn.in_proj_bias")) {
|
||||
size_t prefix_size = new_name.find("attn.in_proj_bias");
|
||||
std::string prefix = new_name.substr(0, prefix_size);
|
||||
|
||||
std::vector<TensorStorage> chunks = tensor_storage.chunk(3);
|
||||
chunks[0].name = prefix + "self_attn.q_proj.bias";
|
||||
chunks[1].name = prefix + "self_attn.k_proj.bias";
|
||||
chunks[2].name = prefix + "self_attn.v_proj.bias";
|
||||
|
||||
processed_tensor_storages.insert(processed_tensor_storages.end(), chunks.begin(), chunks.end());
|
||||
} else {
|
||||
processed_tensor_storages.push_back(tensor_storage);
|
||||
}
|
||||
}
|
||||
|
||||
float bf16_to_f32(uint16_t bfloat16) {
|
||||
uint32_t val_bits = (static_cast<uint32_t>(bfloat16) << 16);
|
||||
return *reinterpret_cast<float*>(&val_bits);
|
||||
@@ -977,44 +915,10 @@ void convert_tensor(void* src,
|
||||
|
||||
/*================================================= ModelLoader ==================================================*/
|
||||
|
||||
// ported from https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py#L16
|
||||
std::map<char, int> unicode_to_byte() {
|
||||
std::map<int, char> byte_to_unicode;
|
||||
|
||||
// List of utf-8 byte ranges
|
||||
for (int b = static_cast<int>('!'); b <= static_cast<int>('~'); ++b) {
|
||||
byte_to_unicode[b] = static_cast<char>(b);
|
||||
}
|
||||
|
||||
for (int b = 49825; b <= 49836; ++b) {
|
||||
byte_to_unicode[b] = static_cast<char>(b);
|
||||
}
|
||||
|
||||
for (int b = 49838; b <= 50111; ++b) {
|
||||
byte_to_unicode[b] = static_cast<char>(b);
|
||||
}
|
||||
// printf("%d %d %d %d\n", static_cast<int>('¡'), static_cast<int>('¬'), static_cast<int>('®'), static_cast<int>('ÿ'));
|
||||
// exit(1);
|
||||
|
||||
int n = 0;
|
||||
for (int b = 0; b < 256; ++b) {
|
||||
if (byte_to_unicode.find(b) == byte_to_unicode.end()) {
|
||||
byte_to_unicode[b] = static_cast<char>(256 + n);
|
||||
n++;
|
||||
}
|
||||
}
|
||||
|
||||
// byte_encoder = bytes_to_unicode()
|
||||
// byte_decoder = {v: k for k, v in byte_encoder.items()}
|
||||
std::map<char, int> byte_decoder;
|
||||
|
||||
for (const auto& entry : byte_to_unicode) {
|
||||
byte_decoder[entry.second] = entry.first;
|
||||
}
|
||||
|
||||
byte_to_unicode.clear();
|
||||
|
||||
return byte_decoder;
|
||||
void ModelLoader::add_tensor_storage(const TensorStorage& tensor_storage) {
|
||||
TensorStorage copy = tensor_storage;
|
||||
copy.name = convert_tensor_name(copy.name);
|
||||
tensor_storage_map[copy.name] = std::move(copy);
|
||||
}
|
||||
|
||||
bool is_zip_file(const std::string& file_path) {
|
||||
@@ -1144,8 +1048,7 @@ bool ModelLoader::init_from_gguf_file(const std::string& file_path, const std::s
|
||||
|
||||
// LOG_DEBUG("%s %s", name.c_str(), tensor_storage.to_string().c_str());
|
||||
|
||||
tensor_storages.push_back(tensor_storage);
|
||||
add_preprocess_tensor_storage_types(tensor_storages_types, tensor_storage.name, tensor_storage.type);
|
||||
add_tensor_storage(tensor_storage);
|
||||
}
|
||||
|
||||
return true;
|
||||
@@ -1170,8 +1073,7 @@ bool ModelLoader::init_from_gguf_file(const std::string& file_path, const std::s
|
||||
|
||||
GGML_ASSERT(ggml_nbytes(dummy) == tensor_storage.nbytes());
|
||||
|
||||
tensor_storages.push_back(tensor_storage);
|
||||
add_preprocess_tensor_storage_types(tensor_storages_types, tensor_storage.name, tensor_storage.type);
|
||||
add_tensor_storage(tensor_storage);
|
||||
}
|
||||
|
||||
gguf_free(ctx_gguf_);
|
||||
@@ -1338,8 +1240,7 @@ bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const
|
||||
GGML_ASSERT(tensor_storage.nbytes() == tensor_data_size);
|
||||
}
|
||||
|
||||
tensor_storages.push_back(tensor_storage);
|
||||
add_preprocess_tensor_storage_types(tensor_storages_types, tensor_storage.name, tensor_storage.type);
|
||||
add_tensor_storage(tensor_storage);
|
||||
|
||||
// LOG_DEBUG("%s %s", tensor_storage.to_string().c_str(), dtype.c_str());
|
||||
}
|
||||
@@ -1358,11 +1259,13 @@ bool ModelLoader::init_from_diffusers_file(const std::string& file_path, const s
|
||||
if (!init_from_safetensors_file(unet_path, "unet.")) {
|
||||
return false;
|
||||
}
|
||||
for (auto ts : tensor_storages) {
|
||||
if (ts.name.find("add_embedding") != std::string::npos || ts.name.find("label_emb") != std::string::npos) {
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (name.find("add_embedding") != std::string::npos || name.find("label_emb") != std::string::npos) {
|
||||
// probably SDXL
|
||||
LOG_DEBUG("Fixing name for SDXL output blocks.2.2");
|
||||
for (auto& tensor_storage : tensor_storages) {
|
||||
String2TensorStorage new_tensor_storage_map;
|
||||
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
int len = 34;
|
||||
auto pos = tensor_storage.name.find("unet.up_blocks.0.upsamplers.0.conv");
|
||||
if (pos == std::string::npos) {
|
||||
@@ -1370,11 +1273,15 @@ bool ModelLoader::init_from_diffusers_file(const std::string& file_path, const s
|
||||
pos = tensor_storage.name.find("model.diffusion_model.output_blocks.2.1.conv");
|
||||
}
|
||||
if (pos != std::string::npos) {
|
||||
tensor_storage.name = "model.diffusion_model.output_blocks.2.2.conv" + tensor_storage.name.substr(len);
|
||||
LOG_DEBUG("NEW NAME: %s", tensor_storage.name.c_str());
|
||||
add_preprocess_tensor_storage_types(tensor_storages_types, tensor_storage.name, tensor_storage.type);
|
||||
std::string new_name = "model.diffusion_model.output_blocks.2.2.conv" + name.substr(len);
|
||||
LOG_DEBUG("NEW NAME: %s", new_name.c_str());
|
||||
tensor_storage.name = new_name;
|
||||
new_tensor_storage_map[new_name] = tensor_storage;
|
||||
} else {
|
||||
new_tensor_storage_map[name] = tensor_storage;
|
||||
}
|
||||
}
|
||||
tensor_storage_map = new_tensor_storage_map;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -1700,8 +1607,7 @@ bool ModelLoader::parse_data_pkl(uint8_t* buffer,
|
||||
name = prefix + name;
|
||||
}
|
||||
reader.tensor_storage.name = name;
|
||||
tensor_storages.push_back(reader.tensor_storage);
|
||||
add_preprocess_tensor_storage_types(tensor_storages_types, reader.tensor_storage.name, reader.tensor_storage.type);
|
||||
add_tensor_storage(reader.tensor_storage);
|
||||
|
||||
// LOG_DEBUG("%s", reader.tensor_storage.name.c_str());
|
||||
// reset
|
||||
@@ -1755,18 +1661,8 @@ bool ModelLoader::init_from_ckpt_file(const std::string& file_path, const std::s
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ModelLoader::model_is_unet() {
|
||||
for (auto& tensor_storage : tensor_storages) {
|
||||
if (tensor_storage.name.find("model.diffusion_model.input_blocks.") != std::string::npos) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
SDVersion ModelLoader::get_sd_version() {
|
||||
TensorStorage token_embedding_weight, input_block_weight;
|
||||
bool input_block_checked = false;
|
||||
|
||||
bool has_multiple_encoders = false;
|
||||
bool is_unet = false;
|
||||
@@ -1776,14 +1672,15 @@ SDVersion ModelLoader::get_sd_version() {
|
||||
bool is_wan = false;
|
||||
int64_t patch_embedding_channels = 0;
|
||||
bool has_img_emb = false;
|
||||
bool has_middle_block_1 = false;
|
||||
|
||||
for (auto& tensor_storage : tensor_storages) {
|
||||
if (!(is_xl || is_flux)) {
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (!(is_xl)) {
|
||||
if (tensor_storage.name.find("model.diffusion_model.double_blocks.") != std::string::npos) {
|
||||
is_flux = true;
|
||||
if (input_block_checked) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (tensor_storage.name.find("model.diffusion_model.nerf_final_layer_conv.") != std::string::npos) {
|
||||
return VERSION_CHROMA_RADIANCE;
|
||||
}
|
||||
if (tensor_storage.name.find("model.diffusion_model.joint_blocks.") != std::string::npos) {
|
||||
return VERSION_SD3;
|
||||
@@ -1800,28 +1697,29 @@ SDVersion ModelLoader::get_sd_version() {
|
||||
if (tensor_storage.name.find("model.diffusion_model.img_emb") != std::string::npos) {
|
||||
has_img_emb = true;
|
||||
}
|
||||
if (tensor_storage.name.find("model.diffusion_model.input_blocks.") != std::string::npos || tensor_storage.name.find("unet.down_blocks.") != std::string::npos) {
|
||||
if (tensor_storage.name.find("model.diffusion_model.input_blocks.") != std::string::npos ||
|
||||
tensor_storage.name.find("unet.down_blocks.") != std::string::npos) {
|
||||
is_unet = true;
|
||||
if (has_multiple_encoders) {
|
||||
is_xl = true;
|
||||
if (input_block_checked) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (tensor_storage.name.find("conditioner.embedders.1") != std::string::npos || tensor_storage.name.find("cond_stage_model.1") != std::string::npos || tensor_storage.name.find("te.1") != std::string::npos) {
|
||||
if (tensor_storage.name.find("conditioner.embedders.1") != std::string::npos ||
|
||||
tensor_storage.name.find("cond_stage_model.1") != std::string::npos ||
|
||||
tensor_storage.name.find("te.1") != std::string::npos) {
|
||||
has_multiple_encoders = true;
|
||||
if (is_unet) {
|
||||
is_xl = true;
|
||||
if (input_block_checked) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (tensor_storage.name.find("model.diffusion_model.input_blocks.8.0.time_mixer.mix_factor") != std::string::npos) {
|
||||
return VERSION_SVD;
|
||||
}
|
||||
}
|
||||
if (tensor_storage.name.find("model.diffusion_model.middle_block.1.") != std::string::npos ||
|
||||
tensor_storage.name.find("unet.mid_block.resnets.1.") != std::string::npos) {
|
||||
has_middle_block_1 = true;
|
||||
}
|
||||
if (tensor_storage.name == "cond_stage_model.transformer.text_model.embeddings.token_embedding.weight" ||
|
||||
tensor_storage.name == "cond_stage_model.model.token_embedding.weight" ||
|
||||
tensor_storage.name == "text_model.embeddings.token_embedding.weight" ||
|
||||
@@ -1831,12 +1729,10 @@ SDVersion ModelLoader::get_sd_version() {
|
||||
token_embedding_weight = tensor_storage;
|
||||
// break;
|
||||
}
|
||||
if (tensor_storage.name == "model.diffusion_model.input_blocks.0.0.weight" || tensor_storage.name == "model.diffusion_model.img_in.weight" || tensor_storage.name == "unet.conv_in.weight") {
|
||||
input_block_weight = tensor_storage;
|
||||
input_block_checked = true;
|
||||
if (is_xl || is_flux) {
|
||||
break;
|
||||
}
|
||||
if (tensor_storage.name == "model.diffusion_model.input_blocks.0.0.weight" ||
|
||||
tensor_storage.name == "model.diffusion_model.img_in.weight" ||
|
||||
tensor_storage.name == "unet.conv_in.weight") {
|
||||
input_block_weight = tensor_storage;
|
||||
}
|
||||
}
|
||||
if (is_wan) {
|
||||
@@ -1858,6 +1754,9 @@ SDVersion ModelLoader::get_sd_version() {
|
||||
if (is_ip2p) {
|
||||
return VERSION_SDXL_PIX2PIX;
|
||||
}
|
||||
if (!has_middle_block_1) {
|
||||
return VERSION_SDXL_SSD1B;
|
||||
}
|
||||
return VERSION_SDXL;
|
||||
}
|
||||
|
||||
@@ -1881,11 +1780,17 @@ SDVersion ModelLoader::get_sd_version() {
|
||||
if (is_ip2p) {
|
||||
return VERSION_SD1_PIX2PIX;
|
||||
}
|
||||
if (!has_middle_block_1) {
|
||||
return VERSION_SD1_TINY_UNET;
|
||||
}
|
||||
return VERSION_SD1;
|
||||
} else if (token_embedding_weight.ne[0] == 1024) {
|
||||
if (is_inpaint) {
|
||||
return VERSION_SD2_INPAINT;
|
||||
}
|
||||
if (!has_middle_block_1) {
|
||||
return VERSION_SD2_TINY_UNET;
|
||||
}
|
||||
return VERSION_SD2;
|
||||
}
|
||||
return VERSION_COUNT;
|
||||
@@ -1893,7 +1798,7 @@ SDVersion ModelLoader::get_sd_version() {
|
||||
|
||||
std::map<ggml_type, uint32_t> ModelLoader::get_wtype_stat() {
|
||||
std::map<ggml_type, uint32_t> wtype_stat;
|
||||
for (auto& tensor_storage : tensor_storages) {
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (is_unused_tensor(tensor_storage.name)) {
|
||||
continue;
|
||||
}
|
||||
@@ -1910,7 +1815,7 @@ std::map<ggml_type, uint32_t> ModelLoader::get_wtype_stat() {
|
||||
|
||||
std::map<ggml_type, uint32_t> ModelLoader::get_conditioner_wtype_stat() {
|
||||
std::map<ggml_type, uint32_t> wtype_stat;
|
||||
for (auto& tensor_storage : tensor_storages) {
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (is_unused_tensor(tensor_storage.name)) {
|
||||
continue;
|
||||
}
|
||||
@@ -1934,7 +1839,7 @@ std::map<ggml_type, uint32_t> ModelLoader::get_conditioner_wtype_stat() {
|
||||
|
||||
std::map<ggml_type, uint32_t> ModelLoader::get_diffusion_model_wtype_stat() {
|
||||
std::map<ggml_type, uint32_t> wtype_stat;
|
||||
for (auto& tensor_storage : tensor_storages) {
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (is_unused_tensor(tensor_storage.name)) {
|
||||
continue;
|
||||
}
|
||||
@@ -1955,7 +1860,7 @@ std::map<ggml_type, uint32_t> ModelLoader::get_diffusion_model_wtype_stat() {
|
||||
|
||||
std::map<ggml_type, uint32_t> ModelLoader::get_vae_wtype_stat() {
|
||||
std::map<ggml_type, uint32_t> wtype_stat;
|
||||
for (auto& tensor_storage : tensor_storages) {
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (is_unused_tensor(tensor_storage.name)) {
|
||||
continue;
|
||||
}
|
||||
@@ -1976,26 +1881,14 @@ std::map<ggml_type, uint32_t> ModelLoader::get_vae_wtype_stat() {
|
||||
}
|
||||
|
||||
void ModelLoader::set_wtype_override(ggml_type wtype, std::string prefix) {
|
||||
for (auto& pair : tensor_storages_types) {
|
||||
if (prefix.size() < 1 || pair.first.substr(0, prefix.size()) == prefix) {
|
||||
bool found = false;
|
||||
for (auto& tensor_storage : tensor_storages) {
|
||||
std::map<std::string, ggml_type> temp;
|
||||
add_preprocess_tensor_storage_types(temp, tensor_storage.name, tensor_storage.type);
|
||||
for (auto& preprocessed_name : temp) {
|
||||
if (preprocessed_name.first == pair.first) {
|
||||
if (tensor_should_be_converted(tensor_storage, wtype)) {
|
||||
pair.second = wtype;
|
||||
}
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (found) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (!starts_with(name, prefix)) {
|
||||
continue;
|
||||
}
|
||||
if (!tensor_should_be_converted(tensor_storage, wtype)) {
|
||||
continue;
|
||||
}
|
||||
tensor_storage.expected_type = wtype;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2030,74 +1923,13 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
|
||||
LOG_DEBUG("using %d threads for model loading", num_threads_to_use);
|
||||
|
||||
int64_t start_time = ggml_time_ms();
|
||||
|
||||
std::vector<TensorStorage> processed_tensor_storages;
|
||||
|
||||
{
|
||||
struct IndexedStorage {
|
||||
size_t index;
|
||||
TensorStorage ts;
|
||||
};
|
||||
|
||||
std::mutex vec_mutex;
|
||||
std::vector<IndexedStorage> all_results;
|
||||
|
||||
int n_threads = std::min(num_threads_to_use, (int)tensor_storages.size());
|
||||
if (n_threads < 1) {
|
||||
n_threads = 1;
|
||||
}
|
||||
std::vector<std::thread> workers;
|
||||
|
||||
for (int i = 0; i < n_threads; ++i) {
|
||||
workers.emplace_back([&, thread_id = i]() {
|
||||
std::vector<IndexedStorage> local_results;
|
||||
std::vector<TensorStorage> temp_storages;
|
||||
|
||||
for (size_t j = thread_id; j < tensor_storages.size(); j += n_threads) {
|
||||
const auto& tensor_storage = tensor_storages[j];
|
||||
if (is_unused_tensor(tensor_storage.name)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
temp_storages.clear();
|
||||
preprocess_tensor(tensor_storage, temp_storages);
|
||||
|
||||
for (const auto& ts : temp_storages) {
|
||||
local_results.push_back({j, ts});
|
||||
}
|
||||
}
|
||||
|
||||
if (!local_results.empty()) {
|
||||
std::lock_guard<std::mutex> lock(vec_mutex);
|
||||
all_results.insert(all_results.end(),
|
||||
local_results.begin(), local_results.end());
|
||||
}
|
||||
});
|
||||
}
|
||||
for (auto& w : workers) {
|
||||
w.join();
|
||||
}
|
||||
|
||||
std::vector<IndexedStorage> deduplicated;
|
||||
deduplicated.reserve(all_results.size());
|
||||
std::unordered_map<std::string, size_t> name_to_pos;
|
||||
for (auto& entry : all_results) {
|
||||
auto it = name_to_pos.find(entry.ts.name);
|
||||
if (it == name_to_pos.end()) {
|
||||
name_to_pos.emplace(entry.ts.name, deduplicated.size());
|
||||
deduplicated.push_back(entry);
|
||||
} else if (deduplicated[it->second].index < entry.index) {
|
||||
deduplicated[it->second] = entry;
|
||||
}
|
||||
}
|
||||
|
||||
std::sort(deduplicated.begin(), deduplicated.end(), [](const IndexedStorage& a, const IndexedStorage& b) {
|
||||
return a.index < b.index;
|
||||
});
|
||||
|
||||
processed_tensor_storages.reserve(deduplicated.size());
|
||||
for (auto& entry : deduplicated) {
|
||||
processed_tensor_storages.push_back(entry.ts);
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (is_unused_tensor(tensor_storage.name)) {
|
||||
continue;
|
||||
}
|
||||
processed_tensor_storages.push_back(tensor_storage);
|
||||
}
|
||||
|
||||
process_time_ms = ggml_time_ms() - start_time;
|
||||
@@ -2214,107 +2046,72 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
|
||||
}
|
||||
};
|
||||
|
||||
char* read_buf = nullptr;
|
||||
char* target_buf = nullptr;
|
||||
char* convert_buf = nullptr;
|
||||
if (dst_tensor->buffer == nullptr || ggml_backend_buffer_is_host(dst_tensor->buffer)) {
|
||||
if (tensor_storage.type == dst_tensor->type) {
|
||||
GGML_ASSERT(ggml_nbytes(dst_tensor) == tensor_storage.nbytes());
|
||||
if (tensor_storage.is_f64 || tensor_storage.is_i64) {
|
||||
read_buffer.resize(tensor_storage.nbytes_to_read());
|
||||
read_data((char*)read_buffer.data(), nbytes_to_read);
|
||||
read_buf = (char*)read_buffer.data();
|
||||
} else {
|
||||
read_data((char*)dst_tensor->data, nbytes_to_read);
|
||||
read_buf = (char*)dst_tensor->data;
|
||||
}
|
||||
t1 = ggml_time_ms();
|
||||
read_time_ms.fetch_add(t1 - t0);
|
||||
|
||||
t0 = ggml_time_ms();
|
||||
if (tensor_storage.is_bf16) {
|
||||
// inplace op
|
||||
bf16_to_f32_vec((uint16_t*)dst_tensor->data, (float*)dst_tensor->data, tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f8_e4m3) {
|
||||
// inplace op
|
||||
f8_e4m3_to_f16_vec((uint8_t*)dst_tensor->data, (uint16_t*)dst_tensor->data, tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f8_e5m2) {
|
||||
// inplace op
|
||||
f8_e5m2_to_f16_vec((uint8_t*)dst_tensor->data, (uint16_t*)dst_tensor->data, tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f64) {
|
||||
f64_to_f32_vec((double*)read_buffer.data(), (float*)dst_tensor->data, tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_i64) {
|
||||
i64_to_i32_vec((int64_t*)read_buffer.data(), (int32_t*)dst_tensor->data, tensor_storage.nelements());
|
||||
}
|
||||
t1 = ggml_time_ms();
|
||||
convert_time_ms.fetch_add(t1 - t0);
|
||||
target_buf = (char*)dst_tensor->data;
|
||||
} else {
|
||||
read_buffer.resize(std::max(tensor_storage.nbytes(), tensor_storage.nbytes_to_read()));
|
||||
read_data((char*)read_buffer.data(), nbytes_to_read);
|
||||
t1 = ggml_time_ms();
|
||||
read_time_ms.fetch_add(t1 - t0);
|
||||
|
||||
t0 = ggml_time_ms();
|
||||
if (tensor_storage.is_bf16) {
|
||||
// inplace op
|
||||
bf16_to_f32_vec((uint16_t*)read_buffer.data(), (float*)read_buffer.data(), tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f8_e4m3) {
|
||||
// inplace op
|
||||
f8_e4m3_to_f16_vec((uint8_t*)read_buffer.data(), (uint16_t*)read_buffer.data(), tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f8_e5m2) {
|
||||
// inplace op
|
||||
f8_e5m2_to_f16_vec((uint8_t*)read_buffer.data(), (uint16_t*)read_buffer.data(), tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f64) {
|
||||
// inplace op
|
||||
f64_to_f32_vec((double*)read_buffer.data(), (float*)read_buffer.data(), tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_i64) {
|
||||
// inplace op
|
||||
i64_to_i32_vec((int64_t*)read_buffer.data(), (int32_t*)read_buffer.data(), tensor_storage.nelements());
|
||||
}
|
||||
convert_tensor((void*)read_buffer.data(), tensor_storage.type, dst_tensor->data, dst_tensor->type, (int)tensor_storage.nelements() / (int)tensor_storage.ne[0], (int)tensor_storage.ne[0]);
|
||||
t1 = ggml_time_ms();
|
||||
convert_time_ms.fetch_add(t1 - t0);
|
||||
read_buf = (char*)read_buffer.data();
|
||||
target_buf = read_buf;
|
||||
convert_buf = (char*)dst_tensor->data;
|
||||
}
|
||||
} else {
|
||||
read_buffer.resize(std::max(tensor_storage.nbytes(), tensor_storage.nbytes_to_read()));
|
||||
read_data((char*)read_buffer.data(), nbytes_to_read);
|
||||
t1 = ggml_time_ms();
|
||||
read_time_ms.fetch_add(t1 - t0);
|
||||
|
||||
t0 = ggml_time_ms();
|
||||
if (tensor_storage.is_bf16) {
|
||||
// inplace op
|
||||
bf16_to_f32_vec((uint16_t*)read_buffer.data(), (float*)read_buffer.data(), tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f8_e4m3) {
|
||||
// inplace op
|
||||
f8_e4m3_to_f16_vec((uint8_t*)read_buffer.data(), (uint16_t*)read_buffer.data(), tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f8_e5m2) {
|
||||
// inplace op
|
||||
f8_e5m2_to_f16_vec((uint8_t*)read_buffer.data(), (uint16_t*)read_buffer.data(), tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f64) {
|
||||
// inplace op
|
||||
f64_to_f32_vec((double*)read_buffer.data(), (float*)read_buffer.data(), tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_i64) {
|
||||
// inplace op
|
||||
i64_to_i32_vec((int64_t*)read_buffer.data(), (int32_t*)read_buffer.data(), tensor_storage.nelements());
|
||||
}
|
||||
|
||||
if (tensor_storage.type == dst_tensor->type) {
|
||||
// copy to device memory
|
||||
t1 = ggml_time_ms();
|
||||
convert_time_ms.fetch_add(t1 - t0);
|
||||
t0 = ggml_time_ms();
|
||||
ggml_backend_tensor_set(dst_tensor, read_buffer.data(), 0, ggml_nbytes(dst_tensor));
|
||||
t1 = ggml_time_ms();
|
||||
copy_to_backend_time_ms.fetch_add(t1 - t0);
|
||||
} else {
|
||||
// convert first, then copy to device memory
|
||||
read_buf = (char*)read_buffer.data();
|
||||
target_buf = read_buf;
|
||||
|
||||
if (tensor_storage.type != dst_tensor->type) {
|
||||
convert_buffer.resize(ggml_nbytes(dst_tensor));
|
||||
convert_tensor((void*)read_buffer.data(), tensor_storage.type, (void*)convert_buffer.data(), dst_tensor->type, (int)tensor_storage.nelements() / (int)tensor_storage.ne[0], (int)tensor_storage.ne[0]);
|
||||
t1 = ggml_time_ms();
|
||||
convert_time_ms.fetch_add(t1 - t0);
|
||||
t0 = ggml_time_ms();
|
||||
ggml_backend_tensor_set(dst_tensor, convert_buffer.data(), 0, ggml_nbytes(dst_tensor));
|
||||
t1 = ggml_time_ms();
|
||||
copy_to_backend_time_ms.fetch_add(t1 - t0);
|
||||
convert_buf = (char*)convert_buffer.data();
|
||||
}
|
||||
}
|
||||
|
||||
t0 = ggml_time_ms();
|
||||
read_data(read_buf, nbytes_to_read);
|
||||
t1 = ggml_time_ms();
|
||||
read_time_ms.fetch_add(t1 - t0);
|
||||
|
||||
t0 = ggml_time_ms();
|
||||
if (tensor_storage.is_bf16) {
|
||||
bf16_to_f32_vec((uint16_t*)read_buf, (float*)target_buf, tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f8_e4m3) {
|
||||
f8_e4m3_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f8_e5m2) {
|
||||
f8_e5m2_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_f64) {
|
||||
f64_to_f32_vec((double*)read_buf, (float*)target_buf, tensor_storage.nelements());
|
||||
} else if (tensor_storage.is_i64) {
|
||||
i64_to_i32_vec((int64_t*)read_buf, (int32_t*)target_buf, tensor_storage.nelements());
|
||||
}
|
||||
if (tensor_storage.type != dst_tensor->type) {
|
||||
convert_tensor((void*)target_buf,
|
||||
tensor_storage.type,
|
||||
convert_buf,
|
||||
dst_tensor->type,
|
||||
(int)tensor_storage.nelements() / (int)tensor_storage.ne[0],
|
||||
(int)tensor_storage.ne[0]);
|
||||
} else {
|
||||
convert_buf = read_buf;
|
||||
}
|
||||
t1 = ggml_time_ms();
|
||||
convert_time_ms.fetch_add(t1 - t0);
|
||||
|
||||
if (dst_tensor->buffer != nullptr && !ggml_backend_buffer_is_host(dst_tensor->buffer)) {
|
||||
t0 = ggml_time_ms();
|
||||
ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor));
|
||||
t1 = ggml_time_ms();
|
||||
copy_to_backend_time_ms.fetch_add(t1 - t0);
|
||||
}
|
||||
}
|
||||
if (zip != nullptr) {
|
||||
zip_close(zip);
|
||||
@@ -2503,7 +2300,7 @@ bool ModelLoader::tensor_should_be_converted(const TensorStorage& tensor_storage
|
||||
bool ModelLoader::save_to_gguf_file(const std::string& file_path, ggml_type type, const std::string& tensor_type_rules_str) {
|
||||
auto backend = ggml_backend_cpu_init();
|
||||
size_t mem_size = 1 * 1024 * 1024; // for padding
|
||||
mem_size += tensor_storages.size() * ggml_tensor_overhead();
|
||||
mem_size += tensor_storage_map.size() * ggml_tensor_overhead();
|
||||
mem_size += get_params_mem_size(backend, type);
|
||||
LOG_INFO("model tensors mem size: %.2fMB", mem_size / 1024.f / 1024.f);
|
||||
ggml_context* ggml_ctx = ggml_init({mem_size, nullptr, false});
|
||||
@@ -2570,14 +2367,10 @@ int64_t ModelLoader::get_params_mem_size(ggml_backend_t backend, ggml_type type)
|
||||
}
|
||||
int64_t mem_size = 0;
|
||||
std::vector<TensorStorage> processed_tensor_storages;
|
||||
for (auto& tensor_storage : tensor_storages) {
|
||||
for (auto [name, tensor_storage] : tensor_storage_map) {
|
||||
if (is_unused_tensor(tensor_storage.name)) {
|
||||
continue;
|
||||
}
|
||||
preprocess_tensor(tensor_storage, processed_tensor_storages);
|
||||
}
|
||||
|
||||
for (auto& tensor_storage : processed_tensor_storages) {
|
||||
if (tensor_should_be_converted(tensor_storage, type)) {
|
||||
tensor_storage.type = type;
|
||||
}
|
||||
|
||||
@@ -23,17 +23,21 @@ enum SDVersion {
|
||||
VERSION_SD1,
|
||||
VERSION_SD1_INPAINT,
|
||||
VERSION_SD1_PIX2PIX,
|
||||
VERSION_SD1_TINY_UNET,
|
||||
VERSION_SD2,
|
||||
VERSION_SD2_INPAINT,
|
||||
VERSION_SD2_TINY_UNET,
|
||||
VERSION_SDXL,
|
||||
VERSION_SDXL_INPAINT,
|
||||
VERSION_SDXL_PIX2PIX,
|
||||
VERSION_SDXL_SSD1B,
|
||||
VERSION_SVD,
|
||||
VERSION_SD3,
|
||||
VERSION_FLUX,
|
||||
VERSION_FLUX_FILL,
|
||||
VERSION_FLUX_CONTROLS,
|
||||
VERSION_FLEX_2,
|
||||
VERSION_CHROMA_RADIANCE,
|
||||
VERSION_WAN2,
|
||||
VERSION_WAN2_2_I2V,
|
||||
VERSION_WAN2_2_TI2V,
|
||||
@@ -42,21 +46,30 @@ enum SDVersion {
|
||||
};
|
||||
|
||||
static inline bool sd_version_is_sd1(SDVersion version) {
|
||||
if (version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX) {
|
||||
if (version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX || version == VERSION_SD1_TINY_UNET) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool sd_version_is_sd2(SDVersion version) {
|
||||
if (version == VERSION_SD2 || version == VERSION_SD2_INPAINT) {
|
||||
if (version == VERSION_SD2 || version == VERSION_SD2_INPAINT || version == VERSION_SD2_TINY_UNET) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool sd_version_is_sdxl(SDVersion version) {
|
||||
if (version == VERSION_SDXL || version == VERSION_SDXL_INPAINT || version == VERSION_SDXL_PIX2PIX) {
|
||||
if (version == VERSION_SDXL || version == VERSION_SDXL_INPAINT || version == VERSION_SDXL_PIX2PIX || version == VERSION_SDXL_SSD1B) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool sd_version_is_unet(SDVersion version) {
|
||||
if (sd_version_is_sd1(version) ||
|
||||
sd_version_is_sd2(version) ||
|
||||
sd_version_is_sdxl(version)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
@@ -70,7 +83,11 @@ static inline bool sd_version_is_sd3(SDVersion version) {
|
||||
}
|
||||
|
||||
static inline bool sd_version_is_flux(SDVersion version) {
|
||||
if (version == VERSION_FLUX || version == VERSION_FLUX_FILL || version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2) {
|
||||
if (version == VERSION_FLUX ||
|
||||
version == VERSION_FLUX_FILL ||
|
||||
version == VERSION_FLUX_CONTROLS ||
|
||||
version == VERSION_FLEX_2 ||
|
||||
version == VERSION_CHROMA_RADIANCE) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
@@ -127,6 +144,7 @@ enum PMVersion {
|
||||
struct TensorStorage {
|
||||
std::string name;
|
||||
ggml_type type = GGML_TYPE_F32;
|
||||
ggml_type expected_type = GGML_TYPE_COUNT;
|
||||
bool is_bf16 = false;
|
||||
bool is_f8_e4m3 = false;
|
||||
bool is_f8_e5m2 = false;
|
||||
@@ -235,12 +253,14 @@ struct TensorStorage {
|
||||
|
||||
typedef std::function<bool(const TensorStorage&, ggml_tensor**)> on_new_tensor_cb_t;
|
||||
|
||||
typedef std::map<std::string, enum ggml_type> String2GGMLType;
|
||||
typedef std::map<std::string, TensorStorage> String2TensorStorage;
|
||||
|
||||
class ModelLoader {
|
||||
protected:
|
||||
std::vector<std::string> file_paths_;
|
||||
std::vector<TensorStorage> tensor_storages;
|
||||
String2TensorStorage tensor_storage_map;
|
||||
|
||||
void add_tensor_storage(const TensorStorage& tensor_storage);
|
||||
|
||||
bool parse_data_pkl(uint8_t* buffer,
|
||||
size_t buffer_size,
|
||||
@@ -255,15 +275,13 @@ protected:
|
||||
bool init_from_diffusers_file(const std::string& file_path, const std::string& prefix = "");
|
||||
|
||||
public:
|
||||
String2GGMLType tensor_storages_types;
|
||||
|
||||
bool init_from_file(const std::string& file_path, const std::string& prefix = "");
|
||||
bool model_is_unet();
|
||||
SDVersion get_sd_version();
|
||||
std::map<ggml_type, uint32_t> get_wtype_stat();
|
||||
std::map<ggml_type, uint32_t> get_conditioner_wtype_stat();
|
||||
std::map<ggml_type, uint32_t> get_diffusion_model_wtype_stat();
|
||||
std::map<ggml_type, uint32_t> get_vae_wtype_stat();
|
||||
String2TensorStorage& get_tensor_storage_map() { return tensor_storage_map; }
|
||||
void set_wtype_override(ggml_type wtype, std::string prefix = "");
|
||||
bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads = 0);
|
||||
bool load_tensors(std::map<std::string, struct ggml_tensor*>& tensors,
|
||||
@@ -272,8 +290,8 @@ public:
|
||||
|
||||
std::vector<std::string> get_tensor_names() const {
|
||||
std::vector<std::string> names;
|
||||
for (const auto& ts : tensor_storages) {
|
||||
names.push_back(ts.name);
|
||||
for (const auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
names.push_back(name);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ public:
|
||||
blocks["layernorm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(in_dim));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [N, channels, h, w]
|
||||
|
||||
auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
|
||||
@@ -29,15 +29,15 @@ public:
|
||||
auto layer_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["layernorm"]);
|
||||
|
||||
struct ggml_tensor* r = x;
|
||||
// x = ggml_nn_layer_norm(ctx, x, ln_w, ln_b);
|
||||
// x = ggml_ext_layer_norm(ctx, x, ln_w, ln_b);
|
||||
x = layer_norm->forward(ctx, x);
|
||||
// x = ggml_add(ctx, ggml_mul_mat(ctx, fc1_w, x), fc1_b);
|
||||
x = fc1->forward(ctx, x);
|
||||
x = ggml_gelu_inplace(ctx, x);
|
||||
x = ggml_gelu_inplace(ctx->ggml_ctx, x);
|
||||
x = fc2->forward(ctx, x);
|
||||
// x = ggml_add(ctx, ggml_mul_mat(ctx, fc2_w, x), fc2_b);
|
||||
if (use_residue)
|
||||
x = ggml_add(ctx, x, r);
|
||||
x = ggml_add(ctx->ggml_ctx, x, r);
|
||||
return x;
|
||||
}
|
||||
};
|
||||
@@ -54,7 +54,7 @@ public:
|
||||
blocks["1"] = std::shared_ptr<GGMLBlock>(new Mlp(dim, inner_dim, dim, false));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x) {
|
||||
auto norm = std::dynamic_pointer_cast<LayerNorm>(blocks["0"]);
|
||||
auto ff = std::dynamic_pointer_cast<Mlp>(blocks["1"]);
|
||||
@@ -100,7 +100,7 @@ public:
|
||||
ggml_cont(ctx, tli)};
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* latents) {
|
||||
// x (torch.Tensor): image features
|
||||
@@ -118,33 +118,33 @@ public:
|
||||
auto to_q = std::dynamic_pointer_cast<Linear>(blocks["to_q"]);
|
||||
auto q = to_q->forward(ctx, latents);
|
||||
|
||||
auto kv_input = ggml_concat(ctx, x, latents, 1);
|
||||
auto kv_input = ggml_concat(ctx->ggml_ctx, x, latents, 1);
|
||||
auto to_kv = std::dynamic_pointer_cast<Linear>(blocks["to_kv"]);
|
||||
auto kv = to_kv->forward(ctx, kv_input);
|
||||
auto k = ggml_view_4d(ctx, kv, kv->ne[0] / 2, kv->ne[1], kv->ne[2], kv->ne[3], kv->nb[1] / 2, kv->nb[2] / 2, kv->nb[3] / 2, 0);
|
||||
auto v = ggml_view_4d(ctx, kv, kv->ne[0] / 2, kv->ne[1], kv->ne[2], kv->ne[3], kv->nb[1] / 2, kv->nb[2] / 2, kv->nb[3] / 2, kv->nb[0] * (kv->ne[0] / 2));
|
||||
k = ggml_cont(ctx, k);
|
||||
v = ggml_cont(ctx, v);
|
||||
q = reshape_tensor(ctx, q, heads);
|
||||
k = reshape_tensor(ctx, k, heads);
|
||||
v = reshape_tensor(ctx, v, heads);
|
||||
auto k = ggml_view_4d(ctx->ggml_ctx, kv, kv->ne[0] / 2, kv->ne[1], kv->ne[2], kv->ne[3], kv->nb[1] / 2, kv->nb[2] / 2, kv->nb[3] / 2, 0);
|
||||
auto v = ggml_view_4d(ctx->ggml_ctx, kv, kv->ne[0] / 2, kv->ne[1], kv->ne[2], kv->ne[3], kv->nb[1] / 2, kv->nb[2] / 2, kv->nb[3] / 2, kv->nb[0] * (kv->ne[0] / 2));
|
||||
k = ggml_cont(ctx->ggml_ctx, k);
|
||||
v = ggml_cont(ctx->ggml_ctx, v);
|
||||
q = reshape_tensor(ctx->ggml_ctx, q, heads);
|
||||
k = reshape_tensor(ctx->ggml_ctx, k, heads);
|
||||
v = reshape_tensor(ctx->ggml_ctx, v, heads);
|
||||
scale = 1.f / sqrt(sqrt((float)dim_head));
|
||||
k = ggml_scale_inplace(ctx, k, scale);
|
||||
q = ggml_scale_inplace(ctx, q, scale);
|
||||
k = ggml_scale_inplace(ctx->ggml_ctx, k, scale);
|
||||
q = ggml_scale_inplace(ctx->ggml_ctx, q, scale);
|
||||
// auto weight = ggml_mul_mat(ctx, q, k);
|
||||
auto weight = ggml_mul_mat(ctx, k, q); // NOTE order of mul is opposite to pytorch
|
||||
auto weight = ggml_mul_mat(ctx->ggml_ctx, k, q); // NOTE order of mul is opposite to pytorch
|
||||
|
||||
// GGML's softmax() is equivalent to pytorch's softmax(x, dim=-1)
|
||||
// in this case, dimension along which Softmax will be computed is the last dim
|
||||
// in torch and the first dim in GGML, consistent with the convention that pytorch's
|
||||
// last dimension (varying most rapidly) corresponds to GGML's first (varying most rapidly).
|
||||
// weight = ggml_soft_max(ctx, weight);
|
||||
weight = ggml_soft_max_inplace(ctx, weight);
|
||||
v = ggml_cont(ctx, ggml_transpose(ctx, v));
|
||||
weight = ggml_soft_max_inplace(ctx->ggml_ctx, weight);
|
||||
v = ggml_cont(ctx->ggml_ctx, ggml_transpose(ctx->ggml_ctx, v));
|
||||
// auto out = ggml_mul_mat(ctx, weight, v);
|
||||
auto out = ggml_mul_mat(ctx, v, weight); // NOTE order of mul is opposite to pytorch
|
||||
out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3));
|
||||
out = ggml_reshape_3d(ctx, out, ne[0], ne[1], ggml_nelements(out) / (ne[0] * ne[1]));
|
||||
auto out = ggml_mul_mat(ctx->ggml_ctx, v, weight); // NOTE order of mul is opposite to pytorch
|
||||
out = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, out, 0, 2, 1, 3));
|
||||
out = ggml_reshape_3d(ctx->ggml_ctx, out, ne[0], ne[1], ggml_nelements(out) / (ne[0] * ne[1]));
|
||||
auto to_out = std::dynamic_pointer_cast<Linear>(blocks["to_out"]);
|
||||
out = to_out->forward(ctx, out);
|
||||
return out;
|
||||
@@ -176,7 +176,7 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* latents,
|
||||
struct ggml_tensor* x) {
|
||||
// x: [N, channels, h, w]
|
||||
@@ -191,9 +191,9 @@ public:
|
||||
name = "layers." + std::to_string(i) + ".1";
|
||||
auto ff = std::dynamic_pointer_cast<PMFeedForward>(blocks[name]);
|
||||
auto t = attn->forward(ctx, x, latents);
|
||||
latents = ggml_add(ctx, t, latents);
|
||||
latents = ggml_add(ctx->ggml_ctx, t, latents);
|
||||
t = ff->forward(ctx, latents);
|
||||
latents = ggml_add(ctx, t, latents);
|
||||
latents = ggml_add(ctx->ggml_ctx, t, latents);
|
||||
}
|
||||
latents = proj_out->forward(ctx, latents);
|
||||
latents = norm_out->forward(ctx, latents);
|
||||
@@ -225,7 +225,7 @@ public:
|
||||
4));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* last_hidden_state) {
|
||||
// x: [N, channels, h, w]
|
||||
@@ -235,11 +235,11 @@ public:
|
||||
|
||||
x = token_proj->forward(ctx, x);
|
||||
int64_t nel = ggml_nelements(x);
|
||||
x = ggml_reshape_3d(ctx, x, cross_attention_dim, num_tokens, nel / (cross_attention_dim * num_tokens));
|
||||
x = ggml_reshape_3d(ctx->ggml_ctx, x, cross_attention_dim, num_tokens, nel / (cross_attention_dim * num_tokens));
|
||||
x = token_norm->forward(ctx, x);
|
||||
struct ggml_tensor* out = perceiver_resampler->forward(ctx, x, last_hidden_state);
|
||||
if (use_residul)
|
||||
out = ggml_add(ctx, x, out);
|
||||
out = ggml_add(ctx->ggml_ctx, x, out);
|
||||
return out;
|
||||
}
|
||||
};
|
||||
@@ -256,24 +256,24 @@ public:
|
||||
blocks["layer_norm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(embed_dim));
|
||||
}
|
||||
|
||||
struct ggml_tensor* fuse_fn(struct ggml_context* ctx,
|
||||
struct ggml_tensor* fuse_fn(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* prompt_embeds,
|
||||
struct ggml_tensor* id_embeds) {
|
||||
auto mlp1 = std::dynamic_pointer_cast<FuseBlock>(blocks["mlp1"]);
|
||||
auto mlp2 = std::dynamic_pointer_cast<FuseBlock>(blocks["mlp2"]);
|
||||
auto layer_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm"]);
|
||||
|
||||
auto stacked_id_embeds = ggml_concat(ctx, prompt_embeds, id_embeds, 0);
|
||||
auto stacked_id_embeds = ggml_concat(ctx->ggml_ctx, prompt_embeds, id_embeds, 0);
|
||||
|
||||
stacked_id_embeds = mlp1->forward(ctx, stacked_id_embeds);
|
||||
stacked_id_embeds = ggml_add(ctx, stacked_id_embeds, prompt_embeds);
|
||||
stacked_id_embeds = ggml_add(ctx->ggml_ctx, stacked_id_embeds, prompt_embeds);
|
||||
stacked_id_embeds = mlp2->forward(ctx, stacked_id_embeds);
|
||||
stacked_id_embeds = layer_norm->forward(ctx, stacked_id_embeds);
|
||||
|
||||
return stacked_id_embeds;
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* prompt_embeds,
|
||||
struct ggml_tensor* id_embeds,
|
||||
struct ggml_tensor* class_tokens_mask,
|
||||
@@ -286,25 +286,25 @@ public:
|
||||
// # slice out the image token embeddings
|
||||
ggml_set_name(class_tokens_mask_pos, "class_tokens_mask_pos");
|
||||
ggml_set_name(prompt_embeds, "prompt_embeds");
|
||||
struct ggml_tensor* image_token_embeds = ggml_get_rows(ctx, prompt_embeds, class_tokens_mask_pos);
|
||||
struct ggml_tensor* image_token_embeds = ggml_get_rows(ctx->ggml_ctx, prompt_embeds, class_tokens_mask_pos);
|
||||
ggml_set_name(image_token_embeds, "image_token_embeds");
|
||||
valid_id_embeds = ggml_reshape_2d(ctx, valid_id_embeds, valid_id_embeds->ne[0],
|
||||
valid_id_embeds = ggml_reshape_2d(ctx->ggml_ctx, valid_id_embeds, valid_id_embeds->ne[0],
|
||||
ggml_nelements(valid_id_embeds) / valid_id_embeds->ne[0]);
|
||||
struct ggml_tensor* stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds);
|
||||
|
||||
if (left && right) {
|
||||
stacked_id_embeds = ggml_concat(ctx, left, stacked_id_embeds, 1);
|
||||
stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right, 1);
|
||||
stacked_id_embeds = ggml_concat(ctx->ggml_ctx, left, stacked_id_embeds, 1);
|
||||
stacked_id_embeds = ggml_concat(ctx->ggml_ctx, stacked_id_embeds, right, 1);
|
||||
} else if (left) {
|
||||
stacked_id_embeds = ggml_concat(ctx, left, stacked_id_embeds, 1);
|
||||
stacked_id_embeds = ggml_concat(ctx->ggml_ctx, left, stacked_id_embeds, 1);
|
||||
} else if (right) {
|
||||
stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right, 1);
|
||||
stacked_id_embeds = ggml_concat(ctx->ggml_ctx, stacked_id_embeds, right, 1);
|
||||
}
|
||||
|
||||
class_tokens_mask = ggml_cont(ctx, ggml_transpose(ctx, class_tokens_mask));
|
||||
class_tokens_mask = ggml_repeat(ctx, class_tokens_mask, prompt_embeds);
|
||||
prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask);
|
||||
struct ggml_tensor* updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds);
|
||||
class_tokens_mask = ggml_cont(ctx->ggml_ctx, ggml_transpose(ctx->ggml_ctx, class_tokens_mask));
|
||||
class_tokens_mask = ggml_repeat(ctx->ggml_ctx, class_tokens_mask, prompt_embeds);
|
||||
prompt_embeds = ggml_mul(ctx->ggml_ctx, prompt_embeds, class_tokens_mask);
|
||||
struct ggml_tensor* updated_prompt_embeds = ggml_add(ctx->ggml_ctx, prompt_embeds, stacked_id_embeds);
|
||||
ggml_set_name(updated_prompt_embeds, "updated_prompt_embeds");
|
||||
return updated_prompt_embeds;
|
||||
}
|
||||
@@ -317,8 +317,7 @@ struct PhotoMakerIDEncoderBlock : public CLIPVisionModelProjection {
|
||||
blocks["fuse_module"] = std::shared_ptr<GGMLBlock>(new FuseModule(2048));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* id_pixel_values,
|
||||
struct ggml_tensor* prompt_embeds,
|
||||
struct ggml_tensor* class_tokens_mask,
|
||||
@@ -331,15 +330,15 @@ struct PhotoMakerIDEncoderBlock : public CLIPVisionModelProjection {
|
||||
auto visual_projection_2 = std::dynamic_pointer_cast<Linear>(blocks["visual_projection_2"]);
|
||||
auto fuse_module = std::dynamic_pointer_cast<FuseModule>(blocks["fuse_module"]);
|
||||
|
||||
struct ggml_tensor* shared_id_embeds = vision_model->forward(ctx, backend, id_pixel_values); // [N, hidden_size]
|
||||
struct ggml_tensor* id_embeds = visual_projection->forward(ctx, shared_id_embeds); // [N, proj_dim(768)]
|
||||
struct ggml_tensor* id_embeds_2 = visual_projection_2->forward(ctx, shared_id_embeds); // [N, 1280]
|
||||
struct ggml_tensor* shared_id_embeds = vision_model->forward(ctx, id_pixel_values); // [N, hidden_size]
|
||||
struct ggml_tensor* id_embeds = visual_projection->forward(ctx, shared_id_embeds); // [N, proj_dim(768)]
|
||||
struct ggml_tensor* id_embeds_2 = visual_projection_2->forward(ctx, shared_id_embeds); // [N, 1280]
|
||||
|
||||
id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3));
|
||||
id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3));
|
||||
id_embeds = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, id_embeds, 2, 0, 1, 3));
|
||||
id_embeds_2 = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, id_embeds_2, 2, 0, 1, 3));
|
||||
|
||||
id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2, 2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right
|
||||
id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3));
|
||||
id_embeds = ggml_concat(ctx->ggml_ctx, id_embeds, id_embeds_2, 2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right
|
||||
id_embeds = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, id_embeds, 1, 2, 0, 3));
|
||||
|
||||
struct ggml_tensor* updated_prompt_embeds = fuse_module->forward(ctx,
|
||||
prompt_embeds,
|
||||
@@ -366,8 +365,7 @@ struct PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock : public CLIPVisionMo
|
||||
num_tokens));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* id_pixel_values,
|
||||
struct ggml_tensor* prompt_embeds,
|
||||
struct ggml_tensor* class_tokens_mask,
|
||||
@@ -381,7 +379,7 @@ struct PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock : public CLIPVisionMo
|
||||
auto qformer_perceiver = std::dynamic_pointer_cast<QFormerPerceiver>(blocks["qformer_perceiver"]);
|
||||
|
||||
// struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values); // [N, hidden_size]
|
||||
struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, backend, id_pixel_values, false); // [N, hidden_size]
|
||||
struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values, false); // [N, hidden_size]
|
||||
id_embeds = qformer_perceiver->forward(ctx, id_embeds, last_hidden_state);
|
||||
|
||||
struct ggml_tensor* updated_prompt_embeds = fuse_module->forward(ctx,
|
||||
@@ -414,7 +412,7 @@ public:
|
||||
public:
|
||||
PhotoMakerIDEncoder(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types,
|
||||
const String2TensorStorage& tensor_storage_map,
|
||||
const std::string prefix,
|
||||
SDVersion version = VERSION_SDXL,
|
||||
PMVersion pm_v = PM_VERSION_1,
|
||||
@@ -424,9 +422,9 @@ public:
|
||||
pm_version(pm_v),
|
||||
style_strength(sty) {
|
||||
if (pm_version == PM_VERSION_1) {
|
||||
id_encoder.init(params_ctx, tensor_types, prefix);
|
||||
id_encoder.init(params_ctx, tensor_storage_map, prefix);
|
||||
} else if (pm_version == PM_VERSION_2) {
|
||||
id_encoder2.init(params_ctx, tensor_types, prefix);
|
||||
id_encoder2.init(params_ctx, tensor_storage_map, prefix);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -458,7 +456,7 @@ public:
|
||||
zeros_right.clear();
|
||||
zeros_right_16.clear();
|
||||
|
||||
ggml_context* ctx0 = compute_ctx;
|
||||
auto runner_ctx = get_context();
|
||||
|
||||
struct ggml_cgraph* gf = ggml_new_graph(compute_ctx);
|
||||
|
||||
@@ -466,7 +464,7 @@ public:
|
||||
int64_t seq_length = prompt_embeds->ne[1];
|
||||
ggml_type type = GGML_TYPE_F32;
|
||||
|
||||
struct ggml_tensor* class_tokens_mask_d = ggml_new_tensor_1d(ctx0, type, class_tokens_mask.size());
|
||||
struct ggml_tensor* class_tokens_mask_d = ggml_new_tensor_1d(runner_ctx.ggml_ctx, type, class_tokens_mask.size());
|
||||
|
||||
struct ggml_tensor* id_pixel_values_d = to_backend(id_pixel_values);
|
||||
struct ggml_tensor* prompt_embeds_d = to_backend(prompt_embeds);
|
||||
@@ -488,16 +486,16 @@ public:
|
||||
}
|
||||
// printf("\n");
|
||||
if (ctmpos[0] > 0) {
|
||||
// left = ggml_new_tensor_3d(ctx0, type, hidden_size, 1, ctmpos[0]);
|
||||
left = ggml_new_tensor_3d(ctx0, type, hidden_size, ctmpos[0], 1);
|
||||
// left = ggml_new_tensor_3d(runner_ctx.ggml_ctx, type, hidden_size, 1, ctmpos[0]);
|
||||
left = ggml_new_tensor_3d(runner_ctx.ggml_ctx, type, hidden_size, ctmpos[0], 1);
|
||||
}
|
||||
if (ctmpos[ctmpos.size() - 1] < seq_length - 1) {
|
||||
// right = ggml_new_tensor_3d(ctx0, type,
|
||||
// right = ggml_new_tensor_3d(runner_ctx.ggml_ctx, type,
|
||||
// hidden_size, 1, seq_length - ctmpos[ctmpos.size() - 1] - 1);
|
||||
right = ggml_new_tensor_3d(ctx0, type,
|
||||
right = ggml_new_tensor_3d(runner_ctx.ggml_ctx, type,
|
||||
hidden_size, seq_length - ctmpos[ctmpos.size() - 1] - 1, 1);
|
||||
}
|
||||
struct ggml_tensor* class_tokens_mask_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ctmpos.size());
|
||||
struct ggml_tensor* class_tokens_mask_pos = ggml_new_tensor_1d(runner_ctx.ggml_ctx, GGML_TYPE_I32, ctmpos.size());
|
||||
|
||||
{
|
||||
if (type == GGML_TYPE_F16)
|
||||
@@ -530,16 +528,14 @@ public:
|
||||
}
|
||||
struct ggml_tensor* updated_prompt_embeds = nullptr;
|
||||
if (pm_version == PM_VERSION_1)
|
||||
updated_prompt_embeds = id_encoder.forward(ctx0,
|
||||
runtime_backend,
|
||||
updated_prompt_embeds = id_encoder.forward(&runner_ctx,
|
||||
id_pixel_values_d,
|
||||
prompt_embeds_d,
|
||||
class_tokens_mask_d,
|
||||
class_tokens_mask_pos,
|
||||
left, right);
|
||||
else if (pm_version == PM_VERSION_2)
|
||||
updated_prompt_embeds = id_encoder2.forward(ctx0,
|
||||
runtime_backend,
|
||||
updated_prompt_embeds = id_encoder2.forward(&runner_ctx,
|
||||
id_pixel_values_d,
|
||||
prompt_embeds_d,
|
||||
class_tokens_mask_d,
|
||||
|
||||
+31
-31
@@ -28,7 +28,7 @@ void gaussian_kernel(struct ggml_tensor* kernel) {
|
||||
for (int x = 0; x < kernel->ne[1]; x++) {
|
||||
float gy = -ks_mid + x;
|
||||
float k_ = expf(-((gx * gx + gy * gy) / (2.0f * powf(sigma, 2.0f)))) * normal;
|
||||
ggml_tensor_set_f32(kernel, k_, x, y);
|
||||
ggml_ext_tensor_set_f32(kernel, k_, x, y);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,11 +36,11 @@ void gaussian_kernel(struct ggml_tensor* kernel) {
|
||||
void grayscale(struct ggml_tensor* rgb_img, struct ggml_tensor* grayscale) {
|
||||
for (int iy = 0; iy < rgb_img->ne[1]; iy++) {
|
||||
for (int ix = 0; ix < rgb_img->ne[0]; ix++) {
|
||||
float r = ggml_tensor_get_f32(rgb_img, ix, iy);
|
||||
float g = ggml_tensor_get_f32(rgb_img, ix, iy, 1);
|
||||
float b = ggml_tensor_get_f32(rgb_img, ix, iy, 2);
|
||||
float r = ggml_ext_tensor_get_f32(rgb_img, ix, iy);
|
||||
float g = ggml_ext_tensor_get_f32(rgb_img, ix, iy, 1);
|
||||
float b = ggml_ext_tensor_get_f32(rgb_img, ix, iy, 2);
|
||||
float gray = 0.2989f * r + 0.5870f * g + 0.1140f * b;
|
||||
ggml_tensor_set_f32(grayscale, gray, ix, iy);
|
||||
ggml_ext_tensor_set_f32(grayscale, gray, ix, iy);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -81,37 +81,37 @@ void normalize_tensor(struct ggml_tensor* g) {
|
||||
void non_max_supression(struct ggml_tensor* result, struct ggml_tensor* G, struct ggml_tensor* D) {
|
||||
for (int iy = 1; iy < result->ne[1] - 1; iy++) {
|
||||
for (int ix = 1; ix < result->ne[0] - 1; ix++) {
|
||||
float angle = ggml_tensor_get_f32(D, ix, iy) * 180.0f / M_PI_;
|
||||
float angle = ggml_ext_tensor_get_f32(D, ix, iy) * 180.0f / M_PI_;
|
||||
angle = angle < 0.0f ? angle += 180.0f : angle;
|
||||
float q = 1.0f;
|
||||
float r = 1.0f;
|
||||
|
||||
// angle 0
|
||||
if ((0 >= angle && angle < 22.5f) || (157.5f >= angle && angle <= 180)) {
|
||||
q = ggml_tensor_get_f32(G, ix, iy + 1);
|
||||
r = ggml_tensor_get_f32(G, ix, iy - 1);
|
||||
q = ggml_ext_tensor_get_f32(G, ix, iy + 1);
|
||||
r = ggml_ext_tensor_get_f32(G, ix, iy - 1);
|
||||
}
|
||||
// angle 45
|
||||
else if (22.5f >= angle && angle < 67.5f) {
|
||||
q = ggml_tensor_get_f32(G, ix + 1, iy - 1);
|
||||
r = ggml_tensor_get_f32(G, ix - 1, iy + 1);
|
||||
q = ggml_ext_tensor_get_f32(G, ix + 1, iy - 1);
|
||||
r = ggml_ext_tensor_get_f32(G, ix - 1, iy + 1);
|
||||
}
|
||||
// angle 90
|
||||
else if (67.5f >= angle && angle < 112.5) {
|
||||
q = ggml_tensor_get_f32(G, ix + 1, iy);
|
||||
r = ggml_tensor_get_f32(G, ix - 1, iy);
|
||||
q = ggml_ext_tensor_get_f32(G, ix + 1, iy);
|
||||
r = ggml_ext_tensor_get_f32(G, ix - 1, iy);
|
||||
}
|
||||
// angle 135
|
||||
else if (112.5 >= angle && angle < 157.5f) {
|
||||
q = ggml_tensor_get_f32(G, ix - 1, iy - 1);
|
||||
r = ggml_tensor_get_f32(G, ix + 1, iy + 1);
|
||||
q = ggml_ext_tensor_get_f32(G, ix - 1, iy - 1);
|
||||
r = ggml_ext_tensor_get_f32(G, ix + 1, iy + 1);
|
||||
}
|
||||
|
||||
float cur = ggml_tensor_get_f32(G, ix, iy);
|
||||
float cur = ggml_ext_tensor_get_f32(G, ix, iy);
|
||||
if ((cur >= q) && (cur >= r)) {
|
||||
ggml_tensor_set_f32(result, cur, ix, iy);
|
||||
ggml_ext_tensor_set_f32(result, cur, ix, iy);
|
||||
} else {
|
||||
ggml_tensor_set_f32(result, 0.0f, ix, iy);
|
||||
ggml_ext_tensor_set_f32(result, 0.0f, ix, iy);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -138,9 +138,9 @@ void threshold_hystersis(struct ggml_tensor* img, float high_threshold, float lo
|
||||
for (int iy = 0; iy < img->ne[1]; iy++) {
|
||||
for (int ix = 0; ix < img->ne[0]; ix++) {
|
||||
if (ix >= 3 && ix <= img->ne[0] - 3 && iy >= 3 && iy <= img->ne[1] - 3) {
|
||||
ggml_tensor_set_f32(img, ggml_tensor_get_f32(img, ix, iy), ix, iy);
|
||||
ggml_ext_tensor_set_f32(img, ggml_ext_tensor_get_f32(img, ix, iy), ix, iy);
|
||||
} else {
|
||||
ggml_tensor_set_f32(img, 0.0f, ix, iy);
|
||||
ggml_ext_tensor_set_f32(img, 0.0f, ix, iy);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -148,14 +148,14 @@ void threshold_hystersis(struct ggml_tensor* img, float high_threshold, float lo
|
||||
// hysteresis
|
||||
for (int iy = 1; iy < img->ne[1] - 1; iy++) {
|
||||
for (int ix = 1; ix < img->ne[0] - 1; ix++) {
|
||||
float imd_v = ggml_tensor_get_f32(img, ix, iy);
|
||||
float imd_v = ggml_ext_tensor_get_f32(img, ix, iy);
|
||||
if (imd_v == weak) {
|
||||
if (ggml_tensor_get_f32(img, ix + 1, iy - 1) == strong || ggml_tensor_get_f32(img, ix + 1, iy) == strong ||
|
||||
ggml_tensor_get_f32(img, ix, iy - 1) == strong || ggml_tensor_get_f32(img, ix, iy + 1) == strong ||
|
||||
ggml_tensor_get_f32(img, ix - 1, iy - 1) == strong || ggml_tensor_get_f32(img, ix - 1, iy) == strong) {
|
||||
ggml_tensor_set_f32(img, strong, ix, iy);
|
||||
if (ggml_ext_tensor_get_f32(img, ix + 1, iy - 1) == strong || ggml_ext_tensor_get_f32(img, ix + 1, iy) == strong ||
|
||||
ggml_ext_tensor_get_f32(img, ix, iy - 1) == strong || ggml_ext_tensor_get_f32(img, ix, iy + 1) == strong ||
|
||||
ggml_ext_tensor_get_f32(img, ix - 1, iy - 1) == strong || ggml_ext_tensor_get_f32(img, ix - 1, iy) == strong) {
|
||||
ggml_ext_tensor_set_f32(img, strong, ix, iy);
|
||||
} else {
|
||||
ggml_tensor_set_f32(img, 0.0f, ix, iy);
|
||||
ggml_ext_tensor_set_f32(img, 0.0f, ix, iy);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -198,7 +198,7 @@ bool preprocess_canny(sd_image_t img, float high_threshold, float low_threshold,
|
||||
struct ggml_tensor* iY = ggml_dup_tensor(work_ctx, image_gray);
|
||||
struct ggml_tensor* G = ggml_dup_tensor(work_ctx, image_gray);
|
||||
struct ggml_tensor* tetha = ggml_dup_tensor(work_ctx, image_gray);
|
||||
sd_image_to_tensor(img, image);
|
||||
sd_image_to_ggml_tensor(img, image);
|
||||
grayscale(image, image_gray);
|
||||
convolve(image_gray, image_gray, gkernel, 2);
|
||||
convolve(image_gray, iX, sf_kx, 1);
|
||||
@@ -211,14 +211,14 @@ bool preprocess_canny(sd_image_t img, float high_threshold, float low_threshold,
|
||||
// to RGB channels
|
||||
for (int iy = 0; iy < img.height; iy++) {
|
||||
for (int ix = 0; ix < img.width; ix++) {
|
||||
float gray = ggml_tensor_get_f32(image_gray, ix, iy);
|
||||
float gray = ggml_ext_tensor_get_f32(image_gray, ix, iy);
|
||||
gray = inverse ? 1.0f - gray : gray;
|
||||
ggml_tensor_set_f32(image, gray, ix, iy);
|
||||
ggml_tensor_set_f32(image, gray, ix, iy, 1);
|
||||
ggml_tensor_set_f32(image, gray, ix, iy, 2);
|
||||
ggml_ext_tensor_set_f32(image, gray, ix, iy);
|
||||
ggml_ext_tensor_set_f32(image, gray, ix, iy, 1);
|
||||
ggml_ext_tensor_set_f32(image, gray, ix, iy, 2);
|
||||
}
|
||||
}
|
||||
sd_tensor_to_image(image, img.data);
|
||||
ggml_tensor_to_sd_image(image, img.data);
|
||||
ggml_free(work_ctx);
|
||||
return true;
|
||||
}
|
||||
|
||||
+74
-87
@@ -27,18 +27,18 @@ namespace Qwen {
|
||||
blocks["linear_2"] = std::shared_ptr<GGMLBlock>(new Linear(time_embed_dim, out_dim, sample_proj_bias));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* sample,
|
||||
struct ggml_tensor* condition = nullptr) {
|
||||
if (condition != nullptr) {
|
||||
auto cond_proj = std::dynamic_pointer_cast<Linear>(blocks["cond_proj"]);
|
||||
sample = ggml_add(ctx, sample, cond_proj->forward(ctx, condition));
|
||||
sample = ggml_add(ctx->ggml_ctx, sample, cond_proj->forward(ctx, condition));
|
||||
}
|
||||
auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["linear_1"]);
|
||||
auto linear_2 = std::dynamic_pointer_cast<Linear>(blocks["linear_2"]);
|
||||
|
||||
sample = linear_1->forward(ctx, sample);
|
||||
sample = ggml_silu_inplace(ctx, sample);
|
||||
sample = ggml_silu_inplace(ctx->ggml_ctx, sample);
|
||||
sample = linear_2->forward(ctx, sample);
|
||||
return sample;
|
||||
}
|
||||
@@ -50,13 +50,13 @@ namespace Qwen {
|
||||
blocks["timestep_embedder"] = std::shared_ptr<GGMLBlock>(new TimestepEmbedding(256, embedding_dim));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* timesteps) {
|
||||
// timesteps: [N,]
|
||||
// return: [N, embedding_dim]
|
||||
auto timestep_embedder = std::dynamic_pointer_cast<TimestepEmbedding>(blocks["timestep_embedder"]);
|
||||
|
||||
auto timesteps_proj = ggml_nn_timestep_embedding(ctx, timesteps, 256, 10000, 1.f);
|
||||
auto timesteps_proj = ggml_ext_timestep_embedding(ctx->ggml_ctx, timesteps, 256, 10000, 1.f);
|
||||
auto timesteps_emb = timestep_embedder->forward(ctx, timesteps_proj);
|
||||
return timesteps_emb;
|
||||
}
|
||||
@@ -65,7 +65,6 @@ namespace Qwen {
|
||||
struct QwenImageAttention : public GGMLBlock {
|
||||
protected:
|
||||
int64_t dim_head;
|
||||
bool flash_attn;
|
||||
|
||||
public:
|
||||
QwenImageAttention(int64_t query_dim,
|
||||
@@ -75,9 +74,8 @@ namespace Qwen {
|
||||
int64_t out_context_dim = 0,
|
||||
bool bias = true,
|
||||
bool out_bias = true,
|
||||
float eps = 1e-6,
|
||||
bool flash_attn = false)
|
||||
: dim_head(dim_head), flash_attn(flash_attn) {
|
||||
float eps = 1e-6)
|
||||
: dim_head(dim_head) {
|
||||
int64_t inner_dim = out_dim > 0 ? out_dim : dim_head * num_heads;
|
||||
out_dim = out_dim > 0 ? out_dim : query_dim;
|
||||
out_context_dim = out_context_dim > 0 ? out_context_dim : query_dim;
|
||||
@@ -105,8 +103,7 @@ namespace Qwen {
|
||||
blocks["to_add_out"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, out_context_dim, out_bias, false, false, scale));
|
||||
}
|
||||
|
||||
std::pair<ggml_tensor*, ggml_tensor*> forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
std::pair<ggml_tensor*, ggml_tensor*> forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* img,
|
||||
struct ggml_tensor* txt,
|
||||
struct ggml_tensor* pe,
|
||||
@@ -138,49 +135,49 @@ namespace Qwen {
|
||||
|
||||
auto img_q = to_q->forward(ctx, img);
|
||||
int64_t num_heads = img_q->ne[0] / dim_head;
|
||||
img_q = ggml_reshape_4d(ctx, img_q, dim_head, num_heads, n_img_token, N); // [N, n_img_token, n_head, d_head]
|
||||
img_q = ggml_reshape_4d(ctx->ggml_ctx, img_q, dim_head, num_heads, n_img_token, N); // [N, n_img_token, n_head, d_head]
|
||||
auto img_k = to_k->forward(ctx, img);
|
||||
img_k = ggml_reshape_4d(ctx, img_k, dim_head, num_heads, n_img_token, N); // [N, n_img_token, n_head, d_head]
|
||||
img_k = ggml_reshape_4d(ctx->ggml_ctx, img_k, dim_head, num_heads, n_img_token, N); // [N, n_img_token, n_head, d_head]
|
||||
auto img_v = to_v->forward(ctx, img);
|
||||
img_v = ggml_reshape_4d(ctx, img_v, dim_head, num_heads, n_img_token, N); // [N, n_img_token, n_head, d_head]
|
||||
img_v = ggml_reshape_4d(ctx->ggml_ctx, img_v, dim_head, num_heads, n_img_token, N); // [N, n_img_token, n_head, d_head]
|
||||
|
||||
img_q = norm_q->forward(ctx, img_q);
|
||||
img_k = norm_k->forward(ctx, img_k);
|
||||
|
||||
auto txt_q = add_q_proj->forward(ctx, txt);
|
||||
txt_q = ggml_reshape_4d(ctx, txt_q, dim_head, num_heads, n_txt_token, N); // [N, n_txt_token, n_head, d_head]
|
||||
txt_q = ggml_reshape_4d(ctx->ggml_ctx, txt_q, dim_head, num_heads, n_txt_token, N); // [N, n_txt_token, n_head, d_head]
|
||||
auto txt_k = add_k_proj->forward(ctx, txt);
|
||||
txt_k = ggml_reshape_4d(ctx, txt_k, dim_head, num_heads, n_txt_token, N); // [N, n_txt_token, n_head, d_head]
|
||||
txt_k = ggml_reshape_4d(ctx->ggml_ctx, txt_k, dim_head, num_heads, n_txt_token, N); // [N, n_txt_token, n_head, d_head]
|
||||
auto txt_v = add_v_proj->forward(ctx, txt);
|
||||
txt_v = ggml_reshape_4d(ctx, txt_v, dim_head, num_heads, n_txt_token, N); // [N, n_txt_token, n_head, d_head]
|
||||
txt_v = ggml_reshape_4d(ctx->ggml_ctx, txt_v, dim_head, num_heads, n_txt_token, N); // [N, n_txt_token, n_head, d_head]
|
||||
|
||||
txt_q = norm_added_q->forward(ctx, txt_q);
|
||||
txt_k = norm_added_k->forward(ctx, txt_k);
|
||||
|
||||
auto q = ggml_concat(ctx, txt_q, img_q, 2); // [N, n_txt_token + n_img_token, n_head, d_head]
|
||||
auto k = ggml_concat(ctx, txt_k, img_k, 2); // [N, n_txt_token + n_img_token, n_head, d_head]
|
||||
auto v = ggml_concat(ctx, txt_v, img_v, 2); // [N, n_txt_token + n_img_token, n_head, d_head]
|
||||
auto q = ggml_concat(ctx->ggml_ctx, txt_q, img_q, 2); // [N, n_txt_token + n_img_token, n_head, d_head]
|
||||
auto k = ggml_concat(ctx->ggml_ctx, txt_k, img_k, 2); // [N, n_txt_token + n_img_token, n_head, d_head]
|
||||
auto v = ggml_concat(ctx->ggml_ctx, txt_v, img_v, 2); // [N, n_txt_token + n_img_token, n_head, d_head]
|
||||
|
||||
auto attn = Rope::attention(ctx, backend, q, k, v, pe, mask, flash_attn, (1.0f / 128.f)); // [N, n_txt_token + n_img_token, n_head*d_head]
|
||||
attn = ggml_cont(ctx, ggml_permute(ctx, attn, 0, 2, 1, 3)); // [n_txt_token + n_img_token, N, hidden_size]
|
||||
auto txt_attn_out = ggml_view_3d(ctx,
|
||||
auto attn = Rope::attention(ctx, q, k, v, pe, mask, (1.0f / 128.f)); // [N, n_txt_token + n_img_token, n_head*d_head]
|
||||
attn = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, attn, 0, 2, 1, 3)); // [n_txt_token + n_img_token, N, hidden_size]
|
||||
auto txt_attn_out = ggml_view_3d(ctx->ggml_ctx,
|
||||
attn,
|
||||
attn->ne[0],
|
||||
attn->ne[1],
|
||||
txt->ne[1],
|
||||
attn->nb[1],
|
||||
attn->nb[2],
|
||||
0); // [n_txt_token, N, hidden_size]
|
||||
txt_attn_out = ggml_cont(ctx, ggml_permute(ctx, txt_attn_out, 0, 2, 1, 3)); // [N, n_txt_token, hidden_size]
|
||||
auto img_attn_out = ggml_view_3d(ctx,
|
||||
0); // [n_txt_token, N, hidden_size]
|
||||
txt_attn_out = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, txt_attn_out, 0, 2, 1, 3)); // [N, n_txt_token, hidden_size]
|
||||
auto img_attn_out = ggml_view_3d(ctx->ggml_ctx,
|
||||
attn,
|
||||
attn->ne[0],
|
||||
attn->ne[1],
|
||||
img->ne[1],
|
||||
attn->nb[1],
|
||||
attn->nb[2],
|
||||
attn->nb[2] * txt->ne[1]); // [n_img_token, N, hidden_size]
|
||||
img_attn_out = ggml_cont(ctx, ggml_permute(ctx, img_attn_out, 0, 2, 1, 3)); // [N, n_img_token, hidden_size]
|
||||
attn->nb[2] * txt->ne[1]); // [n_img_token, N, hidden_size]
|
||||
img_attn_out = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, img_attn_out, 0, 2, 1, 3)); // [N, n_img_token, hidden_size]
|
||||
|
||||
img_attn_out = to_out_0->forward(ctx, img_attn_out);
|
||||
txt_attn_out = to_add_out->forward(ctx, txt_attn_out);
|
||||
@@ -194,8 +191,7 @@ namespace Qwen {
|
||||
QwenImageTransformerBlock(int64_t dim,
|
||||
int64_t num_attention_heads,
|
||||
int64_t attention_head_dim,
|
||||
float eps = 1e-6,
|
||||
bool flash_attn = false) {
|
||||
float eps = 1e-6) {
|
||||
// img_mod.0 is nn.SiLU()
|
||||
blocks["img_mod.1"] = std::shared_ptr<GGMLBlock>(new Linear(dim, 6 * dim, true));
|
||||
|
||||
@@ -217,12 +213,10 @@ namespace Qwen {
|
||||
0, // out_context-dim
|
||||
true, // bias
|
||||
true, // out_bias
|
||||
eps,
|
||||
flash_attn));
|
||||
eps));
|
||||
}
|
||||
|
||||
virtual std::pair<ggml_tensor*, ggml_tensor*> forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
virtual std::pair<ggml_tensor*, ggml_tensor*> forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* img,
|
||||
struct ggml_tensor* txt,
|
||||
struct ggml_tensor* t_emb,
|
||||
@@ -244,40 +238,40 @@ namespace Qwen {
|
||||
|
||||
auto attn = std::dynamic_pointer_cast<QwenImageAttention>(blocks["attn"]);
|
||||
|
||||
auto img_mod_params = ggml_silu(ctx, t_emb);
|
||||
auto img_mod_params = ggml_silu(ctx->ggml_ctx, t_emb);
|
||||
img_mod_params = img_mod_1->forward(ctx, img_mod_params);
|
||||
auto img_mod_param_vec = ggml_chunk(ctx, img_mod_params, 6, 0);
|
||||
auto img_mod_param_vec = ggml_ext_chunk(ctx->ggml_ctx, img_mod_params, 6, 0);
|
||||
|
||||
auto txt_mod_params = ggml_silu(ctx, t_emb);
|
||||
auto txt_mod_params = ggml_silu(ctx->ggml_ctx, t_emb);
|
||||
txt_mod_params = txt_mod_1->forward(ctx, txt_mod_params);
|
||||
auto txt_mod_param_vec = ggml_chunk(ctx, txt_mod_params, 6, 0);
|
||||
auto txt_mod_param_vec = ggml_ext_chunk(ctx->ggml_ctx, txt_mod_params, 6, 0);
|
||||
|
||||
auto img_normed = img_norm1->forward(ctx, img);
|
||||
auto img_modulated = Flux::modulate(ctx, img_normed, img_mod_param_vec[0], img_mod_param_vec[1]);
|
||||
auto img_modulated = Flux::modulate(ctx->ggml_ctx, img_normed, img_mod_param_vec[0], img_mod_param_vec[1]);
|
||||
auto img_gate1 = img_mod_param_vec[2];
|
||||
|
||||
auto txt_normed = txt_norm1->forward(ctx, txt);
|
||||
auto txt_modulated = Flux::modulate(ctx, txt_normed, txt_mod_param_vec[0], txt_mod_param_vec[1]);
|
||||
auto txt_modulated = Flux::modulate(ctx->ggml_ctx, txt_normed, txt_mod_param_vec[0], txt_mod_param_vec[1]);
|
||||
auto txt_gate1 = txt_mod_param_vec[2];
|
||||
|
||||
auto [img_attn_output, txt_attn_output] = attn->forward(ctx, backend, img_modulated, txt_modulated, pe);
|
||||
auto [img_attn_output, txt_attn_output] = attn->forward(ctx, img_modulated, txt_modulated, pe);
|
||||
|
||||
img = ggml_add(ctx, img, ggml_mul(ctx, img_attn_output, img_gate1));
|
||||
txt = ggml_add(ctx, txt, ggml_mul(ctx, txt_attn_output, txt_gate1));
|
||||
img = ggml_add(ctx->ggml_ctx, img, ggml_mul(ctx->ggml_ctx, img_attn_output, img_gate1));
|
||||
txt = ggml_add(ctx->ggml_ctx, txt, ggml_mul(ctx->ggml_ctx, txt_attn_output, txt_gate1));
|
||||
|
||||
auto img_normed2 = img_norm2->forward(ctx, img);
|
||||
auto img_modulated2 = Flux::modulate(ctx, img_normed2, img_mod_param_vec[3], img_mod_param_vec[4]);
|
||||
auto img_modulated2 = Flux::modulate(ctx->ggml_ctx, img_normed2, img_mod_param_vec[3], img_mod_param_vec[4]);
|
||||
auto img_gate2 = img_mod_param_vec[5];
|
||||
|
||||
auto txt_normed2 = txt_norm2->forward(ctx, txt);
|
||||
auto txt_modulated2 = Flux::modulate(ctx, txt_normed2, txt_mod_param_vec[3], txt_mod_param_vec[4]);
|
||||
auto txt_modulated2 = Flux::modulate(ctx->ggml_ctx, txt_normed2, txt_mod_param_vec[3], txt_mod_param_vec[4]);
|
||||
auto txt_gate2 = txt_mod_param_vec[5];
|
||||
|
||||
auto img_mlp_out = img_mlp->forward(ctx, img_modulated2);
|
||||
auto txt_mlp_out = txt_mlp->forward(ctx, txt_modulated2);
|
||||
|
||||
img = ggml_add(ctx, img, ggml_mul(ctx, img_mlp_out, img_gate2));
|
||||
txt = ggml_add(ctx, txt, ggml_mul(ctx, txt_mlp_out, txt_gate2));
|
||||
img = ggml_add(ctx->ggml_ctx, img, ggml_mul(ctx->ggml_ctx, img_mlp_out, img_gate2));
|
||||
txt = ggml_add(ctx->ggml_ctx, txt, ggml_mul(ctx->ggml_ctx, txt_mlp_out, txt_gate2));
|
||||
|
||||
return {img, txt};
|
||||
}
|
||||
@@ -294,7 +288,7 @@ namespace Qwen {
|
||||
blocks["linear"] = std::shared_ptr<GGMLBlock>(new Linear(conditioning_embedding_dim, embedding_dim * 2, bias));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* c) {
|
||||
// x: [N, n_token, hidden_size]
|
||||
@@ -304,13 +298,13 @@ namespace Qwen {
|
||||
auto norm = std::dynamic_pointer_cast<LayerNorm>(blocks["norm"]);
|
||||
auto linear = std::dynamic_pointer_cast<Linear>(blocks["linear"]);
|
||||
|
||||
auto emb = linear->forward(ctx, ggml_silu(ctx, c));
|
||||
auto mods = ggml_chunk(ctx, emb, 2, 0);
|
||||
auto emb = linear->forward(ctx, ggml_silu(ctx->ggml_ctx, c));
|
||||
auto mods = ggml_ext_chunk(ctx->ggml_ctx, emb, 2, 0);
|
||||
auto scale = mods[0];
|
||||
auto shift = mods[1];
|
||||
|
||||
x = norm->forward(ctx, x);
|
||||
x = Flux::modulate(ctx, x, shift, scale);
|
||||
x = Flux::modulate(ctx->ggml_ctx, x, shift, scale);
|
||||
|
||||
return x;
|
||||
}
|
||||
@@ -327,7 +321,6 @@ namespace Qwen {
|
||||
float theta = 10000;
|
||||
std::vector<int> axes_dim = {16, 56, 56};
|
||||
int64_t axes_dim_sum = 128;
|
||||
bool flash_attn = false;
|
||||
};
|
||||
|
||||
class QwenImageModel : public GGMLBlock {
|
||||
@@ -349,8 +342,7 @@ namespace Qwen {
|
||||
auto block = std::shared_ptr<GGMLBlock>(new QwenImageTransformerBlock(inner_dim,
|
||||
params.num_attention_heads,
|
||||
params.attention_head_dim,
|
||||
1e-6f,
|
||||
params.flash_attn));
|
||||
1e-6f));
|
||||
blocks["transformer_blocks." + std::to_string(i)] = block;
|
||||
}
|
||||
|
||||
@@ -421,8 +413,7 @@ namespace Qwen {
|
||||
return x;
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward_orig(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward_orig(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* timestep,
|
||||
struct ggml_tensor* context,
|
||||
@@ -442,7 +433,7 @@ namespace Qwen {
|
||||
for (int i = 0; i < params.num_layers; i++) {
|
||||
auto block = std::dynamic_pointer_cast<QwenImageTransformerBlock>(blocks["transformer_blocks." + std::to_string(i)]);
|
||||
|
||||
auto result = block->forward(ctx, backend, img, txt, t_emb, pe);
|
||||
auto result = block->forward(ctx, img, txt, t_emb, pe);
|
||||
img = result.first;
|
||||
txt = result.second;
|
||||
}
|
||||
@@ -453,8 +444,7 @@ namespace Qwen {
|
||||
return img;
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* timestep,
|
||||
struct ggml_tensor* context,
|
||||
@@ -472,32 +462,32 @@ namespace Qwen {
|
||||
int64_t C = x->ne[2];
|
||||
int64_t N = x->ne[3];
|
||||
|
||||
auto img = process_img(ctx, x);
|
||||
auto img = process_img(ctx->ggml_ctx, x);
|
||||
uint64_t img_tokens = img->ne[1];
|
||||
|
||||
if (ref_latents.size() > 0) {
|
||||
for (ggml_tensor* ref : ref_latents) {
|
||||
ref = process_img(ctx, ref);
|
||||
img = ggml_concat(ctx, img, ref, 1);
|
||||
ref = process_img(ctx->ggml_ctx, ref);
|
||||
img = ggml_concat(ctx->ggml_ctx, img, ref, 1);
|
||||
}
|
||||
}
|
||||
|
||||
int64_t h_len = ((H + (params.patch_size / 2)) / params.patch_size);
|
||||
int64_t w_len = ((W + (params.patch_size / 2)) / params.patch_size);
|
||||
|
||||
auto out = forward_orig(ctx, backend, img, timestep, context, pe); // [N, h_len*w_len, ph*pw*C]
|
||||
auto out = forward_orig(ctx, img, timestep, context, pe); // [N, h_len*w_len, ph*pw*C]
|
||||
|
||||
if (out->ne[1] > img_tokens) {
|
||||
out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); // [num_tokens, N, C * patch_size * patch_size]
|
||||
out = ggml_view_3d(ctx, out, out->ne[0], out->ne[1], img_tokens, out->nb[1], out->nb[2], 0);
|
||||
out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); // [N, h*w, C * patch_size * patch_size]
|
||||
out = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, out, 0, 2, 1, 3)); // [num_tokens, N, C * patch_size * patch_size]
|
||||
out = ggml_view_3d(ctx->ggml_ctx, out, out->ne[0], out->ne[1], img_tokens, out->nb[1], out->nb[2], 0);
|
||||
out = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, out, 0, 2, 1, 3)); // [N, h*w, C * patch_size * patch_size]
|
||||
}
|
||||
|
||||
out = unpatchify(ctx, out, h_len, w_len); // [N, C, H + pad_h, W + pad_w]
|
||||
out = unpatchify(ctx->ggml_ctx, out, h_len, w_len); // [N, C, H + pad_h, W + pad_w]
|
||||
|
||||
// slice
|
||||
out = ggml_slice(ctx, out, 1, 0, H); // [N, C, H, W + pad_w]
|
||||
out = ggml_slice(ctx, out, 0, 0, W); // [N, C, H, W]
|
||||
out = ggml_ext_slice(ctx->ggml_ctx, out, 1, 0, H); // [N, C, H, W + pad_w]
|
||||
out = ggml_ext_slice(ctx->ggml_ctx, out, 0, 0, W); // [N, C, H, W]
|
||||
|
||||
return out;
|
||||
}
|
||||
@@ -512,14 +502,12 @@ namespace Qwen {
|
||||
|
||||
QwenImageRunner(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {},
|
||||
const std::string prefix = "",
|
||||
SDVersion version = VERSION_QWEN_IMAGE,
|
||||
bool flash_attn = false)
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
const std::string prefix = "",
|
||||
SDVersion version = VERSION_QWEN_IMAGE)
|
||||
: GGMLRunner(backend, offload_params_to_cpu) {
|
||||
qwen_image_params.flash_attn = flash_attn;
|
||||
qwen_image_params.num_layers = 0;
|
||||
for (auto pair : tensor_types) {
|
||||
for (auto pair : tensor_storage_map) {
|
||||
std::string tensor_name = pair.first;
|
||||
if (tensor_name.find(prefix) == std::string::npos)
|
||||
continue;
|
||||
@@ -538,7 +526,7 @@ namespace Qwen {
|
||||
}
|
||||
LOG_INFO("qwen_image_params.num_layers: %ld", qwen_image_params.num_layers);
|
||||
qwen_image = QwenImageModel(qwen_image_params);
|
||||
qwen_image.init(params_ctx, tensor_types, prefix);
|
||||
qwen_image.init(params_ctx, tensor_storage_map, prefix);
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -582,8 +570,9 @@ namespace Qwen {
|
||||
// pe->data = nullptr;
|
||||
set_backend_tensor_data(pe, pe_vec.data());
|
||||
|
||||
struct ggml_tensor* out = qwen_image.forward(compute_ctx,
|
||||
runtime_backend,
|
||||
auto runner_ctx = get_context();
|
||||
|
||||
struct ggml_tensor* out = qwen_image.forward(&runner_ctx,
|
||||
x,
|
||||
timesteps,
|
||||
context,
|
||||
@@ -649,7 +638,7 @@ namespace Qwen {
|
||||
|
||||
static void load_from_file_and_test(const std::string& file_path) {
|
||||
// cuda q8: pass
|
||||
// cuda q8 fa: nan
|
||||
// cuda q8 fa: pass
|
||||
// ggml_backend_t backend = ggml_backend_cuda_init(0);
|
||||
ggml_backend_t backend = ggml_backend_cpu_init();
|
||||
ggml_type model_data_type = GGML_TYPE_Q8_0;
|
||||
@@ -660,20 +649,18 @@ namespace Qwen {
|
||||
return;
|
||||
}
|
||||
|
||||
auto tensor_types = model_loader.tensor_storages_types;
|
||||
for (auto& item : tensor_types) {
|
||||
// LOG_DEBUG("%s %u", item.first.c_str(), item.second);
|
||||
if (ends_with(item.first, "weight")) {
|
||||
item.second = model_data_type;
|
||||
auto& tensor_storage_map = model_loader.get_tensor_storage_map();
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (ends_with(name, "weight")) {
|
||||
tensor_storage.expected_type = model_data_type;
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<QwenImageRunner> qwen_image = std::make_shared<QwenImageRunner>(backend,
|
||||
false,
|
||||
tensor_types,
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
VERSION_QWEN_IMAGE,
|
||||
true);
|
||||
VERSION_QWEN_IMAGE);
|
||||
|
||||
qwen_image->alloc_params_buffer();
|
||||
std::map<std::string, ggml_tensor*> tensors;
|
||||
|
||||
+84
-93
@@ -349,15 +349,15 @@ namespace Qwen {
|
||||
blocks["down_proj"] = std::shared_ptr<GGMLBlock>(new Linear(intermediate_size, hidden_size, bias));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [N, n_token, hidden_size]
|
||||
auto gate_proj = std::dynamic_pointer_cast<Linear>(blocks["gate_proj"]);
|
||||
auto up_proj = std::dynamic_pointer_cast<Linear>(blocks["up_proj"]);
|
||||
auto down_proj = std::dynamic_pointer_cast<Linear>(blocks["down_proj"]);
|
||||
|
||||
auto h = gate_proj->forward(ctx, x);
|
||||
h = ggml_silu_inplace(ctx, h);
|
||||
h = ggml_mul_inplace(ctx, h, up_proj->forward(ctx, x));
|
||||
h = ggml_silu_inplace(ctx->ggml_ctx, h);
|
||||
h = ggml_mul_inplace(ctx->ggml_ctx, h, up_proj->forward(ctx, x));
|
||||
h = down_proj->forward(ctx, h);
|
||||
return h;
|
||||
}
|
||||
@@ -409,10 +409,10 @@ namespace Qwen {
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [N*grid_t*grid_h*grid_w, in_channels, temporal_patch_size*patch_size*patch_size]
|
||||
// return: [N*grid_t*grid_h*grid_w, embed_dim]
|
||||
x = ggml_reshape_4d(ctx,
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx,
|
||||
x,
|
||||
patch_size,
|
||||
patch_size,
|
||||
@@ -423,22 +423,22 @@ namespace Qwen {
|
||||
auto proj_0 = std::dynamic_pointer_cast<Conv2d>(blocks["proj.0"]);
|
||||
auto proj_1 = std::dynamic_pointer_cast<Conv2d>(blocks["proj.1"]);
|
||||
|
||||
auto x0 = ggml_slice(ctx, x, 2, 0, 1);
|
||||
x0 = ggml_reshape_4d(ctx, x0, x0->ne[0], x0->ne[1], in_channels, x0->ne[3] / in_channels);
|
||||
auto x0 = ggml_ext_slice(ctx->ggml_ctx, x, 2, 0, 1);
|
||||
x0 = ggml_reshape_4d(ctx->ggml_ctx, x0, x0->ne[0], x0->ne[1], in_channels, x0->ne[3] / in_channels);
|
||||
x0 = proj_0->forward(ctx, x0);
|
||||
|
||||
auto x1 = ggml_slice(ctx, x, 2, 1, 2);
|
||||
x1 = ggml_reshape_4d(ctx, x1, x1->ne[0], x1->ne[1], in_channels, x1->ne[3] / in_channels);
|
||||
auto x1 = ggml_ext_slice(ctx->ggml_ctx, x, 2, 1, 2);
|
||||
x1 = ggml_reshape_4d(ctx->ggml_ctx, x1, x1->ne[0], x1->ne[1], in_channels, x1->ne[3] / in_channels);
|
||||
x1 = proj_1->forward(ctx, x1);
|
||||
|
||||
x = ggml_add(ctx, x0, x1);
|
||||
x = ggml_add(ctx->ggml_ctx, x0, x1);
|
||||
} else {
|
||||
auto proj = std::dynamic_pointer_cast<Conv3d>(blocks["proj"]);
|
||||
|
||||
x = proj->forward(ctx, x);
|
||||
}
|
||||
|
||||
x = ggml_reshape_2d(ctx, x, embed_dim, ggml_nelements(x) / embed_dim);
|
||||
x = ggml_reshape_2d(ctx->ggml_ctx, x, embed_dim, ggml_nelements(x) / embed_dim);
|
||||
return x;
|
||||
}
|
||||
};
|
||||
@@ -458,15 +458,15 @@ namespace Qwen {
|
||||
blocks["mlp.2"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, dim));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
auto ln_q = std::dynamic_pointer_cast<RMSNorm>(blocks["ln_q"]);
|
||||
auto mlp_0 = std::dynamic_pointer_cast<Linear>(blocks["mlp.0"]);
|
||||
auto mlp_2 = std::dynamic_pointer_cast<Linear>(blocks["mlp.2"]);
|
||||
|
||||
x = ln_q->forward(ctx, x);
|
||||
x = ggml_reshape_2d(ctx, x, hidden_size, ggml_nelements(x) / hidden_size);
|
||||
x = ggml_reshape_2d(ctx->ggml_ctx, x, hidden_size, ggml_nelements(x) / hidden_size);
|
||||
x = mlp_0->forward(ctx, x);
|
||||
x = ggml_gelu(ctx, x);
|
||||
x = ggml_gelu(ctx->ggml_ctx, x);
|
||||
x = mlp_2->forward(ctx, x);
|
||||
return x;
|
||||
}
|
||||
@@ -495,8 +495,7 @@ namespace Qwen {
|
||||
blocks["proj"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, hidden_size));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* pe,
|
||||
struct ggml_tensor* mask = nullptr) {
|
||||
@@ -519,14 +518,14 @@ namespace Qwen {
|
||||
} else {
|
||||
auto qkv_proj = std::dynamic_pointer_cast<Linear>(blocks["qkv"]);
|
||||
auto qkv = qkv_proj->forward(ctx, x);
|
||||
qkv_vec = split_qkv(ctx, qkv);
|
||||
qkv_vec = split_qkv(ctx->ggml_ctx, qkv);
|
||||
}
|
||||
|
||||
auto q = ggml_reshape_4d(ctx, qkv_vec[0], head_dim, num_heads, qkv_vec[0]->ne[1], qkv_vec[0]->ne[2]); // [N, n_token, n_head, d_head]
|
||||
auto k = ggml_reshape_4d(ctx, qkv_vec[1], head_dim, num_heads, qkv_vec[1]->ne[1], qkv_vec[1]->ne[2]); // [N, n_token, n_head, d_head]
|
||||
auto v = ggml_reshape_4d(ctx, qkv_vec[2], head_dim, num_heads, qkv_vec[2]->ne[1], qkv_vec[2]->ne[2]); // [N, n_token, n_head, d_head]
|
||||
auto q = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[0], head_dim, num_heads, qkv_vec[0]->ne[1], qkv_vec[0]->ne[2]); // [N, n_token, n_head, d_head]
|
||||
auto k = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[1], head_dim, num_heads, qkv_vec[1]->ne[1], qkv_vec[1]->ne[2]); // [N, n_token, n_head, d_head]
|
||||
auto v = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[2], head_dim, num_heads, qkv_vec[2]->ne[1], qkv_vec[2]->ne[2]); // [N, n_token, n_head, d_head]
|
||||
|
||||
x = Rope::attention(ctx, backend, q, k, v, pe, mask, false, 1.f, false); // [N, n_token, hidden_size]
|
||||
x = Rope::attention(ctx, q, k, v, pe, mask, 1.f, false); // [N, n_token, hidden_size]
|
||||
|
||||
x = proj->forward(ctx, x); // [N, n_token, hidden_size]
|
||||
return x;
|
||||
@@ -546,8 +545,7 @@ namespace Qwen {
|
||||
blocks["norm2"] = std::shared_ptr<GGMLBlock>(new RMSNorm(hidden_size, eps));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* pe,
|
||||
struct ggml_tensor* mask = nullptr) {
|
||||
@@ -559,13 +557,13 @@ namespace Qwen {
|
||||
|
||||
auto residual = x;
|
||||
x = norm1->forward(ctx, x);
|
||||
x = attn->forward(ctx, backend, x, pe, mask);
|
||||
x = ggml_add_inplace(ctx, x, residual);
|
||||
x = attn->forward(ctx, x, pe, mask);
|
||||
x = ggml_add_inplace(ctx->ggml_ctx, x, residual);
|
||||
|
||||
residual = x;
|
||||
x = norm2->forward(ctx, x);
|
||||
x = mlp->forward(ctx, x);
|
||||
x = ggml_add_inplace(ctx, x, residual);
|
||||
x = ggml_add_inplace(ctx->ggml_ctx, x, residual);
|
||||
|
||||
return x;
|
||||
}
|
||||
@@ -607,8 +605,7 @@ namespace Qwen {
|
||||
blocks["merger"] = std::shared_ptr<GGMLBlock>(new Qwen2_5_VLPatchMerger(out_hidden_size, hidden_size, spatial_merge_size));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* pixel_values,
|
||||
struct ggml_tensor* pe,
|
||||
struct ggml_tensor* window_index,
|
||||
@@ -623,9 +620,9 @@ namespace Qwen {
|
||||
|
||||
auto x = patch_embed->forward(ctx, pixel_values);
|
||||
|
||||
x = ggml_reshape_4d(ctx, x, x->ne[0] * spatial_merge_size * spatial_merge_size, x->ne[1] / spatial_merge_size / spatial_merge_size, x->ne[2], x->ne[3]);
|
||||
x = ggml_get_rows(ctx, x, window_index);
|
||||
x = ggml_reshape_4d(ctx, x, x->ne[0] / spatial_merge_size / spatial_merge_size, x->ne[1] * spatial_merge_size * spatial_merge_size, x->ne[2], x->ne[3]);
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx, x, x->ne[0] * spatial_merge_size * spatial_merge_size, x->ne[1] / spatial_merge_size / spatial_merge_size, x->ne[2], x->ne[3]);
|
||||
x = ggml_get_rows(ctx->ggml_ctx, x, window_index);
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx, x, x->ne[0] / spatial_merge_size / spatial_merge_size, x->ne[1] * spatial_merge_size * spatial_merge_size, x->ne[2], x->ne[3]);
|
||||
|
||||
for (int i = 0; i < num_layers; i++) {
|
||||
auto block = std::dynamic_pointer_cast<Qwen2_5_VLVisionBlock>(blocks["blocks." + std::to_string(i)]);
|
||||
@@ -634,12 +631,12 @@ namespace Qwen {
|
||||
if (fullatt_block_indexes.find(i) != fullatt_block_indexes.end()) {
|
||||
mask = nullptr;
|
||||
}
|
||||
x = block->forward(ctx, backend, x, pe, mask);
|
||||
x = block->forward(ctx, x, pe, mask);
|
||||
}
|
||||
|
||||
x = merger->forward(ctx, x);
|
||||
|
||||
x = ggml_get_rows(ctx, x, window_inverse_index);
|
||||
x = ggml_get_rows(ctx->ggml_ctx, x, window_inverse_index);
|
||||
|
||||
return x;
|
||||
}
|
||||
@@ -664,8 +661,7 @@ namespace Qwen {
|
||||
blocks["o_proj"] = std::shared_ptr<GGMLBlock>(new Linear(num_heads * head_dim, hidden_size, false));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* input_pos) {
|
||||
// x: [N, n_token, hidden_size]
|
||||
@@ -680,21 +676,21 @@ namespace Qwen {
|
||||
auto k = k_proj->forward(ctx, x); // [N, n_token, num_kv_heads*head_dim]
|
||||
auto v = v_proj->forward(ctx, x); // [N, n_token, num_kv_heads*head_dim]
|
||||
|
||||
q = ggml_reshape_4d(ctx, q, head_dim, num_heads, n_token, N); // [N, n_token, num_heads, head_dim]
|
||||
k = ggml_reshape_4d(ctx, k, head_dim, num_kv_heads, n_token, N); // [N, n_token, num_kv_heads, head_dim]
|
||||
v = ggml_reshape_4d(ctx, v, head_dim, num_kv_heads, n_token, N); // [N, n_token, num_kv_heads, head_dim]
|
||||
q = ggml_reshape_4d(ctx->ggml_ctx, q, head_dim, num_heads, n_token, N); // [N, n_token, num_heads, head_dim]
|
||||
k = ggml_reshape_4d(ctx->ggml_ctx, k, head_dim, num_kv_heads, n_token, N); // [N, n_token, num_kv_heads, head_dim]
|
||||
v = ggml_reshape_4d(ctx->ggml_ctx, v, head_dim, num_kv_heads, n_token, N); // [N, n_token, num_kv_heads, head_dim]
|
||||
|
||||
int sections[4] = {16, 24, 24, 0};
|
||||
q = ggml_rope_multi(ctx, q, input_pos, nullptr, head_dim, sections, GGML_ROPE_TYPE_MROPE, 128000, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
|
||||
k = ggml_rope_multi(ctx, k, input_pos, nullptr, head_dim, sections, GGML_ROPE_TYPE_MROPE, 128000, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
|
||||
q = ggml_rope_multi(ctx->ggml_ctx, q, input_pos, nullptr, head_dim, sections, GGML_ROPE_TYPE_MROPE, 128000, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
|
||||
k = ggml_rope_multi(ctx->ggml_ctx, k, input_pos, nullptr, head_dim, sections, GGML_ROPE_TYPE_MROPE, 128000, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
|
||||
|
||||
q = ggml_cont(ctx, ggml_torch_permute(ctx, q, 0, 2, 1, 3)); // [N, num_heads, n_token, head_dim]
|
||||
q = ggml_reshape_3d(ctx, q, q->ne[0], q->ne[1], q->ne[2] * q->ne[3]); // [N*num_heads, n_token, head_dim]
|
||||
q = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, q, 0, 2, 1, 3)); // [N, num_heads, n_token, head_dim]
|
||||
q = ggml_reshape_3d(ctx->ggml_ctx, q, q->ne[0], q->ne[1], q->ne[2] * q->ne[3]); // [N*num_heads, n_token, head_dim]
|
||||
|
||||
k = ggml_cont(ctx, ggml_torch_permute(ctx, k, 0, 2, 1, 3)); // [N, num_kv_heads, n_token, head_dim]
|
||||
k = ggml_reshape_3d(ctx, k, k->ne[0], k->ne[1], k->ne[2] * k->ne[3]); // [N*num_kv_heads, n_token, head_dim]
|
||||
k = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, k, 0, 2, 1, 3)); // [N, num_kv_heads, n_token, head_dim]
|
||||
k = ggml_reshape_3d(ctx->ggml_ctx, k, k->ne[0], k->ne[1], k->ne[2] * k->ne[3]); // [N*num_kv_heads, n_token, head_dim]
|
||||
|
||||
x = ggml_nn_attention_ext(ctx, backend, q, k, v, num_heads, nullptr, true, true, false); // [N, n_token, hidden_size]
|
||||
x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, num_heads, nullptr, true, true, false); // [N, n_token, hidden_size]
|
||||
|
||||
x = out_proj->forward(ctx, x); // [N, n_token, hidden_size]
|
||||
return x;
|
||||
@@ -714,8 +710,7 @@ namespace Qwen {
|
||||
blocks["post_attention_layernorm"] = std::shared_ptr<GGMLBlock>(new RMSNorm(hidden_size, eps));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* input_pos) {
|
||||
// x: [N, n_token, hidden_size]
|
||||
@@ -726,13 +721,13 @@ namespace Qwen {
|
||||
|
||||
auto residual = x;
|
||||
x = input_layernorm->forward(ctx, x);
|
||||
x = self_attn->forward(ctx, backend, x, input_pos);
|
||||
x = ggml_add_inplace(ctx, x, residual);
|
||||
x = self_attn->forward(ctx, x, input_pos);
|
||||
x = ggml_add_inplace(ctx->ggml_ctx, x, residual);
|
||||
|
||||
residual = x;
|
||||
x = post_attention_layernorm->forward(ctx, x);
|
||||
x = mlp->forward(ctx, x);
|
||||
x = ggml_add_inplace(ctx, x, residual);
|
||||
x = ggml_add_inplace(ctx->ggml_ctx, x, residual);
|
||||
|
||||
return x;
|
||||
}
|
||||
@@ -761,8 +756,7 @@ namespace Qwen {
|
||||
blocks["norm"] = std::shared_ptr<GGMLBlock>(new RMSNorm(hidden_size, eps));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* input_ids,
|
||||
struct ggml_tensor* input_pos,
|
||||
std::vector<std::pair<int, ggml_tensor*>> image_embeds) {
|
||||
@@ -777,7 +771,7 @@ namespace Qwen {
|
||||
if (image_embeds.size() > 0) {
|
||||
GGML_ASSERT(x->ne[2] == 1); // N == 1
|
||||
|
||||
auto raw_x = ggml_cast(ctx, x, image_embeds[0].second->type);
|
||||
auto raw_x = ggml_cast(ctx->ggml_ctx, x, image_embeds[0].second->type);
|
||||
int64_t txt_token_start = 0;
|
||||
int64_t txt_token_end = 0;
|
||||
|
||||
@@ -791,23 +785,23 @@ namespace Qwen {
|
||||
}
|
||||
txt_token_end = image_embeds[i].first;
|
||||
|
||||
auto txt_embed = ggml_slice(ctx, raw_x, 1, txt_token_start, txt_token_end);
|
||||
auto txt_embed = ggml_ext_slice(ctx->ggml_ctx, raw_x, 1, txt_token_start, txt_token_end);
|
||||
if (input_embed == nullptr) {
|
||||
input_embed = txt_embed;
|
||||
} else {
|
||||
input_embed = ggml_concat(ctx, input_embed, txt_embed, 1);
|
||||
input_embed = ggml_concat(ctx->ggml_ctx, input_embed, txt_embed, 1);
|
||||
}
|
||||
|
||||
auto image_embed = image_embeds[i].second;
|
||||
input_embed = ggml_concat(ctx, input_embed, image_embed, 1);
|
||||
input_embed = ggml_concat(ctx->ggml_ctx, input_embed, image_embed, 1);
|
||||
}
|
||||
|
||||
txt_token_start = image_embeds[image_embeds.size() - 1].first + image_embeds[image_embeds.size() - 1].second->ne[1];
|
||||
txt_token_end = raw_x->ne[1];
|
||||
|
||||
auto final_txt_embed = ggml_slice(ctx, raw_x, 1, txt_token_start, txt_token_end);
|
||||
auto final_txt_embed = ggml_ext_slice(ctx->ggml_ctx, raw_x, 1, txt_token_start, txt_token_end);
|
||||
|
||||
input_embed = ggml_concat(ctx, input_embed, final_txt_embed, 1);
|
||||
input_embed = ggml_concat(ctx->ggml_ctx, input_embed, final_txt_embed, 1);
|
||||
GGML_ASSERT(raw_x->ne[1] == input_embed->ne[1]);
|
||||
|
||||
x = input_embed;
|
||||
@@ -816,7 +810,7 @@ namespace Qwen {
|
||||
for (int i = 0; i < num_layers; i++) {
|
||||
auto block = std::dynamic_pointer_cast<Qwen2_5_VLBlock>(blocks["layers." + std::to_string(i)]);
|
||||
|
||||
x = block->forward(ctx, backend, x, input_pos);
|
||||
x = block->forward(ctx, x, input_pos);
|
||||
}
|
||||
|
||||
x = norm->forward(ctx, x);
|
||||
@@ -880,20 +874,18 @@ namespace Qwen {
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* input_ids,
|
||||
struct ggml_tensor* input_pos,
|
||||
std::vector<std::pair<int, ggml_tensor*>> image_embeds) {
|
||||
// input_ids: [N, n_token]
|
||||
auto model = std::dynamic_pointer_cast<Qwen2_5_VLTextModel>(blocks["model"]);
|
||||
|
||||
auto x = model->forward(ctx, backend, input_ids, input_pos, image_embeds);
|
||||
auto x = model->forward(ctx, input_ids, input_pos, image_embeds);
|
||||
return x;
|
||||
}
|
||||
|
||||
struct ggml_tensor* vision_forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* vision_forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* pixel_values,
|
||||
struct ggml_tensor* pe,
|
||||
struct ggml_tensor* window_index,
|
||||
@@ -901,7 +893,7 @@ namespace Qwen {
|
||||
struct ggml_tensor* window_mask) {
|
||||
GGML_ASSERT(enable_vision);
|
||||
auto vision_model = std::dynamic_pointer_cast<Qwen2_5_VLVisionModel>(blocks["visual"]);
|
||||
return vision_model->forward(ctx, backend, pixel_values, pe, window_index, window_inverse_index, window_mask);
|
||||
return vision_model->forward(ctx, pixel_values, pe, window_index, window_inverse_index, window_mask);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -918,13 +910,13 @@ namespace Qwen {
|
||||
|
||||
Qwen2_5_VLRunner(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types,
|
||||
const String2TensorStorage& tensor_storage_map,
|
||||
const std::string prefix,
|
||||
bool enable_vision_ = false)
|
||||
: GGMLRunner(backend, offload_params_to_cpu), enable_vision(enable_vision_) {
|
||||
bool have_vision_weight = false;
|
||||
bool llama_cpp_style = false;
|
||||
for (auto pair : tensor_types) {
|
||||
for (auto pair : tensor_storage_map) {
|
||||
std::string tensor_name = pair.first;
|
||||
if (tensor_name.find(prefix) == std::string::npos)
|
||||
continue;
|
||||
@@ -948,7 +940,7 @@ namespace Qwen {
|
||||
}
|
||||
}
|
||||
model = Qwen2_5_VL(params, enable_vision, llama_cpp_style);
|
||||
model.init(params_ctx, tensor_types, prefix);
|
||||
model.init(params_ctx, tensor_storage_map, prefix);
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -959,23 +951,21 @@ namespace Qwen {
|
||||
model.get_param_tensors(tensors, prefix);
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* input_ids,
|
||||
struct ggml_tensor* input_pos,
|
||||
std::vector<std::pair<int, ggml_tensor*>> image_embeds) {
|
||||
auto hidden_states = model.forward(ctx, backend, input_ids, input_pos, image_embeds); // [N, n_token, hidden_size]
|
||||
auto hidden_states = model.forward(ctx, input_ids, input_pos, image_embeds); // [N, n_token, hidden_size]
|
||||
return hidden_states;
|
||||
}
|
||||
|
||||
struct ggml_tensor* vision_forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* vision_forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* pixel_values,
|
||||
struct ggml_tensor* input_pos,
|
||||
struct ggml_tensor* window_index,
|
||||
struct ggml_tensor* window_inverse_index,
|
||||
struct ggml_tensor* window_mask) {
|
||||
auto hidden_states = model.vision_forward(ctx, backend, pixel_values, input_pos, window_index, window_inverse_index, window_mask);
|
||||
auto hidden_states = model.vision_forward(ctx, pixel_values, input_pos, window_index, window_inverse_index, window_mask);
|
||||
return hidden_states;
|
||||
}
|
||||
|
||||
@@ -1002,7 +992,9 @@ namespace Qwen {
|
||||
n_tokens * 4);
|
||||
set_backend_tensor_data(input_pos, input_pos_vec.data());
|
||||
|
||||
struct ggml_tensor* hidden_states = forward(compute_ctx, runtime_backend, input_ids, input_pos, image_embeds);
|
||||
auto runner_ctx = get_context();
|
||||
|
||||
struct ggml_tensor* hidden_states = forward(&runner_ctx, input_ids, input_pos, image_embeds);
|
||||
|
||||
ggml_build_forward_expand(gf, hidden_states);
|
||||
|
||||
@@ -1042,16 +1034,16 @@ namespace Qwen {
|
||||
int64_t pw = params.vision.patch_size;
|
||||
|
||||
image = ggml_reshape_4d(ctx, image, pw, mw, (W / mw / pw), H * C); // [C*H, (W/mw/pw), mw, pw]
|
||||
image = ggml_cont(ctx, ggml_torch_permute(ctx, image, 0, 2, 3, 1)); // [mw, C*H, (W/mw/pw), pw]
|
||||
image = ggml_cont(ctx, ggml_ext_torch_permute(ctx, image, 0, 2, 3, 1)); // [mw, C*H, (W/mw/pw), pw]
|
||||
image = ggml_reshape_4d(ctx, image, pw * (W / mw / pw), H, C, mw); // [mw, C, H, (W/mw/pw)*pw]
|
||||
image = ggml_cont(ctx, ggml_torch_permute(ctx, image, 0, 2, 3, 1)); // [H, mw, C, (W/mw/pw)*pw]
|
||||
image = ggml_cont(ctx, ggml_ext_torch_permute(ctx, image, 0, 2, 3, 1)); // [H, mw, C, (W/mw/pw)*pw]
|
||||
image = ggml_reshape_4d(ctx, image, pw, (W / mw / pw) * C * mw, ph, mh * (H / mh / ph)); // [(H/mh/ph)*mh, ph, mw*C*(W/mw/pw), pw]
|
||||
image = ggml_cont(ctx, ggml_torch_permute(ctx, image, 0, 2, 1, 3)); // [(H/mh/ph)*mh, mw*C*(W/mw/pw), ph, pw]
|
||||
image = ggml_cont(ctx, ggml_ext_torch_permute(ctx, image, 0, 2, 1, 3)); // [(H/mh/ph)*mh, mw*C*(W/mw/pw), ph, pw]
|
||||
image = ggml_reshape_4d(ctx, image, pw * ph, (W / mw / pw), C, mw * mh * (H / mh / ph)); // [(H/mh/ph)*mh*mw, C, (W/mw/pw), ph*pw]
|
||||
image = ggml_concat(ctx, image, image, 0); // [(H/mh/ph)*mh*mw, C, (W/mw/pw), pt*ph*pw]
|
||||
image = ggml_cont(ctx, ggml_torch_permute(ctx, image, 0, 2, 1, 3)); // [(H/mh/ph)*mh*mw, (W/mw/pw), C, pt*ph*pw]
|
||||
image = ggml_cont(ctx, ggml_ext_torch_permute(ctx, image, 0, 2, 1, 3)); // [(H/mh/ph)*mh*mw, (W/mw/pw), C, pt*ph*pw]
|
||||
image = ggml_reshape_4d(ctx, image, pw * ph * pt * C, (W / mw / pw), mw * mh, (H / mh / ph)); // [(H/mh/ph), mh*mw, (W/mw/pw), C*pt*ph*pw]
|
||||
image = ggml_cont(ctx, ggml_torch_permute(ctx, image, 0, 2, 1, 3)); // [(H/mh/ph), (W/mw/pw), mh*mw, C*pt*ph*pw]
|
||||
image = ggml_cont(ctx, ggml_ext_torch_permute(ctx, image, 0, 2, 1, 3)); // [(H/mh/ph), (W/mw/pw), mh*mw, C*pt*ph*pw]
|
||||
image = ggml_reshape_2d(ctx, image, pw * ph * pt * C, mw * mh * (W / mw / pw) * (H / mh / ph)); // [(H/mh/ph)*(W/mw/pw)*mh*mw, C*pt*ph*pw]
|
||||
return image;
|
||||
}
|
||||
@@ -1167,8 +1159,8 @@ namespace Qwen {
|
||||
// pe->data = nullptr;
|
||||
set_backend_tensor_data(pe, pe_vec.data());
|
||||
|
||||
struct ggml_tensor* hidden_states = vision_forward(compute_ctx,
|
||||
runtime_backend,
|
||||
auto runnter_ctx = get_context();
|
||||
struct ggml_tensor* hidden_states = vision_forward(&runnter_ctx,
|
||||
pixel_values,
|
||||
pe,
|
||||
window_index,
|
||||
@@ -1196,10 +1188,10 @@ namespace Qwen {
|
||||
|
||||
Qwen2_5_VLEmbedder(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {},
|
||||
const std::string prefix = "",
|
||||
bool enable_vision = false)
|
||||
: model(backend, offload_params_to_cpu, tensor_types, prefix, enable_vision) {
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
const std::string prefix = "",
|
||||
bool enable_vision = false)
|
||||
: model(backend, offload_params_to_cpu, tensor_storage_map, prefix, enable_vision) {
|
||||
}
|
||||
|
||||
void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
|
||||
@@ -1319,7 +1311,7 @@ namespace Qwen {
|
||||
print_ggml_tensor(out, false, "out");
|
||||
|
||||
// auto ref_out = load_tensor_from_file(work_ctx, "qwen2vl.bin");
|
||||
// ggml_tensor_diff(ref_out, out, 0.01f);
|
||||
// ggml_ext_tensor_diff(ref_out, out, 0.01f);
|
||||
|
||||
LOG_DEBUG("qwen2vl test done in %dms", t1 - t0);
|
||||
} else {
|
||||
@@ -1355,17 +1347,16 @@ namespace Qwen {
|
||||
return;
|
||||
}
|
||||
|
||||
auto tensor_types = model_loader.tensor_storages_types;
|
||||
for (auto& item : tensor_types) {
|
||||
// LOG_DEBUG("%s %u", item.first.c_str(), item.second);
|
||||
if (ends_with(item.first, "weight")) {
|
||||
item.second = model_data_type;
|
||||
auto& tensor_storage_map = model_loader.get_tensor_storage_map();
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (ends_with(name, "weight")) {
|
||||
tensor_storage.expected_type = model_data_type;
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<Qwen2_5_VLEmbedder> qwenvl = std::make_shared<Qwen2_5_VLEmbedder>(backend,
|
||||
false,
|
||||
tensor_types,
|
||||
tensor_storage_map,
|
||||
"qwen2vl",
|
||||
true);
|
||||
|
||||
|
||||
@@ -360,8 +360,8 @@ namespace Rope {
|
||||
x = ggml_reshape_4d(ctx, x, 2, d_head / 2, L, n_head * N); // [N * n_head, L, d_head/2, 2]
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 3, 0, 1, 2)); // [2, N * n_head, L, d_head/2]
|
||||
} else {
|
||||
x = ggml_reshape_4d(ctx, x, d_head / 2, 2, L, n_head * N); // [N * n_head, L, 2, d_head/2]
|
||||
x = ggml_cont(ctx, ggml_torch_permute(ctx, x, 0, 2, 3, 1)); // [2, N * n_head, L, d_head/2]
|
||||
x = ggml_reshape_4d(ctx, x, d_head / 2, 2, L, n_head * N); // [N * n_head, L, 2, d_head/2]
|
||||
x = ggml_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 3, 1)); // [2, N * n_head, L, d_head/2]
|
||||
}
|
||||
|
||||
int64_t offset = x->nb[2] * x->ne[2];
|
||||
@@ -386,23 +386,21 @@ namespace Rope {
|
||||
return x_out;
|
||||
}
|
||||
|
||||
__STATIC_INLINE__ struct ggml_tensor* attention(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
__STATIC_INLINE__ struct ggml_tensor* attention(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* q,
|
||||
struct ggml_tensor* k,
|
||||
struct ggml_tensor* v,
|
||||
struct ggml_tensor* pe,
|
||||
struct ggml_tensor* mask,
|
||||
bool flash_attn,
|
||||
float kv_scale = 1.0f,
|
||||
bool rope_interleaved = true) {
|
||||
// q,k,v: [N, L, n_head, d_head]
|
||||
// pe: [L, d_head/2, 2, 2]
|
||||
// return: [N, L, n_head*d_head]
|
||||
q = apply_rope(ctx, q, pe, rope_interleaved); // [N*n_head, L, d_head]
|
||||
k = apply_rope(ctx, k, pe, rope_interleaved); // [N*n_head, L, d_head]
|
||||
q = apply_rope(ctx->ggml_ctx, q, pe, rope_interleaved); // [N*n_head, L, d_head]
|
||||
k = apply_rope(ctx->ggml_ctx, k, pe, rope_interleaved); // [N*n_head, L, d_head]
|
||||
|
||||
auto x = ggml_nn_attention_ext(ctx, backend, q, k, v, v->ne[1], mask, false, true, flash_attn, kv_scale); // [N, L, n_head*d_head]
|
||||
auto x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, v->ne[1], mask, false, true, ctx->flash_attn_enabled, kv_scale); // [N, L, n_head*d_head]
|
||||
return x;
|
||||
}
|
||||
}; // namespace Rope
|
||||
|
||||
+280
-257
File diff suppressed because it is too large
Load Diff
@@ -461,7 +461,7 @@ protected:
|
||||
int64_t hidden_size;
|
||||
float eps;
|
||||
|
||||
void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, const std::string prefix = "") override {
|
||||
void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
||||
enum ggml_type wtype = GGML_TYPE_F32;
|
||||
params["weight"] = ggml_new_tensor_1d(ctx, wtype, hidden_size);
|
||||
}
|
||||
@@ -472,10 +472,10 @@ public:
|
||||
: hidden_size(hidden_size),
|
||||
eps(eps) {}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* w = params["weight"];
|
||||
x = ggml_rms_norm(ctx, x, eps);
|
||||
x = ggml_mul(ctx, x, w);
|
||||
x = ggml_rms_norm(ctx->ggml_ctx, x, eps);
|
||||
x = ggml_mul(ctx->ggml_ctx, x, w);
|
||||
return x;
|
||||
}
|
||||
};
|
||||
@@ -487,13 +487,13 @@ public:
|
||||
blocks["wo"] = std::shared_ptr<GGMLBlock>(new Linear(ff_dim, model_dim, false));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
// x: [N, n_token, model_dim]
|
||||
auto wi = std::dynamic_pointer_cast<Linear>(blocks["wi"]);
|
||||
auto wo = std::dynamic_pointer_cast<Linear>(blocks["wo"]);
|
||||
|
||||
x = wi->forward(ctx, x);
|
||||
x = ggml_relu_inplace(ctx, x);
|
||||
x = ggml_relu_inplace(ctx->ggml_ctx, x);
|
||||
x = wo->forward(ctx, x);
|
||||
return x;
|
||||
}
|
||||
@@ -509,15 +509,15 @@ public:
|
||||
blocks["wo"] = std::shared_ptr<GGMLBlock>(new Linear(ff_dim, model_dim, false, false, false, scale));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
// x: [N, n_token, model_dim]
|
||||
auto wi_0 = std::dynamic_pointer_cast<Linear>(blocks["wi_0"]);
|
||||
auto wi_1 = std::dynamic_pointer_cast<Linear>(blocks["wi_1"]);
|
||||
auto wo = std::dynamic_pointer_cast<Linear>(blocks["wo"]);
|
||||
|
||||
auto hidden_gelu = ggml_gelu_inplace(ctx, wi_0->forward(ctx, x));
|
||||
auto hidden_gelu = ggml_gelu_inplace(ctx->ggml_ctx, wi_0->forward(ctx, x));
|
||||
auto hidden_linear = wi_1->forward(ctx, x);
|
||||
x = ggml_mul_inplace(ctx, hidden_gelu, hidden_linear);
|
||||
x = ggml_mul_inplace(ctx->ggml_ctx, hidden_gelu, hidden_linear);
|
||||
x = wo->forward(ctx, x);
|
||||
return x;
|
||||
}
|
||||
@@ -530,14 +530,14 @@ public:
|
||||
blocks["layer_norm"] = std::shared_ptr<GGMLBlock>(new T5LayerNorm(model_dim));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
// x: [N, n_token, model_dim]
|
||||
auto DenseReluDense = std::dynamic_pointer_cast<T5DenseGatedActDense>(blocks["DenseReluDense"]);
|
||||
auto layer_norm = std::dynamic_pointer_cast<T5LayerNorm>(blocks["layer_norm"]);
|
||||
|
||||
auto forwarded_states = layer_norm->forward(ctx, x);
|
||||
forwarded_states = DenseReluDense->forward(ctx, forwarded_states);
|
||||
x = ggml_add_inplace(ctx, forwarded_states, x);
|
||||
x = ggml_add_inplace(ctx->ggml_ctx, forwarded_states, x);
|
||||
return x;
|
||||
}
|
||||
};
|
||||
@@ -569,18 +569,17 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* compute_bias(struct ggml_context* ctx,
|
||||
struct ggml_tensor* compute_bias(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* relative_position_bucket) {
|
||||
auto relative_attention_bias = std::dynamic_pointer_cast<Embedding>(blocks["relative_attention_bias"]);
|
||||
|
||||
auto values = relative_attention_bias->forward(ctx, relative_position_bucket); // shape (query_length, key_length, num_heads)
|
||||
values = ggml_cont(ctx, ggml_permute(ctx, values, 2, 0, 1, 3)); // shape (1, num_heads, query_length, key_length)
|
||||
auto values = relative_attention_bias->forward(ctx, relative_position_bucket); // shape (query_length, key_length, num_heads)
|
||||
values = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, values, 2, 0, 1, 3)); // shape (1, num_heads, query_length, key_length)
|
||||
return values;
|
||||
}
|
||||
|
||||
// x: [N, n_token, model_dim]
|
||||
std::pair<struct ggml_tensor*, struct ggml_tensor*> forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
std::pair<struct ggml_tensor*, struct ggml_tensor*> forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* past_bias = nullptr,
|
||||
struct ggml_tensor* mask = nullptr,
|
||||
@@ -602,16 +601,16 @@ public:
|
||||
}
|
||||
if (past_bias != nullptr) {
|
||||
if (mask != nullptr) {
|
||||
mask = ggml_repeat(ctx, mask, past_bias);
|
||||
mask = ggml_add(ctx, mask, past_bias);
|
||||
mask = ggml_repeat(ctx->ggml_ctx, mask, past_bias);
|
||||
mask = ggml_add(ctx->ggml_ctx, mask, past_bias);
|
||||
} else {
|
||||
mask = past_bias;
|
||||
}
|
||||
}
|
||||
|
||||
k = ggml_scale_inplace(ctx, k, sqrt(d_head));
|
||||
k = ggml_scale_inplace(ctx->ggml_ctx, k, sqrt(d_head));
|
||||
|
||||
x = ggml_nn_attention_ext(ctx, backend, q, k, v, num_heads, mask); // [N, n_token, d_head * n_head]
|
||||
x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, num_heads, mask); // [N, n_token, d_head * n_head]
|
||||
|
||||
x = out_proj->forward(ctx, x); // [N, n_token, model_dim]
|
||||
return {x, past_bias};
|
||||
@@ -629,8 +628,7 @@ public:
|
||||
blocks["layer_norm"] = std::shared_ptr<GGMLBlock>(new T5LayerNorm(model_dim));
|
||||
}
|
||||
|
||||
std::pair<struct ggml_tensor*, struct ggml_tensor*> forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
std::pair<struct ggml_tensor*, struct ggml_tensor*> forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* past_bias = nullptr,
|
||||
struct ggml_tensor* mask = nullptr,
|
||||
@@ -640,11 +638,11 @@ public:
|
||||
auto layer_norm = std::dynamic_pointer_cast<T5LayerNorm>(blocks["layer_norm"]);
|
||||
|
||||
auto normed_hidden_state = layer_norm->forward(ctx, x);
|
||||
auto ret = SelfAttention->forward(ctx, backend, normed_hidden_state, past_bias, mask, relative_position_bucket);
|
||||
auto ret = SelfAttention->forward(ctx, normed_hidden_state, past_bias, mask, relative_position_bucket);
|
||||
auto output = ret.first;
|
||||
past_bias = ret.second;
|
||||
|
||||
x = ggml_add_inplace(ctx, output, x);
|
||||
x = ggml_add_inplace(ctx->ggml_ctx, output, x);
|
||||
return {x, past_bias};
|
||||
}
|
||||
};
|
||||
@@ -656,8 +654,7 @@ public:
|
||||
blocks["layer.1"] = std::shared_ptr<GGMLBlock>(new T5LayerFF(model_dim, ff_dim));
|
||||
}
|
||||
|
||||
std::pair<struct ggml_tensor*, struct ggml_tensor*> forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
std::pair<struct ggml_tensor*, struct ggml_tensor*> forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* past_bias = nullptr,
|
||||
struct ggml_tensor* mask = nullptr,
|
||||
@@ -666,7 +663,7 @@ public:
|
||||
auto layer_0 = std::dynamic_pointer_cast<T5LayerSelfAttention>(blocks["layer.0"]);
|
||||
auto layer_1 = std::dynamic_pointer_cast<T5LayerFF>(blocks["layer.1"]);
|
||||
|
||||
auto ret = layer_0->forward(ctx, backend, x, past_bias, mask, relative_position_bucket);
|
||||
auto ret = layer_0->forward(ctx, x, past_bias, mask, relative_position_bucket);
|
||||
x = ret.first;
|
||||
past_bias = ret.second;
|
||||
x = layer_1->forward(ctx, x);
|
||||
@@ -692,8 +689,7 @@ public:
|
||||
blocks["final_layer_norm"] = std::shared_ptr<GGMLBlock>(new T5LayerNorm(model_dim));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* past_bias = nullptr,
|
||||
struct ggml_tensor* attention_mask = nullptr,
|
||||
@@ -702,7 +698,7 @@ public:
|
||||
for (int i = 0; i < num_layers; i++) {
|
||||
auto block = std::dynamic_pointer_cast<T5Block>(blocks["block." + std::to_string(i)]);
|
||||
|
||||
auto ret = block->forward(ctx, backend, x, past_bias, attention_mask, relative_position_bucket);
|
||||
auto ret = block->forward(ctx, x, past_bias, attention_mask, relative_position_bucket);
|
||||
x = ret.first;
|
||||
past_bias = ret.second;
|
||||
}
|
||||
@@ -740,8 +736,7 @@ public:
|
||||
params.model_dim));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* input_ids,
|
||||
struct ggml_tensor* past_bias = nullptr,
|
||||
struct ggml_tensor* attention_mask = nullptr,
|
||||
@@ -752,7 +747,7 @@ public:
|
||||
auto encoder = std::dynamic_pointer_cast<T5Stack>(blocks["encoder"]);
|
||||
|
||||
auto x = shared->forward(ctx, input_ids);
|
||||
x = encoder->forward(ctx, backend, x, past_bias, attention_mask, relative_position_bucket);
|
||||
x = encoder->forward(ctx, x, past_bias, attention_mask, relative_position_bucket);
|
||||
return x;
|
||||
}
|
||||
};
|
||||
@@ -764,7 +759,7 @@ struct T5Runner : public GGMLRunner {
|
||||
|
||||
T5Runner(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types,
|
||||
const String2TensorStorage& tensor_storage_map,
|
||||
const std::string prefix,
|
||||
bool is_umt5 = false)
|
||||
: GGMLRunner(backend, offload_params_to_cpu) {
|
||||
@@ -773,7 +768,7 @@ struct T5Runner : public GGMLRunner {
|
||||
params.relative_attention = false;
|
||||
}
|
||||
model = T5(params);
|
||||
model.init(params_ctx, tensor_types, prefix);
|
||||
model.init(params_ctx, tensor_storage_map, prefix);
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -784,15 +779,14 @@ struct T5Runner : public GGMLRunner {
|
||||
model.get_param_tensors(tensors, prefix);
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* input_ids,
|
||||
struct ggml_tensor* relative_position_bucket,
|
||||
struct ggml_tensor* attention_mask = nullptr) {
|
||||
size_t N = input_ids->ne[1];
|
||||
size_t n_token = input_ids->ne[0];
|
||||
|
||||
auto hidden_states = model.forward(ctx, backend, input_ids, nullptr, attention_mask, relative_position_bucket); // [N, n_token, model_dim]
|
||||
auto hidden_states = model.forward(ctx, input_ids, nullptr, attention_mask, relative_position_bucket); // [N, n_token, model_dim]
|
||||
return hidden_states;
|
||||
}
|
||||
|
||||
@@ -818,7 +812,8 @@ struct T5Runner : public GGMLRunner {
|
||||
input_ids->ne[0]);
|
||||
set_backend_tensor_data(relative_position_bucket, relative_position_bucket_vec.data());
|
||||
|
||||
struct ggml_tensor* hidden_states = forward(compute_ctx, runtime_backend, input_ids, relative_position_bucket, attention_mask);
|
||||
auto runner_ctx = get_context();
|
||||
struct ggml_tensor* hidden_states = forward(&runner_ctx, input_ids, relative_position_bucket, attention_mask);
|
||||
|
||||
ggml_build_forward_expand(gf, hidden_states);
|
||||
|
||||
@@ -910,10 +905,10 @@ struct T5Embedder {
|
||||
|
||||
T5Embedder(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types = {},
|
||||
const std::string prefix = "",
|
||||
bool is_umt5 = false)
|
||||
: model(backend, offload_params_to_cpu, tensor_types, prefix, is_umt5), tokenizer(is_umt5) {
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
const std::string prefix = "",
|
||||
bool is_umt5 = false)
|
||||
: model(backend, offload_params_to_cpu, tensor_storage_map, prefix, is_umt5), tokenizer(is_umt5) {
|
||||
}
|
||||
|
||||
void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
|
||||
@@ -1014,15 +1009,14 @@ struct T5Embedder {
|
||||
return;
|
||||
}
|
||||
|
||||
auto tensor_types = model_loader.tensor_storages_types;
|
||||
for (auto& item : tensor_types) {
|
||||
// LOG_DEBUG("%s %u", item.first.c_str(), item.second);
|
||||
if (ends_with(item.first, "weight")) {
|
||||
item.second = model_data_type;
|
||||
auto& tensor_storage_map = model_loader.get_tensor_storage_map();
|
||||
for (auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (ends_with(name, "weight")) {
|
||||
tensor_storage.expected_type = model_data_type;
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<T5Embedder> t5 = std::make_shared<T5Embedder>(backend, false, tensor_types, "", true);
|
||||
std::shared_ptr<T5Embedder> t5 = std::make_shared<T5Embedder>(backend, false, tensor_storage_map, "", true);
|
||||
|
||||
t5->alloc_params_buffer();
|
||||
std::map<std::string, ggml_tensor*> tensors;
|
||||
|
||||
@@ -29,7 +29,7 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
// x: [n, n_in, h, w]
|
||||
// return: [n, n_out, h, w]
|
||||
|
||||
@@ -38,9 +38,9 @@ public:
|
||||
auto conv_4 = std::dynamic_pointer_cast<Conv2d>(blocks["conv.4"]);
|
||||
|
||||
auto h = conv_0->forward(ctx, x);
|
||||
h = ggml_relu_inplace(ctx, h);
|
||||
h = ggml_relu_inplace(ctx->ggml_ctx, h);
|
||||
h = conv_2->forward(ctx, h);
|
||||
h = ggml_relu_inplace(ctx, h);
|
||||
h = ggml_relu_inplace(ctx->ggml_ctx, h);
|
||||
h = conv_4->forward(ctx, h);
|
||||
|
||||
if (n_in != n_out) {
|
||||
@@ -49,8 +49,8 @@ public:
|
||||
x = skip->forward(ctx, x);
|
||||
}
|
||||
|
||||
h = ggml_add(ctx, h, x);
|
||||
h = ggml_relu_inplace(ctx, h);
|
||||
h = ggml_add(ctx->ggml_ctx, h, x);
|
||||
h = ggml_relu_inplace(ctx->ggml_ctx, h);
|
||||
return h;
|
||||
}
|
||||
};
|
||||
@@ -86,7 +86,7 @@ public:
|
||||
blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, z_channels, {3, 3}, {1, 1}, {1, 1}));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
// x: [n, in_channels, h, w]
|
||||
// return: [n, z_channels, h/8, w/8]
|
||||
|
||||
@@ -136,20 +136,20 @@ public:
|
||||
blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, out_channels, {3, 3}, {1, 1}, {1, 1}));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* z) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* z) override {
|
||||
// z: [n, z_channels, h, w]
|
||||
// return: [n, out_channels, h*8, w*8]
|
||||
|
||||
auto h = ggml_scale(ctx, z, 1.0f / 3.0f);
|
||||
h = ggml_tanh_inplace(ctx, h);
|
||||
h = ggml_scale(ctx, h, 3.0f);
|
||||
auto h = ggml_scale(ctx->ggml_ctx, z, 1.0f / 3.0f);
|
||||
h = ggml_tanh_inplace(ctx->ggml_ctx, h);
|
||||
h = ggml_scale(ctx->ggml_ctx, h, 3.0f);
|
||||
|
||||
for (int i = 0; i < num_blocks * 3 + 10; i++) {
|
||||
if (blocks.find(std::to_string(i)) == blocks.end()) {
|
||||
if (i == 1) {
|
||||
h = ggml_relu_inplace(ctx, h);
|
||||
h = ggml_relu_inplace(ctx->ggml_ctx, h);
|
||||
} else {
|
||||
h = ggml_upscale(ctx, h, 2, GGML_SCALE_MODE_NEAREST);
|
||||
h = ggml_upscale(ctx->ggml_ctx, h, 2, GGML_SCALE_MODE_NEAREST);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -180,12 +180,12 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* decode(struct ggml_context* ctx, struct ggml_tensor* z) {
|
||||
struct ggml_tensor* decode(GGMLRunnerContext* ctx, struct ggml_tensor* z) {
|
||||
auto decoder = std::dynamic_pointer_cast<TinyDecoder>(blocks["decoder.layers"]);
|
||||
return decoder->forward(ctx, z);
|
||||
}
|
||||
|
||||
struct ggml_tensor* encode(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* encode(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
auto encoder = std::dynamic_pointer_cast<TinyEncoder>(blocks["encoder.layers"]);
|
||||
return encoder->forward(ctx, x);
|
||||
}
|
||||
@@ -197,25 +197,14 @@ struct TinyAutoEncoder : public GGMLRunner {
|
||||
|
||||
TinyAutoEncoder(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types,
|
||||
const String2TensorStorage& tensor_storage_map,
|
||||
const std::string prefix,
|
||||
bool decoder_only = true,
|
||||
SDVersion version = VERSION_SD1)
|
||||
: decode_only(decoder_only),
|
||||
taesd(decoder_only, version),
|
||||
GGMLRunner(backend, offload_params_to_cpu) {
|
||||
taesd.init(params_ctx, tensor_types, prefix);
|
||||
}
|
||||
|
||||
void enable_conv2d_direct() {
|
||||
std::vector<GGMLBlock*> blocks;
|
||||
taesd.get_all_blocks(blocks);
|
||||
for (auto block : blocks) {
|
||||
if (block->get_desc() == "Conv2d") {
|
||||
auto conv_block = (Conv2d*)block;
|
||||
conv_block->enable_direct();
|
||||
}
|
||||
}
|
||||
taesd.init(params_ctx, tensor_storage_map, prefix);
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -252,7 +241,8 @@ struct TinyAutoEncoder : public GGMLRunner {
|
||||
struct ggml_cgraph* build_graph(struct ggml_tensor* z, bool decode_graph) {
|
||||
struct ggml_cgraph* gf = ggml_new_graph(compute_ctx);
|
||||
z = to_backend(z);
|
||||
struct ggml_tensor* out = decode_graph ? taesd.decode(compute_ctx, z) : taesd.encode(compute_ctx, z);
|
||||
auto runner_ctx = get_context();
|
||||
struct ggml_tensor* out = decode_graph ? taesd.decode(&runner_ctx, z) : taesd.encode(&runner_ctx, z);
|
||||
ggml_build_forward_expand(gf, out);
|
||||
return gf;
|
||||
}
|
||||
|
||||
@@ -20,9 +20,10 @@ public:
|
||||
int64_t d_head,
|
||||
int64_t depth,
|
||||
int64_t context_dim,
|
||||
bool use_linear,
|
||||
int64_t time_depth = 1,
|
||||
int64_t max_time_embed_period = 10000)
|
||||
: SpatialTransformer(in_channels, n_head, d_head, depth, context_dim),
|
||||
: SpatialTransformer(in_channels, n_head, d_head, depth, context_dim, use_linear),
|
||||
max_time_embed_period(max_time_embed_period) {
|
||||
// We will convert unet transformer linear to conv2d 1x1 when loading the weights, so use_linear is always False
|
||||
// use_spatial_context is always True
|
||||
@@ -60,8 +61,7 @@ public:
|
||||
blocks["time_mixer"] = std::shared_ptr<GGMLBlock>(new AlphaBlender());
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* context,
|
||||
int timesteps) {
|
||||
@@ -92,7 +92,7 @@ public:
|
||||
auto time_context = context; // [b*t, n_context, context_dim]
|
||||
auto spatial_context = context;
|
||||
// time_context_first_timestep = time_context[::timesteps]
|
||||
auto time_context_first_timestep = ggml_view_3d(ctx,
|
||||
auto time_context_first_timestep = ggml_view_3d(ctx->ggml_ctx,
|
||||
time_context,
|
||||
time_context->ne[0],
|
||||
time_context->ne[1],
|
||||
@@ -100,26 +100,26 @@ public:
|
||||
time_context->nb[1],
|
||||
time_context->nb[2],
|
||||
0); // [b, n_context, context_dim]
|
||||
time_context = ggml_new_tensor_3d(ctx, GGML_TYPE_F32,
|
||||
time_context = ggml_new_tensor_3d(ctx->ggml_ctx, GGML_TYPE_F32,
|
||||
time_context_first_timestep->ne[0],
|
||||
time_context_first_timestep->ne[1],
|
||||
time_context_first_timestep->ne[2] * h * w);
|
||||
time_context = ggml_repeat(ctx, time_context_first_timestep, time_context); // [b*h*w, n_context, context_dim]
|
||||
time_context = ggml_repeat(ctx->ggml_ctx, time_context_first_timestep, time_context); // [b*h*w, n_context, context_dim]
|
||||
|
||||
x = norm->forward(ctx, x);
|
||||
x = proj_in->forward(ctx, x); // [N, inner_dim, h, w]
|
||||
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 1, 2, 0, 3)); // [N, h, w, inner_dim]
|
||||
x = ggml_reshape_3d(ctx, x, inner_dim, w * h, n); // [N, h * w, inner_dim]
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 2, 0, 3)); // [N, h, w, inner_dim]
|
||||
x = ggml_reshape_3d(ctx->ggml_ctx, x, inner_dim, w * h, n); // [N, h * w, inner_dim]
|
||||
|
||||
auto num_frames = ggml_arange(ctx, 0, timesteps, 1);
|
||||
auto num_frames = ggml_arange(ctx->ggml_ctx, 0, timesteps, 1);
|
||||
// since b is 1, no need to do repeat
|
||||
auto t_emb = ggml_nn_timestep_embedding(ctx, num_frames, in_channels, max_time_embed_period); // [N, in_channels]
|
||||
auto t_emb = ggml_ext_timestep_embedding(ctx->ggml_ctx, num_frames, in_channels, max_time_embed_period); // [N, in_channels]
|
||||
|
||||
auto emb = time_pos_embed_0->forward(ctx, t_emb);
|
||||
emb = ggml_silu_inplace(ctx, emb);
|
||||
emb = time_pos_embed_2->forward(ctx, emb); // [N, in_channels]
|
||||
emb = ggml_reshape_3d(ctx, emb, emb->ne[0], 1, emb->ne[1]); // [N, 1, in_channels]
|
||||
emb = ggml_silu_inplace(ctx->ggml_ctx, emb);
|
||||
emb = time_pos_embed_2->forward(ctx, emb); // [N, in_channels]
|
||||
emb = ggml_reshape_3d(ctx->ggml_ctx, emb, emb->ne[0], 1, emb->ne[1]); // [N, 1, in_channels]
|
||||
|
||||
for (int i = 0; i < depth; i++) {
|
||||
std::string transformer_name = "transformer_blocks." + std::to_string(i);
|
||||
@@ -128,11 +128,11 @@ public:
|
||||
auto block = std::dynamic_pointer_cast<BasicTransformerBlock>(blocks[transformer_name]);
|
||||
auto mix_block = std::dynamic_pointer_cast<BasicTransformerBlock>(blocks[time_stack_name]);
|
||||
|
||||
x = block->forward(ctx, backend, x, spatial_context); // [N, h * w, inner_dim]
|
||||
x = block->forward(ctx, x, spatial_context); // [N, h * w, inner_dim]
|
||||
|
||||
// in_channels == inner_dim
|
||||
auto x_mix = x;
|
||||
x_mix = ggml_add(ctx, x_mix, emb); // [N, h * w, inner_dim]
|
||||
x_mix = ggml_add(ctx->ggml_ctx, x_mix, emb); // [N, h * w, inner_dim]
|
||||
|
||||
int64_t N = x_mix->ne[2];
|
||||
int64_t T = timesteps;
|
||||
@@ -140,26 +140,26 @@ public:
|
||||
int64_t S = x_mix->ne[1];
|
||||
int64_t C = x_mix->ne[0];
|
||||
|
||||
x_mix = ggml_reshape_4d(ctx, x_mix, C, S, T, B); // (b t) s c -> b t s c
|
||||
x_mix = ggml_cont(ctx, ggml_permute(ctx, x_mix, 0, 2, 1, 3)); // b t s c -> b s t c
|
||||
x_mix = ggml_reshape_3d(ctx, x_mix, C, T, S * B); // b s t c -> (b s) t c
|
||||
x_mix = ggml_reshape_4d(ctx->ggml_ctx, x_mix, C, S, T, B); // (b t) s c -> b t s c
|
||||
x_mix = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x_mix, 0, 2, 1, 3)); // b t s c -> b s t c
|
||||
x_mix = ggml_reshape_3d(ctx->ggml_ctx, x_mix, C, T, S * B); // b s t c -> (b s) t c
|
||||
|
||||
x_mix = mix_block->forward(ctx, backend, x_mix, time_context); // [B * h * w, T, inner_dim]
|
||||
x_mix = mix_block->forward(ctx, x_mix, time_context); // [B * h * w, T, inner_dim]
|
||||
|
||||
x_mix = ggml_reshape_4d(ctx, x_mix, C, T, S, B); // (b s) t c -> b s t c
|
||||
x_mix = ggml_cont(ctx, ggml_permute(ctx, x_mix, 0, 2, 1, 3)); // b s t c -> b t s c
|
||||
x_mix = ggml_reshape_3d(ctx, x_mix, C, S, T * B); // b t s c -> (b t) s c
|
||||
x_mix = ggml_reshape_4d(ctx->ggml_ctx, x_mix, C, T, S, B); // (b s) t c -> b s t c
|
||||
x_mix = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x_mix, 0, 2, 1, 3)); // b s t c -> b t s c
|
||||
x_mix = ggml_reshape_3d(ctx->ggml_ctx, x_mix, C, S, T * B); // b t s c -> (b t) s c
|
||||
|
||||
x = time_mixer->forward(ctx, x, x_mix); // [N, h * w, inner_dim]
|
||||
}
|
||||
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 1, 0, 2, 3)); // [N, inner_dim, h * w]
|
||||
x = ggml_reshape_4d(ctx, x, w, h, inner_dim, n); // [N, inner_dim, h, w]
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); // [N, inner_dim, h * w]
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx, x, w, h, inner_dim, n); // [N, inner_dim, h, w]
|
||||
|
||||
// proj_out
|
||||
x = proj_out->forward(ctx, x); // [N, in_channels, h, w]
|
||||
|
||||
x = ggml_add(ctx, x, x_in);
|
||||
x = ggml_add(ctx->ggml_ctx, x, x_in);
|
||||
return x;
|
||||
}
|
||||
};
|
||||
@@ -179,17 +179,20 @@ protected:
|
||||
int num_heads = 8;
|
||||
int num_head_channels = -1; // channels // num_heads
|
||||
int context_dim = 768; // 1024 for VERSION_SD2, 2048 for VERSION_SDXL
|
||||
bool use_linear_projection = false;
|
||||
bool tiny_unet = false;
|
||||
|
||||
public:
|
||||
int model_channels = 320;
|
||||
int adm_in_channels = 2816; // only for VERSION_SDXL/SVD
|
||||
|
||||
UnetModelBlock(SDVersion version = VERSION_SD1, const String2GGMLType& tensor_types = {}, bool flash_attn = false)
|
||||
UnetModelBlock(SDVersion version = VERSION_SD1, const String2TensorStorage& tensor_storage_map = {})
|
||||
: version(version) {
|
||||
if (sd_version_is_sd2(version)) {
|
||||
context_dim = 1024;
|
||||
num_head_channels = 64;
|
||||
num_heads = -1;
|
||||
context_dim = 1024;
|
||||
num_head_channels = 64;
|
||||
num_heads = -1;
|
||||
use_linear_projection = true;
|
||||
} else if (sd_version_is_sdxl(version)) {
|
||||
context_dim = 2048;
|
||||
attention_resolutions = {4, 2};
|
||||
@@ -197,19 +200,26 @@ public:
|
||||
transformer_depth = {1, 2, 10};
|
||||
num_head_channels = 64;
|
||||
num_heads = -1;
|
||||
use_linear_projection = true;
|
||||
} else if (version == VERSION_SVD) {
|
||||
in_channels = 8;
|
||||
out_channels = 4;
|
||||
context_dim = 1024;
|
||||
adm_in_channels = 768;
|
||||
num_head_channels = 64;
|
||||
num_heads = -1;
|
||||
in_channels = 8;
|
||||
out_channels = 4;
|
||||
context_dim = 1024;
|
||||
adm_in_channels = 768;
|
||||
num_head_channels = 64;
|
||||
num_heads = -1;
|
||||
use_linear_projection = true;
|
||||
}
|
||||
if (sd_version_is_inpaint(version)) {
|
||||
in_channels = 9;
|
||||
} else if (sd_version_is_unet_edit(version)) {
|
||||
in_channels = 8;
|
||||
}
|
||||
if (version == VERSION_SD1_TINY_UNET || version == VERSION_SD2_TINY_UNET) {
|
||||
num_res_blocks = 1;
|
||||
channel_mult = {1, 2, 4};
|
||||
tiny_unet = true;
|
||||
}
|
||||
|
||||
// dims is always 2
|
||||
// use_temporal_attention is always True for SVD
|
||||
@@ -247,9 +257,9 @@ public:
|
||||
int64_t depth,
|
||||
int64_t context_dim) -> SpatialTransformer* {
|
||||
if (version == VERSION_SVD) {
|
||||
return new SpatialVideoTransformer(in_channels, n_head, d_head, depth, context_dim);
|
||||
return new SpatialVideoTransformer(in_channels, n_head, d_head, depth, context_dim, use_linear_projection);
|
||||
} else {
|
||||
return new SpatialTransformer(in_channels, n_head, d_head, depth, context_dim, flash_attn);
|
||||
return new SpatialTransformer(in_channels, n_head, d_head, depth, context_dim, use_linear_projection);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -270,13 +280,22 @@ public:
|
||||
n_head = ch / d_head;
|
||||
}
|
||||
std::string name = "input_blocks." + std::to_string(input_block_idx) + ".1";
|
||||
blocks[name] = std::shared_ptr<GGMLBlock>(get_attention_layer(ch,
|
||||
n_head,
|
||||
d_head,
|
||||
transformer_depth[i],
|
||||
context_dim));
|
||||
int td = transformer_depth[i];
|
||||
if (version == VERSION_SDXL_SSD1B) {
|
||||
if (i == 2) {
|
||||
td = 4;
|
||||
}
|
||||
}
|
||||
blocks[name] = std::shared_ptr<GGMLBlock>(get_attention_layer(ch,
|
||||
n_head,
|
||||
d_head,
|
||||
td,
|
||||
context_dim));
|
||||
}
|
||||
input_block_chans.push_back(ch);
|
||||
if (tiny_unet) {
|
||||
input_block_idx++;
|
||||
}
|
||||
}
|
||||
if (i != len_mults - 1) {
|
||||
input_block_idx += 1;
|
||||
@@ -295,14 +314,17 @@ public:
|
||||
d_head = num_head_channels;
|
||||
n_head = ch / d_head;
|
||||
}
|
||||
blocks["middle_block.0"] = std::shared_ptr<GGMLBlock>(get_resblock(ch, time_embed_dim, ch));
|
||||
blocks["middle_block.1"] = std::shared_ptr<GGMLBlock>(get_attention_layer(ch,
|
||||
n_head,
|
||||
d_head,
|
||||
transformer_depth[transformer_depth.size() - 1],
|
||||
context_dim));
|
||||
blocks["middle_block.2"] = std::shared_ptr<GGMLBlock>(get_resblock(ch, time_embed_dim, ch));
|
||||
|
||||
if (!tiny_unet) {
|
||||
blocks["middle_block.0"] = std::shared_ptr<GGMLBlock>(get_resblock(ch, time_embed_dim, ch));
|
||||
if (version != VERSION_SDXL_SSD1B) {
|
||||
blocks["middle_block.1"] = std::shared_ptr<GGMLBlock>(get_attention_layer(ch,
|
||||
n_head,
|
||||
d_head,
|
||||
transformer_depth[transformer_depth.size() - 1],
|
||||
context_dim));
|
||||
blocks["middle_block.2"] = std::shared_ptr<GGMLBlock>(get_resblock(ch, time_embed_dim, ch));
|
||||
}
|
||||
}
|
||||
// output_blocks
|
||||
int output_block_idx = 0;
|
||||
for (int i = (int)len_mults - 1; i >= 0; i--) {
|
||||
@@ -324,12 +346,27 @@ public:
|
||||
n_head = ch / d_head;
|
||||
}
|
||||
std::string name = "output_blocks." + std::to_string(output_block_idx) + ".1";
|
||||
blocks[name] = std::shared_ptr<GGMLBlock>(get_attention_layer(ch, n_head, d_head, transformer_depth[i], context_dim));
|
||||
int td = transformer_depth[i];
|
||||
if (version == VERSION_SDXL_SSD1B) {
|
||||
if (i == 2 && (j == 0 || j == 1)) {
|
||||
td = 4;
|
||||
}
|
||||
if (i == 1 && (j == 1 || j == 2)) {
|
||||
td = 1;
|
||||
}
|
||||
}
|
||||
blocks[name] = std::shared_ptr<GGMLBlock>(get_attention_layer(ch, n_head, d_head, td, context_dim));
|
||||
|
||||
up_sample_idx++;
|
||||
}
|
||||
|
||||
if (i > 0 && j == num_res_blocks) {
|
||||
if (tiny_unet) {
|
||||
output_block_idx++;
|
||||
if (output_block_idx == 2) {
|
||||
up_sample_idx = 1;
|
||||
}
|
||||
}
|
||||
std::string name = "output_blocks." + std::to_string(output_block_idx) + "." + std::to_string(up_sample_idx);
|
||||
blocks[name] = std::shared_ptr<GGMLBlock>(new UpSampleBlock(ch, ch));
|
||||
|
||||
@@ -347,7 +384,7 @@ public:
|
||||
}
|
||||
|
||||
struct ggml_tensor* resblock_forward(std::string name,
|
||||
struct ggml_context* ctx,
|
||||
GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* emb,
|
||||
int num_video_frames) {
|
||||
@@ -363,24 +400,22 @@ public:
|
||||
}
|
||||
|
||||
struct ggml_tensor* attention_layer_forward(std::string name,
|
||||
struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* context,
|
||||
int timesteps) {
|
||||
if (version == VERSION_SVD) {
|
||||
auto block = std::dynamic_pointer_cast<SpatialVideoTransformer>(blocks[name]);
|
||||
|
||||
return block->forward(ctx, backend, x, context, timesteps);
|
||||
return block->forward(ctx, x, context, timesteps);
|
||||
} else {
|
||||
auto block = std::dynamic_pointer_cast<SpatialTransformer>(blocks[name]);
|
||||
|
||||
return block->forward(ctx, backend, x, context);
|
||||
return block->forward(ctx, x, context);
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
ggml_backend_t backend,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* timesteps,
|
||||
struct ggml_tensor* context,
|
||||
@@ -397,20 +432,20 @@ public:
|
||||
// return: [N, out_channels, h, w]
|
||||
if (context != nullptr) {
|
||||
if (context->ne[2] != x->ne[3]) {
|
||||
context = ggml_repeat(ctx, context, ggml_new_tensor_3d(ctx, GGML_TYPE_F32, context->ne[0], context->ne[1], x->ne[3]));
|
||||
context = ggml_repeat(ctx->ggml_ctx, context, ggml_new_tensor_3d(ctx->ggml_ctx, GGML_TYPE_F32, context->ne[0], context->ne[1], x->ne[3]));
|
||||
}
|
||||
}
|
||||
|
||||
if (c_concat != nullptr) {
|
||||
if (c_concat->ne[3] != x->ne[3]) {
|
||||
c_concat = ggml_repeat(ctx, c_concat, x);
|
||||
c_concat = ggml_repeat(ctx->ggml_ctx, c_concat, x);
|
||||
}
|
||||
x = ggml_concat(ctx, x, c_concat, 2);
|
||||
x = ggml_concat(ctx->ggml_ctx, x, c_concat, 2);
|
||||
}
|
||||
|
||||
if (y != nullptr) {
|
||||
if (y->ne[1] != x->ne[3]) {
|
||||
y = ggml_repeat(ctx, y, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, y->ne[0], x->ne[3]));
|
||||
y = ggml_repeat(ctx->ggml_ctx, y, ggml_new_tensor_2d(ctx->ggml_ctx, GGML_TYPE_F32, y->ne[0], x->ne[3]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -421,10 +456,10 @@ public:
|
||||
auto out_0 = std::dynamic_pointer_cast<GroupNorm32>(blocks["out.0"]);
|
||||
auto out_2 = std::dynamic_pointer_cast<Conv2d>(blocks["out.2"]);
|
||||
|
||||
auto t_emb = ggml_nn_timestep_embedding(ctx, timesteps, model_channels); // [N, model_channels]
|
||||
auto t_emb = ggml_ext_timestep_embedding(ctx->ggml_ctx, timesteps, model_channels); // [N, model_channels]
|
||||
|
||||
auto emb = time_embed_0->forward(ctx, t_emb);
|
||||
emb = ggml_silu_inplace(ctx, emb);
|
||||
emb = ggml_silu_inplace(ctx->ggml_ctx, emb);
|
||||
emb = time_embed_2->forward(ctx, emb); // [N, time_embed_dim]
|
||||
|
||||
// SDXL/SVD
|
||||
@@ -433,10 +468,10 @@ public:
|
||||
auto label_embed_2 = std::dynamic_pointer_cast<Linear>(blocks["label_emb.0.2"]);
|
||||
|
||||
auto label_emb = label_embed_0->forward(ctx, y);
|
||||
label_emb = ggml_silu_inplace(ctx, label_emb);
|
||||
label_emb = ggml_silu_inplace(ctx->ggml_ctx, label_emb);
|
||||
label_emb = label_embed_2->forward(ctx, label_emb); // [N, time_embed_dim]
|
||||
|
||||
emb = ggml_add(ctx, emb, label_emb); // [N, time_embed_dim]
|
||||
emb = ggml_add(ctx->ggml_ctx, emb, label_emb); // [N, time_embed_dim]
|
||||
}
|
||||
|
||||
// input_blocks
|
||||
@@ -459,10 +494,13 @@ public:
|
||||
h = resblock_forward(name, ctx, h, emb, num_video_frames); // [N, mult*model_channels, h, w]
|
||||
if (std::find(attention_resolutions.begin(), attention_resolutions.end(), ds) != attention_resolutions.end()) {
|
||||
std::string name = "input_blocks." + std::to_string(input_block_idx) + ".1";
|
||||
h = attention_layer_forward(name, ctx, backend, h, context, num_video_frames); // [N, mult*model_channels, h, w]
|
||||
h = attention_layer_forward(name, ctx, h, context, num_video_frames); // [N, mult*model_channels, h, w]
|
||||
}
|
||||
hs.push_back(h);
|
||||
}
|
||||
if (tiny_unet) {
|
||||
input_block_idx++;
|
||||
}
|
||||
if (i != len_mults - 1) {
|
||||
ds *= 2;
|
||||
input_block_idx += 1;
|
||||
@@ -477,13 +515,16 @@ public:
|
||||
// [N, 4*model_channels, h/8, w/8]
|
||||
|
||||
// middle_block
|
||||
h = resblock_forward("middle_block.0", ctx, h, emb, num_video_frames); // [N, 4*model_channels, h/8, w/8]
|
||||
h = attention_layer_forward("middle_block.1", ctx, backend, h, context, num_video_frames); // [N, 4*model_channels, h/8, w/8]
|
||||
h = resblock_forward("middle_block.2", ctx, h, emb, num_video_frames); // [N, 4*model_channels, h/8, w/8]
|
||||
|
||||
if (!tiny_unet) {
|
||||
h = resblock_forward("middle_block.0", ctx, h, emb, num_video_frames); // [N, 4*model_channels, h/8, w/8]
|
||||
if (version != VERSION_SDXL_SSD1B) {
|
||||
h = attention_layer_forward("middle_block.1", ctx, h, context, num_video_frames); // [N, 4*model_channels, h/8, w/8]
|
||||
h = resblock_forward("middle_block.2", ctx, h, emb, num_video_frames); // [N, 4*model_channels, h/8, w/8]
|
||||
}
|
||||
}
|
||||
if (controls.size() > 0) {
|
||||
auto cs = ggml_scale_inplace(ctx, controls[controls.size() - 1], control_strength);
|
||||
h = ggml_add(ctx, h, cs); // middle control
|
||||
auto cs = ggml_scale_inplace(ctx->ggml_ctx, controls[controls.size() - 1], control_strength);
|
||||
h = ggml_add(ctx->ggml_ctx, h, cs); // middle control
|
||||
}
|
||||
int control_offset = controls.size() - 2;
|
||||
|
||||
@@ -495,12 +536,12 @@ public:
|
||||
hs.pop_back();
|
||||
|
||||
if (controls.size() > 0) {
|
||||
auto cs = ggml_scale_inplace(ctx, controls[control_offset], control_strength);
|
||||
h_skip = ggml_add(ctx, h_skip, cs); // control net condition
|
||||
auto cs = ggml_scale_inplace(ctx->ggml_ctx, controls[control_offset], control_strength);
|
||||
h_skip = ggml_add(ctx->ggml_ctx, h_skip, cs); // control net condition
|
||||
control_offset--;
|
||||
}
|
||||
|
||||
h = ggml_concat(ctx, h, h_skip, 2);
|
||||
h = ggml_concat(ctx->ggml_ctx, h, h_skip, 2);
|
||||
|
||||
std::string name = "output_blocks." + std::to_string(output_block_idx) + ".0";
|
||||
|
||||
@@ -510,12 +551,18 @@ public:
|
||||
if (std::find(attention_resolutions.begin(), attention_resolutions.end(), ds) != attention_resolutions.end()) {
|
||||
std::string name = "output_blocks." + std::to_string(output_block_idx) + ".1";
|
||||
|
||||
h = attention_layer_forward(name, ctx, backend, h, context, num_video_frames);
|
||||
h = attention_layer_forward(name, ctx, h, context, num_video_frames);
|
||||
|
||||
up_sample_idx++;
|
||||
}
|
||||
|
||||
if (i > 0 && j == num_res_blocks) {
|
||||
if (tiny_unet) {
|
||||
output_block_idx++;
|
||||
if (output_block_idx == 2) {
|
||||
up_sample_idx = 1;
|
||||
}
|
||||
}
|
||||
std::string name = "output_blocks." + std::to_string(output_block_idx) + "." + std::to_string(up_sample_idx);
|
||||
auto block = std::dynamic_pointer_cast<UpSampleBlock>(blocks[name]);
|
||||
|
||||
@@ -530,7 +577,7 @@ public:
|
||||
|
||||
// out
|
||||
h = out_0->forward(ctx, h);
|
||||
h = ggml_silu_inplace(ctx, h);
|
||||
h = ggml_silu_inplace(ctx->ggml_ctx, h);
|
||||
h = out_2->forward(ctx, h);
|
||||
ggml_set_name(h, "bench-end");
|
||||
return h; // [N, out_channels, h, w]
|
||||
@@ -542,24 +589,11 @@ struct UNetModelRunner : public GGMLRunner {
|
||||
|
||||
UNetModelRunner(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types,
|
||||
const String2TensorStorage& tensor_storage_map,
|
||||
const std::string prefix,
|
||||
SDVersion version = VERSION_SD1,
|
||||
bool flash_attn = false)
|
||||
: GGMLRunner(backend, offload_params_to_cpu), unet(version, tensor_types, flash_attn) {
|
||||
unet.init(params_ctx, tensor_types, prefix);
|
||||
}
|
||||
|
||||
void enable_conv2d_direct() {
|
||||
std::vector<GGMLBlock*> blocks;
|
||||
unet.get_all_blocks(blocks);
|
||||
for (auto block : blocks) {
|
||||
if (block->get_desc() == "Conv2d") {
|
||||
LOG_DEBUG("block %s", block->get_desc().c_str());
|
||||
auto conv_block = (Conv2d*)block;
|
||||
conv_block->enable_direct();
|
||||
}
|
||||
}
|
||||
SDVersion version = VERSION_SD1)
|
||||
: GGMLRunner(backend, offload_params_to_cpu), unet(version, tensor_storage_map) {
|
||||
unet.init(params_ctx, tensor_storage_map, prefix);
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
@@ -594,8 +628,9 @@ struct UNetModelRunner : public GGMLRunner {
|
||||
controls[i] = to_backend(controls[i]);
|
||||
}
|
||||
|
||||
struct ggml_tensor* out = unet.forward(compute_ctx,
|
||||
runtime_backend,
|
||||
auto runner_ctx = get_context();
|
||||
|
||||
struct ggml_tensor* out = unet.forward(&runner_ctx,
|
||||
x,
|
||||
timesteps,
|
||||
context,
|
||||
|
||||
+5
-5
@@ -51,9 +51,9 @@ struct UpscalerGGML {
|
||||
backend = ggml_backend_cpu_init();
|
||||
}
|
||||
LOG_INFO("Upscaler weight type: %s", ggml_type_name(model_data_type));
|
||||
esrgan_upscaler = std::make_shared<ESRGAN>(backend, offload_params_to_cpu, model_loader.tensor_storages_types);
|
||||
esrgan_upscaler = std::make_shared<ESRGAN>(backend, offload_params_to_cpu, model_loader.get_tensor_storage_map());
|
||||
if (direct) {
|
||||
esrgan_upscaler->enable_conv2d_direct();
|
||||
esrgan_upscaler->set_conv2d_direct_enabled(true);
|
||||
}
|
||||
if (!esrgan_upscaler->load_from_file(esrgan_path, n_threads)) {
|
||||
return false;
|
||||
@@ -82,7 +82,7 @@ struct UpscalerGGML {
|
||||
}
|
||||
// LOG_DEBUG("upscale work buffer size: %.2f MB", params.mem_size / 1024.f / 1024.f);
|
||||
ggml_tensor* input_image_tensor = ggml_new_tensor_4d(upscale_ctx, GGML_TYPE_F32, input_image.width, input_image.height, 3, 1);
|
||||
sd_image_to_tensor(input_image, input_image_tensor);
|
||||
sd_image_to_ggml_tensor(input_image, input_image_tensor);
|
||||
|
||||
ggml_tensor* upscaled = ggml_new_tensor_4d(upscale_ctx, GGML_TYPE_F32, output_width, output_height, 3, 1);
|
||||
auto on_tiling = [&](ggml_tensor* in, ggml_tensor* out, bool init) {
|
||||
@@ -91,8 +91,8 @@ struct UpscalerGGML {
|
||||
int64_t t0 = ggml_time_ms();
|
||||
sd_tiling(input_image_tensor, upscaled, esrgan_upscaler->scale, esrgan_upscaler->tile_size, 0.25f, on_tiling);
|
||||
esrgan_upscaler->free_compute_buffer();
|
||||
ggml_tensor_clamp(upscaled, 0.f, 1.f);
|
||||
uint8_t* upscaled_data = sd_tensor_to_image(upscaled);
|
||||
ggml_ext_tensor_clamp_inplace(upscaled, 0.f, 1.f);
|
||||
uint8_t* upscaled_data = ggml_tensor_to_sd_image(upscaled);
|
||||
ggml_free(upscale_ctx);
|
||||
int64_t t3 = ggml_time_ms();
|
||||
LOG_INFO("input_image_tensor upscaled, taking %.2fs", (t3 - t0) / 1000.0f);
|
||||
|
||||
@@ -26,9 +26,6 @@
|
||||
#include "ggml.h"
|
||||
#include "stable-diffusion.h"
|
||||
|
||||
#define STB_IMAGE_RESIZE_IMPLEMENTATION
|
||||
#include "stb_image_resize.h"
|
||||
|
||||
bool ends_with(const std::string& str, const std::string& ending) {
|
||||
if (str.length() >= ending.length()) {
|
||||
return (str.compare(str.length() - ending.length(), ending.length(), ending) == 0);
|
||||
|
||||
@@ -30,7 +30,7 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
// x: [N, in_channels, h, w]
|
||||
// t_emb is always None
|
||||
auto norm1 = std::dynamic_pointer_cast<GroupNorm32>(blocks["norm1"]);
|
||||
@@ -40,12 +40,12 @@ public:
|
||||
|
||||
auto h = x;
|
||||
h = norm1->forward(ctx, h);
|
||||
h = ggml_silu_inplace(ctx, h); // swish
|
||||
h = ggml_silu_inplace(ctx->ggml_ctx, h); // swish
|
||||
h = conv1->forward(ctx, h);
|
||||
// return h;
|
||||
|
||||
h = norm2->forward(ctx, h);
|
||||
h = ggml_silu_inplace(ctx, h); // swish
|
||||
h = ggml_silu_inplace(ctx->ggml_ctx, h); // swish
|
||||
// dropout, skip for inference
|
||||
h = conv2->forward(ctx, h);
|
||||
|
||||
@@ -56,7 +56,7 @@ public:
|
||||
x = nin_shortcut->forward(ctx, x); // [N, out_channels, h, w]
|
||||
}
|
||||
|
||||
h = ggml_add(ctx, h, x);
|
||||
h = ggml_add(ctx->ggml_ctx, h, x);
|
||||
return h; // [N, out_channels, h, w]
|
||||
}
|
||||
};
|
||||
@@ -64,25 +64,51 @@ public:
|
||||
class AttnBlock : public UnaryBlock {
|
||||
protected:
|
||||
int64_t in_channels;
|
||||
bool use_linear;
|
||||
|
||||
public:
|
||||
AttnBlock(int64_t in_channels)
|
||||
: in_channels(in_channels) {
|
||||
blocks["norm"] = std::shared_ptr<GGMLBlock>(new GroupNorm32(in_channels));
|
||||
blocks["q"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_channels, in_channels, {1, 1}));
|
||||
blocks["k"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_channels, in_channels, {1, 1}));
|
||||
blocks["v"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_channels, in_channels, {1, 1}));
|
||||
|
||||
blocks["proj_out"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_channels, in_channels, {1, 1}));
|
||||
void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") {
|
||||
auto iter = tensor_storage_map.find(prefix + "proj_out.weight");
|
||||
if (iter != tensor_storage_map.end()) {
|
||||
if (iter->second.n_dims == 4 && use_linear) {
|
||||
use_linear = false;
|
||||
blocks["q"] = std::make_shared<Conv2d>(in_channels, in_channels, std::pair{1, 1});
|
||||
blocks["k"] = std::make_shared<Conv2d>(in_channels, in_channels, std::pair{1, 1});
|
||||
blocks["v"] = std::make_shared<Conv2d>(in_channels, in_channels, std::pair{1, 1});
|
||||
blocks["proj_out"] = std::make_shared<Conv2d>(in_channels, in_channels, std::pair{1, 1});
|
||||
} else if (iter->second.n_dims == 2 && !use_linear) {
|
||||
use_linear = true;
|
||||
blocks["q"] = std::make_shared<Linear>(in_channels, in_channels);
|
||||
blocks["k"] = std::make_shared<Linear>(in_channels, in_channels);
|
||||
blocks["v"] = std::make_shared<Linear>(in_channels, in_channels);
|
||||
blocks["proj_out"] = std::make_shared<Linear>(in_channels, in_channels);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
public:
|
||||
AttnBlock(int64_t in_channels, bool use_linear)
|
||||
: in_channels(in_channels), use_linear(use_linear) {
|
||||
blocks["norm"] = std::shared_ptr<GGMLBlock>(new GroupNorm32(in_channels));
|
||||
if (use_linear) {
|
||||
blocks["q"] = std::shared_ptr<GGMLBlock>(new Linear(in_channels, in_channels));
|
||||
blocks["k"] = std::shared_ptr<GGMLBlock>(new Linear(in_channels, in_channels));
|
||||
blocks["v"] = std::shared_ptr<GGMLBlock>(new Linear(in_channels, in_channels));
|
||||
blocks["proj_out"] = std::shared_ptr<GGMLBlock>(new Linear(in_channels, in_channels));
|
||||
} else {
|
||||
blocks["q"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_channels, in_channels, {1, 1}));
|
||||
blocks["k"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_channels, in_channels, {1, 1}));
|
||||
blocks["v"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_channels, in_channels, {1, 1}));
|
||||
blocks["proj_out"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_channels, in_channels, {1, 1}));
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
// x: [N, in_channels, h, w]
|
||||
auto norm = std::dynamic_pointer_cast<GroupNorm32>(blocks["norm"]);
|
||||
auto q_proj = std::dynamic_pointer_cast<Conv2d>(blocks["q"]);
|
||||
auto k_proj = std::dynamic_pointer_cast<Conv2d>(blocks["k"]);
|
||||
auto v_proj = std::dynamic_pointer_cast<Conv2d>(blocks["v"]);
|
||||
auto proj_out = std::dynamic_pointer_cast<Conv2d>(blocks["proj_out"]);
|
||||
auto q_proj = std::dynamic_pointer_cast<UnaryBlock>(blocks["q"]);
|
||||
auto k_proj = std::dynamic_pointer_cast<UnaryBlock>(blocks["k"]);
|
||||
auto v_proj = std::dynamic_pointer_cast<UnaryBlock>(blocks["v"]);
|
||||
auto proj_out = std::dynamic_pointer_cast<UnaryBlock>(blocks["proj_out"]);
|
||||
|
||||
auto h_ = norm->forward(ctx, x);
|
||||
|
||||
@@ -91,25 +117,46 @@ public:
|
||||
const int64_t h = h_->ne[1];
|
||||
const int64_t w = h_->ne[0];
|
||||
|
||||
auto q = q_proj->forward(ctx, h_); // [N, in_channels, h, w]
|
||||
q = ggml_cont(ctx, ggml_permute(ctx, q, 1, 2, 0, 3)); // [N, h, w, in_channels]
|
||||
q = ggml_reshape_3d(ctx, q, c, h * w, n); // [N, h * w, in_channels]
|
||||
ggml_tensor* q;
|
||||
ggml_tensor* k;
|
||||
ggml_tensor* v;
|
||||
if (use_linear) {
|
||||
h_ = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, h_, 1, 2, 0, 3)); // [N, h, w, in_channels]
|
||||
h_ = ggml_reshape_3d(ctx->ggml_ctx, h_, c, h * w, n); // [N, h * w, in_channels]
|
||||
|
||||
auto k = k_proj->forward(ctx, h_); // [N, in_channels, h, w]
|
||||
k = ggml_cont(ctx, ggml_permute(ctx, k, 1, 2, 0, 3)); // [N, h, w, in_channels]
|
||||
k = ggml_reshape_3d(ctx, k, c, h * w, n); // [N, h * w, in_channels]
|
||||
q = q_proj->forward(ctx, h_); // [N, h * w, in_channels]
|
||||
k = k_proj->forward(ctx, h_); // [N, h * w, in_channels]
|
||||
v = v_proj->forward(ctx, h_); // [N, h * w, in_channels]
|
||||
|
||||
auto v = v_proj->forward(ctx, h_); // [N, in_channels, h, w]
|
||||
v = ggml_reshape_3d(ctx, v, h * w, c, n); // [N, in_channels, h * w]
|
||||
v = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, v, 1, 0, 2, 3)); // [N, in_channels, h * w]
|
||||
} else {
|
||||
q = q_proj->forward(ctx, h_); // [N, in_channels, h, w]
|
||||
q = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, q, 1, 2, 0, 3)); // [N, h, w, in_channels]
|
||||
q = ggml_reshape_3d(ctx->ggml_ctx, q, c, h * w, n); // [N, h * w, in_channels]
|
||||
|
||||
h_ = ggml_nn_attention(ctx, q, k, v, false); // [N, h * w, in_channels]
|
||||
k = k_proj->forward(ctx, h_); // [N, in_channels, h, w]
|
||||
k = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, k, 1, 2, 0, 3)); // [N, h, w, in_channels]
|
||||
k = ggml_reshape_3d(ctx->ggml_ctx, k, c, h * w, n); // [N, h * w, in_channels]
|
||||
|
||||
h_ = ggml_cont(ctx, ggml_permute(ctx, h_, 1, 0, 2, 3)); // [N, in_channels, h * w]
|
||||
h_ = ggml_reshape_4d(ctx, h_, w, h, c, n); // [N, in_channels, h, w]
|
||||
v = v_proj->forward(ctx, h_); // [N, in_channels, h, w]
|
||||
v = ggml_reshape_3d(ctx->ggml_ctx, v, h * w, c, n); // [N, in_channels, h * w]
|
||||
}
|
||||
|
||||
h_ = proj_out->forward(ctx, h_); // [N, in_channels, h, w]
|
||||
h_ = ggml_ext_attention(ctx->ggml_ctx, q, k, v, false); // [N, h * w, in_channels]
|
||||
|
||||
h_ = ggml_add(ctx, h_, x);
|
||||
if (use_linear) {
|
||||
h_ = proj_out->forward(ctx, h_); // [N, h * w, in_channels]
|
||||
|
||||
h_ = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, h_, 1, 0, 2, 3)); // [N, in_channels, h * w]
|
||||
h_ = ggml_reshape_4d(ctx->ggml_ctx, h_, w, h, c, n); // [N, in_channels, h, w]
|
||||
} else {
|
||||
h_ = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, h_, 1, 0, 2, 3)); // [N, in_channels, h * w]
|
||||
h_ = ggml_reshape_4d(ctx->ggml_ctx, h_, w, h, c, n); // [N, in_channels, h, w]
|
||||
|
||||
h_ = proj_out->forward(ctx, h_); // [N, in_channels, h, w]
|
||||
}
|
||||
|
||||
h_ = ggml_add(ctx->ggml_ctx, h_, x);
|
||||
return h_;
|
||||
}
|
||||
};
|
||||
@@ -133,7 +180,7 @@ public:
|
||||
kernel_padding));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
struct ggml_tensor* x) override {
|
||||
// timesteps always None
|
||||
// skip_video always False
|
||||
@@ -152,24 +199,24 @@ public:
|
||||
int64_t H = x->ne[1];
|
||||
int64_t W = x->ne[0];
|
||||
|
||||
x = ggml_reshape_4d(ctx, x, W * H, C, T, B); // (b t) c h w -> b t c (h w)
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); // b t c (h w) -> b c t (h w)
|
||||
x = time_mix_conv->forward(ctx, x); // [B, OC, T, OH * OW]
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); // b c t (h w) -> b t c (h w)
|
||||
x = ggml_reshape_4d(ctx, x, W, H, C, T * B); // b t c (h w) -> (b t) c h w
|
||||
return x; // [B*T, OC, OH, OW]
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx, x, W * H, C, T, B); // (b t) c h w -> b t c (h w)
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 0, 2, 1, 3)); // b t c (h w) -> b c t (h w)
|
||||
x = time_mix_conv->forward(ctx, x); // [B, OC, T, OH * OW]
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 0, 2, 1, 3)); // b c t (h w) -> b t c (h w)
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx, x, W, H, C, T * B); // b t c (h w) -> (b t) c h w
|
||||
return x; // [B*T, OC, OH, OW]
|
||||
}
|
||||
};
|
||||
|
||||
class VideoResnetBlock : public ResnetBlock {
|
||||
protected:
|
||||
void init_params(struct ggml_context* ctx, const String2GGMLType& tensor_types = {}, const std::string prefix = "") override {
|
||||
enum ggml_type wtype = get_type(prefix + "mix_factor", tensor_types, GGML_TYPE_F32);
|
||||
void init_params(struct ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
||||
enum ggml_type wtype = get_type(prefix + "mix_factor", tensor_storage_map, GGML_TYPE_F32);
|
||||
params["mix_factor"] = ggml_new_tensor_1d(ctx, wtype, 1);
|
||||
}
|
||||
|
||||
float get_alpha() {
|
||||
float alpha = ggml_backend_tensor_get_f32(params["mix_factor"]);
|
||||
float alpha = ggml_ext_backend_tensor_get_f32(params["mix_factor"]);
|
||||
return sigmoid(alpha);
|
||||
}
|
||||
|
||||
@@ -182,7 +229,7 @@ public:
|
||||
blocks["time_stack"] = std::shared_ptr<GGMLBlock>(new ResBlock(out_channels, 0, out_channels, {video_kernel_size, 1}, 3, false, true));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) override {
|
||||
struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) override {
|
||||
// x: [N, in_channels, h, w] aka [b*t, in_channels, h, w]
|
||||
// return: [N, out_channels, h, w] aka [b*t, out_channels, h, w]
|
||||
// t_emb is always None
|
||||
@@ -199,19 +246,19 @@ public:
|
||||
int64_t H = x->ne[1];
|
||||
int64_t W = x->ne[0];
|
||||
|
||||
x = ggml_reshape_4d(ctx, x, W * H, C, T, B); // (b t) c h w -> b t c (h w)
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); // b t c (h w) -> b c t (h w)
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx, x, W * H, C, T, B); // (b t) c h w -> b t c (h w)
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 0, 2, 1, 3)); // b t c (h w) -> b c t (h w)
|
||||
auto x_mix = x;
|
||||
|
||||
x = time_stack->forward(ctx, x); // b t c (h w)
|
||||
|
||||
float alpha = get_alpha();
|
||||
x = ggml_add(ctx,
|
||||
ggml_scale(ctx, x, alpha),
|
||||
ggml_scale(ctx, x_mix, 1.0f - alpha));
|
||||
x = ggml_add(ctx->ggml_ctx,
|
||||
ggml_scale(ctx->ggml_ctx, x, alpha),
|
||||
ggml_scale(ctx->ggml_ctx, x_mix, 1.0f - alpha));
|
||||
|
||||
x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); // b c t (h w) -> b t c (h w)
|
||||
x = ggml_reshape_4d(ctx, x, W, H, C, T * B); // b t c (h w) -> (b t) c h w
|
||||
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 0, 2, 1, 3)); // b c t (h w) -> b t c (h w)
|
||||
x = ggml_reshape_4d(ctx->ggml_ctx, x, W, H, C, T * B); // b t c (h w) -> (b t) c h w
|
||||
|
||||
return x;
|
||||
}
|
||||
@@ -233,7 +280,8 @@ public:
|
||||
int num_res_blocks,
|
||||
int in_channels,
|
||||
int z_channels,
|
||||
bool double_z = true)
|
||||
bool double_z = true,
|
||||
bool use_linear_projection = false)
|
||||
: ch(ch),
|
||||
ch_mult(ch_mult),
|
||||
num_res_blocks(num_res_blocks),
|
||||
@@ -264,14 +312,14 @@ public:
|
||||
}
|
||||
|
||||
blocks["mid.block_1"] = std::shared_ptr<GGMLBlock>(new ResnetBlock(block_in, block_in));
|
||||
blocks["mid.attn_1"] = std::shared_ptr<GGMLBlock>(new AttnBlock(block_in));
|
||||
blocks["mid.attn_1"] = std::shared_ptr<GGMLBlock>(new AttnBlock(block_in, use_linear_projection));
|
||||
blocks["mid.block_2"] = std::shared_ptr<GGMLBlock>(new ResnetBlock(block_in, block_in));
|
||||
|
||||
blocks["norm_out"] = std::shared_ptr<GGMLBlock>(new GroupNorm32(block_in));
|
||||
blocks["conv_out"] = std::shared_ptr<GGMLBlock>(new Conv2d(block_in, double_z ? z_channels * 2 : z_channels, {3, 3}, {1, 1}, {1, 1}));
|
||||
}
|
||||
|
||||
virtual struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
virtual struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [N, in_channels, h, w]
|
||||
|
||||
auto conv_in = std::dynamic_pointer_cast<Conv2d>(blocks["conv_in"]);
|
||||
@@ -307,8 +355,8 @@ public:
|
||||
|
||||
// end
|
||||
h = norm_out->forward(ctx, h);
|
||||
h = ggml_silu_inplace(ctx, h); // nonlinearity/swish
|
||||
h = conv_out->forward(ctx, h); // [N, z_channels*2, h, w]
|
||||
h = ggml_silu_inplace(ctx->ggml_ctx, h); // nonlinearity/swish
|
||||
h = conv_out->forward(ctx, h); // [N, z_channels*2, h, w]
|
||||
return h;
|
||||
}
|
||||
};
|
||||
@@ -351,8 +399,9 @@ public:
|
||||
std::vector<int> ch_mult,
|
||||
int num_res_blocks,
|
||||
int z_channels,
|
||||
bool video_decoder = false,
|
||||
int video_kernel_size = 3)
|
||||
bool use_linear_projection = false,
|
||||
bool video_decoder = false,
|
||||
int video_kernel_size = 3)
|
||||
: ch(ch),
|
||||
out_ch(out_ch),
|
||||
ch_mult(ch_mult),
|
||||
@@ -366,7 +415,7 @@ public:
|
||||
blocks["conv_in"] = std::shared_ptr<GGMLBlock>(new Conv2d(z_channels, block_in, {3, 3}, {1, 1}, {1, 1}));
|
||||
|
||||
blocks["mid.block_1"] = get_resnet_block(block_in, block_in);
|
||||
blocks["mid.attn_1"] = std::shared_ptr<GGMLBlock>(new AttnBlock(block_in));
|
||||
blocks["mid.attn_1"] = std::shared_ptr<GGMLBlock>(new AttnBlock(block_in, use_linear_projection));
|
||||
blocks["mid.block_2"] = get_resnet_block(block_in, block_in);
|
||||
|
||||
for (int i = num_resolutions - 1; i >= 0; i--) {
|
||||
@@ -388,7 +437,7 @@ public:
|
||||
blocks["conv_out"] = get_conv_out(block_in, out_ch, {3, 3}, {1, 1}, {1, 1});
|
||||
}
|
||||
|
||||
virtual struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* z) {
|
||||
virtual struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* z) {
|
||||
// z: [N, z_channels, h, w]
|
||||
// alpha is always 0
|
||||
// merge_strategy is always learned
|
||||
@@ -429,8 +478,8 @@ public:
|
||||
}
|
||||
|
||||
h = norm_out->forward(ctx, h);
|
||||
h = ggml_silu_inplace(ctx, h); // nonlinearity/swish
|
||||
h = conv_out->forward(ctx, h); // [N, out_ch, h*8, w*8]
|
||||
h = ggml_silu_inplace(ctx->ggml_ctx, h); // nonlinearity/swish
|
||||
h = conv_out->forward(ctx, h); // [N, out_ch, h*8, w*8]
|
||||
return h;
|
||||
}
|
||||
};
|
||||
@@ -454,9 +503,10 @@ protected:
|
||||
} dd_config;
|
||||
|
||||
public:
|
||||
AutoencodingEngine(bool decode_only = true,
|
||||
bool use_video_decoder = false,
|
||||
SDVersion version = VERSION_SD1)
|
||||
AutoencodingEngine(SDVersion version = VERSION_SD1,
|
||||
bool decode_only = true,
|
||||
bool use_linear_projection = false,
|
||||
bool use_video_decoder = false)
|
||||
: decode_only(decode_only), use_video_decoder(use_video_decoder) {
|
||||
if (sd_version_is_dit(version)) {
|
||||
dd_config.z_channels = 16;
|
||||
@@ -470,6 +520,7 @@ public:
|
||||
dd_config.ch_mult,
|
||||
dd_config.num_res_blocks,
|
||||
dd_config.z_channels,
|
||||
use_linear_projection,
|
||||
use_video_decoder));
|
||||
if (use_quant) {
|
||||
blocks["post_quant_conv"] = std::shared_ptr<GGMLBlock>(new Conv2d(dd_config.z_channels,
|
||||
@@ -482,7 +533,8 @@ public:
|
||||
dd_config.num_res_blocks,
|
||||
dd_config.in_channels,
|
||||
dd_config.z_channels,
|
||||
dd_config.double_z));
|
||||
dd_config.double_z,
|
||||
use_linear_projection));
|
||||
if (use_quant) {
|
||||
int factor = dd_config.double_z ? 2 : 1;
|
||||
|
||||
@@ -493,7 +545,7 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_tensor* decode(struct ggml_context* ctx, struct ggml_tensor* z) {
|
||||
struct ggml_tensor* decode(GGMLRunnerContext* ctx, struct ggml_tensor* z) {
|
||||
// z: [N, z_channels, h, w]
|
||||
if (use_quant) {
|
||||
auto post_quant_conv = std::dynamic_pointer_cast<Conv2d>(blocks["post_quant_conv"]);
|
||||
@@ -507,7 +559,7 @@ public:
|
||||
return h;
|
||||
}
|
||||
|
||||
struct ggml_tensor* encode(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* encode(GGMLRunnerContext* ctx, struct ggml_tensor* x) {
|
||||
// x: [N, in_channels, h, w]
|
||||
auto encoder = std::dynamic_pointer_cast<Encoder>(blocks["encoder"]);
|
||||
|
||||
@@ -529,34 +581,59 @@ struct VAE : public GGMLRunner {
|
||||
struct ggml_tensor** output,
|
||||
struct ggml_context* output_ctx) = 0;
|
||||
virtual void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) = 0;
|
||||
virtual void enable_conv2d_direct(){};
|
||||
virtual void set_conv2d_scale(float scale) { SD_UNUSED(scale); };
|
||||
};
|
||||
|
||||
struct FakeVAE : public VAE {
|
||||
FakeVAE(ggml_backend_t backend, bool offload_params_to_cpu)
|
||||
: VAE(backend, offload_params_to_cpu) {}
|
||||
void compute(const int n_threads,
|
||||
struct ggml_tensor* z,
|
||||
bool decode_graph,
|
||||
struct ggml_tensor** output,
|
||||
struct ggml_context* output_ctx) override {
|
||||
if (*output == nullptr && output_ctx != nullptr) {
|
||||
*output = ggml_dup_tensor(output_ctx, z);
|
||||
}
|
||||
ggml_ext_tensor_iter(z, [&](ggml_tensor* z, int64_t i0, int64_t i1, int64_t i2, int64_t i3) {
|
||||
float value = ggml_ext_tensor_get_f32(z, i0, i1, i2, i3);
|
||||
ggml_ext_tensor_set_f32(*output, value, i0, i1, i2, i3);
|
||||
});
|
||||
}
|
||||
|
||||
void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) override {}
|
||||
|
||||
std::string get_desc() override {
|
||||
return "fake_vae";
|
||||
}
|
||||
};
|
||||
|
||||
struct AutoEncoderKL : public VAE {
|
||||
bool decode_only = true;
|
||||
AutoencodingEngine ae;
|
||||
|
||||
AutoEncoderKL(ggml_backend_t backend,
|
||||
bool offload_params_to_cpu,
|
||||
const String2GGMLType& tensor_types,
|
||||
const String2TensorStorage& tensor_storage_map,
|
||||
const std::string prefix,
|
||||
bool decode_only = false,
|
||||
bool use_video_decoder = false,
|
||||
SDVersion version = VERSION_SD1)
|
||||
: decode_only(decode_only), ae(decode_only, use_video_decoder, version), VAE(backend, offload_params_to_cpu) {
|
||||
ae.init(params_ctx, tensor_types, prefix);
|
||||
}
|
||||
|
||||
void enable_conv2d_direct() override {
|
||||
std::vector<GGMLBlock*> blocks;
|
||||
ae.get_all_blocks(blocks);
|
||||
for (auto block : blocks) {
|
||||
if (block->get_desc() == "Conv2d") {
|
||||
auto conv_block = (Conv2d*)block;
|
||||
conv_block->enable_direct();
|
||||
: decode_only(decode_only), VAE(backend, offload_params_to_cpu) {
|
||||
bool use_linear_projection = false;
|
||||
for (const auto& [name, tensor_storage] : tensor_storage_map) {
|
||||
if (!starts_with(name, prefix)) {
|
||||
continue;
|
||||
}
|
||||
if (ends_with(name, "attn_1.proj_out.weight")) {
|
||||
if (tensor_storage.n_dims == 2) {
|
||||
use_linear_projection = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
ae = AutoencodingEngine(version, decode_only, use_linear_projection, use_video_decoder);
|
||||
ae.init(params_ctx, tensor_storage_map, prefix);
|
||||
}
|
||||
|
||||
void set_conv2d_scale(float scale) override {
|
||||
@@ -583,7 +660,9 @@ struct AutoEncoderKL : public VAE {
|
||||
|
||||
z = to_backend(z);
|
||||
|
||||
struct ggml_tensor* out = decode_graph ? ae.decode(compute_ctx, z) : ae.encode(compute_ctx, z);
|
||||
auto runner_ctx = get_context();
|
||||
|
||||
struct ggml_tensor* out = decode_graph ? ae.decode(&runner_ctx, z) : ae.encode(&runner_ctx, z);
|
||||
|
||||
ggml_build_forward_expand(gf, out);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user