From 8f120c93e77d1c57ee38723fc90e0a89532efe19 Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 18 Oct 2026 14:35:05 +0800 Subject: [PATCH] feat: support Diffusers-format MiniMax-H3 LoRAs --- src/model/adapter/lora.hpp | 43 +++++++++++++++++++++++++++++++++++++- src/model_loader.h | 2 ++ src/model_loader_files.cpp | 18 ++++++++++++++++ src/name_conversion.cpp | 20 ++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp index e9d20a4ece..1e5377ccc8 100644 --- a/src/model/adapter/lora.hpp +++ b/src/model/adapter/lora.hpp @@ -1,6 +1,7 @@ #ifndef __SD_MODEL_ADAPTER_LORA_HPP__ #define __SD_MODEL_ADAPTER_LORA_HPP__ +#include #include #include "core/ggml_extend.h" #include "core/ggml_extend_backend.h" @@ -11,15 +12,18 @@ #include "model/adapter/lora_ops.h" #include "model_loader.h" #include "model_manager.h" +#include "name_conversion.h" #define LORA_GRAPH_BASE_SIZE 10240 struct LoraModel : public GGMLRunner { std::string lora_id; - float multiplier = 1.0f; + float multiplier = 1.0f; + float metadata_alpha = -1.f; std::unordered_map lora_tensors; std::map original_tensor_to_final_tensor; std::set applied_lora_tensors; + std::set swapped_lora_up_tensors; std::set skipped_incompatible_lora_tensors; std::set warned_incompatible_model_tensors; std::string file_path; @@ -62,6 +66,31 @@ struct LoraModel : public GGMLRunner { if (load_failed || !registered_params.empty()) return false; model_manager->set_n_threads(n_threads); + metadata_alpha = -1.f; + const auto metadata = model_manager->loader().file_metadata(source_file); + const auto alpha = metadata.find("alpha"); + if (alpha != metadata.end()) { + try { + size_t parsed = 0; + metadata_alpha = std::stof(alpha->second, &parsed); + if (parsed != alpha->second.size() || !std::isfinite(metadata_alpha) || metadata_alpha < 0.f) { + throw std::invalid_argument("invalid alpha"); + } + } catch (const std::exception&) { + LOG_ERROR("invalid LoRA alpha metadata '%s' in '%s'", alpha->second.c_str(), file_path.c_str()); + return false; + } + LOG_VERBOSE("LoRA '%s' uses metadata alpha %.6g when tensor alpha/scale is absent", file_path.c_str(), metadata_alpha); + } + if (sd_version_is_minimax_h3(source_version)) { + for (const auto& entry : model_manager->loader().file_tensors(source_file)) { + const auto name = convert_tensor_name(entry.first, VERSION_COUNT); + if (ends_with(name, ".ff.net.0.proj.weight.lora_up")) { + // Diffusers SwiGLU stores [value, gate]; MiniMax-H3 uses [gate, value]. + swapped_lora_up_tensors.insert(convert_tensor_name(entry.first, source_version)); + } + } + } const auto sources = model_manager->loader().file_tensors(source_file, source_version); std::map tensors; std::map scalars; @@ -116,6 +145,8 @@ struct LoraModel : public GGMLRunner { scalar_values.clear(); original_tensor_to_final_tensor.clear(); applied_lora_tensors.clear(); + swapped_lora_up_tensors.clear(); + metadata_alpha = -1.f; skipped_incompatible_lora_tensors.clear(); warned_incompatible_model_tensors.clear(); applied = false; @@ -210,6 +241,10 @@ struct LoraModel : public GGMLRunner { auto iter = lora_tensors.find(lora_up_name); if (iter != lora_tensors.end()) { lora_up = ggml_ext_cast_f32(ctx, backend, iter->second); + if (swapped_lora_up_tensors.count(lora_up_name)) { + auto halves = ggml_ext_chunk(ctx, lora_up, 2, 1); + lora_up = ggml_concat(ctx, halves[1], halves[0], 1); + } } iter = lora_tensors.find(lora_mid_name); @@ -236,6 +271,7 @@ struct LoraModel : public GGMLRunner { float scale_value = 1.0f; int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1]; + scale_value = metadata_alpha >= 0.f ? metadata_alpha / rank : 1.f; iter = lora_tensors.find(scale_name); if (iter != lora_tensors.end()) { scale_value = scalar_value(iter->second); @@ -723,6 +759,10 @@ struct LoraModel : public GGMLRunner { iter = lora_tensors.find(lora_up_name); if (iter != lora_tensors.end()) { lora_up = iter->second; + if (swapped_lora_up_tensors.count(lora_up_name)) { + auto halves = ggml_ext_chunk(ctx, lora_up, 2, 1); + lora_up = ggml_concat(ctx, halves[1], halves[0], 1); + } if (is_conv2d && lora_up->type != GGML_TYPE_F16) { lora_up = ggml_cast(ctx, lora_up, GGML_TYPE_F16); } @@ -791,6 +831,7 @@ struct LoraModel : public GGMLRunner { std::string scale_tensor_name; int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1]; + scale_value = metadata_alpha >= 0.f ? metadata_alpha / rank : 1.f; iter = lora_tensors.find(scale_name); if (iter != lora_tensors.end()) { scale_value = scalar_value(iter->second); diff --git a/src/model_loader.h b/src/model_loader.h index 63ecfb63b8..aae359099e 100644 --- a/src/model_loader.h +++ b/src/model_loader.h @@ -105,6 +105,8 @@ class ModelLoader { bool del_file(FileId id); uint64_t file_revision(FileId id) const; std::string file_path(FileId id) const; + std::map file_metadata(FileId id) const; + String2TensorStorage file_tensors(FileId id) const; String2TensorStorage file_tensors(FileId id, SDVersion version) const; bool load_file_tensors(FileId id, SDVersion version, on_new_tensor_cb_t callback, const std::set& names, bool use_mmap = false) const; bool refresh_files(bool include_isolated = true); diff --git a/src/model_loader_files.cpp b/src/model_loader_files.cpp index 8e838b1190..9ac5456cc1 100644 --- a/src/model_loader_files.cpp +++ b/src/model_loader_files.cpp @@ -324,6 +324,24 @@ ModelLoader ModelLoader::file_reader(FileId id, SDVersion version) const { return reader; } +std::map ModelLoader::file_metadata(FileId id) const { + for (const auto& file : files_) { + if (file.id == id) { + return file.metadata; + } + } + return {}; +} + +String2TensorStorage ModelLoader::file_tensors(FileId id) const { + for (const auto& file : files_) { + if (file.id == id) { + return file.tensors; + } + } + return {}; +} + String2TensorStorage ModelLoader::file_tensors(FileId id, SDVersion version) const { return file_reader(id, version).tensor_storage_map; } diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp index a342eb82ec..8f9f1fc4ed 100644 --- a/src/name_conversion.cpp +++ b/src/name_conversion.cpp @@ -1024,6 +1024,24 @@ static std::string convert_ming_image_dit_name(std::string name) { return name; } +std::string convert_diffusers_dit_to_original_minimax_h3(std::string name) { + static const std::vector> prefix_map = { + {"transformer_blocks.", "blocks."}, + {"token_refiner.refiner_blocks.", "token_refiner.blocks."}, + }; + static const std::vector> name_map = { + {".attn.to_q.weight", ".attn.qkv_proj.weight"}, + {".attn.to_k.weight", ".attn.qkv_proj.weight.1"}, + {".attn.to_v.weight", ".attn.qkv_proj.weight.2"}, + {".attn.to_out.0.", ".attn.out_proj."}, + {".ff.net.0.proj.", ".mlp.fc1."}, + {".ff.net.2.", ".mlp.fc2."}, + }; + replace_with_prefix_map(name, prefix_map); + replace_with_name_map(name, name_map); + return name; +} + std::string convert_diffusion_model_name(std::string name, std::string prefix, SDVersion version) { if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) { name = convert_diffusers_unet_to_original_sd1(name); @@ -1047,6 +1065,8 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S name = convert_diffusers_dit_to_original_krea2(name); } else if (sd_version_is_pixart(version)) { name = convert_diffusers_dit_to_original_pixart(name); + } else if (sd_version_is_minimax_h3(version)) { + name = convert_diffusers_dit_to_original_minimax_h3(name); } return name; }