From f10bc52a7bdcf2befb7fb0eb6cb2d4a4af3a0625 Mon Sep 17 00:00:00 2001 From: leejet Date: Thu, 6 Aug 2026 01:15:12 +0800 Subject: [PATCH 1/2] feat: support INT8 ConvRot safetensors --- README.md | 1 + docs/int8_convrot.md | 102 +++++++++++++++++++++++++ ggml | 2 +- src/core/ggml_extend.hpp | 130 ++++++++++++++++++++++++++++---- src/model/adapter/lora.hpp | 28 +++++++ src/model_io/safetensors_io.cpp | 103 +++++++++++++++++++++++++ src/model_io/tensor_storage.h | 19 +++-- src/model_loader.cpp | 3 + src/stable-diffusion.cpp | 21 +++++- 9 files changed, 384 insertions(+), 25 deletions(-) create mode 100644 docs/int8_convrot.md diff --git a/README.md b/README.md index cdbbf7735..a80ae5f1b 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ For runtime and parameter backend placement, see the [backend selection guide](. - [LCM/LCM-LoRA](./docs/lcm.md) - [Docker](./docs/docker.md) - [Quantization and GGUF](./docs/quantization_and_gguf.md) +- [INT8 convrot safetensors](./docs/int8_convrot.md) - [Inference acceleration via caching](./docs/caching.md) ## Bindings diff --git a/docs/int8_convrot.md b/docs/int8_convrot.md new file mode 100644 index 000000000..d41df8bd4 --- /dev/null +++ b/docs/int8_convrot.md @@ -0,0 +1,102 @@ +# INT8 Convrot Safetensors + +sd.cpp can load and execute ComfyUI `int8_tensorwise` safetensors with `convrot` metadata directly. The stored INT8 weights are not converted to another weight type at load time. + +## Checkpoint format + +Each quantized linear module contains the following tensors: + +- `.weight`: an I8 weight matrix. +- `.weight_scale`: one floating-point scale for each output row. ComfyUI's two-dimensional `[out_features, 1]` representation is normalized to a one-dimensional tensor while loading. +- `.comfy_quant`: a U8 tensor containing the JSON quantization configuration. + +A supported configuration has this form: + +```json +{ + "format": "int8_tensorwise", + "convrot": true, + "convrot_groupsize": 256 +} +``` + +The convrot group size must be a power of four and must divide the input feature dimension. The commonly used configuration is H256, with `convrot_groupsize` set to `256`. + +## How INT8 convrot works + +Convrot combines an offline rotation of the weights with the same rotation of the activations at runtime. The rotation uses a normalized regular Hadamard matrix constructed recursively from + +```text + [ 1 1 1 -1 ] + [ 1 1 -1 1 ] +H4 = [ 1 -1 1 1 ] / 2 + [-1 1 1 1 ] +``` + +For a group size `G = 4^n`, the transform is the normalized Kronecker power of `H4`. It is applied independently to every contiguous group of `G` input features. The resulting block-diagonal rotation matrix `R` is orthogonal and symmetric, so `R R^T = I`. + +For an original floating-point linear layer + +```text +Y = X W^T + b +``` + +the checkpoint stores a rotated weight matrix `W_rot = W R`, quantized per output row. At runtime sd.cpp computes `X_rot = X R`. Ignoring quantization error, + +```text +X_rot W_rot^T = X R (W R)^T = X R R^T W^T = X W^T +``` + +The rotation therefore preserves the linear operation. Its purpose is to spread isolated large values across each feature group, reducing the effect of outliers on tensorwise INT8 quantization. + +### Weight quantization + +The rotated weights are quantized offline with one scale per output row: + +```text +s_w[o] = max_i(abs(W_rot[o, i])) / 127 +Q_w[o, i] = clamp(round(W_rot[o, i] / s_w[o]), -127, 127) +``` + +`Q_w` is stored in `.weight`, and `s_w` is stored in `.weight_scale`. + +### Runtime activation quantization + +For every activation row, sd.cpp applies the group-wise Hadamard rotation and then calculates one dynamic scale across the entire rotated row: + +```text +s_x[r] = max_i(abs(X_rot[r, i])) / 127 +Q_x[r, i] = clamp(round(X_rot[r, i] / s_x[r]), -127, 127) +``` + +The matrix multiplication accumulates into signed 32-bit integers: + +```text +A[r, o] = sum_i(Q_x[r, i] * Q_w[o, i]) +``` + +The floating-point output is reconstructed as + +```text +Y[r, o] ~= A[r, o] * s_x[r] * s_w[o] + b[o] +``` + +The packed runtime activation tensor contains the I8 activation rows and their floating-point row scales. Linear layers that share the same input and convrot group size reuse this packed tensor, avoiding repeated rotation and activation quantization within the graph. + +## Backend support + +- CPU provides the portable regular Hadamard, activation quantization, INT8 matrix multiplication, and scale restoration implementations. +- NVIDIA CUDA devices with compute capability 7.5 or newer use the native accelerated path. For H256, CUDA fuses the rotation, row-wise maximum reduction, and activation quantization. It uses cuBLAS for I8 x I8 to I32 GEMM and a CUDA kernel for scale restoration and bias addition. +- Vulkan and other GPU backends do not currently have dedicated INT8 convrot kernels. They use the backend scheduler to fall back to CPU, which is expected to be substantially slower than the CUDA path. + +LoRA adapters are applied at runtime without modifying the INT8 weights. The INT8 convrot path computes the base linear output, while LoRA, LoHa, LoKr, and raw weight-difference adapters compute their output corrections from the original, unrotated activation and add them to the base output. `--lora-apply-mode auto` selects this path for models containing INT8 tensorwise weights. If `immediately` is requested, sd.cpp falls back to runtime application because merging an adapter would require dequantizing and rotating its weight update, then recalculating the per-row scales and requantizing the result. + +The dedicated CUDA convrot activation path currently requires a group size of `256`; other supported group sizes use CPU execution. + +## Example + +ComfyUI INT8 convrot safetensors can be passed to `--diffusion-model` without conversion: + +```powershell +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\krea2_turbo_int8_convrot.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'" --steps 8 --cfg-scale 1 --diffusion-fa -v --offload-to-cpu +``` diff --git a/ggml b/ggml index eced84c86..6d8fbd43c 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit eced84c86f8b012c752c016f7fe789adea168e1e +Subproject commit 6d8fbd43c9b6400dcb8f51af216cef169365e5d4 diff --git a/src/core/ggml_extend.hpp b/src/core/ggml_extend.hpp index d14d2875b..b7f348f96 100644 --- a/src/core/ggml_extend.hpp +++ b/src/core/ggml_extend.hpp @@ -1039,6 +1039,38 @@ __STATIC_INLINE__ ggml_tensor* ggml_ext_linear(ggml_context* ctx, return x; } +__STATIC_INLINE__ ggml_tensor* ggml_ext_linear_i8_tensorwise(ggml_context* ctx, + ggml_tensor* x, + ggml_tensor* w, + ggml_tensor* weight_scale, + ggml_tensor* b, + int convrot_group_size, + float scale = 1.f) { + GGML_ASSERT(x->type == GGML_TYPE_F32 || (x->type == GGML_TYPE_I8 && scale == 1.f)); + if (scale != 1.f) { + x = ggml_ext_scale(ctx, x, scale); + } + + ggml_tensor* fused_bias = scale == 1.f ? b : nullptr; + if (x->ne[2] * x->ne[3] > 1024) { + int64_t ne2 = x->ne[2]; + int64_t ne3 = x->ne[3]; + x = ggml_reshape_2d(ctx, x, x->ne[0], x->ne[1] * x->ne[2] * x->ne[3]); + x = ggml_mul_mat_i8_tensorwise(ctx, w, x, weight_scale, fused_bias, convrot_group_size); + x = ggml_reshape_4d(ctx, x, x->ne[0], x->ne[1] / ne2 / ne3, ne2, ne3); + } else { + x = ggml_mul_mat_i8_tensorwise(ctx, w, x, weight_scale, fused_bias, convrot_group_size); + } + + if (scale != 1.f) { + x = ggml_ext_scale(ctx, x, 1.f / scale); + if (b != nullptr) { + x = ggml_add_inplace(ctx, x, b); + } + } + return x; +} + __STATIC_INLINE__ ggml_tensor* ggml_ext_pad_ext(ggml_context* ctx, ggml_backend_t backend, ggml_tensor* x, @@ -1679,6 +1711,13 @@ struct WeightAdapter { ggml_tensor* b, const std::string& prefix, ForwardParams forward_params) = 0; + virtual ggml_tensor* add_lora_to_output(ggml_context* ctx, + ggml_backend_t backend, + ggml_tensor* x, + ggml_tensor* w, + ggml_tensor* output, + const std::string& prefix, + ForwardParams forward_params) = 0; virtual size_t get_extra_graph_size() = 0; }; @@ -1696,6 +1735,7 @@ struct GGMLRunnerContext { std::function get_cache_tensor; std::function cache_tensor; std::function set_backend_tensor_data; + std::map, ggml_tensor*> int8_convrot_cache; void capture_tensor(const std::string& name, ggml_tensor* tensor) { if (debug_tensors == nullptr || tensor == nullptr) { @@ -1754,7 +1794,8 @@ struct GGMLRunner { std::vector extra_runtime_backends; // borrowed (SDBackendManager-owned) ggml_backend_sched_t sched = nullptr; // owned - ggml_backend_t cpu_fallback_backend = nullptr; // owned, sched requires a trailing CPU backend + size_t sched_graph_capacity = 0; + ggml_backend_t cpu_fallback_backend = nullptr; // owned, sched requires a trailing CPU backend bool multi_device_eval_callback_warned = false; std::shared_ptr weight_adapter = nullptr; @@ -2040,9 +2081,19 @@ struct GGMLRunner { // Pass explicit buffer types: synthesized defaults can make CUDA devices // report supporting each other's buffers and skip a required copy. bool ensure_sched(ggml_cgraph* gf) { - if (sched != nullptr) { + const size_t required_graph_size = gf != nullptr + ? std::max(1, + (size_t)ggml_graph_n_nodes(gf) + + sd::ggml_graph_cut::leaf_count(gf)) + : 1; + if (sched != nullptr && sched_graph_capacity >= required_graph_size) { return true; } + if (sched != nullptr) { + ggml_backend_sched_free(sched); + sched = nullptr; + sched_graph_capacity = 0; + } std::vector backends; backends.reserve(extra_runtime_backends.size() + 2); backends.push_back(runtime_backend); @@ -2070,20 +2121,17 @@ struct GGMLRunner { bufts.push_back(buft); } - size_t graph_size = MAX_GRAPH_SIZE; - if (gf != nullptr) { - graph_size = std::max(graph_size, (size_t)ggml_graph_n_nodes(gf)); - } sched = ggml_backend_sched_new(backends.data(), bufts.data(), (int)backends.size(), - graph_size, + required_graph_size, /*parallel=*/false, /*op_offload=*/false); if (sched == nullptr) { LOG_ERROR("%s: failed to create backend sched", get_desc().c_str()); return false; } + sched_graph_capacity = required_graph_size; return true; } @@ -3030,7 +3078,8 @@ struct GGMLRunner { } if (sched != nullptr) { ggml_backend_sched_free(sched); - sched = nullptr; + sched = nullptr; + sched_graph_capacity = 0; } } @@ -3356,14 +3405,18 @@ class Linear : public UnaryBlock { bool force_f32; bool force_prec_f32; bool allow_weight_scale; - bool has_weight_scale = false; + bool has_weight_scale = false; + bool int8_convrot = false; + int int8_convrot_group_size = 0; float scale; std::string prefix; void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override { - this->prefix = prefix; - has_weight_scale = false; - enum ggml_type wtype = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32); + this->prefix = prefix; + has_weight_scale = false; + int8_convrot = false; + int8_convrot_group_size = 0; + enum ggml_type wtype = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32); if (in_features % ggml_blck_size(wtype) != 0 || force_f32) { wtype = GGML_TYPE_F32; } @@ -3372,10 +3425,18 @@ class Linear : public UnaryBlock { enum ggml_type wtype = GGML_TYPE_F32; params["bias"] = ggml_new_tensor_1d(ctx, wtype, out_features); } - if (allow_weight_scale && tensor_storage_map.find(prefix + "weight_scale") != tensor_storage_map.end()) { + auto weight_storage = tensor_storage_map.find(prefix + "weight"); + const bool is_int8_tensorwise = weight_storage != tensor_storage_map.end() && weight_storage->second.is_int8_tensorwise; + if ((allow_weight_scale || is_int8_tensorwise) && tensor_storage_map.find(prefix + "weight_scale") != tensor_storage_map.end()) { params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, out_features); has_weight_scale = true; } + if (is_int8_tensorwise) { + GGML_ASSERT(wtype == GGML_TYPE_I8); + GGML_ASSERT(has_weight_scale); + int8_convrot = weight_storage->second.int8_convrot; + int8_convrot_group_size = weight_storage->second.int8_convrot_group_size; + } } public: @@ -3410,6 +3471,49 @@ class Linear : public UnaryBlock { } ggml_tensor* linear_bias = has_weight_scale ? nullptr : b; ggml_tensor* out = nullptr; + if (w->type == GGML_TYPE_I8) { + if (x->type != GGML_TYPE_F32) { + x = ggml_ext_cast_f32(ctx->ggml_ctx, ctx->backend, x); + } + if (!ggml_is_contiguous(x)) { + x = ggml_cont(ctx->ggml_ctx, x); + } + ggml_tensor* lora_input = x; + if (ctx->weight_adapter && b != nullptr) { + b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias"); + } + if (int8_convrot && scale == 1.f) { + const auto cache_key = std::make_pair(x, int8_convrot_group_size); + auto cached = ctx->int8_convrot_cache.find(cache_key); + if (cached == ctx->int8_convrot_cache.end()) { + x = ggml_quantize_i8_convrot(ctx->ggml_ctx, x, int8_convrot_group_size); + ctx->int8_convrot_cache.emplace(cache_key, x); + } else { + x = cached->second; + } + } + out = ggml_ext_linear_i8_tensorwise(ctx->ggml_ctx, + x, + w, + params["weight_scale"], + b, + int8_convrot ? int8_convrot_group_size : 0, + scale); + if (ctx->weight_adapter) { + WeightAdapter::ForwardParams forward_params; + forward_params.op_type = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR; + forward_params.linear.force_prec_f32 = force_prec_f32; + forward_params.linear.scale = scale; + out = ctx->weight_adapter->add_lora_to_output(ctx->ggml_ctx, + ctx->backend, + lora_input, + w, + out, + prefix, + forward_params); + } + return out; + } if (ctx->weight_adapter) { WeightAdapter::ForwardParams forward_params; forward_params.op_type = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR; diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp index 8ddfca24f..8eb88d543 100644 --- a/src/model/adapter/lora.hpp +++ b/src/model/adapter/lora.hpp @@ -1072,6 +1072,34 @@ struct MultiLoraAdapter : public WeightAdapter { return out; } + ggml_tensor* add_lora_to_output(ggml_context* ctx, + ggml_backend_t backend, + ggml_tensor* x, + ggml_tensor* w, + ggml_tensor* output, + const std::string& prefix, + WeightAdapter::ForwardParams forward_params) override { + for (auto& lora_model : lora_models) { + ggml_tensor* weight_diff = lora_model->get_weight_diff(prefix + "weight", backend, ctx, w, false); + if (weight_diff != nullptr) { + GGML_ASSERT(forward_params.op_type == ForwardParams::op_type_t::OP_LINEAR); + ggml_tensor* out_diff = ggml_ext_linear(ctx, + x, + weight_diff, + nullptr, + forward_params.linear.force_prec_f32, + forward_params.linear.scale); + output = ggml_add_inplace(ctx, output, out_diff); + } + + ggml_tensor* out_diff = lora_model->get_out_diff(ctx, backend, x, w, forward_params, prefix + "weight"); + if (out_diff != nullptr) { + output = ggml_add_inplace(ctx, output, out_diff); + } + } + return output; + } + size_t get_extra_graph_size() override { size_t lora_tensor_num = 0; for (auto& lora_model : lora_models) { diff --git a/src/model_io/safetensors_io.cpp b/src/model_io/safetensors_io.cpp index df71eab11..69bcaa1ec 100644 --- a/src/model_io/safetensors_io.cpp +++ b/src/model_io/safetensors_io.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -93,10 +94,52 @@ static ggml_type safetensors_dtype_to_ggml_type(const std::string& dtype) { ttype = GGML_TYPE_I32; } else if (dtype == "I64") { ttype = GGML_TYPE_I32; + } else if (dtype == "I8") { + ttype = GGML_TYPE_I8; } return ttype; } +struct ComfyQuantConfig { + std::string format; + bool convrot = false; + int group_size = 0; +}; + +static bool read_comfy_quant_config(std::ifstream& file, + const std::string& file_path, + const std::string& tensor_name, + size_t offset, + size_t size, + ComfyQuantConfig& config, + std::string* error) { + static constexpr size_t MAX_COMFY_QUANT_CONFIG_SIZE = 64 * 1024; + if (size == 0 || size > MAX_COMFY_QUANT_CONFIG_SIZE) { + set_error(error, "invalid ComfyUI quantization metadata tensor '" + tensor_name + "' in '" + file_path + "'"); + return false; + } + + std::vector data(size + 1, '\0'); + file.clear(); + file.seekg((std::streamoff)offset, std::ios::beg); + file.read(data.data(), (std::streamsize)size); + if (!file) { + set_error(error, "read ComfyUI quantization metadata tensor failed: '" + tensor_name + "'"); + return false; + } + + try { + const nlohmann::json json = nlohmann::json::parse(data.data(), data.data() + size); + config.format = json.value("format", ""); + config.convrot = json.value("convrot", false); + config.group_size = json.value("convrot_groupsize", 0); + } catch (const std::exception&) { + set_error(error, "parsing ComfyUI quantization metadata tensor failed: '" + tensor_name + "'"); + return false; + } + return true; +} + // https://huggingface.co/docs/safetensors/index bool read_safetensors_file(const std::string& file_path, std::vector& tensor_storages, @@ -163,6 +206,33 @@ bool read_safetensors_file(const std::string& file_path, } } + std::unordered_map comfy_quant_configs; + for (const auto& item : header_.items()) { + const std::string& name = item.key(); + if (name == "__metadata__" || !ends_with(name, ".comfy_quant")) { + continue; + } + + const nlohmann::json& tensor_info = item.value(); + if (tensor_info.value("dtype", "") != "U8") { + set_error(error, "invalid dtype for ComfyUI quantization metadata tensor '" + name + "'"); + return false; + } + const size_t begin = tensor_info["data_offsets"][0].get(); + const size_t end = tensor_info["data_offsets"][1].get(); + if (begin > end || end > file_size_ - data_start) { + set_error(error, "data offsets out of bounds for tensor '" + name + "'"); + return false; + } + + ComfyQuantConfig config; + if (!read_comfy_quant_config(file, file_path, name, data_start + begin, end - begin, config, error)) { + return false; + } + const std::string module_name = name.substr(0, name.size() - std::string(".comfy_quant").size()); + comfy_quant_configs.emplace(module_name, std::move(config)); + } + tensor_storages.clear(); for (auto& item : header_.items()) { std::string name = item.key(); @@ -220,6 +290,39 @@ bool read_safetensors_file(const std::string& file_path, TensorStorage tensor_storage(name, type, ne, n_dims, 0, data_start + begin); tensor_storage.reverse_ne(); + if (ends_with(name, ".weight")) { + const std::string module_name = name.substr(0, name.size() - std::string(".weight").size()); + auto config = comfy_quant_configs.find(module_name); + if (config != comfy_quant_configs.end() && config->second.format == "int8_tensorwise") { + if (type != GGML_TYPE_I8) { + set_error(error, "ComfyUI int8_tensorwise weight is not I8: '" + name + "'"); + return false; + } + if (config->second.convrot) { + int group_size_remainder = config->second.group_size; + while (group_size_remainder > 1 && group_size_remainder % 4 == 0) { + group_size_remainder /= 4; + } + if (group_size_remainder != 1 || tensor_storage.ne[0] % config->second.group_size != 0) { + set_error(error, "invalid ComfyUI convrot group size for tensor '" + name + "'"); + return false; + } + } + tensor_storage.is_int8_tensorwise = true; + tensor_storage.int8_convrot = config->second.convrot; + tensor_storage.int8_convrot_group_size = config->second.group_size; + } + } else if (ends_with(name, ".weight_scale")) { + const std::string module_name = name.substr(0, name.size() - std::string(".weight_scale").size()); + auto config = comfy_quant_configs.find(module_name); + if (config != comfy_quant_configs.end() && config->second.format == "int8_tensorwise" && + tensor_storage.n_dims == 2 && tensor_storage.ne[0] == 1) { + tensor_storage.ne[0] = tensor_storage.ne[1]; + tensor_storage.ne[1] = 1; + tensor_storage.n_dims = 1; + } + } + size_t tensor_data_size = end - begin; bool tensor_size_ok; diff --git a/src/model_io/tensor_storage.h b/src/model_io/tensor_storage.h index 307535a53..5672c9437 100644 --- a/src/model_io/tensor_storage.h +++ b/src/model_io/tensor_storage.h @@ -15,14 +15,17 @@ struct TensorStorage { std::string name; - ggml_type type = GGML_TYPE_F32; - ggml_type expected_type = GGML_TYPE_COUNT; - bool is_f8_e4m3 = false; - bool is_f8_e5m2 = false; - bool is_f64 = false; - bool is_i64 = false; - int64_t ne[SD_MAX_DIMS] = {1, 1, 1, 1, 1}; - int n_dims = 0; + ggml_type type = GGML_TYPE_F32; + ggml_type expected_type = GGML_TYPE_COUNT; + bool is_f8_e4m3 = false; + bool is_f8_e5m2 = false; + bool is_f64 = false; + bool is_i64 = false; + bool is_int8_tensorwise = false; + bool int8_convrot = false; + int int8_convrot_group_size = 0; + int64_t ne[SD_MAX_DIMS] = {1, 1, 1, 1, 1}; + int n_dims = 0; std::string storage_key; size_t file_index = 0; diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 047c9d996..891a41736 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -1508,6 +1508,9 @@ bool ModelLoader::load_tensors(std::map& tensors, bool ModelLoader::tensor_should_be_converted(const TensorStorage& tensor_storage, ggml_type type) { const std::string& name = tensor_storage.name; + if (tensor_storage.is_int8_tensorwise) { + return false; + } if (type != GGML_TYPE_COUNT) { if (ggml_is_quantized(type) && tensor_storage.ne[0] % ggml_blck_size(type) != 0) { // Pass, do not convert diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index bdb8a968b..9cb3b0f27 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -960,8 +960,16 @@ class StableDiffusionGGML { LOG_DEBUG("ggml tensor size = %d bytes", (int)sizeof(ggml_tensor)); + bool have_int8_tensorwise = false; + for (const auto& [_, tensor_storage] : model_loader.get_tensor_storage_map()) { + if (tensor_storage.is_int8_tensorwise) { + have_int8_tensorwise = true; + break; + } + } + if (sd_ctx_params->lora_apply_mode == LORA_APPLY_AUTO) { - bool have_quantized_weight = false; + bool have_quantized_weight = have_int8_tensorwise; for (const auto& [type, _] : wtype_stat) { if (ggml_is_quantized(type)) { have_quantized_weight = true; @@ -977,12 +985,19 @@ class StableDiffusionGGML { apply_lora_immediately = true; } } else if (sd_ctx_params->lora_apply_mode == LORA_APPLY_IMMEDIATELY) { - if (row_split_active()) { + if (have_int8_tensorwise) { + LOG_WARN( + "INT8 tensorwise weights do not support the immediately LoRA apply mode; " + "using at_runtime instead"); + apply_lora_immediately = false; + } else if (row_split_active()) { LOG_WARN( "row-split tensors do not support the immediately LoRA apply mode; " "LoRAs will not be applied to them (use --lora-apply-mode at_runtime)"); + apply_lora_immediately = false; + } else { + apply_lora_immediately = true; } - apply_lora_immediately = true; } else { apply_lora_immediately = false; } From 36ea7424642cf2b428a618356d154318e0366662 Mon Sep 17 00:00:00 2001 From: leejet Date: Thu, 6 Aug 2026 01:43:38 +0800 Subject: [PATCH 2/2] fix HIP build for INT8 convrot --- ggml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml b/ggml index 6d8fbd43c..7fa9a07cf 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit 6d8fbd43c9b6400dcb8f51af216cef169365e5d4 +Subproject commit 7fa9a07cf89771317377dad7c358ec4c0c2331cf