From bad009db69f0e6c65711ad6903e02f2ae9e76784 Mon Sep 17 00:00:00 2001 From: Aditya Gupta <84338325+MaiAditya@users.noreply.github.com> Date: Mon, 16 Nov 2026 07:56:03 +0530 Subject: [PATCH 1/3] [LoRA] Only swap lora_B in final_layer scale/shift remaps (Flux Control, HunyuanVideo) The original final layer orders its modulation (shift, scale) and diffusers' norm_out (scale, shift). The Flux Control and HunyuanVideo converters swapped the halves of lora_A as well as lora_B, which scrambles lora_A's rank dimension and gives a wrong delta. Swap only lora_B, as the kohya Flux converter already does. --- .../loaders/lora_conversion_utils.py | 14 +++++--- .../test_models_transformer_flux.py | 34 +++++++++++++++++++ .../test_models_transformer_hunyuan_video.py | 20 ++++++++++- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/diffusers/loaders/lora_conversion_utils.py b/src/diffusers/loaders/lora_conversion_utils.py index b6f626842ee2..e7eb4287076a 100644 --- a/src/diffusers/loaders/lora_conversion_utils.py +++ b/src/diffusers/loaders/lora_conversion_utils.py @@ -1352,11 +1352,14 @@ def _convert_bfl_flux_control_lora_to_diffusers(original_state_dict): f"final_layer.linear.{lora_key}.bias" ) - converted_state_dict[f"norm_out.linear.{lora_key}.weight"] = swap_scale_shift( + # The original final layer orders its modulation (shift, scale), diffusers' `norm_out` (scale, shift). Only + # lora_B's rows are the layer's outputs; lora_A's rows are the rank dimension and must stay as they are. + maybe_swap = swap_scale_shift if lora_key == "lora_B" else (lambda weight: weight) + converted_state_dict[f"norm_out.linear.{lora_key}.weight"] = maybe_swap( original_state_dict.pop(f"final_layer.adaLN_modulation.1.{lora_key}.weight") ) if f"final_layer.adaLN_modulation.1.{lora_key}.bias" in original_state_dict_keys: - converted_state_dict[f"norm_out.linear.{lora_key}.bias"] = swap_scale_shift( + converted_state_dict[f"norm_out.linear.{lora_key}.bias"] = maybe_swap( original_state_dict.pop(f"final_layer.adaLN_modulation.1.{lora_key}.bias") ) @@ -1596,9 +1599,10 @@ def _convert_hunyuan_video_lora_to_diffusers(original_state_dict): def remap_norm_scale_shift_(key, state_dict): weight = state_dict.pop(key) - shift, scale = weight.chunk(2, dim=0) - new_weight = torch.cat([scale, shift], dim=0) - state_dict[key.replace("final_layer.adaLN_modulation.1", "norm_out.linear")] = new_weight + # Only lora_B's rows are the layer's (shift, scale) outputs; lora_A's rows are the rank dimension. + if "lora_B" in key: + weight = swap_scale_shift(weight) + state_dict[key.replace("final_layer.adaLN_modulation.1", "norm_out.linear")] = weight def remap_txt_in_(key, state_dict): def rename_key(key): diff --git a/tests/models/transformers/test_models_transformer_flux.py b/tests/models/transformers/test_models_transformer_flux.py index 827987cf47c8..6897e2e05bbb 100644 --- a/tests/models/transformers/test_models_transformer_flux.py +++ b/tests/models/transformers/test_models_transformer_flux.py @@ -20,6 +20,7 @@ import torch from diffusers import BitsAndBytesConfig, FluxTransformer2DModel, GGUFQuantizationConfig +from diffusers.loaders import FluxLoraLoaderMixin from diffusers.models.embeddings import ImageProjection from diffusers.models.transformers.transformer_flux import FluxIPAdapterAttnProcessor from diffusers.utils.torch_utils import randn_tensor @@ -311,6 +312,39 @@ def create_ip_adapter_state_dict(self, model: Any) -> dict[str, dict[str, Any]]: class TestFluxTransformerLoRA(FluxTransformerTesterConfig, LoraTesterMixin): """LoRA adapter tests for Flux Transformer.""" + def test_lora_bfl_control_final_layer_modulation(self): + # The original final layer orders its modulation (shift, scale) and diffusers' `norm_out` (scale, shift). Only + # lora_B's rows are the layer's outputs, so the converted update must be the original one with its halves swapped. + rank, inner_dim, mlp_dim = 1, 3072, 4 * 3072 + modules = ["img_in", "txt_in", "time_in.in_layer", "time_in.out_layer", "vector_in.in_layer"] + modules += ["vector_in.out_layer", "final_layer.linear"] + for i in range(19): + for stream in ("img", "txt"): + modules += [f"double_blocks.{i}.{stream}_{name}" for name in ("mod.lin", "attn.qkv", "attn.proj")] + modules += [f"double_blocks.{i}.{stream}_mlp.{j}" for j in (0, 2)] + for i in range(38): + modules += [f"single_blocks.{i}.{name}" for name in ("modulation.lin", "linear1", "linear2")] + state_dict = {} + for module in modules: + state_dict[f"{module}.lora_A.weight"] = torch.zeros(rank, 1) + state_dict[f"{module}.lora_B.weight"] = torch.zeros(3 * inner_dim + mlp_dim, rank) + norms = [f"double_blocks.{i}.{s}_attn.norm" for i in range(19) for s in ("img", "txt")] + norms += [f"single_blocks.{i}.norm" for i in range(38)] + for norm in norms: + state_dict[f"{norm}.query_norm.scale"] = state_dict[f"{norm}.key_norm.scale"] = torch.ones(1) + + torch.manual_seed(0) + lora_A, lora_B = torch.randn(2, 16), torch.randn(32, 2) + state_dict["final_layer.adaLN_modulation.1.lora_A.weight"] = lora_A + state_dict["final_layer.adaLN_modulation.1.lora_B.weight"] = lora_B + converted = FluxLoraLoaderMixin.lora_state_dict(state_dict) + delta = ( + converted["transformer.norm_out.linear.lora_B.weight"] + @ converted["transformer.norm_out.linear.lora_A.weight"] + ) + shift, scale = (lora_B @ lora_A).chunk(2, dim=0) + assert torch.allclose(delta, torch.cat([scale, shift])) + class TestFluxTransformerLoRAHotSwap(FluxTransformerTesterConfig, LoraHotSwappingForModelTesterMixin): """LoRA hot-swapping tests for Flux Transformer.""" diff --git a/tests/models/transformers/test_models_transformer_hunyuan_video.py b/tests/models/transformers/test_models_transformer_hunyuan_video.py index bb51ed71e008..3324f8262c20 100644 --- a/tests/models/transformers/test_models_transformer_hunyuan_video.py +++ b/tests/models/transformers/test_models_transformer_hunyuan_video.py @@ -15,6 +15,7 @@ import torch from diffusers import HunyuanVideoTransformer3DModel +from diffusers.loaders import HunyuanVideoLoraLoaderMixin from diffusers.utils.torch_utils import randn_tensor from ...testing_utils import enable_full_determinism, torch_device @@ -305,4 +306,21 @@ def test_output(self, base_model_output): class TestHunyuanVideoTransformerLoRA(HunyuanVideoTransformerTesterConfig, LoraTesterMixin): - pass + def test_lora_original_final_layer_modulation(self): + # The original final layer orders its modulation (shift, scale) and diffusers' `norm_out` (scale, shift). Only + # lora_B's rows are the layer's outputs, so the converted update must be the original one with its halves swapped. + torch.manual_seed(0) + lora_A, lora_B = torch.randn(2, 16), torch.randn(32, 2) + state_dict = { + "diffusion_model.final_layer.adaLN_modulation.1.lora_A.weight": lora_A, + "diffusion_model.final_layer.adaLN_modulation.1.lora_B.weight": lora_B, + "diffusion_model.double_blocks.0.img_attn_qkv.lora_A.weight": torch.randn(2, 16), + "diffusion_model.double_blocks.0.img_attn_qkv.lora_B.weight": torch.randn(48, 2), + } + converted = HunyuanVideoLoraLoaderMixin.lora_state_dict(state_dict) + delta = ( + converted["transformer.norm_out.linear.lora_B.weight"] + @ converted["transformer.norm_out.linear.lora_A.weight"] + ) + shift, scale = (lora_B @ lora_A).chunk(2, dim=0) + assert torch.allclose(delta, torch.cat([scale, shift])) From 30ebd63ab7244e3f99717ab7a1fb83a565c22803 Mon Sep 17 00:00:00 2001 From: Aditya Gupta <84338325+MaiAditya@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:15:41 +0530 Subject: [PATCH 2/3] Use minimal tensors in the Flux Control final layer LoRA test --- .../transformers/test_models_transformer_flux.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/models/transformers/test_models_transformer_flux.py b/tests/models/transformers/test_models_transformer_flux.py index 6897e2e05bbb..7f1297ca1a39 100644 --- a/tests/models/transformers/test_models_transformer_flux.py +++ b/tests/models/transformers/test_models_transformer_flux.py @@ -315,7 +315,9 @@ class TestFluxTransformerLoRA(FluxTransformerTesterConfig, LoraTesterMixin): def test_lora_bfl_control_final_layer_modulation(self): # The original final layer orders its modulation (shift, scale) and diffusers' `norm_out` (scale, shift). Only # lora_B's rows are the layer's outputs, so the converted update must be the original one with its halves swapped. - rank, inner_dim, mlp_dim = 1, 3072, 4 * 3072 + # `_convert_bfl_flux_control_lora_to_diffusers` expects every block of the 19 double / 38 single block model and + # raises on missing or leftover keys, so all blocks are present. Their LoRAs are minimal zero tensors; only + # `linear1` must have the full (q, k, v, mlp) output size to be split, so it is an expanded view of one element. modules = ["img_in", "txt_in", "time_in.in_layer", "time_in.out_layer", "vector_in.in_layer"] modules += ["vector_in.out_layer", "final_layer.linear"] for i in range(19): @@ -326,8 +328,10 @@ def test_lora_bfl_control_final_layer_modulation(self): modules += [f"single_blocks.{i}.{name}" for name in ("modulation.lin", "linear1", "linear2")] state_dict = {} for module in modules: - state_dict[f"{module}.lora_A.weight"] = torch.zeros(rank, 1) - state_dict[f"{module}.lora_B.weight"] = torch.zeros(3 * inner_dim + mlp_dim, rank) + state_dict[f"{module}.lora_A.weight"] = torch.zeros(1, 1) + state_dict[f"{module}.lora_B.weight"] = torch.zeros(3, 1) + for i in range(38): + state_dict[f"single_blocks.{i}.linear1.lora_B.weight"] = torch.zeros(1, 1).expand(3 * 3072 + 4 * 3072, 1) norms = [f"double_blocks.{i}.{s}_attn.norm" for i in range(19) for s in ("img", "txt")] norms += [f"single_blocks.{i}.norm" for i in range(38)] for norm in norms: From 3f34c58a00f2b459d128a0d02fe6eb04063c12b2 Mon Sep 17 00:00:00 2001 From: Aditya Gupta <84338325+MaiAditya@users.noreply.github.com> Date: Tue, 17 Nov 2026 10:34:42 +0530 Subject: [PATCH 3/3] Take the Flux Control LoRA block counts from the state dict and shrink its test --- .../loaders/lora_conversion_utils.py | 9 ++++++-- .../test_models_transformer_flux.py | 22 ++++--------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/diffusers/loaders/lora_conversion_utils.py b/src/diffusers/loaders/lora_conversion_utils.py index e7eb4287076a..236f0ee7c942 100644 --- a/src/diffusers/loaders/lora_conversion_utils.py +++ b/src/diffusers/loaders/lora_conversion_utils.py @@ -1071,8 +1071,13 @@ def _custom_replace(key: str, substrings: list[str]) -> str: def _convert_bfl_flux_control_lora_to_diffusers(original_state_dict): converted_state_dict = {} original_state_dict_keys = list(original_state_dict.keys()) - num_layers = 19 - num_single_layers = 38 + num_layers = 0 + num_single_layers = 0 + for key in original_state_dict_keys: + if key.startswith("single_blocks."): + num_single_layers = max(num_single_layers, int(key.split(".")[1]) + 1) + elif key.startswith("double_blocks."): + num_layers = max(num_layers, int(key.split(".")[1]) + 1) inner_dim = 3072 mlp_ratio = 4.0 diff --git a/tests/models/transformers/test_models_transformer_flux.py b/tests/models/transformers/test_models_transformer_flux.py index 7f1297ca1a39..d4801e214436 100644 --- a/tests/models/transformers/test_models_transformer_flux.py +++ b/tests/models/transformers/test_models_transformer_flux.py @@ -20,7 +20,7 @@ import torch from diffusers import BitsAndBytesConfig, FluxTransformer2DModel, GGUFQuantizationConfig -from diffusers.loaders import FluxLoraLoaderMixin +from diffusers.loaders.lora_conversion_utils import _convert_bfl_flux_control_lora_to_diffusers from diffusers.models.embeddings import ImageProjection from diffusers.models.transformers.transformer_flux import FluxIPAdapterAttnProcessor from diffusers.utils.torch_utils import randn_tensor @@ -315,33 +315,19 @@ class TestFluxTransformerLoRA(FluxTransformerTesterConfig, LoraTesterMixin): def test_lora_bfl_control_final_layer_modulation(self): # The original final layer orders its modulation (shift, scale) and diffusers' `norm_out` (scale, shift). Only # lora_B's rows are the layer's outputs, so the converted update must be the original one with its halves swapped. - # `_convert_bfl_flux_control_lora_to_diffusers` expects every block of the 19 double / 38 single block model and - # raises on missing or leftover keys, so all blocks are present. Their LoRAs are minimal zero tensors; only - # `linear1` must have the full (q, k, v, mlp) output size to be split, so it is an expanded view of one element. + # The converter takes the number of blocks from the keys, so the state dict only needs the non-block layers. modules = ["img_in", "txt_in", "time_in.in_layer", "time_in.out_layer", "vector_in.in_layer"] modules += ["vector_in.out_layer", "final_layer.linear"] - for i in range(19): - for stream in ("img", "txt"): - modules += [f"double_blocks.{i}.{stream}_{name}" for name in ("mod.lin", "attn.qkv", "attn.proj")] - modules += [f"double_blocks.{i}.{stream}_mlp.{j}" for j in (0, 2)] - for i in range(38): - modules += [f"single_blocks.{i}.{name}" for name in ("modulation.lin", "linear1", "linear2")] state_dict = {} for module in modules: state_dict[f"{module}.lora_A.weight"] = torch.zeros(1, 1) - state_dict[f"{module}.lora_B.weight"] = torch.zeros(3, 1) - for i in range(38): - state_dict[f"single_blocks.{i}.linear1.lora_B.weight"] = torch.zeros(1, 1).expand(3 * 3072 + 4 * 3072, 1) - norms = [f"double_blocks.{i}.{s}_attn.norm" for i in range(19) for s in ("img", "txt")] - norms += [f"single_blocks.{i}.norm" for i in range(38)] - for norm in norms: - state_dict[f"{norm}.query_norm.scale"] = state_dict[f"{norm}.key_norm.scale"] = torch.ones(1) + state_dict[f"{module}.lora_B.weight"] = torch.zeros(1, 1) torch.manual_seed(0) lora_A, lora_B = torch.randn(2, 16), torch.randn(32, 2) state_dict["final_layer.adaLN_modulation.1.lora_A.weight"] = lora_A state_dict["final_layer.adaLN_modulation.1.lora_B.weight"] = lora_B - converted = FluxLoraLoaderMixin.lora_state_dict(state_dict) + converted = _convert_bfl_flux_control_lora_to_diffusers(state_dict) delta = ( converted["transformer.norm_out.linear.lora_B.weight"] @ converted["transformer.norm_out.linear.lora_A.weight"]