diff --git a/src/diffusers/loaders/lora_conversion_utils.py b/src/diffusers/loaders/lora_conversion_utils.py index b6f626842ee2..236f0ee7c942 100644 --- a/src/diffusers/loaders/lora_conversion_utils.py +++ b/src/diffusers/loaders/lora_conversion_utils.py @@ -1071,8 +1071,13 @@ def _custom_replace(key: str, substrings: list[str]) -> str: def _convert_bfl_flux_control_lora_to_diffusers(original_state_dict): converted_state_dict = {} original_state_dict_keys = list(original_state_dict.keys()) - num_layers = 19 - num_single_layers = 38 + num_layers = 0 + num_single_layers = 0 + for key in original_state_dict_keys: + if key.startswith("single_blocks."): + num_single_layers = max(num_single_layers, int(key.split(".")[1]) + 1) + elif key.startswith("double_blocks."): + num_layers = max(num_layers, int(key.split(".")[1]) + 1) inner_dim = 3072 mlp_ratio = 4.0 @@ -1352,11 +1357,14 @@ def _convert_bfl_flux_control_lora_to_diffusers(original_state_dict): f"final_layer.linear.{lora_key}.bias" ) - converted_state_dict[f"norm_out.linear.{lora_key}.weight"] = swap_scale_shift( + # The original final layer orders its modulation (shift, scale), diffusers' `norm_out` (scale, shift). Only + # lora_B's rows are the layer's outputs; lora_A's rows are the rank dimension and must stay as they are. + maybe_swap = swap_scale_shift if lora_key == "lora_B" else (lambda weight: weight) + converted_state_dict[f"norm_out.linear.{lora_key}.weight"] = maybe_swap( original_state_dict.pop(f"final_layer.adaLN_modulation.1.{lora_key}.weight") ) if f"final_layer.adaLN_modulation.1.{lora_key}.bias" in original_state_dict_keys: - converted_state_dict[f"norm_out.linear.{lora_key}.bias"] = swap_scale_shift( + converted_state_dict[f"norm_out.linear.{lora_key}.bias"] = maybe_swap( original_state_dict.pop(f"final_layer.adaLN_modulation.1.{lora_key}.bias") ) @@ -1596,9 +1604,10 @@ def _convert_hunyuan_video_lora_to_diffusers(original_state_dict): def remap_norm_scale_shift_(key, state_dict): weight = state_dict.pop(key) - shift, scale = weight.chunk(2, dim=0) - new_weight = torch.cat([scale, shift], dim=0) - state_dict[key.replace("final_layer.adaLN_modulation.1", "norm_out.linear")] = new_weight + # Only lora_B's rows are the layer's (shift, scale) outputs; lora_A's rows are the rank dimension. + if "lora_B" in key: + weight = swap_scale_shift(weight) + state_dict[key.replace("final_layer.adaLN_modulation.1", "norm_out.linear")] = weight def remap_txt_in_(key, state_dict): def rename_key(key): diff --git a/tests/models/transformers/test_models_transformer_flux.py b/tests/models/transformers/test_models_transformer_flux.py index 827987cf47c8..d4801e214436 100644 --- a/tests/models/transformers/test_models_transformer_flux.py +++ b/tests/models/transformers/test_models_transformer_flux.py @@ -20,6 +20,7 @@ import torch from diffusers import BitsAndBytesConfig, FluxTransformer2DModel, GGUFQuantizationConfig +from diffusers.loaders.lora_conversion_utils import _convert_bfl_flux_control_lora_to_diffusers from diffusers.models.embeddings import ImageProjection from diffusers.models.transformers.transformer_flux import FluxIPAdapterAttnProcessor from diffusers.utils.torch_utils import randn_tensor @@ -311,6 +312,29 @@ def create_ip_adapter_state_dict(self, model: Any) -> dict[str, dict[str, Any]]: class TestFluxTransformerLoRA(FluxTransformerTesterConfig, LoraTesterMixin): """LoRA adapter tests for Flux Transformer.""" + def test_lora_bfl_control_final_layer_modulation(self): + # The original final layer orders its modulation (shift, scale) and diffusers' `norm_out` (scale, shift). Only + # lora_B's rows are the layer's outputs, so the converted update must be the original one with its halves swapped. + # The converter takes the number of blocks from the keys, so the state dict only needs the non-block layers. + modules = ["img_in", "txt_in", "time_in.in_layer", "time_in.out_layer", "vector_in.in_layer"] + modules += ["vector_in.out_layer", "final_layer.linear"] + state_dict = {} + for module in modules: + state_dict[f"{module}.lora_A.weight"] = torch.zeros(1, 1) + state_dict[f"{module}.lora_B.weight"] = torch.zeros(1, 1) + + torch.manual_seed(0) + lora_A, lora_B = torch.randn(2, 16), torch.randn(32, 2) + state_dict["final_layer.adaLN_modulation.1.lora_A.weight"] = lora_A + state_dict["final_layer.adaLN_modulation.1.lora_B.weight"] = lora_B + converted = _convert_bfl_flux_control_lora_to_diffusers(state_dict) + delta = ( + converted["transformer.norm_out.linear.lora_B.weight"] + @ converted["transformer.norm_out.linear.lora_A.weight"] + ) + shift, scale = (lora_B @ lora_A).chunk(2, dim=0) + assert torch.allclose(delta, torch.cat([scale, shift])) + class TestFluxTransformerLoRAHotSwap(FluxTransformerTesterConfig, LoraHotSwappingForModelTesterMixin): """LoRA hot-swapping tests for Flux Transformer.""" diff --git a/tests/models/transformers/test_models_transformer_hunyuan_video.py b/tests/models/transformers/test_models_transformer_hunyuan_video.py index bb51ed71e008..3324f8262c20 100644 --- a/tests/models/transformers/test_models_transformer_hunyuan_video.py +++ b/tests/models/transformers/test_models_transformer_hunyuan_video.py @@ -15,6 +15,7 @@ import torch from diffusers import HunyuanVideoTransformer3DModel +from diffusers.loaders import HunyuanVideoLoraLoaderMixin from diffusers.utils.torch_utils import randn_tensor from ...testing_utils import enable_full_determinism, torch_device @@ -305,4 +306,21 @@ def test_output(self, base_model_output): class TestHunyuanVideoTransformerLoRA(HunyuanVideoTransformerTesterConfig, LoraTesterMixin): - pass + def test_lora_original_final_layer_modulation(self): + # The original final layer orders its modulation (shift, scale) and diffusers' `norm_out` (scale, shift). Only + # lora_B's rows are the layer's outputs, so the converted update must be the original one with its halves swapped. + torch.manual_seed(0) + lora_A, lora_B = torch.randn(2, 16), torch.randn(32, 2) + state_dict = { + "diffusion_model.final_layer.adaLN_modulation.1.lora_A.weight": lora_A, + "diffusion_model.final_layer.adaLN_modulation.1.lora_B.weight": lora_B, + "diffusion_model.double_blocks.0.img_attn_qkv.lora_A.weight": torch.randn(2, 16), + "diffusion_model.double_blocks.0.img_attn_qkv.lora_B.weight": torch.randn(48, 2), + } + converted = HunyuanVideoLoraLoaderMixin.lora_state_dict(state_dict) + delta = ( + converted["transformer.norm_out.linear.lora_B.weight"] + @ converted["transformer.norm_out.linear.lora_A.weight"] + ) + shift, scale = (lora_B @ lora_A).chunk(2, dim=0) + assert torch.allclose(delta, torch.cat([scale, shift]))