diff --git a/src/diffusers/loaders/lora_conversion_utils.py b/src/diffusers/loaders/lora_conversion_utils.py index b6f626842ee2..da9b0486bc71 100644 --- a/src/diffusers/loaders/lora_conversion_utils.py +++ b/src/diffusers/loaders/lora_conversion_utils.py @@ -1766,6 +1766,52 @@ def remap_single_transformer_blocks_(key, state_dict): return converted_state_dict +def _convert_musubi_hunyuan_video_lora_to_diffusers(state_dict): + # https://github.com/kohya-ss/musubi-tuner + # Keys look like `lora_unet_double_blocks_0_img_attn_qkv.lora_down.weight`. Rewrite them into the original + # HunyuanVideo layout (`double_blocks.0.img_attn_qkv.lora_A.weight`), fold `alpha / rank` into the weights and + # reuse `_convert_hunyuan_video_lora_to_diffusers` for the rest. + module_renames = { + "img_mod_linear": "img_mod.linear", + "txt_mod_linear": "txt_mod.linear", + "img_mlp_fc1": "img_mlp.fc1", + "img_mlp_fc2": "img_mlp.fc2", + "txt_mlp_fc1": "txt_mlp.fc1", + "txt_mlp_fc2": "txt_mlp.fc2", + "modulation_linear": "modulation.linear", + } + original_state_dict = {k[len("lora_unet_") :]: v for k, v in state_dict.items()} + converted_state_dict = {} + + for key in [k for k in original_state_dict if k.endswith(".lora_down.weight")]: + module_key = key[: -len(".lora_down.weight")] + match = re.fullmatch(r"(double_blocks|single_blocks)_(\d+)_(\w+)", module_key) + if match is None: + continue + block_type, block_idx, module_name = match.groups() + module_name = module_renames.get(module_name, module_name) + + down_weight = original_state_dict.pop(f"{module_key}.lora_down.weight") + up_weight = original_state_dict.pop(f"{module_key}.lora_up.weight") + alpha = original_state_dict.pop(f"{module_key}.alpha").item() + # LoRA is scaled by `alpha / rank` in the forward pass, so fold that scale into the weights. + scale = alpha / down_weight.shape[0] + scale_down = scale + scale_up = 1.0 + while scale_down * 2 < scale_up: + scale_down *= 2 + scale_up /= 2 + + new_key = f"{block_type}.{block_idx}.{module_name}" + converted_state_dict[f"{new_key}.lora_A.weight"] = down_weight * scale_down + converted_state_dict[f"{new_key}.lora_B.weight"] = up_weight * scale_up + + if len(original_state_dict) > 0: + raise ValueError(f"`state_dict` should be empty at this point but has {original_state_dict.keys()=}") + + return _convert_hunyuan_video_lora_to_diffusers(converted_state_dict) + + def _convert_non_diffusers_lumina2_lora_to_diffusers(state_dict): # Remove "diffusion_model." prefix from keys. state_dict = {k[len("diffusion_model.") :]: v for k, v in state_dict.items()} diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 43d455551f26..cf1849ec49a3 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -46,6 +46,7 @@ _convert_kohya_flux2_lora_to_diffusers, _convert_kohya_flux_lora_to_diffusers, _convert_lycoris_flux2_lokr_to_diffusers, + _convert_musubi_hunyuan_video_lora_to_diffusers, _convert_musubi_wan_lora_to_diffusers, _convert_non_diffusers_ace_step_lora_to_diffusers, _convert_non_diffusers_anima_lora_to_diffusers, @@ -3571,7 +3572,9 @@ def lora_state_dict( state_dict = {k: v for k, v in state_dict.items() if "dora_scale" not in k} is_original_hunyuan_video = any("img_attn_qkv" in k for k in state_dict) - if is_original_hunyuan_video: + if any(k.startswith("lora_unet_") for k in state_dict): + state_dict = _convert_musubi_hunyuan_video_lora_to_diffusers(state_dict) + elif is_original_hunyuan_video: state_dict = _convert_hunyuan_video_lora_to_diffusers(state_dict) out = (state_dict, metadata) if return_lora_metadata else state_dict