Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions src/diffusers/loaders/lora_conversion_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1766,6 +1766,52 @@ def remap_single_transformer_blocks_(key, state_dict):
return converted_state_dict


def _convert_musubi_hunyuan_video_lora_to_diffusers(state_dict):
# https://github.com/kohya-ss/musubi-tuner
# Keys look like `lora_unet_double_blocks_0_img_attn_qkv.lora_down.weight`. Rewrite them into the original
# HunyuanVideo layout (`double_blocks.0.img_attn_qkv.lora_A.weight`), fold `alpha / rank` into the weights and
# reuse `_convert_hunyuan_video_lora_to_diffusers` for the rest.
module_renames = {
"img_mod_linear": "img_mod.linear",
"txt_mod_linear": "txt_mod.linear",
"img_mlp_fc1": "img_mlp.fc1",
"img_mlp_fc2": "img_mlp.fc2",
"txt_mlp_fc1": "txt_mlp.fc1",
"txt_mlp_fc2": "txt_mlp.fc2",
"modulation_linear": "modulation.linear",
}
original_state_dict = {k[len("lora_unet_") :]: v for k, v in state_dict.items()}
converted_state_dict = {}

for key in [k for k in original_state_dict if k.endswith(".lora_down.weight")]:
module_key = key[: -len(".lora_down.weight")]
match = re.fullmatch(r"(double_blocks|single_blocks)_(\d+)_(\w+)", module_key)
if match is None:
continue
block_type, block_idx, module_name = match.groups()
module_name = module_renames.get(module_name, module_name)

down_weight = original_state_dict.pop(f"{module_key}.lora_down.weight")
up_weight = original_state_dict.pop(f"{module_key}.lora_up.weight")
alpha = original_state_dict.pop(f"{module_key}.alpha").item()
# LoRA is scaled by `alpha / rank` in the forward pass, so fold that scale into the weights.
scale = alpha / down_weight.shape[0]
scale_down = scale
scale_up = 1.0
while scale_down * 2 < scale_up:
scale_down *= 2
scale_up /= 2

new_key = f"{block_type}.{block_idx}.{module_name}"
converted_state_dict[f"{new_key}.lora_A.weight"] = down_weight * scale_down
converted_state_dict[f"{new_key}.lora_B.weight"] = up_weight * scale_up

if len(original_state_dict) > 0:
raise ValueError(f"`state_dict` should be empty at this point but has {original_state_dict.keys()=}")

return _convert_hunyuan_video_lora_to_diffusers(converted_state_dict)


def _convert_non_diffusers_lumina2_lora_to_diffusers(state_dict):
# Remove "diffusion_model." prefix from keys.
state_dict = {k[len("diffusion_model.") :]: v for k, v in state_dict.items()}
Expand Down
5 changes: 4 additions & 1 deletion src/diffusers/loaders/lora_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
_convert_kohya_flux2_lora_to_diffusers,
_convert_kohya_flux_lora_to_diffusers,
_convert_lycoris_flux2_lokr_to_diffusers,
_convert_musubi_hunyuan_video_lora_to_diffusers,
_convert_musubi_wan_lora_to_diffusers,
_convert_non_diffusers_ace_step_lora_to_diffusers,
_convert_non_diffusers_anima_lora_to_diffusers,
Expand Down Expand Up @@ -3571,7 +3572,9 @@ def lora_state_dict(
state_dict = {k: v for k, v in state_dict.items() if "dora_scale" not in k}

is_original_hunyuan_video = any("img_attn_qkv" in k for k in state_dict)
if is_original_hunyuan_video:
if any(k.startswith("lora_unet_") for k in state_dict):
state_dict = _convert_musubi_hunyuan_video_lora_to_diffusers(state_dict)
elif is_original_hunyuan_video:
state_dict = _convert_hunyuan_video_lora_to_diffusers(state_dict)

out = (state_dict, metadata) if return_lora_metadata else state_dict
Expand Down