From 7debf19a7ff7b2ab24239fb325d04b7da4ad2248 Mon Sep 17 00:00:00 2001 From: "ChanFu, Hsu" <228416682+IvenHsu01@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:08:33 +0800 Subject: [PATCH 1/4] [LoRA] support musubi-tuner HunyuanVideo LoRAs --- .../loaders/lora_conversion_utils.py | 46 +++++++++++++++++++ src/diffusers/loaders/lora_pipeline.py | 8 +++- .../test_models_transformer_hunyuan_video.py | 29 +++++++++++- 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/src/diffusers/loaders/lora_conversion_utils.py b/src/diffusers/loaders/lora_conversion_utils.py index b6f626842ee2..e755706c6535 100644 --- a/src/diffusers/loaders/lora_conversion_utils.py +++ b/src/diffusers/loaders/lora_conversion_utils.py @@ -1766,6 +1766,52 @@ def remap_single_transformer_blocks_(key, state_dict): return converted_state_dict +def _convert_musubi_hunyuan_video_lora_to_diffusers(state_dict): + # https://github.com/kohya-ss/musubi-tuner + # Keys look like `lora_unet_double_blocks_0_img_attn_qkv.lora_down.weight`. Rewrite them into the original + # HunyuanVideo layout (`double_blocks.0.img_attn_qkv.lora_A.weight`), fold `alpha / rank` into the weights and + # reuse `_convert_hunyuan_video_lora_to_diffusers` for the rest. + module_renames = { + "img_mod_linear": "img_mod.linear", + "txt_mod_linear": "txt_mod.linear", + "img_mlp_fc1": "img_mlp.fc1", + "img_mlp_fc2": "img_mlp.fc2", + "txt_mlp_fc1": "txt_mlp.fc1", + "txt_mlp_fc2": "txt_mlp.fc2", + "modulation_linear": "modulation.linear", + } + original_state_dict = {k[len("lora_unet_") :]: v for k, v in state_dict.items()} + converted_state_dict = {} + + for key in [k for k in original_state_dict if k.endswith(".lora_down.weight")]: + module_key = key[: -len(".lora_down.weight")] + match = re.fullmatch(r"(double_blocks|single_blocks)_(\d+)_(\w+)", module_key) + if match is None: + continue + block_type, block_idx, module_name = match.groups() + module_name = module_renames.get(module_name, module_name) + + down_weight = original_state_dict.pop(f"{module_key}.lora_down.weight") + up_weight = original_state_dict.pop(f"{module_key}.lora_up.weight") + alpha = original_state_dict.pop(f"{module_key}.alpha", None) + # LoRA is scaled by `alpha / rank` in the forward pass, so fold that scale into the weights. + scale = alpha.item() / down_weight.shape[0] if alpha is not None else 1.0 + scale_down = scale + scale_up = 1.0 + while scale_down * 2 < scale_up: + scale_down *= 2 + scale_up /= 2 + + new_key = f"{block_type}.{block_idx}.{module_name}" + converted_state_dict[f"{new_key}.lora_A.weight"] = down_weight * scale_down + converted_state_dict[f"{new_key}.lora_B.weight"] = up_weight * scale_up + + if len(original_state_dict) > 0: + raise ValueError(f"`state_dict` should be empty at this point but has {original_state_dict.keys()=}") + + return _convert_hunyuan_video_lora_to_diffusers(converted_state_dict) + + def _convert_non_diffusers_lumina2_lora_to_diffusers(state_dict): # Remove "diffusion_model." prefix from keys. state_dict = {k[len("diffusion_model.") :]: v for k, v in state_dict.items()} diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 43d455551f26..9bd1bb987cbe 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -46,6 +46,7 @@ _convert_kohya_flux2_lora_to_diffusers, _convert_kohya_flux_lora_to_diffusers, _convert_lycoris_flux2_lokr_to_diffusers, + _convert_musubi_hunyuan_video_lora_to_diffusers, _convert_musubi_wan_lora_to_diffusers, _convert_non_diffusers_ace_step_lora_to_diffusers, _convert_non_diffusers_anima_lora_to_diffusers, @@ -3571,7 +3572,12 @@ def lora_state_dict( state_dict = {k: v for k, v in state_dict.items() if "dora_scale" not in k} is_original_hunyuan_video = any("img_attn_qkv" in k for k in state_dict) - if is_original_hunyuan_video: + is_musubi_hunyuan_video = any( + k.startswith(("lora_unet_double_blocks_", "lora_unet_single_blocks_")) for k in state_dict + ) + if is_musubi_hunyuan_video: + state_dict = _convert_musubi_hunyuan_video_lora_to_diffusers(state_dict) + elif is_original_hunyuan_video: state_dict = _convert_hunyuan_video_lora_to_diffusers(state_dict) out = (state_dict, metadata) if return_lora_metadata else state_dict diff --git a/tests/models/transformers/test_models_transformer_hunyuan_video.py b/tests/models/transformers/test_models_transformer_hunyuan_video.py index bb51ed71e008..380f7ff6f8ba 100644 --- a/tests/models/transformers/test_models_transformer_hunyuan_video.py +++ b/tests/models/transformers/test_models_transformer_hunyuan_video.py @@ -15,6 +15,7 @@ import torch from diffusers import HunyuanVideoTransformer3DModel +from diffusers.loaders import HunyuanVideoLoraLoaderMixin from diffusers.utils.torch_utils import randn_tensor from ...testing_utils import enable_full_determinism, torch_device @@ -305,4 +306,30 @@ def test_output(self, base_model_output): class TestHunyuanVideoTransformerLoRA(HunyuanVideoTransformerTesterConfig, LoraTesterMixin): - pass + def test_lora_musubi_state_dict(self): + # musubi-tuner / kohya keys (`lora_unet_*`, `lora_down` / `lora_up` / `alpha`), as in the LoRA used by the + # HunyuanVideo docs. The converted update must be the original one scaled by `alpha / rank`. + torch.manual_seed(0) + rank, alpha = 4, 1.0 + qkv_down, qkv_up = torch.randn(rank, 16), torch.randn(48, rank) + fc1_down, fc1_up = torch.randn(rank, 16), torch.randn(64, rank) + state_dict = { + "lora_unet_double_blocks_0_img_attn_qkv.lora_down.weight": qkv_down, + "lora_unet_double_blocks_0_img_attn_qkv.lora_up.weight": qkv_up, + "lora_unet_double_blocks_0_img_attn_qkv.alpha": torch.tensor(alpha), + "lora_unet_double_blocks_0_img_mlp_fc1.lora_down.weight": fc1_down, + "lora_unet_double_blocks_0_img_mlp_fc1.lora_up.weight": fc1_up, + "lora_unet_double_blocks_0_img_mlp_fc1.alpha": torch.tensor(alpha), + } + converted = HunyuanVideoLoraLoaderMixin.lora_state_dict(state_dict) + + def delta(module): + prefix = f"transformer.transformer_blocks.0.{module}" + return converted[f"{prefix}.lora_B.weight"] @ converted[f"{prefix}.lora_A.weight"] + + expected_q, expected_k, expected_v = (alpha / rank * qkv_up @ qkv_down).chunk(3, dim=0) + assert torch.allclose(delta("attn.to_q"), expected_q, atol=1e-6) + assert torch.allclose(delta("attn.to_k"), expected_k, atol=1e-6) + assert torch.allclose(delta("attn.to_v"), expected_v, atol=1e-6) + assert torch.allclose(delta("ff.net.0.proj"), alpha / rank * fc1_up @ fc1_down, atol=1e-6) + assert len(converted) == 8 From 7448ebcbeca04e5315100ba1dde79d9183d2cfee Mon Sep 17 00:00:00 2001 From: "ChanFu, Hsu" <228416682+IvenHsu01@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:34:58 +0800 Subject: [PATCH 2/4] Address self-review: pop alpha like the Wan converter, move musubi test to the pipeline LoRA tests --- .../loaders/lora_conversion_utils.py | 4 +-- .../test_models_transformer_hunyuan_video.py | 29 +-------------- .../hunyuan_video/test_hunyuan_video.py | 35 +++++++++++++++++++ 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/diffusers/loaders/lora_conversion_utils.py b/src/diffusers/loaders/lora_conversion_utils.py index e755706c6535..da9b0486bc71 100644 --- a/src/diffusers/loaders/lora_conversion_utils.py +++ b/src/diffusers/loaders/lora_conversion_utils.py @@ -1793,9 +1793,9 @@ def _convert_musubi_hunyuan_video_lora_to_diffusers(state_dict): down_weight = original_state_dict.pop(f"{module_key}.lora_down.weight") up_weight = original_state_dict.pop(f"{module_key}.lora_up.weight") - alpha = original_state_dict.pop(f"{module_key}.alpha", None) + alpha = original_state_dict.pop(f"{module_key}.alpha").item() # LoRA is scaled by `alpha / rank` in the forward pass, so fold that scale into the weights. - scale = alpha.item() / down_weight.shape[0] if alpha is not None else 1.0 + scale = alpha / down_weight.shape[0] scale_down = scale scale_up = 1.0 while scale_down * 2 < scale_up: diff --git a/tests/models/transformers/test_models_transformer_hunyuan_video.py b/tests/models/transformers/test_models_transformer_hunyuan_video.py index 380f7ff6f8ba..bb51ed71e008 100644 --- a/tests/models/transformers/test_models_transformer_hunyuan_video.py +++ b/tests/models/transformers/test_models_transformer_hunyuan_video.py @@ -15,7 +15,6 @@ import torch from diffusers import HunyuanVideoTransformer3DModel -from diffusers.loaders import HunyuanVideoLoraLoaderMixin from diffusers.utils.torch_utils import randn_tensor from ...testing_utils import enable_full_determinism, torch_device @@ -306,30 +305,4 @@ def test_output(self, base_model_output): class TestHunyuanVideoTransformerLoRA(HunyuanVideoTransformerTesterConfig, LoraTesterMixin): - def test_lora_musubi_state_dict(self): - # musubi-tuner / kohya keys (`lora_unet_*`, `lora_down` / `lora_up` / `alpha`), as in the LoRA used by the - # HunyuanVideo docs. The converted update must be the original one scaled by `alpha / rank`. - torch.manual_seed(0) - rank, alpha = 4, 1.0 - qkv_down, qkv_up = torch.randn(rank, 16), torch.randn(48, rank) - fc1_down, fc1_up = torch.randn(rank, 16), torch.randn(64, rank) - state_dict = { - "lora_unet_double_blocks_0_img_attn_qkv.lora_down.weight": qkv_down, - "lora_unet_double_blocks_0_img_attn_qkv.lora_up.weight": qkv_up, - "lora_unet_double_blocks_0_img_attn_qkv.alpha": torch.tensor(alpha), - "lora_unet_double_blocks_0_img_mlp_fc1.lora_down.weight": fc1_down, - "lora_unet_double_blocks_0_img_mlp_fc1.lora_up.weight": fc1_up, - "lora_unet_double_blocks_0_img_mlp_fc1.alpha": torch.tensor(alpha), - } - converted = HunyuanVideoLoraLoaderMixin.lora_state_dict(state_dict) - - def delta(module): - prefix = f"transformer.transformer_blocks.0.{module}" - return converted[f"{prefix}.lora_B.weight"] @ converted[f"{prefix}.lora_A.weight"] - - expected_q, expected_k, expected_v = (alpha / rank * qkv_up @ qkv_down).chunk(3, dim=0) - assert torch.allclose(delta("attn.to_q"), expected_q, atol=1e-6) - assert torch.allclose(delta("attn.to_k"), expected_k, atol=1e-6) - assert torch.allclose(delta("attn.to_v"), expected_v, atol=1e-6) - assert torch.allclose(delta("ff.net.0.proj"), alpha / rank * fc1_up @ fc1_down, atol=1e-6) - assert len(converted) == 8 + pass diff --git a/tests/pipelines/hunyuan_video/test_hunyuan_video.py b/tests/pipelines/hunyuan_video/test_hunyuan_video.py index 6b857eecea3b..49929f287a00 100644 --- a/tests/pipelines/hunyuan_video/test_hunyuan_video.py +++ b/tests/pipelines/hunyuan_video/test_hunyuan_video.py @@ -26,6 +26,7 @@ HunyuanVideoTransformer3DModel, ) +from ...models.testing_utils.lora import check_if_lora_correctly_set from ...testing_utils import ( Expectations, assert_tensors_close, @@ -271,6 +272,40 @@ class TestHunyuanVideoPipelineTaylorSeerCache(HunyuanVideoPipelineTesterConfig, class TestHunyuanVideoPipelineLoRA(HunyuanVideoPipelineTesterConfig, LoraTesterMixin): """LoRA tests for the HunyuanVideo pipeline.""" + def test_lora_musubi_state_dict(self, base_pipe_output): + # musubi-tuner keys (`lora_unet_*` with `lora_down` / `lora_up` / `alpha`), as in the LoRA used by the + # HunyuanVideo docs. The loaded update must be the original one scaled by `alpha / rank`. + pipe = self.get_pipeline().to(torch_device) + inner_dim = pipe.transformer.config.num_attention_heads * pipe.transformer.config.attention_head_dim + rank, alpha = 4, 1.0 + + torch.manual_seed(0) + qkv_down, qkv_up = torch.randn(rank, inner_dim), torch.randn(3 * inner_dim, rank) + fc1_down, fc1_up = torch.randn(rank, inner_dim), torch.randn(4 * inner_dim, rank) + state_dict = { + "lora_unet_double_blocks_0_img_attn_qkv.lora_down.weight": qkv_down, + "lora_unet_double_blocks_0_img_attn_qkv.lora_up.weight": qkv_up, + "lora_unet_double_blocks_0_img_attn_qkv.alpha": torch.tensor(alpha), + "lora_unet_double_blocks_0_img_mlp_fc1.lora_down.weight": fc1_down, + "lora_unet_double_blocks_0_img_mlp_fc1.lora_up.weight": fc1_up, + "lora_unet_double_blocks_0_img_mlp_fc1.alpha": torch.tensor(alpha), + } + pipe.load_lora_weights(state_dict, adapter_name="musubi") + assert check_if_lora_correctly_set(pipe.transformer), "Lora not correctly set in transformer" + + block = pipe.transformer.transformer_blocks[0] + expected_q, expected_k, expected_v = (alpha / rank * qkv_up @ qkv_down).chunk(3, dim=0) + for module, expected in [ + (block.attn.to_q, expected_q), + (block.attn.to_k, expected_k), + (block.attn.to_v, expected_v), + (block.ff.net[0].proj, alpha / rank * fc1_up @ fc1_down), + ]: + assert_tensors_close(module.get_delta_weight("musubi").cpu(), expected) + + output_lora = self.run_pipe(pipe) + assert not torch.allclose(output_lora, base_pipe_output, atol=1e-3, rtol=1e-3), "Lora should change the output" + class TestHunyuanVideoPipelineLoRAMemory(HunyuanVideoPipelineTesterConfig, LoraMemoryTesterMixin): """LoRA x memory-optimization tests (group offload, CPU offload) for the HunyuanVideo pipeline.""" From 7e1c21791bb9c84042bdabfc313da2261a3d279d Mon Sep 17 00:00:00 2001 From: "ChanFu, Hsu" <228416682+IvenHsu01@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:42:36 +0800 Subject: [PATCH 3/4] Detect musubi HunyuanVideo LoRAs by the lora_unet_ prefix, like the Wan loader --- src/diffusers/loaders/lora_pipeline.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 9bd1bb987cbe..cf1849ec49a3 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -3572,10 +3572,7 @@ def lora_state_dict( state_dict = {k: v for k, v in state_dict.items() if "dora_scale" not in k} is_original_hunyuan_video = any("img_attn_qkv" in k for k in state_dict) - is_musubi_hunyuan_video = any( - k.startswith(("lora_unet_double_blocks_", "lora_unet_single_blocks_")) for k in state_dict - ) - if is_musubi_hunyuan_video: + if any(k.startswith("lora_unet_") for k in state_dict): state_dict = _convert_musubi_hunyuan_video_lora_to_diffusers(state_dict) elif is_original_hunyuan_video: state_dict = _convert_hunyuan_video_lora_to_diffusers(state_dict) From 32f1d524cca04e01604849c8a8573f05f7519084 Mon Sep 17 00:00:00 2001 From: "ChanFu, Hsu" <228416682+IvenHsu01@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:20:45 +0800 Subject: [PATCH 4/4] Remove the musubi HunyuanVideo LoRA test (review) --- .../hunyuan_video/test_hunyuan_video.py | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/tests/pipelines/hunyuan_video/test_hunyuan_video.py b/tests/pipelines/hunyuan_video/test_hunyuan_video.py index 49929f287a00..6b857eecea3b 100644 --- a/tests/pipelines/hunyuan_video/test_hunyuan_video.py +++ b/tests/pipelines/hunyuan_video/test_hunyuan_video.py @@ -26,7 +26,6 @@ HunyuanVideoTransformer3DModel, ) -from ...models.testing_utils.lora import check_if_lora_correctly_set from ...testing_utils import ( Expectations, assert_tensors_close, @@ -272,40 +271,6 @@ class TestHunyuanVideoPipelineTaylorSeerCache(HunyuanVideoPipelineTesterConfig, class TestHunyuanVideoPipelineLoRA(HunyuanVideoPipelineTesterConfig, LoraTesterMixin): """LoRA tests for the HunyuanVideo pipeline.""" - def test_lora_musubi_state_dict(self, base_pipe_output): - # musubi-tuner keys (`lora_unet_*` with `lora_down` / `lora_up` / `alpha`), as in the LoRA used by the - # HunyuanVideo docs. The loaded update must be the original one scaled by `alpha / rank`. - pipe = self.get_pipeline().to(torch_device) - inner_dim = pipe.transformer.config.num_attention_heads * pipe.transformer.config.attention_head_dim - rank, alpha = 4, 1.0 - - torch.manual_seed(0) - qkv_down, qkv_up = torch.randn(rank, inner_dim), torch.randn(3 * inner_dim, rank) - fc1_down, fc1_up = torch.randn(rank, inner_dim), torch.randn(4 * inner_dim, rank) - state_dict = { - "lora_unet_double_blocks_0_img_attn_qkv.lora_down.weight": qkv_down, - "lora_unet_double_blocks_0_img_attn_qkv.lora_up.weight": qkv_up, - "lora_unet_double_blocks_0_img_attn_qkv.alpha": torch.tensor(alpha), - "lora_unet_double_blocks_0_img_mlp_fc1.lora_down.weight": fc1_down, - "lora_unet_double_blocks_0_img_mlp_fc1.lora_up.weight": fc1_up, - "lora_unet_double_blocks_0_img_mlp_fc1.alpha": torch.tensor(alpha), - } - pipe.load_lora_weights(state_dict, adapter_name="musubi") - assert check_if_lora_correctly_set(pipe.transformer), "Lora not correctly set in transformer" - - block = pipe.transformer.transformer_blocks[0] - expected_q, expected_k, expected_v = (alpha / rank * qkv_up @ qkv_down).chunk(3, dim=0) - for module, expected in [ - (block.attn.to_q, expected_q), - (block.attn.to_k, expected_k), - (block.attn.to_v, expected_v), - (block.ff.net[0].proj, alpha / rank * fc1_up @ fc1_down), - ]: - assert_tensors_close(module.get_delta_weight("musubi").cpu(), expected) - - output_lora = self.run_pipe(pipe) - assert not torch.allclose(output_lora, base_pipe_output, atol=1e-3, rtol=1e-3), "Lora should change the output" - class TestHunyuanVideoPipelineLoRAMemory(HunyuanVideoPipelineTesterConfig, LoraMemoryTesterMixin): """LoRA x memory-optimization tests (group offload, CPU offload) for the HunyuanVideo pipeline."""