Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 16 additions & 7 deletions src/diffusers/loaders/lora_conversion_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1071,8 +1071,13 @@ def _custom_replace(key: str, substrings: list[str]) -> str:
def _convert_bfl_flux_control_lora_to_diffusers(original_state_dict):
converted_state_dict = {}
original_state_dict_keys = list(original_state_dict.keys())
num_layers = 19
num_single_layers = 38
num_layers = 0
num_single_layers = 0
for key in original_state_dict_keys:
if key.startswith("single_blocks."):
num_single_layers = max(num_single_layers, int(key.split(".")[1]) + 1)
elif key.startswith("double_blocks."):
num_layers = max(num_layers, int(key.split(".")[1]) + 1)
inner_dim = 3072
mlp_ratio = 4.0

Expand Down Expand Up @@ -1352,11 +1357,14 @@ def _convert_bfl_flux_control_lora_to_diffusers(original_state_dict):
f"final_layer.linear.{lora_key}.bias"
)

converted_state_dict[f"norm_out.linear.{lora_key}.weight"] = swap_scale_shift(
# The original final layer orders its modulation (shift, scale), diffusers' `norm_out` (scale, shift). Only
# lora_B's rows are the layer's outputs; lora_A's rows are the rank dimension and must stay as they are.
maybe_swap = swap_scale_shift if lora_key == "lora_B" else (lambda weight: weight)
converted_state_dict[f"norm_out.linear.{lora_key}.weight"] = maybe_swap(
original_state_dict.pop(f"final_layer.adaLN_modulation.1.{lora_key}.weight")
)
if f"final_layer.adaLN_modulation.1.{lora_key}.bias" in original_state_dict_keys:
converted_state_dict[f"norm_out.linear.{lora_key}.bias"] = swap_scale_shift(
converted_state_dict[f"norm_out.linear.{lora_key}.bias"] = maybe_swap(
original_state_dict.pop(f"final_layer.adaLN_modulation.1.{lora_key}.bias")
)

Expand Down Expand Up @@ -1596,9 +1604,10 @@ def _convert_hunyuan_video_lora_to_diffusers(original_state_dict):

def remap_norm_scale_shift_(key, state_dict):
weight = state_dict.pop(key)
shift, scale = weight.chunk(2, dim=0)
new_weight = torch.cat([scale, shift], dim=0)
state_dict[key.replace("final_layer.adaLN_modulation.1", "norm_out.linear")] = new_weight
# Only lora_B's rows are the layer's (shift, scale) outputs; lora_A's rows are the rank dimension.
if "lora_B" in key:
weight = swap_scale_shift(weight)
state_dict[key.replace("final_layer.adaLN_modulation.1", "norm_out.linear")] = weight

def remap_txt_in_(key, state_dict):
def rename_key(key):
Expand Down
24 changes: 24 additions & 0 deletions tests/models/transformers/test_models_transformer_flux.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import torch

from diffusers import BitsAndBytesConfig, FluxTransformer2DModel, GGUFQuantizationConfig
from diffusers.loaders.lora_conversion_utils import _convert_bfl_flux_control_lora_to_diffusers
from diffusers.models.embeddings import ImageProjection
from diffusers.models.transformers.transformer_flux import FluxIPAdapterAttnProcessor
from diffusers.utils.torch_utils import randn_tensor
Expand Down Expand Up @@ -311,6 +312,29 @@ def create_ip_adapter_state_dict(self, model: Any) -> dict[str, dict[str, Any]]:
class TestFluxTransformerLoRA(FluxTransformerTesterConfig, LoraTesterMixin):
"""LoRA adapter tests for Flux Transformer."""

def test_lora_bfl_control_final_layer_modulation(self):
# The original final layer orders its modulation (shift, scale) and diffusers' `norm_out` (scale, shift). Only
# lora_B's rows are the layer's outputs, so the converted update must be the original one with its halves swapped.
# The converter takes the number of blocks from the keys, so the state dict only needs the non-block layers.
modules = ["img_in", "txt_in", "time_in.in_layer", "time_in.out_layer", "vector_in.in_layer"]
modules += ["vector_in.out_layer", "final_layer.linear"]
state_dict = {}
for module in modules:
state_dict[f"{module}.lora_A.weight"] = torch.zeros(1, 1)
state_dict[f"{module}.lora_B.weight"] = torch.zeros(1, 1)

torch.manual_seed(0)
lora_A, lora_B = torch.randn(2, 16), torch.randn(32, 2)
state_dict["final_layer.adaLN_modulation.1.lora_A.weight"] = lora_A
state_dict["final_layer.adaLN_modulation.1.lora_B.weight"] = lora_B
converted = _convert_bfl_flux_control_lora_to_diffusers(state_dict)
delta = (
converted["transformer.norm_out.linear.lora_B.weight"]
@ converted["transformer.norm_out.linear.lora_A.weight"]
)
Comment on lines +331 to +334

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we're checking against just norm_out, then why construct this big a state_dict?

shift, scale = (lora_B @ lora_A).chunk(2, dim=0)
assert torch.allclose(delta, torch.cat([scale, shift]))


class TestFluxTransformerLoRAHotSwap(FluxTransformerTesterConfig, LoraHotSwappingForModelTesterMixin):
"""LoRA hot-swapping tests for Flux Transformer."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import torch

from diffusers import HunyuanVideoTransformer3DModel
from diffusers.loaders import HunyuanVideoLoraLoaderMixin
from diffusers.utils.torch_utils import randn_tensor

from ...testing_utils import enable_full_determinism, torch_device
Expand Down Expand Up @@ -305,4 +306,21 @@ def test_output(self, base_model_output):


class TestHunyuanVideoTransformerLoRA(HunyuanVideoTransformerTesterConfig, LoraTesterMixin):
pass
def test_lora_original_final_layer_modulation(self):
# The original final layer orders its modulation (shift, scale) and diffusers' `norm_out` (scale, shift). Only
# lora_B's rows are the layer's outputs, so the converted update must be the original one with its halves swapped.
torch.manual_seed(0)
lora_A, lora_B = torch.randn(2, 16), torch.randn(32, 2)
state_dict = {
"diffusion_model.final_layer.adaLN_modulation.1.lora_A.weight": lora_A,
"diffusion_model.final_layer.adaLN_modulation.1.lora_B.weight": lora_B,
"diffusion_model.double_blocks.0.img_attn_qkv.lora_A.weight": torch.randn(2, 16),
"diffusion_model.double_blocks.0.img_attn_qkv.lora_B.weight": torch.randn(48, 2),
}
converted = HunyuanVideoLoraLoaderMixin.lora_state_dict(state_dict)
delta = (
converted["transformer.norm_out.linear.lora_B.weight"]
@ converted["transformer.norm_out.linear.lora_A.weight"]
)
shift, scale = (lora_B @ lora_A).chunk(2, dim=0)
assert torch.allclose(delta, torch.cat([scale, shift]))
Loading