Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions src/diffusers/models/transformers/transformer_ernie_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,15 +268,16 @@ def forward(
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = temb
residual = x
x = self.adaLN_sa_ln(x)
x = (x.float() * (1 + scale_msa.float()) + shift_msa.float()).to(x.dtype)
# The float32 scale / shift / gate promote each product to float32, so the activations are not upcast first.
x = (x * (1 + scale_msa.float()) + shift_msa.float()).to(x.dtype)
x_bsh = x.permute(1, 0, 2) # [S, B, H] → [B, S, H] for diffusers Attention (batch-first)
attn_out = self.self_attention(x_bsh, attention_mask=attention_mask, image_rotary_emb=rotary_pos_emb)
attn_out = attn_out.permute(1, 0, 2) # [B, S, H] → [S, B, H]
x = residual + (gate_msa.float() * attn_out.float()).to(x.dtype)
x = residual + (gate_msa.float() * attn_out).to(x.dtype)
residual = x
x = self.adaLN_mlp_ln(x)
x = (x.float() * (1 + scale_mlp.float()) + shift_mlp.float()).to(x.dtype)
return residual + (gate_mlp.float() * self.mlp(x).float()).to(x.dtype)
x = (x * (1 + scale_mlp.float()) + shift_mlp.float()).to(x.dtype)
return residual + (gate_mlp.float() * self.mlp(x)).to(x.dtype)


class ErnieImageAdaLNContinuous(nn.Module):
Expand Down Expand Up @@ -389,7 +390,6 @@ def forward(
text_sbh = text_bth.transpose(0, 1).contiguous()

x = torch.cat([img_sbh, text_sbh], dim=0)
S = x.shape[0]

# Position IDs
text_ids = (
Expand Down Expand Up @@ -431,8 +431,9 @@ def forward(
sample = self.time_proj(timestep)
sample = sample.to(dtype=dtype)
c = self.time_embedding(sample)
# [1, B, H] each, broadcast over the sequence inside the blocks
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = [
t.unsqueeze(0).expand(S, -1, -1).contiguous() for t in self.adaLN_modulation(c).chunk(6, dim=-1)
t.unsqueeze(0) for t in self.adaLN_modulation(c).chunk(6, dim=-1)
]
for layer in self.layers:
temb = [shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp]
Expand Down
45 changes: 45 additions & 0 deletions tests/models/transformers/test_models_transformer_ernie_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@
# limitations under the License.

import os
from unittest import mock

import pytest
import torch

from diffusers import ErnieImageTransformer2DModel
from diffusers.models.transformers.transformer_ernie_image import ErnieImageSharedAdaLNBlock
from diffusers.utils.torch_utils import randn_tensor

from ...testing_utils import torch_device
Expand Down Expand Up @@ -104,6 +106,49 @@ class TestErnieImageTransformer(ErnieImageTransformerTesterConfig, ModelTesterMi
pass


class TestErnieImageTransformerAdaLN(ErnieImageTransformerTesterConfig):
def test_adaln_broadcast_matches_expanded_float32_path(self):
"""Broadcasting the [1, B, H] modulation tensors gives the same bfloat16 output, bit for bit, as expanding
them to [S, B, H] and upcasting the activations to float32 before the products."""

def block_forward_reference(block, x, rotary_pos_emb, temb, attention_mask=None):
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = [
t.expand(x.shape[0], -1, -1).contiguous() for t in temb
]
residual = x
x = block.adaLN_sa_ln(x)
x = (x.float() * (1 + scale_msa.float()) + shift_msa.float()).to(x.dtype)
attn_out = block.self_attention(
x.permute(1, 0, 2), attention_mask=attention_mask, image_rotary_emb=rotary_pos_emb
)
x = residual + (gate_msa.float() * attn_out.permute(1, 0, 2).float()).to(x.dtype)
residual = x
x = block.adaLN_mlp_ln(x)
x = (x.float() * (1 + scale_mlp.float()) + shift_mlp.float()).to(x.dtype)
return residual + (gate_mlp.float() * block.mlp(x).float()).to(x.dtype)

torch.manual_seed(0)
model = self.model_class(**self.get_init_dict()).to(torch_device, torch.bfloat16).eval()
with torch.no_grad():
# zero-initialised: gate = 0 would hide the attention and MLP outputs, and the output would be all zeros
for linear in (model.adaLN_modulation[-1], model.final_linear):
linear.weight.normal_(std=0.5)
linear.bias.normal_(std=0.5)

inputs = self.get_dummy_inputs(batch_size=2)
inputs["hidden_states"] = inputs["hidden_states"].to(torch.bfloat16)
inputs["text_bth"] = inputs["text_bth"].to(torch.bfloat16)
# a different timestep per sample, so the two samples get different scale / shift / gate
inputs["timestep"] = torch.tensor([1.0, 500.0], device=torch_device)
inputs["text_lens"] = torch.tensor([16, 9], device=torch_device)

with torch.no_grad():
out = model(**inputs).sample
with mock.patch.object(ErnieImageSharedAdaLNBlock, "forward", block_forward_reference):
ref = model(**inputs).sample
assert torch.equal(out, ref)


class TestErnieImageTransformerTraining(ErnieImageTransformerTesterConfig, TrainingTesterMixin):
def test_gradient_checkpointing_is_applied(self):
expected_set = {"ErnieImageTransformer2DModel"}
Expand Down
Loading