From bb5d9798b5c77f60292a6911875c687e545d06e8 Mon Sep 17 00:00:00 2001 From: "Hsu, Chan Fu" Date: Fri, 13 Nov 2026 11:25:07 +0800 Subject: [PATCH] Broadcast the ERNIE-Image adaLN modulation, drop float32 copies ErnieImageTransformer2DModel expanded the six adaLN modulation tensors to (seq_len, batch, hidden) with expand(S, -1, -1).contiguous(), and every ErnieImageSharedAdaLNBlock cast them and the activations to float32 before the products (x.float() twice, attn_out.float(), self.mlp(x).float()). Keep the six tensors as (1, batch, hidden) and let them broadcast over the sequence, and drop the activation upcasts. Scale, shift and gate are still cast to float32, so every product still runs in float32 by type promotion and is cast back at the same points. Add a test that checks the bfloat16 transformer output, bit for bit, against the block with the expanded modulation and the float32 upcasts. Bit-identical baidu/ERNIE-Image pipeline output in eager, bf16, on RTX 5090 at the model card settings (1264x848, 50 steps, use_pe=True) and on AMD Radeon AI PRO R9700. Under torch.compile the output is not bit-identical to compiled main. Signed-off-by: Hsu, Chan Fu --- .../transformers/transformer_ernie_image.py | 13 +++--- .../test_models_transformer_ernie_image.py | 45 +++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/diffusers/models/transformers/transformer_ernie_image.py b/src/diffusers/models/transformers/transformer_ernie_image.py index 791d4e0bce42..447d51242446 100644 --- a/src/diffusers/models/transformers/transformer_ernie_image.py +++ b/src/diffusers/models/transformers/transformer_ernie_image.py @@ -268,15 +268,16 @@ def forward( shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = temb residual = x x = self.adaLN_sa_ln(x) - x = (x.float() * (1 + scale_msa.float()) + shift_msa.float()).to(x.dtype) + # The float32 scale / shift / gate promote each product to float32, so the activations are not upcast first. + x = (x * (1 + scale_msa.float()) + shift_msa.float()).to(x.dtype) x_bsh = x.permute(1, 0, 2) # [S, B, H] → [B, S, H] for diffusers Attention (batch-first) attn_out = self.self_attention(x_bsh, attention_mask=attention_mask, image_rotary_emb=rotary_pos_emb) attn_out = attn_out.permute(1, 0, 2) # [B, S, H] → [S, B, H] - x = residual + (gate_msa.float() * attn_out.float()).to(x.dtype) + x = residual + (gate_msa.float() * attn_out).to(x.dtype) residual = x x = self.adaLN_mlp_ln(x) - x = (x.float() * (1 + scale_mlp.float()) + shift_mlp.float()).to(x.dtype) - return residual + (gate_mlp.float() * self.mlp(x).float()).to(x.dtype) + x = (x * (1 + scale_mlp.float()) + shift_mlp.float()).to(x.dtype) + return residual + (gate_mlp.float() * self.mlp(x)).to(x.dtype) class ErnieImageAdaLNContinuous(nn.Module): @@ -389,7 +390,6 @@ def forward( text_sbh = text_bth.transpose(0, 1).contiguous() x = torch.cat([img_sbh, text_sbh], dim=0) - S = x.shape[0] # Position IDs text_ids = ( @@ -431,8 +431,9 @@ def forward( sample = self.time_proj(timestep) sample = sample.to(dtype=dtype) c = self.time_embedding(sample) + # [1, B, H] each, broadcast over the sequence inside the blocks shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = [ - t.unsqueeze(0).expand(S, -1, -1).contiguous() for t in self.adaLN_modulation(c).chunk(6, dim=-1) + t.unsqueeze(0) for t in self.adaLN_modulation(c).chunk(6, dim=-1) ] for layer in self.layers: temb = [shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp] diff --git a/tests/models/transformers/test_models_transformer_ernie_image.py b/tests/models/transformers/test_models_transformer_ernie_image.py index aff444c6941e..a8e1f91ccfe3 100644 --- a/tests/models/transformers/test_models_transformer_ernie_image.py +++ b/tests/models/transformers/test_models_transformer_ernie_image.py @@ -14,11 +14,13 @@ # limitations under the License. import os +from unittest import mock import pytest import torch from diffusers import ErnieImageTransformer2DModel +from diffusers.models.transformers.transformer_ernie_image import ErnieImageSharedAdaLNBlock from diffusers.utils.torch_utils import randn_tensor from ...testing_utils import torch_device @@ -104,6 +106,49 @@ class TestErnieImageTransformer(ErnieImageTransformerTesterConfig, ModelTesterMi pass +class TestErnieImageTransformerAdaLN(ErnieImageTransformerTesterConfig): + def test_adaln_broadcast_matches_expanded_float32_path(self): + """Broadcasting the [1, B, H] modulation tensors gives the same bfloat16 output, bit for bit, as expanding + them to [S, B, H] and upcasting the activations to float32 before the products.""" + + def block_forward_reference(block, x, rotary_pos_emb, temb, attention_mask=None): + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = [ + t.expand(x.shape[0], -1, -1).contiguous() for t in temb + ] + residual = x + x = block.adaLN_sa_ln(x) + x = (x.float() * (1 + scale_msa.float()) + shift_msa.float()).to(x.dtype) + attn_out = block.self_attention( + x.permute(1, 0, 2), attention_mask=attention_mask, image_rotary_emb=rotary_pos_emb + ) + x = residual + (gate_msa.float() * attn_out.permute(1, 0, 2).float()).to(x.dtype) + residual = x + x = block.adaLN_mlp_ln(x) + x = (x.float() * (1 + scale_mlp.float()) + shift_mlp.float()).to(x.dtype) + return residual + (gate_mlp.float() * block.mlp(x).float()).to(x.dtype) + + torch.manual_seed(0) + model = self.model_class(**self.get_init_dict()).to(torch_device, torch.bfloat16).eval() + with torch.no_grad(): + # zero-initialised: gate = 0 would hide the attention and MLP outputs, and the output would be all zeros + for linear in (model.adaLN_modulation[-1], model.final_linear): + linear.weight.normal_(std=0.5) + linear.bias.normal_(std=0.5) + + inputs = self.get_dummy_inputs(batch_size=2) + inputs["hidden_states"] = inputs["hidden_states"].to(torch.bfloat16) + inputs["text_bth"] = inputs["text_bth"].to(torch.bfloat16) + # a different timestep per sample, so the two samples get different scale / shift / gate + inputs["timestep"] = torch.tensor([1.0, 500.0], device=torch_device) + inputs["text_lens"] = torch.tensor([16, 9], device=torch_device) + + with torch.no_grad(): + out = model(**inputs).sample + with mock.patch.object(ErnieImageSharedAdaLNBlock, "forward", block_forward_reference): + ref = model(**inputs).sample + assert torch.equal(out, ref) + + class TestErnieImageTransformerTraining(ErnieImageTransformerTesterConfig, TrainingTesterMixin): def test_gradient_checkpointing_is_applied(self): expected_set = {"ErnieImageTransformer2DModel"}