diff --git a/src/diffusers/models/embeddings.py b/src/diffusers/models/embeddings.py index e1938a99b265..7232dbffe52b 100644 --- a/src/diffusers/models/embeddings.py +++ b/src/diffusers/models/embeddings.py @@ -994,7 +994,11 @@ def apply_rotary_emb( else: raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.") - out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + # Compute in float32 through type promotion rather than `x.float()` / `x_rotated.float()`, which would + # materialize two float32 copies of x. Same values: bf16 / fp16 -> float32 is exact. + compute_dtype = torch.promote_types(torch.float32, cos.dtype) + cos, sin = cos.to(compute_dtype), sin.to(compute_dtype) + out = (x * cos + x_rotated * sin).to(x.dtype) return out else: @@ -1013,7 +1017,10 @@ def apply_1d_rope(tokens, pos, cos, sin): sin = F.embedding(pos, sin)[:, None, :, :] x1, x2 = tokens[..., : tokens.shape[-1] // 2], tokens[..., tokens.shape[-1] // 2 :] tokens_rotated = torch.cat((-x2, x1), dim=-1) - return (tokens.float() * cos + tokens_rotated.float() * sin).to(tokens.dtype) + # float32 math through type promotion, without float32 copies of the input + compute_dtype = torch.promote_types(torch.float32, cos.dtype) + cos, sin = cos.to(compute_dtype), sin.to(compute_dtype) + return (tokens * cos + tokens_rotated * sin).to(tokens.dtype) (t_cos, t_sin), (h_cos, h_sin), (w_cos, w_sin) = freqs_cis t, h, w = x.chunk(3, dim=-1) diff --git a/src/diffusers/models/transformers/transformer_joyimage.py b/src/diffusers/models/transformers/transformer_joyimage.py index d2fec631f1ac..f6866d379c65 100644 --- a/src/diffusers/models/transformers/transformer_joyimage.py +++ b/src/diffusers/models/transformers/transformer_joyimage.py @@ -48,11 +48,14 @@ def _apply_rotary_emb( sin = freqs_cis[1].view(*shape).to(xq.device) def _rotate_half(x): - x_real, x_imag = x.float().reshape(*x.shape[:-1], -1, 2).unbind(-1) + x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) return torch.stack([-x_imag, x_real], dim=-1).flatten(3) - xq_out = (xq.float() * cos + _rotate_half(xq) * sin).type_as(xq) - xk_out = (xk.float() * cos + _rotate_half(xk) * sin).type_as(xk) + # float32 math through type promotion, without float32 copies of the input + compute_dtype = torch.promote_types(torch.float32, cos.dtype) + cos, sin = cos.to(compute_dtype), sin.to(compute_dtype) + xq_out = (xq * cos + _rotate_half(xq) * sin).type_as(xq) + xk_out = (xk * cos + _rotate_half(xk) * sin).type_as(xk) return xq_out, xk_out diff --git a/src/diffusers/models/transformers/transformer_joyimage_edit_plus.py b/src/diffusers/models/transformers/transformer_joyimage_edit_plus.py index a81027b5f40b..04643ec6f804 100644 --- a/src/diffusers/models/transformers/transformer_joyimage_edit_plus.py +++ b/src/diffusers/models/transformers/transformer_joyimage_edit_plus.py @@ -45,11 +45,14 @@ def _apply_rotary_emb_batched( sin = sin.unsqueeze(2) def _rotate_half(x): - x_real, x_imag = x.float().reshape(*x.shape[:-1], -1, 2).unbind(-1) + x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) return torch.stack([-x_imag, x_real], dim=-1).flatten(3) - xq_out = (xq.float() * cos + _rotate_half(xq) * sin).type_as(xq) - xk_out = (xk.float() * cos + _rotate_half(xk) * sin).type_as(xk) + # float32 math through type promotion, without float32 copies of the input + compute_dtype = torch.promote_types(torch.float32, cos.dtype) + cos, sin = cos.to(compute_dtype), sin.to(compute_dtype) + xq_out = (xq * cos + _rotate_half(xq) * sin).type_as(xq) + xk_out = (xk * cos + _rotate_half(xk) * sin).type_as(xk) return xq_out, xk_out diff --git a/src/diffusers/models/transformers/transformer_longcat_audio_dit.py b/src/diffusers/models/transformers/transformer_longcat_audio_dit.py index 9b8c0b4bf147..b03139f731d7 100644 --- a/src/diffusers/models/transformers/transformer_longcat_audio_dit.py +++ b/src/diffusers/models/transformers/transformer_longcat_audio_dit.py @@ -93,7 +93,10 @@ def _apply_rotary_emb(hidden_states: torch.Tensor, rope: tuple[torch.Tensor, tor cos, sin = rope cos = cos[None, :, None].to(hidden_states.device) sin = sin[None, :, None].to(hidden_states.device) - return (hidden_states.float() * cos + _rotate_half(hidden_states).float() * sin).to(hidden_states.dtype) + # float32 math through type promotion, without float32 copies of the input + compute_dtype = torch.promote_types(torch.float32, cos.dtype) + cos, sin = cos.to(compute_dtype), sin.to(compute_dtype) + return (hidden_states * cos + _rotate_half(hidden_states) * sin).to(hidden_states.dtype) class AudioDiTGRN(nn.Module): diff --git a/src/diffusers/models/transformers/transformer_ltx.py b/src/diffusers/models/transformers/transformer_ltx.py index c33e0f6141fc..9ca5ee8281e0 100644 --- a/src/diffusers/models/transformers/transformer_ltx.py +++ b/src/diffusers/models/transformers/transformer_ltx.py @@ -597,5 +597,8 @@ def apply_rotary_emb(x, freqs): cos, sin = freqs x_real, x_imag = x.unflatten(2, (-1, 2)).unbind(-1) # [B, S, C // 2] x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(2) - out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + # float32 math through type promotion, without float32 copies of the input + compute_dtype = torch.promote_types(torch.float32, cos.dtype) + cos, sin = cos.to(compute_dtype), sin.to(compute_dtype) + out = (x * cos + x_rotated * sin).to(x.dtype) return out diff --git a/src/diffusers/models/transformers/transformer_ltx2.py b/src/diffusers/models/transformers/transformer_ltx2.py index e5462e4b047a..8e9c274caad9 100644 --- a/src/diffusers/models/transformers/transformer_ltx2.py +++ b/src/diffusers/models/transformers/transformer_ltx2.py @@ -39,7 +39,10 @@ def apply_interleaved_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, tor cos, sin = freqs x_real, x_imag = x.unflatten(2, (-1, 2)).unbind(-1) # [B, S, C // 2] x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(2) - out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + # float32 math through type promotion, without float32 copies of the input + compute_dtype = torch.promote_types(torch.float32, cos.dtype) + cos, sin = cos.to(compute_dtype), sin.to(compute_dtype) + out = (x * cos + x_rotated * sin).to(x.dtype) return out diff --git a/src/diffusers/models/transformers/transformer_nucleusmoe_image.py b/src/diffusers/models/transformers/transformer_nucleusmoe_image.py index 5488c02c579f..6a365f9bf212 100644 --- a/src/diffusers/models/transformers/transformer_nucleusmoe_image.py +++ b/src/diffusers/models/transformers/transformer_nucleusmoe_image.py @@ -74,7 +74,10 @@ def _apply_rotary_emb_nucleus( else: raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.") - out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + # float32 math through type promotion, without float32 copies of the input + compute_dtype = torch.promote_types(torch.float32, cos.dtype) + cos, sin = cos.to(compute_dtype), sin.to(compute_dtype) + out = (x * cos + x_rotated * sin).to(x.dtype) return out else: diff --git a/src/diffusers/models/transformers/transformer_qwenimage.py b/src/diffusers/models/transformers/transformer_qwenimage.py index 29a98c0e9e57..c06cbca37552 100644 --- a/src/diffusers/models/transformers/transformer_qwenimage.py +++ b/src/diffusers/models/transformers/transformer_qwenimage.py @@ -130,7 +130,10 @@ def apply_rotary_emb_qwen( else: raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.") - out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + # float32 math through type promotion, without float32 copies of the input + compute_dtype = torch.promote_types(torch.float32, cos.dtype) + cos, sin = cos.to(compute_dtype), sin.to(compute_dtype) + out = (x * cos + x_rotated * sin).to(x.dtype) return out else: diff --git a/src/diffusers/models/transformers/transformer_qwenimage21.py b/src/diffusers/models/transformers/transformer_qwenimage21.py index 1a9ef250c519..78ebcf5cd7df 100644 --- a/src/diffusers/models/transformers/transformer_qwenimage21.py +++ b/src/diffusers/models/transformers/transformer_qwenimage21.py @@ -122,7 +122,10 @@ def apply_rotary_emb_qwen( else: raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.") - out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + # float32 math through type promotion, without float32 copies of the input + compute_dtype = torch.promote_types(torch.float32, cos.dtype) + cos, sin = cos.to(compute_dtype), sin.to(compute_dtype) + out = (x * cos + x_rotated * sin).to(x.dtype) return out else: diff --git a/src/diffusers/models/transformers/transformer_sana_wm_refiner.py b/src/diffusers/models/transformers/transformer_sana_wm_refiner.py index 892eca5eb8e7..a22784929d7c 100644 --- a/src/diffusers/models/transformers/transformer_sana_wm_refiner.py +++ b/src/diffusers/models/transformers/transformer_sana_wm_refiner.py @@ -37,7 +37,10 @@ def apply_interleaved_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, tor cos, sin = freqs x_real, x_imag = x.unflatten(2, (-1, 2)).unbind(-1) # [B, S, C // 2] x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(2) - out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + # float32 math through type promotion, without float32 copies of the input + compute_dtype = torch.promote_types(torch.float32, cos.dtype) + cos, sin = cos.to(compute_dtype), sin.to(compute_dtype) + out = (x * cos + x_rotated * sin).to(x.dtype) return out diff --git a/tests/models/test_layers_utils.py b/tests/models/test_layers_utils.py index bcdf64abe6b8..52c9a2d1bee9 100644 --- a/tests/models/test_layers_utils.py +++ b/tests/models/test_layers_utils.py @@ -21,7 +21,7 @@ from torch import nn from diffusers.models.attention import GEGLU, AdaLayerNorm, ApproximateGELU -from diffusers.models.embeddings import get_timestep_embedding +from diffusers.models.embeddings import apply_rotary_emb, get_timestep_embedding from diffusers.models.resnet import Downsample2D, ResnetBlock2D, Upsample2D from diffusers.models.transformers.transformer_2d import Transformer2DModel @@ -108,6 +108,36 @@ def test_sinoid_embeddings_hardcoded(self): 1e-3, ) + def test_apply_rotary_emb_matches_float32_reference(self): + # Must equal the explicit float32 formula bit for bit, for every input dtype and layout. + def reference(x, cos, sin, unbind_dim): + if unbind_dim == -1: + x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) + x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3) + else: + x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2) + x_rotated = torch.cat([-x_imag, x_real], dim=-1) + return (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + + generator = torch.Generator().manual_seed(0) + seq_len, dim = 16, 8 + angles = torch.rand(seq_len, dim, generator=generator) * 10 + cos, sin = angles.cos().to(torch_device), angles.sin().to(torch_device) + for dtype in (torch.float32, torch.float16, torch.bfloat16): + for unbind_dim in (-1, -2): + for sequence_dim in (1, 2): + shape = (2, seq_len, 3, dim) if sequence_dim == 1 else (2, 3, seq_len, dim) + x = torch.randn(shape, generator=generator).to(torch_device, dtype) + if sequence_dim == 1: + cos_b, sin_b = cos[None, :, None], sin[None, :, None] + else: + cos_b, sin_b = cos[None, None], sin[None, None] + + out = apply_rotary_emb(x, (cos, sin), use_real_unbind_dim=unbind_dim, sequence_dim=sequence_dim) + + assert out.dtype == dtype + assert torch.equal(out, reference(x, cos_b, sin_b, unbind_dim)), (dtype, unbind_dim, sequence_dim) + class Upsample2DBlockTests(unittest.TestCase): def test_upsample_default(self):