diff --git a/src/diffusers/models/transformers/transformer_cogview4.py b/src/diffusers/models/transformers/transformer_cogview4.py index 66d4970fafcf..34b8f4b26a6d 100644 --- a/src/diffusers/models/transformers/transformer_cogview4.py +++ b/src/diffusers/models/transformers/transformer_cogview4.py @@ -113,6 +113,54 @@ def forward( ) +def _apply_rotary_emb_to_latents_( + query: torch.Tensor, + key: torch.Tensor, + text_seq_length: int, + image_rotary_emb: tuple[torch.Tensor, torch.Tensor], +) -> None: + """ + Applies the rotary embedding in place to the latent part of `query` and `key`, with the same result as + `apply_rotary_emb(..., use_real_unbind_dim=-2)`. + """ + cos, sin = image_rotary_emb + use_split_half = ( + not torch.is_grad_enabled() + and isinstance(cos, torch.Tensor) + and isinstance(sin, torch.Tensor) + and cos.dim() == 2 + and sin.dim() == 2 + and cos.dtype == torch.float32 + and sin.dtype == torch.float32 + and cos.device == query.device + and sin.device == query.device + and query.shape[-1] % 2 == 0 + ) + if not use_split_half: + from ..embeddings import apply_rotary_emb + + query[:, :, text_seq_length:, :] = apply_rotary_emb( + query[:, :, text_seq_length:, :], image_rotary_emb, use_real_unbind_dim=-2 + ) + key[:, :, text_seq_length:, :] = apply_rotary_emb( + key[:, :, text_seq_length:, :], image_rotary_emb, use_real_unbind_dim=-2 + ) + return + + # Same fp32 products and sums as `apply_rotary_emb`, written into the two halves of `x` without building the + # rotated copy; the `out=` store rounds to the input dtype once, like `.to(x.dtype)`. + half = query.shape[-1] // 2 + sin1, sin2 = sin[:, :half], sin[:, half:] + for x in (query, key): + x = x[:, :, text_seq_length:, :] + x1, x2 = x[..., :half], x[..., half:] + x_cos = x * cos + x2_sin1 = x2 * sin1 + x1_sin2 = x1 * sin2 + torch.sub(x_cos[..., :half], x2_sin1, out=x1) + torch.add(x_cos[..., half:], x1_sin2, out=x2) + + class CogView4AttnProcessor: """ Processor for implementing scaled dot-product attention for the CogView4 model. It applies a rotary embedding on @@ -157,14 +205,7 @@ def __call__( # 3. Rotational positional embeddings applied to latent stream if image_rotary_emb is not None: - from ..embeddings import apply_rotary_emb - - query[:, :, text_seq_length:, :] = apply_rotary_emb( - query[:, :, text_seq_length:, :], image_rotary_emb, use_real_unbind_dim=-2 - ) - key[:, :, text_seq_length:, :] = apply_rotary_emb( - key[:, :, text_seq_length:, :], image_rotary_emb, use_real_unbind_dim=-2 - ) + _apply_rotary_emb_to_latents_(query, key, text_seq_length, image_rotary_emb) # 4. Attention if attention_mask is not None: @@ -751,7 +792,9 @@ def forward( # 1. RoPE if image_rotary_emb is None: - image_rotary_emb = self.rope(hidden_states) + freqs_cos, freqs_sin = self.rope(hidden_states) + # The RoPE tables are built on CPU; move them once here instead of once per attention layer. + image_rotary_emb = (freqs_cos.to(hidden_states.device), freqs_sin.to(hidden_states.device)) # 2. Patch & Timestep embeddings p = self.config.patch_size diff --git a/tests/models/transformers/test_models_transformer_cogview4.py b/tests/models/transformers/test_models_transformer_cogview4.py index 9acd2393a21e..042f1e07c25d 100644 --- a/tests/models/transformers/test_models_transformer_cogview4.py +++ b/tests/models/transformers/test_models_transformer_cogview4.py @@ -12,9 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from unittest.mock import patch + +import pytest import torch from diffusers import CogView4Transformer2DModel +from diffusers.models.embeddings import apply_rotary_emb +from diffusers.models.transformers.transformer_cogview4 import _apply_rotary_emb_to_latents_ from diffusers.utils.torch_utils import randn_tensor from ...testing_utils import enable_full_determinism, torch_device @@ -109,3 +114,47 @@ def test_gradient_checkpointing_is_applied(self): class TestCogView4TransformerLoRA(CogView4TransformerTesterConfig, LoraTesterMixin): pass + + +class TestCogView4RotaryEmbedding: + text_seq_length = 5 + image_seq_length = 12 + + def get_inputs(self, tables_device): + generator = torch.Generator("cpu").manual_seed(0) + shape = (2, 3, self.text_seq_length + self.image_seq_length, 16) + query = torch.randn(shape, generator=generator).to(torch_device, torch.bfloat16) + key = torch.randn(shape, generator=generator).to(torch_device, torch.bfloat16) + cos = torch.randn(self.image_seq_length, 16, generator=generator).to(tables_device) + sin = torch.randn(self.image_seq_length, 16, generator=generator).to(tables_device) + return query, key, (cos, sin) + + def check_against_apply_rotary_emb(self, tables_device, expected_calls): + query, key, image_rotary_emb = self.get_inputs(tables_device) + expected = [] + for x in (query, key): + x = x.clone() + x[:, :, self.text_seq_length :, :] = apply_rotary_emb( + x[:, :, self.text_seq_length :, :], image_rotary_emb, use_real_unbind_dim=-2 + ) + expected.append(x) + assert not torch.equal(query, expected[0]) + + with patch("diffusers.models.embeddings.apply_rotary_emb", wraps=apply_rotary_emb) as mock: + _apply_rotary_emb_to_latents_(query, key, self.text_seq_length, image_rotary_emb) + + assert mock.call_count == expected_calls + assert torch.equal(query, expected[0]) + assert torch.equal(key, expected[1]) + + @torch.no_grad() + def test_split_half_matches_apply_rotary_emb(self): + self.check_against_apply_rotary_emb(torch_device, expected_calls=0) + + @pytest.mark.skipif(torch_device == "cpu", reason="CPU tables are on the input device") + @torch.no_grad() + def test_cpu_tables_fall_back_to_apply_rotary_emb(self): + self.check_against_apply_rotary_emb("cpu", expected_calls=2) + + def test_grad_enabled_falls_back_to_apply_rotary_emb(self): + self.check_against_apply_rotary_emb(torch_device, expected_calls=2)