diff --git a/src/diffusers/models/attention_dispatch.py b/src/diffusers/models/attention_dispatch.py index a9805676dde7..9d2135034a9f 100644 --- a/src/diffusers/models/attention_dispatch.py +++ b/src/diffusers/models/attention_dispatch.py @@ -3813,6 +3813,33 @@ def _prepare_additive_attn_mask( return attn_mask +# On ROCm, SDPA runs AOTriton kernels whose configs are tuned for contiguous (B, H, S, D) inputs. When K/V reach SDPA as +# the strided (B, H, S, D) view of a (B, S, H, D) tensor and no longer fit in the on-die cache, the kernel gets several +# times slower, and copying K/V first is much faster and bit-identical. For small K/V the copy costs more than it saves. +# Values are the K+V size (bytes) from which copying was measured to be faster; architectures not listed are unchanged. +_ROCM_SDPA_PACK_KV_MIN_BYTES = { + # Radeon PRO W7800 (64 MB Infinity Cache): copying was slower at 54 MiB of K/V and faster from 63.8 MiB up. + "gfx1100": 60 * 1024**2, +} + + +@functools.lru_cache(maxsize=None) +def _rocm_sdpa_pack_kv_min_bytes(device: torch.device) -> int | None: + arch = torch.cuda.get_device_properties(device).gcnArchName.split(":")[0] + return _ROCM_SDPA_PACK_KV_MIN_BYTES.get(arch) + + +def _maybe_pack_kv_for_rocm_sdpa(key: torch.Tensor, value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + if torch.version.hip is None or key.device.type != "cuda" or torch.compiler.is_compiling(): + return key, value + min_bytes = _rocm_sdpa_pack_kv_min_bytes(key.device) + # Below the threshold K/V still fit in the on-die cache, where reading the strided view is cheap and the copy only + # adds time. Architectures without a measured threshold keep the current path. + if min_bytes is None or (key.numel() + value.numel()) * key.element_size() < min_bytes: + return key, value + return key.contiguous(), value.contiguous() + + @_AttentionBackendRegistry.register( AttentionBackendName.NATIVE, constraints=[_check_device, _check_shape], @@ -3847,6 +3874,7 @@ def _native_attention( if _parallel_config is None: query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value)) + key, value = _maybe_pack_kv_for_rocm_sdpa(key, value) out = torch.nn.functional.scaled_dot_product_attention( query=query, key=key, diff --git a/tests/models/test_attention_dispatch.py b/tests/models/test_attention_dispatch.py index 099a6fd5e913..aabf8eeb3f2c 100644 --- a/tests/models/test_attention_dispatch.py +++ b/tests/models/test_attention_dispatch.py @@ -35,6 +35,7 @@ from diffusers.models.transformers.transformer_flux import FluxAttention from ..testing_utils import ( + IS_ROCM_SYSTEM, is_attention, is_context_parallel, is_kernels_available, @@ -350,3 +351,40 @@ def test_output_close_to_native(self): assert not torch.equal(out, ref) cos = F.cosine_similarity(out.float().flatten(), ref.float().flatten(), dim=0) assert cos > 0.99 + + +@is_attention +@require_torch_accelerator +@pytest.mark.skipif(not IS_ROCM_SYSTEM, reason="K/V are only copied before native SDPA on ROCm.") +class TestNativeAttentionRocmKVPacking: + """On ROCm, the native backend copies strided K/V before SDPA once K+V reach the architecture's threshold.""" + + @pytest.fixture + def kv_contiguity(self, monkeypatch): + seen = [] + sdpa = F.scaled_dot_product_attention + + def recording_sdpa(query, key, value, **kwargs): + seen.append((key.is_contiguous(), value.is_contiguous())) + return sdpa(query, key, value, **kwargs) + + monkeypatch.setattr(F, "scaled_dot_product_attention", recording_sdpa) + return seen + + # K+V of the (1, 64, 2, 64) bf16 inputs below take 32 KiB. + @pytest.mark.parametrize( + "min_bytes, packed", + [(32 * 1024, True), (32 * 1024 + 1, False), (None, False)], + ids=["at_threshold", "below_threshold", "arch_not_listed"], + ) + def test_kv_copied_only_from_threshold(self, monkeypatch, kv_contiguity, min_bytes, packed): + torch.manual_seed(0) + query, key, value = (torch.randn(1, 64, 2, 64, device=torch_device, dtype=torch.bfloat16) for _ in range(3)) + + monkeypatch.setattr(attention_dispatch, "_rocm_sdpa_pack_kv_min_bytes", lambda device: min_bytes) + out = dispatch_attention_fn(query, key, value, backend=AttentionBackendName.NATIVE) + assert kv_contiguity == [(packed, packed)] + + monkeypatch.setattr(attention_dispatch, "_rocm_sdpa_pack_kv_min_bytes", lambda device: None) + ref = dispatch_attention_fn(query, key, value, backend=AttentionBackendName.NATIVE) + assert torch.equal(out, ref)