Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions src/diffusers/models/attention_dispatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -3813,6 +3813,33 @@ def _prepare_additive_attn_mask(
return attn_mask


# On ROCm, SDPA runs AOTriton kernels whose configs are tuned for contiguous (B, H, S, D) inputs. When K/V reach SDPA as
# the strided (B, H, S, D) view of a (B, S, H, D) tensor and no longer fit in the on-die cache, the kernel gets several
# times slower, and copying K/V first is much faster and bit-identical. For small K/V the copy costs more than it saves.
# Values are the K+V size (bytes) from which copying was measured to be faster; architectures not listed are unchanged.
_ROCM_SDPA_PACK_KV_MIN_BYTES = {
# Radeon PRO W7800 (64 MB Infinity Cache): copying was slower at 54 MiB of K/V and faster from 63.8 MiB up.
"gfx1100": 60 * 1024**2,
}


@functools.lru_cache(maxsize=None)
def _rocm_sdpa_pack_kv_min_bytes(device: torch.device) -> int | None:
arch = torch.cuda.get_device_properties(device).gcnArchName.split(":")[0]
return _ROCM_SDPA_PACK_KV_MIN_BYTES.get(arch)


def _maybe_pack_kv_for_rocm_sdpa(key: torch.Tensor, value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
if torch.version.hip is None or key.device.type != "cuda" or torch.compiler.is_compiling():
return key, value
min_bytes = _rocm_sdpa_pack_kv_min_bytes(key.device)
# Below the threshold K/V still fit in the on-die cache, where reading the strided view is cheap and the copy only
# adds time. Architectures without a measured threshold keep the current path.
if min_bytes is None or (key.numel() + value.numel()) * key.element_size() < min_bytes:

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's supplement this rule with an explanation.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the quick review! Done in f383ab1: added a comment above the check explaining that below the threshold K/V still fit in the on-die cache, so the copy only adds time, and that architectures without a measured threshold keep the current path.

return key, value
return key.contiguous(), value.contiguous()


@_AttentionBackendRegistry.register(
AttentionBackendName.NATIVE,
constraints=[_check_device, _check_shape],
Expand Down Expand Up @@ -3847,6 +3874,7 @@ def _native_attention(

if _parallel_config is None:
query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value))
key, value = _maybe_pack_kv_for_rocm_sdpa(key, value)
out = torch.nn.functional.scaled_dot_product_attention(
query=query,
key=key,
Expand Down
38 changes: 38 additions & 0 deletions tests/models/test_attention_dispatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
from diffusers.models.transformers.transformer_flux import FluxAttention

from ..testing_utils import (
IS_ROCM_SYSTEM,
is_attention,
is_context_parallel,
is_kernels_available,
Expand Down Expand Up @@ -350,3 +351,40 @@ def test_output_close_to_native(self):
assert not torch.equal(out, ref)
cos = F.cosine_similarity(out.float().flatten(), ref.float().flatten(), dim=0)
assert cos > 0.99


@is_attention
@require_torch_accelerator
@pytest.mark.skipif(not IS_ROCM_SYSTEM, reason="K/V are only copied before native SDPA on ROCm.")
class TestNativeAttentionRocmKVPacking:
"""On ROCm, the native backend copies strided K/V before SDPA once K+V reach the architecture's threshold."""

@pytest.fixture
def kv_contiguity(self, monkeypatch):
seen = []
sdpa = F.scaled_dot_product_attention

def recording_sdpa(query, key, value, **kwargs):
seen.append((key.is_contiguous(), value.is_contiguous()))
return sdpa(query, key, value, **kwargs)

monkeypatch.setattr(F, "scaled_dot_product_attention", recording_sdpa)
return seen

# K+V of the (1, 64, 2, 64) bf16 inputs below take 32 KiB.
@pytest.mark.parametrize(
"min_bytes, packed",
[(32 * 1024, True), (32 * 1024 + 1, False), (None, False)],
ids=["at_threshold", "below_threshold", "arch_not_listed"],
)
def test_kv_copied_only_from_threshold(self, monkeypatch, kv_contiguity, min_bytes, packed):
torch.manual_seed(0)
query, key, value = (torch.randn(1, 64, 2, 64, device=torch_device, dtype=torch.bfloat16) for _ in range(3))

monkeypatch.setattr(attention_dispatch, "_rocm_sdpa_pack_kv_min_bytes", lambda device: min_bytes)
out = dispatch_attention_fn(query, key, value, backend=AttentionBackendName.NATIVE)
assert kv_contiguity == [(packed, packed)]

monkeypatch.setattr(attention_dispatch, "_rocm_sdpa_pack_kv_min_bytes", lambda device: None)
ref = dispatch_attention_fn(query, key, value, backend=AttentionBackendName.NATIVE)
assert torch.equal(out, ref)
Loading