diff --git a/examples/controlnet/train_controlnet_flux.py b/examples/controlnet/train_controlnet_flux.py index 72a887256ffd..43bec0feca55 100644 --- a/examples/controlnet/train_controlnet_flux.py +++ b/examples/controlnet/train_controlnet_flux.py @@ -985,7 +985,7 @@ def load_model_hook(models, input_dir): if args.enable_npu_flash_attention: if is_torch_npu_available(): logger.info("npu flash attention enabled.") - flux_transformer.enable_npu_flash_attention() + flux_transformer.set_attention_backend("_native_npu") else: raise ValueError("npu flash attention requires torch_npu extensions and is supported only on npu devices.") diff --git a/examples/controlnet/train_controlnet_sdxl.py b/examples/controlnet/train_controlnet_sdxl.py index 4488dd39273d..59108da85bd4 100644 --- a/examples/controlnet/train_controlnet_sdxl.py +++ b/examples/controlnet/train_controlnet_sdxl.py @@ -51,6 +51,7 @@ UNet2DConditionModel, UniPCMultistepScheduler, ) +from diffusers.models.attention_processor import AttnProcessorNPU from diffusers.optimization import get_scheduler from diffusers.utils import check_min_version, is_wandb_available, make_image_grid from diffusers.utils.hub_utils import load_or_create_model_card, populate_model_card @@ -979,7 +980,7 @@ def load_model_hook(models, input_dir): if args.enable_npu_flash_attention: if is_torch_npu_available(): logger.info("npu flash attention enabled.") - unet.enable_npu_flash_attention() + unet.set_attn_processor(AttnProcessorNPU()) else: raise ValueError("npu flash attention requires torch_npu extensions and is supported only on npu devices.") diff --git a/examples/dreambooth/train_dreambooth_lora_sana.py b/examples/dreambooth/train_dreambooth_lora_sana.py index 360507cfe1de..3279246c6a59 100644 --- a/examples/dreambooth/train_dreambooth_lora_sana.py +++ b/examples/dreambooth/train_dreambooth_lora_sana.py @@ -69,6 +69,7 @@ SanaPipeline, SanaTransformer2DModel, ) +from diffusers.models.attention_processor import AttnProcessorNPU from diffusers.optimization import get_scheduler from diffusers.training_utils import ( _collate_lora_metadata, @@ -1023,7 +1024,7 @@ def main(args): if is_torch_npu_available(): logger.info("npu flash attention enabled.") for block in transformer.transformer_blocks: - block.attn2.set_use_npu_flash_attention(True) + block.attn2.set_processor(AttnProcessorNPU()) else: raise ValueError("npu flash attention requires torch_npu extensions and is supported only on npu device ") diff --git a/examples/research_projects/lpl/train_sdxl_lpl.py b/examples/research_projects/lpl/train_sdxl_lpl.py index 2d03cee6e0f5..f002e12d34b8 100644 --- a/examples/research_projects/lpl/train_sdxl_lpl.py +++ b/examples/research_projects/lpl/train_sdxl_lpl.py @@ -50,6 +50,7 @@ import diffusers from diffusers import AutoencoderKL, DDPMScheduler, StableDiffusionXLPipeline, UNet2DConditionModel +from diffusers.models.attention_processor import AttnProcessorNPU from diffusers.optimization import get_scheduler from diffusers.training_utils import EMAModel, compute_snr from diffusers.utils import check_min_version, is_wandb_available @@ -852,7 +853,7 @@ def main(args): if args.enable_npu_flash_attention: if is_torch_npu_available(): logger.info("npu flash attention enabled.") - unet.enable_npu_flash_attention() + unet.set_attn_processor(AttnProcessorNPU()) else: raise ValueError("npu flash attention requires torch_npu extensions and is supported only on npu devices.") if args.enable_xformers_memory_efficient_attention: diff --git a/examples/research_projects/pytorch_xla/inference/flux/flux_inference.py b/examples/research_projects/pytorch_xla/inference/flux/flux_inference.py index 35cb015a6cc7..5be0039c25f2 100644 --- a/examples/research_projects/pytorch_xla/inference/flux/flux_inference.py +++ b/examples/research_projects/pytorch_xla/inference/flux/flux_inference.py @@ -36,7 +36,7 @@ def _main(index, args, text_pipe, ckpt_id): flux_pipe = FluxPipeline.from_pretrained( ckpt_id, text_encoder=None, tokenizer=None, text_encoder_2=None, tokenizer_2=None, torch_dtype=torch.bfloat16 ).to(device0) - flux_pipe.transformer.enable_xla_flash_attention(partition_spec=("data", None, None, None), is_flux=True) + flux_pipe.transformer.set_attention_backend("_native_xla") FlashAttention.DEFAULT_BLOCK_SIZES = { "block_q": 1536, "block_k_major": 1536, diff --git a/examples/research_projects/pytorch_xla/inference/flux/flux_inference_spmd.py b/examples/research_projects/pytorch_xla/inference/flux/flux_inference_spmd.py index 9d1eeeae1b0d..dccfec4e1e3b 100644 --- a/examples/research_projects/pytorch_xla/inference/flux/flux_inference_spmd.py +++ b/examples/research_projects/pytorch_xla/inference/flux/flux_inference_spmd.py @@ -104,7 +104,7 @@ def main(args): spec[largest_dim] = "model" xs.mark_sharding(param, mesh, tuple(spec)) - flux_pipe.transformer.enable_xla_flash_attention(partition_spec=("data", None, None, None), is_flux=True) + flux_pipe.transformer.set_attention_backend("_native_xla") FlashAttention.DEFAULT_BLOCK_SIZES = { "block_q": 1536, "block_k_major": 1536, diff --git a/examples/research_projects/pytorch_xla/training/text_to_image/train_text_to_image_xla.py b/examples/research_projects/pytorch_xla/training/text_to_image/train_text_to_image_xla.py index 021b732ad438..eef5df1bce01 100644 --- a/examples/research_projects/pytorch_xla/training/text_to_image/train_text_to_image_xla.py +++ b/examples/research_projects/pytorch_xla/training/text_to_image/train_text_to_image_xla.py @@ -24,6 +24,7 @@ StableDiffusionPipeline, UNet2DConditionModel, ) +from diffusers.models.attention_processor import XLAFlashAttnProcessor2_0 from diffusers.training_utils import compute_snr from diffusers.utils import is_wandb_available from diffusers.utils.hub_utils import load_or_create_model_card, populate_model_card @@ -520,7 +521,7 @@ def main(args): from torch_xla.distributed.fsdp.utils import apply_xla_patch_to_nn_linear unet = apply_xla_patch_to_nn_linear(unet, xs.xla_patched_nn_linear_forward) - unet.enable_xla_flash_attention(partition_spec=("data", None, None, None)) + unet.set_attn_processor(XLAFlashAttnProcessor2_0(partition_spec=("data", None, None, None))) vae.requires_grad_(False) text_encoder.requires_grad_(False) diff --git a/examples/research_projects/sana/train_sana_sprint_diffusers.py b/examples/research_projects/sana/train_sana_sprint_diffusers.py index bd164ea2119e..61031fcf77a4 100644 --- a/examples/research_projects/sana/train_sana_sprint_diffusers.py +++ b/examples/research_projects/sana/train_sana_sprint_diffusers.py @@ -52,7 +52,7 @@ SanaSprintPipeline, SanaTransformer2DModel, ) -from diffusers.models.attention_processor import Attention +from diffusers.models.attention_processor import Attention, AttnProcessorNPU from diffusers.optimization import get_scheduler from diffusers.training_utils import ( free_memory, @@ -1106,9 +1106,9 @@ def main(args): if is_torch_npu_available(): logger.info("npu flash attention enabled.") for block in transformer.transformer_blocks: - block.attn2.set_use_npu_flash_attention(True) + block.attn2.set_processor(AttnProcessorNPU()) for block in pretrained_model.transformer_blocks: - block.attn2.set_use_npu_flash_attention(True) + block.attn2.set_processor(AttnProcessorNPU()) else: raise ValueError("npu flash attention requires torch_npu extensions and is supported only on npu device ") diff --git a/examples/text_to_image/train_text_to_image_lora_sdxl.py b/examples/text_to_image/train_text_to_image_lora_sdxl.py index 442b9c7e9e48..21bb698be190 100644 --- a/examples/text_to_image/train_text_to_image_lora_sdxl.py +++ b/examples/text_to_image/train_text_to_image_lora_sdxl.py @@ -51,6 +51,7 @@ UNet2DConditionModel, ) from diffusers.loaders import StableDiffusionLoraLoaderMixin +from diffusers.models.attention_processor import AttnProcessorNPU from diffusers.optimization import get_scheduler from diffusers.training_utils import _set_state_dict_into_text_encoder, cast_training_params, compute_snr from diffusers.utils import ( @@ -679,7 +680,7 @@ def main(args): if args.enable_npu_flash_attention: if is_torch_npu_available(): logger.info("npu flash attention enabled.") - unet.enable_npu_flash_attention() + unet.set_attn_processor(AttnProcessorNPU()) else: raise ValueError("npu flash attention requires torch_npu extensions and is supported only on npu devices.") diff --git a/examples/text_to_image/train_text_to_image_sdxl.py b/examples/text_to_image/train_text_to_image_sdxl.py index dcab1ddefaa7..bff0725c9554 100644 --- a/examples/text_to_image/train_text_to_image_sdxl.py +++ b/examples/text_to_image/train_text_to_image_sdxl.py @@ -46,6 +46,7 @@ import diffusers from diffusers import AutoencoderKL, DDPMScheduler, StableDiffusionXLPipeline, UNet2DConditionModel +from diffusers.models.attention_processor import AttnProcessorNPU from diffusers.optimization import get_scheduler from diffusers.training_utils import EMAModel, compute_snr from diffusers.utils import check_min_version, is_wandb_available @@ -736,7 +737,7 @@ def main(args): if args.enable_npu_flash_attention: if is_torch_npu_available(): logger.info("npu flash attention enabled.") - unet.enable_npu_flash_attention() + unet.set_attn_processor(AttnProcessorNPU()) else: raise ValueError("npu flash attention requires torch_npu extensions and is supported only on npu devices.") if args.enable_xformers_memory_efficient_attention: diff --git a/src/diffusers/models/attention.py b/src/diffusers/models/attention.py index 3a7e810697b4..7113e4e09e3a 100644 --- a/src/diffusers/models/attention.py +++ b/src/diffusers/models/attention.py @@ -175,13 +175,15 @@ def set_attention_backend(self, backend: str): self.processor._attention_backend = backend def set_use_npu_flash_attention(self, use_npu_flash_attention: bool) -> None: - """ - Set whether to use NPU flash attention from `torch_npu` or not. - - Args: - use_npu_flash_attention (`bool`): Whether to use NPU flash attention or not. - """ + # Deprecated and scheduled for removal in version 0.46.0. + deprecation_message = ( + "`set_use_npu_flash_attention` is deprecated and will be removed in version 0.46.0. Use" + ' `set_attention_backend("_native_npu")` instead.' + ) + deprecate("set_use_npu_flash_attention", "0.46.0", deprecation_message, standard_warn=False) + self._set_use_npu_flash_attention(use_npu_flash_attention) + def _set_use_npu_flash_attention(self, use_npu_flash_attention: bool) -> None: if use_npu_flash_attention: if not is_torch_npu_available(): raise ImportError("torch_npu is not available") @@ -194,17 +196,20 @@ def set_use_xla_flash_attention( partition_spec: tuple[str | None, ...] | None = None, is_flux=False, ) -> None: - """ - Set whether to use XLA flash attention from `torch_xla` or not. + # Deprecated and scheduled for removal in version 0.46.0. + deprecation_message = ( + "`set_use_xla_flash_attention` is deprecated and will be removed in version 0.46.0. Use" + ' `set_attention_backend("_native_xla")` instead.' + ) + deprecate("set_use_xla_flash_attention", "0.46.0", deprecation_message, standard_warn=False) + self._set_use_xla_flash_attention(use_xla_flash_attention, partition_spec, is_flux) - Args: - use_xla_flash_attention (`bool`): - Whether to use pallas flash attention kernel from `torch_xla` or not. - partition_spec (`tuple[]`, *optional*): - Specify the partition specification if using SPMD. Otherwise None. - is_flux (`bool`, *optional*, defaults to `False`): - Whether the model is a Flux model. - """ + def _set_use_xla_flash_attention( + self, + use_xla_flash_attention: bool, + partition_spec: tuple[str | None, ...] | None = None, + is_flux=False, + ) -> None: if use_xla_flash_attention: if not is_torch_xla_available(): raise ImportError("torch_xla is not available") diff --git a/src/diffusers/models/attention_processor.py b/src/diffusers/models/attention_processor.py index d01348639ee6..861bfbf1fc8a 100755 --- a/src/diffusers/models/attention_processor.py +++ b/src/diffusers/models/attention_processor.py @@ -314,15 +314,20 @@ def set_use_xla_flash_attention( partition_spec: tuple[str | None, ...] | None = None, is_flux=False, ) -> None: - r""" - Set whether to use xla flash attention from `torch_xla` or not. + # Deprecated and scheduled for removal in version 0.46.0. + deprecation_message = ( + "`set_use_xla_flash_attention` is deprecated and will be removed in version 0.46.0. Use" + " `set_processor(XLAFlashAttnProcessor2_0(partition_spec))` instead." + ) + deprecate("set_use_xla_flash_attention", "0.46.0", deprecation_message, standard_warn=False) + self._set_use_xla_flash_attention(use_xla_flash_attention, partition_spec, is_flux) - Args: - use_xla_flash_attention (`bool`): - Whether to use pallas flash attention kernel from `torch_xla` or not. - partition_spec (`tuple[]`, *optional*): - Specify the partition specification if using SPMD. Otherwise None. - """ + def _set_use_xla_flash_attention( + self, + use_xla_flash_attention: bool, + partition_spec: tuple[str | None, ...] | None = None, + is_flux=False, + ) -> None: if use_xla_flash_attention: if not is_torch_xla_available: raise "torch_xla is not available" @@ -342,10 +347,15 @@ def set_use_xla_flash_attention( self.set_processor(processor) def set_use_npu_flash_attention(self, use_npu_flash_attention: bool) -> None: - r""" - Set whether to use npu flash attention from `torch_npu` or not. + # Deprecated and scheduled for removal in version 0.46.0. + deprecation_message = ( + "`set_use_npu_flash_attention` is deprecated and will be removed in version 0.46.0. Use" + " `set_processor(AttnProcessorNPU())` instead." + ) + deprecate("set_use_npu_flash_attention", "0.46.0", deprecation_message, standard_warn=False) + self._set_use_npu_flash_attention(use_npu_flash_attention) - """ + def _set_use_npu_flash_attention(self, use_npu_flash_attention: bool) -> None: if use_npu_flash_attention: processor = AttnProcessorNPU() else: diff --git a/src/diffusers/models/modeling_utils.py b/src/diffusers/models/modeling_utils.py index 2b2d78f7c23d..561ad1ce4d58 100644 --- a/src/diffusers/models/modeling_utils.py +++ b/src/diffusers/models/modeling_utils.py @@ -129,6 +129,17 @@ def __exit__(self, *args, **kwargs): _LOW_CPU_MEM_USAGE_DEFAULT = True +NPU_FLASH_ATTENTION_DEPRECATION_MESSAGE = ( + "`enable_npu_flash_attention`, `disable_npu_flash_attention` and `set_use_npu_flash_attention` are deprecated and" + ' will be removed in version 0.46.0. Use `set_attention_backend("_native_npu")` instead. For models that do not' + " support attention backends, such as UNets, use `set_attn_processor(AttnProcessorNPU())`." +) +XLA_FLASH_ATTENTION_DEPRECATION_MESSAGE = ( + "`enable_xla_flash_attention`, `disable_xla_flash_attention` and `set_use_xla_flash_attention` are deprecated and" + ' will be removed in version 0.46.0. Use `set_attention_backend("_native_xla")` instead. For models that do not' + " support attention backends, such as UNets, use `set_attn_processor(XLAFlashAttnProcessor2_0(partition_spec))`." +) + if is_accelerate_available(): import accelerate @@ -329,12 +340,17 @@ def disable_gradient_checkpointing(self) -> None: self._set_gradient_checkpointing(enable=False) def set_use_npu_flash_attention(self, valid: bool) -> None: - r""" - Set the switch for the npu flash attention. - """ + # Deprecated and scheduled for removal in version 0.46.0. + deprecate( + "set_use_npu_flash_attention", "0.46.0", NPU_FLASH_ATTENTION_DEPRECATION_MESSAGE, standard_warn=False + ) + self._set_use_npu_flash_attention(valid) + def _set_use_npu_flash_attention(self, valid: bool) -> None: def fn_recursive_set_npu_flash_attention(module: torch.nn.Module): - if hasattr(module, "set_use_npu_flash_attention"): + if hasattr(module, "_set_use_npu_flash_attention"): + module._set_use_npu_flash_attention(valid) + elif hasattr(module, "set_use_npu_flash_attention"): module.set_use_npu_flash_attention(valid) for child in module.children(): @@ -345,27 +361,36 @@ def fn_recursive_set_npu_flash_attention(module: torch.nn.Module): fn_recursive_set_npu_flash_attention(module) def enable_npu_flash_attention(self) -> None: - r""" - Enable npu flash attention from torch_npu - - """ - self.set_use_npu_flash_attention(True) + # Deprecated and scheduled for removal in version 0.46.0. + deprecate("enable_npu_flash_attention", "0.46.0", NPU_FLASH_ATTENTION_DEPRECATION_MESSAGE, standard_warn=False) + self._set_use_npu_flash_attention(True) def disable_npu_flash_attention(self) -> None: - r""" - disable npu flash attention from torch_npu - - """ - self.set_use_npu_flash_attention(False) + # Deprecated and scheduled for removal in version 0.46.0. + deprecate( + "disable_npu_flash_attention", "0.46.0", NPU_FLASH_ATTENTION_DEPRECATION_MESSAGE, standard_warn=False + ) + self._set_use_npu_flash_attention(False) def set_use_xla_flash_attention( self, use_xla_flash_attention: bool, partition_spec: Callable | None = None, **kwargs + ) -> None: + # Deprecated and scheduled for removal in version 0.46.0. + deprecate( + "set_use_xla_flash_attention", "0.46.0", XLA_FLASH_ATTENTION_DEPRECATION_MESSAGE, standard_warn=False + ) + self._set_use_xla_flash_attention(use_xla_flash_attention, partition_spec, **kwargs) + + def _set_use_xla_flash_attention( + self, use_xla_flash_attention: bool, partition_spec: Callable | None = None, **kwargs ) -> None: # Recursively walk through all the children. # Any children which exposes the set_use_xla_flash_attention method # gets the message def fn_recursive_set_flash_attention(module: torch.nn.Module): - if hasattr(module, "set_use_xla_flash_attention"): + if hasattr(module, "_set_use_xla_flash_attention"): + module._set_use_xla_flash_attention(use_xla_flash_attention, partition_spec, **kwargs) + elif hasattr(module, "set_use_xla_flash_attention"): module.set_use_xla_flash_attention(use_xla_flash_attention, partition_spec, **kwargs) for child in module.children(): @@ -376,16 +401,16 @@ def fn_recursive_set_flash_attention(module: torch.nn.Module): fn_recursive_set_flash_attention(module) def enable_xla_flash_attention(self, partition_spec: Callable | None = None, **kwargs): - r""" - Enable the flash attention pallals kernel for torch_xla. - """ - self.set_use_xla_flash_attention(True, partition_spec, **kwargs) + # Deprecated and scheduled for removal in version 0.46.0. + deprecate("enable_xla_flash_attention", "0.46.0", XLA_FLASH_ATTENTION_DEPRECATION_MESSAGE, standard_warn=False) + self._set_use_xla_flash_attention(True, partition_spec, **kwargs) def disable_xla_flash_attention(self): - r""" - Disable the flash attention pallals kernel for torch_xla. - """ - self.set_use_xla_flash_attention(False) + # Deprecated and scheduled for removal in version 0.46.0. + deprecate( + "disable_xla_flash_attention", "0.46.0", XLA_FLASH_ATTENTION_DEPRECATION_MESSAGE, standard_warn=False + ) + self._set_use_xla_flash_attention(False) def set_use_memory_efficient_attention_xformers(self, valid: bool, attention_op: Callable | None = None) -> None: # Recursively walk through all the children. diff --git a/tests/others/test_flash_attention_toggles_deprecation.py b/tests/others/test_flash_attention_toggles_deprecation.py new file mode 100644 index 000000000000..0d0287848fc3 --- /dev/null +++ b/tests/others/test_flash_attention_toggles_deprecation.py @@ -0,0 +1,77 @@ +# coding=utf-8 +# Copyright 2026 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch + +import pytest + +from diffusers import UNet2DModel +from diffusers.models.attention_processor import Attention, AttnProcessor2_0 +from diffusers.models.transformers.transformer_flux import FluxAttention + + +def get_model(): + return UNet2DModel( + block_out_channels=(4, 8), + layers_per_block=1, + norm_num_groups=4, + sample_size=8, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "AttnDownBlock2D"), + up_block_types=("AttnUpBlock2D", "UpBlock2D"), + ) + + +def assert_warns_once(fn, name): + with pytest.warns(FutureWarning, match=f"`set_use_{name}_flash_attention` (is|are) deprecated") as record: + fn() + assert len([w for w in record if w.category is FutureWarning]) == 1 + + +# `deprecate()` raises once the library reaches the removal version, so these tests fail until the methods are removed. +@pytest.mark.parametrize("name", ["npu", "xla"]) +class TestFlashAttentionTogglesDeprecation: + def test_enable_on_model(self, name): + model = get_model() + with patch.object(model, f"_set_use_{name}_flash_attention") as set_use: + assert_warns_once(getattr(model, f"enable_{name}_flash_attention"), name) + assert set_use.call_args.args[0] is True + + def test_disable_on_model(self, name): + model = get_model() + assert_warns_once(getattr(model, f"disable_{name}_flash_attention"), name) + + processors = [module.processor for module in model.modules() if isinstance(module, Attention)] + assert len(processors) > 0 + assert all(isinstance(processor, AttnProcessor2_0) for processor in processors) + + def test_set_use_on_model(self, name): + model = get_model() + assert_warns_once(lambda: getattr(model, f"set_use_{name}_flash_attention")(False), name) + + def test_set_use_on_attention(self, name): + attention = Attention(query_dim=8, heads=1, dim_head=8) + assert_warns_once(lambda: getattr(attention, f"set_use_{name}_flash_attention")(False), name) + assert isinstance(attention.processor, AttnProcessor2_0) + + def test_set_use_on_attention_module_mixin(self, name): + attention = FluxAttention(query_dim=8, heads=1, dim_head=8) + with ( + patch(f"diffusers.models.attention.is_torch_{name}_available", return_value=True), + patch.object(attention, "set_attention_backend") as set_attention_backend, + ): + assert_warns_once(lambda: getattr(attention, f"set_use_{name}_flash_attention")(True), name) + set_attention_backend.assert_called_once_with(f"_native_{name}")