From d9baefa32fd049880268d35cb5af53dd62f693e8 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Fri, 20 Nov 2026 12:48:59 +0530 Subject: [PATCH] docs: fix autodocstring layout for components in modular diffusers. --- .../anima/modular_blocks_anima.py | 37 ++-- .../modular_pipelines/cosmos/denoise.py | 3 +- .../cosmos/modular_blocks_cosmos3.py | 54 ++++-- .../modular_blocks_cosmos3_distilled.py | 13 +- .../modular_pipelines/echo/denoise.py | 2 +- .../echo/modular_blocks_echo.py | 23 ++- .../ernie_image/modular_blocks_ernie_image.py | 22 ++- .../flux/modular_blocks_flux.py | 26 ++- .../flux/modular_blocks_flux_kontext.py | 26 ++- .../flux2/modular_blocks_flux2.py | 21 ++- .../flux2/modular_blocks_flux2_klein.py | 23 ++- .../flux2/modular_blocks_flux2_klein_base.py | 31 ++-- .../modular_pipelines/flux3/denoise.py | 8 +- .../flux3/modular_blocks_flux3.py | 34 ++-- .../flux3/modular_blocks_flux3_distilled.py | 21 ++- .../helios/modular_blocks_helios.py | 29 ++- .../helios/modular_blocks_helios_pyramid.py | 33 ++-- ...modular_blocks_helios_pyramid_distilled.py | 29 ++- .../modular_blocks_hunyuan_video1_5.py | 69 ++++--- .../ideogram4/before_denoise.py | 4 +- .../modular_pipelines/ideogram4/decoders.py | 3 +- .../modular_pipelines/ideogram4/denoise.py | 5 +- .../modular_pipelines/ideogram4/encoders.py | 11 +- .../ideogram4/modular_blocks_ideogram4.py | 19 +- .../kandinsky_s2v/encoders.py | 6 +- .../modular_blocks_kandinsky_s2v.py | 18 +- .../modular_blocks_kandinsky_s2v_distilled.py | 14 +- .../modular_pipelines/krea2/before_denoise.py | 6 +- .../modular_pipelines/krea2/decoders.py | 3 +- .../modular_pipelines/krea2/denoise.py | 8 +- .../modular_pipelines/krea2/encoders.py | 9 +- .../krea2/modular_blocks_krea2.py | 15 +- .../krea2/modular_blocks_krea2_turbo.py | 12 +- .../ltx/modular_blocks_ltx.py | 54 ++++-- .../ltx2/modular_blocks_ltx2.py | 173 +++++++++++++----- .../ltx2/modular_blocks_ltx25.py | 38 ++-- .../minimax_h3/modular_blocks_minimax_h3.py | 50 +++-- .../modular_blocks_minimax_h3_image.py | 27 ++- .../modular_blocks_minimax_music3.py | 23 ++- .../modular_pipeline_utils.py | 13 +- .../p_h3_image/modular_blocks_p_h3_image.py | 41 +++-- .../p_h3_image/prompt_upsampler.py | 4 +- .../pan2/modular_blocks_pan2.py | 29 ++- .../modular_blocks_pika_h3_flex.py | 50 +++-- .../qwenimage/before_denoise.py | 16 +- .../modular_pipelines/qwenimage/decoders.py | 13 +- .../modular_pipelines/qwenimage/denoise.py | 37 ++-- .../modular_pipelines/qwenimage/encoders.py | 41 +++-- .../modular_pipelines/qwenimage/inputs.py | 6 +- .../qwenimage/modular_blocks_qwenimage.py | 84 ++++++--- .../modular_blocks_qwenimage_edit.py | 55 ++++-- .../modular_blocks_qwenimage_edit_plus.py | 35 ++-- .../modular_blocks_qwenimage_layered.py | 34 ++-- .../qwenimage21/before_denoise.py | 4 +- .../modular_pipelines/qwenimage21/decoders.py | 3 +- .../modular_pipelines/qwenimage21/denoise.py | 16 +- .../modular_pipelines/qwenimage21/encoders.py | 18 +- .../qwenimage21/modular_blocks_qwenimage21.py | 41 +++-- .../modular_blocks_qwenimage21_turbo.py | 34 ++-- .../modular_blocks_stable_diffusion_3.py | 36 ++-- .../modular_blocks_stable_diffusion_xl.py | 19 +- .../wan/modular_blocks_wan.py | 14 +- .../wan/modular_blocks_wan22.py | 19 +- .../wan/modular_blocks_wan22_i2v.py | 22 ++- .../wan/modular_blocks_wan_i2v.py | 35 ++-- .../wan/modular_blocks_wan_vace.py | 19 +- .../modular_blocks_wan_animate_2.py | 24 ++- .../modular_blocks_wan_animate_2_distilled.py | 24 ++- .../z_image/modular_blocks_z_image.py | 28 ++- 69 files changed, 1207 insertions(+), 609 deletions(-) diff --git a/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py b/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py index f17538fd258f..dc5a0d0a3403 100644 --- a/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py +++ b/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py @@ -34,8 +34,10 @@ class AnimaCoreDenoiseStep(SequentialPipelineBlocks): Denoise block that takes encoded Anima text inputs and runs the denoising process. Components: - text_conditioner (`AnimaTextConditioner`) transformer (`CosmosTransformer3DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) + - text_conditioner (`AnimaTextConditioner`) + - transformer (`CosmosTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: qwen_prompt_embeds (`Tensor`): @@ -100,7 +102,8 @@ class AnimaDecodeStep(SequentialPipelineBlocks): Decode Anima latents into generated images. Components: - vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) Inputs: latents (`Tensor`): @@ -132,8 +135,10 @@ class AnimaImg2ImgCoreDenoiseStep(SequentialPipelineBlocks): AnimaImg2ImgVaeEncoderStep. Components: - text_conditioner (`AnimaTextConditioner`) transformer (`CosmosTransformer3DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) + - text_conditioner (`AnimaTextConditioner`) + - transformer (`CosmosTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: qwen_prompt_embeds (`Tensor`): @@ -208,8 +213,10 @@ class AnimaAutoCoreDenoiseStep(AutoPipelineBlocks): `AnimaImg2ImgCoreDenoiseStep` (img2img) is used when image_latents are present. Components: - text_conditioner (`AnimaTextConditioner`) transformer (`CosmosTransformer3DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) + - text_conditioner (`AnimaTextConditioner`) + - transformer (`CosmosTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: qwen_prompt_embeds (`Tensor`): @@ -275,7 +282,8 @@ class AnimaAutoVaeImageEncoderStep(AutoPipelineBlocks): (text-to-image workflow). Components: - vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) Inputs: image (`Image | list`, *optional*): @@ -318,10 +326,15 @@ class AnimaAutoBlocks(SequentialPipelineBlocks): - `img2img`: requires `image`, `prompt` Components: - text_encoder (`Qwen3Model`) tokenizer (`Qwen2Tokenizer`) t5_tokenizer (`T5Tokenizer`) guider - (`ClassifierFreeGuidance`) vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) - text_conditioner (`AnimaTextConditioner`) transformer (`CosmosTransformer3DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) + - text_encoder (`Qwen3Model`) + - tokenizer (`Qwen2Tokenizer`) + - t5_tokenizer (`T5Tokenizer`) + - guider (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) + - text_conditioner (`AnimaTextConditioner`) + - transformer (`CosmosTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/cosmos/denoise.py b/src/diffusers/modular_pipelines/cosmos/denoise.py index 46b1684c3712..b8a2427ce8ca 100644 --- a/src/diffusers/modular_pipelines/cosmos/denoise.py +++ b/src/diffusers/modular_pipelines/cosmos/denoise.py @@ -963,7 +963,8 @@ class Cosmos3TransferDenoiseStep(Cosmos3DenoiseLoopWrapper): Runs the per-chunk transfer denoising loop over scheduler timesteps. Components: - transformer (`Cosmos3OmniTransformer`) scheduler (`UniPCMultistepScheduler`) + - transformer (`Cosmos3OmniTransformer`) + - scheduler (`UniPCMultistepScheduler`) Inputs: timesteps (`Tensor`): diff --git a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py index 864e687adad2..a0b6bcb3c05a 100644 --- a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py +++ b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py @@ -57,7 +57,8 @@ class Cosmos3TransferTextBlocks(SequentialPipelineBlocks): transfer mode using the per-chunk frame count. Components: - video_processor (`VideoProcessor`) text_tokenizer (`AutoTokenizer`) + - video_processor (`VideoProcessor`) + - text_tokenizer (`AutoTokenizer`) Inputs: control_videos (`dict`): @@ -121,7 +122,8 @@ class Cosmos3AutoTextEncoderStep(AutoPipelineBlocks): - Cosmos3TextEncoderStep runs otherwise. Components: - video_processor (`VideoProcessor`) text_tokenizer (`AutoTokenizer`) + - video_processor (`VideoProcessor`) + - text_tokenizer (`AutoTokenizer`) Configs: default_use_system_prompt (default: True) enable_safety_checker (default: True) @@ -204,7 +206,8 @@ class Cosmos3AutoVaeEncoderStep(ConditionalPipelineBlocks): - when no action, image, or video conditioning is provided, this block is skipped. Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Inputs: action (`CosmosActionCondition`, *optional*): @@ -280,7 +283,7 @@ class Cosmos3AutoSoundDecodeStep(AutoPipelineBlocks): - if sound_latents are not provided, this block is skipped. Components: - sound_tokenizer (`Cosmos3AVAEAudioTokenizer`) + - sound_tokenizer (`Cosmos3AVAEAudioTokenizer`) Inputs: sound_latents (`Tensor`, *optional*): @@ -313,7 +316,9 @@ class Cosmos3DecodeStep(SequentialPipelineBlocks): Decodes denoised latents into modality outputs. Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) sound_tokenizer (`Cosmos3AVAEAudioTokenizer`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - sound_tokenizer (`Cosmos3AVAEAudioTokenizer`) Inputs: latents (`Tensor`): @@ -368,7 +373,8 @@ class Cosmos3VisionCoreDenoiseStep(SequentialPipelineBlocks): Runs the text-and-vision Cosmos3 denoising workflow. Components: - transformer (`Cosmos3OmniTransformer`) scheduler (`UniPCMultistepScheduler`) + - transformer (`Cosmos3OmniTransformer`) + - scheduler (`UniPCMultistepScheduler`) Configs: use_native_flow_schedule (default: False) @@ -448,7 +454,8 @@ class Cosmos3VisionSoundCoreDenoiseStep(SequentialPipelineBlocks): Runs the text, vision, and sound Cosmos3 denoising workflow. Components: - transformer (`Cosmos3OmniTransformer`) scheduler (`UniPCMultistepScheduler`) + - transformer (`Cosmos3OmniTransformer`) + - scheduler (`UniPCMultistepScheduler`) Configs: use_native_flow_schedule (default: False) @@ -541,7 +548,8 @@ class Cosmos3VisionActionCoreDenoiseStep(SequentialPipelineBlocks): Runs the text, vision, and action Cosmos3 denoising workflow. Components: - transformer (`Cosmos3OmniTransformer`) scheduler (`UniPCMultistepScheduler`) + - transformer (`Cosmos3OmniTransformer`) + - scheduler (`UniPCMultistepScheduler`) Configs: use_native_flow_schedule (default: False) @@ -638,7 +646,8 @@ class Cosmos3VisionSoundActionCoreDenoiseStep(SequentialPipelineBlocks): Runs the text, vision, sound, and action Cosmos3 denoising workflow. Components: - transformer (`Cosmos3OmniTransformer`) scheduler (`UniPCMultistepScheduler`) + - transformer (`Cosmos3OmniTransformer`) + - scheduler (`UniPCMultistepScheduler`) Configs: use_native_flow_schedule (default: False) @@ -748,8 +757,10 @@ class Cosmos3TransferChunkDenoiseStep(SequentialPipelineBlocks): cross-carry (previous_output, output_chunks) lives on PipelineState. Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) transformer (`Cosmos3OmniTransformer`) scheduler - (`UniPCMultistepScheduler`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - transformer (`Cosmos3OmniTransformer`) + - scheduler (`UniPCMultistepScheduler`) Inputs: chunk_id (`int`, *optional*, defaults to 0): @@ -903,8 +914,10 @@ class Cosmos3TransferCoreDenoiseStep(SequentialPipelineBlocks): Transfer denoise stage: prepare shared text segments once, then run the autoregressive chunk loop. Components: - transformer (`Cosmos3OmniTransformer`) vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) scheduler - (`UniPCMultistepScheduler`) + - transformer (`Cosmos3OmniTransformer`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - scheduler (`UniPCMultistepScheduler`) Inputs: cond_input_ids (`None`): @@ -1031,8 +1044,10 @@ class Cosmos3AutoCoreDenoiseStep(ConditionalPipelineBlocks): - vision runs otherwise. Components: - transformer (`Cosmos3OmniTransformer`) vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) scheduler - (`UniPCMultistepScheduler`) + - transformer (`Cosmos3OmniTransformer`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - scheduler (`UniPCMultistepScheduler`) Configs: use_native_flow_schedule (default: False) @@ -1229,9 +1244,12 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): - `action_inverse_dynamics`: requires `prompt`, `action` Components: - video_processor (`VideoProcessor`) text_tokenizer (`AutoTokenizer`) vae (`AutoencoderKLWan`) transformer - (`Cosmos3OmniTransformer`) scheduler (`UniPCMultistepScheduler`) sound_tokenizer - (`Cosmos3AVAEAudioTokenizer`) + - video_processor (`VideoProcessor`) + - text_tokenizer (`AutoTokenizer`) + - vae (`AutoencoderKLWan`) + - transformer (`Cosmos3OmniTransformer`) + - scheduler (`UniPCMultistepScheduler`) + - sound_tokenizer (`Cosmos3AVAEAudioTokenizer`) Configs: default_use_system_prompt (default: True) enable_safety_checker (default: True) use_native_flow_schedule diff --git a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py index 3d101ffb879a..f0f8ac1835b2 100644 --- a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py +++ b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py @@ -25,7 +25,8 @@ class Cosmos3DistilledAutoVaeEncoderStep(ConditionalPipelineBlocks): - when no image or video conditioning is provided, this block is skipped. Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Inputs: video (`None`, *optional*): @@ -83,7 +84,8 @@ class Cosmos3DistilledVisionCoreDenoiseStep(SequentialPipelineBlocks): Runs the text-and-vision distilled Cosmos3 denoising workflow. Components: - transformer (`Cosmos3OmniTransformer`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Cosmos3OmniTransformer`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Configs: is_distilled (default: True) distilled_sigmas (default: None) @@ -170,8 +172,11 @@ class Cosmos3DistilledBlocks(SequentialPipelineBlocks): - `video2video`: requires `prompt`, `video` Components: - text_tokenizer (`AutoTokenizer`) vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) transformer - (`Cosmos3OmniTransformer`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - text_tokenizer (`AutoTokenizer`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - transformer (`Cosmos3OmniTransformer`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Configs: default_use_system_prompt (default: True) enable_safety_checker (default: True) is_distilled (default: True) diff --git a/src/diffusers/modular_pipelines/echo/denoise.py b/src/diffusers/modular_pipelines/echo/denoise.py index ea6b4c891c82..9f9108bc0f5f 100644 --- a/src/diffusers/modular_pipelines/echo/denoise.py +++ b/src/diffusers/modular_pipelines/echo/denoise.py @@ -359,7 +359,7 @@ class EchoDenoiseLoopStep(LoopSequentialPipelineBlocks): sigma schedule. Components: - transformer (`LTX2VideoTransformer3DModel`) + - transformer (`LTX2VideoTransformer3DModel`) Inputs: sigmas (`list | tuple`): diff --git a/src/diffusers/modular_pipelines/echo/modular_blocks_echo.py b/src/diffusers/modular_pipelines/echo/modular_blocks_echo.py index 16c95d3372bd..57d99c02eeb0 100644 --- a/src/diffusers/modular_pipelines/echo/modular_blocks_echo.py +++ b/src/diffusers/modular_pipelines/echo/modular_blocks_echo.py @@ -36,7 +36,9 @@ class EchoTextConditioningStep(SequentialPipelineBlocks): Positive-only text conditioning for the guidance-free Echo DMD checkpoint. Components: - text_encoder (`PreTrainedModel`) tokenizer (`PreTrainedTokenizerBase`) connectors (`LTX2TextConnectors`) + - text_encoder (`PreTrainedModel`) + - tokenizer (`PreTrainedTokenizerBase`) + - connectors (`LTX2TextConnectors`) Inputs: prompt (`str`): @@ -83,7 +85,7 @@ class EchoDenoiseStep(SequentialPipelineBlocks): Prepare Echo conditioning and target latents, then run stochastic DMD denoising. Components: - transformer (`LTX2VideoTransformer3DModel`) + - transformer (`LTX2VideoTransformer3DModel`) Inputs: first_frame_latents (`Tensor`, *optional*): @@ -166,8 +168,10 @@ class EchoDecoderStep(SequentialPipelineBlocks): Decode Echo video and audio outputs with mixed-precision-safe audio vocoding. Components: - vae (`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) audio_vae (`AutoencoderKLLTX2Audio`) - vocoder (`LTX2Vocoder`) + - vae (`AutoencoderKLLTX2Video`) + - video_processor (`VideoProcessor`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - vocoder (`LTX2Vocoder`) Inputs: latents (`Tensor`): @@ -223,9 +227,14 @@ class EchoBlocks(SequentialPipelineBlocks): stochastic DMD denoising. Components: - text_encoder (`PreTrainedModel`) tokenizer (`PreTrainedTokenizerBase`) connectors (`LTX2TextConnectors`) vae - (`AutoencoderKLLTX2Video`) audio_vae (`AutoencoderKLLTX2Audio`) transformer (`LTX2VideoTransformer3DModel`) - video_processor (`VideoProcessor`) vocoder (`LTX2Vocoder`) + - text_encoder (`PreTrainedModel`) + - tokenizer (`PreTrainedTokenizerBase`) + - connectors (`LTX2TextConnectors`) + - vae (`AutoencoderKLLTX2Video`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - transformer (`LTX2VideoTransformer3DModel`) + - video_processor (`VideoProcessor`) + - vocoder (`LTX2Vocoder`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py b/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py index 17e4eebaffda..ad4b5a29cfba 100644 --- a/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py +++ b/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py @@ -36,7 +36,8 @@ class ErnieImageAutoPromptEnhancerStep(ConditionalPipelineBlocks): - If `use_pe` is `None` or `False`, the step is skipped. Components: - pe (`Ministral3ForCausalLM`) pe_tokenizer (`AutoTokenizer`) + - pe (`Ministral3ForCausalLM`) + - pe_tokenizer (`AutoTokenizer`) Inputs: prompt (`str`, *optional*): @@ -86,8 +87,9 @@ class ErnieImageCoreDenoiseStep(SequentialPipelineBlocks): Denoise block that takes encoded conditions and runs the denoising process for ErnieImage. Components: - transformer (`ErnieImageTransformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) + - transformer (`ErnieImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: prompt_embeds (`list`): @@ -140,10 +142,16 @@ class ErnieImageAutoBlocks(SequentialPipelineBlocks): - `text2image`: requires `prompt` Components: - pe (`Ministral3ForCausalLM`) pe_tokenizer (`AutoTokenizer`) text_encoder (`Mistral3Model`) tokenizer - (`AutoTokenizer`) guider (`ClassifierFreeGuidance`) transformer (`ErnieImageTransformer2DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) vae (`AutoencoderKLFlux2`) pachifier (`ErnieImagePachifier`) - image_processor (`VaeImageProcessor`) + - pe (`Ministral3ForCausalLM`) + - pe_tokenizer (`AutoTokenizer`) + - text_encoder (`Mistral3Model`) + - tokenizer (`AutoTokenizer`) + - guider (`ClassifierFreeGuidance`) + - transformer (`ErnieImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - vae (`AutoencoderKLFlux2`) + - pachifier (`ErnieImagePachifier`) + - image_processor (`VaeImageProcessor`) Inputs: prompt (`str`, *optional*): diff --git a/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py b/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py index a725304c6af6..ab7956505156 100644 --- a/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py +++ b/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py @@ -47,7 +47,8 @@ class FluxImg2ImgVaeEncoderStep(SequentialPipelineBlocks): Vae encoder step that preprocess andencode the image inputs into their latent representations. Components: - image_processor (`VaeImageProcessor`) vae (`AutoencoderKL`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKL`) Inputs: resized_image (`None`, *optional*): @@ -87,7 +88,8 @@ class FluxAutoVaeEncoderStep(AutoPipelineBlocks): step will be skipped. Components: - image_processor (`VaeImageProcessor`) vae (`AutoencoderKL`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKL`) Inputs: resized_image (`None`, *optional*): @@ -130,7 +132,7 @@ class FluxBeforeDenoiseStep(SequentialPipelineBlocks): Before denoise step that prepares the inputs for the denoise step in text-to-image generation. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: height (`int`, *optional*): @@ -190,7 +192,7 @@ class FluxImg2ImgBeforeDenoiseStep(SequentialPipelineBlocks): Before denoise step that prepare the inputs for the denoise step for img2img task. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: height (`int`, *optional*): @@ -265,7 +267,7 @@ class FluxAutoBeforeDenoiseStep(AutoPipelineBlocks): - `FluxImg2ImgBeforeDenoiseStep` (img2img) is used when only `image_latents` is provided. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: height (`int`): @@ -442,7 +444,8 @@ class FluxCoreDenoiseStep(SequentialPipelineBlocks): - for text-to-image generation, all you need to provide is prompt embeddings. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`FluxTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`FluxTransformer2DModel`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -520,9 +523,14 @@ class FluxAutoBlocks(SequentialPipelineBlocks): - `image2image`: requires `image`, `prompt` Components: - text_encoder (`CLIPTextModel`) tokenizer (`CLIPTokenizer`) text_encoder_2 (`T5EncoderModel`) tokenizer_2 - (`T5Tokenizer`) image_processor (`VaeImageProcessor`) vae (`AutoencoderKL`) scheduler - (`FlowMatchEulerDiscreteScheduler`) transformer (`FluxTransformer2DModel`) + - text_encoder (`CLIPTextModel`) + - tokenizer (`CLIPTokenizer`) + - text_encoder_2 (`T5EncoderModel`) + - tokenizer_2 (`T5Tokenizer`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKL`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`FluxTransformer2DModel`) Inputs: prompt (`None`, *optional*): diff --git a/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py b/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py index f937d2555b1a..11d31236b6ba 100644 --- a/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py +++ b/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py @@ -45,7 +45,8 @@ class FluxKontextVaeEncoderStep(SequentialPipelineBlocks): Vae encoder step that preprocess andencode the image inputs into their latent representations. Components: - image_processor (`VaeImageProcessor`) vae (`AutoencoderKL`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKL`) Inputs: image (`None`, *optional*): @@ -81,7 +82,8 @@ class FluxKontextAutoVaeEncoderStep(AutoPipelineBlocks): provided, step will be skipped. Components: - image_processor (`VaeImageProcessor`) vae (`AutoencoderKL`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKL`) Inputs: image (`None`, *optional*): @@ -121,7 +123,7 @@ class FluxKontextBeforeDenoiseStep(SequentialPipelineBlocks): Before denoise step that prepares the inputs for the denoise step for Flux Kontext Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: height (`int`, *optional*): @@ -184,7 +186,7 @@ class FluxKontextImageConditionedBeforeDenoiseStep(SequentialPipelineBlocks): for image-conditioned tasks. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: height (`int`, *optional*): @@ -255,7 +257,7 @@ class FluxKontextAutoBeforeDenoiseStep(AutoPipelineBlocks): provided. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: height (`int`, *optional*): @@ -443,7 +445,8 @@ class FluxKontextCoreDenoiseStep(SequentialPipelineBlocks): - for text-to-image generation, all you need to provide is prompt embeddings. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`FluxTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`FluxTransformer2DModel`) Inputs: height (`None`, *optional*): @@ -525,9 +528,14 @@ class FluxKontextAutoBlocks(SequentialPipelineBlocks): - `text2image`: requires `prompt` Components: - text_encoder (`CLIPTextModel`) tokenizer (`CLIPTokenizer`) text_encoder_2 (`T5EncoderModel`) tokenizer_2 - (`T5Tokenizer`) image_processor (`VaeImageProcessor`) vae (`AutoencoderKL`) scheduler - (`FlowMatchEulerDiscreteScheduler`) transformer (`FluxTransformer2DModel`) + - text_encoder (`CLIPTextModel`) + - tokenizer (`CLIPTokenizer`) + - text_encoder_2 (`T5EncoderModel`) + - tokenizer_2 (`T5Tokenizer`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKL`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`FluxTransformer2DModel`) Inputs: prompt (`None`, *optional*): diff --git a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2.py b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2.py index 2bbb7975a983..b432d75ed559 100644 --- a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2.py +++ b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2.py @@ -44,7 +44,8 @@ class Flux2VaeEncoderSequentialStep(SequentialPipelineBlocks): VAE encoder step that preprocesses, encodes, and prepares image latents for Flux2 conditioning. Components: - image_processor (`Flux2ImageProcessor`) vae (`AutoencoderKLFlux2`) + - image_processor (`Flux2ImageProcessor`) + - vae (`AutoencoderKLFlux2`) Inputs: image (`None`, *optional*): @@ -82,7 +83,8 @@ class Flux2AutoVaeEncoderStep(AutoPipelineBlocks): - If `image` is not provided, step will be skipped. Components: - image_processor (`Flux2ImageProcessor`) vae (`AutoencoderKLFlux2`) + - image_processor (`Flux2ImageProcessor`) + - vae (`AutoencoderKLFlux2`) Inputs: image (`None`, *optional*): @@ -134,7 +136,8 @@ class Flux2CoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that performs the denoising process for Flux2-dev. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`Flux2Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux2Transformer2DModel`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -205,7 +208,8 @@ class Flux2ImageConditionedCoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that performs the denoising process for Flux2-dev with image conditioning. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`Flux2Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux2Transformer2DModel`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -291,9 +295,12 @@ class Flux2AutoBlocks(SequentialPipelineBlocks): - `image_conditioned`: requires `image`, `prompt` Components: - text_encoder (`Mistral3ForConditionalGeneration`) tokenizer (`AutoProcessor`) image_processor - (`Flux2ImageProcessor`) vae (`AutoencoderKLFlux2`) scheduler (`FlowMatchEulerDiscreteScheduler`) transformer - (`Flux2Transformer2DModel`) + - text_encoder (`Mistral3ForConditionalGeneration`) + - tokenizer (`AutoProcessor`) + - image_processor (`Flux2ImageProcessor`) + - vae (`AutoencoderKLFlux2`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux2Transformer2DModel`) Inputs: prompt (`None`, *optional*): diff --git a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein.py b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein.py index 689cf808c4ba..3efb1b56a02f 100644 --- a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein.py +++ b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein.py @@ -47,7 +47,8 @@ class Flux2KleinVaeEncoderSequentialStep(SequentialPipelineBlocks): VAE encoder step that preprocesses and encodes the image inputs into their latent representations. Components: - image_processor (`Flux2ImageProcessor`) vae (`AutoencoderKLFlux2`) + - image_processor (`Flux2ImageProcessor`) + - vae (`AutoencoderKLFlux2`) Inputs: image (`None`, *optional*): @@ -85,7 +86,8 @@ class Flux2KleinAutoVaeEncoderStep(AutoPipelineBlocks): - If `image` is not provided, step will be skipped. Components: - image_processor (`Flux2ImageProcessor`) vae (`AutoencoderKLFlux2`) + - image_processor (`Flux2ImageProcessor`) + - vae (`AutoencoderKLFlux2`) Inputs: image (`None`, *optional*): @@ -143,7 +145,8 @@ class Flux2KleinCoreDenoiseStep(SequentialPipelineBlocks): generation. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`Flux2Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux2Transformer2DModel`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -211,7 +214,8 @@ class Flux2KleinImageConditionedCoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that performs the denoising process for Flux2-Klein (distilled model) with image conditioning. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`Flux2Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux2Transformer2DModel`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -267,7 +271,8 @@ class Flux2KleinAutoCoreDenoiseStep(AutoPipelineBlocks): - `Flux2KleinImageConditionedCoreDenoiseStep` is used for image-conditioned generation. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`Flux2Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux2Transformer2DModel`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -330,8 +335,12 @@ class Flux2KleinAutoBlocks(SequentialPipelineBlocks): - `image_conditioned`: requires `image`, `prompt` Components: - text_encoder (`Qwen3ForCausalLM`) tokenizer (`Qwen2Tokenizer`) image_processor (`Flux2ImageProcessor`) vae - (`AutoencoderKLFlux2`) scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`Flux2Transformer2DModel`) + - text_encoder (`Qwen3ForCausalLM`) + - tokenizer (`Qwen2Tokenizer`) + - image_processor (`Flux2ImageProcessor`) + - vae (`AutoencoderKLFlux2`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux2Transformer2DModel`) Configs: is_distilled (default: True) diff --git a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein_base.py b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein_base.py index f3108bdadeac..25872cce7d50 100644 --- a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein_base.py +++ b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein_base.py @@ -47,7 +47,8 @@ class Flux2KleinBaseVaeEncoderSequentialStep(SequentialPipelineBlocks): VAE encoder step that preprocesses and encodes the image inputs into their latent representations. Components: - image_processor (`Flux2ImageProcessor`) vae (`AutoencoderKLFlux2`) + - image_processor (`Flux2ImageProcessor`) + - vae (`AutoencoderKLFlux2`) Inputs: image (`None`, *optional*): @@ -85,7 +86,8 @@ class Flux2KleinBaseAutoVaeEncoderStep(AutoPipelineBlocks): - If `image` is not provided, step will be skipped. Components: - image_processor (`Flux2ImageProcessor`) vae (`AutoencoderKLFlux2`) + - image_processor (`Flux2ImageProcessor`) + - vae (`AutoencoderKLFlux2`) Inputs: image (`None`, *optional*): @@ -140,8 +142,9 @@ class Flux2KleinBaseCoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that performs the denoising process for Flux2-Klein (base model), for text-to-image generation. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`Flux2Transformer2DModel`) guider - (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux2Transformer2DModel`) + - guider (`ClassifierFreeGuidance`) Configs: is_distilled (default: False) @@ -213,8 +216,9 @@ class Flux2KleinBaseImageConditionedCoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that performs the denoising process for Flux2-Klein (base model) with image conditioning. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`Flux2Transformer2DModel`) guider - (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux2Transformer2DModel`) + - guider (`ClassifierFreeGuidance`) Configs: is_distilled (default: False) @@ -274,8 +278,9 @@ class Flux2KleinBaseAutoCoreDenoiseStep(AutoPipelineBlocks): - `Flux2KleinBaseImageConditionedCoreDenoiseStep` is used for image-conditioned generation. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`Flux2Transformer2DModel`) guider - (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux2Transformer2DModel`) + - guider (`ClassifierFreeGuidance`) Configs: is_distilled (default: False) @@ -343,9 +348,13 @@ class Flux2KleinBaseAutoBlocks(SequentialPipelineBlocks): - `image_conditioned`: requires `image`, `prompt` Components: - text_encoder (`Qwen3ForCausalLM`) tokenizer (`Qwen2Tokenizer`) guider (`ClassifierFreeGuidance`) - image_processor (`Flux2ImageProcessor`) vae (`AutoencoderKLFlux2`) scheduler - (`FlowMatchEulerDiscreteScheduler`) transformer (`Flux2Transformer2DModel`) + - text_encoder (`Qwen3ForCausalLM`) + - tokenizer (`Qwen2Tokenizer`) + - guider (`ClassifierFreeGuidance`) + - image_processor (`Flux2ImageProcessor`) + - vae (`AutoencoderKLFlux2`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux2Transformer2DModel`) Configs: is_distilled (default: False) diff --git a/src/diffusers/modular_pipelines/flux3/denoise.py b/src/diffusers/modular_pipelines/flux3/denoise.py index ed91f8569cb5..f23fce491674 100644 --- a/src/diffusers/modular_pipelines/flux3/denoise.py +++ b/src/diffusers/modular_pipelines/flux3/denoise.py @@ -321,8 +321,9 @@ class Flux3DenoiseStep(Flux3DenoiseLoopWrapper): This block supports text-to-image and image-to-image generation. Components: - transformer (`Flux3Transformer2DModel`) guider (`ClassifierFreeGuidance`) scheduler - (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux3Transformer2DModel`) + - guider (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): @@ -367,7 +368,8 @@ class Flux3DistilledDenoiseStep(Flux3DenoiseLoopWrapper): This block supports text-to-image and image-to-image generation, without classifier-free guidance. Components: - transformer (`Flux3Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): diff --git a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py index 31dcc62f7b0d..8bcf9cf0216d 100644 --- a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py +++ b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py @@ -43,7 +43,8 @@ class Flux3VaeEncoderSequentialStep(SequentialPipelineBlocks): VAE encoder step that preprocesses and encodes the reference images into their latent representations. Components: - image_processor (`Flux3ImageProcessor`) vae (`AutoencoderKLFlux2`) + - image_processor (`Flux3ImageProcessor`) + - vae (`AutoencoderKLFlux2`) Inputs: image (`Image | list`): @@ -79,7 +80,8 @@ class Flux3AutoVaeEncoderStep(AutoPipelineBlocks): - If `image` is not provided, step will be skipped. Components: - image_processor (`Flux3ImageProcessor`) vae (`AutoencoderKLFlux2`) + - image_processor (`Flux3ImageProcessor`) + - vae (`AutoencoderKLFlux2`) Inputs: image (`Image | list`, *optional*): @@ -134,8 +136,9 @@ class Flux3CoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that performs the denoising process for FLUX.3, for text-to-image generation. Components: - transformer (`Flux3Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) + - transformer (`Flux3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -203,8 +206,9 @@ class Flux3ImageConditionedCoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that performs the denoising process for FLUX.3 with reference images (image-to-image). Components: - transformer (`Flux3Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) + - transformer (`Flux3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -267,8 +271,9 @@ class Flux3AutoCoreDenoiseStep(AutoPipelineBlocks): - `Flux3CoreDenoiseStep` is used otherwise (text-to-image). Components: - transformer (`Flux3Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) + - transformer (`Flux3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -346,10 +351,15 @@ class Flux3AutoBlocks(SequentialPipelineBlocks): - `image_conditioned`: requires `prompt`, `image` Components: - prompt_upsampler (`Qwen3_5ForConditionalGeneration`): The prompt upsampler. prompt_upsampler_processor - (`AutoProcessor`): Its processor. text_encoder (`Qwen3VLForConditionalGeneration`) tokenizer - (`AutoProcessor`) image_processor (`Flux3ImageProcessor`) vae (`AutoencoderKLFlux2`) transformer - (`Flux3Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) + - prompt_upsampler (`Qwen3_5ForConditionalGeneration`): The prompt upsampler. + - prompt_upsampler_processor (`AutoProcessor`): Its processor. + - text_encoder (`Qwen3VLForConditionalGeneration`) + - tokenizer (`AutoProcessor`) + - image_processor (`Flux3ImageProcessor`) + - vae (`AutoencoderKLFlux2`) + - transformer (`Flux3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py index d23f894c6df7..66fb14b257d0 100644 --- a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py +++ b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py @@ -61,7 +61,8 @@ class Flux3DistilledCoreDenoiseStep(SequentialPipelineBlocks): generation. Components: - transformer (`Flux3Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -129,7 +130,8 @@ class Flux3DistilledImageConditionedCoreDenoiseStep(SequentialPipelineBlocks): (image-to-image). Components: - transformer (`Flux3Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -190,7 +192,8 @@ class Flux3DistilledAutoCoreDenoiseStep(AutoPipelineBlocks): - `Flux3DistilledCoreDenoiseStep` is used otherwise (text-to-image). Components: - transformer (`Flux3Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Flux3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -266,10 +269,14 @@ class Flux3DistilledAutoBlocks(SequentialPipelineBlocks): - `image_conditioned`: requires `prompt`, `image` Components: - prompt_upsampler (`Qwen3_5ForConditionalGeneration`): The prompt upsampler. prompt_upsampler_processor - (`AutoProcessor`): Its processor. text_encoder (`Qwen3VLForConditionalGeneration`) tokenizer - (`AutoProcessor`) image_processor (`Flux3ImageProcessor`) vae (`AutoencoderKLFlux2`) transformer - (`Flux3Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - prompt_upsampler (`Qwen3_5ForConditionalGeneration`): The prompt upsampler. + - prompt_upsampler_processor (`AutoProcessor`): Its processor. + - text_encoder (`Qwen3VLForConditionalGeneration`) + - tokenizer (`AutoProcessor`) + - image_processor (`Flux3ImageProcessor`) + - vae (`AutoencoderKLFlux2`) + - transformer (`Flux3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py index c3d5cb4efc77..0d54a8d1bfc1 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py @@ -49,7 +49,8 @@ class HeliosAutoVaeEncoderStep(AutoPipelineBlocks): - If neither is provided, step will be skipped. Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Inputs: video (`None`, *optional*): @@ -100,7 +101,9 @@ class HeliosCoreDenoiseStep(SequentialPipelineBlocks): Denoise block that takes encoded conditions and runs the chunk-based denoising process. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider (`ClassifierFreeGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -166,7 +169,9 @@ class HeliosI2VCoreDenoiseStep(SequentialPipelineBlocks): I2V denoise block that seeds history with image latents and uses I2V-aware chunk preparation. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider (`ClassifierFreeGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -260,7 +265,9 @@ class HeliosV2VCoreDenoiseStep(SequentialPipelineBlocks): V2V denoise block that seeds history with video latents and uses I2V-aware chunk preparation. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider (`ClassifierFreeGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -355,7 +362,9 @@ class HeliosAutoCoreDenoiseStep(ConditionalPipelineBlocks): - `HeliosCoreDenoiseStep` (text2video) for text-to-video tasks. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider (`ClassifierFreeGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -457,9 +466,13 @@ class HeliosAutoBlocks(SequentialPipelineBlocks): - `video2video`: requires `prompt`, `video` Components: - text_encoder (`UMT5EncoderModel`) tokenizer (`AutoTokenizer`) guider (`ClassifierFreeGuidance`) vae - (`AutoencoderKLWan`) video_processor (`VideoProcessor`) transformer (`HeliosTransformer3DModel`) scheduler - (`HeliosScheduler`) + - text_encoder (`UMT5EncoderModel`) + - tokenizer (`AutoTokenizer`) + - guider (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py index fea11786de21..8c5c6cb0dbdf 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py @@ -48,7 +48,8 @@ class HeliosPyramidAutoVaeEncoderStep(AutoPipelineBlocks): - If neither is provided, step will be skipped. Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Inputs: video (`None`, *optional*): @@ -99,8 +100,9 @@ class HeliosPyramidCoreDenoiseStep(SequentialPipelineBlocks): T2V pyramid denoise block with progressive multi-resolution denoising. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider - (`ClassifierFreeZeroStarGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeZeroStarGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -161,8 +163,9 @@ class HeliosPyramidI2VCoreDenoiseStep(SequentialPipelineBlocks): I2V pyramid denoise block with progressive multi-resolution denoising. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider - (`ClassifierFreeZeroStarGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeZeroStarGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -250,8 +253,9 @@ class HeliosPyramidV2VCoreDenoiseStep(SequentialPipelineBlocks): V2V pyramid denoise block with progressive multi-resolution denoising. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider - (`ClassifierFreeZeroStarGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeZeroStarGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -340,8 +344,9 @@ class HeliosPyramidAutoCoreDenoiseStep(ConditionalPipelineBlocks): - `HeliosPyramidCoreDenoiseStep` (text2video) for text-to-video tasks. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider - (`ClassifierFreeZeroStarGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeZeroStarGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -439,9 +444,13 @@ class HeliosPyramidAutoBlocks(SequentialPipelineBlocks): - `video2video`: requires `prompt`, `video` Components: - text_encoder (`UMT5EncoderModel`) tokenizer (`AutoTokenizer`) guider (`ClassifierFreeGuidance`) vae - (`AutoencoderKLWan`) video_processor (`VideoProcessor`) transformer (`HeliosTransformer3DModel`) scheduler - (`HeliosScheduler`) + - text_encoder (`UMT5EncoderModel`) + - tokenizer (`AutoTokenizer`) + - guider (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py index 3e0b32f0df7e..81b40d12d824 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py @@ -48,7 +48,8 @@ class HeliosPyramidDistilledAutoVaeEncoderStep(AutoPipelineBlocks): - If neither is provided, step will be skipped. Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Inputs: video (`None`, *optional*): @@ -99,7 +100,9 @@ class HeliosPyramidDistilledCoreDenoiseStep(SequentialPipelineBlocks): T2V distilled pyramid denoise block with DMD scheduler and no CFG. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider (`ClassifierFreeGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -162,7 +165,9 @@ class HeliosPyramidDistilledI2VCoreDenoiseStep(SequentialPipelineBlocks): I2V distilled pyramid denoise block with DMD scheduler and no CFG. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider (`ClassifierFreeGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -252,7 +257,9 @@ class HeliosPyramidDistilledV2VCoreDenoiseStep(SequentialPipelineBlocks): V2V distilled pyramid denoise block with DMD scheduler and no CFG. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider (`ClassifierFreeGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -343,7 +350,9 @@ class HeliosPyramidDistilledAutoCoreDenoiseStep(ConditionalPipelineBlocks): - `HeliosPyramidDistilledCoreDenoiseStep` (text2video) for text-to-video tasks. Components: - transformer (`HeliosTransformer3DModel`) scheduler (`HeliosScheduler`) guider (`ClassifierFreeGuidance`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -447,9 +456,13 @@ class HeliosPyramidDistilledAutoBlocks(SequentialPipelineBlocks): - `video2video`: requires `prompt`, `video` Components: - text_encoder (`UMT5EncoderModel`) tokenizer (`AutoTokenizer`) guider (`ClassifierFreeGuidance`) vae - (`AutoencoderKLWan`) video_processor (`VideoProcessor`) transformer (`HeliosTransformer3DModel`) scheduler - (`HeliosScheduler`) + - text_encoder (`UMT5EncoderModel`) + - tokenizer (`AutoTokenizer`) + - guider (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - transformer (`HeliosTransformer3DModel`) + - scheduler (`HeliosScheduler`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py b/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py index bdbdba1ecdd9..fa6d9b71696c 100644 --- a/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py +++ b/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py @@ -39,8 +39,10 @@ class HunyuanVideo15CoreDenoiseStep(SequentialPipelineBlocks): Denoise block that takes encoded conditions and runs the denoising process. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`HunyuanVideo15Transformer3DModel`) - video_processor (`HunyuanVideo15ImageProcessor`) guider (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`HunyuanVideo15Transformer3DModel`) + - video_processor (`HunyuanVideo15ImageProcessor`) + - guider (`ClassifierFreeGuidance`) Inputs: prompt_embeds (`Tensor`): @@ -110,10 +112,15 @@ class HunyuanVideo15Blocks(SequentialPipelineBlocks): Modular pipeline blocks for HunyuanVideo 1.5 text-to-video. Components: - text_encoder (`Qwen2_5_VLTextModel`) tokenizer (`Qwen2Tokenizer`) text_encoder_2 (`T5EncoderModel`) - tokenizer_2 (`ByT5Tokenizer`) guider (`ClassifierFreeGuidance`) scheduler (`FlowMatchEulerDiscreteScheduler`) - transformer (`HunyuanVideo15Transformer3DModel`) video_processor (`HunyuanVideo15ImageProcessor`) vae - (`AutoencoderKLHunyuanVideo15`) + - text_encoder (`Qwen2_5_VLTextModel`) + - tokenizer (`Qwen2Tokenizer`) + - text_encoder_2 (`T5EncoderModel`) + - tokenizer_2 (`ByT5Tokenizer`) + - guider (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`HunyuanVideo15Transformer3DModel`) + - video_processor (`HunyuanVideo15ImageProcessor`) + - vae (`AutoencoderKLHunyuanVideo15`) Inputs: prompt (`str`, *optional*): @@ -172,8 +179,10 @@ class HunyuanVideo15Image2VideoCoreDenoiseStep(SequentialPipelineBlocks): Denoise block for image-to-video that takes encoded conditions and runs the denoising process. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`HunyuanVideo15Transformer3DModel`) - video_processor (`HunyuanVideo15ImageProcessor`) guider (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`HunyuanVideo15Transformer3DModel`) + - video_processor (`HunyuanVideo15ImageProcessor`) + - guider (`ClassifierFreeGuidance`) Inputs: prompt_embeds (`Tensor`): @@ -249,7 +258,8 @@ class HunyuanVideo15AutoVaeEncoderStep(AutoPipelineBlocks): - If `image` is not provided, step will be skipped. Components: - vae (`AutoencoderKLHunyuanVideo15`) video_processor (`HunyuanVideo15ImageProcessor`) + - vae (`AutoencoderKLHunyuanVideo15`) + - video_processor (`HunyuanVideo15ImageProcessor`) Inputs: image (`Image | list`, *optional*): @@ -292,7 +302,8 @@ class HunyuanVideo15AutoImageEncoderStep(AutoPipelineBlocks): - If `image` is not provided, step will be skipped. Components: - image_encoder (`SiglipVisionModel`) feature_extractor (`SiglipImageProcessor`) + - image_encoder (`SiglipVisionModel`) + - feature_extractor (`SiglipImageProcessor`) Inputs: image (`Image | list`, *optional*): @@ -326,8 +337,10 @@ class HunyuanVideo15AutoCoreDenoiseStep(AutoPipelineBlocks): - `HunyuanVideo15CoreDenoiseStep` is used otherwise (text-to-video). Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`HunyuanVideo15Transformer3DModel`) - video_processor (`HunyuanVideo15ImageProcessor`) guider (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`HunyuanVideo15Transformer3DModel`) + - video_processor (`HunyuanVideo15ImageProcessor`) + - guider (`ClassifierFreeGuidance`) Inputs: prompt_embeds (`Tensor`): @@ -399,11 +412,17 @@ class HunyuanVideo15AutoBlocks(SequentialPipelineBlocks): - `image2video`: requires `image`, `prompt` Components: - text_encoder (`Qwen2_5_VLTextModel`) tokenizer (`Qwen2Tokenizer`) text_encoder_2 (`T5EncoderModel`) - tokenizer_2 (`ByT5Tokenizer`) guider (`ClassifierFreeGuidance`) vae (`AutoencoderKLHunyuanVideo15`) - video_processor (`HunyuanVideo15ImageProcessor`) image_encoder (`SiglipVisionModel`) feature_extractor - (`SiglipImageProcessor`) scheduler (`FlowMatchEulerDiscreteScheduler`) transformer - (`HunyuanVideo15Transformer3DModel`) + - text_encoder (`Qwen2_5_VLTextModel`) + - tokenizer (`Qwen2Tokenizer`) + - text_encoder_2 (`T5EncoderModel`) + - tokenizer_2 (`ByT5Tokenizer`) + - guider (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLHunyuanVideo15`) + - video_processor (`HunyuanVideo15ImageProcessor`) + - image_encoder (`SiglipVisionModel`) + - feature_extractor (`SiglipImageProcessor`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`HunyuanVideo15Transformer3DModel`) Inputs: prompt (`str`, *optional*): @@ -472,11 +491,17 @@ class HunyuanVideo15Image2VideoBlocks(SequentialPipelineBlocks): Modular pipeline blocks for HunyuanVideo 1.5 image-to-video. Components: - text_encoder (`Qwen2_5_VLTextModel`) tokenizer (`Qwen2Tokenizer`) text_encoder_2 (`T5EncoderModel`) - tokenizer_2 (`ByT5Tokenizer`) guider (`ClassifierFreeGuidance`) vae (`AutoencoderKLHunyuanVideo15`) - video_processor (`HunyuanVideo15ImageProcessor`) image_encoder (`SiglipVisionModel`) feature_extractor - (`SiglipImageProcessor`) scheduler (`FlowMatchEulerDiscreteScheduler`) transformer - (`HunyuanVideo15Transformer3DModel`) + - text_encoder (`Qwen2_5_VLTextModel`) + - tokenizer (`Qwen2Tokenizer`) + - text_encoder_2 (`T5EncoderModel`) + - tokenizer_2 (`ByT5Tokenizer`) + - guider (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLHunyuanVideo15`) + - video_processor (`HunyuanVideo15ImageProcessor`) + - image_encoder (`SiglipVisionModel`) + - feature_extractor (`SiglipImageProcessor`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`HunyuanVideo15Transformer3DModel`) Inputs: prompt (`str`, *optional*): diff --git a/src/diffusers/modular_pipelines/ideogram4/before_denoise.py b/src/diffusers/modular_pipelines/ideogram4/before_denoise.py index c29ee38e085b..adecfc33d1f9 100644 --- a/src/diffusers/modular_pipelines/ideogram4/before_denoise.py +++ b/src/diffusers/modular_pipelines/ideogram4/before_denoise.py @@ -203,7 +203,7 @@ class Ideogram4PrepareLatentsStep(ModularPipelineBlocks): Step that prepares the packed image latents (B, num_image_tokens, latent_dim) for the denoising loop. Components: - transformer (`Ideogram4Transformer2DModel`) + - transformer (`Ideogram4Transformer2DModel`) Inputs: latents (`Tensor`, *optional*): @@ -291,7 +291,7 @@ class Ideogram4SetTimestepsStep(ModularPipelineBlocks): weights. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: num_inference_steps (`int`, *optional*, defaults to 48): diff --git a/src/diffusers/modular_pipelines/ideogram4/decoders.py b/src/diffusers/modular_pipelines/ideogram4/decoders.py index 710734ee7fb2..40996a41cd76 100644 --- a/src/diffusers/modular_pipelines/ideogram4/decoders.py +++ b/src/diffusers/modular_pipelines/ideogram4/decoders.py @@ -34,7 +34,8 @@ class Ideogram4DecodeStep(ModularPipelineBlocks): batch-norm statistics and decodes through the VAE. Components: - vae (`AutoencoderKLFlux2`) image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLFlux2`) + - image_processor (`VaeImageProcessor`) Inputs: output_type (`str`, *optional*, defaults to pil): diff --git a/src/diffusers/modular_pipelines/ideogram4/denoise.py b/src/diffusers/modular_pipelines/ideogram4/denoise.py index db1a708c4315..e6799ab0b021 100644 --- a/src/diffusers/modular_pipelines/ideogram4/denoise.py +++ b/src/diffusers/modular_pipelines/ideogram4/denoise.py @@ -220,8 +220,9 @@ class Ideogram4DenoiseStep(LoopSequentialPipelineBlocks): and unconditional transformers and blending with the per-step guidance schedule. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`Ideogram4Transformer2DModel`) - unconditional_transformer (`Ideogram4Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Ideogram4Transformer2DModel`) + - unconditional_transformer (`Ideogram4Transformer2DModel`) Inputs: timesteps (`Tensor`): diff --git a/src/diffusers/modular_pipelines/ideogram4/encoders.py b/src/diffusers/modular_pipelines/ideogram4/encoders.py index fa7fc765ea9b..c146361dc413 100644 --- a/src/diffusers/modular_pipelines/ideogram4/encoders.py +++ b/src/diffusers/modular_pipelines/ideogram4/encoders.py @@ -46,9 +46,10 @@ class Ideogram4PromptUpsampleStep(ModularPipelineBlocks): `Qwen3VLForConditionalGeneration`); install `outlines` for schema-constrained captions. Components: - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. tokenizer (`Qwen2Tokenizer`): The tokenizer paired - with the text encoder. prompt_enhancer_head (`Ideogram4PromptEnhancerHead`): LM head grafted onto the text - encoder for prompt upsampling. + - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. + - tokenizer (`Qwen2Tokenizer`): The tokenizer paired with the text encoder. + - prompt_enhancer_head (`Ideogram4PromptEnhancerHead`): LM head grafted onto the text encoder for prompt + upsampling. Inputs: prompt (`str`): @@ -182,8 +183,8 @@ class Ideogram4TextEncoderStep(ModularPipelineBlocks): tokens are appended later (the encoder is causal with image after text, so they never affect the text features). Components: - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. tokenizer (`Qwen2Tokenizer`): The tokenizer paired - with the text encoder. + - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. + - tokenizer (`Qwen2Tokenizer`): The tokenizer paired with the text encoder. Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/ideogram4/modular_blocks_ideogram4.py b/src/diffusers/modular_pipelines/ideogram4/modular_blocks_ideogram4.py index 0b788fe236be..e281b2d236d1 100644 --- a/src/diffusers/modular_pipelines/ideogram4/modular_blocks_ideogram4.py +++ b/src/diffusers/modular_pipelines/ideogram4/modular_blocks_ideogram4.py @@ -52,8 +52,9 @@ class Ideogram4CoreDenoiseStep(SequentialPipelineBlocks): unpatchifies the result for the decoder. Components: - transformer (`Ideogram4Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) - unconditional_transformer (`Ideogram4Transformer2DModel`) + - transformer (`Ideogram4Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - unconditional_transformer (`Ideogram4Transformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -115,11 +116,15 @@ class Ideogram4AutoBlocks(SequentialPipelineBlocks): - `text2image`: requires `prompt` Components: - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. tokenizer (`Qwen2Tokenizer`): The tokenizer paired - with the text encoder. prompt_enhancer_head (`Ideogram4PromptEnhancerHead`): LM head grafted onto the text - encoder for prompt upsampling. transformer (`Ideogram4Transformer2DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) unconditional_transformer (`Ideogram4Transformer2DModel`) vae - (`AutoencoderKLFlux2`) image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. + - tokenizer (`Qwen2Tokenizer`): The tokenizer paired with the text encoder. + - prompt_enhancer_head (`Ideogram4PromptEnhancerHead`): LM head grafted onto the text encoder for prompt + upsampling. + - transformer (`Ideogram4Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - unconditional_transformer (`Ideogram4Transformer2DModel`) + - vae (`AutoencoderKLFlux2`) + - image_processor (`VaeImageProcessor`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/kandinsky_s2v/encoders.py b/src/diffusers/modular_pipelines/kandinsky_s2v/encoders.py index a248582ae732..ad4cedeb4dd1 100644 --- a/src/diffusers/modular_pipelines/kandinsky_s2v/encoders.py +++ b/src/diffusers/modular_pipelines/kandinsky_s2v/encoders.py @@ -344,8 +344,10 @@ class KandinskyS2VDistilledTextAudioEncoderStep(KandinskyS2VTextAudioEncoderStep prompt: it plans the chunks the speech track needs and encodes the prompt and the speech. Components: - text_encoder (`Qwen2_5OmniThinkerForConditionalGeneration`) tokenizer (`Qwen2_5OmniProcessor`) text_encoder_2 - (`CLIPTextModel`) tokenizer_2 (`CLIPTokenizer`) + - text_encoder (`Qwen2_5OmniThinkerForConditionalGeneration`) + - tokenizer (`Qwen2_5OmniProcessor`) + - text_encoder_2 (`CLIPTextModel`) + - tokenizer_2 (`CLIPTokenizer`) Inputs: prompt (`str`, *optional*, defaults to The video presents a stable medium close-up of a subject facing the diff --git a/src/diffusers/modular_pipelines/kandinsky_s2v/modular_blocks_kandinsky_s2v.py b/src/diffusers/modular_pipelines/kandinsky_s2v/modular_blocks_kandinsky_s2v.py index 099b4353a25f..f9fc3d0b55f7 100644 --- a/src/diffusers/modular_pipelines/kandinsky_s2v/modular_blocks_kandinsky_s2v.py +++ b/src/diffusers/modular_pipelines/kandinsky_s2v/modular_blocks_kandinsky_s2v.py @@ -39,8 +39,9 @@ class KandinskyS2VCoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that prepares the conditions and the FramePack memory, then denoises the video chunk by chunk. Components: - transformer (`KandinskyS2VTransformer3DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) + - transformer (`KandinskyS2VTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: reference_latents (`Tensor`): @@ -111,10 +112,15 @@ class KandinskyS2VAutoBlocks(SequentialPipelineBlocks): a speech track, generated chunk by chunk. Components: - text_encoder (`Qwen2_5OmniThinkerForConditionalGeneration`) tokenizer (`Qwen2_5OmniProcessor`) text_encoder_2 - (`CLIPTextModel`) tokenizer_2 (`CLIPTokenizer`) vae (`AutoencoderKLHunyuanVideo`) video_processor - (`VideoProcessor`) transformer (`KandinskyS2VTransformer3DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) + - text_encoder (`Qwen2_5OmniThinkerForConditionalGeneration`) + - tokenizer (`Qwen2_5OmniProcessor`) + - text_encoder_2 (`CLIPTextModel`) + - tokenizer_2 (`CLIPTokenizer`) + - vae (`AutoencoderKLHunyuanVideo`) + - video_processor (`VideoProcessor`) + - transformer (`KandinskyS2VTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: prompt (`str`, *optional*, defaults to The video presents a stable medium close-up of a subject facing the diff --git a/src/diffusers/modular_pipelines/kandinsky_s2v/modular_blocks_kandinsky_s2v_distilled.py b/src/diffusers/modular_pipelines/kandinsky_s2v/modular_blocks_kandinsky_s2v_distilled.py index f8fbb72c15e6..34f319e0b135 100644 --- a/src/diffusers/modular_pipelines/kandinsky_s2v/modular_blocks_kandinsky_s2v_distilled.py +++ b/src/diffusers/modular_pipelines/kandinsky_s2v/modular_blocks_kandinsky_s2v_distilled.py @@ -39,7 +39,8 @@ class KandinskyS2VDistilledCoreDenoiseStep(SequentialPipelineBlocks): the video chunk by chunk with the pi-Flow scheduler. Components: - transformer (`KandinskyS2VTransformer3DModel`) scheduler (`PiflowScheduler`) + - transformer (`KandinskyS2VTransformer3DModel`) + - scheduler (`PiflowScheduler`) Inputs: reference_latents (`Tensor`): @@ -106,9 +107,14 @@ class KandinskyS2VDistilledAutoBlocks(SequentialPipelineBlocks): segments per chunk and no guidance, which is distilled into the student. Components: - text_encoder (`Qwen2_5OmniThinkerForConditionalGeneration`) tokenizer (`Qwen2_5OmniProcessor`) text_encoder_2 - (`CLIPTextModel`) tokenizer_2 (`CLIPTokenizer`) vae (`AutoencoderKLHunyuanVideo`) video_processor - (`VideoProcessor`) transformer (`KandinskyS2VTransformer3DModel`) scheduler (`PiflowScheduler`) + - text_encoder (`Qwen2_5OmniThinkerForConditionalGeneration`) + - tokenizer (`Qwen2_5OmniProcessor`) + - text_encoder_2 (`CLIPTextModel`) + - tokenizer_2 (`CLIPTokenizer`) + - vae (`AutoencoderKLHunyuanVideo`) + - video_processor (`VideoProcessor`) + - transformer (`KandinskyS2VTransformer3DModel`) + - scheduler (`PiflowScheduler`) Inputs: prompt (`str`, *optional*, defaults to The video presents a stable medium close-up of a subject facing the diff --git a/src/diffusers/modular_pipelines/krea2/before_denoise.py b/src/diffusers/modular_pipelines/krea2/before_denoise.py index 17ed6a3cd376..20eaffe12f4e 100644 --- a/src/diffusers/modular_pipelines/krea2/before_denoise.py +++ b/src/diffusers/modular_pipelines/krea2/before_denoise.py @@ -262,7 +262,7 @@ class Krea2PrepareLatentsStep(ModularPipelineBlocks): denoising loop. Components: - transformer (`Krea2Transformer2DModel`) + - transformer (`Krea2Transformer2DModel`) Inputs: latents (`Tensor`, *optional*): @@ -373,7 +373,7 @@ class Krea2SetTimestepsStep(ModularPipelineBlocks): dynamic time shift `mu`. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: num_inference_steps (`int`, *optional*, defaults to 28): @@ -459,7 +459,7 @@ class Krea2TurboSetTimestepsStep(ModularPipelineBlocks): sigma schedule with the fixed time shift `mu=1.15` the checkpoint was distilled with. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: num_inference_steps (`int`, *optional*, defaults to 8): diff --git a/src/diffusers/modular_pipelines/krea2/decoders.py b/src/diffusers/modular_pipelines/krea2/decoders.py index 751b312037d0..5790c90ea799 100644 --- a/src/diffusers/modular_pipelines/krea2/decoders.py +++ b/src/diffusers/modular_pipelines/krea2/decoders.py @@ -109,7 +109,8 @@ class Krea2DecodeStep(ModularPipelineBlocks): Qwen-Image VAE into images. Components: - vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) Inputs: output_type (`str`, *optional*, defaults to pil): diff --git a/src/diffusers/modular_pipelines/krea2/denoise.py b/src/diffusers/modular_pipelines/krea2/denoise.py index 8fc019052a54..b10214f13f05 100644 --- a/src/diffusers/modular_pipelines/krea2/denoise.py +++ b/src/diffusers/modular_pipelines/krea2/denoise.py @@ -290,8 +290,9 @@ class Krea2DenoiseStep(Krea2DenoiseLoopWrapper): conditional (and, when the guider enables CFG, the negative) text features and combining them through the `guider`. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) transformer - (`Krea2Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`Krea2Transformer2DModel`) Inputs: timesteps (`Tensor`): @@ -341,7 +342,8 @@ class Krea2TurboDenoiseStep(Krea2DenoiseLoopWrapper): classifier-free guidance. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`Krea2Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Krea2Transformer2DModel`) Inputs: timesteps (`Tensor`): diff --git a/src/diffusers/modular_pipelines/krea2/encoders.py b/src/diffusers/modular_pipelines/krea2/encoders.py index 154f1d905ade..ce0fb181aa3e 100644 --- a/src/diffusers/modular_pipelines/krea2/encoders.py +++ b/src/diffusers/modular_pipelines/krea2/encoders.py @@ -51,8 +51,9 @@ class Krea2TextEncoderStep(ModularPipelineBlocks): prompt is encoded the same way when the guider enables CFG. Components: - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. tokenizer (`AutoTokenizer`): The tokenizer paired - with the text encoder. guider (`ClassifierFreeGuidance`) + - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. + - tokenizer (`AutoTokenizer`): The tokenizer paired with the text encoder. + - guider (`ClassifierFreeGuidance`) Inputs: prompt (`str`): @@ -211,8 +212,8 @@ class Krea2TurboTextEncoderStep(Krea2TextEncoderStep): negative prompt and has no guider. Components: - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. tokenizer (`AutoTokenizer`): The tokenizer paired - with the text encoder. + - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. + - tokenizer (`AutoTokenizer`): The tokenizer paired with the text encoder. Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/krea2/modular_blocks_krea2.py b/src/diffusers/modular_pipelines/krea2/modular_blocks_krea2.py index 1a1f11f60656..33c120e9a349 100644 --- a/src/diffusers/modular_pipelines/krea2/modular_blocks_krea2.py +++ b/src/diffusers/modular_pipelines/krea2/modular_blocks_krea2.py @@ -49,8 +49,9 @@ class Krea2CoreDenoiseStep(SequentialPipelineBlocks): then runs the symmetric-CFG denoising loop, producing the denoised packed latents for the decoder. Components: - transformer (`Krea2Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) + - transformer (`Krea2Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -112,9 +113,13 @@ class Krea2AutoBlocks(SequentialPipelineBlocks): - `text2image`: requires `prompt` Components: - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. tokenizer (`AutoTokenizer`): The tokenizer paired - with the text encoder. guider (`ClassifierFreeGuidance`) transformer (`Krea2Transformer2DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. + - tokenizer (`AutoTokenizer`): The tokenizer paired with the text encoder. + - guider (`ClassifierFreeGuidance`) + - transformer (`Krea2Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/krea2/modular_blocks_krea2_turbo.py b/src/diffusers/modular_pipelines/krea2/modular_blocks_krea2_turbo.py index e3c8f1c288b3..ee7036a03b7b 100644 --- a/src/diffusers/modular_pipelines/krea2/modular_blocks_krea2_turbo.py +++ b/src/diffusers/modular_pipelines/krea2/modular_blocks_krea2_turbo.py @@ -50,7 +50,8 @@ class Krea2TurboCoreDenoiseStep(SequentialPipelineBlocks): denoised packed latents for the decoder. Components: - transformer (`Krea2Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`Krea2Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -108,9 +109,12 @@ class Krea2TurboAutoBlocks(SequentialPipelineBlocks): - `text2image`: requires `prompt` Components: - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. tokenizer (`AutoTokenizer`): The tokenizer paired - with the text encoder. transformer (`Krea2Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) - vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLModel`): The Qwen3-VL text encoder. + - tokenizer (`AutoTokenizer`): The tokenizer paired with the text encoder. + - transformer (`Krea2Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py b/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py index 0bdaf6ec8816..6b29afce3a7d 100644 --- a/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py +++ b/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py @@ -35,8 +35,10 @@ class LTXCoreDenoiseStep(SequentialPipelineBlocks): Denoise block that takes encoded conditions and runs the denoising process. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) pachifier (`LTXVideoPachifier`) guider - (`ClassifierFreeGuidance`) transformer (`LTXVideoTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - pachifier (`LTXVideoPachifier`) + - guider (`ClassifierFreeGuidance`) + - transformer (`LTXVideoTransformer3DModel`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -100,8 +102,10 @@ class LTXImage2VideoCoreDenoiseStep(SequentialPipelineBlocks): Denoise block for image-to-video that takes encoded conditions and image latents, and runs the denoising process. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) pachifier (`LTXVideoPachifier`) guider - (`ClassifierFreeGuidance`) transformer (`LTXVideoTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - pachifier (`LTXVideoPachifier`) + - guider (`ClassifierFreeGuidance`) + - transformer (`LTXVideoTransformer3DModel`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -168,9 +172,14 @@ class LTXBlocks(SequentialPipelineBlocks): Modular pipeline blocks for LTX Video text-to-video. Components: - text_encoder (`T5EncoderModel`) tokenizer (`T5Tokenizer`) guider (`ClassifierFreeGuidance`) scheduler - (`FlowMatchEulerDiscreteScheduler`) pachifier (`LTXVideoPachifier`) transformer - (`LTXVideoTransformer3DModel`) vae (`AutoencoderKLLTXVideo`) video_processor (`VideoProcessor`) + - text_encoder (`T5EncoderModel`) + - tokenizer (`T5Tokenizer`) + - guider (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - pachifier (`LTXVideoPachifier`) + - transformer (`LTXVideoTransformer3DModel`) + - vae (`AutoencoderKLLTXVideo`) + - video_processor (`VideoProcessor`) Inputs: prompt (`str`): @@ -239,7 +248,8 @@ class LTXAutoVaeEncoderStep(AutoPipelineBlocks): - If `image` is not provided, step will be skipped. Components: - vae (`AutoencoderKLLTXVideo`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLLTXVideo`) + - video_processor (`VideoProcessor`) Inputs: image (`Image | list`, *optional*): @@ -279,8 +289,10 @@ class LTXAutoCoreDenoiseStep(AutoPipelineBlocks): - `LTXCoreDenoiseStep` is used otherwise (text-to-video). Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) pachifier (`LTXVideoPachifier`) guider - (`ClassifierFreeGuidance`) transformer (`LTXVideoTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - pachifier (`LTXVideoPachifier`) + - guider (`ClassifierFreeGuidance`) + - transformer (`LTXVideoTransformer3DModel`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -345,9 +357,14 @@ class LTXAutoBlocks(SequentialPipelineBlocks): - `image2video`: requires `image`, `prompt` Components: - text_encoder (`T5EncoderModel`) tokenizer (`T5Tokenizer`) guider (`ClassifierFreeGuidance`) vae - (`AutoencoderKLLTXVideo`) video_processor (`VideoProcessor`) scheduler (`FlowMatchEulerDiscreteScheduler`) - pachifier (`LTXVideoPachifier`) transformer (`LTXVideoTransformer3DModel`) + - text_encoder (`T5EncoderModel`) + - tokenizer (`T5Tokenizer`) + - guider (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLLTXVideo`) + - video_processor (`VideoProcessor`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - pachifier (`LTXVideoPachifier`) + - transformer (`LTXVideoTransformer3DModel`) Inputs: prompt (`str`): @@ -422,9 +439,14 @@ class LTXImage2VideoBlocks(SequentialPipelineBlocks): Modular pipeline blocks for LTX Video image-to-video. Components: - text_encoder (`T5EncoderModel`) tokenizer (`T5Tokenizer`) guider (`ClassifierFreeGuidance`) vae - (`AutoencoderKLLTXVideo`) video_processor (`VideoProcessor`) scheduler (`FlowMatchEulerDiscreteScheduler`) - pachifier (`LTXVideoPachifier`) transformer (`LTXVideoTransformer3DModel`) + - text_encoder (`T5EncoderModel`) + - tokenizer (`T5Tokenizer`) + - guider (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLLTXVideo`) + - video_processor (`VideoProcessor`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - pachifier (`LTXVideoPachifier`) + - transformer (`LTXVideoTransformer3DModel`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py index aa0990f3cc80..b48d29212820 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py @@ -65,7 +65,8 @@ class LTX2AutoPromptEnhancerStep(ConditionalPipelineBlocks): - Skipped when `enable_prompt_enhancement` is falsy / not provided. Components: - prompt_enhancer (`PreTrainedModel`) processor (`ProcessorMixin`) + - prompt_enhancer (`PreTrainedModel`) + - processor (`ProcessorMixin`) Inputs: prompt (`str`, *optional*): @@ -130,7 +131,9 @@ class LTX2TextConditioningStep(SequentialPipelineBlocks): stage expands them by `num_videos_per_prompt` -- so they can be reused across denoise runs. Components: - text_encoder (`PreTrainedModel`) tokenizer (`PreTrainedTokenizerBase`) connectors (`LTX2TextConnectors`) + - text_encoder (`PreTrainedModel`) + - tokenizer (`PreTrainedTokenizerBase`) + - connectors (`LTX2TextConnectors`) Inputs: prompt (`str`): @@ -189,7 +192,7 @@ class LTX2AutoDurationStep(ConditionalPipelineBlocks): - Skipped when `num_frames` is supplied as an integer. Components: - duration_head (`LTX2DurationHead`) + - duration_head (`LTX2DurationHead`) Inputs: min_seconds (`float`, *optional*, defaults to 1.0): @@ -235,7 +238,9 @@ class LTX2AutoVaeEncoderStep(AutoPipelineBlocks): - Skipped otherwise. Components: - vae (`AutoencoderKLLTX2Video`) text_encoder (`PreTrainedModel`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLLTX2Video`) + - text_encoder (`PreTrainedModel`) + - video_processor (`VideoProcessor`) Inputs: image (`Image | list`, *optional*): @@ -278,8 +283,11 @@ class LTX2CoreDenoiseStep(SequentialPipelineBlocks): latents and runs the joint denoising loop. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`LTX2VideoTransformer3DModel`) audio_vae - (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`) audio_guider (`LTX2Guidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`LTX2VideoTransformer3DModel`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - guider (`LTX2Guidance`) + - audio_guider (`LTX2Guidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -382,8 +390,11 @@ class LTX2Image2VideoCoreDenoiseStep(SequentialPipelineBlocks): conditioning and runs the joint denoising loop. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`LTX2VideoTransformer3DModel`) audio_vae - (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`) audio_guider (`LTX2Guidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`LTX2VideoTransformer3DModel`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - guider (`LTX2Guidance`) + - audio_guider (`LTX2Guidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -490,9 +501,12 @@ class LTX2ConditionCoreDenoiseStep(SequentialPipelineBlocks): conditions to the video latents and runs the joint denoising loop. Components: - transformer (`LTX2VideoTransformer3DModel`) vae (`AutoencoderKLLTX2Video`) scheduler - (`FlowMatchEulerDiscreteScheduler`) audio_vae (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`) audio_guider - (`LTX2Guidance`) + - transformer (`LTX2VideoTransformer3DModel`) + - vae (`AutoencoderKLLTX2Video`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - guider (`LTX2Guidance`) + - audio_guider (`LTX2Guidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -603,7 +617,8 @@ class LTX2AutoConditionEncoderStep(ConditionalPipelineBlocks): - Skipped for text-to-video and image-to-video. Components: - vae (`AutoencoderKLLTX2Video`) text_encoder (`PreTrainedModel`) + - vae (`AutoencoderKLLTX2Video`) + - text_encoder (`PreTrainedModel`) Inputs: conditions (`list`, *optional*): @@ -662,7 +677,9 @@ class LTX2AutoReferenceEncoderStep(ConditionalPipelineBlocks): - Skipped otherwise, for IC-LoRAs that take no reference video. Components: - vae (`AutoencoderKLLTX2Video`) transformer (`LTX2VideoTransformer3DModel`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLLTX2Video`) + - transformer (`LTX2VideoTransformer3DModel`) + - video_processor (`VideoProcessor`) Inputs: reference_conditions (`list`, *optional*): @@ -799,9 +816,12 @@ class LTX2InContextCoreDenoiseStep(SequentialPipelineBlocks): the reference implementation's uniform treatment of both. Components: - transformer (`LTX2VideoTransformer3DModel`) vae (`AutoencoderKLLTX2Video`) scheduler - (`FlowMatchEulerDiscreteScheduler`) audio_vae (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`) audio_guider - (`LTX2Guidance`) + - transformer (`LTX2VideoTransformer3DModel`) + - vae (`AutoencoderKLLTX2Video`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - guider (`LTX2Guidance`) + - audio_guider (`LTX2Guidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -934,9 +954,12 @@ class LTX2AutoCoreDenoiseStep(ConditionalPipelineBlocks): - `LTX2CoreDenoiseStep` otherwise (text-to-video). Components: - transformer (`LTX2VideoTransformer3DModel`) vae (`AutoencoderKLLTX2Video`) scheduler - (`FlowMatchEulerDiscreteScheduler`) audio_vae (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`) audio_guider - (`LTX2Guidance`) + - transformer (`LTX2VideoTransformer3DModel`) + - vae (`AutoencoderKLLTX2Video`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - guider (`LTX2Guidance`) + - audio_guider (`LTX2Guidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -1062,8 +1085,10 @@ class LTX2DecoderStep(SequentialPipelineBlocks): Decode stage: VAE-decodes the video latents and vocodes the audio latents (or returns latents). Components: - vae (`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) audio_vae (`AutoencoderKLLTX2Audio`) - vocoder (`LTX2Vocoder`) + - vae (`AutoencoderKLLTX2Video`) + - video_processor (`VideoProcessor`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - vocoder (`LTX2Vocoder`) Inputs: latents (`Tensor`): @@ -1123,8 +1148,10 @@ class LTX2ConditionDecoderStep(SequentialPipelineBlocks): latents and vocodes the audio latents (or returns latents). Components: - vae (`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) audio_vae (`AutoencoderKLLTX2Audio`) - vocoder (`LTX2Vocoder`) + - vae (`AutoencoderKLLTX2Video`) + - video_processor (`VideoProcessor`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - vocoder (`LTX2Vocoder`) Inputs: latents (`Tensor`): @@ -1191,8 +1218,10 @@ class LTX2AutoDecoderStep(AutoPipelineBlocks): - `LTX2DecoderStep` otherwise (text-to-video, image-to-video). Components: - vae (`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) audio_vae (`AutoencoderKLLTX2Audio`) - vocoder (`LTX2Vocoder`) + - vae (`AutoencoderKLLTX2Video`) + - video_processor (`VideoProcessor`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - vocoder (`LTX2Vocoder`) Inputs: latents (`Tensor`): @@ -1261,11 +1290,20 @@ class LTX2Blocks(SequentialPipelineBlocks): Modular pipeline blocks for LTX-2 text-to-video (joint video + audio). Components: - prompt_enhancer (`PreTrainedModel`) processor (`ProcessorMixin`) text_encoder (`PreTrainedModel`) tokenizer - (`PreTrainedTokenizerBase`) connectors (`LTX2TextConnectors`) duration_head (`LTX2DurationHead`) scheduler - (`FlowMatchEulerDiscreteScheduler`) transformer (`LTX2VideoTransformer3DModel`) audio_vae - (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`) audio_guider (`LTX2Guidance`) vae - (`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) vocoder (`LTX2Vocoder`) + - prompt_enhancer (`PreTrainedModel`) + - processor (`ProcessorMixin`) + - text_encoder (`PreTrainedModel`) + - tokenizer (`PreTrainedTokenizerBase`) + - connectors (`LTX2TextConnectors`) + - duration_head (`LTX2DurationHead`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`LTX2VideoTransformer3DModel`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - guider (`LTX2Guidance`) + - audio_guider (`LTX2Guidance`) + - vae (`AutoencoderKLLTX2Video`) + - video_processor (`VideoProcessor`) + - vocoder (`LTX2Vocoder`) Inputs: prompt (`str`, *optional*): @@ -1373,11 +1411,20 @@ class LTX2ImageToVideoBlocks(SequentialPipelineBlocks): Modular pipeline blocks for LTX-2 image-to-video (joint video + audio). Components: - prompt_enhancer (`PreTrainedModel`) processor (`ProcessorMixin`) text_encoder (`PreTrainedModel`) tokenizer - (`PreTrainedTokenizerBase`) connectors (`LTX2TextConnectors`) duration_head (`LTX2DurationHead`) vae - (`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) scheduler (`FlowMatchEulerDiscreteScheduler`) - transformer (`LTX2VideoTransformer3DModel`) audio_vae (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`) - audio_guider (`LTX2Guidance`) vocoder (`LTX2Vocoder`) + - prompt_enhancer (`PreTrainedModel`) + - processor (`ProcessorMixin`) + - text_encoder (`PreTrainedModel`) + - tokenizer (`PreTrainedTokenizerBase`) + - connectors (`LTX2TextConnectors`) + - duration_head (`LTX2DurationHead`) + - vae (`AutoencoderKLLTX2Video`) + - video_processor (`VideoProcessor`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`LTX2VideoTransformer3DModel`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - guider (`LTX2Guidance`) + - audio_guider (`LTX2Guidance`) + - vocoder (`LTX2Vocoder`) Inputs: prompt (`str`, *optional*): @@ -1494,11 +1541,20 @@ class LTX2ConditionBlocks(SequentialPipelineBlocks): arbitrary latent frame indices of the generated video. Components: - prompt_enhancer (`PreTrainedModel`) processor (`ProcessorMixin`) text_encoder (`PreTrainedModel`) tokenizer - (`PreTrainedTokenizerBase`) connectors (`LTX2TextConnectors`) duration_head (`LTX2DurationHead`) vae - (`AutoencoderKLLTX2Video`) transformer (`LTX2VideoTransformer3DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) audio_vae (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`) audio_guider - (`LTX2Guidance`) video_processor (`VideoProcessor`) vocoder (`LTX2Vocoder`) + - prompt_enhancer (`PreTrainedModel`) + - processor (`ProcessorMixin`) + - text_encoder (`PreTrainedModel`) + - tokenizer (`PreTrainedTokenizerBase`) + - connectors (`LTX2TextConnectors`) + - duration_head (`LTX2DurationHead`) + - vae (`AutoencoderKLLTX2Video`) + - transformer (`LTX2VideoTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - guider (`LTX2Guidance`) + - audio_guider (`LTX2Guidance`) + - video_processor (`VideoProcessor`) + - vocoder (`LTX2Vocoder`) Inputs: prompt (`str`, *optional*): @@ -1614,11 +1670,19 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): `LTX2InContextPipeline`. Components: - prompt_enhancer (`PreTrainedModel`) processor (`ProcessorMixin`) text_encoder (`PreTrainedModel`) tokenizer - (`PreTrainedTokenizerBase`) connectors (`LTX2TextConnectors`) vae (`AutoencoderKLLTX2Video`) transformer - (`LTX2VideoTransformer3DModel`) video_processor (`VideoProcessor`) scheduler - (`FlowMatchEulerDiscreteScheduler`) audio_vae (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`) audio_guider - (`LTX2Guidance`) vocoder (`LTX2Vocoder`) + - prompt_enhancer (`PreTrainedModel`) + - processor (`ProcessorMixin`) + - text_encoder (`PreTrainedModel`) + - tokenizer (`PreTrainedTokenizerBase`) + - connectors (`LTX2TextConnectors`) + - vae (`AutoencoderKLLTX2Video`) + - transformer (`LTX2VideoTransformer3DModel`) + - video_processor (`VideoProcessor`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - guider (`LTX2Guidance`) + - audio_guider (`LTX2Guidance`) + - vocoder (`LTX2Vocoder`) Inputs: prompt (`str`, *optional*): @@ -1770,11 +1834,20 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): - `in_context`: requires `reference_conditions`, `num_frames`, `prompt` Components: - prompt_enhancer (`PreTrainedModel`) processor (`ProcessorMixin`) text_encoder (`PreTrainedModel`) tokenizer - (`PreTrainedTokenizerBase`) connectors (`LTX2TextConnectors`) duration_head (`LTX2DurationHead`) vae - (`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) transformer (`LTX2VideoTransformer3DModel`) - scheduler (`FlowMatchEulerDiscreteScheduler`) audio_vae (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`) - audio_guider (`LTX2Guidance`) vocoder (`LTX2Vocoder`) + - prompt_enhancer (`PreTrainedModel`) + - processor (`ProcessorMixin`) + - text_encoder (`PreTrainedModel`) + - tokenizer (`PreTrainedTokenizerBase`) + - connectors (`LTX2TextConnectors`) + - duration_head (`LTX2DurationHead`) + - vae (`AutoencoderKLLTX2Video`) + - video_processor (`VideoProcessor`) + - transformer (`LTX2VideoTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - guider (`LTX2Guidance`) + - audio_guider (`LTX2Guidance`) + - vocoder (`LTX2Vocoder`) Inputs: prompt (`str`, *optional*): diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py index 3280820eab0c..7061d8cdbb6e 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py @@ -39,8 +39,10 @@ class LTX25DecoderStep(SequentialPipelineBlocks): returns latents). Components: - diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) video_processor (`VideoProcessor`) audio_vae - (`AutoencoderKLLTX2Audio`) vocoder (`LTX2Vocoder`) + - diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) + - video_processor (`VideoProcessor`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - vocoder (`LTX2Vocoder`) Inputs: latents (`Tensor`): @@ -95,8 +97,10 @@ class LTX25ConditionDecoderStep(SequentialPipelineBlocks): latents with the diffusion decoder and vocodes the audio latents (or returns latents). Components: - diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) video_processor (`VideoProcessor`) audio_vae - (`AutoencoderKLLTX2Audio`) vocoder (`LTX2Vocoder`) + - diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) + - video_processor (`VideoProcessor`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - vocoder (`LTX2Vocoder`) Inputs: latents (`Tensor`): @@ -156,8 +160,10 @@ class LTX25AutoDecoderStep(AutoPipelineBlocks): - `LTX25DecoderStep` otherwise (text-to-video, image-to-video). Components: - diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) video_processor (`VideoProcessor`) audio_vae - (`AutoencoderKLLTX2Audio`) vocoder (`LTX2Vocoder`) + - diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) + - video_processor (`VideoProcessor`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - vocoder (`LTX2Vocoder`) Inputs: latents (`Tensor`): @@ -226,11 +232,21 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): - `in_context`: requires `reference_conditions`, `num_frames`, `prompt` Components: - prompt_enhancer (`PreTrainedModel`) processor (`ProcessorMixin`) text_encoder (`PreTrainedModel`) tokenizer - (`PreTrainedTokenizerBase`) connectors (`LTX2TextConnectors`) duration_head (`LTX2DurationHead`) vae - (`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) transformer (`LTX2VideoTransformer3DModel`) - scheduler (`FlowMatchEulerDiscreteScheduler`) audio_vae (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`) - audio_guider (`LTX2Guidance`) diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) vocoder (`LTX2Vocoder`) + - prompt_enhancer (`PreTrainedModel`) + - processor (`ProcessorMixin`) + - text_encoder (`PreTrainedModel`) + - tokenizer (`PreTrainedTokenizerBase`) + - connectors (`LTX2TextConnectors`) + - duration_head (`LTX2DurationHead`) + - vae (`AutoencoderKLLTX2Video`) + - video_processor (`VideoProcessor`) + - transformer (`LTX2VideoTransformer3DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - audio_vae (`AutoencoderKLLTX2Audio`) + - guider (`LTX2Guidance`) + - audio_guider (`LTX2Guidance`) + - diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) + - vocoder (`LTX2Vocoder`) Inputs: prompt (`str`, *optional*): diff --git a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py index e96454a44fa4..9b3024f3d863 100644 --- a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py +++ b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py @@ -49,7 +49,7 @@ class MiniMaxH3AutoBeforeEncodeStep(ConditionalPipelineBlocks): - a text-only request (`t2va`) skips this block, and the layout step falls back to MiniMax-H3's own 16:9 canvas. Components: - image_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) Configs: canvas_short_edge (default: 768) canvas_max_pixels (default: 1032192) reference_image_short_edge (default: @@ -135,7 +135,9 @@ class MiniMaxH3AutoTextEncoderStep(ConditionalPipelineBlocks): - `MiniMaxH3TextEncoderStep` runs otherwise (`t2va`), presenting the prompt alone. Components: - text_encoder (`Qwen3VLForConditionalGeneration`) tokenizer (`Qwen2Tokenizer`) processor (`Qwen3VLProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - tokenizer (`Qwen2Tokenizer`) + - processor (`Qwen3VLProcessor`) Inputs: prompt (`str`): @@ -188,7 +190,8 @@ class MiniMaxH3AutoVaeEncoderStep(ConditionalPipelineBlocks): - a text-only request (`t2va`) skips this block. Components: - vae (`AutoencoderKLMiniMaxH3`) audio_vae (`AutoencoderKLMiniMaxH3Audio`) + - vae (`AutoencoderKLMiniMaxH3`) + - audio_vae (`AutoencoderKLMiniMaxH3Audio`) Inputs: normalized_references (`list`, *optional*): @@ -238,8 +241,9 @@ class MiniMaxH3CoreDenoiseStep(SequentialPipelineBlocks): latents. Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer - (`MiniMaxH3Transformer3DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3Transformer3DModel`) Configs: canvas_short_edge (default: 768) canvas_max_pixels (default: 1032192) @@ -331,8 +335,9 @@ class MiniMaxH3FL2VACoreDenoiseStep(SequentialPipelineBlocks): `transformer` partition and unpacks the denoised rows back into latents. Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer - (`MiniMaxH3Transformer3DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3Transformer3DModel`) Configs: canvas_short_edge (default: 768) canvas_max_pixels (default: 1032192) @@ -432,8 +437,9 @@ class MiniMaxH3Ref2VACoreDenoiseStep(SequentialPipelineBlocks): denoised rows back into latents. Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer_ref - (`MiniMaxH3Transformer3DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer_ref (`MiniMaxH3Transformer3DModel`) Inputs: text_token_tags (`Tensor`): @@ -534,8 +540,10 @@ class MiniMaxH3AutoDenoiseStep(ConditionalPipelineBlocks): - `MiniMaxH3CoreDenoiseStep` runs otherwise (`t2va`), against the `transformer` partition. Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer_ref - (`MiniMaxH3Transformer3DModel`) transformer (`MiniMaxH3Transformer3DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer_ref (`MiniMaxH3Transformer3DModel`) + - transformer (`MiniMaxH3Transformer3DModel`) Configs: canvas_short_edge (default: 768) canvas_max_pixels (default: 1032192) @@ -615,7 +623,9 @@ class MiniMaxH3DecodeStep(SequentialPipelineBlocks): Decodes the denoised rows of the packed sequence into the generated video and its soundtrack. Components: - vae (`AutoencoderKLMiniMaxH3`) video_processor (`VideoProcessor`) audio_vae (`AutoencoderKLMiniMaxH3Audio`) + - vae (`AutoencoderKLMiniMaxH3`) + - video_processor (`VideoProcessor`) + - audio_vae (`AutoencoderKLMiniMaxH3Audio`) Inputs: latents (`Tensor`): @@ -669,11 +679,17 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): - `ref2va`: requires `prompt`, `references` Components: - image_processor (`VaeImageProcessor`) text_encoder (`Qwen3VLForConditionalGeneration`) tokenizer - (`Qwen2Tokenizer`) processor (`Qwen3VLProcessor`) vae (`AutoencoderKLMiniMaxH3`) audio_vae - (`AutoencoderKLMiniMaxH3Audio`) scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) - transformer_ref (`MiniMaxH3Transformer3DModel`) transformer (`MiniMaxH3Transformer3DModel`) video_processor - (`VideoProcessor`) + - image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - tokenizer (`Qwen2Tokenizer`) + - processor (`Qwen3VLProcessor`) + - vae (`AutoencoderKLMiniMaxH3`) + - audio_vae (`AutoencoderKLMiniMaxH3Audio`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer_ref (`MiniMaxH3Transformer3DModel`) + - transformer (`MiniMaxH3Transformer3DModel`) + - video_processor (`VideoProcessor`) Configs: canvas_short_edge (default: 768) canvas_max_pixels (default: 1032192) reference_image_short_edge (default: diff --git a/src/diffusers/modular_pipelines/minimax_h3_image/modular_blocks_minimax_h3_image.py b/src/diffusers/modular_pipelines/minimax_h3_image/modular_blocks_minimax_h3_image.py index 86bc6e0d326b..1d8019a199eb 100644 --- a/src/diffusers/modular_pipelines/minimax_h3_image/modular_blocks_minimax_h3_image.py +++ b/src/diffusers/modular_pipelines/minimax_h3_image/modular_blocks_minimax_h3_image.py @@ -43,7 +43,7 @@ class MiniMaxH3ImageAutoBeforeEncodeStep(AutoPipelineBlocks): - a `text2image` request skips this block. Components: - image_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) Configs: reference_min_short_edge (default: 1024) reference_max_short_edge (default: 2048) vision_min_short_edge @@ -84,7 +84,9 @@ class MiniMaxH3ImageAutoTextEncoderStep(AutoPipelineBlocks): - `MiniMaxH3ImageTextEncoderStep` runs otherwise (`text2image`). Components: - text_encoder (`Qwen3VLForConditionalGeneration`) tokenizer (`Qwen2Tokenizer`) processor (`Qwen3VLProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - tokenizer (`Qwen2Tokenizer`) + - processor (`Qwen3VLProcessor`) Inputs: prompt (`str`): @@ -124,7 +126,7 @@ class MiniMaxH3ImageAutoVaeEncoderStep(AutoPipelineBlocks): - a `text2image` request skips this block. Components: - vae (`AutoencoderKLMiniMaxH3Image`) + - vae (`AutoencoderKLMiniMaxH3Image`) Inputs: reference_images (`list`, *optional*): @@ -159,7 +161,8 @@ class MiniMaxH3ImageCoreDenoiseStep(SequentialPipelineBlocks): denoising loop and unpacks the latents. Components: - scheduler (`MiniMaxH3Scheduler`) transformer (`MiniMaxH3ImageTransformer2DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3ImageTransformer2DModel`) Configs: sample_sigmas (default: [1.0, 0.9675236940383911, 0.8875485062599182, 0.75, 0.5327739119529724, @@ -228,7 +231,8 @@ class MiniMaxH3ImageEditCoreDenoiseStep(SequentialPipelineBlocks): puts the noised reference rows in front of the generated rows, runs the denoising loop and unpacks the latents. Components: - scheduler (`MiniMaxH3Scheduler`) transformer (`MiniMaxH3ImageTransformer2DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3ImageTransformer2DModel`) Configs: sample_sigmas (default: [1.0, 0.9675236940383911, 0.8875485062599182, 0.75, 0.5327739119529724, @@ -314,7 +318,8 @@ class MiniMaxH3ImageAutoDenoiseStep(AutoPipelineBlocks): - `MiniMaxH3ImageCoreDenoiseStep` runs otherwise (`text2image`). Components: - scheduler (`MiniMaxH3Scheduler`) transformer (`MiniMaxH3ImageTransformer2DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3ImageTransformer2DModel`) Configs: sample_sigmas (default: [1.0, 0.9675236940383911, 0.8875485062599182, 0.75, 0.5327739119529724, @@ -378,9 +383,13 @@ class MiniMaxH3ImageBlocks(SequentialPipelineBlocks): - `image_conditioned`: requires `prompt`, `image` Components: - image_processor (`VaeImageProcessor`) text_encoder (`Qwen3VLForConditionalGeneration`) tokenizer - (`Qwen2Tokenizer`) processor (`Qwen3VLProcessor`) vae (`AutoencoderKLMiniMaxH3Image`) scheduler - (`MiniMaxH3Scheduler`) transformer (`MiniMaxH3ImageTransformer2DModel`) + - image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - tokenizer (`Qwen2Tokenizer`) + - processor (`Qwen3VLProcessor`) + - vae (`AutoencoderKLMiniMaxH3Image`) + - scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3ImageTransformer2DModel`) Configs: reference_min_short_edge (default: 1024) reference_max_short_edge (default: 2048) vision_min_short_edge diff --git a/src/diffusers/modular_pipelines/minimax_music3/modular_blocks_minimax_music3.py b/src/diffusers/modular_pipelines/minimax_music3/modular_blocks_minimax_music3.py index 2fee8c484ec6..0397de2012cd 100644 --- a/src/diffusers/modular_pipelines/minimax_music3/modular_blocks_minimax_music3.py +++ b/src/diffusers/modular_pipelines/minimax_music3/modular_blocks_minimax_music3.py @@ -43,8 +43,9 @@ class MiniMaxMusic3SemanticGenerationStep(SequentialPipelineBlocks): condition the flow-matching stage. Components: - tokenizer (`Qwen2Tokenizer`) language_model (`Qwen3ForCausalLM`) rvq_depth_decoder - (`MiniMaxMusic3RVQDepthDecoder`) + - tokenizer (`Qwen2Tokenizer`) + - language_model (`Qwen3ForCausalLM`) + - rvq_depth_decoder (`MiniMaxMusic3RVQDepthDecoder`) Inputs: prompt (`str`): @@ -96,8 +97,10 @@ class MiniMaxMusic3CoreDenoiseStep(SequentialPipelineBlocks): Flow-VAE latents from noise, blending every window into the previous one over their overlap. Components: - condition_encoder (`MiniMaxMusic3ConditionEncoder`) transformer (`MiniMaxMusic3Transformer1DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) + - condition_encoder (`MiniMaxMusic3ConditionEncoder`) + - transformer (`MiniMaxMusic3Transformer1DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: frame_hiddens (`Tensor`): @@ -135,10 +138,14 @@ class MiniMaxMusic3Blocks(SequentialPipelineBlocks): DAC-style vocoder decodes them into a stereo waveform at 44.1 kHz. Components: - tokenizer (`Qwen2Tokenizer`) language_model (`Qwen3ForCausalLM`) rvq_depth_decoder - (`MiniMaxMusic3RVQDepthDecoder`) condition_encoder (`MiniMaxMusic3ConditionEncoder`) transformer - (`MiniMaxMusic3Transformer1DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) vocoder (`MiniMaxMusic3Vocoder`) + - tokenizer (`Qwen2Tokenizer`) + - language_model (`Qwen3ForCausalLM`) + - rvq_depth_decoder (`MiniMaxMusic3RVQDepthDecoder`) + - condition_encoder (`MiniMaxMusic3ConditionEncoder`) + - transformer (`MiniMaxMusic3Transformer1DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - vocoder (`MiniMaxMusic3Vocoder`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/modular_pipeline_utils.py b/src/diffusers/modular_pipelines/modular_pipeline_utils.py index 09960243e8b2..3271b6cfd7e3 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline_utils.py +++ b/src/diffusers/modular_pipelines/modular_pipeline_utils.py @@ -828,14 +828,14 @@ def get_type_str(type_hint): return "\n".join(lines) -def format_components(components, indent_level=4, max_line_length=115, add_empty_lines=True): +def format_components(components, indent_level=4, *, add_empty_lines=True, list_marker=""): """Format a list of ComponentSpec objects into a readable string representation. Args: components: list of ComponentSpec objects to format indent_level: Number of spaces to indent each component line (default: 4) - max_line_length: Maximum length for each line before wrapping (default: 115) add_empty_lines: Whether to add empty lines between components (default: True) + list_marker: Prefix for each component name, such as `"- "` for a bullet. Defaults to an empty string. Returns: A formatted string representing all components @@ -859,7 +859,7 @@ def format_components(components, indent_level=4, max_line_length=115, add_empty component.type_hint.__name__ if hasattr(component.type_hint, "__name__") else str(component.type_hint) ) - component_desc = f"{component_indent}{component.name} (`{type_name}`)" + component_desc = f"{component_indent}{list_marker}{component.name} (`{type_name}`)" if component.description: component_desc += f": {component.description}" @@ -883,13 +883,12 @@ def format_components(components, indent_level=4, max_line_length=115, add_empty return "\n".join(formatted_components) -def format_configs(configs, indent_level=4, max_line_length=115, add_empty_lines=True): +def format_configs(configs, indent_level=4, *, add_empty_lines=True): """Format a list of ConfigSpec objects into a readable string representation. Args: configs: list of ConfigSpec objects to format indent_level: Number of spaces to indent each config line (default: 4) - max_line_length: Maximum length for each line before wrapping (default: 115) add_empty_lines: Whether to add empty lines between configs (default: True) Returns: @@ -986,7 +985,9 @@ def make_doc_string( # Add components section if provided if expected_components and len(expected_components) > 0: - components_str = format_components(expected_components, indent_level=2, add_empty_lines=False) + components_str = format_components( + expected_components, indent_level=2, add_empty_lines=False, list_marker="- " + ) output += components_str + "\n\n" # Add configs section if provided diff --git a/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py b/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py index 020201616522..794cfa104a7a 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py +++ b/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py @@ -40,7 +40,7 @@ class PH3ImageAutoBeforeEncodeStep(AutoPipelineBlocks): - a `text2image` request skips this block. Components: - image_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) Configs: reference_min_pixels (default: 1048576) reference_max_pixels (default: 8388608) canvas_min_pixels (default: @@ -91,8 +91,10 @@ class PH3ImageAutoTextEncoderStep(AutoPipelineBlocks): - `PH3ImageTextEncoderStep` runs otherwise (`text2image`). Components: - text_encoder (`Qwen3VLForConditionalGeneration`) text_connector (`PH3ImageTextConnector`) tokenizer - (`Qwen2Tokenizer`) processor (`Qwen3VLProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - text_connector (`PH3ImageTextConnector`) + - tokenizer (`Qwen2Tokenizer`) + - processor (`Qwen3VLProcessor`) Inputs: prompt (`str`): @@ -129,7 +131,7 @@ class PH3ImageAutoVaeEncoderStep(AutoPipelineBlocks): - a `text2image` request skips this block. Components: - vae (`AutoencoderKLPH3Image`) + - vae (`AutoencoderKLPH3Image`) Inputs: reference_images (`list`, *optional*): @@ -162,8 +164,9 @@ class PH3ImageCoreDenoiseStep(SequentialPipelineBlocks): unpacks the latents. Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer - (`MiniMaxH3Transformer3DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3Transformer3DModel`) Configs: sample_sigmas (default: [1.0, 0.984753, 0.967905, 0.951688, 0.925372, 0.881923, 0.810083, 0.528718, 0.0]): @@ -234,8 +237,9 @@ class PH3ImageEditCoreDenoiseStep(SequentialPipelineBlocks): edit packed in front of the generated rows. Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer - (`MiniMaxH3Transformer3DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3Transformer3DModel`) Configs: sample_sigmas (default: [1.0, 0.984753, 0.967905, 0.951688, 0.925372, 0.881923, 0.810083, 0.528718, 0.0]): @@ -319,8 +323,9 @@ class PH3ImageAutoDenoiseStep(AutoPipelineBlocks): - `PH3ImageCoreDenoiseStep` runs otherwise (`text2image`). Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer - (`MiniMaxH3Transformer3DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3Transformer3DModel`) Configs: sample_sigmas (default: [1.0, 0.984753, 0.967905, 0.951688, 0.925372, 0.881923, 0.810083, 0.528718, 0.0]): @@ -384,11 +389,17 @@ class PH3ImageBlocks(SequentialPipelineBlocks): - `image_conditioned`: requires `prompt`, `image` Components: - image_processor (`VaeImageProcessor`) prompt_upsampler (`Qwen3_5ForConditionalGeneration`): The prompt - upsampler. prompt_upsampler_processor (`AutoProcessor`): Its processor. text_encoder - (`Qwen3VLForConditionalGeneration`) text_connector (`PH3ImageTextConnector`) tokenizer (`Qwen2Tokenizer`) - processor (`Qwen3VLProcessor`) vae (`AutoencoderKLPH3Image`) scheduler (`MiniMaxH3Scheduler`) audio_scheduler - (`MiniMaxH3Scheduler`) transformer (`MiniMaxH3Transformer3DModel`) + - image_processor (`VaeImageProcessor`) + - prompt_upsampler (`Qwen3_5ForConditionalGeneration`): The prompt upsampler. + - prompt_upsampler_processor (`AutoProcessor`): Its processor. + - text_encoder (`Qwen3VLForConditionalGeneration`) + - text_connector (`PH3ImageTextConnector`) + - tokenizer (`Qwen2Tokenizer`) + - processor (`Qwen3VLProcessor`) + - vae (`AutoencoderKLPH3Image`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3Transformer3DModel`) Configs: reference_min_pixels (default: 1048576) reference_max_pixels (default: 8388608) canvas_min_pixels (default: diff --git a/src/diffusers/modular_pipelines/p_h3_image/prompt_upsampler.py b/src/diffusers/modular_pipelines/p_h3_image/prompt_upsampler.py index 60302b9d4439..a6516421de6c 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/prompt_upsampler.py +++ b/src/diffusers/modular_pipelines/p_h3_image/prompt_upsampler.py @@ -258,8 +258,8 @@ class PH3ImageAutoPromptUpsampleStep(AutoPipelineBlocks): To render `prompt` as given, remove the step from the blocks: `blocks.sub_blocks.pop("prompt_upsampler")`. Components: - prompt_upsampler (`Qwen3_5ForConditionalGeneration`): The prompt upsampler. prompt_upsampler_processor - (`AutoProcessor`): Its processor. + - prompt_upsampler (`Qwen3_5ForConditionalGeneration`): The prompt upsampler. + - prompt_upsampler_processor (`AutoProcessor`): Its processor. Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/pan2/modular_blocks_pan2.py b/src/diffusers/modular_pipelines/pan2/modular_blocks_pan2.py index f860f712202d..9ade0d7bd800 100644 --- a/src/diffusers/modular_pipelines/pan2/modular_blocks_pan2.py +++ b/src/diffusers/modular_pipelines/pan2/modular_blocks_pan2.py @@ -39,7 +39,8 @@ class PAN2AutoVaeEncoderStep(AutoPipelineBlocks): - If `image` is not provided, this step is skipped (text-to-video). Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Inputs: image (`Image`, *optional*): @@ -85,8 +86,9 @@ class PAN2CoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that runs the denoising process for text-to-video. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`PAN2Transformer3DModel`) guider - (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`PAN2Transformer3DModel`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -148,8 +150,9 @@ class PAN2Image2VideoCoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that runs the denoising process for image-to-video, conditioned on the first frame. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`PAN2Transformer3DModel`) guider - (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`PAN2Transformer3DModel`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -199,8 +202,9 @@ class PAN2AutoCoreDenoiseStep(AutoPipelineBlocks): - `PAN2CoreDenoiseStep` is used otherwise (text-to-video). Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`PAN2Transformer3DModel`) guider - (`ClassifierFreeGuidance`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`PAN2Transformer3DModel`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -257,9 +261,14 @@ class PAN2Blocks(SequentialPipelineBlocks): - `i2v`: requires `prompt`, `image` Components: - text_encoder (`Qwen3_5TextModel`) tokenizer (`Qwen2Tokenizer`) safety_checker (`PAN2SafetyChecker`) vae - (`AutoencoderKLWan`) video_processor (`VideoProcessor`) scheduler (`FlowMatchEulerDiscreteScheduler`) - transformer (`PAN2Transformer3DModel`) guider (`ClassifierFreeGuidance`) + - text_encoder (`Qwen3_5TextModel`) + - tokenizer (`Qwen2Tokenizer`) + - safety_checker (`PAN2SafetyChecker`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`PAN2Transformer3DModel`) + - guider (`ClassifierFreeGuidance`) Configs: enable_safety_checker (default: True) diff --git a/src/diffusers/modular_pipelines/pika_h3_flex/modular_blocks_pika_h3_flex.py b/src/diffusers/modular_pipelines/pika_h3_flex/modular_blocks_pika_h3_flex.py index 06dd9265a3c6..b07a3cbca80c 100644 --- a/src/diffusers/modular_pipelines/pika_h3_flex/modular_blocks_pika_h3_flex.py +++ b/src/diffusers/modular_pipelines/pika_h3_flex/modular_blocks_pika_h3_flex.py @@ -49,7 +49,7 @@ class PikaH3FlexAutoBeforeEncodeStep(ConditionalPipelineBlocks): - a text-only request (`t2va`) skips this block, and the layout step falls back to the 768p 16:9 canvas. Components: - image_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) Configs: canvas_short_edge (default: 768) canvas_max_pixels (default: 1032192) @@ -138,7 +138,9 @@ class PikaH3FlexAutoTextEncoderStep(ConditionalPipelineBlocks): - `PikaH3FlexTextEncoderStep` runs otherwise (`t2va`), presenting the prompt alone. Components: - text_encoder (`Qwen3VLForConditionalGeneration`) tokenizer (`Qwen2Tokenizer`) processor (`Qwen3VLProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - tokenizer (`Qwen2Tokenizer`) + - processor (`Qwen3VLProcessor`) Inputs: prompt (`str`): @@ -192,7 +194,8 @@ class PikaH3FlexAutoVaeEncoderStep(ConditionalPipelineBlocks): - a text-only request (`t2va`) skips this block. Components: - vae (`AutoencoderKLMiniMaxH3`) audio_vae (`AutoencoderKLMiniMaxH3Audio`) + - vae (`AutoencoderKLMiniMaxH3`) + - audio_vae (`AutoencoderKLMiniMaxH3Audio`) Inputs: normalized_references (`list`, *optional*): @@ -242,8 +245,9 @@ class PikaH3FlexCoreDenoiseStep(SequentialPipelineBlocks): and unpacks the denoised rows back into latents. Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer - (`MiniMaxH3Transformer3DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3Transformer3DModel`) Configs: canvas_short_edge (default: 768) canvas_max_pixels (default: 1032192) denoising_step_list (default: [1000, @@ -334,8 +338,9 @@ class PikaH3FlexFL2VACoreDenoiseStep(SequentialPipelineBlocks): against the `transformer` partition with the `fl2va` adapter and unpacks the denoised rows back into latents. Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer - (`MiniMaxH3Transformer3DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer (`MiniMaxH3Transformer3DModel`) Configs: canvas_short_edge (default: 768) canvas_max_pixels (default: 1032192) denoising_step_list (default: [1000, @@ -435,8 +440,9 @@ class PikaH3FlexRef2VACoreDenoiseStep(SequentialPipelineBlocks): partition with the `ref2va` adapter and unpacks the denoised rows back into latents. Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer_ref - (`MiniMaxH3Transformer3DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer_ref (`MiniMaxH3Transformer3DModel`) Configs: denoising_step_list (default: [1000, 750, 500, 250, 100, 40, 0]) @@ -540,8 +546,10 @@ class PikaH3FlexAutoDenoiseStep(ConditionalPipelineBlocks): adapter. Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer_ref - (`MiniMaxH3Transformer3DModel`) transformer (`MiniMaxH3Transformer3DModel`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer_ref (`MiniMaxH3Transformer3DModel`) + - transformer (`MiniMaxH3Transformer3DModel`) Configs: denoising_step_list (default: [1000, 750, 500, 250, 100, 40, 0]) canvas_short_edge (default: 768) @@ -621,7 +629,9 @@ class PikaH3FlexDecodeStep(SequentialPipelineBlocks): Decodes the denoised rows of the packed sequence into the generated video and its soundtrack. Components: - vae (`AutoencoderKLMiniMaxH3`) video_processor (`VideoProcessor`) audio_vae (`AutoencoderKLMiniMaxH3Audio`) + - vae (`AutoencoderKLMiniMaxH3`) + - video_processor (`VideoProcessor`) + - audio_vae (`AutoencoderKLMiniMaxH3Audio`) Inputs: latents (`Tensor`): @@ -680,11 +690,17 @@ class PikaH3FlexBlocks(SequentialPipelineBlocks): - `ref2va`: requires `prompt`, `references` Components: - image_processor (`VaeImageProcessor`) text_encoder (`Qwen3VLForConditionalGeneration`) tokenizer - (`Qwen2Tokenizer`) processor (`Qwen3VLProcessor`) vae (`AutoencoderKLMiniMaxH3`) audio_vae - (`AutoencoderKLMiniMaxH3Audio`) scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) - transformer_ref (`MiniMaxH3Transformer3DModel`) transformer (`MiniMaxH3Transformer3DModel`) video_processor - (`VideoProcessor`) + - image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - tokenizer (`Qwen2Tokenizer`) + - processor (`Qwen3VLProcessor`) + - vae (`AutoencoderKLMiniMaxH3`) + - audio_vae (`AutoencoderKLMiniMaxH3Audio`) + - scheduler (`MiniMaxH3Scheduler`) + - audio_scheduler (`MiniMaxH3Scheduler`) + - transformer_ref (`MiniMaxH3Transformer3DModel`) + - transformer (`MiniMaxH3Transformer3DModel`) + - video_processor (`VideoProcessor`) Configs: canvas_short_edge (default: 768) canvas_max_pixels (default: 1032192) denoising_step_list (default: [1000, diff --git a/src/diffusers/modular_pipelines/qwenimage/before_denoise.py b/src/diffusers/modular_pipelines/qwenimage/before_denoise.py index e4720b16244d..3eee7e673cba 100644 --- a/src/diffusers/modular_pipelines/qwenimage/before_denoise.py +++ b/src/diffusers/modular_pipelines/qwenimage/before_denoise.py @@ -123,7 +123,7 @@ class QwenImagePrepareLatentsStep(ModularPipelineBlocks): Prepare initial random noise for the generation process Components: - pachifier (`QwenImagePachifier`) + - pachifier (`QwenImagePachifier`) Inputs: latents (`Tensor`, *optional*): @@ -241,7 +241,7 @@ class QwenImageLayeredPrepareLatentsStep(ModularPipelineBlocks): Prepare initial random noise (B, layers+1, C, H, W) for the generation process Components: - pachifier (`QwenImageLayeredPachifier`) + - pachifier (`QwenImageLayeredPachifier`) Inputs: latents (`Tensor`, *optional*): @@ -363,7 +363,7 @@ class QwenImagePrepareLatentsWithStrengthStep(ModularPipelineBlocks): prepare_latents. Both noise and image latents should alreadybe patchified. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: latents (`Tensor`): @@ -469,7 +469,7 @@ class QwenImageCreateMaskLatentsStep(ModularPipelineBlocks): Step that creates mask latents from preprocessed mask_image by interpolating to latent space. Components: - pachifier (`QwenImagePachifier`) + - pachifier (`QwenImagePachifier`) Inputs: processed_mask_image (`Tensor`): @@ -561,7 +561,7 @@ class QwenImageSetTimestepsStep(ModularPipelineBlocks): Step that sets the scheduler's timesteps for text-to-image generation. Should be run after prepare latents step. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: num_inference_steps (`int`, *optional*, defaults to 50): @@ -649,7 +649,7 @@ class QwenImageLayeredSetTimestepsStep(ModularPipelineBlocks): Set timesteps step for QwenImage Layered with custom mu calculation based on image_latents. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: num_inference_steps (`int`, *optional*, defaults to 50): @@ -730,7 +730,7 @@ class QwenImageSetTimestepsWithStrengthStep(ModularPipelineBlocks): latents step. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: num_inference_steps (`int`, *optional*, defaults to 50): @@ -1240,7 +1240,7 @@ class QwenImageControlNetBeforeDenoiserStep(ModularPipelineBlocks): step that prepare inputs for controlnet. Insert before the Denoise Step, after set_timesteps step. Components: - controlnet (`QwenImageControlNetModel`) + - controlnet (`QwenImageControlNetModel`) Inputs: control_guidance_start (`float`, *optional*, defaults to 0.0): diff --git a/src/diffusers/modular_pipelines/qwenimage/decoders.py b/src/diffusers/modular_pipelines/qwenimage/decoders.py index 9ba2365416b2..5184f51ae965 100644 --- a/src/diffusers/modular_pipelines/qwenimage/decoders.py +++ b/src/diffusers/modular_pipelines/qwenimage/decoders.py @@ -39,7 +39,7 @@ class QwenImageAfterDenoiseStep(ModularPipelineBlocks): channels, 1, height, width) Components: - pachifier (`QwenImagePachifier`) + - pachifier (`QwenImagePachifier`) Inputs: height (`int`): @@ -110,7 +110,7 @@ class QwenImageLayeredAfterDenoiseStep(ModularPipelineBlocks): Unpack latents from (B, seq, C*4) to (B, C, layers+1, H, W) after denoising. Components: - pachifier (`QwenImageLayeredPachifier`) + - pachifier (`QwenImageLayeredPachifier`) Inputs: latents (`Tensor`): @@ -185,7 +185,7 @@ class QwenImageDecoderStep(ModularPipelineBlocks): Step that decodes the latents to images Components: - vae (`AutoencoderKLQwenImage`) + - vae (`AutoencoderKLQwenImage`) Inputs: latents (`Tensor`): @@ -262,7 +262,8 @@ class QwenImageLayeredDecoderStep(ModularPipelineBlocks): Decode unpacked latents (B, C, layers+1, H, W) into layer images. Components: - vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) Inputs: latents (`Tensor`): @@ -361,7 +362,7 @@ class QwenImageProcessImagesOutputStep(ModularPipelineBlocks): postprocess the generated image Components: - image_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) Inputs: images (`Tensor`): @@ -435,7 +436,7 @@ class QwenImageInpaintProcessImagesOutputStep(ModularPipelineBlocks): postprocess the generated image, optional apply the mask overally to the original image.. Components: - image_mask_processor (`InpaintProcessor`) + - image_mask_processor (`InpaintProcessor`) Inputs: images (`Tensor`): diff --git a/src/diffusers/modular_pipelines/qwenimage/denoise.py b/src/diffusers/modular_pipelines/qwenimage/denoise.py index 7b1ec4844ad8..927c716445ef 100644 --- a/src/diffusers/modular_pipelines/qwenimage/denoise.py +++ b/src/diffusers/modular_pipelines/qwenimage/denoise.py @@ -527,8 +527,9 @@ class QwenImageDenoiseStep(QwenImageDenoiseLoopWrapper): This block supports text2image and image2image tasks for QwenImage. Components: - guider (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): @@ -585,8 +586,9 @@ class QwenImageInpaintDenoiseStep(QwenImageDenoiseLoopWrapper): This block supports inpainting tasks for QwenImage. Components: - guider (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): @@ -650,8 +652,10 @@ class QwenImageControlNetDenoiseStep(QwenImageDenoiseLoopWrapper): This block supports text2img/img2img tasks with controlnet for QwenImage. Components: - guider (`ClassifierFreeGuidance`) controlnet (`QwenImageControlNetModel`) transformer - (`QwenImageTransformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - controlnet (`QwenImageControlNetModel`) + - transformer (`QwenImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): @@ -716,8 +720,10 @@ class QwenImageInpaintControlNetDenoiseStep(QwenImageDenoiseLoopWrapper): This block supports inpainting tasks with controlnet for QwenImage. Components: - guider (`ClassifierFreeGuidance`) controlnet (`QwenImageControlNetModel`) transformer - (`QwenImageTransformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - controlnet (`QwenImageControlNetModel`) + - transformer (`QwenImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): @@ -794,8 +800,9 @@ class QwenImageEditDenoiseStep(QwenImageDenoiseLoopWrapper): This block supports QwenImage Edit. Components: - guider (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): @@ -853,8 +860,9 @@ class QwenImageEditInpaintDenoiseStep(QwenImageDenoiseLoopWrapper): This block supports inpainting tasks for QwenImage Edit. Components: - guider (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): @@ -917,8 +925,9 @@ class QwenImageLayeredDenoiseStep(QwenImageDenoiseLoopWrapper): This block supports QwenImage Layered. Components: - guider (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): diff --git a/src/diffusers/modular_pipelines/qwenimage/encoders.py b/src/diffusers/modular_pipelines/qwenimage/encoders.py index 1c414bb1f3c5..acf85529fbda 100644 --- a/src/diffusers/modular_pipelines/qwenimage/encoders.py +++ b/src/diffusers/modular_pipelines/qwenimage/encoders.py @@ -282,7 +282,7 @@ class QwenImageEditResizeStep(ModularPipelineBlocks): Image Resize step that resize the image to target area while maintaining the aspect ratio. Components: - image_resize_processor (`VaeImageProcessor`) + - image_resize_processor (`VaeImageProcessor`) Inputs: image (`Image | list`): @@ -358,7 +358,7 @@ class QwenImageLayeredResizeStep(ModularPipelineBlocks): maintaining the aspect ratio. Components: - image_resize_processor (`VaeImageProcessor`) + - image_resize_processor (`VaeImageProcessor`) Inputs: image (`Image | list`): @@ -453,7 +453,7 @@ class QwenImageEditPlusResizeStep(ModularPipelineBlocks): encoding. Each image is resized independently based on its own aspect ratio. Components: - image_resize_processor (`VaeImageProcessor`) + - image_resize_processor (`VaeImageProcessor`) Inputs: image (`Image | list`): @@ -557,7 +557,8 @@ class QwenImageLayeredGetImagePromptStep(ModularPipelineBlocks): passes through unchanged. Components: - text_encoder (`Qwen2_5_VLForConditionalGeneration`) processor (`Qwen2VLProcessor`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`) + - processor (`Qwen2VLProcessor`) Inputs: prompt (`str`, *optional*): @@ -671,8 +672,9 @@ class QwenImageTextEncoderStep(ModularPipelineBlocks): Text Encoder step that generates text embeddings to guide the image generation. Components: - text_encoder (`Qwen2_5_VLForConditionalGeneration`): The text encoder to use tokenizer (`Qwen2Tokenizer`): - The tokenizer to use guider (`ClassifierFreeGuidance`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`): The text encoder to use + - tokenizer (`Qwen2Tokenizer`): The tokenizer to use + - guider (`ClassifierFreeGuidance`) Inputs: prompt (`str`): @@ -803,8 +805,9 @@ class QwenImageEditTextEncoderStep(ModularPipelineBlocks): generation. Components: - text_encoder (`Qwen2_5_VLForConditionalGeneration`) processor (`Qwen2VLProcessor`) guider - (`ClassifierFreeGuidance`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`) + - processor (`Qwen2VLProcessor`) + - guider (`ClassifierFreeGuidance`) Inputs: prompt (`str`): @@ -928,8 +931,9 @@ class QwenImageEditPlusTextEncoderStep(ModularPipelineBlocks): embeddings for guiding image generation. Components: - text_encoder (`Qwen2_5_VLForConditionalGeneration`) processor (`Qwen2VLProcessor`) guider - (`ClassifierFreeGuidance`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`) + - processor (`Qwen2VLProcessor`) + - guider (`ClassifierFreeGuidance`) Inputs: prompt (`str`): @@ -1067,7 +1071,7 @@ class QwenImageInpaintProcessImagesInputStep(ModularPipelineBlocks): resized to the given height and width. Components: - image_mask_processor (`InpaintProcessor`) + - image_mask_processor (`InpaintProcessor`) Inputs: mask_image (`Image`): @@ -1178,7 +1182,7 @@ class QwenImageEditInpaintProcessImagesInputStep(ModularPipelineBlocks): resized first. Components: - image_mask_processor (`InpaintProcessor`) + - image_mask_processor (`InpaintProcessor`) Inputs: mask_image (`Image`): @@ -1271,7 +1275,7 @@ class QwenImageProcessImagesInputStep(ModularPipelineBlocks): Image Preprocess step. will resize the image to the given height and width. Components: - image_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) Inputs: image (`Image | list`): @@ -1357,7 +1361,7 @@ class QwenImageEditProcessImagesInputStep(ModularPipelineBlocks): Image Preprocess step. Images needs to be resized first. Components: - image_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) Inputs: resized_image (`list`): @@ -1431,7 +1435,7 @@ class QwenImageEditPlusProcessImagesInputStep(ModularPipelineBlocks): processed images. Components: - image_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) Inputs: resized_image (`list`): @@ -1520,7 +1524,7 @@ class QwenImageVaeEncoderStep(ModularPipelineBlocks): Handles both single images and lists of images with varied resolutions. Components: - vae (`AutoencoderKLQwenImage`) + - vae (`AutoencoderKLQwenImage`) Inputs: processed_image (`Tensor`): @@ -1629,8 +1633,9 @@ class QwenImageControlNetVaeEncoderStep(ModularPipelineBlocks): VAE Encoder step that converts `control_image` into latent representations control_image_latents. Components: - vae (`AutoencoderKLQwenImage`) controlnet (`QwenImageControlNetModel`) control_image_processor - (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) + - controlnet (`QwenImageControlNetModel`) + - control_image_processor (`VaeImageProcessor`) Inputs: control_image (`Image`): diff --git a/src/diffusers/modular_pipelines/qwenimage/inputs.py b/src/diffusers/modular_pipelines/qwenimage/inputs.py index 2d35d833c60f..f22a877fa44c 100644 --- a/src/diffusers/modular_pipelines/qwenimage/inputs.py +++ b/src/diffusers/modular_pipelines/qwenimage/inputs.py @@ -266,7 +266,7 @@ class QwenImageAdditionalInputsStep(ModularPipelineBlocks): This block should be placed after the encoder steps and the text input step. Components: - pachifier (`QwenImagePachifier`) + - pachifier (`QwenImagePachifier`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -482,7 +482,7 @@ class QwenImageEditPlusAdditionalInputsStep(ModularPipelineBlocks): This block should be placed after the encoder steps and the text input step. Components: - pachifier (`QwenImagePachifier`) + - pachifier (`QwenImagePachifier`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -718,7 +718,7 @@ class QwenImageLayeredAdditionalInputsStep(ModularPipelineBlocks): This block should be placed after the encoder steps and the text input step. Components: - pachifier (`QwenImageLayeredPachifier`) + - pachifier (`QwenImageLayeredPachifier`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py index bf87028b2f90..31329a05a613 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py @@ -66,8 +66,9 @@ class QwenImageAutoTextEncoderStep(AutoPipelineBlocks): Text encoder step that encodes the text prompt into a text embedding. This is an auto pipeline block. Components: - text_encoder (`Qwen2_5_VLForConditionalGeneration`): The text encoder to use tokenizer (`Qwen2Tokenizer`): - The tokenizer to use guider (`ClassifierFreeGuidance`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`): The text encoder to use + - tokenizer (`Qwen2Tokenizer`): The tokenizer to use + - guider (`ClassifierFreeGuidance`) Inputs: prompt (`str`, *optional*): @@ -114,7 +115,8 @@ class QwenImageInpaintVaeEncoderStep(SequentialPipelineBlocks): - Creates `image_latents`. Components: - image_mask_processor (`InpaintProcessor`) vae (`AutoencoderKLQwenImage`) + - image_mask_processor (`InpaintProcessor`) + - vae (`AutoencoderKLQwenImage`) Inputs: mask_image (`Image`): @@ -161,7 +163,8 @@ class QwenImageImg2ImgVaeEncoderStep(SequentialPipelineBlocks): Vae encoder step that preprocess andencode the image inputs into their latent representations. Components: - image_processor (`VaeImageProcessor`) vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) Inputs: image (`Image | list`): @@ -216,8 +219,9 @@ class QwenImageOptionalControlNetVaeEncoderStep(AutoPipelineBlocks): - if `control_image` is not provided, step will be skipped. Components: - vae (`AutoencoderKLQwenImage`) controlnet (`QwenImageControlNetModel`) control_image_processor - (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) + - controlnet (`QwenImageControlNetModel`) + - control_image_processor (`VaeImageProcessor`) Inputs: control_image (`Image`, *optional*): @@ -260,7 +264,7 @@ class QwenImageImg2ImgInputStep(SequentialPipelineBlocks): Input step that prepares the inputs for the img2img denoising step. It: Components: - pachifier (`QwenImagePachifier`) + - pachifier (`QwenImagePachifier`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -323,7 +327,7 @@ class QwenImageInpaintInputStep(SequentialPipelineBlocks): Input step that prepares the inputs for the inpainting denoising step. It: Components: - pachifier (`QwenImagePachifier`) + - pachifier (`QwenImagePachifier`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -400,7 +404,8 @@ class QwenImageInpaintPrepareLatentsStep(SequentialPipelineBlocks): - Create the pachified latents `mask` based on the processedmask image. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - pachifier (`QwenImagePachifier`) Inputs: latents (`Tensor`): @@ -452,8 +457,10 @@ class QwenImageCoreDenoiseStep(SequentialPipelineBlocks): (timesteps, latents, rope inputs etc.). Components: - pachifier (`QwenImagePachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -525,8 +532,10 @@ class QwenImageInpaintCoreDenoiseStep(SequentialPipelineBlocks): task. Components: - pachifier (`QwenImagePachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -606,8 +615,10 @@ class QwenImageImg2ImgCoreDenoiseStep(SequentialPipelineBlocks): task. Components: - pachifier (`QwenImagePachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -685,8 +696,11 @@ class QwenImageControlNetCoreDenoiseStep(SequentialPipelineBlocks): (timesteps, latents, rope inputs etc.). Components: - pachifier (`QwenImagePachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) controlnet - (`QwenImageControlNetModel`) guider (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - controlnet (`QwenImageControlNetModel`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -771,8 +785,11 @@ class QwenImageControlNetInpaintCoreDenoiseStep(SequentialPipelineBlocks): task. Components: - pachifier (`QwenImagePachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) controlnet - (`QwenImageControlNetModel`) guider (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - controlnet (`QwenImageControlNetModel`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -865,8 +882,11 @@ class QwenImageControlNetImg2ImgCoreDenoiseStep(SequentialPipelineBlocks): task. Components: - pachifier (`QwenImagePachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) controlnet - (`QwenImageControlNetModel`) guider (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - controlnet (`QwenImageControlNetModel`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -1022,7 +1042,8 @@ class QwenImageDecodeStep(SequentialPipelineBlocks): Decode step that decodes the latents to images and postprocess the generated image. Components: - vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) Inputs: latents (`Tensor`): @@ -1053,7 +1074,8 @@ class QwenImageInpaintDecodeStep(SequentialPipelineBlocks): overally to the original image. Components: - vae (`AutoencoderKLQwenImage`) image_mask_processor (`InpaintProcessor`) + - vae (`AutoencoderKLQwenImage`) + - image_mask_processor (`InpaintProcessor`) Inputs: latents (`Tensor`): @@ -1123,11 +1145,17 @@ class QwenImageAutoBlocks(SequentialPipelineBlocks): - `controlnet_inpainting`: requires `prompt`, `mask_image`, `image`, `control_image` Components: - text_encoder (`Qwen2_5_VLForConditionalGeneration`): The text encoder to use tokenizer (`Qwen2Tokenizer`): - The tokenizer to use guider (`ClassifierFreeGuidance`) image_mask_processor (`InpaintProcessor`) vae - (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) controlnet (`QwenImageControlNetModel`) - control_image_processor (`VaeImageProcessor`) pachifier (`QwenImagePachifier`) scheduler - (`FlowMatchEulerDiscreteScheduler`) transformer (`QwenImageTransformer2DModel`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`): The text encoder to use + - tokenizer (`Qwen2Tokenizer`): The tokenizer to use + - guider (`ClassifierFreeGuidance`) + - image_mask_processor (`InpaintProcessor`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) + - controlnet (`QwenImageControlNetModel`) + - control_image_processor (`VaeImageProcessor`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`QwenImageTransformer2DModel`) Inputs: prompt (`str`, *optional*): diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py index 37b80b69ec7e..65f9ac1754f9 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py @@ -63,8 +63,10 @@ class QwenImageEditVLEncoderStep(SequentialPipelineBlocks): QwenImage-Edit VL encoder step that encode the image and text prompts together. Components: - image_resize_processor (`VaeImageProcessor`) text_encoder (`Qwen2_5_VLForConditionalGeneration`) processor - (`Qwen2VLProcessor`) guider (`ClassifierFreeGuidance`) + - image_resize_processor (`VaeImageProcessor`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`) + - processor (`Qwen2VLProcessor`) + - guider (`ClassifierFreeGuidance`) Inputs: image (`Image | list`): @@ -111,8 +113,9 @@ class QwenImageEditVaeEncoderStep(SequentialPipelineBlocks): Vae encoder step that encode the image inputs into their latent representations. Components: - image_resize_processor (`VaeImageProcessor`) image_processor (`VaeImageProcessor`) vae - (`AutoencoderKLQwenImage`) + - image_resize_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) Inputs: image (`Image | list`): @@ -152,8 +155,9 @@ class QwenImageEditInpaintVaeEncoderStep(SequentialPipelineBlocks): - create image latents. Components: - image_resize_processor (`VaeImageProcessor`) image_mask_processor (`InpaintProcessor`) vae - (`AutoencoderKLQwenImage`) + - image_resize_processor (`VaeImageProcessor`) + - image_mask_processor (`InpaintProcessor`) + - vae (`AutoencoderKLQwenImage`) Inputs: image (`Image | list`): @@ -227,7 +231,7 @@ class QwenImageEditInputStep(SequentialPipelineBlocks): - update height/width based `image_latents`, patchify `image_latents`. Components: - pachifier (`QwenImagePachifier`) + - pachifier (`QwenImagePachifier`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -297,7 +301,7 @@ class QwenImageEditInpaintInputStep(SequentialPipelineBlocks): - update height/width based `image_latents`, patchify `image_latents`. Components: - pachifier (`QwenImagePachifier`) + - pachifier (`QwenImagePachifier`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -376,7 +380,8 @@ class QwenImageEditInpaintPrepareLatentsStep(SequentialPipelineBlocks): - Create the patchified latents `mask` based on the processed mask image. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - pachifier (`QwenImagePachifier`) Inputs: latents (`Tensor`): @@ -424,8 +429,10 @@ class QwenImageEditCoreDenoiseStep(SequentialPipelineBlocks): Core denoising workflow for QwenImage-Edit edit (img2img) task. Components: - pachifier (`QwenImagePachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -498,8 +505,10 @@ class QwenImageEditInpaintCoreDenoiseStep(SequentialPipelineBlocks): Core denoising workflow for QwenImage-Edit edit inpaint task. Components: - pachifier (`QwenImagePachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -617,7 +626,8 @@ class QwenImageEditDecodeStep(SequentialPipelineBlocks): Decode step that decodes the latents to images and postprocess the generated image. Components: - vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) Inputs: latents (`Tensor`): @@ -648,7 +658,8 @@ class QwenImageEditInpaintDecodeStep(SequentialPipelineBlocks): overlay to the original image. Components: - vae (`AutoencoderKLQwenImage`) image_mask_processor (`InpaintProcessor`) + - vae (`AutoencoderKLQwenImage`) + - image_mask_processor (`InpaintProcessor`) Inputs: latents (`Tensor`): @@ -724,10 +735,16 @@ class QwenImageEditAutoBlocks(SequentialPipelineBlocks): - `image_conditioned_inpainting`: requires `prompt`, `mask_image`, `image` Components: - image_resize_processor (`VaeImageProcessor`) text_encoder (`Qwen2_5_VLForConditionalGeneration`) processor - (`Qwen2VLProcessor`) guider (`ClassifierFreeGuidance`) image_mask_processor (`InpaintProcessor`) vae - (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) pachifier (`QwenImagePachifier`) scheduler - (`FlowMatchEulerDiscreteScheduler`) transformer (`QwenImageTransformer2DModel`) + - image_resize_processor (`VaeImageProcessor`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`) + - processor (`Qwen2VLProcessor`) + - guider (`ClassifierFreeGuidance`) + - image_mask_processor (`InpaintProcessor`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`QwenImageTransformer2DModel`) Inputs: image (`Image | list`): diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit_plus.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit_plus.py index 4a1f418d7b45..a1bd156368fb 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit_plus.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit_plus.py @@ -54,8 +54,10 @@ class QwenImageEditPlusVLEncoderStep(SequentialPipelineBlocks): QwenImage-Edit Plus VL encoder step that encodes the image and text prompts together. Components: - image_resize_processor (`VaeImageProcessor`) text_encoder (`Qwen2_5_VLForConditionalGeneration`) processor - (`Qwen2VLProcessor`) guider (`ClassifierFreeGuidance`) + - image_resize_processor (`VaeImageProcessor`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`) + - processor (`Qwen2VLProcessor`) + - guider (`ClassifierFreeGuidance`) Inputs: image (`Image | list`): @@ -104,8 +106,9 @@ class QwenImageEditPlusVaeEncoderStep(SequentialPipelineBlocks): Each image is resized independently based on its own aspect ratio to 1024x1024 target area. Components: - image_resize_processor (`VaeImageProcessor`) image_processor (`VaeImageProcessor`) vae - (`AutoencoderKLQwenImage`) + - image_resize_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) Inputs: image (`Image | list`): @@ -156,7 +159,7 @@ class QwenImageEditPlusInputStep(SequentialPipelineBlocks): - Defaults height/width from last image in the list. Components: - pachifier (`QwenImagePachifier`) + - pachifier (`QwenImagePachifier`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -227,8 +230,10 @@ class QwenImageEditPlusCoreDenoiseStep(SequentialPipelineBlocks): Core denoising workflow for QwenImage-Edit Plus edit (img2img) task. Components: - pachifier (`QwenImagePachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -305,7 +310,8 @@ class QwenImageEditPlusDecodeStep(SequentialPipelineBlocks): Decode step that decodes the latents to images and postprocesses the generated image. Components: - vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) + - image_processor (`VaeImageProcessor`) Inputs: latents (`Tensor`): @@ -351,10 +357,15 @@ class QwenImageEditPlusAutoBlocks(SequentialPipelineBlocks): - VL encoder uses 384x384 target area, VAE encoder uses 1024x1024 target area. Components: - image_resize_processor (`VaeImageProcessor`) text_encoder (`Qwen2_5_VLForConditionalGeneration`) processor - (`Qwen2VLProcessor`) guider (`ClassifierFreeGuidance`) image_processor (`VaeImageProcessor`) vae - (`AutoencoderKLQwenImage`) pachifier (`QwenImagePachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) - transformer (`QwenImageTransformer2DModel`) + - image_resize_processor (`VaeImageProcessor`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`) + - processor (`Qwen2VLProcessor`) + - guider (`ClassifierFreeGuidance`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) + - pachifier (`QwenImagePachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`QwenImageTransformer2DModel`) Inputs: image (`Image | list`): diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_layered.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_layered.py index a10454f1fb0c..6f4ab3fcd1f8 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_layered.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_layered.py @@ -56,8 +56,11 @@ class QwenImageLayeredTextEncoderStep(SequentialPipelineBlocks): provided. Components: - image_resize_processor (`VaeImageProcessor`) text_encoder (`Qwen2_5_VLForConditionalGeneration`) processor - (`Qwen2VLProcessor`) tokenizer (`Qwen2Tokenizer`): The tokenizer to use guider (`ClassifierFreeGuidance`) + - image_resize_processor (`VaeImageProcessor`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`) + - processor (`Qwen2VLProcessor`) + - tokenizer (`Qwen2Tokenizer`): The tokenizer to use + - guider (`ClassifierFreeGuidance`) Inputs: image (`Image | list`): @@ -113,8 +116,9 @@ class QwenImageLayeredVaeEncoderStep(SequentialPipelineBlocks): Vae encoder step that encode the image inputs into their latent representations. Components: - image_resize_processor (`VaeImageProcessor`) image_processor (`VaeImageProcessor`) vae - (`AutoencoderKLQwenImage`) + - image_resize_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) Inputs: image (`Image | list`): @@ -161,7 +165,7 @@ class QwenImageLayeredInputStep(SequentialPipelineBlocks): - update height/width based `image_latents`, patchify `image_latents`. Components: - pachifier (`QwenImageLayeredPachifier`) + - pachifier (`QwenImageLayeredPachifier`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -226,8 +230,10 @@ class QwenImageLayeredCoreDenoiseStep(SequentialPipelineBlocks): Core denoising workflow for QwenImage-Layered img2img task. Components: - pachifier (`QwenImageLayeredPachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) transformer (`QwenImageTransformer2DModel`) + - pachifier (`QwenImageLayeredPachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImageTransformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -311,10 +317,16 @@ class QwenImageLayeredAutoBlocks(SequentialPipelineBlocks): Auto Modular pipeline for layered denoising tasks using QwenImage-Layered. Components: - image_resize_processor (`VaeImageProcessor`) text_encoder (`Qwen2_5_VLForConditionalGeneration`) processor - (`Qwen2VLProcessor`) tokenizer (`Qwen2Tokenizer`): The tokenizer to use guider (`ClassifierFreeGuidance`) - image_processor (`VaeImageProcessor`) vae (`AutoencoderKLQwenImage`) pachifier (`QwenImageLayeredPachifier`) - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`QwenImageTransformer2DModel`) + - image_resize_processor (`VaeImageProcessor`) + - text_encoder (`Qwen2_5_VLForConditionalGeneration`) + - processor (`Qwen2VLProcessor`) + - tokenizer (`Qwen2Tokenizer`): The tokenizer to use + - guider (`ClassifierFreeGuidance`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage`) + - pachifier (`QwenImageLayeredPachifier`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`QwenImageTransformer2DModel`) Inputs: image (`Image | list`): diff --git a/src/diffusers/modular_pipelines/qwenimage21/before_denoise.py b/src/diffusers/modular_pipelines/qwenimage21/before_denoise.py index 01123f742fef..95b848c2b849 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/before_denoise.py +++ b/src/diffusers/modular_pipelines/qwenimage21/before_denoise.py @@ -219,7 +219,7 @@ class QwenImage21SetTimestepsStep(ModularPipelineBlocks): `num_inference_steps`. Should be run after the prepare latents step. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: num_inference_steps (`int`, *optional*, defaults to 40): @@ -464,7 +464,7 @@ class QwenImage21TurboSetTimestepsStep(ModularPipelineBlocks): after the prepare latents step. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Configs: sample_sigmas (default: None): Sampling grid the checkpoint ships with, used when `sigmas` is not passed. diff --git a/src/diffusers/modular_pipelines/qwenimage21/decoders.py b/src/diffusers/modular_pipelines/qwenimage21/decoders.py index e49eb23f20b4..c62c59dc779e 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/decoders.py +++ b/src/diffusers/modular_pipelines/qwenimage21/decoders.py @@ -100,7 +100,8 @@ class QwenImage21DecodeStep(ModularPipelineBlocks): Step that decodes the latents to RGBA images and postprocesses them Components: - vae (`AutoencoderKLQwenImage21`) image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage21`) + - image_processor (`VaeImageProcessor`) Inputs: latents (`Tensor`): diff --git a/src/diffusers/modular_pipelines/qwenimage21/denoise.py b/src/diffusers/modular_pipelines/qwenimage21/denoise.py index be2358ef5055..87e11bb84cc2 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/denoise.py +++ b/src/diffusers/modular_pipelines/qwenimage21/denoise.py @@ -444,8 +444,9 @@ class QwenImage21DenoiseStep(QwenImage21DenoiseLoopWrapper): This block supports text-to-image generation. Components: - guider (`ClassifierFreeGuidance`) transformer (`QwenImage21Transformer2DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImage21Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): @@ -509,8 +510,9 @@ class QwenImage21ImageConditionedDenoiseStep(QwenImage21DenoiseLoopWrapper): This block supports image-conditioned generation. Components: - guider (`ClassifierFreeGuidance`) transformer (`QwenImage21Transformer2DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImage21Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): @@ -576,7 +578,8 @@ class QwenImage21TurboDenoiseStep(QwenImage21DenoiseLoopWrapper): This block supports text-to-image generation with the Turbo checkpoint. Components: - transformer (`QwenImage21Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`QwenImage21Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): @@ -637,7 +640,8 @@ class QwenImage21TurboImageConditionedDenoiseStep(QwenImage21DenoiseLoopWrapper) This block supports image-conditioned generation with the Turbo checkpoint. Components: - transformer (`QwenImage21Transformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`QwenImage21Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: timesteps (`Tensor`): diff --git a/src/diffusers/modular_pipelines/qwenimage21/encoders.py b/src/diffusers/modular_pipelines/qwenimage21/encoders.py index 1160abd3ac7f..2cec495c7eda 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/encoders.py +++ b/src/diffusers/modular_pipelines/qwenimage21/encoders.py @@ -203,7 +203,7 @@ class QwenImage21ResizeStep(ModularPipelineBlocks): the VAE encoder. Components: - image_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) Inputs: image (`Image | list`): @@ -302,7 +302,8 @@ class QwenImage21TextEncoderStep(ModularPipelineBlocks): Text Encoder step that generates text embeddings with Qwen3-VL to guide text-to-image generation. Components: - text_encoder (`Qwen3VLForConditionalGeneration`) processor (`Qwen3VLProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - processor (`Qwen3VLProcessor`) Inputs: prompt (`str`): @@ -427,7 +428,8 @@ class QwenImage21VLTextEncoderStep(ModularPipelineBlocks): image-conditioned generation. Components: - text_encoder (`Qwen3VLForConditionalGeneration`) processor (`Qwen3VLProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - processor (`Qwen3VLProcessor`) Inputs: prompt (`str`): @@ -564,7 +566,7 @@ class QwenImage21ProcessImagesInputStep(ModularPipelineBlocks): Image preprocess step that turns the resized RGBA condition images into normalized tensors for the VAE. Components: - image_processor (`VaeImageProcessor`) + - image_processor (`VaeImageProcessor`) Inputs: resized_image (`list`): @@ -637,7 +639,7 @@ class QwenImage21VaeEncoderStep(ModularPipelineBlocks): resolutions, so the latents are returned as a list. Components: - vae (`AutoencoderKLQwenImage21`) + - vae (`AutoencoderKLQwenImage21`) Inputs: processed_image (`list`): @@ -717,7 +719,8 @@ class QwenImage21TurboTextEncoderStep(ModularPipelineBlocks): checkpoint runs without guidance, so there is no negative prompt. Components: - text_encoder (`Qwen3VLForConditionalGeneration`) processor (`Qwen3VLProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - processor (`Qwen3VLProcessor`) Inputs: prompt (`str`): @@ -796,7 +799,8 @@ class QwenImage21TurboVLTextEncoderStep(ModularPipelineBlocks): image-conditioned generation. The Turbo checkpoint runs without guidance, so there is no negative prompt. Components: - text_encoder (`Qwen3VLForConditionalGeneration`) processor (`Qwen3VLProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - processor (`Qwen3VLProcessor`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py index 569ffe530186..8d2d425ddc15 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py +++ b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py @@ -54,8 +54,9 @@ class QwenImage21VLEncoderStep(SequentialPipelineBlocks): Vision-language encoder step that resizes the condition images and encodes them together with the prompt. Components: - image_processor (`VaeImageProcessor`) text_encoder (`Qwen3VLForConditionalGeneration`) processor - (`Qwen3VLProcessor`) + - image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - processor (`Qwen3VLProcessor`) Inputs: image (`Image | list`): @@ -104,8 +105,9 @@ class QwenImage21AutoTextEncoderStep(AutoPipelineBlocks): - `QwenImage21TextEncoderStep` is used otherwise. Components: - image_processor (`VaeImageProcessor`) text_encoder (`Qwen3VLForConditionalGeneration`) processor - (`Qwen3VLProcessor`) + - image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - processor (`Qwen3VLProcessor`) Inputs: image (`Image | list`, *optional*): @@ -168,7 +170,8 @@ class QwenImage21VaeEncoderSequentialStep(SequentialPipelineBlocks): VAE encoder step that resizes, preprocesses and encodes the condition images into latents. Components: - image_processor (`VaeImageProcessor`) vae (`AutoencoderKLQwenImage21`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage21`) Inputs: image (`Image | list`): @@ -205,7 +208,8 @@ class QwenImage21AutoVaeEncoderStep(AutoPipelineBlocks): - If `image` is not provided, step will be skipped. Components: - image_processor (`VaeImageProcessor`) vae (`AutoencoderKLQwenImage21`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage21`) Inputs: image (`Image | list`, *optional*): @@ -261,8 +265,9 @@ class QwenImage21CoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that performs the denoising process for text-to-image generation. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) transformer - (`QwenImage21Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImage21Transformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -342,8 +347,9 @@ class QwenImage21ImageConditionedCoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that performs the denoising process for image-conditioned generation. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) transformer - (`QwenImage21Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImage21Transformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -415,8 +421,9 @@ class QwenImage21AutoCoreDenoiseStep(AutoPipelineBlocks): - `QwenImage21CoreDenoiseStep` is used otherwise. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) transformer - (`QwenImage21Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImage21Transformer2DModel`) Inputs: num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -505,9 +512,13 @@ class QwenImage21AutoBlocks(SequentialPipelineBlocks): - `image_conditioned`: requires `prompt`, `image` Components: - image_processor (`VaeImageProcessor`) text_encoder (`Qwen3VLForConditionalGeneration`) processor - (`Qwen3VLProcessor`) vae (`AutoencoderKLQwenImage21`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) transformer (`QwenImage21Transformer2DModel`) + - image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - processor (`Qwen3VLProcessor`) + - vae (`AutoencoderKLQwenImage21`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`QwenImage21Transformer2DModel`) Inputs: image (`Image | list`, *optional*): diff --git a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py index e7ba91f01627..65de561f28f6 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py +++ b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py @@ -54,8 +54,9 @@ class QwenImage21TurboVLEncoderStep(SequentialPipelineBlocks): Vision-language encoder step that resizes the condition images and encodes them together with the prompt. Components: - image_processor (`VaeImageProcessor`) text_encoder (`Qwen3VLForConditionalGeneration`) processor - (`Qwen3VLProcessor`) + - image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - processor (`Qwen3VLProcessor`) Inputs: image (`Image | list`): @@ -96,8 +97,9 @@ class QwenImage21TurboAutoTextEncoderStep(AutoPipelineBlocks): - `QwenImage21TurboTextEncoderStep` is used otherwise. Components: - image_processor (`VaeImageProcessor`) text_encoder (`Qwen3VLForConditionalGeneration`) processor - (`Qwen3VLProcessor`) + - image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - processor (`Qwen3VLProcessor`) Inputs: image (`Image | list`, *optional*): @@ -152,7 +154,8 @@ class QwenImage21TurboVaeEncoderSequentialStep(SequentialPipelineBlocks): VAE encoder step that resizes, preprocesses and encodes the condition images into latents. Components: - image_processor (`VaeImageProcessor`) vae (`AutoencoderKLQwenImage21`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage21`) Inputs: image (`Image | list`): @@ -189,7 +192,8 @@ class QwenImage21TurboAutoVaeEncoderStep(AutoPipelineBlocks): - If `image` is not provided, step will be skipped. Components: - image_processor (`VaeImageProcessor`) vae (`AutoencoderKLQwenImage21`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKLQwenImage21`) Inputs: image (`Image | list`, *optional*): @@ -245,7 +249,8 @@ class QwenImage21TurboCoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that performs the denoising process for text-to-image generation with the Turbo checkpoint. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`QwenImage21Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`QwenImage21Transformer2DModel`) Configs: sample_sigmas (default: None): Sampling grid the checkpoint ships with, used when `sigmas` is not passed. @@ -321,7 +326,8 @@ class QwenImage21TurboImageConditionedCoreDenoiseStep(SequentialPipelineBlocks): Core denoise step that performs the denoising process for image-conditioned generation with the Turbo checkpoint. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`QwenImage21Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`QwenImage21Transformer2DModel`) Configs: sample_sigmas (default: None): Sampling grid the checkpoint ships with, used when `sigmas` is not passed. @@ -389,7 +395,8 @@ class QwenImage21TurboAutoCoreDenoiseStep(AutoPipelineBlocks): - `QwenImage21TurboCoreDenoiseStep` is used otherwise. Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) transformer (`QwenImage21Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`QwenImage21Transformer2DModel`) Configs: sample_sigmas (default: None): Sampling grid the checkpoint ships with, used when `sigmas` is not passed. @@ -475,9 +482,12 @@ class QwenImage21TurboAutoBlocks(SequentialPipelineBlocks): - `image_conditioned`: requires `prompt`, `image` Components: - image_processor (`VaeImageProcessor`) text_encoder (`Qwen3VLForConditionalGeneration`) processor - (`Qwen3VLProcessor`) vae (`AutoencoderKLQwenImage21`) scheduler (`FlowMatchEulerDiscreteScheduler`) - transformer (`QwenImage21Transformer2DModel`) + - image_processor (`VaeImageProcessor`) + - text_encoder (`Qwen3VLForConditionalGeneration`) + - processor (`Qwen3VLProcessor`) + - vae (`AutoencoderKLQwenImage21`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - transformer (`QwenImage21Transformer2DModel`) Configs: sample_sigmas (default: None): Sampling grid the checkpoint ships with, used when `sigmas` is not passed. diff --git a/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py b/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py index dc2aae05577f..8f36136925c2 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py @@ -38,7 +38,8 @@ class StableDiffusion3Img2ImgVaeEncoderStep(SequentialPipelineBlocks): """ Components: - image_processor (`VaeImageProcessor`) vae (`AutoencoderKL`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKL`) Inputs: image (`None`, *optional*): @@ -69,7 +70,8 @@ class StableDiffusion3Img2ImgVaeEncoderStep(SequentialPipelineBlocks): class StableDiffusion3AutoVaeEncoderStep(AutoPipelineBlocks): """ Components: - image_processor (`VaeImageProcessor`) vae (`AutoencoderKL`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKL`) Inputs: image (`None`, *optional*): @@ -98,8 +100,9 @@ class StableDiffusion3AutoVaeEncoderStep(AutoPipelineBlocks): class StableDiffusion3T2ICoreDenoiseStep(SequentialPipelineBlocks): """ Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) transformer - (`SD3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`SD3Transformer2DModel`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -154,8 +157,9 @@ def outputs(self): class StableDiffusion3I2ICoreDenoiseStep(SequentialPipelineBlocks): """ Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) transformer - (`SD3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`SD3Transformer2DModel`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -223,8 +227,9 @@ def outputs(self): class StableDiffusion3AutoCoreDenoiseStep(AutoPipelineBlocks): """ Components: - scheduler (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) transformer - (`SD3Transformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`SD3Transformer2DModel`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -296,10 +301,17 @@ class StableDiffusion3AutoBlocks(SequentialPipelineBlocks): - `image2image`: requires `image`, `prompt` Components: - text_encoder (`CLIPTextModelWithProjection`) tokenizer (`CLIPTokenizer`) text_encoder_2 - (`CLIPTextModelWithProjection`) tokenizer_2 (`CLIPTokenizer`) text_encoder_3 (`T5EncoderModel`) tokenizer_3 - (`T5Tokenizer`) image_processor (`VaeImageProcessor`) vae (`AutoencoderKL`) scheduler - (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) transformer (`SD3Transformer2DModel`) + - text_encoder (`CLIPTextModelWithProjection`) + - tokenizer (`CLIPTokenizer`) + - text_encoder_2 (`CLIPTextModelWithProjection`) + - tokenizer_2 (`CLIPTokenizer`) + - text_encoder_3 (`T5EncoderModel`) + - tokenizer_3 (`T5Tokenizer`) + - image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKL`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) + - transformer (`SD3Transformer2DModel`) Inputs: prompt (`None`, *optional*): diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py index ebe43e4ff23c..62950ea7e048 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py @@ -308,11 +308,20 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): `mask_image`, `image`, `prompt` Components: - text_encoder (`CLIPTextModel`) text_encoder_2 (`CLIPTextModelWithProjection`) tokenizer (`CLIPTokenizer`) - tokenizer_2 (`CLIPTokenizer`) guider (`ClassifierFreeGuidance`) image_encoder - (`CLIPVisionModelWithProjection`) feature_extractor (`CLIPImageProcessor`) unet (`UNet2DConditionModel`) vae - (`AutoencoderKL`) image_processor (`VaeImageProcessor`) mask_processor (`VaeImageProcessor`) scheduler - (`EulerDiscreteScheduler`) controlnet (`ControlNetUnionModel`) control_image_processor (`VaeImageProcessor`) + - text_encoder (`CLIPTextModel`) + - text_encoder_2 (`CLIPTextModelWithProjection`) + - tokenizer (`CLIPTokenizer`) + - tokenizer_2 (`CLIPTokenizer`) + - guider (`ClassifierFreeGuidance`) + - image_encoder (`CLIPVisionModelWithProjection`) + - feature_extractor (`CLIPImageProcessor`) + - unet (`UNet2DConditionModel`) + - vae (`AutoencoderKL`) + - image_processor (`VaeImageProcessor`) + - mask_processor (`VaeImageProcessor`) + - scheduler (`EulerDiscreteScheduler`) + - controlnet (`ControlNetUnionModel`) + - control_image_processor (`VaeImageProcessor`) Configs: force_zeros_for_empty_prompt (default: True) requires_aesthetics_score (default: False) diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan.py index a664fff3a1c1..63076abfc7b4 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan.py @@ -44,7 +44,9 @@ class WanCoreDenoiseStep(SequentialPipelineBlocks): denoise block that takes encoded conditions and runs the denoising process. Components: - transformer (`WanTransformer3DModel`) scheduler (`UniPCMultistepScheduler`) guider (`ClassifierFreeGuidance`) + - transformer (`WanTransformer3DModel`) + - scheduler (`UniPCMultistepScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`None`, *optional*, defaults to 1): @@ -106,9 +108,13 @@ class WanBlocks(SequentialPipelineBlocks): Modular pipeline blocks for Wan2.1. Components: - text_encoder (`UMT5EncoderModel`) tokenizer (`AutoTokenizer`) guider (`ClassifierFreeGuidance`) transformer - (`WanTransformer3DModel`) scheduler (`UniPCMultistepScheduler`) vae (`AutoencoderKLWan`) video_processor - (`VideoProcessor`) + - text_encoder (`UMT5EncoderModel`) + - tokenizer (`AutoTokenizer`) + - guider (`ClassifierFreeGuidance`) + - transformer (`WanTransformer3DModel`) + - scheduler (`UniPCMultistepScheduler`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Inputs: prompt (`None`, *optional*): diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22.py index 24bf2a6341cd..8fda169b5c33 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22.py @@ -45,8 +45,11 @@ class Wan22CoreDenoiseStep(SequentialPipelineBlocks): denoise block that takes encoded conditions and runs the denoising process. Components: - transformer (`WanTransformer3DModel`) scheduler (`UniPCMultistepScheduler`) guider (`ClassifierFreeGuidance`) - guider_2 (`ClassifierFreeGuidance`) transformer_2 (`WanTransformer3DModel`) + - transformer (`WanTransformer3DModel`) + - scheduler (`UniPCMultistepScheduler`) + - guider (`ClassifierFreeGuidance`) + - guider_2 (`ClassifierFreeGuidance`) + - transformer_2 (`WanTransformer3DModel`) Configs: boundary_ratio (default: 0.875): The boundary ratio to divide the denoising loop into high noise and low @@ -114,9 +117,15 @@ class Wan22Blocks(SequentialPipelineBlocks): Modular pipeline for text-to-video using Wan2.2. Components: - text_encoder (`UMT5EncoderModel`) tokenizer (`AutoTokenizer`) guider (`ClassifierFreeGuidance`) transformer - (`WanTransformer3DModel`) scheduler (`UniPCMultistepScheduler`) guider_2 (`ClassifierFreeGuidance`) - transformer_2 (`WanTransformer3DModel`) vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) + - text_encoder (`UMT5EncoderModel`) + - tokenizer (`AutoTokenizer`) + - guider (`ClassifierFreeGuidance`) + - transformer (`WanTransformer3DModel`) + - scheduler (`UniPCMultistepScheduler`) + - guider_2 (`ClassifierFreeGuidance`) + - transformer_2 (`WanTransformer3DModel`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Configs: boundary_ratio (default: 0.875): The boundary ratio to divide the denoising loop into high noise and low diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py index 583bcaff1a7c..7dc6be38b582 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py @@ -48,7 +48,8 @@ class WanImage2VideoVaeEncoderStep(SequentialPipelineBlocks): representation Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Inputs: image (`Image`): @@ -92,8 +93,11 @@ class Wan22Image2VideoCoreDenoiseStep(SequentialPipelineBlocks): denoise block that takes encoded text and image latent conditions and runs the denoising process. Components: - transformer (`WanTransformer3DModel`) scheduler (`UniPCMultistepScheduler`) guider (`ClassifierFreeGuidance`) - guider_2 (`ClassifierFreeGuidance`) transformer_2 (`WanTransformer3DModel`) + - transformer (`WanTransformer3DModel`) + - scheduler (`UniPCMultistepScheduler`) + - guider (`ClassifierFreeGuidance`) + - guider_2 (`ClassifierFreeGuidance`) + - transformer_2 (`WanTransformer3DModel`) Configs: boundary_ratio (default: 0.875): The boundary ratio to divide the denoising loop into high noise and low @@ -170,9 +174,15 @@ class Wan22Image2VideoBlocks(SequentialPipelineBlocks): Modular pipeline for image-to-video using Wan2.2. Components: - text_encoder (`UMT5EncoderModel`) tokenizer (`AutoTokenizer`) guider (`ClassifierFreeGuidance`) vae - (`AutoencoderKLWan`) video_processor (`VideoProcessor`) transformer (`WanTransformer3DModel`) scheduler - (`UniPCMultistepScheduler`) guider_2 (`ClassifierFreeGuidance`) transformer_2 (`WanTransformer3DModel`) + - text_encoder (`UMT5EncoderModel`) + - tokenizer (`AutoTokenizer`) + - guider (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - transformer (`WanTransformer3DModel`) + - scheduler (`UniPCMultistepScheduler`) + - guider_2 (`ClassifierFreeGuidance`) + - transformer_2 (`WanTransformer3DModel`) Configs: boundary_ratio (default: 0.875): The boundary ratio to divide the denoising loop into high noise and low diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py index cb00e9395169..9872bde0919c 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py @@ -52,7 +52,8 @@ class WanImage2VideoImageEncoderStep(SequentialPipelineBlocks): Image2Video Image Encoder step that resize the image and encode the image to generate the image embeddings Components: - image_processor (`CLIPImageProcessor`) image_encoder (`CLIPVisionModel`) + - image_processor (`CLIPImageProcessor`) + - image_encoder (`CLIPVisionModel`) Inputs: image (`Image`): @@ -86,7 +87,8 @@ class WanFLF2VImageEncoderStep(SequentialPipelineBlocks): embeddings Components: - image_processor (`CLIPImageProcessor`) image_encoder (`CLIPVisionModel`) + - image_processor (`CLIPImageProcessor`) + - image_encoder (`CLIPVisionModel`) Inputs: image (`Image`): @@ -126,7 +128,8 @@ class WanAutoImageEncoderStep(AutoPipelineBlocks): not provided, step will be skipped. Components: - image_processor (`CLIPImageProcessor`) image_encoder (`CLIPVisionModel`) + - image_processor (`CLIPImageProcessor`) + - image_encoder (`CLIPVisionModel`) Inputs: image (`Image`, *optional*): @@ -176,7 +179,8 @@ class WanImage2VideoVaeEncoderStep(SequentialPipelineBlocks): representation Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Inputs: image (`Image`): @@ -216,7 +220,8 @@ class WanFLF2VVaeEncoderStep(SequentialPipelineBlocks): latent conditions Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Inputs: image (`Image`): @@ -267,7 +272,8 @@ class WanAutoVaeEncoderStep(AutoPipelineBlocks): provided, step will be skipped. Components: - vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) Inputs: image (`Image`, *optional*): @@ -325,7 +331,9 @@ class WanImage2VideoCoreDenoiseStep(SequentialPipelineBlocks): denoise block that takes encoded text and image latent conditions and runs the denoising process. Components: - transformer (`WanTransformer3DModel`) scheduler (`UniPCMultistepScheduler`) guider (`ClassifierFreeGuidance`) + - transformer (`WanTransformer3DModel`) + - scheduler (`UniPCMultistepScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`None`, *optional*, defaults to 1): @@ -403,10 +411,15 @@ class WanImage2VideoAutoBlocks(SequentialPipelineBlocks): - `flf2v`: requires `last_image`, `image`, `prompt` Components: - text_encoder (`UMT5EncoderModel`) tokenizer (`AutoTokenizer`) guider (`ClassifierFreeGuidance`) - image_processor (`CLIPImageProcessor`) image_encoder (`CLIPVisionModel`) vae (`AutoencoderKLWan`) - video_processor (`VideoProcessor`) transformer (`WanTransformer3DModel`) scheduler - (`UniPCMultistepScheduler`) + - text_encoder (`UMT5EncoderModel`) + - tokenizer (`AutoTokenizer`) + - guider (`ClassifierFreeGuidance`) + - image_processor (`CLIPImageProcessor`) + - image_encoder (`CLIPVisionModel`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - transformer (`WanTransformer3DModel`) + - scheduler (`UniPCMultistepScheduler`) Inputs: prompt (`None`, *optional*): diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py index 1689ed91a59f..98c26cd9b58a 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py @@ -47,8 +47,11 @@ class Wan22VaceCoreDenoiseStep(SequentialPipelineBlocks): denoise block that takes encoded text and vace conditioning latents and runs the denoising process. Components: - transformer (`WanTransformer3DModel`) scheduler (`UniPCMultistepScheduler`) guider (`ClassifierFreeGuidance`) - guider_2 (`ClassifierFreeGuidance`) transformer_2 (`WanVACETransformer3DModel`) + - transformer (`WanTransformer3DModel`) + - scheduler (`UniPCMultistepScheduler`) + - guider (`ClassifierFreeGuidance`) + - guider_2 (`ClassifierFreeGuidance`) + - transformer_2 (`WanVACETransformer3DModel`) Configs: boundary_ratio (default: 0.875): The boundary ratio to divide the denoising loop into high noise and low @@ -128,9 +131,15 @@ class Wan22VaceBlocks(SequentialPipelineBlocks): Modular pipeline for controllable video generation using Wan2.2 VACE. Components: - text_encoder (`UMT5EncoderModel`) tokenizer (`AutoTokenizer`) guider (`ClassifierFreeGuidance`) vae - (`AutoencoderKLWan`) video_processor (`VideoProcessor`) transformer (`WanTransformer3DModel`) scheduler - (`UniPCMultistepScheduler`) guider_2 (`ClassifierFreeGuidance`) transformer_2 (`WanVACETransformer3DModel`) + - text_encoder (`UMT5EncoderModel`) + - tokenizer (`AutoTokenizer`) + - guider (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLWan`) + - video_processor (`VideoProcessor`) + - transformer (`WanTransformer3DModel`) + - scheduler (`UniPCMultistepScheduler`) + - guider_2 (`ClassifierFreeGuidance`) + - transformer_2 (`WanVACETransformer3DModel`) Configs: boundary_ratio (default: 0.875): The boundary ratio to divide the denoising loop into high noise and low diff --git a/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2.py b/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2.py index f77eb378c15b..fc14f88e4913 100644 --- a/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2.py +++ b/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2.py @@ -53,7 +53,8 @@ class WanAnimate2ImageEncodeStep(SequentialPipelineBlocks): into `encoder_hidden_states_image`. Components: - image_processor (`WanAnimate2VideoProcessor`) image_encoder (`CLIPVisionModel`) + - image_processor (`WanAnimate2VideoProcessor`) + - image_encoder (`CLIPVisionModel`) Inputs: image (`Image | list`): @@ -100,7 +101,8 @@ class WanAnimate2VideoEncodeStep(SequentialPipelineBlocks): of segments) and CLIP-encodes its first frame into `condition_clip_context`. Components: - video_processor (`WanAnimate2VideoProcessor`) image_encoder (`CLIPVisionModel`) + - video_processor (`WanAnimate2VideoProcessor`) + - image_encoder (`CLIPVisionModel`) Inputs: driving_video (`list`): @@ -165,8 +167,10 @@ class WanAnimate2CoreDenoiseStep(SequentialPipelineBlocks): decoding each segment inside the loop because the next segment conditions on its decoded pixels. Components: - vae (`AutoencoderKLWan`) transformer (`WanAnimate2Transformer3DModel`) scheduler (`SchedulerMixin`) guider - (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLWan`) + - transformer (`WanAnimate2Transformer3DModel`) + - scheduler (`SchedulerMixin`) + - guider (`ClassifierFreeGuidance`) Inputs: segment_frame_length (`int`, *optional*, defaults to 81): @@ -255,9 +259,15 @@ class WanAnimate2Blocks(SequentialPipelineBlocks): produce a video of the character following the driving motion. Components: - text_encoder (`UMT5EncoderModel`) tokenizer (`AutoTokenizer`) image_processor (`WanAnimate2VideoProcessor`) - image_encoder (`CLIPVisionModel`) video_processor (`WanAnimate2VideoProcessor`) vae (`AutoencoderKLWan`) - transformer (`WanAnimate2Transformer3DModel`) scheduler (`SchedulerMixin`) guider (`ClassifierFreeGuidance`) + - text_encoder (`UMT5EncoderModel`) + - tokenizer (`AutoTokenizer`) + - image_processor (`WanAnimate2VideoProcessor`) + - image_encoder (`CLIPVisionModel`) + - video_processor (`WanAnimate2VideoProcessor`) + - vae (`AutoencoderKLWan`) + - transformer (`WanAnimate2Transformer3DModel`) + - scheduler (`SchedulerMixin`) + - guider (`ClassifierFreeGuidance`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2_distilled.py b/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2_distilled.py index 8eab815897da..99277e32e7df 100644 --- a/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2_distilled.py +++ b/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2_distilled.py @@ -53,7 +53,8 @@ class WanAnimate2DistilledImageEncodeStep(SequentialPipelineBlocks): into `encoder_hidden_states_image`. Components: - image_processor (`WanAnimate2VideoProcessor`) image_encoder (`CLIPVisionModel`) + - image_processor (`WanAnimate2VideoProcessor`) + - image_encoder (`CLIPVisionModel`) Inputs: image (`Image | list`): @@ -100,7 +101,8 @@ class WanAnimate2DistilledVideoEncodeStep(SequentialPipelineBlocks): of segments) and CLIP-encodes its first frame into `condition_clip_context`. Components: - video_processor (`WanAnimate2VideoProcessor`) image_encoder (`CLIPVisionModel`) + - video_processor (`WanAnimate2VideoProcessor`) + - image_encoder (`CLIPVisionModel`) Inputs: driving_video (`list`): @@ -165,8 +167,10 @@ class WanAnimate2DistilledCoreDenoiseStep(SequentialPipelineBlocks): segment-by-segment denoising loop in few steps without classifier-free guidance. Components: - vae (`AutoencoderKLWan`) transformer (`WanAnimate2Transformer3DModel`) scheduler (`SchedulerMixin`) guider - (`ClassifierFreeGuidance`) + - vae (`AutoencoderKLWan`) + - transformer (`WanAnimate2Transformer3DModel`) + - scheduler (`SchedulerMixin`) + - guider (`ClassifierFreeGuidance`) Inputs: segment_frame_length (`int`, *optional*, defaults to 81): @@ -257,9 +261,15 @@ class WanAnimate2DistilledBlocks(SequentialPipelineBlocks): classifier-free guidance. Components: - text_encoder (`UMT5EncoderModel`) tokenizer (`AutoTokenizer`) image_processor (`WanAnimate2VideoProcessor`) - image_encoder (`CLIPVisionModel`) video_processor (`WanAnimate2VideoProcessor`) vae (`AutoencoderKLWan`) - transformer (`WanAnimate2Transformer3DModel`) scheduler (`SchedulerMixin`) guider (`ClassifierFreeGuidance`) + - text_encoder (`UMT5EncoderModel`) + - tokenizer (`AutoTokenizer`) + - image_processor (`WanAnimate2VideoProcessor`) + - image_encoder (`CLIPVisionModel`) + - video_processor (`WanAnimate2VideoProcessor`) + - vae (`AutoencoderKLWan`) + - transformer (`WanAnimate2Transformer3DModel`) + - scheduler (`SchedulerMixin`) + - guider (`ClassifierFreeGuidance`) Inputs: prompt (`str`): diff --git a/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py b/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py index 1440128a3c55..742679ccf4f2 100644 --- a/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py +++ b/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py @@ -48,8 +48,9 @@ class ZImageCoreDenoiseStep(SequentialPipelineBlocks): denoise block that takes encoded conditions and runs the denoising process. Components: - transformer (`ZImageTransformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) + - transformer (`ZImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -102,8 +103,9 @@ class ZImageImage2ImageCoreDenoiseStep(SequentialPipelineBlocks): denoise block that takes encoded text and image latent conditions and runs the denoising process. Components: - transformer (`ZImageTransformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) + - transformer (`ZImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -174,8 +176,9 @@ class ZImageAutoDenoiseStep(AutoPipelineBlocks): - if `image_latents` is not provided, `ZImageCoreDenoiseStep` will be used. Components: - transformer (`ZImageTransformer2DModel`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider - (`ClassifierFreeGuidance`) + - transformer (`ZImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) + - guider (`ClassifierFreeGuidance`) Inputs: num_images_per_prompt (`None`, *optional*, defaults to 1): @@ -233,7 +236,8 @@ class ZImageAutoVaeImageEncoderStep(AutoPipelineBlocks): Vae Image Encoder step that encode the image to generate the image latents Components: - vae (`AutoencoderKL`) image_processor (`VaeImageProcessor`) + - vae (`AutoencoderKL`) + - image_processor (`VaeImageProcessor`) Inputs: image (`Image`, *optional*): @@ -272,9 +276,13 @@ class ZImageAutoBlocks(SequentialPipelineBlocks): - `image2image`: requires `image`, `prompt` Components: - text_encoder (`Qwen3Model`) tokenizer (`Qwen2Tokenizer`) guider (`ClassifierFreeGuidance`) vae - (`AutoencoderKL`) image_processor (`VaeImageProcessor`) transformer (`ZImageTransformer2DModel`) scheduler - (`FlowMatchEulerDiscreteScheduler`) + - text_encoder (`Qwen3Model`) + - tokenizer (`Qwen2Tokenizer`) + - guider (`ClassifierFreeGuidance`) + - vae (`AutoencoderKL`) + - image_processor (`VaeImageProcessor`) + - transformer (`ZImageTransformer2DModel`) + - scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: prompt (`None`, *optional*):